<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# pytype: skip-file

from __future__ import absolute_import

import contextlib
import logging
import os
import re
import shutil
import signal
import socket
import subprocess
import tempfile
import threading
import time

import grpc
from future.moves.urllib.error import URLError
from future.moves.urllib.request import urlopen

from apache_beam.version import __version__ as beam_version

_LOGGER = logging.getLogger(__name__)

class SubprocessServer(object):
  """An abstract base class for running GRPC Servers as an external process.

  This class acts as a context which will start up a server, provides a stub
  to connect to it, and then shuts the server down.  For example::

      with SubprocessServer(GrpcStubClass, [executable, arg, ...]) as stub:
          stub.CallService(...)
  """
  def __init__(self, stub_class, cmd, port=None):
    """Creates the server object.

    :param stub_class: the auto-generated GRPC client stub class used for
        connecting to the GRPC service
    :param cmd: command (including arguments) for starting up the server,
        suitable for passing to `subprocess.POpen`.
    :param port: (optional) the port at which the subprocess will serve its
        service.  If not given, one will be randomly chosen and the special
        string "{{PORT}}" will be substituted in the command line arguments
        with the chosen port.
    """
    self._process_lock = threading.RLock()
    self._process = None
    self._stub_class = stub_class
    self._cmd = [str(arg) for arg in cmd]
    self._port = port

  def __enter__(self):
    return self.start()

  def __exit__(self, *unused_args):
    self.stop()

  def start(self):
    try:
      endpoint = self.start_process()
      wait_secs = .1
      channel_options = [("grpc.max_receive_message_length", -1),
                         ("grpc.max_send_message_length", -1)]
      channel = grpc.insecure_channel(endpoint, options=channel_options)
      channel_ready = grpc.channel_ready_future(channel)
      while True:
        if self._process is not None and self._process.poll() is not None:
          _LOGGER.error("Starting job service with %s", self._process.args)
          raise RuntimeError(
              'Service failed to start up with error %s' % self._process.poll())
        try:
          channel_ready.result(timeout=wait_secs)
          break
        except (grpc.FutureTimeoutError, grpc.RpcError):
          wait_secs *= 1.2
          logging.log(
              logging.WARNING if wait_secs > 1 else logging.DEBUG,
              'Waiting for grpc channel to be ready at %s.',
              endpoint)
      return self._stub_class(channel)
    except:  # pylint: disable=bare-except
      _LOGGER.exception("Error bringing up service")
      self.stop()
      raise

  def start_process(self):
    with self._process_lock:
      if self._process:
        self.stop()
      if self._port:
        port = self._port
        cmd = self._cmd
      else:
        port, = pick_port(None)
        cmd = [arg.replace('{{PORT}}', str(port)) for arg in self._cmd]
      endpoint = 'localhost:%s' % port
      _LOGGER.info("Starting service with %s", str(cmd).replace("',", "'"))
      self._process = subprocess.Popen(
          cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

      # Emit the output of this command as info level logging.
      def log_stdout():
        line = self._process.stdout.readline()
        while line:
          # Remove newline via rstrip() to not print an empty line
          _LOGGER.info(line.rstrip())
          line = self._process.stdout.readline()

      t = threading.Thread(target=log_stdout)
      t.daemon = True
      t.start()
      return endpoint

  def stop(self):
    self.stop_process()

  def stop_process(self):
    with self._process_lock:
      if not self._process:
        return
      for _ in range(5):
        if self._process.poll() is not None:
          break
        logging.debug("Sending SIGINT to job_server")
        self._process.send_signal(signal.SIGINT)
        time.sleep(1)
      if self._process.poll() is None:
        self._process.kill()
      self._process = None

  def local_temp_dir(self, **kwargs):
    return tempfile.mkdtemp(dir=self._local_temp_root, **kwargs)

class JavaJarServer(SubprocessServer):

  APACHE_REPOSITORY = 'https://repo.maven.apache.org/maven2'
  BEAM_GROUP_ID = 'org.apache.beam'
  JAR_CACHE = os.path.expanduser("~/.apache_beam/cache/jars")

  _BEAM_SERVICES = type(
      'local', (threading.local, ),
      dict(__init__=lambda self: setattr(self, 'replacements', {})))()

  def __init__(self, stub_class, path_to_jar, java_arguments):
    super(JavaJarServer, self).__init__(
        stub_class, ['java', '-jar', path_to_jar] + list(java_arguments))
    self._existing_service = path_to_jar if _is_service_endpoint(
        path_to_jar) else None

  def start_process(self):
    if self._existing_service:
      return self._existing_service
    else:
      return super(JavaJarServer, self).start_process()

  def stop_process(self):
    if self._existing_service:
      pass
    else:
      return super(JavaJarServer, self).stop_process()

  @classmethod
  def jar_name(cls, artifact_id, version, classifier=None, appendix=None):
    return '-'.join(
        filter(None, [artifact_id, appendix, version, classifier])) + '.jar'

  @classmethod
  def path_to_maven_jar(
      cls,
      artifact_id,
      group_id,
      version,
      repository=APACHE_REPOSITORY,
      classifier=None,
      appendix=None):
    return '/'.join([
        repository,
        group_id.replace('.', '/'),
        artifact_id,
        version,
        cls.jar_name(artifact_id, version, classifier, appendix)
    ])

  @classmethod
  def path_to_beam_jar(cls, gradle_target, appendix=None, version=beam_version):
    if gradle_target in cls._BEAM_SERVICES.replacements:
      return cls._BEAM_SERVICES.replacements[gradle_target]

    gradle_package = gradle_target.strip(':').rsplit(':', 1)[0]
    artifact_id = 'beam-' + gradle_package.replace(':', '-')
    project_root = os.path.sep.join(
        os.path.abspath(__file__).split(os.path.sep)[:-5])
    local_path = os.path.join(
        project_root,
        gradle_package.replace(':', os.path.sep),
        'build',
        'libs',
        cls.jar_name(
            artifact_id,
            version.replace('.dev', ''),
            classifier='SNAPSHOT',
            appendix=appendix))
    if os.path.exists(local_path):
      _LOGGER.info('Using pre-built snapshot at %s', local_path)
      return local_path
    elif '.dev' in version:
      # TODO: Attempt to use nightly snapshots?
      raise RuntimeError(
          (
              '%s not found. '
              'Please build the server with \n  cd %s; ./gradlew %s') %
          (local_path, os.path.abspath(project_root), gradle_target))
    else:
      return cls.path_to_maven_jar(
          artifact_id,
          cls.BEAM_GROUP_ID,
          version,
          cls.APACHE_REPOSITORY,
          appendix=appendix)

  @classmethod
  def local_jar(cls, url, cache_dir=None):
    if cache_dir is None:
      cache_dir = cls.JAR_CACHE
    # TODO: Verify checksum?
    if _is_service_endpoint(url):
      return url
    elif os.path.exists(url):
      return url
    else:
      cached_jar = os.path.join(cache_dir, os.path.basename(url))
      if os.path.exists(cached_jar):
        _LOGGER.info('Using cached job server jar from %s' % url)
      else:
        _LOGGER.info('Downloading job server jar from %s' % url)
        if not os.path.exists(cache_dir):
          os.makedirs(cache_dir)
          # TODO: Clean up this cache according to some policy.
        try:
          url_read = urlopen(url)
          with open(cached_jar + '.tmp', 'wb') as jar_write:
            shutil.copyfileobj(url_read, jar_write, length=1 << 20)
          os.rename(cached_jar + '.tmp', cached_jar)
        except URLError as e:
          raise RuntimeError(
              'Unable to fetch remote job server jar at %s: %s' % (url, e))
      return cached_jar

  @classmethod
  @contextlib.contextmanager
  def beam_services(cls, replacements):
    try:
      old = cls._BEAM_SERVICES.replacements
      cls._BEAM_SERVICES.replacements = dict(old, **replacements)
      yield
    finally:
      cls._BEAM_SERVICES.replacements = old

def _is_service_endpoint(path):
  return re.match(r'^[a-zA-Z0-9.-]+:\d+$', path)

def pick_port(*ports):
  """
  Returns a list of ports, same length as input ports list, but replaces
  all None or 0 ports with a random free port.
  """
  sockets = []

  def find_free_port(port):
    if port:
      return port
    else:
      s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
      sockets.append(s)
      s.bind(('localhost', 0))
      _, free_port = s.getsockname()
      return free_port

  ports = list(map(find_free_port, ports))
  # Close sockets only now to avoid the same port to be chosen twice
  for s in sockets:
    s.close()
  return ports
<commit_msg>[BEAM-10618] subprocess_server.py: Fallback to AF_INET6 family when finding free port (#12438)<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# pytype: skip-file

from __future__ import absolute_import

import contextlib
import logging
import os
import re
import shutil
import signal
import socket
import subprocess
import tempfile
import threading
import time

import grpc
from future.moves.urllib.error import URLError
from future.moves.urllib.request import urlopen

from apache_beam.version import __version__ as beam_version

_LOGGER = logging.getLogger(__name__)

class SubprocessServer(object):
  """An abstract base class for running GRPC Servers as an external process.

  This class acts as a context which will start up a server, provides a stub
  to connect to it, and then shuts the server down.  For example::

      with SubprocessServer(GrpcStubClass, [executable, arg, ...]) as stub:
          stub.CallService(...)
  """
  def __init__(self, stub_class, cmd, port=None):
    """Creates the server object.

    :param stub_class: the auto-generated GRPC client stub class used for
        connecting to the GRPC service
    :param cmd: command (including arguments) for starting up the server,
        suitable for passing to `subprocess.POpen`.
    :param port: (optional) the port at which the subprocess will serve its
        service.  If not given, one will be randomly chosen and the special
        string "{{PORT}}" will be substituted in the command line arguments
        with the chosen port.
    """
    self._process_lock = threading.RLock()
    self._process = None
    self._stub_class = stub_class
    self._cmd = [str(arg) for arg in cmd]
    self._port = port

  def __enter__(self):
    return self.start()

  def __exit__(self, *unused_args):
    self.stop()

  def start(self):
    try:
      endpoint = self.start_process()
      wait_secs = .1
      channel_options = [("grpc.max_receive_message_length", -1),
                         ("grpc.max_send_message_length", -1)]
      channel = grpc.insecure_channel(endpoint, options=channel_options)
      channel_ready = grpc.channel_ready_future(channel)
      while True:
        if self._process is not None and self._process.poll() is not None:
          _LOGGER.error("Starting job service with %s", self._process.args)
          raise RuntimeError(
              'Service failed to start up with error %s' % self._process.poll())
        try:
          channel_ready.result(timeout=wait_secs)
          break
        except (grpc.FutureTimeoutError, grpc.RpcError):
          wait_secs *= 1.2
          logging.log(
              logging.WARNING if wait_secs > 1 else logging.DEBUG,
              'Waiting for grpc channel to be ready at %s.',
              endpoint)
      return self._stub_class(channel)
    except:  # pylint: disable=bare-except
      _LOGGER.exception("Error bringing up service")
      self.stop()
      raise

  def start_process(self):
    with self._process_lock:
      if self._process:
        self.stop()
      if self._port:
        port = self._port
        cmd = self._cmd
      else:
        port, = pick_port(None)
        cmd = [arg.replace('{{PORT}}', str(port)) for arg in self._cmd]
      endpoint = 'localhost:%s' % port
      _LOGGER.info("Starting service with %s", str(cmd).replace("',", "'"))
      self._process = subprocess.Popen(
          cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

      # Emit the output of this command as info level logging.
      def log_stdout():
        line = self._process.stdout.readline()
        while line:
          # Remove newline via rstrip() to not print an empty line
          _LOGGER.info(line.rstrip())
          line = self._process.stdout.readline()

      t = threading.Thread(target=log_stdout)
      t.daemon = True
      t.start()
      return endpoint

  def stop(self):
    self.stop_process()

  def stop_process(self):
    with self._process_lock:
      if not self._process:
        return
      for _ in range(5):
        if self._process.poll() is not None:
          break
        logging.debug("Sending SIGINT to job_server")
        self._process.send_signal(signal.SIGINT)
        time.sleep(1)
      if self._process.poll() is None:
        self._process.kill()
      self._process = None

  def local_temp_dir(self, **kwargs):
    return tempfile.mkdtemp(dir=self._local_temp_root, **kwargs)

class JavaJarServer(SubprocessServer):

  APACHE_REPOSITORY = 'https://repo.maven.apache.org/maven2'
  BEAM_GROUP_ID = 'org.apache.beam'
  JAR_CACHE = os.path.expanduser("~/.apache_beam/cache/jars")

  _BEAM_SERVICES = type(
      'local', (threading.local, ),
      dict(__init__=lambda self: setattr(self, 'replacements', {})))()

  def __init__(self, stub_class, path_to_jar, java_arguments):
    super(JavaJarServer, self).__init__(
        stub_class, ['java', '-jar', path_to_jar] + list(java_arguments))
    self._existing_service = path_to_jar if _is_service_endpoint(
        path_to_jar) else None

  def start_process(self):
    if self._existing_service:
      return self._existing_service
    else:
      return super(JavaJarServer, self).start_process()

  def stop_process(self):
    if self._existing_service:
      pass
    else:
      return super(JavaJarServer, self).stop_process()

  @classmethod
  def jar_name(cls, artifact_id, version, classifier=None, appendix=None):
    return '-'.join(
        filter(None, [artifact_id, appendix, version, classifier])) + '.jar'

  @classmethod
  def path_to_maven_jar(
      cls,
      artifact_id,
      group_id,
      version,
      repository=APACHE_REPOSITORY,
      classifier=None,
      appendix=None):
    return '/'.join([
        repository,
        group_id.replace('.', '/'),
        artifact_id,
        version,
        cls.jar_name(artifact_id, version, classifier, appendix)
    ])

  @classmethod
  def path_to_beam_jar(cls, gradle_target, appendix=None, version=beam_version):
    if gradle_target in cls._BEAM_SERVICES.replacements:
      return cls._BEAM_SERVICES.replacements[gradle_target]

    gradle_package = gradle_target.strip(':').rsplit(':', 1)[0]
    artifact_id = 'beam-' + gradle_package.replace(':', '-')
    project_root = os.path.sep.join(
        os.path.abspath(__file__).split(os.path.sep)[:-5])
    local_path = os.path.join(
        project_root,
        gradle_package.replace(':', os.path.sep),
        'build',
        'libs',
        cls.jar_name(
            artifact_id,
            version.replace('.dev', ''),
            classifier='SNAPSHOT',
            appendix=appendix))
    if os.path.exists(local_path):
      _LOGGER.info('Using pre-built snapshot at %s', local_path)
      return local_path
    elif '.dev' in version:
      # TODO: Attempt to use nightly snapshots?
      raise RuntimeError(
          (
              '%s not found. '
              'Please build the server with \n  cd %s; ./gradlew %s') %
          (local_path, os.path.abspath(project_root), gradle_target))
    else:
      return cls.path_to_maven_jar(
          artifact_id,
          cls.BEAM_GROUP_ID,
          version,
          cls.APACHE_REPOSITORY,
          appendix=appendix)

  @classmethod
  def local_jar(cls, url, cache_dir=None):
    if cache_dir is None:
      cache_dir = cls.JAR_CACHE
    # TODO: Verify checksum?
    if _is_service_endpoint(url):
      return url
    elif os.path.exists(url):
      return url
    else:
      cached_jar = os.path.join(cache_dir, os.path.basename(url))
      if os.path.exists(cached_jar):
        _LOGGER.info('Using cached job server jar from %s' % url)
      else:
        _LOGGER.info('Downloading job server jar from %s' % url)
        if not os.path.exists(cache_dir):
          os.makedirs(cache_dir)
          # TODO: Clean up this cache according to some policy.
        try:
          url_read = urlopen(url)
          with open(cached_jar + '.tmp', 'wb') as jar_write:
            shutil.copyfileobj(url_read, jar_write, length=1 << 20)
          os.rename(cached_jar + '.tmp', cached_jar)
        except URLError as e:
          raise RuntimeError(
              'Unable to fetch remote job server jar at %s: %s' % (url, e))
      return cached_jar

  @classmethod
  @contextlib.contextmanager
  def beam_services(cls, replacements):
    try:
      old = cls._BEAM_SERVICES.replacements
      cls._BEAM_SERVICES.replacements = dict(old, **replacements)
      yield
    finally:
      cls._BEAM_SERVICES.replacements = old

def _is_service_endpoint(path):
  return re.match(r'^[a-zA-Z0-9.-]+:\d+$', path)

def pick_port(*ports):
  """
  Returns a list of ports, same length as input ports list, but replaces
  all None or 0 ports with a random free port.
  """
  sockets = []

  def find_free_port(port):
    if port:
      return port
    else:
      try:
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
      except OSError as e:
        # [Errno 97] Address family not supported by protocol
        # Likely indicates we are in an IPv6-only environment (BEAM-10618). Try
        # again with AF_INET6.
        if e.errno == 97:
          s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
        else:
          raise e

      sockets.append(s)
      s.bind(('localhost', 0))
      return s.getsockname()[1]

  ports = list(map(find_free_port, ports))
  # Close sockets only now to avoid the same port to be chosen twice
  for s in sockets:
    s.close()
  return ports
<|endoftext|>


<commit_before># Copyright 2019 Patrick Wilson <patrickraymondwilson@gmail.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo.tests import common

class TestProjectTemplate(common.TransactionCase):

    def setUp(self):
        super().setUp()
        self.test_customer = self.env['res.partner'].create({
            'name': 'TestCustomer'})
        self.test_project = self.env['project.project'].create({
            'name': 'TestProject',
            'alias_name': 'test_alias',
            'total_planned_hours': 0.0,
            'partner_id': self.test_customer.id})
        self.env['project.task'].create({
            'name': 'TestTask',
            'project_id': self.test_project.id})

    # TEST 01: Set project to be a template and test name change
    def test_on_change_is_template(self):
        # Test when changing project to a template
        project_01 = self.test_project
        project_01.is_template = True
        project_01.on_change_is_template()
        self.assertEqual(project_01.name, 'TestProject (TEMPLATE)')

        # Test when changing template back to project
        project_01.is_template = False
        project_01.on_change_is_template()
        self.assertEqual(project_01.name, 'TestProject')

    # TEST 02: Create project from template
    def test_create_project_from_template(self):
        # Set Project Template
        project_01 = self.test_project
        project_01.is_template = True
        project_01.on_change_is_template()

        # Create new Project from Template
        project_01.create_project_from_template()
        new_project = self.env['project.project'].search([(
            'name', '=', 'TestProject (COPY)')])
        self.assertEqual(len(new_project), 1)
<commit_msg>[UPD] Add to Test<commit_after># Copyright 2019 Patrick Wilson <patrickraymondwilson@gmail.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo.tests import common

class TestProjectTemplate(common.TransactionCase):

    def setUp(self):
        super().setUp()
        self.test_customer = self.env['res.partner'].create({
            'name': 'TestCustomer'})
        self.test_project = self.env['project.project'].create({
            'name': 'TestProject',
            'alias_name': 'test_alias',
            'total_planned_hours': 0.0,
            'partner_id': self.test_customer.id})
        self.env['project.task'].create({
            'name': 'TestTask',
            'project_id': self.test_project.id})

    # TEST 01: Set project to be a template and test name change
    def test_on_change_is_template(self):
        # Test when changing project to a template
        project_01 = self.test_project
        project_01.is_template = True
        project_01.on_change_is_template()
        self.assertEqual(project_01.name, 'TestProject (TEMPLATE)')

        # Test when changing template back to project
        project_01.is_template = False
        project_01.on_change_is_template()
        self.assertEqual(project_01.name, 'TestProject')

    # TEST 02: Create project from template
    def test_create_project_from_template(self):
        # Set Project Template
        project_01 = self.test_project
        project_01.is_template = True
        project_01.on_change_is_template()

        # Create new Project from Template
        project_01.create_project_from_template()
        new_project = self.env['project.project'].search([(
            'name', '=', 'TestProject (COPY)')])
        self.assertEqual(len(new_project), 1)

    # TEST 03: Create project from template using non-standard name
    def test_create_project_from_template_non_standard_name(self):
        # Set Project Template
        project_01 = self.test_project
        project_01.is_template = True
        project_01.on_change_is_template()
        # Change the name of project template
        project_01.name = "TestProject(TEST)"

        # Create new Project from Template
        project_01.create_project_from_template()
        new_project = self.env['project.project'].search([(
            'name', '=', 'TestProject(TEST) (COPY)')])
        self.assertEqual(len(new_project), 1)
<|endoftext|>


<commit_before>from chainer import functions

def depthwise_convolution_2d(x, W, b=None, stride=1, pad=0):
    """Two-dimensional depthwise convolution function.

    This is an implementation of two-dimensional depthwise convolution.
    It takes two or three variables: the input image ``x``, the filter weight
    ``W``, and optionally, the bias vector ``b``.

    Notation: here is a notation for dimensionalities.

    - :math:`n` is the batch size.
    - :math:`c_I` is the number of the input.
    - :math:`c_M` is the channel multiplier.
    - :math:`h` and :math:`w` are the height and width of the input image,
      respectively.
    - :math:`h_O` and :math:`w_O` are the height and width of the output image,
      respectively.
    - :math:`k_H` and :math:`k_W` are the height and width of the filters,
      respectively.

    Args:
        x (chainer.Variable or :class:`numpy.ndarray` or cupy.ndarray):
            Input variable of shape :math:`(n, c_I, h, w)`.
        W (~chainer.Variable): Weight variable of shape
            :math:`(c_M, c_I, k_H, k_W)`.
        b (~chainer.Variable):
            Bias variable of length :math:`c_M * c_I` (optional).
        stride (int or pair of ints): Stride of filter applications.
            ``stride=s`` and ``stride=(s, s)`` are equivalent.
        pad (int or pair of ints): Spatial padding width for input arrays.
            ``pad=p`` and ``pad=(p, p)`` are equivalent.

    Returns:
        ~chainer.Variable:
            Output variable. Its shape is :math:`(n, c_I * c_M, h_O, w_O)`.

    Like ``Convolution2D``, ``DepthwiseConvolution2D`` function computes
    correlations between filters and patches of size :math:`(k_H, k_W)` in
    ``x``.
    But unlike ``Convolution2D``, ``DepthwiseConvolution2D`` does not add up
    input channels of filters but concatenates them.
    For that reason, the shape of outputs of depthwise convolution are
    :math:`(n, c_I * c_M, h_O, w_O)`, :math:`c_M` is called channel_multiplier.

    :math:`(h_O, w_O)` is determined by the equivalent equation of
    ``Convolution2D``.

    If the bias vector is given, then it is added to all spatial locations of
    the output of convolution.

    See: `L. Sifre. Rigid-motion scattering for image classification\
          <https://www.di.ens.fr/data/publications/papers/phd_sifre.pdf>`_

    .. seealso:: :class:`~chainer.links.DepthwiseConvolution2D`

    .. admonition:: Example

        >>> x = np.random.uniform(0, 1, (2, 3, 4, 7))
        >>> W = np.random.uniform(0, 1, (2, 3, 3, 3))
        >>> b = np.random.uniform(0, 1, (6,))
        >>> y = F.depthwise_convolution_2d(x, W, b)
        >>> y.shape
        (2, 6, 2, 5)

    """
    multiplier, in_channels, kh, kw = W.shape
    W = functions.transpose(W, (1, 0, 2, 3))
    W = functions.reshape(W, (multiplier * in_channels, 1, kh, kw))
    return functions.convolution_2d(x, W, b, stride, pad, groups=in_channels)
<commit_msg>Fix circular import<commit_after>import chainer

def depthwise_convolution_2d(x, W, b=None, stride=1, pad=0):
    """Two-dimensional depthwise convolution function.

    This is an implementation of two-dimensional depthwise convolution.
    It takes two or three variables: the input image ``x``, the filter weight
    ``W``, and optionally, the bias vector ``b``.

    Notation: here is a notation for dimensionalities.

    - :math:`n` is the batch size.
    - :math:`c_I` is the number of the input.
    - :math:`c_M` is the channel multiplier.
    - :math:`h` and :math:`w` are the height and width of the input image,
      respectively.
    - :math:`h_O` and :math:`w_O` are the height and width of the output image,
      respectively.
    - :math:`k_H` and :math:`k_W` are the height and width of the filters,
      respectively.

    Args:
        x (chainer.Variable or :class:`numpy.ndarray` or cupy.ndarray):
            Input variable of shape :math:`(n, c_I, h, w)`.
        W (~chainer.Variable): Weight variable of shape
            :math:`(c_M, c_I, k_H, k_W)`.
        b (~chainer.Variable):
            Bias variable of length :math:`c_M * c_I` (optional).
        stride (int or pair of ints): Stride of filter applications.
            ``stride=s`` and ``stride=(s, s)`` are equivalent.
        pad (int or pair of ints): Spatial padding width for input arrays.
            ``pad=p`` and ``pad=(p, p)`` are equivalent.

    Returns:
        ~chainer.Variable:
            Output variable. Its shape is :math:`(n, c_I * c_M, h_O, w_O)`.

    Like ``Convolution2D``, ``DepthwiseConvolution2D`` function computes
    correlations between filters and patches of size :math:`(k_H, k_W)` in
    ``x``.
    But unlike ``Convolution2D``, ``DepthwiseConvolution2D`` does not add up
    input channels of filters but concatenates them.
    For that reason, the shape of outputs of depthwise convolution are
    :math:`(n, c_I * c_M, h_O, w_O)`, :math:`c_M` is called channel_multiplier.

    :math:`(h_O, w_O)` is determined by the equivalent equation of
    ``Convolution2D``.

    If the bias vector is given, then it is added to all spatial locations of
    the output of convolution.

    See: `L. Sifre. Rigid-motion scattering for image classification\
          <https://www.di.ens.fr/data/publications/papers/phd_sifre.pdf>`_

    .. seealso:: :class:`~chainer.links.DepthwiseConvolution2D`

    .. admonition:: Example

        >>> x = np.random.uniform(0, 1, (2, 3, 4, 7))
        >>> W = np.random.uniform(0, 1, (2, 3, 3, 3))
        >>> b = np.random.uniform(0, 1, (6,))
        >>> y = F.depthwise_convolution_2d(x, W, b)
        >>> y.shape
        (2, 6, 2, 5)

    """
    multiplier, in_channels, kh, kw = W.shape
    F = chainer.functions
    W = F.transpose(W, (1, 0, 2, 3))
    W = F.reshape(W, (multiplier * in_channels, 1, kh, kw))
    return F.convolution_2d(x, W, b, stride, pad, groups=in_channels)
<|endoftext|>


<commit_before>import json
import os
import os.path
import threading
from Queue import Queue
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from SocketServer import ThreadingMixIn
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.db_proxy import DBProxy
from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestermessenger import shared_file_messenger

# logger
_logger = core_utils.setup_logger()
shared_file_messenger._logger = _logger

# handler for http front-end
class HttpHandler(BaseHTTPRequestHandler):

    def __init__(self, *args, **kwargs):
        self.dbProxy = DBProxy()
        self.tmpLog = None
        BaseHTTPRequestHandler.__init__(self, *args, **kwargs)

    def log_message(self, format, *args):
        # suppress console logging
        pass

    def do_POST(self):
        # logger
        if self.tmpLog is None:
            self.tmpLog = core_utils.make_logger(_logger)
        # 400 by default
        self.send_response(400)
        toSkip = False
        form = None
        methodName = None
        dataStr = None
        message = ''
        # parse the form data posted
        try:
            dataStr = self.rfile.read(int(self.headers['Content-Length']))
            form = json.loads(dataStr)
        except:
            message = 'corrupted json'
            toSkip = True
        # check parameters
        if not toSkip:
            toSkip = True
            # method is not set
            if 'methodName' not in form:
                message = 'methodName is not given'
            elif 'workerID' not in form:
                message = 'workerID is not given'
            elif 'data' not in form:
                message = 'data is not given'
            else:
                toSkip = False
        # get worker
        if not toSkip:
            try:
                workerID = form['workerID']
                workSpec = self.dbProxy.get_worker_with_id(workerID)
                if workSpec is None:
                    message = 'workerID={0} not found in DB'.format(workerID)
                else:
                    # chose file and operation for each action
                    methodName = form['methodName']
                    opType = None
                    filePath = ''
                    if methodName == 'requestJobs':
                        filePath = os.path.join(workSpec.get_access_point(),
                                                shared_file_messenger.jsonJobRequestFileName)
                        opType = 'w'
                    elif methodName == 'getJobs':
                        filePath = os.path.join(workSpec.get_access_point(),
                                                shared_file_messenger.jsonJobSpecFileName)
                        opType = 'r'
                    elif methodName == 'requestEventRanges':
                        filePath = os.path.join(workSpec.get_access_point(),
                                                shared_file_messenger.jsonEventsRequestFileName)
                        opType = 'w'
                    elif methodName == 'getEventRanges':
                        filePath = os.path.join(workSpec.get_access_point(),
                                                shared_file_messenger.jsonEventsFeedFileName)
                        opType = 'r'
                    elif methodName == 'updateJobs':
                        filePath = os.path.join(workSpec.get_access_point(),
                                                shared_file_messenger.jsonAttrsFileName)
                        opType = 'w'
                    elif methodName == 'uploadJobReport':
                        filePath = os.path.join(workSpec.get_access_point(),
                                                shared_file_messenger.jsonJobReport)
                        opType = 'w'
                    elif methodName == 'uploadEventOutputDump':
                        filePath = os.path.join(workSpec.get_access_point(),
                                                shared_file_messenger.jsonOutputsFileName)
                        opType = 'w'
                    else:
                        self.send_response(501)
                        message = 'method not implemented'
                        toSkip = True
                    # take action
                    if not toSkip:
                        # write actions
                        if opType == 'w':
                            # check if file exists
                            if os.path.exists(filePath):
                                message = 'previous operation is not yet done'
                                self.send_response(503)
                            else:
                                with open(filePath, 'w') as fileHandle:
                                    json.dump(form['data'], fileHandle)
                                    message = 'OK'
                                    self.send_response(200)
                        else:
                            # read actions
                            if os.path.exists(filePath):
                                with open(filePath) as fileHandle:
                                    message = json.load(fileHandle)
                                    self.send_header('Content-Type', 'application/json')
                                    self.send_response(200)
                            else:
                                message = 'previous operation is not yet done'
                                self.send_response(503)
            except:
                self.send_response(500)
                message = core_utils.dump_error_message(_logger)
        if harvester_config.frontend.verbose:
            self.tmpLog.debug('method={0} json={1} msg={2}'.format(methodName, dataStr, message))
        # set the response
        self.end_headers()
        self.wfile.write(message)
        return

# http front-end with a thread pool
class ThreadedHttpServer(ThreadingMixIn, HTTPServer):
    allow_reuse_address = True

    # override to make a thread pool
    def serve_forever(self, *args, **kwargs):
        # queue for requests
        self.requests = Queue(harvester_config.frontend.nThreads)
        # make thread pool
        for x in range(harvester_config.frontend.nThreads):
            thr = threading.Thread(target=self.thread_main_loop)
            thr.daemon = True
            thr.start()
        # original server main loop
        HTTPServer.serve_forever(self, *args, **kwargs)

    # main loop of the thread
    def thread_main_loop(self):
        while True:
            # get a request from queue to process
            ThreadingMixIn.process_request_thread(self, *self.requests.get())

    # override to queue requests
    def process_request(self, request, client_address):
        self.requests.put((request, client_address))

# singleton launcher for http front-end
class FrontendLauncher(object):
    instance = None
    lock = threading.Lock()

    # override __new__ to have a singleton
    def __new__(cls, *args, **kwargs):
        if cls.instance is None:
            with cls.lock:
                if cls.instance is None:
                    httpd = ThreadedHttpServer(('', harvester_config.frontend.portNumber), HttpHandler)
                    thr = threading.Thread(target=httpd.serve_forever)
                    thr.daemon = True
                    thr.start()
                    cls.instance = thr
        return cls.instance

# start frontend
frontend = FrontendLauncher()

# messenger with http frontend
class HttpServerMessenger(shared_file_messenger.SharedFileMessenger):
    # constructor
    def __init__(self, **kwarg):
        shared_file_messenger.SharedFileMessenger.__init__(self, **kwarg)
<commit_msg>fixed http header<commit_after>import json
import os
import os.path
import threading
from Queue import Queue
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from SocketServer import ThreadingMixIn
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.db_proxy import DBProxy
from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestermessenger import shared_file_messenger

# logger
_logger = core_utils.setup_logger()
shared_file_messenger._logger = _logger

# handler for http front-end
class HttpHandler(BaseHTTPRequestHandler):

    def __init__(self, *args, **kwargs):
        self.dbProxy = DBProxy()
        self.tmpLog = None
        BaseHTTPRequestHandler.__init__(self, *args, **kwargs)

    def log_message(self, format, *args):
        # suppress console logging
        pass

    def do_POST(self):
        # logger
        if self.tmpLog is None:
            self.tmpLog = core_utils.make_logger(_logger)
        toSkip = False
        form = None
        methodName = None
        dataStr = None
        message = ''
        # parse the form data posted
        try:
            dataStr = self.rfile.read(int(self.headers['Content-Length']))
            form = json.loads(dataStr)
        except:
            message = 'corrupted json'
            toSkip = True
        # check parameters
        if not toSkip:
            toSkip = True
            # method is not set
            if 'methodName' not in form:
                message = 'methodName is not given'
                self.send_response(400)
            elif 'workerID' not in form:
                message = 'workerID is not given'
                self.send_response(400)
            elif 'data' not in form:
                message = 'data is not given'
                self.send_response(400)
            else:
                toSkip = False
        # get worker
        if not toSkip:
            try:
                workerID = form['workerID']
                workSpec = self.dbProxy.get_worker_with_id(workerID)
                if workSpec is None:
                    message = 'workerID={0} not found in DB'.format(workerID)
                else:
                    # chose file and operation for each action
                    methodName = form['methodName']
                    opType = None
                    filePath = ''
                    if methodName == 'requestJobs':
                        filePath = os.path.join(workSpec.get_access_point(),
                                                shared_file_messenger.jsonJobRequestFileName)
                        opType = 'w'
                    elif methodName == 'getJobs':
                        filePath = os.path.join(workSpec.get_access_point(),
                                                shared_file_messenger.jsonJobSpecFileName)
                        opType = 'r'
                    elif methodName == 'requestEventRanges':
                        filePath = os.path.join(workSpec.get_access_point(),
                                                shared_file_messenger.jsonEventsRequestFileName)
                        opType = 'w'
                    elif methodName == 'getEventRanges':
                        filePath = os.path.join(workSpec.get_access_point(),
                                                shared_file_messenger.jsonEventsFeedFileName)
                        opType = 'r'
                    elif methodName == 'updateJobs':
                        filePath = os.path.join(workSpec.get_access_point(),
                                                shared_file_messenger.jsonAttrsFileName)
                        opType = 'w'
                    elif methodName == 'uploadJobReport':
                        filePath = os.path.join(workSpec.get_access_point(),
                                                shared_file_messenger.jsonJobReport)
                        opType = 'w'
                    elif methodName == 'uploadEventOutputDump':
                        filePath = os.path.join(workSpec.get_access_point(),
                                                shared_file_messenger.jsonOutputsFileName)
                        opType = 'w'
                    else:
                        self.send_response(501)
                        message = 'method not implemented'
                        toSkip = True
                    # take action
                    if not toSkip:
                        # write actions
                        if opType == 'w':
                            # check if file exists
                            if os.path.exists(filePath):
                                message = 'previous request is not yet processed'
                                self.send_response(503)
                            else:
                                with open(filePath, 'w') as fileHandle:
                                    json.dump(form['data'], fileHandle)
                                    message = 'OK'
                                    self.send_response(200)
                        else:
                            # read actions
                            if os.path.exists(filePath):
                                with open(filePath) as fileHandle:
                                    message = json.load(fileHandle)
                                    self.send_response(200)
                                    self.send_header('Content-Type', 'application/json')
                            else:
                                message = 'previous request is not yet processed'
                                self.send_response(503)
            except:
                self.send_response(500)
                message = core_utils.dump_error_message(_logger)
        if harvester_config.frontend.verbose:
            self.tmpLog.debug('method={0} json={1} msg={2}'.format(methodName, dataStr, message))
        # set the response
        self.end_headers()
        self.wfile.write(message)
        return

# http front-end with a thread pool
class ThreadedHttpServer(ThreadingMixIn, HTTPServer):
    allow_reuse_address = True

    # override to make a thread pool
    def serve_forever(self, *args, **kwargs):
        # queue for requests
        self.requests = Queue(harvester_config.frontend.nThreads)
        # make thread pool
        for x in range(harvester_config.frontend.nThreads):
            thr = threading.Thread(target=self.thread_main_loop)
            thr.daemon = True
            thr.start()
        # original server main loop
        HTTPServer.serve_forever(self, *args, **kwargs)

    # main loop of the thread
    def thread_main_loop(self):
        while True:
            # get a request from queue to process
            ThreadingMixIn.process_request_thread(self, *self.requests.get())

    # override to queue requests
    def process_request(self, request, client_address):
        self.requests.put((request, client_address))

# singleton launcher for http front-end
class FrontendLauncher(object):
    instance = None
    lock = threading.Lock()

    # override __new__ to have a singleton
    def __new__(cls, *args, **kwargs):
        if cls.instance is None:
            with cls.lock:
                if cls.instance is None:
                    httpd = ThreadedHttpServer(('', harvester_config.frontend.portNumber), HttpHandler)
                    thr = threading.Thread(target=httpd.serve_forever)
                    thr.daemon = True
                    thr.start()
                    cls.instance = thr
        return cls.instance

# start frontend
frontend = FrontendLauncher()

# messenger with http frontend
class HttpServerMessenger(shared_file_messenger.SharedFileMessenger):
    # constructor
    def __init__(self, **kwarg):
        shared_file_messenger.SharedFileMessenger.__init__(self, **kwarg)
<|endoftext|>


<commit_before>import warnings

import numpy

from chainer import backend
from chainer.backends import cuda
from chainer import configuration
from chainer import function
from chainer.functions.normalization import batch_normalization
from chainer.utils import type_check

def _as4darray(arr):
    if arr.ndim == 0:
        return arr.reshape(1, 1, 1, 1)
    elif arr.ndim == 4:
        return arr
    else:
        return arr.reshape(arr.shape[0], -1, 1, 1)

def _xhat(x, mean, std, expander):
    x_mu = x - mean[expander]
    x_mu /= std[expander]
    return x_mu

class BatchRenormalizationFunction(function.Function):

    def __init__(self, eps=2e-5, mean=None, var=None, decay=0.9,
                 rmax=1, dmax=0, update_statistics=True):
        self._running_mean = mean
        self._running_var = var
        self.rmax = rmax
        self.dmax = dmax
        self.r = None
        self.update_statistics = update_statistics

        self.eps = eps
        self.decay = decay

    def _warn_accessing_property(self):
        msg = '''\
batch_renormalization function updates running statistics by default. The \
properties of BatchRenormalizationFunction should not be accessed.

Set update_statistics=False to stop updating.'''
        if self.update_statistics:
            warnings.warn(msg, DeprecationWarning)

    @property
    def running_mean(self):
        self._warn_accessing_property()
        return self._running_mean

    @property
    def running_var(self):
        self._warn_accessing_property()
        return self._running_var

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() == 3)
        x_type, gamma_type, beta_type = in_types
        M = type_check.eval(gamma_type.ndim)
        type_check.expect(
            x_type.dtype.kind == 'f',
            x_type.ndim >= gamma_type.ndim + 1,
            x_type.shape[1:1 + M] == gamma_type.shape,
            # TODO(tkerola): Check shape
            gamma_type.dtype == x_type.dtype,
            beta_type.dtype == x_type.dtype,
            gamma_type.shape == beta_type.shape,
        )

    def forward(self, inputs):
        xp = backend.get_array_module(*inputs)
        x, gamma, beta = inputs

        # Note: we must be in train mode.
        assert configuration.config.train

        if not self.update_statistics:
            self._running_mean = xp.array(self._running_mean)
            self._running_var = xp.array(self._running_var)

        head_ndim = gamma.ndim + 1
        expander = (None, Ellipsis) + (None,) * (x.ndim - head_ndim)

        # NOTE(tommi): cuDNN is not used since it does not support
        # batch renormalization
        axis = (0,) + tuple(range(head_ndim, x.ndim))
        mean = x.mean(axis=axis)
        var = x.var(axis=axis) + self.eps
        self.std = xp.sqrt(var, dtype=var.dtype)

        running_sigma = xp.sqrt(self._running_var + self.eps,
                                dtype=self._running_mean.dtype)
        self.r = xp.clip(self.std / running_sigma,
                         1.0 / self.rmax, self.rmax)
        d = xp.clip(
            (mean - self._running_mean) / running_sigma,
            -self.dmax, self.dmax)

        # Update running statistics:
        m = x.size // gamma[expander].size
        self._running_mean *= self.decay
        adjust = m / max(m - 1., 1.)  # unbiased estimation
        temp_ar = xp.array(mean)
        temp_ar *= (1 - self.decay)
        self._running_mean += temp_ar
        del temp_ar
        self._running_var *= self.decay
        temp_ar = xp.array(var)
        temp_ar *= (1 - self.decay) * adjust
        self._running_var += temp_ar
        del temp_ar

        gamma = gamma[expander]
        beta = beta[expander]

        if xp is numpy:
            self.x_hat = _xhat(x, mean, self.std, expander)
            self.x_hat_renorm = self.x_hat * self.r[expander] + d[expander]
            y = gamma * self.x_hat_renorm
            y += beta
        else:
            self.x_hat, self.x_hat_renorm, y = cuda.elementwise(
                'T x, T mean, T std, T gamma, T beta, T r, T d',
                'T x_hat, T x_hat_renorm, T y',
                '''
                x_hat = (x - mean) / std;
                x_hat_renorm = x_hat * r + d;
                y = gamma * x_hat_renorm + beta;
                ''',
                'bn_fwd')(x, mean[expander], self.std[expander], gamma,
                          beta, self.r[expander], d[expander])

        return y,

    def backward(self, inputs, grad_outputs):
        x, gamma, _ = inputs
        gy = grad_outputs[0]
        head_ndim = gamma.ndim + 1
        expander = (None, Ellipsis) + (None,) * (x.ndim - head_ndim)
        m = gamma.dtype.type(x.size // gamma.size)
        axis = (0,) + tuple(range(head_ndim, x.ndim))
        xp = backend.get_array_module(x)

        # Note: we must be in train mode.
        assert configuration.config.train
        # NOTE(tommi): cuDNN is not used since it does not support
        # batch renormalization
        gbeta = gy.sum(axis=axis)
        ggamma = (gy * self.x_hat_renorm).sum(axis=axis)
        gsigma_batch = (gy * self.x_hat).sum(axis=axis)
        if xp is numpy:
            scale = (self.r * gamma / self.std)[expander]
            gx = scale * (gy - (self.x_hat * gsigma_batch[expander] +
                                gbeta[expander]) / m)
        else:
            inv_m = numpy.float32(1) / m
            gx = cuda.elementwise(
                'T gy, T x_hat, T gamma, T std, T gsigma_batch, T gbeta, \
                T inv_m, T r',
                'T gx',
                'gx = (r * gamma / std) * (gy - (x_hat * gsigma_batch + gbeta) * \
                inv_m)',
                'bn_bwd')(gy, self.x_hat, gamma[expander],
                          self.std[expander], gsigma_batch[expander],
                          gbeta[expander], inv_m, self.r[expander])
        return gx, ggamma, gbeta

def batch_renormalization(x, gamma, beta, rmax, dmax, eps=2e-5,
                          running_mean=None, running_var=None, decay=0.9,
                          update_statistics=False):
    """Batch renormalization function.

    This is an extension of batch normalization, which ensures that the
    training and inference models generate the same outputs that depend on
    individual examples rather than the entire minibatch.

    .. note::
        This function does not perform in-place update to
        ``running_mean`` and ``running_var`` by default, contrary to
        :func:`~chainer.functions.batch_normalization`.
        If the function is called, it will not be possible to access the
        updated running mean and variance statistics, because they are members
        of the function object, which cannot be accessed by the caller.
        If it is desired to update the running statistics, call the function
        with `update_statistics=True` option.

    See: `Batch Renormalization: Towards Reducing Minibatch Dependence in \
          Batch-Normalized Models <https://arxiv.org/abs/1702.03275>`_

    .. seealso:: :class:`links.BatchRenormalization`
    .. seealso:: :class:`functions.normalization.batch_normalization.BatchNormalization`  # NOQA

    """
    return BatchRenormalizationFunction(
        eps, running_mean, running_var, decay, rmax, dmax, update_statistics
    )(x, gamma, beta)

def fixed_batch_renormalization(x, gamma, beta, mean, var, eps=2e-5):
    warnings.warn(
        'fixed_batch_renormalization is deprecated. '
        'Use fixed_batch_normalization instead.',
        DeprecationWarning)
    with configuration.using_config('train', False):
        return batch_normalization.fixed_batch_normalization(
            x, gamma, beta, mean, var, eps
        )
<commit_msg>Fix syntax<commit_after>import warnings

import numpy

from chainer import backend
from chainer.backends import cuda
from chainer import configuration
from chainer import function
from chainer.functions.normalization import batch_normalization
from chainer.utils import type_check

def _as4darray(arr):
    if arr.ndim == 0:
        return arr.reshape(1, 1, 1, 1)
    elif arr.ndim == 4:
        return arr
    else:
        return arr.reshape(arr.shape[0], -1, 1, 1)

def _xhat(x, mean, std, expander):
    x_mu = x - mean[expander]
    x_mu /= std[expander]
    return x_mu

class BatchRenormalizationFunction(function.Function):

    def __init__(self, eps=2e-5, mean=None, var=None, decay=0.9,
                 rmax=1, dmax=0, update_statistics=True):
        self._running_mean = mean
        self._running_var = var
        self.rmax = rmax
        self.dmax = dmax
        self.r = None
        self.update_statistics = update_statistics

        self.eps = eps
        self.decay = decay

    def _warn_accessing_property(self):
        msg = '''\
batch_renormalization function updates running statistics by default. The \
properties of BatchRenormalizationFunction should not be accessed.

Set update_statistics=False to stop updating.'''
        if self.update_statistics:
            warnings.warn(msg, DeprecationWarning)

    @property
    def running_mean(self):
        self._warn_accessing_property()
        return self._running_mean

    @property
    def running_var(self):
        self._warn_accessing_property()
        return self._running_var

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() == 3)
        x_type, gamma_type, beta_type = in_types
        M = type_check.eval(gamma_type.ndim)
        type_check.expect(
            x_type.dtype.kind == 'f',
            x_type.ndim >= gamma_type.ndim + 1,
            x_type.shape[1:1 + M] == gamma_type.shape,
            # TODO(tkerola): Check shape
            gamma_type.dtype == x_type.dtype,
            beta_type.dtype == x_type.dtype,
            gamma_type.shape == beta_type.shape,
        )

    def forward(self, inputs):
        xp = backend.get_array_module(*inputs)
        x, gamma, beta = inputs

        # Note: we must be in train mode.
        assert configuration.config.train

        if not self.update_statistics:
            self._running_mean = xp.array(self._running_mean)
            self._running_var = xp.array(self._running_var)

        head_ndim = gamma.ndim + 1
        expander = (None, Ellipsis) + (None,) * (x.ndim - head_ndim)

        # NOTE(tommi): cuDNN is not used since it does not support
        # batch renormalization
        axis = (0,) + tuple(range(head_ndim, x.ndim))
        mean = x.mean(axis=axis)
        var = x.var(axis=axis) + self.eps
        self.std = xp.sqrt(var, dtype=var.dtype)

        running_sigma = xp.sqrt(self._running_var + self.eps,
                                dtype=self._running_mean.dtype)
        self.r = xp.clip(self.std / running_sigma,
                         1.0 / self.rmax, self.rmax)
        d = xp.clip(
            (mean - self._running_mean) / running_sigma,
            -self.dmax, self.dmax)

        # Update running statistics:
        m = x.size // gamma[expander].size
        self._running_mean *= self.decay
        adjust = m / max(m - 1., 1.)  # unbiased estimation
        temp_ar = xp.array(mean)
        temp_ar *= (1 - self.decay)
        self._running_mean += temp_ar
        del temp_ar
        self._running_var *= self.decay
        temp_ar = xp.array(var)
        temp_ar *= (1 - self.decay) * adjust
        self._running_var += temp_ar
        del temp_ar

        gamma = gamma[expander]
        beta = beta[expander]

        if xp is numpy:
            self.x_hat = _xhat(x, mean, self.std, expander)
            self.x_hat_renorm = self.x_hat * self.r[expander] + d[expander]
            y = gamma * self.x_hat_renorm
            y += beta
        else:
            self.x_hat, self.x_hat_renorm, y = cuda.elementwise(
                'T x, T mean, T std, T gamma, T beta, T r, T d',
                'T x_hat, T x_hat_renorm, T y',
                '''
                x_hat = (x - mean) / std;
                x_hat_renorm = x_hat * r + d;
                y = gamma * x_hat_renorm + beta;
                ''',
                'bn_fwd')(x, mean[expander], self.std[expander], gamma,
                          beta, self.r[expander], d[expander])

        return y,

    def backward(self, inputs, grad_outputs):
        x, gamma, _ = inputs
        gy = grad_outputs[0]
        head_ndim = gamma.ndim + 1
        expander = (None, Ellipsis) + (None,) * (x.ndim - head_ndim)
        m = gamma.dtype.type(x.size // gamma.size)
        axis = (0,) + tuple(range(head_ndim, x.ndim))
        xp = backend.get_array_module(x)

        # Note: we must be in train mode.
        assert configuration.config.train
        # NOTE(tommi): cuDNN is not used since it does not support
        # batch renormalization
        gbeta = gy.sum(axis=axis)
        ggamma = (gy * self.x_hat_renorm).sum(axis=axis)
        gsigma_batch = (gy * self.x_hat).sum(axis=axis)
        if xp is numpy:
            scale = (self.r * gamma / self.std)[expander]
            gx = scale * (gy - (self.x_hat * gsigma_batch[expander] +
                                gbeta[expander]) / m)
        else:
            inv_m = numpy.float32(1) / m
            gx = cuda.elementwise(
                'T gy, T x_hat, T gamma, T std, T gsigma_batch, T gbeta, \
                T inv_m, T r',
                'T gx',
                'gx = (r * gamma / std) * (gy - (x_hat * gsigma_batch + gbeta) * \
                inv_m)',
                'bn_bwd')(gy, self.x_hat, gamma[expander],
                          self.std[expander], gsigma_batch[expander],
                          gbeta[expander], inv_m, self.r[expander])
        return gx, ggamma, gbeta

def batch_renormalization(x, gamma, beta, rmax, dmax, eps=2e-5,
                          running_mean=None, running_var=None, decay=0.9,
                          update_statistics=False):
    """Batch renormalization function.

    This is an extension of batch normalization, which ensures that the
    training and inference models generate the same outputs that depend on
    individual examples rather than the entire minibatch.

    .. note::
        This function does not perform in-place update to
        ``running_mean`` and ``running_var`` by default, contrary to
        :func:`~chainer.functions.batch_normalization`.
        If the function is called, it will not be possible to access the
        updated running mean and variance statistics, because they are members
        of the function object, which cannot be accessed by the caller.
        If it is desired to update the running statistics, call the function
        with `update_statistics=True` option.

    See: `Batch Renormalization: Towards Reducing Minibatch Dependence in \
          Batch-Normalized Models <https://arxiv.org/abs/1702.03275>`_

    .. seealso:: :class:`~chainer.links.BatchRenormalization`
    .. seealso:: :class:`chainer.functions.batch_normalization`

    """
    return BatchRenormalizationFunction(
        eps, running_mean, running_var, decay, rmax, dmax, update_statistics
    )(x, gamma, beta)

def fixed_batch_renormalization(x, gamma, beta, mean, var, eps=2e-5):
    warnings.warn(
        'fixed_batch_renormalization is deprecated. '
        'Use fixed_batch_normalization instead.',
        DeprecationWarning)
    with configuration.using_config('train', False):
        return batch_normalization.fixed_batch_normalization(
            x, gamma, beta, mean, var, eps
        )
<|endoftext|>


<commit_before>import django.template

def renderFile(path, context):
	t = django.template.loader.get_template(path)
	return t.render(django.template.Context(context))
<commit_msg>Added template.renderString<commit_after>import django.template

def renderFile(path, context):
	t = django.template.loader.get_template(path)
	return t.render(django.template.Context(context))

def renderString(template, context):
	t = django.template.Template(template)
	return t.render(django.template.Context(context))
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014 Compassion CH (http://www.compassion.ch)
#    Releasing children from poverty in Jesus' name
#    @author: Emanuel Cino <ecino@compassion.ch>
#
#    The licence is in the file __openerp__.py
#
##############################################################################
from openerp import api, fields, models, _
from openerp.osv import orm
from . import gp_connector

class ResPartner(models.Model):
    """ This class upgrade the partners to match Compassion needs.
        It also synchronize all changes with the MySQL server of GP.
    """

    _inherit = 'res.partner'

    def _get_receipt_types(self, cursor, uid, context=None):
        """ Display values for the receipt selection fields. """
        return [
            ('no', _('No receipt')),
            ('default', _('Default')),
            ('email', _('By e-mail')),
            ('paper', _('On paper'))]

    ##########################################################################
    #                        NEW PARTNER FIELDS                              #
    # ########################################################################
    ref = fields.Char(default=False)
    lang = fields.Selection(default=False)
    opt_out = fields.Boolean(default=True)
    nbmag = fields.integer('Number of Magazines', size=2,
                           required=True, default=0)
    tax_certificate = fields.selection(
        _get_receipt_types, required=True, default='default')
    thankyou_letter = fields.selection(
        _get_receipt_types, _('Thank you letter'),
        required=True, default='default')
    calendar = fields.boolean(
        help=_("Indicates if the partner wants to receive the Compassion "
               "calendar."), default=True)
    christmas_card = fields.boolean(
        help=_("Indicates if the partner wants to receive the "
               "christmas card."), default=True)
    birthday_reminder = fields.boolean(
        help=_("Indicates if the partner wants to receive a birthday "
               "reminder of his child."), default=True)
    abroad = fields.boolean(
        'Abroad/Only e-mail',
        help=_("Indicates if the partner is abroad and should only be "
               "updated by e-mail"))

    ##########################################################################
    #                           PUBLIC METHODS                               #
    ##########################################################################
    def create_from_gp(self, cr, uid, vals, context=None):
        """ Simple create method that skips MySQL insertion, since it is
            called from GP in order to export the addresses. """
        return super(ResPartner, self).create(cr, uid, vals, context)

    def write_from_gp(self, cr, uid, ids, vals, context=None):
        """ Simple write method that skips MySQL insertion, since it is
        called from GP in order to export the addresses. """
        return super(ResPartner, self).write(cr, uid, ids, vals, context)

    def create(self, cr, uid, vals, context=None):
        """ We override the create method so that each partner creation will
            also propagate in the MySQL table used by GP. This method is also
            called by GP with XMLRPC, so that the Odoo holds all the logic
            to sync the two databases.
        """
        gp = gp_connector.GPConnect()
        fieldsUpdate = vals.keys()

        # If the reference is not defined, we automatically set it.
        if 'ref' not in fieldsUpdate:
            # If the contact has a parent and uses his address, plus the parent
            # has a reference, we use the same reference.
            if vals.get('parent_id') and vals.get('use_parent_address'):
                vals['ref'] = self.browse(
                    cr, uid, vals['parent_id'], context=context).ref
            # Otherwise we attribute a new reference number
            else:
                vals['ref'] = gp.nextRef()

        # Create the partner in Odoo, with original values, before exporting
        # to MySQL.
        new_id = super(ResPartner, self).create(cr, uid, vals, context)
        partner = self.browse(cr, uid, new_id, {'lang': 'en_US'})

        # Set some special fields that need a conversion before sending to
        # MySQL
        categories = self._get_category_names(partner, cr, uid)
        self._update_vals_with_gp_partner(partner, vals)

        gp.createPartner(uid, vals, partner, categories)

        return new_id

    def write(self, cr, uid, ids, vals, context=None):
        """ We override the write method so that each update will also update
            the partner in the MySQL table used by GP. This method is also
            called by GP with XMLRPC, so that the update syncs the two
            databases. """
        # We first call the original write method and update the records.
        result = super(ResPartner, self).write(
            cr, uid, ids, vals, context=context)

        # Some fields that need some conversion before sending to MySQL
        categories = list()

        records = self.browse(cr, uid, ids, {'lang': 'en_US'})
        records = [records] if not isinstance(records, list) else records
        create = False
        gp = gp_connector.GPConnect()

        for record in records:
            if record.ref:
                # Handle the change of parent_id and use_parent_address
                if 'use_parent_address' in vals.keys() and record.parent_id:
                    if vals['use_parent_address']:
                        # Change the MySQL partner so that it is linked to the
                        # contact
                        gp.linkContact(uid, record.parent_id.ref, record.id)
                        # Merge contact with parent
                        vals['ref'] = record.parent_id.ref
                    else:
                        # Change the MySQL company partner so that it is no
                        # more linked to the contact of the company
                        gp.unlinkContact(
                            uid, record.parent_id,
                            self._get_category_names(record.parent_id, cr,
                                                     uid))
                        # Link with contact in GP and sync or create new
                        # contact if it doesn't exist.
                        ref = gp.getRefContact(record.id)
                        if ref < 0:
                            ref = str(gp.nextRef())
                            create = True
                        vals['ref'] = ref
                    # Write the new reference in OpenERP
                    super(ResPartner, self).write(
                        cr, uid, record.id, vals, context)
                    record.ref = vals['ref']
                    # We need to copy all partner fields in the new referenced
                    # partner in MySQL
                    self._fill_fields(record, vals)

                # Handle the change of the company status (weird update, but
                # could happen!)
                if 'is_company' in vals.keys():
                    if vals['is_company']:
                        gp.changeToCompany(uid, record.id, record.name)
                    else:
                        # Unlink the contact, if any exists.
                        gp.unlinkContact(
                            uid, record,
                            self._get_category_names(record, cr, uid))
                        gp.changeToPrivate(uid, record.id)
                        # Add name information to contact
                        vals['lastname'] = record.lastname
            else:
                # The partner has no relation to MySQL ! Fix it.
                self._fill_fields(record, vals)
                ref = gp.getRefContact(record.id)
                if ref < 0:
                    ref = str(gp.nextRef())
                    create = True
                vals['ref'] = ref
                # Write the new reference in OpenERP
                record.ref = ref
                super(ResPartner, self).write(
                    cr, uid, record.id, vals, context=context)

            # Get the special fields values
            categories = self._get_category_names(record, cr, uid)

            # Update the MySQL table with the retrieved values.
            self._update_vals_with_gp_partner(record, vals)
            if create:
                gp.createPartner(uid, vals, record, categories)
            else:
                gp.updatePartner(uid, vals, record, categories)

        return result

    def unlink(self, cr, uid, ids, context=None):
        """ We want to perform some checks before deleting a partner ! """
        records = self.browse(cr, uid, ids, context)
        records = [records] if not isinstance(records, list) else records
        gp = gp_connector.GPConnect()
        for record in records:
            # If it is a company, unlink contact in MySQL if there is any.
            if record.is_company:
                for child in record.child_ids:
                    if child.use_parent_address:
                        gp.unlinkContact(
                            uid, record, self._get_category_names(record, cr,
                                                                  uid))

            # If it is a contact, unlink from company.
            if record.use_parent_address and record.parent_id:
                gp.unlinkContact(
                    uid, record.parent_id, self._get_category_names(
                        record.parent_id, cr, uid))

            # Unlink from MySQL
            gp.unlink(uid, record.id)

        return super(ResPartner, self).unlink(cr, uid, ids, context)

    def get_unreconciled_amount(self, cr, uid, partner_id, context=None):
        """Returns the amount of unreconciled credits in Account 1050"""
        partner = self._find_accounting_partner(self.browse(
            cr, uid, partner_id, context))
        mv_line_obj = self.pool.get('account.move.line')
        move_line_ids = mv_line_obj.search(cr, uid, [
            ('partner_id', '=', partner.id),
            ('account_id.code', '=', '1050'),
            ('credit', '>', '0'),
            ('reconcile_id', '=', False)], context=context)
        res = 0
        for move_line in mv_line_obj.browse(cr, uid, move_line_ids, context):
            res += move_line.credit
        return res

    ##########################################################################
    #                           PRIVATE METHODS                              #
    ##########################################################################
    def _update_vals_with_gp_partner(self, partner, vals):
        """ Updates the vals dictionary with correct values for GP
        given the type of the partner (company / contact). """
        count = 0

        # By default, update the GP partner with erp_id = partner.id
        vals['id'] = partner.id

        # Don't unset the "Mailing complet" category when the parent wants to
        # receive it.
        if partner.use_parent_address and not partner.parent_id.opt_out and \
           'opt_out' in vals:
            del vals['opt_out']

        # Check if the record is a company with children and update fields
        # accordingly.
        for child_record in partner.child_ids:
            if child_record.use_parent_address:
                # Set the correct reference id
                vals['id'] = child_record.id
                # Don't update some values that are defined by the child
                # partner.
                vals.pop('abroad', None)
                vals.pop('opt_out', None)
                vals.pop('thankyou_letter', None)
                vals.pop('tax_certificate', None)
                vals.pop('title', None)
                count += 1
        if count > 1:
            raise orm.except_orm(
                _("Partner Error"),
                _("You cannot more than one contact with same address than "
                  "the company. GP does not handle that!"))

    def _get_category_names(self, record, cr, uid, context={'lang': 'en_US'}):
        """ Get a list of the category names of all linked partners.

        Args:
            record (res.partner) : The current partner for which we search
                                   the categories

        Returns:
            A list of strings containing the category names of the current
            partner, his parent and children.
        """
        categories = []
        ids = map(lambda category: category.id, record.category_id)
        category_obj = self.pool.get('res.partner.category')
        for category in category_obj.browse(cr, uid, ids, context):
            categories.append(category.name)
        if record.use_parent_address and record.parent_id:
            for category in category_obj.browse(cr, uid,
                                                record.parent_id.category_id,
                                                context):
                categories.append(category.id.name)
        for child_record in record.child_ids:
            if child_record.use_parent_address:
                for category in category_obj.browse(cr, uid,
                                                    child_record.category_id,
                                                    context):
                    categories.append(category.id.name)
        return categories

    def _fill_fields(self, record, vals):
        """ Fills all needed vals for the MySQL table. """
        vals['title'] = record.title.id
        vals['lang'] = record.lang
        vals['firstname'] = record.firstname
        vals['lastname'] = record.lastname
        vals['function'] = record.function
        vals['nbmag'] = record.nbmag
        vals['calendar'] = record.calendar
        vals['tax_certificate'] = record.tax_certificate
        vals['thankyou_letter'] = record.thankyou_letter
        vals['abroad'] = record.abroad
        vals['opt_out'] = record.opt_out
        vals['mobile'] = record.mobile
        vals['category_id'] = record.category_id
        if not record.use_parent_address:
            # Creation of new partner, add all possible fields.
            vals['street'] = record.street
            vals['street2'] = record.street2
            vals['street3'] = record.street3
            vals['city'] = record.city
            vals['zip'] = record.zip
            vals['country_id'] = record.country_id.id
            vals['birthdate'] = record.birthdate
            vals['deathdate'] = record.deathdate
            vals['phone'] = record.phone
            vals['email'] = record.email
            vals['church_id'] = record.church_id.id
            vals['church_unlinked'] = record.church_unlinked
            vals['date'] = record.date
<commit_msg>Porting partner_compassion<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014 Compassion CH (http://www.compassion.ch)
#    Releasing children from poverty in Jesus' name
#    @author: Emanuel Cino <ecino@compassion.ch>
#
#    The licence is in the file __openerp__.py
#
##############################################################################
from openerp import api, exceptions, fields, models, _
from . import gp_connector

class ResPartner(models.Model):
    """ This class upgrade the partners to match Compassion needs.
        It also synchronize all changes with the MySQL server of GP.
    """

    _inherit = 'res.partner'

    def _get_receipt_types(self):
        """ Display values for the receipt selection fields. """
        return [
            ('no', _('No receipt')),
            ('default', _('Default')),
            ('email', _('By e-mail')),
            ('paper', _('On paper'))]

    ##########################################################################
    #                        NEW PARTNER FIELDS                              #
    ##########################################################################
    ref = fields.Char(default=False)
    lang = fields.Selection(default=False)
    opt_out = fields.Boolean(default=True)
    nbmag = fields.integer('Number of Magazines', size=2,
                           required=True, default=0)
    tax_certificate = fields.selection(
        _get_receipt_types, required=True, default='default')
    thankyou_letter = fields.selection(
        _get_receipt_types, _('Thank you letter'),
        required=True, default='default')
    calendar = fields.boolean(
        help=_("Indicates if the partner wants to receive the Compassion "
               "calendar."), default=True)
    christmas_card = fields.boolean(
        help=_("Indicates if the partner wants to receive the "
               "christmas card."), default=True)
    birthday_reminder = fields.boolean(
        help=_("Indicates if the partner wants to receive a birthday "
               "reminder of his child."), default=True)
    abroad = fields.boolean(
        'Abroad/Only e-mail',
        help=_("Indicates if the partner is abroad and should only be "
               "updated by e-mail"))

    ##########################################################################
    #                           PUBLIC METHODS                               #
    ##########################################################################
    @api.multi
    def create_from_gp(self, vals):
        """ Simple create method that skips MySQL insertion, since it is
            called from GP in order to export the addresses. """
        return super(ResPartner, self).create(vals)

    @api.multi
    def write_from_gp(self, vals):
        """ Simple write method that skips MySQL insertion, since it is
        called from GP in order to export the addresses. """
        return super(ResPartner, self).write(vals)

    @api.model
    def create(self, vals):
        """ We override the create method so that each partner creation will
            also propagate in the MySQL table used by GP. This method is also
            called by GP with XMLRPC, so that the Odoo holds all the logic
            to sync the two databases.
        """
        uid = self.env.user.id
        gp = gp_connector.GPConnect()
        fieldsUpdate = vals.keys()
        obj_partner = self.env['res.partner']
        # If the reference is not defined, we automatically set it.
        if 'ref' not in fieldsUpdate:
            # If the contact has a parent and uses his address, plus the parent
            # has a reference, we use the same reference.
            if vals.get('parent_id') and vals.get('use_parent_address'):
                vals['ref'] = obj_partner.browse(vals['parent_id']).ref
            # Otherwise we attribute a new reference number
            else:
                vals['ref'] = gp.nextRef()

        # Create the partner in Odoo, with original values, before exporting
        # to MySQL.
        partner = super(ResPartner, self).create(
            vals).with_context(lang='en_US')

        # Set some special fields that need a conversion before sending to
        # MySQL
        categories = partner._get_category_names()
        partner._update_vals_with_gp_partner(vals)

        gp.createPartner(uid, vals, partner, categories)

        return partner

    @api.multi
    def write(self, vals):
        """ We override the write method so that each update will also update
            the partner in the MySQL table used by GP. This method is also
            called by GP with XMLRPC, so that the update syncs the two
            databases. """
        uid = self.env.user.id
        # We first call the original write method and update the records.
        result = super(ResPartner, self).write(vals)

        # Some fields that need some conversion before sending to MySQL
        categories = list()

        create = False
        gp = gp_connector.GPConnect()

        for record in self.with_context(lang='en_US'):
            if record.ref:
                # Handle the change of parent_id and use_parent_address
                if 'use_parent_address' in vals.keys() and record.parent_id:
                    if vals['use_parent_address']:
                        # Change the MySQL partner so that it is linked to the
                        # contact
                        gp.linkContact(uid, record.parent_id.ref, record.id)
                        # Merge contact with parent
                        vals['ref'] = record.parent_id.ref
                    else:
                        # Change the MySQL company partner so that it is no
                        # more linked to the contact of the company
                        gp.unlinkContact(
                            uid, record.parent_id,
                            record.parent_id._get_category_names())
                        # Link with contact in GP and sync or create new
                        # contact if it doesn't exist.
                        ref = gp.getRefContact(uid, record.id)
                        if ref < 0:
                            ref = str(gp.nextRef())
                            create = True
                        vals['ref'] = ref
                    # Write the new reference in OpenERP
                    super(ResPartner, record).write(vals)
                    record.ref = vals['ref']
                    # We need to copy all partner fields in the new referenced
                    # partner in MySQL
                    record._fill_fields(vals)

                # Handle the change of the company status (weird update, but
                # could happen!)
                if 'is_company' in vals.keys():
                    if vals['is_company']:
                        gp.changeToCompany(uid, record.id, record.name)
                    else:
                        # Unlink the contact, if any exists.
                        gp.unlinkContact(
                            uid, record, record._get_category_names())
                        gp.changeToPrivate(uid, record.id)
                        # Add name information to contact
                        vals['lastname'] = record.lastname
            else:
                # The partner has no relation to MySQL ! Fix it.
                record._fill_fields(vals)
                ref = gp.getRefContact(record.id)
                if ref < 0:
                    ref = str(gp.nextRef())
                    create = True
                vals['ref'] = ref
                # Write the new reference in OpenERP
                record.ref = ref
                super(ResPartner, record).write(vals)

            # Get the special fields values
            categories = record._get_category_names()

            # Update the MySQL table with the retrieved values.
            record._update_vals_with_gp_partner(vals)
            if create:
                gp.createPartner(uid, vals, record, categories)
            else:
                gp.updatePartner(uid, vals, record, categories)

        return result

    def unlink(self):
        """ We want to perform some checks before deleting a partner ! """
        uid = self.env.user.id
        gp = gp_connector.GPConnect()
        for record in self:
            # If it is a company, unlink contact in MySQL if there is any.
            if record.is_company:
                for child in record.child_ids:
                    if child.use_parent_address:
                        gp.unlinkContact(
                            uid, record, record._get_category_names())

            # If it is a contact, unlink from company.
            if record.use_parent_address and record.parent_id:
                gp.unlinkContact(
                    uid, record.parent_id,
                    record.parent_id._get_category_names())

            # Unlink from MySQL
            gp.unlink(uid, record.id)

        return super(ResPartner, self).unlink()

    @api.one
    def get_unreconciled_amount(self):
        """Returns the amount of unreconciled credits in Account 1050"""
        mv_line_obj = self.env['account.move.line']
        move_line_ids = mv_line_obj.search([
            ('partner_id', '=', self.id),
            ('account_id.code', '=', '1050'),
            ('credit', '>', '0'),
            ('reconcile_id', '=', False)])
        res = 0
        for move_line in move_line_ids:
            res += move_line.credit
        return res

    ##########################################################################
    #                           PRIVATE METHODS                              #
    ##########################################################################
    @api.one
    def _update_vals_with_gp_partner(self, vals):
        """ Updates the vals dictionary with correct values for GP
        given the type of the partner (company / contact). """
        count = 0

        # By default, update the GP partner with erp_id = partner.id
        vals['id'] = self.id

        # Don't unset the "Mailing complet" category when the parent wants to
        # receive it.
        if self.use_parent_address and not self.parent_id.opt_out and \
           'opt_out' in vals:
            del vals['opt_out']

        # Check if the record is a company with children and update fields
        # accordingly.
        for child_record in self.child_ids:
            if child_record.use_parent_address:
                # Set the correct reference id
                vals['id'] = child_record.id
                # Don't update some values that are defined by the child
                # partner.
                vals.pop('abroad', None)
                vals.pop('opt_out', None)
                vals.pop('thankyou_letter', None)
                vals.pop('tax_certificate', None)
                vals.pop('title', None)
                count += 1
        if count > 1:
            raise exceptions.Wraning(
                _("Partner Error"),
                _("You cannot more than one contact with same address than "
                  "the company. GP does not handle that!"))

    @api.one
    def _get_category_names(self):
        """ Get a list of the category names of all linked partners.

        Args:
            record (res.partner) : The current partner for which we search
                                   the categories

        Returns:
            A list of strings containing the category names of the current
            partner, his parent and children.
        """
        categories_name = self.mapped('category_id.name')

        parent_category_names = self.filtered(
            lambda partner: partner.use_parent_address).mapped(
            'parent_id.category.name').with_context(lang='en_US')
        child_category_names = self.child_ids.filtered(
            lambda partner: partner.use_parent_address).mapped(
                'category.name').with_context(lang='en_US')

        categories_name.append(parent_category_names)
        categories_name.append(child_category_names)

        return categories_name

    @api.multi
    def _fill_fields(self, vals):
        """ Fills all needed vals for the MySQL table. """
        self.ensure_one()
        vals['title'] = self.title.id
        vals['lang'] = self.lang
        vals['firstname'] = self.firstname
        vals['lastname'] = self.lastname
        vals['function'] = self.function
        vals['nbmag'] = self.nbmag
        vals['calendar'] = self.calendar
        vals['tax_certificate'] = self.tax_certificate
        vals['thankyou_letter'] = self.thankyou_letter
        vals['abroad'] = self.abroad
        vals['opt_out'] = self.opt_out
        vals['mobile'] = self.mobile
        vals['category_id'] = self.category_id
        if not self.use_parent_address:
            # Creation of new partner, add all possible fields.
            vals['street'] = self.street
            vals['street2'] = self.street2
            vals['street3'] = self.street3
            vals['city'] = self.city
            vals['zip'] = self.zip
            vals['country_id'] = self.country_id.id
            vals['birthdate'] = self.birthdate
            vals['deathdate'] = self.deathdate
            vals['phone'] = self.phone
            vals['email'] = self.email
            vals['church_id'] = self.church_id.id
            vals['church_unlinked'] = self.church_unlinked
            vals['date'] = self.date
<|endoftext|>