<commit_before>#!/usr/bin/env python
# -- Content-Encoding: UTF-8 --
"""
COHORTE JSON configuration file reader.

Parses JSON files and handles "import-files" and "from-file" special fields.

:author: Thomas Calmant
:license: Apache Software License 2.0

..

    Copyright 2014 isandlaTech

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
"""

# Documentation strings format
__docformat__ = "restructuredtext en"

# Version
__version__ = '1.0.1'

# ------------------------------------------------------------------------------

# COHORTE constants
import cohorte

# iPOPO Decorators
from pelix.ipopo.decorators import ComponentFactory, Provides, Instantiate, \
    Requires, Property

# Python standard library
import json
import logging

# ------------------------------------------------------------------------------

KEY_FILE_FROM = 'from-file'
"""
Key used in configuration file to replace a JSON object by the content of
another file
"""

KEY_FILES_IMPORT = 'import-files'
"""
Key used to merge a JSON object with the content of other files
"""

KEY_OVERRIDDEN_PROPERTIES = 'overriddenProperties'
"""
Key used in configuration file to override the properties of imported files
"""

KEY_PROPERTIES = 'properties'
""" Standard key to store object properties (used by overriddenProperties) """

_logger = logging.getLogger(__name__)

# ------------------------------------------------------------------------------

@ComponentFactory('cohorte-reader-json-factory')
@Provides(cohorte.SERVICE_FILE_READER)
@Property('_handled_formats', 'file.format', ('json', 'js'))
@Requires('_finder', cohorte.SERVICE_FILE_FINDER)
@Instantiate('cohorte-reader-json')
class ConfigurationFileReader(object):
    """
    Reader for isolate configuration files, returning the combined and parsed
    JSON content.

    It handles file imports using the 'from' key
    The overridden properties, defined using the 'overriddenProperties' key,
    are applied on the 'properties' field, then on the object itself.
    """
    def __init__(self):
        """
        Sets up the parser
        """
        # The file finder
        self._finder = None

    def _compute_overridden_props(self, json_object, overriding_props):
        """
        Parses the given properties object and overrides it with the given
        properties

        :param json_object: A properties JSON object (can't be None)
        :param overriding_props: Overriding properties (can be None)
        :return: The overridden properties
        """
        overridden_props = json_object.get(KEY_OVERRIDDEN_PROPERTIES, None)
        if not overridden_props:
            return overriding_props

        if overriding_props is not None:
            overridden_props = overridden_props.copy()
            overridden_props.update(overriding_props)

        return overridden_props

    def _update_properties(self, json_data, overridden_props):
        """
        Updates the properties in the given parsed JSON content in-place

        :param json_data: A parsed JSON content
        :param overridden_props: Dictionary to describe the new values of the
                                 parsed file properties
        """
        if not overridden_props:
            # Nothing to do
            return

        if type(json_data) is list:
            # We have an array, update all of its children
            for json_object in json_data:
                self._update_properties(json_object, overridden_props)
        elif type(json_data) is dict:
            # Update the 'properties' field first
            properties_dict = json_data.get(KEY_PROPERTIES, None)
            if properties_dict and type(properties_dict) is dict:
                properties_dict.update(overridden_props)

            # Update the JSON object fields
            for key, value in overridden_props.items():
                if key in json_data:
                    json_data[key] = value

    def _do_recursive_imports(self, filename, json_data, overridden_props,
                              include_stack):
        """
        Recursively does the file merging according to the indications in the
        given JSON data (object or array).

        **TODO:**
        * Refactor to avoid duplicated lines

        :param filename: The name of the file used to parse this data
                         (can be None)
        :param json_data: A parsed JSON data (object or array)
        :param overridden_props: Properties to override in imported files
        :return: The combined JSON result
        :raise ValueError: Error parsing an imported JSON file
        :raise IOError: Error reading an imported JSON file
        """
        if type(json_data) is list:
            # We have an array, update all of its children
            i = 0
            while i < len(json_data):
                entry = json_data[i]
                new_data = self._do_recursive_imports(filename, entry,
                                                      overridden_props,
                                                      include_stack)
                if new_data is not entry:
                    # The entry has been updated
                    json_data[i] = new_data

                i += 1

            return json_data

        elif type(json_data) is dict:
            # We have an object
            from_filename = json_data.get(KEY_FILE_FROM)
            import_filenames = json_data.get(KEY_FILES_IMPORT)

            if from_filename:
                # Load & return the content of the imported file
                # (this method will be called by _load_file)
                new_props = self._compute_overridden_props(json_data,
                                                           overridden_props)

                imported_data = self._load_file(from_filename, filename,
                                                new_props, include_stack)

                # Update properties
                self._update_properties(imported_data, new_props)

                # Return the imported object
                return imported_data

            elif import_filenames:
                # Load the content of the imported file and merge with local
                # values, i.e. add entries existing only in the imported file
                # and merge arrays
                if type(import_filenames) is not list:
                    import_filenames = [import_filenames]

                # Remove import keys
                del json_data[KEY_FILES_IMPORT]
                if KEY_OVERRIDDEN_PROPERTIES in json_data:
                    del json_data[KEY_OVERRIDDEN_PROPERTIES]

                for import_filename in import_filenames:
                    # Import files
                    imported_data = self._load_file(
                        import_filename, filename, overridden_props,
                        include_stack)

                    # Update properties in imported data
                    self._update_properties(imported_data, overridden_props)

                    # Merge arrays with imported data
                    json_data = self.merge_object(json_data, imported_data)

                    # Do the recursive import
                    for key, value in json_data.items():
                        new_value = self._do_recursive_imports(
                            filename, value, overridden_props, include_stack)
                        if new_value is not value:
                            # The value has been changed
                            json_data[key] = value

                return json_data

            else:
                # Standard object, look into its entries
                for key, value in json_data.items():
                    new_value = self._do_recursive_imports(
                        filename, value, overridden_props, include_stack)
                    if new_value is not value:
                        # The value has been changed
                        json_data[key] = new_value

                return json_data

        # Nothing to do
        return json_data

    def _find_equivalent(self, searched_dict, dicts_list):
        """
        Finds the item in the given list which has the same ID than the given
        dictionary.

        A dictionary is equivalent to another if they have the same value for
        one of the following keys: 'id', 'uid', 'name'.

        :param searched_dict: The dictionary to look for into the list
        :param dicts_list: A list of potential equivalents
        :return: The first item found in the list equivalent to the given
                 dictionary, or None
        """
        for id_key in ('id', 'uid', 'name'):
            # Recognize the ID key used, if any
            local_id = searched_dict.get(id_key)
            if local_id:
                # Found an ID
                for other_item in dicts_list:
                    if other_item.get(id_key) == local_id:
                        # Found an item with the same ID
                        return other_item

        # Found nothing
        return None

    def merge_object(self, local, imported):
        """
        Merges recursively two JSON objects.

        The local values have priority on imported ones.
        Arrays of objects are also merged.

        :param local: The local object, which will receive the merged values
                      (modified in-place)
        :param imported: The imported object, which will be merged into local
        :return: The merge result, i.e. local
        """
        for key, imp_value in imported.items():
            if key not in local:
                # Missing key
                local[key] = imp_value
            else:
                # Get current value
                cur_value = local[key]
                cur_type = type(cur_value)

                if cur_type is not type(imp_value):
                    # Different types found
                    _logger.warning("Trying to merge different types. "
                                    "Ignoring. (key: %s, types: %s / %s)",
                                    key, cur_type, type(imp_value))
                    continue

                if cur_type is dict:
                    # Merge children
                    local[key] = self.merge_object(cur_value, imp_value)
                elif cur_type is list:
                    # Merge arrays
                    new_array = imp_value[:]

                    for cur_item in cur_value:
                        # Merge items
                        if type(cur_item) is dict:
                            # Recognize the ID key used
                            imp_item = self._find_equivalent(cur_item,
                                                             imp_value)
                            if not imp_item:
                                # No equivalent found, append the item
                                new_array.append(cur_item)

                            elif imp_item != cur_item:
                                # Found an equivalent that must be merged
                                self.merge_object(cur_item, imp_item)

                                # Replace the existing entry
                                idx = new_array.index(imp_item)
                                del new_array[idx]
                                new_array.insert(idx, cur_item)

                        elif cur_item not in imp_value:
                            # Append new values
                            new_array.append(cur_item)

                    # Update the object
                    local[key] = new_array

        return local

    def _parse_file(self, filename, overridden_props, include_stack):
        """
        Returns the parsed JSON content of the given file

        :param filename: A JSON file to parse
        :param overridden_props: Properties to override in imported files
        :return: The parsed content (array or dictionary)
        :raise ValueError: Error parsing the file
        :raise IOError: Error reading an imported JSON file
        """
        # Read the file content, removing commented lines
        lines = []
        with open(filename) as filep:
            # Comment block flag
            commented = False
            for line in filep:
                if not commented:
                    # Non commented line
                    try:
                        # Look for a multiline comment
                        start_idx = line.index('/*')

                        # Is the end of the comment on the same line ?
                        end_idx = line.find('*/', start_idx)

                        # We enter a comment block if the end of the comment
                        # is not on this line
                        commented = (end_idx == -1)

                        if commented:
                            # Remove the beginning of the comment
                            line = line[:start_idx]
                        else:
                            # Remove the in-line comment
                            end_idx += len('*/')
                            line = ''.join((line[:start_idx], line[end_idx:]))
                    except ValueError:
                        # No multiline comment found
                        pass

                    try:
                        # Single line comment
                        line = line[:line.index('//')]
                    except ValueError:
                        # No comment
                        pass

                    # Store the treated line
                    if line:
                        lines.append(line)

                else:
                    # We are in a comment block
                    try:
                        # Look for the end of the multiline comment block
                        end_idx = line.index('*/')
                        end_idx += len('*/')
                    except ValueError:
                        # No end of comment
                        pass
                    else:
                        # End of comment found
                        commented = False

                        # Store the line, without comments
                        lines.append(line[end_idx:])

        # Load the JSON data
        json_data = json.loads(''.join(lines))

        # Check imports
        return self._do_recursive_imports(filename, json_data,
                                          overridden_props, include_stack)

    def _load_file(self, filename, base_file, overridden_props, include_stack):
        """
        Parses a configuration file.
        This method shall not be called directly, as it doesn't performs clean
        up if an error occurs

        :param filename: Base name or relative name of the file to load
        :param base_file: If given, search the file near the base file first
        :param overridden_props: Properties to override in imported files
        :return: The parsed JSON content.
        :raise ValueError: Error parsing a JSON file
        :raise IOError: Error reading the configuration file
        """
        # Parse the first matching file
        finder = self._finder.find_rel(filename, base_file)
        try:
            conffile = next(finder)
        except StopIteration:
            # No file found
            raise IOError("File not found: '{0}' (base: {1})"
                          .format(filename, base_file))

        try:
            # Get the first file that is not yet in the include stack
            while conffile in include_stack:
                conffile = next(finder)
        except StopIteration:
            # All found files are already in the inclusion stack
            raise ValueError("Recursive import detected: '{0}' - '{1}'"
                             .format(conffile, include_stack))

        # Store the selected file in the inclusion stack
        include_stack.append(conffile)

        # Parse the file and resolve inclusions
        json_data = self._parse_file(conffile, overridden_props, include_stack)

        # Remove the top of the stack before returning
        include_stack.pop()
        return json_data

    def load_file(self, filename, base_file=None, overridden_props=None,
                  log_error=True):
        """
        Parses a configuration file.
        If a configuration entry has a "from" key, then this entry is replaced
        by the content of the linked file.

        :param filename: Base name or relative name of the file to load
        :param base_file: If given, search the file near the base file first
        :param overridden_props: Properties to override in imported files
        :param log_error: If True, log the encountered I/O error (if any)
        :return: The parsed JSON content.
        :raise ValueError: Error parsing a JSON file
        :raise IOError: Error reading the configuration file
        """
        include_stack = []

        try:
            # Load the file
            return self._load_file(filename, base_file, overridden_props,
                                   include_stack)
        except ValueError as ex:
            # Log parsing errors
            _logger.error("Error parsing file '%s': %s", include_stack[-1], ex)
            raise
        except IOError as ex:
            if log_error:
                # Log the access error
                if include_stack:
                    _logger.error("Error looking for file imported by '%s': "
                                  "%s", include_stack[-1], ex)
                else:
                    _logger.error("Can't read file '%s': %s", filename, ex)

            # Re-raise the error
            raise
<commit_msg>Config Reader: avoid reader '//' in strings as in-line comments<commit_after>#!/usr/bin/env python
# -- Content-Encoding: UTF-8 --
"""
COHORTE JSON configuration file reader.

Parses JSON files and handles "import-files" and "from-file" special fields.

:author: Thomas Calmant
:license: Apache Software License 2.0

..

    Copyright 2014 isandlaTech

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
"""

# Documentation strings format
__docformat__ = "restructuredtext en"

# Version
__version__ = '1.0.1'

# ------------------------------------------------------------------------------

# COHORTE constants
import cohorte

# iPOPO Decorators
from pelix.ipopo.decorators import ComponentFactory, Provides, Instantiate, \
    Requires, Property

# Python standard library
import json
import logging

# ------------------------------------------------------------------------------

KEY_FILE_FROM = 'from-file'
"""
Key used in configuration file to replace a JSON object by the content of
another file
"""

KEY_FILES_IMPORT = 'import-files'
"""
Key used to merge a JSON object with the content of other files
"""

KEY_OVERRIDDEN_PROPERTIES = 'overriddenProperties'
"""
Key used in configuration file to override the properties of imported files
"""

KEY_PROPERTIES = 'properties'
""" Standard key to store object properties (used by overriddenProperties) """

_logger = logging.getLogger(__name__)

# ------------------------------------------------------------------------------

@ComponentFactory('cohorte-reader-json-factory')
@Provides(cohorte.SERVICE_FILE_READER)
@Property('_handled_formats', 'file.format', ('json', 'js'))
@Requires('_finder', cohorte.SERVICE_FILE_FINDER)
@Instantiate('cohorte-reader-json')
class ConfigurationFileReader(object):
    """
    Reader for isolate configuration files, returning the combined and parsed
    JSON content.

    It handles file imports using the 'from' key
    The overridden properties, defined using the 'overriddenProperties' key,
    are applied on the 'properties' field, then on the object itself.
    """
    def __init__(self):
        """
        Sets up the parser
        """
        # The file finder
        self._finder = None

    def _compute_overridden_props(self, json_object, overriding_props):
        """
        Parses the given properties object and overrides it with the given
        properties

        :param json_object: A properties JSON object (can't be None)
        :param overriding_props: Overriding properties (can be None)
        :return: The overridden properties
        """
        overridden_props = json_object.get(KEY_OVERRIDDEN_PROPERTIES, None)
        if not overridden_props:
            return overriding_props

        if overriding_props is not None:
            overridden_props = overridden_props.copy()
            overridden_props.update(overriding_props)

        return overridden_props

    def _update_properties(self, json_data, overridden_props):
        """
        Updates the properties in the given parsed JSON content in-place

        :param json_data: A parsed JSON content
        :param overridden_props: Dictionary to describe the new values of the
                                 parsed file properties
        """
        if not overridden_props:
            # Nothing to do
            return

        if type(json_data) is list:
            # We have an array, update all of its children
            for json_object in json_data:
                self._update_properties(json_object, overridden_props)
        elif type(json_data) is dict:
            # Update the 'properties' field first
            properties_dict = json_data.get(KEY_PROPERTIES, None)
            if properties_dict and type(properties_dict) is dict:
                properties_dict.update(overridden_props)

            # Update the JSON object fields
            for key, value in overridden_props.items():
                if key in json_data:
                    json_data[key] = value

    def _do_recursive_imports(self, filename, json_data, overridden_props,
                              include_stack):
        """
        Recursively does the file merging according to the indications in the
        given JSON data (object or array).

        **TODO:**
        * Refactor to avoid duplicated lines

        :param filename: The name of the file used to parse this data
                         (can be None)
        :param json_data: A parsed JSON data (object or array)
        :param overridden_props: Properties to override in imported files
        :return: The combined JSON result
        :raise ValueError: Error parsing an imported JSON file
        :raise IOError: Error reading an imported JSON file
        """
        if type(json_data) is list:
            # We have an array, update all of its children
            i = 0
            while i < len(json_data):
                entry = json_data[i]
                new_data = self._do_recursive_imports(filename, entry,
                                                      overridden_props,
                                                      include_stack)
                if new_data is not entry:
                    # The entry has been updated
                    json_data[i] = new_data

                i += 1

            return json_data

        elif type(json_data) is dict:
            # We have an object
            from_filename = json_data.get(KEY_FILE_FROM)
            import_filenames = json_data.get(KEY_FILES_IMPORT)

            if from_filename:
                # Load & return the content of the imported file
                # (this method will be called by _load_file)
                new_props = self._compute_overridden_props(json_data,
                                                           overridden_props)

                imported_data = self._load_file(from_filename, filename,
                                                new_props, include_stack)

                # Update properties
                self._update_properties(imported_data, new_props)

                # Return the imported object
                return imported_data

            elif import_filenames:
                # Load the content of the imported file and merge with local
                # values, i.e. add entries existing only in the imported file
                # and merge arrays
                if type(import_filenames) is not list:
                    import_filenames = [import_filenames]

                # Remove import keys
                del json_data[KEY_FILES_IMPORT]
                if KEY_OVERRIDDEN_PROPERTIES in json_data:
                    del json_data[KEY_OVERRIDDEN_PROPERTIES]

                for import_filename in import_filenames:
                    # Import files
                    imported_data = self._load_file(
                        import_filename, filename, overridden_props,
                        include_stack)

                    # Update properties in imported data
                    self._update_properties(imported_data, overridden_props)

                    # Merge arrays with imported data
                    json_data = self.merge_object(json_data, imported_data)

                    # Do the recursive import
                    for key, value in json_data.items():
                        new_value = self._do_recursive_imports(
                            filename, value, overridden_props, include_stack)
                        if new_value is not value:
                            # The value has been changed
                            json_data[key] = value

                return json_data

            else:
                # Standard object, look into its entries
                for key, value in json_data.items():
                    new_value = self._do_recursive_imports(
                        filename, value, overridden_props, include_stack)
                    if new_value is not value:
                        # The value has been changed
                        json_data[key] = new_value

                return json_data

        # Nothing to do
        return json_data

    def _find_equivalent(self, searched_dict, dicts_list):
        """
        Finds the item in the given list which has the same ID than the given
        dictionary.

        A dictionary is equivalent to another if they have the same value for
        one of the following keys: 'id', 'uid', 'name'.

        :param searched_dict: The dictionary to look for into the list
        :param dicts_list: A list of potential equivalents
        :return: The first item found in the list equivalent to the given
                 dictionary, or None
        """
        for id_key in ('id', 'uid', 'name'):
            # Recognize the ID key used, if any
            local_id = searched_dict.get(id_key)
            if local_id:
                # Found an ID
                for other_item in dicts_list:
                    if other_item.get(id_key) == local_id:
                        # Found an item with the same ID
                        return other_item

        # Found nothing
        return None

    def merge_object(self, local, imported):
        """
        Merges recursively two JSON objects.

        The local values have priority on imported ones.
        Arrays of objects are also merged.

        :param local: The local object, which will receive the merged values
                      (modified in-place)
        :param imported: The imported object, which will be merged into local
        :return: The merge result, i.e. local
        """
        for key, imp_value in imported.items():
            if key not in local:
                # Missing key
                local[key] = imp_value
            else:
                # Get current value
                cur_value = local[key]
                cur_type = type(cur_value)

                if cur_type is not type(imp_value):
                    # Different types found
                    _logger.warning("Trying to merge different types. "
                                    "Ignoring. (key: %s, types: %s / %s)",
                                    key, cur_type, type(imp_value))
                    continue

                if cur_type is dict:
                    # Merge children
                    local[key] = self.merge_object(cur_value, imp_value)
                elif cur_type is list:
                    # Merge arrays
                    new_array = imp_value[:]

                    for cur_item in cur_value:
                        # Merge items
                        if type(cur_item) is dict:
                            # Recognize the ID key used
                            imp_item = self._find_equivalent(cur_item,
                                                             imp_value)
                            if not imp_item:
                                # No equivalent found, append the item
                                new_array.append(cur_item)

                            elif imp_item != cur_item:
                                # Found an equivalent that must be merged
                                self.merge_object(cur_item, imp_item)

                                # Replace the existing entry
                                idx = new_array.index(imp_item)
                                del new_array[idx]
                                new_array.insert(idx, cur_item)

                        elif cur_item not in imp_value:
                            # Append new values
                            new_array.append(cur_item)

                    # Update the object
                    local[key] = new_array

        return local

    def _parse_file(self, filename, overridden_props, include_stack):
        """
        Returns the parsed JSON content of the given file

        :param filename: A JSON file to parse
        :param overridden_props: Properties to override in imported files
        :return: The parsed content (array or dictionary)
        :raise ValueError: Error parsing the file
        :raise IOError: Error reading an imported JSON file
        """
        # Read the file content, removing commented lines
        lines = []
        with open(filename) as filep:
            # Comment block flag
            commented = False
            for line in filep:
                if not commented:
                    # Non commented line
                    try:
                        # Look for a multiline comment
                        start_idx = line.index('/*')

                        # Is the end of the comment on the same line ?
                        end_idx = line.find('*/', start_idx)

                        # We enter a comment block if the end of the comment
                        # is not on this line
                        commented = (end_idx == -1)

                        if commented:
                            # Remove the beginning of the comment
                            line = line[:start_idx]
                        else:
                            # Remove the in-line comment
                            end_idx += len('*/')
                            line = ''.join((line[:start_idx], line[end_idx:]))
                    except ValueError:
                        # No multiline comment found
                        pass

                    # Single line comment
                    can_comment = True
                    for idx, char in enumerate(line):
                        if char == '"':
                            can_comment = not can_comment
                        elif char == "/" and line[idx-1] == '/' \
                                and can_comment:
                            # Found a '//' and we're not in a string
                            break
                    else:
                        # No break, no comment found
                        idx = -1

                    if idx > -1:
                        # Valid index, remove the comment
                        line = line[:idx]

                    # Store the treated line
                    if line:
                        lines.append(line)

                else:
                    # We are in a comment block
                    try:
                        # Look for the end of the multi-line comment block
                        end_idx = line.index('*/')
                        end_idx += len('*/')
                    except ValueError:
                        # No end of comment
                        pass
                    else:
                        # End of comment found
                        commented = False

                        # Store the line, without comments
                        lines.append(line[end_idx:])

        # Load the JSON data
        json_data = json.loads(''.join(lines))

        # Check imports
        return self._do_recursive_imports(filename, json_data,
                                          overridden_props, include_stack)

    def _load_file(self, filename, base_file, overridden_props, include_stack):
        """
        Parses a configuration file.
        This method shall not be called directly, as it doesn't performs clean
        up if an error occurs

        :param filename: Base name or relative name of the file to load
        :param base_file: If given, search the file near the base file first
        :param overridden_props: Properties to override in imported files
        :return: The parsed JSON content.
        :raise ValueError: Error parsing a JSON file
        :raise IOError: Error reading the configuration file
        """
        # Parse the first matching file
        finder = self._finder.find_rel(filename, base_file)
        try:
            conffile = next(finder)
        except StopIteration:
            # No file found
            raise IOError("File not found: '{0}' (base: {1})"
                          .format(filename, base_file))

        try:
            # Get the first file that is not yet in the include stack
            while conffile in include_stack:
                conffile = next(finder)
        except StopIteration:
            # All found files are already in the inclusion stack
            raise ValueError("Recursive import detected: '{0}' - '{1}'"
                             .format(conffile, include_stack))

        # Store the selected file in the inclusion stack
        include_stack.append(conffile)

        # Parse the file and resolve inclusions
        json_data = self._parse_file(conffile, overridden_props, include_stack)

        # Remove the top of the stack before returning
        include_stack.pop()
        return json_data

    def load_file(self, filename, base_file=None, overridden_props=None,
                  log_error=True):
        """
        Parses a configuration file.
        If a configuration entry has a "from" key, then this entry is replaced
        by the content of the linked file.

        :param filename: Base name or relative name of the file to load
        :param base_file: If given, search the file near the base file first
        :param overridden_props: Properties to override in imported files
        :param log_error: If True, log the encountered I/O error (if any)
        :return: The parsed JSON content.
        :raise ValueError: Error parsing a JSON file
        :raise IOError: Error reading the configuration file
        """
        include_stack = []

        try:
            # Load the file
            return self._load_file(filename, base_file, overridden_props,
                                   include_stack)
        except ValueError as ex:
            # Log parsing errors
            _logger.error("Error parsing file '%s': %s", include_stack[-1], ex)
            raise
        except IOError as ex:
            if log_error:
                # Log the access error
                if include_stack:
                    _logger.error("Error looking for file imported by '%s': "
                                  "%s", include_stack[-1], ex)
                else:
                    _logger.error("Can't read file '%s': %s", filename, ex)

            # Re-raise the error
            raise
<|endoftext|>


<commit_before>#!/usr/bin/env python2

import os
import sys
import StringIO
import psycopg2
import psycopg2.extensions
from optparse import OptionParser
from ConfigParser import SafeConfigParser
try:
    import bzrlib.plugin
    import bzrlib.builtins
except ImportError:
    pass

def copy_database(conn_parms):
    db_old = conn_parms['database']
    db_new = '%s_migrated' % conn_parms['database']
    print('copying database %(db_old)s to %(db_new)s...' % {'db_old': db_old,
                                                            'db_new': db_new})
    if conn_parms.get('host') == 'False':
        del conn_parms['host']
        del conn_parms['port']
    conn = psycopg2.connect(**conn_parms)
    conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
    cur = conn.cursor()
    cur.execute('drop database if exists "%(db)s"' % {'db': db_new})
    try:
        print "Copying the database using 'with template'"
        cur.execute('create database "%(db_new)s" with template "%(db_old)s"' %
                    {'db_new': db_new, 'db_old': db_old})
        cur.close()
    except psycopg2.OperationalError:
        print "Failed, fallback on creating empty database + loading a dump"
        cur.execute('create database "%(db)s"' % {'db': db_new})
        cur.close()

        os.environ['PGUSER'] = conn_parms['user']
        if conn_parms.get('host') and not os.environ.get('PGHOST'):
            os.environ['PGHOST'] = conn_parms['host']

        if conn_parms.get('port') and not os.environ.get('PGPORT'):
            os.environ['PGPORT'] = conn_parms['port']

        password_set = False
        if conn_parms.get('password') and not os.environ.get('PGPASSWORD'):
            os.environ['PGPASSWORD'] = conn_parms['password']
            password_set = True

        os.system(
            ('pg_dump --format=custom --no-password %(db_old)s ' +
             '| pg_restore --no-password --dbname=%(db_new)s') %
            {'db_old': db_old, 'db_new': db_new}
        )

        if password_set:
            del os.environ['PGPASSWORD']
    return db_new

migrations = {
    '8.0': {
        'addons': {
            'addons': {
                'type': 'link',
                'url': os.path.join('server', 'addons'),
            },
        },
        'server': {
            'type': 'git',
            'url': 'git://github.com/OpenUpgrade/OpenUpgrade.git',
            'branch': '8.0',
            'addons_dir': os.path.join('openerp', 'addons'),
            'root_dir': os.path.join(''),
            'cmd': 'openerp-server --update=all --database=%(db)s '
                   '--config=%(config)s --stop-after-init --no-xmlrpc',
            },
    },
    '7.0': {
        'addons': {
            'addons': {
                'type': 'link',
                'url': os.path.join('server', 'addons'),
            },
        },
        'server': {
            'type': 'git',
            'url': 'git://github.com/OpenUpgrade/OpenUpgrade.git',
            'branch': '7.0',
            'addons_dir': os.path.join('openerp', 'addons'),
            'root_dir': os.path.join(''),
            'cmd': 'openerp-server --update=all --database=%(db)s '
                   '--config=%(config)s --stop-after-init --no-xmlrpc '
                   '--no-netrpc',
        },
    },
    '6.1': {
        'addons': {
            'addons': {
                'type': 'link',
                'url': os.path.join('server', 'addons'),
            },
        },
        'server': {
            'type': 'git',
            'url': 'git://github.com/OpenUpgrade/OpenUpgrade.git',
            'branch': '6.1',
            'addons_dir': os.path.join('openerp', 'addons'),
            'root_dir': os.path.join(''),
            'cmd': 'openerp-server --update=all --database=%(db)s '
                   '--config=%(config)s --stop-after-init --no-xmlrpc '
                   '--no-netrpc',
        },
    },
    '6.0': {
        'addons': {
            'addons': {
                'type': 'link',
                'url': os.path.join('server', 'addons'),
            },
        },
        'server': {
            'type': 'git',
            'url': 'git://github.com/OpenUpgrade/OpenUpgrade.git',
            'branch': '6.0',
            'addons_dir': os.path.join('bin', 'addons'),
            'root_dir': os.path.join('bin'),
            'cmd': 'bin/openerp-server.py --update=all --database=%(db)s '
                   '--config=%(config)s --stop-after-init --no-xmlrpc '
                   '--no-netrpc',
        },
    },
}
config = SafeConfigParser()
parser = OptionParser(
    description='Migrate script for the impatient or lazy. '
    'Makes a copy of your database, downloads the files necessary to migrate '
    'it as requested and runs the migration on the copy (so your original '
    'database will not be touched). While the migration is running only '
    'errors are shown, for a detailed log see ${branch-dir}/migration.log')
parser.add_option(
    "-C", "--config", action="store", type="string",
    dest="config",
    help="current openerp config (required)")
parser.add_option(
    "-D", "--database", action="store", type="string",
    dest="database",
    help="current openerp database (required if not given in config)")
parser.add_option(
    "-B", "--branch-dir", action="store", type="string",
    dest="branch_dir",
    help="the directory to download openupgrade-server code to [%default]",
    default='/var/tmp/openupgrade')
parser.add_option(
    "-R", "--run-migrations", action="store", type="string",
    dest="migrations",
    help="comma separated list of migrations to run, ie. \"" +
    ','.join(sorted([a for a in migrations])) +
    "\" (required)")
parser.add_option(
    "-A", "--add", action="store", type="string", dest="add",
    help="load a python module that declares a dict "
    "'migrations' which is merged with the one of this script "
    "(see the source for details). You also can pass a string "
    "that evaluates to a dict. For the banking addons, pass "
    "\"{'6.1': {'addons': {'banking': 'lp:banking-addons/6.1'}}}\"")
parser.add_option("-I", "--inplace", action="store_true", dest="inplace",
                  help="don't copy database before attempting upgrade "
                  "(dangerous)")
(options, args) = parser.parse_args()

if not options.config or not options.migrations\
        or not reduce(lambda a, b: a and (b in migrations),
                      options.migrations.split(','),
                      True):
    parser.print_help()
    sys.exit()

config.read(options.config)

conn_parms = {}
for parm in ('host', 'port', 'user', 'password'):
    db_parm = 'db_' + parm
    if config.has_option('options', db_parm):
        conn_parms[parm] = config.get('options', db_parm)

if 'user' not in conn_parms:
    print 'No user found in configuration'
    sys.exit()

db_name = options.database or config.get('options', 'db_name')

if not db_name or db_name == '' or db_name.isspace()\
        or db_name.lower() == 'false':
    parser.print_help()
    sys.exit()

conn_parms['database'] = db_name

if options.add:
    merge_migrations = {}
    if os.path.isfile(options.add):
        import imp
        merge_migrations_mod = imp.load_source('merge_migrations_mod',
                                               options.add)
        merge_migrations = merge_migrations_mod.migrations
    else:
        merge_migrations = eval(options.add)

    def deep_update(dict1, dict2):
        result = {}
        for (name, value) in dict1.iteritems():
            if name in dict2:
                if isinstance(dict1[name], dict) and isinstance(dict2[name],
                                                                dict):
                    result[name] = deep_update(dict1[name], dict2[name])
                else:
                    result[name] = dict2[name]
            else:
                result[name] = dict1[name]
        for (name, value) in dict2.iteritems():
            if name not in dict1:
                result[name] = value
        return result

    migrations = deep_update(migrations, merge_migrations)

for version in options.migrations.split(','):
    if version not in migrations:
        print '%s is not a valid version! (valid verions are %s)' % (
            version,
            ','.join(sorted([a for a in migrations])))

bzrlib.plugin.load_plugins()
bzrlib.trace.enable_default_logging()
logfile = os.path.join(options.branch_dir, 'migration.log')

if not os.path.exists(options.branch_dir):
    os.mkdir(options.branch_dir)

for version in options.migrations.split(','):
    if not os.path.exists(os.path.join(options.branch_dir, version)):
        os.mkdir(os.path.join(options.branch_dir, version))
    for (name, addon_config) in dict(
            migrations[version]['addons'],
            server=migrations[version]['server']).iteritems():
        addon_config = addon_config\
            if isinstance(addon_config, dict)\
            else {'url': addon_config}
        addon_config_type = addon_config.get('type', 'bzr')
        if os.path.exists(os.path.join(options.branch_dir, version, name)):
            if addon_config_type == 'link':
                continue
            elif addon_config_type == 'bzr':
                cmd_revno = bzrlib.builtins.cmd_revno()
                cmd_revno.outf = StringIO.StringIO()
                cmd_revno.run(location=os.path.join(options.branch_dir,
                                                    version,
                                                    name))
                print 'updating %s rev%s' % (
                    os.path.join(version, name),
                    cmd_revno.outf.getvalue().strip())
                cmd_update = bzrlib.builtins.cmd_update()
                cmd_update.outf = StringIO.StringIO()
                cmd_update.outf.encoding = 'utf8'
                cmd_update.run(
                    dir=os.path.join(options.branch_dir, version, name))
                if hasattr(cmd_update, '_operation'):
                    cmd_update.cleanup_now()
                print 'now at rev' + cmd_revno.outf.getvalue().strip()
            elif addon_config_type == 'git':
                os.system('cd %(location)s; git pull origin %(branch)s' % {
                    'branch': addon_config.get('branch', 'master'),
                    'location': os.path.join(options.branch_dir,
                                             version,
                                             name),
                })
            else:
                raise Exception('Unknown type %s' % addon_config_type)
        else:
            if addon_config_type == 'link':
                print 'linking %s to %s' % (addon_config['url'],
                                            os.path.join(options.branch_dir,
                                                         version,
                                                         name))
                os.symlink(addon_config['url'],
                           os.path.join(options.branch_dir, version, name))
            elif addon_config_type == 'bzr':
                print 'getting ' + addon_config['url']
                cmd_checkout = bzrlib.builtins.cmd_checkout()
                cmd_checkout.outf = StringIO.StringIO()
                cmd_checkout.run(
                    addon_config['url'],
                    os.path.join(options.branch_dir, version, name),
                    lightweight=True)
            elif addon_config_type == 'git':
                print 'getting ' + addon_config['url']
                os.system('git clone --branch %(branch)s --single-branch '
                          '--depth=1 %(url)s %(target)s' %
                          {
                              'branch': addon_config.get('branch', 'master'),
                              'url': addon_config['url'],
                              'target': os.path.join(options.branch_dir,
                                                     version,
                                                     name),
                          })
            else:
                raise Exception('Unknown type %s' % addon_config_type)

db_name = conn_parms['database']
if not options.inplace:
    db_name = copy_database(conn_parms)

for version in options.migrations.split(','):
    print 'running migration for '+version
    config.set('options', 'without_demo', 'True')
    config.set('options', 'logfile', logfile)
    config.set('options', 'port', 'False')
    config.set('options', 'netport', 'False')
    config.set('options', 'xmlrpc_port', 'False')
    config.set('options', 'netrpc_port', 'False')
    config.set(
        'options',
        'addons_path',
        ','.join(
            [os.path.join(options.branch_dir,
                          version,
                          'server',
                          migrations[version]['server']['addons_dir'])] +
            [os.path.join(options.branch_dir,
                          version,
                          name,
                          addon_conf.get('addons_dir', '')
                          if isinstance(addon_conf, dict) else '')
             for (name, addon_conf)
             in migrations[version]['addons'].iteritems()]))
    config.set(
        'options',
        'root_path',
        os.path.join(
            options.branch_dir,
            version,
            'server',
            migrations[version]['server']['root_dir']))
    config.write(
        open(
            os.path.join(options.branch_dir, version, 'server.cfg'), 'w+'))

    os.system(
        os.path.join(
            options.branch_dir,
            version,
            'server',
            migrations[version]['server']['cmd'] % {
                'db': db_name,
                'config': os.path.join(options.branch_dir, version,
                                       'server.cfg')
            }))
<commit_msg>[ADD] allow to pass force_deps to migrate.py<commit_after>#!/usr/bin/env python2

import os
import sys
import StringIO
import psycopg2
import psycopg2.extensions
from optparse import OptionParser
from ConfigParser import SafeConfigParser
try:
    import bzrlib.plugin
    import bzrlib.builtins
except ImportError:
    pass

def copy_database(conn_parms):
    db_old = conn_parms['database']
    db_new = '%s_migrated' % conn_parms['database']
    print('copying database %(db_old)s to %(db_new)s...' % {'db_old': db_old,
                                                            'db_new': db_new})
    if conn_parms.get('host') == 'False':
        del conn_parms['host']
        del conn_parms['port']
    conn = psycopg2.connect(**conn_parms)
    conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
    cur = conn.cursor()
    cur.execute('drop database if exists "%(db)s"' % {'db': db_new})
    try:
        print "Copying the database using 'with template'"
        cur.execute('create database "%(db_new)s" with template "%(db_old)s"' %
                    {'db_new': db_new, 'db_old': db_old})
        cur.close()
    except psycopg2.OperationalError:
        print "Failed, fallback on creating empty database + loading a dump"
        cur.execute('create database "%(db)s"' % {'db': db_new})
        cur.close()

        os.environ['PGUSER'] = conn_parms['user']
        if conn_parms.get('host') and not os.environ.get('PGHOST'):
            os.environ['PGHOST'] = conn_parms['host']

        if conn_parms.get('port') and not os.environ.get('PGPORT'):
            os.environ['PGPORT'] = conn_parms['port']

        password_set = False
        if conn_parms.get('password') and not os.environ.get('PGPASSWORD'):
            os.environ['PGPASSWORD'] = conn_parms['password']
            password_set = True

        os.system(
            ('pg_dump --format=custom --no-password %(db_old)s ' +
             '| pg_restore --no-password --dbname=%(db_new)s') %
            {'db_old': db_old, 'db_new': db_new}
        )

        if password_set:
            del os.environ['PGPASSWORD']
    return db_new

migrations = {
    '8.0': {
        'addons': {
            'addons': {
                'type': 'link',
                'url': os.path.join('server', 'addons'),
            },
        },
        'server': {
            'type': 'git',
            'url': 'git://github.com/OpenUpgrade/OpenUpgrade.git',
            'branch': '8.0',
            'addons_dir': os.path.join('openerp', 'addons'),
            'root_dir': os.path.join(''),
            'cmd': 'openerp-server --update=all --database=%(db)s '
                   '--config=%(config)s --stop-after-init --no-xmlrpc',
            },
    },
    '7.0': {
        'addons': {
            'addons': {
                'type': 'link',
                'url': os.path.join('server', 'addons'),
            },
        },
        'server': {
            'type': 'git',
            'url': 'git://github.com/OpenUpgrade/OpenUpgrade.git',
            'branch': '7.0',
            'addons_dir': os.path.join('openerp', 'addons'),
            'root_dir': os.path.join(''),
            'cmd': 'openerp-server --update=all --database=%(db)s '
                   '--config=%(config)s --stop-after-init --no-xmlrpc '
                   '--no-netrpc',
        },
    },
    '6.1': {
        'addons': {
            'addons': {
                'type': 'link',
                'url': os.path.join('server', 'addons'),
            },
        },
        'server': {
            'type': 'git',
            'url': 'git://github.com/OpenUpgrade/OpenUpgrade.git',
            'branch': '6.1',
            'addons_dir': os.path.join('openerp', 'addons'),
            'root_dir': os.path.join(''),
            'cmd': 'openerp-server --update=all --database=%(db)s '
                   '--config=%(config)s --stop-after-init --no-xmlrpc '
                   '--no-netrpc',
        },
    },
    '6.0': {
        'addons': {
            'addons': {
                'type': 'link',
                'url': os.path.join('server', 'addons'),
            },
        },
        'server': {
            'type': 'git',
            'url': 'git://github.com/OpenUpgrade/OpenUpgrade.git',
            'branch': '6.0',
            'addons_dir': os.path.join('bin', 'addons'),
            'root_dir': os.path.join('bin'),
            'cmd': 'bin/openerp-server.py --update=all --database=%(db)s '
                   '--config=%(config)s --stop-after-init --no-xmlrpc '
                   '--no-netrpc',
        },
    },
}
config = SafeConfigParser()
parser = OptionParser(
    description='Migrate script for the impatient or lazy. '
    'Makes a copy of your database, downloads the files necessary to migrate '
    'it as requested and runs the migration on the copy (so your original '
    'database will not be touched). While the migration is running only '
    'errors are shown, for a detailed log see ${branch-dir}/migration.log')
parser.add_option(
    "-C", "--config", action="store", type="string",
    dest="config",
    help="current openerp config (required)")
parser.add_option(
    "-D", "--database", action="store", type="string",
    dest="database",
    help="current openerp database (required if not given in config)")
parser.add_option(
    "-B", "--branch-dir", action="store", type="string",
    dest="branch_dir",
    help="the directory to download openupgrade-server code to [%default]",
    default='/var/tmp/openupgrade')
parser.add_option(
    "-R", "--run-migrations", action="store", type="string",
    dest="migrations",
    help="comma separated list of migrations to run, ie. \"" +
    ','.join(sorted([a for a in migrations])) +
    "\" (required)")
parser.add_option(
    "-A", "--add", action="store", type="string", dest="add",
    help="load a python module that declares a dict "
    "'migrations' which is merged with the one of this script "
    "(see the source for details). You also can pass a string "
    "that evaluates to a dict. For the banking addons, pass "
    "\"{'6.1': {'addons': {'banking': 'lp:banking-addons/6.1'}}}\"")
parser.add_option("-I", "--inplace", action="store_true", dest="inplace",
                  help="don't copy database before attempting upgrade "
                  "(dangerous)")
parser.add_option(
    "-F", "--force-deps", action="store", dest="force_deps",
    help="force dependencies from a dict of the form \"{'module_name': "
    "['new_dependency1', 'new_dependency2']}\"")
(options, args) = parser.parse_args()

if not options.config or not options.migrations\
        or not reduce(lambda a, b: a and (b in migrations),
                      options.migrations.split(','),
                      True):
    parser.print_help()
    sys.exit()

config.read(options.config)

conn_parms = {}
for parm in ('host', 'port', 'user', 'password'):
    db_parm = 'db_' + parm
    if config.has_option('options', db_parm):
        conn_parms[parm] = config.get('options', db_parm)

if 'user' not in conn_parms:
    print 'No user found in configuration'
    sys.exit()

db_name = options.database or config.get('options', 'db_name')

if not db_name or db_name == '' or db_name.isspace()\
        or db_name.lower() == 'false':
    parser.print_help()
    sys.exit()

conn_parms['database'] = db_name

if options.add:
    merge_migrations = {}
    if os.path.isfile(options.add):
        import imp
        merge_migrations_mod = imp.load_source('merge_migrations_mod',
                                               options.add)
        merge_migrations = merge_migrations_mod.migrations
    else:
        merge_migrations = eval(options.add)

    def deep_update(dict1, dict2):
        result = {}
        for (name, value) in dict1.iteritems():
            if name in dict2:
                if isinstance(dict1[name], dict) and isinstance(dict2[name],
                                                                dict):
                    result[name] = deep_update(dict1[name], dict2[name])
                else:
                    result[name] = dict2[name]
            else:
                result[name] = dict1[name]
        for (name, value) in dict2.iteritems():
            if name not in dict1:
                result[name] = value
        return result

    migrations = deep_update(migrations, merge_migrations)

for version in options.migrations.split(','):
    if version not in migrations:
        print '%s is not a valid version! (valid verions are %s)' % (
            version,
            ','.join(sorted([a for a in migrations])))

bzrlib.plugin.load_plugins()
bzrlib.trace.enable_default_logging()
logfile = os.path.join(options.branch_dir, 'migration.log')

if not os.path.exists(options.branch_dir):
    os.mkdir(options.branch_dir)

for version in options.migrations.split(','):
    if not os.path.exists(os.path.join(options.branch_dir, version)):
        os.mkdir(os.path.join(options.branch_dir, version))
    for (name, addon_config) in dict(
            migrations[version]['addons'],
            server=migrations[version]['server']).iteritems():
        addon_config = addon_config\
            if isinstance(addon_config, dict)\
            else {'url': addon_config}
        addon_config_type = addon_config.get('type', 'bzr')
        if os.path.exists(os.path.join(options.branch_dir, version, name)):
            if addon_config_type == 'link':
                continue
            elif addon_config_type == 'bzr':
                cmd_revno = bzrlib.builtins.cmd_revno()
                cmd_revno.outf = StringIO.StringIO()
                cmd_revno.run(location=os.path.join(options.branch_dir,
                                                    version,
                                                    name))
                print 'updating %s rev%s' % (
                    os.path.join(version, name),
                    cmd_revno.outf.getvalue().strip())
                cmd_update = bzrlib.builtins.cmd_update()
                cmd_update.outf = StringIO.StringIO()
                cmd_update.outf.encoding = 'utf8'
                cmd_update.run(
                    dir=os.path.join(options.branch_dir, version, name))
                if hasattr(cmd_update, '_operation'):
                    cmd_update.cleanup_now()
                print 'now at rev' + cmd_revno.outf.getvalue().strip()
            elif addon_config_type == 'git':
                os.system('cd %(location)s; git pull origin %(branch)s' % {
                    'branch': addon_config.get('branch', 'master'),
                    'location': os.path.join(options.branch_dir,
                                             version,
                                             name),
                })
            else:
                raise Exception('Unknown type %s' % addon_config_type)
        else:
            if addon_config_type == 'link':
                print 'linking %s to %s' % (addon_config['url'],
                                            os.path.join(options.branch_dir,
                                                         version,
                                                         name))
                os.symlink(addon_config['url'],
                           os.path.join(options.branch_dir, version, name))
            elif addon_config_type == 'bzr':
                print 'getting ' + addon_config['url']
                cmd_checkout = bzrlib.builtins.cmd_checkout()
                cmd_checkout.outf = StringIO.StringIO()
                cmd_checkout.run(
                    addon_config['url'],
                    os.path.join(options.branch_dir, version, name),
                    lightweight=True)
            elif addon_config_type == 'git':
                print 'getting ' + addon_config['url']
                os.system('git clone --branch %(branch)s --single-branch '
                          '--depth=1 %(url)s %(target)s' %
                          {
                              'branch': addon_config.get('branch', 'master'),
                              'url': addon_config['url'],
                              'target': os.path.join(options.branch_dir,
                                                     version,
                                                     name),
                          })
            else:
                raise Exception('Unknown type %s' % addon_config_type)

db_name = conn_parms['database']
if not options.inplace:
    db_name = copy_database(conn_parms)

for version in options.migrations.split(','):
    print 'running migration for '+version
    config.set('options', 'without_demo', 'True')
    config.set('options', 'logfile', logfile)
    config.set('options', 'port', 'False')
    config.set('options', 'netport', 'False')
    config.set('options', 'xmlrpc_port', 'False')
    config.set('options', 'netrpc_port', 'False')
    config.set(
        'options',
        'addons_path',
        ','.join(
            [os.path.join(options.branch_dir,
                          version,
                          'server',
                          migrations[version]['server']['addons_dir'])] +
            [os.path.join(options.branch_dir,
                          version,
                          name,
                          addon_conf.get('addons_dir', '')
                          if isinstance(addon_conf, dict) else '')
             for (name, addon_conf)
             in migrations[version]['addons'].iteritems()]))
    config.set(
        'options',
        'root_path',
        os.path.join(
            options.branch_dir,
            version,
            'server',
            migrations[version]['server']['root_dir']))
    if options.force_deps:
        if not config.has_section('openupgrade'):
            config.add_section('openupgrade')
        config.set('openupgrade', 'force_deps', options.force_deps)
    config.write(
        open(
            os.path.join(options.branch_dir, version, 'server.cfg'), 'w+'))

    os.system(
        os.path.join(
            options.branch_dir,
            version,
            'server',
            migrations[version]['server']['cmd'] % {
                'db': db_name,
                'config': os.path.join(options.branch_dir, version,
                                       'server.cfg')
            }))
<|endoftext|>


<commit_before>#coding=utf-8

import re
import json
import logging
import random
from channels import Group
from channels import Channel
from channels.sessions import channel_session
from .models import Room
from .models import Player

log = logging.getLogger(__name__)

noEnoughPeople = 'not enough people in the room'
gameHasStarted = 'game has started'
gameNotStarted = 'game does not start'
notReady = 'someone is not ready'
notRightPerson = 'You are not the right person to vote'
voteInfo = 'You vote '
dayerror = 'This is in the day'
nighterror = 'This is in the night'
identification = 'Your identification is '

identificationDict = dict()
identificationDict[0] = 'cunmin'
identificationDict[1] = 'langren'
identificationDict[2] = 'yuyanjia'
identificationDict[3] = 'nvwu'
identificationDict[4] = 'lieren'
identificationDict[5] = 'shouwei'

def sendMessage(label, name, messageInfo, typo):
    message = dict()
    message['handle'] = 'system'
    message['typo'] = typo
    message['message'] = messageInfo
    try:
        room = Room.objects.get(label=label)
    except Room.DoesNotExist:
        log.debug('ws room does not exist label=%s', label)
        return
    m = room.messages.create(**message)
    Channel(name).send({'text': json.dumps(m.as_dict())})

def sendGroupMessage(label, messageInfo, typo):
    message = dict()
    message['handle'] = 'system'
    message['typo'] = typo
    message['message'] = messageInfo
    try:
        room = Room.objects.get(label=label)
    except Room.DoesNotExist:
        log.debug('ws room does not exist label=%s', label)
        return
    m = room.messages.create(**message)
    Group('chat-'+label).send({'text': json.dumps(m.as_dict())})

def judgement(label):
    try:
        room = Room.objects.get(label=label)
    except Room.DoesNotExist:
        log.debug('ws room does not exist label=%s', label)
        return -1
    roleList = room.roleList.split(",")
    cunMin = roleList[0]
    langRen = roleList[1]
    shenMin = room.playerNumber - cunMin - langRen
    for player in room.players.all():
        if player.alive == 0:
            if player.identification == 0:
                cunMin = cunMin - 1
            elif player.identification == 1:
                langRen = langRen - 1
            else:
                shenMin = shenMin - 1
    if cunMin == 0 or shenMin == 0 or langRen >= cunMin + shenMin:
        return 1
    elif langRen == 0:
        return 2
    else:
        return 3

def judgementView(label, name):
    try:
        room = Room.objects.get(label=label)
    except Room.DoesNotExist:
        log.debug('ws room does not exist label=%s', label)
        sendMessage(label, name, 'room does not exist!', 'error')
        return
    cunmin = ''
    langren = ''
    yuyanjia = ''
    nvwu = ''
    shouwei = ''
    lieren = ''
    for player in room.players.all():
        if player.identification == 0:
            cunmin = cunmin + player.position + ' '
        elif player.identification == 1:
            langren = langren + player.position + ' '
        elif player.identification == 2:
            yuyanjia = yuyanjia + player.position + ' '
        elif player.identification == 3:
            lieren = lieren + player.position + ' '
        elif player.identification == 4:
            nvwu = nvwu + player.position + ' '
        elif player.identification == 5:
            shouwei = shouwei + player.position + ' '
    Info = 'Identification list \ '
    # if cunmin.len() > 0:
    #     Info = Info + 'cunmin: ' + cunmin + '\ '
    # if langren.len() > 0:
    #     Info = Info + 'langren: ' + langren + '\ '
    # if yuyanjia.len() > 0:
    #     Info = Info + 'yuyanjia: ' + yuyanjia + '\ '
    # if lieren.len() > 0:
    #     Info = Info + 'lieren: ' + lieren + '\ '
    # if nvwu.len() > 0:
    #     Info = Info + 'nvwu: ' + nvwu + '\ '
    # if shouwei.len() > 0:
    #     Info = Info + 'shouwei: ' + shouwei + '\ '
    sendMessage(label, name, Info, 'message')

def room_status(label, number, gameStatus):
    try:
        room = Room.objects.get(label=label)
    except Room.DoesNotExist:
        log.debug('ws room does not exist label=%s', label)
        return -2

def startGame(label):
    try:
        room = Room.objects.get(label=label)
    except Room.DoesNotExist:
        log.debug('ws room does not exist label=%s', label)
        sendGroupMessage(label, 'room does not exist!', 'error')
        return
    room.gameStart = 1
    room.save()
    roleList = room.roleList.split(",")
    playerList = []
    gameStatus = []
    gameStatus.append(0)
    gameStatus.append(1)
    for i in range(0, int(roleList[0])):
        playerList.append(0)
    for i in range(0, int(roleList[1])):
        playerList.append(1)
    for i in range(0, int(roleList[2])):
        playerList.append(2)
    if roleList[2] is not 0:
        gameStatus.append(2)
    for i in range(0, int(roleList[3])):
        playerList.append(3)
    if roleList[3] is not 0:
        gameStatus.append(3)
    for i in range(0, int(roleList[4])):
        playerList.append(4)
    if roleList[4] is not 0:
        gameStatus.append(4)
    for i in range(0, int(roleList[5])):
        playerList.append(5)
    if roleList[5] is not 0:
        gameStatus.append(5)
    random.shuffle(playerList)
    for i in range(1, room.playerNumber + 1):
        player = room.players.filter(position=i).first()
        player.identification = playerList[i - 1]
        player.save()
    sendGroupMessage(label, 'identification is ready!', 'message')

@channel_session
def ws_connect(message):
    # Extract the room from the message. This expects message.path to be of the
    # form /chat/{label}/, and finds a Room if the message path is applicable,
    # and if the Room exists. Otherwise, bails (meaning this is a some othersort
    # of websocket). So, this is effectively a version of _get_object_or_404.
    try:
        prefix, label = message['path'].decode('ascii').strip('/').split('/')
        if prefix != 'chat':
            log.debug('invalid ws path=%s', message['path'])
            return
        room = Room.objects.get(label=label)

    except ValueError:
        log.debug('invalid ws path=%s', message['path'])
        return
    except Room.DoesNotExist:
        log.debug('ws room does not exist label=%s', label)
        return

    log.debug('chat connect room=%s client=%s:%s',
        room.label, message['client'][0], message['client'][1])

    if room.playerNumber == room.currentNumber:
        log.debug('room is full')
        return
    if room.gameStart == 1:
        log.debug('game has been started!')
        return
    # Need to be explicit about the channel layer so that testability works
    # This may be a FIXME?
    Room.objects.filter(label=label).update(currentNumber=room.currentNumber + 1)
    Group('chat-'+label).add(message.reply_channel)
    message.channel_session['room'] = room.label

@channel_session
def ws_receive(message):
    # Look up the room from the channel session, bailing if it doesn't exist
    try:
        label = message.channel_session['room']
        room = Room.objects.get(label=label)
    except KeyError:
        log.debug('no room in channel_session')
        return
    except Room.DoesNotExist:
        log.debug('recieved message, buy room does not exist label=%s', label)
        return

    # Parse out a chat message from the content text, bailing if it doesn't
    # conform to the expected message format.
    try:
        data = json.loads(message['text'])
    except ValueError:
        log.debug("ws message isn't json text=%s", text)
        return

    if set(data.keys()) != set(('handle', 'message', 'typo')):
        log.debug("ws message unexpected format data=%s", data)
        return

    if data:
        player = None
        try:
            player = room.players.filter(position=data['handle']).first()
        except ValueError:
            log.debug("something is wrong")
        if player is not None:
            if player.address != message.reply_channel.name:
                log.debug("this room's position has been occupied by another guy")
                sendMessage(room.label, message.reply_channel.name, "this room's position has been occupied by another guy", 'error')
        else:
            room.players.create(position=data['handle'],address=message.reply_channel.name)
        log.debug('chat message room=%s handle=%s message=%s',
            room.label, data['handle'], data['message'])
        if data['typo'] == 'startGame':
            if room.currentNumber < room.playerNumber:
                sendMessage(room.label, message.reply_channel.name, noEnoughPeople, 'error')
            elif room.gameStart == 1:
                sendMessage(room.label, message.reply_channel.name, gameHasStarted, 'error')
            elif room.players.all().count() < room.playerNumber:
                sendMessage(room.label, message.reply_channel.name, notReady, 'error')
            else:
                sendGroupMessage(room.label, 'Game Starts!', 'message')
                startGame(label)
        elif data['typo'] == 'Vote':
                sendMessage(room.label, message.reply_channel.name, voteInfo + data['message'], 'message')
        elif data['typo'] == 'posion':
            if room.gameStart == 0:
                sendMessage(room.label, message.reply_channel.name, gameNotStarted, 'error')
        elif data['typo'] == 'heal':
            if room.gameStart == 0:
                sendMessage(room.label, message.reply_channel.name, gameNotStarted, 'error')
        elif data['typo'] == 'bloom':
            if room.gameStart == 0:
                sendMessage(room.label, message.reply_channel.name, gameNotStarted, 'error')
        elif data['typo'] == 'identification':
            if room.gameStart == 0:
                sendMessage(room.label, message.reply_channel.name, gameNotStarted, 'error')
            else:
                player = room.players.filter(position=data['handle']).first()
                sendMessage(room.label, message.reply_channel.name, identification + identificationDict[player.identification], 'message')
        elif data['typo'] == 'judgement':
            if room.gameStart == 0:
                sendMessage(room.label, message.reply_channel.name, gameNotStarted, 'error')
            else:
                player = room.players.filter(position=data['handle']).first()
                judgementView(label, player.address)

        #m = room.messages.create(**data)

        # See above for the note about Group
        #Group('chat-'+label).send({'text': json.dumps(m.as_dict())})

@channel_session
def ws_disconnect(message):
    try:
        label = message.channel_session['room']
        room = Room.objects.get(label=label)
        Group('chat-'+label).discard(message.reply_channel)
        player = room.players.filter(address=message.reply_channel.name).first()
        if player is not None:
            Room.objects.filter(label=label).update(currentNumber=room.currentNumber - 1)
            room.players.filter(address=message.reply_channel.name).delete()
    except (KeyError, Room.DoesNotExist):
        pass
<commit_msg>add backend function for judgement and judgementview<commit_after>#coding=utf-8

import re
import json
import logging
import random
from channels import Group
from channels import Channel
from channels.sessions import channel_session
from .models import Room
from .models import Player

log = logging.getLogger(__name__)

noEnoughPeople = 'not enough people in the room'
gameHasStarted = 'game has started'
gameNotStarted = 'game does not start'
notReady = 'someone is not ready'
notRightPerson = 'You are not the right person to vote'
voteInfo = 'You vote '
dayerror = 'This is in the day'
nighterror = 'This is in the night'
identification = 'Your identification is '

identificationDict = dict()
identificationDict[0] = 'cunmin'
identificationDict[1] = 'langren'
identificationDict[2] = 'yuyanjia'
identificationDict[3] = 'nvwu'
identificationDict[4] = 'lieren'
identificationDict[5] = 'shouwei'

def sendMessage(label, name, messageInfo, typo):
    message = dict()
    message['handle'] = 'system'
    message['typo'] = typo
    message['message'] = messageInfo
    try:
        room = Room.objects.get(label=label)
    except Room.DoesNotExist:
        log.debug('ws room does not exist label=%s', label)
        return
    m = room.messages.create(**message)
    Channel(name).send({'text': json.dumps(m.as_dict())})

def sendGroupMessage(label, messageInfo, typo):
    message = dict()
    message['handle'] = 'system'
    message['typo'] = typo
    message['message'] = messageInfo
    try:
        room = Room.objects.get(label=label)
    except Room.DoesNotExist:
        log.debug('ws room does not exist label=%s', label)
        return
    m = room.messages.create(**message)
    Group('chat-'+label).send({'text': json.dumps(m.as_dict())})

def judgement(label):
    try:
        room = Room.objects.get(label=label)
    except Room.DoesNotExist:
        log.debug('ws room does not exist label=%s', label)
        return -1
    roleList = room.roleList.split(",")
    cunMin = roleList[0]
    langRen = roleList[1]
    shenMin = room.playerNumber - cunMin - langRen
    for player in room.players.all():
        if player.alive == 0:
            if player.identification == 0:
                cunMin = cunMin - 1
            elif player.identification == 1:
                langRen = langRen - 1
            else:
                shenMin = shenMin - 1
    if cunMin == 0 or shenMin == 0 or langRen >= cunMin + shenMin:
        return 1
    elif langRen == 0:
        return 2
    else:
        return 3

def judgementView(label, name):
    try:
        room = Room.objects.get(label=label)
    except Room.DoesNotExist:
        log.debug('ws room does not exist label=%s', label)
        sendMessage(label, name, 'room does not exist!', 'error')
        return
    cunmin = ''
    langren = ''
    yuyanjia = ''
    nvwu = ''
    shouwei = ''
    lieren = ''
    for player in room.players.all():
        if player.identification == 0:
            cunmin = cunmin + player.position + ' '
        elif player.identification == 1:
            langren = langren + player.position + ' '
        elif player.identification == 2:
            yuyanjia = yuyanjia + player.position + ' '
        elif player.identification == 3:
            lieren = lieren + player.position + ' '
        elif player.identification == 4:
            nvwu = nvwu + player.position + ' '
        elif player.identification == 5:
            shouwei = shouwei + player.position + ' '
    Info = 'Identification list \ '
    # if cunmin.len() > 0:
    #     Info = Info + 'cunmin: ' + cunmin + '\ '
    # if langren.len() > 0:
    #     Info = Info + 'langren: ' + langren + '\ '
    # if yuyanjia.len() > 0:
    #     Info = Info + 'yuyanjia: ' + yuyanjia + '\ '
    # if lieren.len() > 0:
    #     Info = Info + 'lieren: ' + lieren + '\ '
    # if nvwu.len() > 0:
    #     Info = Info + 'nvwu: ' + nvwu + '\ '
    # if shouwei.len() > 0:
    #     Info = Info + 'shouwei: ' + shouwei + '\ '
    sendMessage(label, name, Info, 'message')

def room_status(label, number, gameStatus):
    try:
        room = Room.objects.get(label=label)
    except Room.DoesNotExist:
        log.debug('ws room does not exist label=%s', label)
        return -2

def startGame(label):
    try:
        room = Room.objects.get(label=label)
    except Room.DoesNotExist:
        log.debug('ws room does not exist label=%s', label)
        sendGroupMessage(label, 'room does not exist!', 'error')
        return
    room.gameStart = 1
    room.save()
    roleList = room.roleList.split(",")
    playerList = []
    gameStatus = []
    gameStatus.append(0)
    gameStatus.append(1)
    for i in range(0, int(roleList[0])):
        playerList.append(0)
    for i in range(0, int(roleList[1])):
        playerList.append(1)
    for i in range(0, int(roleList[2])):
        playerList.append(2)
    if roleList[2] is not 0:
        gameStatus.append(2)
    for i in range(0, int(roleList[3])):
        playerList.append(3)
    if roleList[3] is not 0:
        gameStatus.append(3)
    for i in range(0, int(roleList[4])):
        playerList.append(4)
    if roleList[4] is not 0:
        gameStatus.append(4)
    for i in range(0, int(roleList[5])):
        playerList.append(5)
    if roleList[5] is not 0:
        gameStatus.append(5)
    random.shuffle(playerList)
    for i in range(1, room.playerNumber + 1):
        player = room.players.filter(position=i).first()
        player.identification = playerList[i - 1]
        player.save()
    sendGroupMessage(label, 'identification is ready!', 'message')

@channel_session
def ws_connect(message):
    # Extract the room from the message. This expects message.path to be of the
    # form /chat/{label}/, and finds a Room if the message path is applicable,
    # and if the Room exists. Otherwise, bails (meaning this is a some othersort
    # of websocket). So, this is effectively a version of _get_object_or_404.
    try:
        prefix, label = message['path'].decode('ascii').strip('/').split('/')
        if prefix != 'chat':
            log.debug('invalid ws path=%s', message['path'])
            return
        room = Room.objects.get(label=label)

    except ValueError:
        log.debug('invalid ws path=%s', message['path'])
        return
    except Room.DoesNotExist:
        log.debug('ws room does not exist label=%s', label)
        return

    log.debug('chat connect room=%s client=%s:%s',
        room.label, message['client'][0], message['client'][1])

    if room.playerNumber == room.currentNumber:
        log.debug('room is full')
        return
    if room.gameStart == 1:
        log.debug('game has been started!')
        return
    # Need to be explicit about the channel layer so that testability works
    # This may be a FIXME?
    Room.objects.filter(label=label).update(currentNumber=room.currentNumber + 1)
    Group('chat-'+label).add(message.reply_channel)
    message.channel_session['room'] = room.label

@channel_session
def ws_receive(message):
    # Look up the room from the channel session, bailing if it doesn't exist
    try:
        label = message.channel_session['room']
        room = Room.objects.get(label=label)
    except KeyError:
        log.debug('no room in channel_session')
        return
    except Room.DoesNotExist:
        log.debug('recieved message, buy room does not exist label=%s', label)
        return

    # Parse out a chat message from the content text, bailing if it doesn't
    # conform to the expected message format.
    try:
        data = json.loads(message['text'])
    except ValueError:
        log.debug("ws message isn't json text=%s", text)
        return

    if set(data.keys()) != set(('handle', 'message', 'typo')):
        log.debug("ws message unexpected format data=%s", data)
        return

    if data:
        player = None
        try:
            player = room.players.filter(position=data['handle']).first()
        except ValueError:
            log.debug("something is wrong")
        if player is not None:
            if player.address != message.reply_channel.name:
                log.debug("this room's position has been occupied by another guy")
                sendMessage(room.label, message.reply_channel.name, "this room's position has been occupied by another guy", 'error')
        else:
            room.players.create(position=data['handle'],address=message.reply_channel.name)
        log.debug('chat message room=%s handle=%s message=%s',
            room.label, data['handle'], data['message'])
        if data['typo'] == 'startGame':
            if room.currentNumber < room.playerNumber:
                sendMessage(room.label, message.reply_channel.name, noEnoughPeople, 'error')
            elif room.gameStart == 1:
                sendMessage(room.label, message.reply_channel.name, gameHasStarted, 'error')
            elif room.players.all().count() < room.playerNumber:
                sendMessage(room.label, message.reply_channel.name, notReady, 'error')
            else:
                sendGroupMessage(room.label, 'Game Starts!', 'message')
                startGame(label)
        elif data['typo'] == 'Vote':
                sendMessage(room.label, message.reply_channel.name, voteInfo + data['message'], 'message')
        elif data['typo'] == 'posion':
            if room.gameStart == 0:
                sendMessage(room.label, message.reply_channel.name, gameNotStarted, 'error')
        elif data['typo'] == 'heal':
            if room.gameStart == 0:
                sendMessage(room.label, message.reply_channel.name, gameNotStarted, 'error')
        elif data['typo'] == 'bloom':
            if room.gameStart == 0:
                sendMessage(room.label, message.reply_channel.name, gameNotStarted, 'error')
        elif data['typo'] == 'identification':
            if room.gameStart == 0:
                sendMessage(room.label, message.reply_channel.name, gameNotStarted, 'error')
            else:
                player = room.players.filter(position=data['handle']).first()
                sendMessage(room.label, message.reply_channel.name, identification + identificationDict[player.identification], 'message')
        elif data['typo'] == 'judgement':
            if room.gameStart == 0:
                sendMessage(room.label, message.reply_channel.name, gameNotStarted, 'error')
            else:
                judgementView(room.label, message.reply_channel.name)

        #m = room.messages.create(**data)

        # See above for the note about Group
        #Group('chat-'+label).send({'text': json.dumps(m.as_dict())})

@channel_session
def ws_disconnect(message):
    try:
        label = message.channel_session['room']
        room = Room.objects.get(label=label)
        Group('chat-'+label).discard(message.reply_channel)
        player = room.players.filter(address=message.reply_channel.name).first()
        if player is not None:
            Room.objects.filter(label=label).update(currentNumber=room.currentNumber - 1)
            room.players.filter(address=message.reply_channel.name).delete()
    except (KeyError, Room.DoesNotExist):
        pass
<|endoftext|>