<commit_before>"""Parses a given header in order to generate the RPC interface of it."""

import datetime
import os
import re
import sys

import CppHeaderParser as header_parser

CLASS_HEADER = """
class Rpc{name} {{
  private:
    foobar::{name} api;
  public:
{methods}
}};"""

HEADER_HEADER = """// Generated on {date}
#pragma once
#include <string>
#include <map>
#include <tuple>

#include "stdafx.h"
#include "event.h"
#include "percolate.h"
#include "api/{header}"
#include "factory.h"

using namespace std;
using namespace serialization;

{cls}"""

EVENT_ONLY = """
    Payload {name}(vector<char> & _unused) {{
      Event event;
      fb2k::inMainThread([&] {{
        api.{name}(event);
      }});
      event.wait();
      return serialization::serializer.packed_bool(true);
    }}"""

PARAM_ONLY = """
    Payload {name}(vector<char> & buffer) {{
      {param_type} param(serialization::serializer.unpack<{param_param_type}>(buffer));
      Event event;
      fb2k::inMainThread([&] {{
        api.{name}(param, event);
      }});
      event.wait();
      return serialization::serializer.packed_bool(true);
    }}"""

RESULT_ONLY = """
    Payload {name}(vector<char> & buffer) {{
      {type} result;
      fb2k::inMainThread([&] {{
        api.{name}(result);
      }});
      result.wait();
      return serialization::serializer.packed_result(result);
    }}"""

RESULT_AND_PARAM = """
    Payload {name}(vector<char> & buffer) {{
      {param_type} param(serialization::serializer.unpack<{param_param_type}>(buffer));
      {result_type} result;
      fb2k::inMainThread([&] {{
        api.{name}(param, result);
      }});
      result.wait();
      return serialization::serializer.packed_result(result);
    }}"""

def _generate_method_from_parameters(name, parameters):
    if len(parameters) == 1:
        parameter = parameters[0]
        parameter_type = parameter['raw_type']
        if parameter_type.find('ApiResult') > -1:
            return RESULT_ONLY.format(name=name, type=parameter_type)

        elif parameter_type.find('Event') > -1:
            return EVENT_ONLY.format(name=name)

    if len(parameters) == 2:
        first, second = parameters
        first_parameter_type = first['raw_type']
        second_parameter_type = second['raw_type']
        param_type = re.sub("^ApiParam<(.*?)>$", r"\1", first_parameter_type)
        param_type = param_type.strip()

        if second_parameter_type.find('Event') > -1:
            return PARAM_ONLY.format(name=name,
                                     param_type=first_parameter_type,
                                     param_param_type=param_type)

        elif second_parameter_type.find('ApiResult') > -1:
            return RESULT_AND_PARAM.format(name=name,
                                           param_type=first_parameter_type,
                                           param_param_type=param_type,
                                           result_type=second_parameter_type)
    else:
        raise ValueError('Cannot interpreter the given method.')

class RpcClass(object):

    def __init__(self, name, body):
        self.name = name
        self.body = body

    def public_methods(self):
        for method in self.body['methods']['public']:
            name = method['name']
            parameters = method['parameters']
            if name == self.name:
                # It is a constructor
                continue

            yield _generate_method_from_parameters(name, parameters)

    # Generation of data

    def generate(self):
        methods = "\n".join(self.public_methods())
        return CLASS_HEADER.format(name=self.name, methods=methods)

def parse_header(header_file):
    return header_parser.CppHeader(header_file)

if __name__ == '__main__':
    header = parse_header(sys.argv[1])
    for cls, cls_body in header.classes.items():
        class_header = RpcClass(cls, cls_body).generate()
        header_header = HEADER_HEADER.format(
            header=os.path.basename(sys.argv[1]),
            cls=class_header,
            date=datetime.datetime.now().strftime("%Y-%M-%d %H:%m:%S"))
        print(header_header)
<commit_msg>Add support for picking what the generate script should do.<commit_after>"""Parses a given header in order to generate the RPC interface of it."""

import argparse
import datetime
import os
import re
import sys

import CppHeaderParser as header_parser

CLASS_HEADER = """
class Rpc{name} {{
  private:
    foobar::{name} api;
  public:
{methods}
}};"""

HEADER_HEADER = """// Generated on {date}
#pragma once
#include <string>
#include <map>
#include <tuple>

#include "stdafx.h"
#include "event.h"
#include "percolate.h"
#include "api/{header}"
#include "factory.h"

using namespace std;
using namespace serialization;

{cls}"""

EVENT_ONLY = """
    Payload {name}(vector<char> & _unused) {{
      Event event;
      fb2k::inMainThread([&] {{
        api.{name}(event);
      }});
      event.wait();
      return serialization::serializer.packed_bool(true);
    }}"""

PARAM_ONLY = """
    Payload {name}(vector<char> & buffer) {{
      {param_type} param(serialization::serializer.unpack<{param_param_type}>(buffer));
      Event event;
      fb2k::inMainThread([&] {{
        api.{name}(param, event);
      }});
      event.wait();
      return serialization::serializer.packed_bool(true);
    }}"""

RESULT_ONLY = """
    Payload {name}(vector<char> & buffer) {{
      {type} result;
      fb2k::inMainThread([&] {{
        api.{name}(result);
      }});
      result.wait();
      return serialization::serializer.packed_result(result);
    }}"""

RESULT_AND_PARAM = """
    Payload {name}(vector<char> & buffer) {{
      {param_type} param(serialization::serializer.unpack<{param_param_type}>(buffer));
      {result_type} result;
      fb2k::inMainThread([&] {{
        api.{name}(param, result);
      }});
      result.wait();
      return serialization::serializer.packed_result(result);
    }}"""

def _generate_method_from_parameters(name, parameters):
    if len(parameters) == 1:
        parameter = parameters[0]
        parameter_type = parameter['raw_type']
        if parameter_type.find('ApiResult') > -1:
            return RESULT_ONLY.format(name=name, type=parameter_type)

        elif parameter_type.find('Event') > -1:
            return EVENT_ONLY.format(name=name)

    if len(parameters) == 2:
        first, second = parameters
        first_parameter_type = first['raw_type']
        second_parameter_type = second['raw_type']
        param_type = re.sub("^ApiParam<(.*?)>$", r"\1", first_parameter_type)
        param_type = param_type.strip()

        if second_parameter_type.find('Event') > -1:
            return PARAM_ONLY.format(name=name,
                                     param_type=first_parameter_type,
                                     param_param_type=param_type)

        elif second_parameter_type.find('ApiResult') > -1:
            return RESULT_AND_PARAM.format(name=name,
                                           param_type=first_parameter_type,
                                           param_param_type=param_type,
                                           result_type=second_parameter_type)
    else:
        raise ValueError('Cannot interpreter the given method.')

class RpcClass(object):

    def __init__(self, name, body):
        self.name = name
        self.body = body

    def public_methods(self):
        for method in self.body['methods']['public']:
            name = method['name']
            parameters = method['parameters']
            if name == self.name:
                # It is a constructor
                continue

            yield _generate_method_from_parameters(name, parameters)

    # Generation of data

    def generate(self):
        methods = "\n".join(self.public_methods())
        return CLASS_HEADER.format(name=self.name, methods=methods)

def parse_header(header_file):
    return header_parser.CppHeader(header_file)

def generate_header(args):
    header = parse_header(args.header)
    for cls, cls_body in header.classes.items():
        class_header = RpcClass(cls, cls_body).generate()

        header_header = HEADER_HEADER.format(
            header=os.path.basename(sys.argv[1]),
            cls=class_header,
            date=datetime.datetime.now().strftime("%Y-%M-%d %H:%m:%S"))

        with args.output as stream:
            stream.write(header_header)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers(dest="cmd")
    generate_header_parser = subparsers.add_parser(
        "generate_header",
        description="Generate a RPC header file from an API header file.")

    generate_header_parser.add_argument("--header", type=str)
    generate_header_parser.add_argument("--output",
                                        required=True,
                                        type=argparse.FileType('w+'))
    args = parser.parse_args()

    if args.cmd is None:
        parser.print_help()
        exit()

    globals()[args.cmd](args)
<|endoftext|>


<commit_before>#  Copyright 2008-2011 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import re

from robot import utils
from robot.result.visitor import SuiteVisitor
from robot.model.tags import TagPatterns

class StatisticsBuilder(SuiteVisitor):

    def __init__(self, suite_stats, tag_stats):
        self._current_suite_stat = suite_stats
        self._tag_stats = tag_stats
        self._parents = []
        self._first = True

    def start_suite(self, suite):
        if not self._first:
            new  = SuiteStatistics(suite)
            self._current_suite_stat.suites.append(new)
            self._parents.append(self._current_suite_stat)
            self._current_suite_stat = new
        self._current_suite = suite
        self._first = False

    def end_suite(self, suite):
        if self._parents:
            self._parents[-1].all.add_stat(self._current_suite_stat.all)
            self._parents[-1].critical.add_stat(self._current_suite_stat.critical)
            self._current_suite_stat = self._parents.pop(-1)

    def start_test(self, test):
        self._current_suite_stat.add_test(test)
        self._tag_stats.add_test(test, self._current_suite.critical)

class Statistics(object):

    def __init__(self, suite, tag_stat_include=None, tag_stat_exclude=None,
                 tag_stat_combine=None, tag_doc=None, tag_stat_link=None):
        self.suite = SuiteStatistics(suite)
        self.tags = TagStatistics(tag_stat_include, tag_stat_exclude,
                                  tag_stat_combine, tag_doc, tag_stat_link)
        StatisticsBuilder(self.suite, self.tags).visit_suite(suite)
        self.tags.sort()
        self.total = TotalStatistics(self.suite)

    #TODO: Replace with visit
    def serialize(self, serializer):
        serializer.start_statistics(self)
        self.total.serialize(serializer)
        self.tags.serialize(serializer)
        self.suite.serialize(serializer)
        serializer.end_statistics(self)

    def visit(self, visitor):
        self.serialize(visitor)

class SuiteStatistics(object):

    def __init__(self, suite):
        self.all = SuiteStat(suite)
        self.critical = SuiteStat(suite)
        self.suites = []

    def add_test(self, test):
        self.all.add_test(test)
        if test.critical == 'yes':
            self.critical.add_test(test)

class Stat(object):

    def __init__(self, name=''):
        self.name = name
        self.passed = 0
        self.failed = 0

    @property
    def total(self):
        return self.passed + self.failed

    def add_stat(self, other):
        self.passed += other.passed
        self.failed += other.failed

    def add_test(self, test):
        if test.status == 'PASS':
            self.passed += 1
        else:
            self.failed += 1

    def fail_all(self):
        self.failed += self.passed
        self.passed = 0

    def add_suite(self, suite):
        for test in suite.tests:
            if self._is_included(test):
                self.add_test(test)
        for suite in suite.suites:
            self.add_stat(self._subsuite_stats(suite))

    def _is_included(self, test):
        return True

    def _subsuite_stats(self, suite):
        return suite.all_stats

    def __cmp__(self, other):
        return cmp(self.name, other.name)

    def __nonzero__(self):
        return self.failed == 0

class SuiteStat(Stat):
    type = 'suite'

    def __init__(self, suite):
        Stat.__init__(self, suite.name)
        self.longname = suite.longname
        self.id = suite.id

    def serialize(self, serializer):
        serializer.suite_stat(self)

class TagStat(Stat):
    type = 'tag'

    def __init__(self, name, doc='', links=[], critical=False,
                 non_critical=False, combined=''):
        Stat.__init__(self, name)
        self.doc = doc
        self.links = links
        self.critical = critical
        self.non_critical = non_critical
        self.combined = combined
        self.tests = []

    def add_test(self, test):
        Stat.add_test(self, test)
        self.tests.append(test)

    def __cmp__(self, other):
        if self.critical != other.critical:
            return cmp(other.critical, self.critical)
        if self.non_critical != other.non_critical:
            return cmp(other.non_critical, self.non_critical)
        if bool(self.combined) != bool(other.combined):
            return cmp(bool(other.combined), bool(self.combined))
        return cmp(self.name, other.name)

    def serialize(self, serializer):
        serializer.tag_stat(self)

class TotalStat(Stat):
    type = 'total'

    def __init__(self, name, suite_stat):
        Stat.__init__(self, name)
        self.passed = suite_stat.passed
        self.failed = suite_stat.failed

    def serialize(self, serializer):
        serializer.total_stat(self)

class CombinedTag(object):

    def __init__(self, pattern, name):
        self.pattern = pattern
        self._matcher = TagPatterns(pattern)
        self.name = name or pattern

    def match(self, tags):
        return self._matcher.match(tags)

class TagStatistics:

    def __init__(self, include=None, exclude=None, combine=None, docs=None,
                 links=None):
        self.stats = utils.NormalizedDict(ignore=['_'])
        self._include = TagPatterns(include)
        self._exclude = TagPatterns(exclude)
        self._info = TagStatInfo(docs or [], links or [])
        self._combine = self._create_combined(combine or [])

    def _create_combined(self, combines):
        combines = [CombinedTag(pattern, name) for pattern, name in combines]
        for comb in combines:
            self.stats[comb.name] = TagStat(comb.name,
                                            self._info.get_doc(comb.name),
                                            self._info.get_links(comb.name),
                                            combined=comb.pattern)
        return combines

    def add_test(self, test, critical):
        self._add_tags_statistics(test, critical)
        self._add_combined_statistics(test)

    def _add_tags_statistics(self, test, critical):
        for tag in test.tags:
            if not self._is_included(tag):
                continue
            if tag not in self.stats:
                self.stats[tag] = TagStat(tag,
                                          self._info.get_doc(tag),
                                          self._info.get_links(tag),
                                          critical.is_critical(tag),
                                          critical.is_non_critical(tag))
            self.stats[tag].add_test(test)

    def _is_included(self, tag):
        if self._include and not self._include.match(tag):
            return False
        return not self._exclude.match(tag)

    def _add_combined_statistics(self, test):
        for comb in self._combine:
            if comb.match(test.tags):
                self.stats[comb.name].add_test(test)

    def serialize(self, serializer):
        serializer.start_tag_stats(self)
        for stat in sorted(self.stats.values()):
            stat.serialize(serializer)
        serializer.end_tag_stats(self)

    def sort(self):
        # TODO: Is this needed?
        for stat in self.stats.values():
            stat.tests.sort()

class TotalStatistics:

    def __init__(self, suite):
        self.critical = TotalStat('Critical Tests', suite.critical)
        self.all = TotalStat('All Tests', suite.all)

    def serialize(self, serializer):
        serializer.start_total_stats(self)
        self.critical.serialize(serializer)
        self.all.serialize(serializer)
        serializer.end_total_stats(self)

class TagStatInfo:

    def __init__(self, docs, links):
        self._docs = [TagStatDoc(*doc) for doc in docs]
        self._links = [TagStatLink(*link) for link in links]

    def get_doc(self, tag):
        return ' & '.join(doc.text for doc in self._docs if doc.matches(tag))

    def get_links(self, tag):
        return [link.get_link(tag) for link in self._links if link.matches(tag)]

class TagStatDoc:

    def __init__(self, pattern, doc):
        self.text = doc
        self._pattern = TagPatterns(pattern)

    def matches(self, tag):
        return self._pattern.match(tag)

class TagStatLink:
    _match_pattern_tokenizer = re.compile('(\*|\?)')

    def __init__(self, pattern, link, title):
        self._regexp = self._get_match_regexp(pattern)
        self._link = link
        self._title = title.replace('_', ' ')

    def matches(self, tag):
        return self._regexp.match(tag) is not None

    def get_link(self, tag):
        match = self._regexp.match(tag)
        if not match:
            return None
        link, title = self._replace_groups(self._link, self._title, match)
        return link, title

    def _replace_groups(self, link, title, match):
        for index, group in enumerate(match.groups()):
            placefolder = '%' + str(index+1)
            link = link.replace(placefolder, group)
            title = title.replace(placefolder, group)
        return link, title

    def _get_match_regexp(self, pattern):
        regexp = []
        open_parenthesis = False
        for token in self._match_pattern_tokenizer.split(pattern):
            if token == '':
                continue
            if token == '?':
                if not open_parenthesis:
                    regexp.append('(')
                    open_parenthesis = True
                regexp.append('.')
                continue
            if open_parenthesis:
                regexp.append(')')
                open_parenthesis = False
            if token == '*':
                regexp.append('(.*)')
                continue
            regexp.append(re.escape(token))
        if open_parenthesis:
            regexp.append(')')
        return re.compile('^%s$' % ''.join(regexp), re.IGNORECASE)
<commit_msg>Automated merge with https://code.google.com/p/robotframework/<commit_after>#  Copyright 2008-2011 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import re

from robot import utils
from robot.result.visitor import SuiteVisitor
from robot.model.tags import TagPatterns

class StatisticsBuilder(SuiteVisitor):

    def __init__(self, tag_stats):
        self._tag_stats = tag_stats
        self._parents = []

    def start_suite(self, suite):
        new  = SuiteStatistics(suite)
        self._current_suite_stat.suites.append(new)
        self._parents.append(self._current_suite_stat)
        self._current_suite_stat = new
        self._current_suite = suite

    def build(self, suite):
        self._current_suite_stat = SuiteStatistics(suite)
        self.visit_suite(suite)
        return self._current_suite_stat.suites[0]

    def end_suite(self, suite):
        if self._parents:
            self._parents[-1].all.add_stat(self._current_suite_stat.all)
            self._parents[-1].critical.add_stat(self._current_suite_stat.critical)
            self._current_suite_stat = self._parents.pop(-1)

    def visit_test(self, test):
        self._current_suite_stat.add_test(test)
        self._tag_stats.add_test(test, self._current_suite.critical)

class Statistics(object):

    def __init__(self, suite, tag_stat_include=None, tag_stat_exclude=None,
                 tag_stat_combine=None, tag_doc=None, tag_stat_link=None):
        self.tags = TagStatistics(tag_stat_include, tag_stat_exclude,
                                  tag_stat_combine, tag_doc, tag_stat_link)
        self.suite = StatisticsBuilder(self.tags).build(suite)
        self.tags.sort()
        self.total = TotalStatistics(self.suite)

    #TODO: Replace with visit
    def serialize(self, serializer):
        serializer.start_statistics(self)
        self.total.serialize(serializer)
        self.tags.serialize(serializer)
        self.suite.serialize(serializer)
        serializer.end_statistics(self)

    def visit(self, visitor):
        self.serialize(visitor)

class SuiteStatistics(object):

    def __init__(self, suite):
        self.all = SuiteStat(suite)
        self.critical = SuiteStat(suite)
        self.suites = []

    def add_test(self, test):
        self.all.add_test(test)
        if test.critical == 'yes':
            self.critical.add_test(test)

class Stat(object):

    def __init__(self, name=''):
        self.name = name
        self.passed = 0
        self.failed = 0

    @property
    def total(self):
        return self.passed + self.failed

    def add_stat(self, other):
        self.passed += other.passed
        self.failed += other.failed

    def add_test(self, test):
        if test.status == 'PASS':
            self.passed += 1
        else:
            self.failed += 1

    def fail_all(self):
        self.failed += self.passed
        self.passed = 0

    def add_suite(self, suite):
        for test in suite.tests:
            if self._is_included(test):
                self.add_test(test)
        for suite in suite.suites:
            self.add_stat(self._subsuite_stats(suite))

    def _is_included(self, test):
        return True

    def _subsuite_stats(self, suite):
        return suite.all_stats

    def __cmp__(self, other):
        return cmp(self.name, other.name)

    def __nonzero__(self):
        return self.failed == 0

class SuiteStat(Stat):
    type = 'suite'

    def __init__(self, suite):
        Stat.__init__(self, suite.name)
        self.longname = suite.longname
        self.id = suite.id

    def serialize(self, serializer):
        serializer.suite_stat(self)

class TagStat(Stat):
    type = 'tag'

    def __init__(self, name, doc='', links=[], critical=False,
                 non_critical=False, combined=''):
        Stat.__init__(self, name)
        self.doc = doc
        self.links = links
        self.critical = critical
        self.non_critical = non_critical
        self.combined = combined
        self.tests = []

    def add_test(self, test):
        Stat.add_test(self, test)
        self.tests.append(test)

    def __cmp__(self, other):
        if self.critical != other.critical:
            return cmp(other.critical, self.critical)
        if self.non_critical != other.non_critical:
            return cmp(other.non_critical, self.non_critical)
        if bool(self.combined) != bool(other.combined):
            return cmp(bool(other.combined), bool(self.combined))
        return cmp(self.name, other.name)

    def serialize(self, serializer):
        serializer.tag_stat(self)

class TotalStat(Stat):
    type = 'total'

    def __init__(self, name, suite_stat):
        Stat.__init__(self, name)
        self.passed = suite_stat.passed
        self.failed = suite_stat.failed

    def serialize(self, serializer):
        serializer.total_stat(self)

class CombinedTag(object):

    def __init__(self, pattern, name):
        self.pattern = pattern
        self._matcher = TagPatterns(pattern)
        self.name = name or pattern

    def match(self, tags):
        return self._matcher.match(tags)

class TagStatistics:

    def __init__(self, include=None, exclude=None, combine=None, docs=None,
                 links=None):
        self.stats = utils.NormalizedDict(ignore=['_'])
        self._include = TagPatterns(include)
        self._exclude = TagPatterns(exclude)
        self._info = TagStatInfo(docs or [], links or [])
        self._combine = self._create_combined(combine or [])

    def _create_combined(self, combines):
        combines = [CombinedTag(pattern, name) for pattern, name in combines]
        for comb in combines:
            self.stats[comb.name] = TagStat(comb.name,
                                            self._info.get_doc(comb.name),
                                            self._info.get_links(comb.name),
                                            combined=comb.pattern)
        return combines

    def add_test(self, test, critical):
        self._add_tags_statistics(test, critical)
        self._add_combined_statistics(test)

    def _add_tags_statistics(self, test, critical):
        for tag in test.tags:
            if not self._is_included(tag):
                continue
            if tag not in self.stats:
                self.stats[tag] = TagStat(tag,
                                          self._info.get_doc(tag),
                                          self._info.get_links(tag),
                                          critical.is_critical(tag),
                                          critical.is_non_critical(tag))
            self.stats[tag].add_test(test)

    def _is_included(self, tag):
        if self._include and not self._include.match(tag):
            return False
        return not self._exclude.match(tag)

    def _add_combined_statistics(self, test):
        for comb in self._combine:
            if comb.match(test.tags):
                self.stats[comb.name].add_test(test)

    def serialize(self, serializer):
        serializer.start_tag_stats(self)
        for stat in sorted(self.stats.values()):
            stat.serialize(serializer)
        serializer.end_tag_stats(self)

    def sort(self):
        # TODO: Is this needed?
        for stat in self.stats.values():
            stat.tests.sort()

class TotalStatistics:

    def __init__(self, suite):
        self.critical = TotalStat('Critical Tests', suite.critical)
        self.all = TotalStat('All Tests', suite.all)

    def serialize(self, serializer):
        serializer.start_total_stats(self)
        self.critical.serialize(serializer)
        self.all.serialize(serializer)
        serializer.end_total_stats(self)

class TagStatInfo:

    def __init__(self, docs, links):
        self._docs = [TagStatDoc(*doc) for doc in docs]
        self._links = [TagStatLink(*link) for link in links]

    def get_doc(self, tag):
        return ' & '.join(doc.text for doc in self._docs if doc.matches(tag))

    def get_links(self, tag):
        return [link.get_link(tag) for link in self._links if link.matches(tag)]

class TagStatDoc:

    def __init__(self, pattern, doc):
        self.text = doc
        self._pattern = TagPatterns(pattern)

    def matches(self, tag):
        return self._pattern.match(tag)

class TagStatLink:
    _match_pattern_tokenizer = re.compile('(\*|\?)')

    def __init__(self, pattern, link, title):
        self._regexp = self._get_match_regexp(pattern)
        self._link = link
        self._title = title.replace('_', ' ')

    def matches(self, tag):
        return self._regexp.match(tag) is not None

    def get_link(self, tag):
        match = self._regexp.match(tag)
        if not match:
            return None
        link, title = self._replace_groups(self._link, self._title, match)
        return link, title

    def _replace_groups(self, link, title, match):
        for index, group in enumerate(match.groups()):
            placefolder = '%' + str(index+1)
            link = link.replace(placefolder, group)
            title = title.replace(placefolder, group)
        return link, title

    def _get_match_regexp(self, pattern):
        regexp = []
        open_parenthesis = False
        for token in self._match_pattern_tokenizer.split(pattern):
            if token == '':
                continue
            if token == '?':
                if not open_parenthesis:
                    regexp.append('(')
                    open_parenthesis = True
                regexp.append('.')
                continue
            if open_parenthesis:
                regexp.append(')')
                open_parenthesis = False
            if token == '*':
                regexp.append('(.*)')
                continue
            regexp.append(re.escape(token))
        if open_parenthesis:
            regexp.append(')')
        return re.compile('^%s$' % ''.join(regexp), re.IGNORECASE)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf8 -*-

import dbutils
from yamlns import namespace as ns

def getGenerationMovelinesByPartner(db, partner_id):
    with db.cursor() as cr:
        cr.execute("""\
            select
                p.*
            from
                res_partner as p
            where
                p.id = %(partner_id)s
            order by p.id
        """, dict(
            partner_id=partner_id,
        ))
        partner = dbutils.nsList(cr)[0]
    partner.account = '1635{:08}'.format(int(partner.ref[1:]))
    with db.cursor() as cr:
        cr.execute("""\
            select
                %(partner_name)s as partner_name,
                ml.*,
                false
            from
                account_move_line as ml
            left join
                account_period as period
            on
                period.id = ml.period_id
            left join
                account_account as a
            on
                ml.account_id=a.id
            where
                a.code = %(account)s and
                not period.special and
                true
            order by ml.id
        """, dict(
            partner_id = partner_id,
            partner_name = partner.name,
            account = partner.account,
        ))
        return dbutils.nsList(cr)

def getOrderLines(db, order_id):
    with db.cursor() as cr:
        cr.execute("""\
            select
                pl.*,
                po.date_done as order_sent_date,
                p.name as partner_name,
                false
            from
                payment_line as pl
            left join
                payment_order as po
            on
                po.id = pl.order_id
            left join
                res_partner as p
            on
               p.id = pl.partner_id
            where
                pl.order_id = %(order_id)s
            order by pl.id
        """, dict(
            order_id=order_id,
        ))
        return dbutils.nsList(cr)

def getMoveLines(db, move_id):
    with db.cursor() as cr:
        cr.execute("""\
            select
                ml.*,
                p.name as partner_name
            from
                account_move_line as ml
            left join
                account_account as ac
            on ac.id = ml.account_id
            left join
                res_partner as p
            on
               p.id = ml.partner_id
            where
                ac.code ilike '1635%%' and
                ml.move_id = %(move_id)s
            order by ml.id
        """, dict(
            move_id=move_id,
        ))
        return dbutils.nsList(cr)

def getActiveInvestments(db):
    with db.cursor() as cr:
        cr.execute("""\
            select
                inv.*,
                ml.name as mlname,
                ml.date_created as move_date_created,
                soci.partner_id,
                p.name as partner_name
            from
                generationkwh_investment as inv
            left join
                account_move_line as ml
            on
                inv.move_line_id = ml.id
            left join
                somenergia_soci as soci
            on
                soci.id = inv.member_id
            left join
                res_partner as p
            on
                p.id = soci.partner_id
            where
                inv.active and
                true
            order by inv.id
        """, dict(
        ))
        return dbutils.nsList(cr)

def lastInvestment(db):
    with db.cursor() as cr:
        cr.execute("""\
            select
                max(move_line_id)
            from
                generationkwh_investment as inv
            where
                inv.active and
                true
        """, dict(
        ))
        return dbutils.nsList(cr)[0].max

def investmentOrderAndMovements(db, modeName='GENERATION kWh'):
    with db.cursor() as cr:
        cr.execute("""\
            select
                o.id as order_id,
                m.move_id as move_id,
                o.n_lines as order_nlines,
                m.nlines as move_nlines,
                o.date_done as order_sent_date,
                m.date_created as move_create_date
            from
            (
                select
                    po.id,
                    po.n_lines,
                    po.date_done,
                    true
                from
                    payment_order as po
                left join
                    payment_mode as pm
                on
                    pm.id = po.mode
                where
                    pm.name = 'GENERATION kWh' or
                    po.id = 1389 or -- has mode 'ENGINYERS'
                    false
            ) as o
            full outer join (
                select
                    l.move_id,
                    count(l.id) as nlines,
                    l.date_created
                from
                    account_move_line as l
                left join
                    account_account as a
                on
                    l.account_id=a.id
                where
                    a.code ilike '1635%%' and
                    l.name = '/' and
                    true
                group by
                    move_id,
                    l.date_created
                order by
                    l.date_created
            ) as m
                on o.date_done = m.date_created
            order by
                m.date_created
            ;
            """)
        return dbutils.nsList(cr)

def appendToKey(d, key, value):
    values = d.get(key, [])
    wasEmpty = bool(values)
    values.append(value)
    d[key] = values
    return wasEmpty

def getAndRemoveFirst(d, key):
    values = d.get(key, [])
    if not values: return None
    return values.pop(0)

def bindMoveLineAndPaymentLine(moveline, orderline):
    True and success(
        "Match ml-po ml-{moveline.id} {orderline.name} {orderline.create_date} {amount:=8}€ {moveline.partner_name}"
        .format(**ns(
            moveline=moveline,
            orderline=orderline,
            amount = moveline.credit-moveline.debit,
            )))
    solvedMovelines[moveline.id] = ns(
        ref=orderline.name,
        order_date=orderline.create_date,
        partner_id = orderline.partner_id,
        partner_name = orderline.partner_name,
        )

def bindInvestmentWithOrder(investment, moveline):
    True and success(
        "Match inv-od {investment.id} {moveline.ref} {moveline.order_date} {amount:=8}€ {moveline.partner_name}"
        .format(**ns(
            investment=investment,
            moveline=moveline,
            amount=investment.nshares*100.,
            )))

def displayPartnersMovements(db, partner_id):
    movelines = getGenerationMovelinesByPartner(db,partner_id)
    for m in movelines:
        m.amount = m.credit -m.debit
        error("    {date_created} {id} {partner_name} {amount} {name}"
            .format(**m))

import configdb
import psycopg2
from consolemsg import step, warn, success, error

with psycopg2.connect(**configdb.psycopg) as db:

    solvedMovelines = ns()
    step("Pairing movelines and orderlines by order/move")

    for data in investmentOrderAndMovements(db):
        step("Quadrant remesa feta el {move_create_date}".format(**data))

        step(" Check that movement and order are paired")

        if not data.move_id:
            warn("{}: Order without movement. Id {}, with {} entries" .format(
                data.order_sent_date or "not yet",
                data.order_id,
                data.order_nlines,
                ))
            continue
        if not data.order_id:
            warn("{}: Movement without order. Id {}, with {} entries".format(
                data.move_create_date or "not yet",
                data.move_id,
                data.move_nlines,
                ))
            continue
        if data.move_nlines != data.order_nlines:
            warn("{}: Different number of lines. Payment order {}, Movement {}".format(
                data.move_create_date,
                data.order_nlines,
                data.move_nlines,
                ))

        orderLines = getOrderLines(db, data.order_id)
        moveLines = getMoveLines(db, data.move_id)

        if len(orderLines) != data.order_nlines:
            warn("Got different number of lines {} than reported {} in Order"
                .format(len(orderLines), data.order_nlines))

        if len(moveLines) != data.move_nlines:
            warn("Got different number of lines {} than reported {} in Move"
                .format(len(orderLines), data.order_nlines))

        step(" Matching order lines and move lines by partner, amount and order")
        orderLinesDict = ns()
        repesca = []

        #step("  Create a map partner,amount -> orderline")
        for line in orderLines:
            key = (line.partner_id, -line.amount)
            appendToKey(orderLinesDict, key, line)

        #step("  for each move line look up orderlines by partner,amount")
        for moveline in moveLines:
            key = (moveline.partner_id, moveline.credit)
            orderline = getAndRemoveFirst(orderLinesDict, key)
            if not orderline:
                repesca.append(moveline)
                continue

            # orderline-movementline match found
            bindMoveLineAndPaymentLine(moveline, orderline)

        step(" Matching missed order lines just by partner")

        #step("  Create a partner -> unpaired movelines map")
        movelinesByPartnerId = {}
        for moveline in repesca:
            appendToKey(movelinesByPartnerId, moveline.partner_id, moveline)

        #step("  Lookup for each unpaired order")
        for key, options in orderLinesDict.items():
            for orderline in options:
                moveline = getAndRemoveFirst(movelinesByPartnerId, orderline.partner_id)
                if not moveline:
                    error(
                        "Linia de Remesa sense parella: "
                        "{name} id {id} {order_sent_date} {amount}€ {partner_name}"
                        .format(**orderline))
                    print key
                    continue
                bindMoveLineAndPaymentLine(moveline, orderline)
                warn(
                    "Amount missmatch {orderline.name} "
                    "order {orderline.amount}€ move {moveline.credit}"
                    .format(**ns(
                        orderline=orderline,
                        moveline=moveline,
                    )))
                displayPartnersMovements(db, orderline.partner_id)

        for partner_id, movelines in movelinesByPartnerId.items():
            for moveline in movelines:
                error(
                    "Linia de Moviment sense parella "
                    "id {id} {create_date} {credit}€ {partner_name}"
                    .format(**moveline))

    step("Pairing investments")
    step(" Pairing investments using existing relation")

    investments = getActiveInvestments(db)

    unmatchedMoveLines = list(solvedMovelines.keys())
    unmatchedInvestments = []

    for inv in investments:
        moveline_id = inv.move_line_id

        if moveline_id not in solvedMovelines.keys():
            warn("Missing moveline {move_line_id} "
                "for partner {partner_name} {nshares}00€ '{mlname}'"
                .format(**inv))
            #warn(inv.dump())
            unmatchedInvestments.append(inv.id)
        else:
            bindInvestmentWithOrder(inv, solvedMovelines[moveline_id])
            if moveline_id not in  unmatchedMoveLines:
                error("Move line referenced twice {move_line_id} {partner_name}".format(**inv))
                displayPartnersMovements(db, inv.partner_id)
            else:
                unmatchedMoveLines.remove(moveline_id)

    lastGeneratedMoveLine = lastInvestment(db)
    ungenerated = [
        line_id
        for line_id in unmatchedMoveLines
        if line_id > lastGeneratedMoveLine
    ]

    unmatchedMoveLines = [
        line_id
        for line_id in unmatchedMoveLines
        if line_id <= lastGeneratedMoveLine
    ]

    step("Retry match investments just by partner")
    orphanMoveLinesByPartner = {}

    for moveline_id in unmatchedMoveLines:
        moveline = solvedMovelines[moveline_id]

        wasEmpty = appendToKey(orphanMoveLinesByPartner, moveline.partner_id, moveline)
        if wasEmpty:
            warn("partner with two pending movements id {id} {partner_name}".format(id=moveline_id, **moveline))

    for inv in investments:
        if inv.id not in unmatchedInvestments:
            continue

        partner_id = inv.partner_id
        solution = getAndRemoveFirst(orphanMoveLinesByPartner, partner_id)
        if not solution:
            error("No name for Investment {move_date_created} {move_line_id} {nshares:=5}00.00€ {partner_name}".format(**inv))
            displayPartnersMovements(db, inv.partner_id)
            continue
        bindInvestmentWithOrder(inv, solution)

    for orphanNames in orphanMoveLinesByPartner.values():
        for orphanName in orphanNames:
            error("Unassigned name {ref} {order_date} {partner_name}"
                .format(**orphanName))
            displayPartnersMovements(db, orphanName.partner_id)

    for moveline_id in ungenerated:
        moveline = solvedMovelines[moveline_id]
        warn("Not yet generated investment {ref} {order_date} {partner_name}".format(**moveline))

<commit_msg>providing more info on outlier cases<commit_after>#!/usr/bin/env python
# -*- coding: utf8 -*-

import dbutils
from yamlns import namespace as ns

def activeInvestmentRelatedToFiscalEndYear(db):
    """Returns the list of active investment related to
    movelines of a fiscal closure.
    Investments of such movelines should be disabled."""
    query = """\
        select
            inv.id as id
        from
            generationkwh_investment as inv
        left join
            account_move_line as ml
        on
            inv.move_line_id = ml.id
        left join
            account_period as p
        on
            p.id = ml.period_id
        where
            p.special and
            inv.active
        """
    with db.cursor() as cr:
        cr.execute(query)

def deleteInactiveInvestmentRelatedToFiscalEndYear(db):
    query = """\
        delete
        from generationkwh_investment
        where id in (
            select
                inv.id as id
            from
                generationkwh_investment as inv
            left join
                account_move_line as ml
            on
                inv.move_line_id = ml.id
            left join
                account_period as p
            on
                p.id = ml.period_id
            where
                p.special and
                not inv.active
            )
        """
    with db.cursor() as cr:
        cr.execute(query)

def getGenerationMovelinesByPartner(db, partner_id):
    with db.cursor() as cr:
        cr.execute("""\
            select
                p.*
            from
                res_partner as p
            where
                p.id = %(partner_id)s
            order by p.id
        """, dict(
            partner_id=partner_id,
        ))
        partner = dbutils.nsList(cr)[0]
    partner.account = '1635{:08}'.format(int(partner.ref[1:]))
    with db.cursor() as cr:
        cr.execute("""\
            select
                %(partner_name)s as partner_name,
                ml.*,
                false
            from
                account_move_line as ml
            left join
                account_period as period
            on
                period.id = ml.period_id
            left join
                account_account as a
            on
                ml.account_id=a.id
            where
                a.code = %(account)s and
                not period.special and
                true
            order by ml.id
        """, dict(
            partner_id = partner_id,
            partner_name = partner.name,
            account = partner.account,
        ))
        return dbutils.nsList(cr)

def getInvestmentsByPartner(db, partner_id):
    with db.cursor() as cr:
        cr.execute("""\
            select
                p.*
            from
                res_partner as p
            where
                p.id = %(partner_id)s
            order by p.id
        """, dict(
            partner_id=partner_id,
        ))
        partner = dbutils.nsList(cr)[0]
    with db.cursor() as cr:
        cr.execute("""\
            select
                %(partner_name)s as partner_name,
                inv.*,
                false
            from
                generationkwh_investment as inv
            left join
                somenergia_soci as member
            on
                member.id = inv.member_id
            where
                member.partner_id = %(partner_id)s and
                true
            order by inv.id
        """, dict(
            partner_id = partner_id,
            partner_name = partner.name,
        ))
        return dbutils.nsList(cr)

def getOrderLines(db, order_id):
    with db.cursor() as cr:
        cr.execute("""\
            select
                pl.*,
                po.date_done as order_sent_date,
                p.name as partner_name,
                false
            from
                payment_line as pl
            left join
                payment_order as po
            on
                po.id = pl.order_id
            left join
                res_partner as p
            on
               p.id = pl.partner_id
            where
                pl.order_id = %(order_id)s
            order by pl.id
        """, dict(
            order_id=order_id,
        ))
        return dbutils.nsList(cr)

def getMoveLines(db, move_id):
    with db.cursor() as cr:
        cr.execute("""\
            select
                ml.*,
                p.name as partner_name
            from
                account_move_line as ml
            left join
                account_account as ac
            on ac.id = ml.account_id
            left join
                res_partner as p
            on
               p.id = ml.partner_id
            where
                ac.code ilike '1635%%' and
                ml.move_id = %(move_id)s
            order by ml.id
        """, dict(
            move_id=move_id,
        ))
        return dbutils.nsList(cr)

def getActiveInvestments(db):
    with db.cursor() as cr:
        cr.execute("""\
            select
                inv.*,
                ml.name as mlname,
                ml.date_created as move_date_created,
                soci.partner_id,
                p.name as partner_name
            from
                generationkwh_investment as inv
            left join
                account_move_line as ml
            on
                inv.move_line_id = ml.id
            left join
                somenergia_soci as soci
            on
                soci.id = inv.member_id
            left join
                res_partner as p
            on
                p.id = soci.partner_id
            where
                inv.active and
                true
            order by inv.id
        """, dict(
        ))
        return dbutils.nsList(cr)

def lastInvestment(db):
    with db.cursor() as cr:
        cr.execute("""\
            select
                max(move_line_id)
            from
                generationkwh_investment as inv
            where
                inv.active and
                true
        """, dict(
        ))
        return dbutils.nsList(cr)[0].max

def investmentOrderAndMovements(db, modeName='GENERATION kWh'):
    with db.cursor() as cr:
        cr.execute("""\
            select
                o.id as order_id,
                m.move_id as move_id,
                o.n_lines as order_nlines,
                m.nlines as move_nlines,
                o.date_done as order_sent_date,
                m.date_created as move_create_date
            from
            (
                select
                    po.id,
                    po.n_lines,
                    po.date_done,
                    true
                from
                    payment_order as po
                left join
                    payment_mode as pm
                on
                    pm.id = po.mode
                where
                    pm.name = 'GENERATION kWh' or
                    po.id = 1389 or -- has mode 'ENGINYERS'
                    false
            ) as o
            full outer join (
                select
                    l.move_id,
                    count(l.id) as nlines,
                    l.date_created
                from
                    account_move_line as l
                left join
                    account_account as a
                on
                    l.account_id=a.id
                where
                    a.code ilike '1635%%' and
                    l.name = '/' and
                    true
                group by
                    move_id,
                    l.date_created
                order by
                    l.date_created
            ) as m
                on o.date_done = m.date_created
            order by
                m.date_created
            ;
            """)
        return dbutils.nsList(cr)

def appendToKey(d, key, value):
    values = d.get(key, [])
    wasEmpty = bool(values)
    values.append(value)
    d[key] = values
    return wasEmpty

def getAndRemoveFirst(d, key):
    values = d.get(key, [])
    if not values: return None
    return values.pop(0)

def bindMoveLineAndPaymentLine(moveline, orderline):
    True and success(
        "Match ml-po ml-{moveline.id} {orderline.name} {orderline.create_date} {amount:=8}€ {moveline.partner_name}"
        .format(**ns(
            moveline=moveline,
            orderline=orderline,
            amount = moveline.credit-moveline.debit,
            )))
    solvedMovelines[moveline.id] = ns(
        ref=orderline.name,
        order_date=orderline.create_date,
        partner_id = orderline.partner_id,
        partner_name = orderline.partner_name,
        amount = moveline.credit-moveline.debit,
        )

def bindInvestmentWithOrder(db, investment, moveline):
    True and success(
        "Match inv-od {investment.id} {moveline.ref} {moveline.order_date} {amount:=8}€ {moveline.partner_name}"
        .format(**ns(
            investment=investment,
            moveline=moveline,
            amount=investment.nshares*100.,
            )))

    with db.cursor() as cr:
        cr.execute("""\
            UPDATE generationkwh_investment as inv
            SET
                name = %(name)s,
                order_date = %(order_date)s
            WHERE
                inv.id = %(id)s
            """, dict(
                id = investment.id,
                order_date = moveline.order_date,
                name = moveline.ref,
            ))

def displayPartnersMovements(db, partner_id):
    movelines = getGenerationMovelinesByPartner(db,partner_id)
    for m in movelines:
        m.amount = m.credit -m.debit
        error("    mov {date_created} {id} {partner_name} {amount} {name}"
            .format(**m))
    investments = getInvestmentsByPartner(db, partner_id)
    for inv in investments:
        inv.amount = inv.nshares*100.
        inv.active = 'y' if inv.active else 'n'
        error("    inv {active} {order_date_or_not} {purchase_date_or_not} {first_effective_date_or_not} {last_effective_date_or_not} {id} {partner_name} {amount} {name}"
            .format(
            order_date_or_not = inv.order_date or '____-__-__',
            purchase_date_or_not = inv.purchase_date or '____-__-__',
            first_effective_date_or_not = inv.first_effective_date or '____-__-__',
            last_effective_date_or_not = inv.last_effective_date or '____-__-__',
            **inv))

import configdb
import psycopg2
from consolemsg import step, warn, success, error, fail

with psycopg2.connect(**configdb.psycopg) as db:

    step("Clean up cases")
    fiscalEndYearInvestments = activeInvestmentRelatedToFiscalEndYear(db)
    if fiscalEndYearInvestments:
        fail("There are active investments which are related to fiscal end year")
    print deleteInactiveInvestmentRelatedToFiscalEndYear(db)

    solvedMovelines = ns()
    step("Pairing movelines and orderlines by order/move")

    for data in investmentOrderAndMovements(db):
        step("Quadrant remesa feta el {move_create_date}".format(**data))

        step(" Check that movement and order are paired")

        if not data.move_id:
            warn("{}: Order without movement. Id {}, with {} entries" .format(
                data.order_sent_date or "not yet",
                data.order_id,
                data.order_nlines,
                ))
            continue
        if not data.order_id:
            warn("{}: Movement without order. Id {}, with {} entries".format(
                data.move_create_date or "not yet",
                data.move_id,
                data.move_nlines,
                ))
            continue
        if data.move_nlines != data.order_nlines:
            warn("{}: Different number of lines. Payment order {}, Movement {}".format(
                data.move_create_date,
                data.order_nlines,
                data.move_nlines,
                ))

        orderLines = getOrderLines(db, data.order_id)
        moveLines = getMoveLines(db, data.move_id)

        if len(orderLines) != data.order_nlines:
            warn("Got different number of lines {} than reported {} in Order"
                .format(len(orderLines), data.order_nlines))

        if len(moveLines) != data.move_nlines:
            warn("Got different number of lines {} than reported {} in Move"
                .format(len(orderLines), data.order_nlines))

        step(" Matching order lines and move lines by partner, amount and order")
        orderLinesDict = ns()
        repesca = []

        #step("  Create a map partner,amount -> orderline")
        for line in orderLines:
            key = (line.partner_id, -line.amount)
            appendToKey(orderLinesDict, key, line)

        #step("  for each move line look up orderlines by partner,amount")
        for moveline in moveLines:
            key = (moveline.partner_id, moveline.credit)
            orderline = getAndRemoveFirst(orderLinesDict, key)
            if not orderline:
                repesca.append(moveline)
                continue

            # orderline-movementline match found
            bindMoveLineAndPaymentLine(moveline, orderline)

        step(" Matching missed order lines just by partner")

        #step("  Create a partner -> unpaired movelines map")
        movelinesByPartnerId = {}
        for moveline in repesca:
            appendToKey(movelinesByPartnerId, moveline.partner_id, moveline)

        #step("  Lookup for each unpaired order")
        for key, options in orderLinesDict.items():
            for orderline in options:
                moveline = getAndRemoveFirst(movelinesByPartnerId, orderline.partner_id)
                if not moveline:
                    error(
                        "Linia de Remesa sense parella: "
                        "{name} id {id} {order_sent_date} {amount}€ {partner_name}"
                        .format(**orderline))
                    print key
                    continue
                bindMoveLineAndPaymentLine(moveline, orderline)
                warn(
                    "Amount missmatch {orderline.name} "
                    "order {orderline.amount}€ move {moveline.credit}"
                    .format(**ns(
                        orderline=orderline,
                        moveline=moveline,
                    )))
                displayPartnersMovements(db, orderline.partner_id)

        for partner_id, movelines in movelinesByPartnerId.items():
            for moveline in movelines:
                error(
                    "Linia de Moviment sense parella "
                    "id {id} {create_date} {credit}€ {partner_name}"
                    .format(**moveline))

    step("Pairing investments")
    step(" Pairing investments using existing relation")

    investments = getActiveInvestments(db)

    unmatchedMoveLines = list(solvedMovelines.keys())
    unmatchedInvestments = []

    for inv in investments:
        moveline_id = inv.move_line_id

        if moveline_id not in solvedMovelines.keys():
            warn("Moveline {move_line_id} refered by inv {id} not in a payment order: "
                "partner {partner_name} {nshares}00€ '{mlname}'"
                .format(**inv))
            #warn(inv.dump())
            unmatchedInvestments.append(inv.id)
            continue
        related_moveline = solvedMovelines[moveline_id]
        if related_moveline.partner_id != inv.partner_id:
            error("Moveline partner missmatch inv {partner_name} mov {mov_partner} {move_line_id} {nshares}00€ {mlname}".format(
                mov_partner=related_moveline.partner_name,
                **inv
                ))
            displayPartnersMovements(db, inv.partner_id)
            displayPartnersMovements(db, related_moveline.partner_id)
            unmatchedInvestments.append(inv.id)
            continue
        if related_moveline.amount != inv.nshares*100:
            warn("Amount missmatch {partner_name} inv {nshares}00€ mov {mov_amount}€ {mlname}"
                .format(
                    mov_amount=related_moveline.amount,
                    **inv))
            displayPartnersMovements(db, inv.partner_id)
        bindInvestmentWithOrder(db, inv, solvedMovelines[moveline_id])
        if moveline_id not in  unmatchedMoveLines:
            error("Move line referenced twice {move_line_id} {partner_name}".format(**inv))
            displayPartnersMovements(db, inv.partner_id)
        else:
            unmatchedMoveLines.remove(moveline_id)

    lastGeneratedMoveLine = lastInvestment(db)
    ungenerated = [
        line_id
        for line_id in unmatchedMoveLines
        if line_id > lastGeneratedMoveLine
    ]

    unmatchedMoveLines = [
        line_id
        for line_id in unmatchedMoveLines
        if line_id <= lastGeneratedMoveLine
    ]

    step("Retry match investments just by partner")
    orphanMoveLinesByPartner = {}

    for moveline_id in unmatchedMoveLines:
        moveline = solvedMovelines[moveline_id]

        wasEmpty = appendToKey(orphanMoveLinesByPartner, moveline.partner_id, moveline)
        if wasEmpty:
            warn("partner with two pending movements id {id} {partner_name}".format(id=moveline_id, **moveline))

    for inv in investments:
        if inv.id not in unmatchedInvestments:
            continue

        partner_id = inv.partner_id
        solution = getAndRemoveFirst(orphanMoveLinesByPartner, partner_id)
        if not solution:
            error("No name for Investment {move_date_created} {move_line_id} {nshares:=5}00.00€ {partner_name}".format(**inv))
            displayPartnersMovements(db, inv.partner_id)
            continue
        bindInvestmentWithOrder(db, inv, solution)

    for orphanNames in orphanMoveLinesByPartner.values():
        for orphanName in orphanNames:
            error("Unassigned name {ref} {order_date} {partner_name}"
                .format(**orphanName))
            displayPartnersMovements(db, orphanName.partner_id)

    for moveline_id in ungenerated:
        moveline = solvedMovelines[moveline_id]
        warn("Not yet generated investment {ref} {order_date} {partner_name}".format(**moveline))

<|endoftext|>


<commit_before>#  Copyright 2008-2009 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import os
import sys

from robot import utils
from robot.errors import DataError

class _Metadata:

    def __init__(self, metadata=None):
        self._data = {}
        for name in self._names.values():
            self._data[name.replace(' ','')] = None
        if metadata is not None:
            for item in metadata:
                self.set(item)

    def set(self, item):
        try:
            name = self._names[utils.normalize(item.name)].replace(' ', '')
        except KeyError:
            self._report_invalid_meta(item)
        else:
            if self._data[name] is None:
                self._data[name] = item.value
            else:
                self._data[name].extend(item.value)

    def _report_invalid_meta(self, item):
        name = self.__class__.__name__.replace('Metadata','')
        name = utils.printable_name(name, code_style=True)
        available = self._get_available_names()
        msg = "%s does not have metadata '%s'. Available: %s"
        item.report_invalid_syntax(msg % (name, item.name, available))

    def _get_available_names(self):
        names = dict([ (name, 1) for name in self._names.values() ]).keys()
        names.sort()
        return utils.seq2str(names)

    def get(self, key, default=None):
        value = self._data[key]
        if value is None:
            return default
        if key == 'Documentation':
            return ' '.join(value)
        return value

    def __getitem__(self, key):
        return self.get(key)

class TestCaseMetadata(_Metadata):
    _names = { 'documentation': 'Documentation',
               'document': 'Documentation',
               'setup': 'Setup',
               'precondition': 'Setup',
               'teardown': 'Teardown',
               'postcondition': 'Teardown',
               'tags': 'Tags',
               'timeout': 'Timeout' }

class UserKeywordMetadata(_Metadata):
    _names = { 'documentation': 'Documentation',
               'document': 'Documentation',
               'arguments': 'Arguments',
               'return': 'Return',
               'timeout': 'Timeout' }

class TestCaseFileMetadata(_Metadata):
    _names = { 'documentation': 'Documentation',
               'document': 'Documentation',
               'suitesetup': 'Suite Setup',
               'suiteprecondition': 'Suite Setup',
               'suiteteardown': 'Suite Teardown',
               'suitepostcondition': 'Suite Teardown',
               'testsetup': 'Test Setup',
               'testprecondition': 'Test Setup',
               'testteardown': 'Test Teardown',
               'testpostcondition': 'Test Teardown',
               'defaulttags': 'Default Tags',
               'forcetags': 'Force Tags',
               'testtimeout': 'Test Timeout' }

    def __init__(self, rawdata=None):
        _Metadata.__init__(self)
        self.user_metadata = {}
        self.imports = []
        if rawdata is not None:
            self._set_rawdata(rawdata)

    def _set_rawdata(self, rawdata):
        for item in rawdata.settings:
            name = item.name.lower()
            if name in ['library', 'resource', 'variables']:
                self.imports.append(ImportSetting(item))
            elif name.startswith('meta:'):
                self._set_user_metadata(item.name[5:].strip(), item.value)
            else:
                self.set(item)

    def _set_user_metadata(self, name, value):
        name = utils.printable_name(name)
        value = ' '.join(value)
        if self.user_metadata.has_key(name):
            value = '%s %s' % (self.user_metadata[name], value)
        self.user_metadata[name] = value

class TestSuiteInitFileMetadata(TestCaseFileMetadata):
    _deprecated_metadata = ['testsetup', 'testteardown', 'defaulttags',
                            'testtimeout']

    def set(self, item):
        TestCaseFileMetadata.set(self, item)
        if utils.normalize(item.name) in self._deprecated_metadata:
            msg = "Using setting '%s' in init file is deprecated and will be " \
                    "removed in the next major release." % (item.name)
            item.report_invalid_syntax(msg, 'WARN')

class ImportSetting:

    def __init__(self, item):
        self._item = item
        self.name = item.name = utils.normalize(item.name).capitalize()
        self.value = None

    def original_value(self):
        return self._item.copy().value

    def replace_variables(self, variables):
        item = self._item.copy()
        try:
            self.value = variables.replace_list(item.value)
        except DataError:
            raise DataError("Replacing variables from setting '%s' failed: %s"
                            % (self.name, utils.get_error_message()))
        if len(self.value) == 0:
            raise DataError("Setting '%s' requires a value" % self.name)
        if self.name == 'Resource' and len(self.value) > 1:
            raise DataError('Invalid resource import parameters: %s' % utils.seq2str2(self.value))
        basedir = utils.get_directory(item._parent._source)
        self.value[0] = self._get_path(self.name, self.value[0], basedir)

    def resolve_name(self, name):
        """Returns absolute path to file if name points to one, otherwise return name unchanged."""
        return self._get_path(self.name, name,
                              utils.get_directory(self._item._parent._source))

    def _get_path(self, name, path, basedir):
        if name == 'Library' and not self._is_library_by_path(path, basedir):
            return path
        try:
            path = self._resolve_path(path.replace('/', os.sep), basedir)
        except DataError:
            if name == 'Variables': name = 'Variable file'
            if name == 'Resource': name = 'Resource file'
            raise DataError("%s '%s' does not exist" % (name, path))
        if name in ['Variables','Library']:
            dirname, filename = os.path.split(os.path.normpath(path))
            path = os.path.join(utils.normpath(dirname), filename)
        else:
            path = utils.normpath(path)
        return path

    def _is_library_by_path(self, path, basedir):
        if os.path.splitext(path)[1].lower() in ['.py','.java','.class']:
            return os.path.isfile(os.path.join(basedir, path))
        if not path.endswith('/'):
            return False
        init = os.path.join(basedir, path.replace('/',os.sep), '__init__.py')
        return os.path.isfile(init)

    def _resolve_path(self, respath, basedir):
        for base in [basedir] + sys.path:
            if not os.path.isdir(base):
                continue
            path = os.path.join(base, respath)
            if os.path.exists(path):
                return path
        raise DataError()

    def report_invalid_syntax(self):
        self._item.report_invalid_syntax()
<commit_msg>proper error message when library setting has no value<commit_after>#  Copyright 2008-2009 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import os
import sys

from robot import utils
from robot.errors import DataError

class _Metadata:

    def __init__(self, metadata=None):
        self._data = {}
        for name in self._names.values():
            self._data[name.replace(' ','')] = None
        if metadata is not None:
            for item in metadata:
                self.set(item)

    def set(self, item):
        try:
            name = self._names[utils.normalize(item.name)].replace(' ', '')
        except KeyError:
            self._report_invalid_meta(item)
        else:
            if self._data[name] is None:
                self._data[name] = item.value
            else:
                self._data[name].extend(item.value)

    def _report_invalid_meta(self, item):
        name = self.__class__.__name__.replace('Metadata','')
        name = utils.printable_name(name, code_style=True)
        available = self._get_available_names()
        msg = "%s does not have metadata '%s'. Available: %s"
        item.report_invalid_syntax(msg % (name, item.name, available))

    def _get_available_names(self):
        names = dict([ (name, 1) for name in self._names.values() ]).keys()
        names.sort()
        return utils.seq2str(names)

    def get(self, key, default=None):
        value = self._data[key]
        if value is None:
            return default
        if key == 'Documentation':
            return ' '.join(value)
        return value

    def __getitem__(self, key):
        return self.get(key)

class TestCaseMetadata(_Metadata):
    _names = { 'documentation': 'Documentation',
               'document': 'Documentation',
               'setup': 'Setup',
               'precondition': 'Setup',
               'teardown': 'Teardown',
               'postcondition': 'Teardown',
               'tags': 'Tags',
               'timeout': 'Timeout' }

class UserKeywordMetadata(_Metadata):
    _names = { 'documentation': 'Documentation',
               'document': 'Documentation',
               'arguments': 'Arguments',
               'return': 'Return',
               'timeout': 'Timeout' }

class TestCaseFileMetadata(_Metadata):
    _names = { 'documentation': 'Documentation',
               'document': 'Documentation',
               'suitesetup': 'Suite Setup',
               'suiteprecondition': 'Suite Setup',
               'suiteteardown': 'Suite Teardown',
               'suitepostcondition': 'Suite Teardown',
               'testsetup': 'Test Setup',
               'testprecondition': 'Test Setup',
               'testteardown': 'Test Teardown',
               'testpostcondition': 'Test Teardown',
               'defaulttags': 'Default Tags',
               'forcetags': 'Force Tags',
               'testtimeout': 'Test Timeout' }

    def __init__(self, rawdata=None):
        _Metadata.__init__(self)
        self.user_metadata = {}
        self.imports = []
        if rawdata is not None:
            self._set_rawdata(rawdata)

    def _set_rawdata(self, rawdata):
        for item in rawdata.settings:
            name = item.name.lower()
            if name in ['library', 'resource', 'variables']:
                self.imports.append(ImportSetting(item))
            elif name.startswith('meta:'):
                self._set_user_metadata(item.name[5:].strip(), item.value)
            else:
                self.set(item)

    def _set_user_metadata(self, name, value):
        name = utils.printable_name(name)
        value = ' '.join(value)
        if self.user_metadata.has_key(name):
            value = '%s %s' % (self.user_metadata[name], value)
        self.user_metadata[name] = value

class TestSuiteInitFileMetadata(TestCaseFileMetadata):
    _deprecated_metadata = ['testsetup', 'testteardown', 'defaulttags',
                            'testtimeout']

    def set(self, item):
        TestCaseFileMetadata.set(self, item)
        if utils.normalize(item.name) in self._deprecated_metadata:
            msg = "Using setting '%s' in init file is deprecated and will be " \
                    "removed in the next major release." % (item.name)
            item.report_invalid_syntax(msg, 'WARN')

class ImportSetting:

    def __init__(self, item):
        self._item = item
        self.name = item.name = utils.normalize(item.name).capitalize()
        self.value = None

    def original_value(self):
        orig = self._item.copy().value
        self._at_least_one_value_required(orig)
        return orig

    def _at_least_one_value_required(self, values):
        if len(values) == 0:
            raise DataError("Setting '%s' requires a value" % self.name)

    def replace_variables(self, variables):
        item = self._item.copy()
        try:
            self.value = variables.replace_list(item.value)
        except DataError:
            raise DataError("Replacing variables from setting '%s' failed: %s"
                            % (self.name, utils.get_error_message()))
        self._at_least_one_value_required(self.value)
        if self.name == 'Resource' and len(self.value) > 1:
            raise DataError('Invalid resource import parameters: %s' % utils.seq2str2(self.value))
        basedir = utils.get_directory(item._parent._source)
        self.value[0] = self._get_path(self.name, self.value[0], basedir)

    def resolve_name(self, name):
        """Returns absolute path to file if name points to one, otherwise return name unchanged."""
        return self._get_path(self.name, name,
                              utils.get_directory(self._item._parent._source))

    def _get_path(self, name, path, basedir):
        if name == 'Library' and not self._is_library_by_path(path, basedir):
            return path
        try:
            path = self._resolve_path(path.replace('/', os.sep), basedir)
        except DataError:
            if name == 'Variables': name = 'Variable file'
            if name == 'Resource': name = 'Resource file'
            raise DataError("%s '%s' does not exist" % (name, path))
        if name in ['Variables','Library']:
            dirname, filename = os.path.split(os.path.normpath(path))
            path = os.path.join(utils.normpath(dirname), filename)
        else:
            path = utils.normpath(path)
        return path

    def _is_library_by_path(self, path, basedir):
        if os.path.splitext(path)[1].lower() in ['.py','.java','.class']:
            return os.path.isfile(os.path.join(basedir, path))
        if not path.endswith('/'):
            return False
        init = os.path.join(basedir, path.replace('/',os.sep), '__init__.py')
        return os.path.isfile(init)

    def _resolve_path(self, respath, basedir):
        for base in [basedir] + sys.path:
            if not os.path.isdir(base):
                continue
            path = os.path.join(base, respath)
            if os.path.exists(path):
                return path
        raise DataError()

    def report_invalid_syntax(self):
        self._item.report_invalid_syntax()
<|endoftext|>


<commit_before>'''
geodata.osm.extract
-------------------

Extracts nodes/ways/relations, their metadata and dependencies
from .osm XML files.
'''

import re
import six
import urllib
import HTMLParser

from collections import OrderedDict
from lxml import etree

from geodata.csv_utils import unicode_csv_reader
from geodata.text.normalize import normalize_string, NORMALIZE_STRING_DECOMPOSE, NORMALIZE_STRING_LATIN_ASCII
from geodata.encoding import safe_decode

WAY_OFFSET = 10 ** 15
RELATION_OFFSET = 2 * 10 ** 15

NODE = 'node'
WAY = 'way'
RELATION = 'relation'

ALL_OSM_TAGS = set([NODE, WAY, RELATION])
WAYS_RELATIONS = set([WAY, RELATION])

OSM_NAME_TAGS = (
    'name',
    'alt_name',
    'int_name',
    'nat_name',
    'reg_name',
    'loc_name',
    'official_name',
    'commonname',
    'common_name',
    'place_name',
    'short_name',
)

OSM_BASE_NAME_TAGS = (
    'tiger:name_base',
)

def parse_osm(filename, allowed_types=ALL_OSM_TAGS, dependencies=False):
    '''
    Parse a file in .osm format iteratively, generating tuples like:
    ('node:1', OrderedDict([('lat', '12.34'), ('lon', '23.45')])),
    ('node:2', OrderedDict([('lat', '12.34'), ('lon', '23.45')])),
    ('node:3', OrderedDict([('lat', '12.34'), ('lon', '23.45')])),
    ('node:4', OrderedDict([('lat', '12.34'), ('lon', '23.45')])),
    ('way:4444', OrderedDict([('name', 'Main Street')]), [1,2,3,4])
    '''
    f = open(filename)
    parser = etree.iterparse(f)

    single_type = len(allowed_types) == 1

    for (_, elem) in parser:
        elem_id = long(elem.attrib.pop('id', 0))
        item_type = elem.tag
        if elem_id >= WAY_OFFSET and elem_id < RELATION_OFFSET:
            elem_id -= WAY_OFFSET
            item_type = 'way'
        elif elem_id >= RELATION_OFFSET:
            elem_id -= RELATION_OFFSET
            item_type = 'relation'

        if item_type in allowed_types:
            attrs = OrderedDict(elem.attrib)
            deps = [] if dependencies else None

            for e in elem.getchildren():
                if e.tag == 'tag':
                    attrs[e.attrib['k']] = e.attrib['v']
                elif dependencies and item_type == 'way' and e.tag == 'nd':
                    deps.append(long(e.attrib['ref']))
                elif dependencies and item_type == 'relation' and e.tag == 'member' and \
                        e.attrib.get('type') in ('way', 'relation') and \
                        e.attrib.get('role') in ('inner', 'outer'):
                    deps.append((long(e.attrib['ref']), e.attrib.get('role')))

            key = elem_id if single_type else '{}:{}'.format(item_type, elem_id)
            yield key, attrs, deps

        if elem.tag in ALL_OSM_TAGS:
            elem.clear()
            while elem.getprevious() is not None:
                del elem.getparent()[0]

def osm_type_and_id(element_id):
    element_id = long(element_id)
    if element_id >= RELATION_OFFSET:
        id_type = RELATION
        element_id -= RELATION_OFFSET
    elif element_id >= WAY_OFFSET:
        id_type = WAY
        element_id -= WAY_OFFSET
    else:
        id_type = NODE

    return id_type, element_id

apposition_regex = re.compile('(.*[^\s])[\s]*\([\s]*(.*[^\s])[\s]*\)$', re.I)

html_parser = HTMLParser.HTMLParser()

def normalize_wikipedia_title(title):
    match = apposition_regex.match(title)
    if match:
        title = match.group(1)

    title = safe_decode(title)
    title = html_parser.unescape(title)
    title = urllib.unquote_plus(title)

    return title.replace(u'_', u' ').strip()

def osm_wikipedia_title_and_language(key, value):
    language = None
    if u':' in key:
        key, language = key.rsplit(u':', 1)

    if u':' in value:
        possible_language = value.split(u':', 1)[0]
        if len(possible_language) == 2 and language is None:
            language = possible_language
            value = value.rsplit(u':', 1)[-1]

    return normalize_wikipedia_title(value), language

non_breaking_dash = six.u('[-\u058a\u05be\u1400\u1806\u2010-\u2013\u2212\u2e17\u2e1a\ufe32\ufe63\uff0d]')
simple_number = six.u('(?:{})?[0-9]+(?:\.[0-9]+)?').format(non_breaking_dash)
simple_number_regex = re.compile(simple_number, re.UNICODE)

non_breaking_dash_regex = re.compile(non_breaking_dash, re.UNICODE)
number_range_regex = re.compile(six.u('({}){}({})').format(simple_number, non_breaking_dash, simple_number), re.UNICODE)
letter_range_regex = re.compile(r'([^\W\d_]){}([^\W\d_])'.format(non_breaking_dash.encode('unicode-escape')), re.UNICODE)

def parse_osm_number_range(value):
    value = normalize_string(value, string_options=NORMALIZE_STRING_LATIN_ASCII | NORMALIZE_STRING_DECOMPOSE)
    numbers = []
    values = value.split(six.u(';'))
    for val in values:
        val = val.strip()
        match = number_range_regex.match(val)
        if match:
            start_num, end_num = match.groups()
            try:
                start_num = int(start_num)
                end_num = int(end_num)
                if end_num > start_num:
                    if end_num - start_num > 100:
                        end_num = start_num + 100
                    for i in xrange(start_num, end_num + 1):
                        numbers.append(safe_decode(i))
                else:
                    numbers.extend([start_num, end_num])
                    continue
            except (TypeError, ValueError):
                numbers.extend([start_num, end_num])
                continue

        else:
            letter_match = letter_range_regex.match(val)
            if letter_match:
                start_num, end_num = letter_match.groups()
                start_num = ord(start_num)
                end_num = ord(end_num)
                if end_num > start_num:
                    if end_num - start_num > 100:
                        end_num = start_num + 100
                    for i in xrange(start_num, end_num + 1):
                        numbers.append(six.unichr(i))
                else:
                    numbers.extend([six.unichr(start_num), six.unichr(end_num)])
                    continue
            else:
                numbers.extend(non_breaking_dash_regex.split(safe_decode(val)))
    return numbers
<commit_msg>[osm] Extract all members with role= as dependencies<commit_after>'''
geodata.osm.extract
-------------------

Extracts nodes/ways/relations, their metadata and dependencies
from .osm XML files.
'''

import re
import six
import urllib
import HTMLParser

from collections import OrderedDict
from lxml import etree

from geodata.csv_utils import unicode_csv_reader
from geodata.text.normalize import normalize_string, NORMALIZE_STRING_DECOMPOSE, NORMALIZE_STRING_LATIN_ASCII
from geodata.encoding import safe_decode

WAY_OFFSET = 10 ** 15
RELATION_OFFSET = 2 * 10 ** 15

NODE = 'node'
WAY = 'way'
RELATION = 'relation'

ALL_OSM_TAGS = set([NODE, WAY, RELATION])
WAYS_RELATIONS = set([WAY, RELATION])

OSM_NAME_TAGS = (
    'name',
    'alt_name',
    'int_name',
    'nat_name',
    'reg_name',
    'loc_name',
    'official_name',
    'commonname',
    'common_name',
    'place_name',
    'short_name',
)

OSM_BASE_NAME_TAGS = (
    'tiger:name_base',
)

def parse_osm(filename, allowed_types=ALL_OSM_TAGS, dependencies=False):
    '''
    Parse a file in .osm format iteratively, generating tuples like:
    ('node:1', OrderedDict([('lat', '12.34'), ('lon', '23.45')])),
    ('node:2', OrderedDict([('lat', '12.34'), ('lon', '23.45')])),
    ('node:3', OrderedDict([('lat', '12.34'), ('lon', '23.45')])),
    ('node:4', OrderedDict([('lat', '12.34'), ('lon', '23.45')])),
    ('way:4444', OrderedDict([('name', 'Main Street')]), [1,2,3,4])
    '''
    f = open(filename)
    parser = etree.iterparse(f)

    single_type = len(allowed_types) == 1

    for (_, elem) in parser:
        elem_id = long(elem.attrib.pop('id', 0))
        item_type = elem.tag
        if elem_id >= WAY_OFFSET and elem_id < RELATION_OFFSET:
            elem_id -= WAY_OFFSET
            item_type = 'way'
        elif elem_id >= RELATION_OFFSET:
            elem_id -= RELATION_OFFSET
            item_type = 'relation'

        if item_type in allowed_types:
            attrs = OrderedDict(elem.attrib)
            deps = [] if dependencies else None

            for e in elem.getchildren():
                if e.tag == 'tag':
                    attrs[e.attrib['k']] = e.attrib['v']
                elif dependencies and item_type == 'way' and e.tag == 'nd':
                    deps.append(long(e.attrib['ref']))
                elif dependencies and item_type == 'relation' and e.tag == 'member' and \
                        e.attrib.get('type') in ('way', 'relation') and \
                        'role' in e.attrib:
                    deps.append((long(e.attrib['ref']), e.attrib['role']))

            key = elem_id if single_type else '{}:{}'.format(item_type, elem_id)
            yield key, attrs, deps

        if elem.tag in ALL_OSM_TAGS:
            elem.clear()
            while elem.getprevious() is not None:
                del elem.getparent()[0]

def osm_type_and_id(element_id):
    element_id = long(element_id)
    if element_id >= RELATION_OFFSET:
        id_type = RELATION
        element_id -= RELATION_OFFSET
    elif element_id >= WAY_OFFSET:
        id_type = WAY
        element_id -= WAY_OFFSET
    else:
        id_type = NODE

    return id_type, element_id

apposition_regex = re.compile('(.*[^\s])[\s]*\([\s]*(.*[^\s])[\s]*\)$', re.I)

html_parser = HTMLParser.HTMLParser()

def normalize_wikipedia_title(title):
    match = apposition_regex.match(title)
    if match:
        title = match.group(1)

    title = safe_decode(title)
    title = html_parser.unescape(title)
    title = urllib.unquote_plus(title)

    return title.replace(u'_', u' ').strip()

def osm_wikipedia_title_and_language(key, value):
    language = None
    if u':' in key:
        key, language = key.rsplit(u':', 1)

    if u':' in value:
        possible_language = value.split(u':', 1)[0]
        if len(possible_language) == 2 and language is None:
            language = possible_language
            value = value.rsplit(u':', 1)[-1]

    return normalize_wikipedia_title(value), language

non_breaking_dash = six.u('[-\u058a\u05be\u1400\u1806\u2010-\u2013\u2212\u2e17\u2e1a\ufe32\ufe63\uff0d]')
simple_number = six.u('(?:{})?[0-9]+(?:\.[0-9]+)?').format(non_breaking_dash)
simple_number_regex = re.compile(simple_number, re.UNICODE)

non_breaking_dash_regex = re.compile(non_breaking_dash, re.UNICODE)
number_range_regex = re.compile(six.u('({}){}({})').format(simple_number, non_breaking_dash, simple_number), re.UNICODE)
letter_range_regex = re.compile(r'([^\W\d_]){}([^\W\d_])'.format(non_breaking_dash.encode('unicode-escape')), re.UNICODE)

def parse_osm_number_range(value):
    value = normalize_string(value, string_options=NORMALIZE_STRING_LATIN_ASCII | NORMALIZE_STRING_DECOMPOSE)
    numbers = []
    values = value.split(six.u(';'))
    for val in values:
        val = val.strip()
        match = number_range_regex.match(val)
        if match:
            start_num, end_num = match.groups()
            try:
                start_num = int(start_num)
                end_num = int(end_num)
                if end_num > start_num:
                    if end_num - start_num > 100:
                        end_num = start_num + 100
                    for i in xrange(start_num, end_num + 1):
                        numbers.append(safe_decode(i))
                else:
                    numbers.extend([start_num, end_num])
                    continue
            except (TypeError, ValueError):
                numbers.extend([start_num, end_num])
                continue

        else:
            letter_match = letter_range_regex.match(val)
            if letter_match:
                start_num, end_num = letter_match.groups()
                start_num = ord(start_num)
                end_num = ord(end_num)
                if end_num > start_num:
                    if end_num - start_num > 100:
                        end_num = start_num + 100
                    for i in xrange(start_num, end_num + 1):
                        numbers.append(six.unichr(i))
                else:
                    numbers.extend([six.unichr(start_num), six.unichr(end_num)])
                    continue
            else:
                numbers.extend(non_breaking_dash_regex.split(safe_decode(val)))
    return numbers
<|endoftext|>