<commit_before>###########################################################################
# Joshua R. Boverhof, LBNL
# See LBNLCopyright for copyright notice!
###########################################################################
import types, copy
import ZSI
from ZSI import TC, TCtimes, TCcompound
from ZSI.TC import TypeCode
from ZSI import _copyright, _children, _child_elements, \
        _inttypes, _stringtypes, _seqtypes, _find_arraytype, _find_href, \
        _find_type, \
        EvaluateException
from ZSI.TCcompound import _check_typecode_list,_find_arrayoffset , \
     _find_arrayposition,_offset_pat,_position_pat

from xml.dom.ext import SplitQName
from xml.ns import SOAP, SCHEMA

DEBUG  = 0
def print_debug(msg, level=1, *l, **kw):
    if DEBUG >= level:
        for i in l:
            msg += '\n*\t' + str(i)
        for k,v in kw.items():
            msg += '\n**\t%s: %s' %(k,v)
        print 'TYPECODES(%d): %s' %(level,msg)

###########################################################################
# Module Classes: BaseTypeInterpreter
###########################################################################

TC.Boolean.tag = 'boolean'
TCcompound.Array.tag = 'array'
TC.Decimal.tag = 'decimal'
TC.Base64String.tag = 'base64Binary'

class NamespaceException(Exception): pass
class BaseTypeInterpreter:
    """
    Example mapping of xsd/soapenc types to zsi python types.
    Checks against all available classes in ZSI.TC
    Used in TypeCodeSequenceGenerator, and TC_Template.
    """

    def __init__(self):
        self.__type_list = [TC.Boolean, TC.Iinteger, TC.IunsignedShort, \
                            TC.gYearMonth, TC.Base64String, \
                            TC.InonNegativeInteger, TC.Iint, TC.String, \
                            TC.gDateTime, TC.IunsignedInt, \
                            TC.IpositiveInteger, TC.FPfloat, TC.gDay, \
                            TC.InegativeInteger, TC.gDate, TC.URI, \
                            TC.HexBinaryString, TC.IunsignedByte, \
                            TC.gMonthDay, TC.InonPositiveInteger, \
                            TC.Ibyte, TC.FPdouble, TC.gTime, TC.gYear, \
                            TC.Ilong, TC.IunsignedLong, TC.Ishort, \
                            TC.Decimal]

        self.__tc_to_int = [
            ZSI.TCnumbers.IEnumeration,
            ZSI.TCnumbers.Iint,
            ZSI.TCnumbers.Iinteger,
            ZSI.TCnumbers.Ilong,
            ZSI.TCnumbers.InegativeInteger,
            ZSI.TCnumbers.InonNegativeInteger,
            ZSI.TCnumbers.InonPositiveInteger,
            ZSI.TC.Integer,
            ZSI.TCnumbers.IpositiveInteger,
            ZSI.TCnumbers.Ishort]

        self.__tc_to_float = [
            ZSI.TC.Decimal,
            ZSI.TCnumbers.FPEnumeration,
            ZSI.TCnumbers.FPdouble,
            ZSI.TCnumbers.FPfloat]

        self.__tc_to_string = [
            ZSI.TC.Base64String,
            ZSI.TC.Enumeration,
            ZSI.TC.HexBinaryString,
            ZSI.TCnumbers.Ibyte,
            ZSI.TCnumbers.IunsignedByte,
            ZSI.TCnumbers.IunsignedInt,
            ZSI.TCnumbers.IunsignedLong,
            ZSI.TCnumbers.IunsignedShort,
            ZSI.TC.String,
            ZSI.TC.URI,
            ZSI.TC.XMLString]

        self.__tc_to_date = [
            ZSI.TCtimes.gDate,
            ZSI.TCtimes.gDateTime,
            ZSI.TCtimes.gDay,
            ZSI.TCtimes.gMonthDay,
            ZSI.TCtimes.gTime,
            ZSI.TCtimes.gYear,
            ZSI.TCtimes.gYearMonth]

        return

    def __get_xsd_typecode(self, msg_type):
        for tc in self.__type_list:
            if tc.tag == msg_type:
                break
        else:
            tc = TC.Any
        return tc

    def __get_soapenc_typecode(self, msg_type):
        if msg_type == 'Array':
            return TCcompound.Array
        if msg_type == 'base64':
            return TC.Base64String
        else:
            raise NamespaceException

    def get_typeclass(self, msg_type, targetNamespace):
        prefix, name = SplitQName(msg_type)
        if targetNamespace in SCHEMA.XSD_LIST:
            return self.__get_xsd_typecode(name)
        elif targetNamespace in [SOAP.ENC]:
            return self.__get_soapenc_typecode(name)
        return None

    def get_pythontype(self, msg_type, targetNamespace):
        tc = self.get_typeclass(msg_type, targetNamespace)
        if tc in self.__tc_to_int:
            return 'int'
        elif tc in self.__tc_to_float:
            return 'float'
        elif tc in self.__tc_to_string:
            return 'str'
        elif tc in self.__tc_to_date:
            return 'tuple'
        elif tc in [TCcompound.Array]:
            return 'list'
        elif isinstance(tc, TypeCode):
            raise EvaluateException,\
               'failed to map zsi typecode to a python type'
        return None

<commit_msg>minor tweek<commit_after>###########################################################################
# Joshua R. Boverhof, LBNL
# See LBNLCopyright for copyright notice!
###########################################################################
import types, copy
import ZSI
from ZSI import TC, TCtimes, TCcompound
from ZSI.TC import TypeCode
from ZSI import _copyright, _children, _child_elements, \
        _inttypes, _stringtypes, _seqtypes, _find_arraytype, _find_href, \
        _find_type, \
        EvaluateException
from ZSI.TCcompound import _check_typecode_list,_find_arrayoffset , \
     _find_arrayposition,_offset_pat,_position_pat

from xml.dom.ext import SplitQName
from xml.ns import SOAP, SCHEMA

DEBUG  = 0
def print_debug(msg, level=1, *l, **kw):
    if DEBUG >= level:
        for i in l:
            msg += '\n*\t' + str(i)
        for k,v in kw.items():
            msg += '\n**\t%s: %s' %(k,v)
        print 'TYPECODES(%d): %s' %(level,msg)

###########################################################################
# Module Classes: BaseTypeInterpreter
###########################################################################

TC.Boolean.tag = 'boolean'
TCcompound.Array.tag = 'array'
TC.Decimal.tag = 'decimal'
TC.Base64String.tag = 'base64Binary'

class NamespaceException(Exception): pass
class BaseTypeInterpreter:
    """
    Example mapping of xsd/soapenc types to zsi python types.
    Checks against all available classes in ZSI.TC
    Used in TypeCodeSequenceGenerator, and TC_Template.
    """

    def __init__(self):
        self.__type_list = [TC.Boolean, TC.Iinteger, TC.IunsignedShort, \
                            TC.gYearMonth, TC.Base64String, \
                            TC.InonNegativeInteger, TC.Iint, TC.String, \
                            TC.gDateTime, TC.IunsignedInt, \
                            TC.IpositiveInteger, TC.FPfloat, TC.gDay, \
                            TC.InegativeInteger, TC.gDate, TC.URI, \
                            TC.HexBinaryString, TC.IunsignedByte, \
                            TC.gMonthDay, TC.InonPositiveInteger, \
                            TC.Ibyte, TC.FPdouble, TC.gTime, TC.gYear, \
                            TC.Ilong, TC.IunsignedLong, TC.Ishort, \
                            TC.Decimal]

        self.__tc_to_int = [
            ZSI.TCnumbers.IEnumeration,
            ZSI.TCnumbers.Iint,
            ZSI.TCnumbers.Iinteger,
            ZSI.TCnumbers.Ilong,
            ZSI.TCnumbers.InegativeInteger,
            ZSI.TCnumbers.InonNegativeInteger,
            ZSI.TCnumbers.InonPositiveInteger,
            ZSI.TC.Integer,
            ZSI.TCnumbers.IpositiveInteger,
            ZSI.TCnumbers.Ishort]

        self.__tc_to_float = [
            ZSI.TC.Decimal,
            ZSI.TCnumbers.FPEnumeration,
            ZSI.TCnumbers.FPdouble,
            ZSI.TCnumbers.FPfloat]

        self.__tc_to_string = [
            ZSI.TC.Base64String,
            ZSI.TC.Enumeration,
            ZSI.TC.HexBinaryString,
            ZSI.TCnumbers.Ibyte,
            ZSI.TCnumbers.IunsignedByte,
            ZSI.TCnumbers.IunsignedInt,
            ZSI.TCnumbers.IunsignedLong,
            ZSI.TCnumbers.IunsignedShort,
            ZSI.TC.String,
            ZSI.TC.URI,
            ZSI.TC.XMLString]

        self.__tc_to_date = [
            ZSI.TCtimes.gDate,
            ZSI.TCtimes.gDateTime,
            ZSI.TCtimes.gDay,
            ZSI.TCtimes.gMonthDay,
            ZSI.TCtimes.gTime,
            ZSI.TCtimes.gYear,
            ZSI.TCtimes.gYearMonth]

        return

    def __get_xsd_typecode(self, msg_type):
        for tc in self.__type_list:
            if tc.tag == msg_type:
                break
        else:
            tc = TC.Any
        return tc

    def __get_soapenc_typecode(self, msg_type):
        if msg_type == 'Array':
            return TCcompound.Array
        elif msg_type == 'base64':
            return TC.Base64String
        else:
            raise NamespaceException

    def get_typeclass(self, msg_type, targetNamespace):
        prefix, name = SplitQName(msg_type)
        if targetNamespace in SCHEMA.XSD_LIST:
            return self.__get_xsd_typecode(name)
        elif targetNamespace in [SOAP.ENC]:
            return self.__get_soapenc_typecode(name)
        return None

    def get_pythontype(self, msg_type, targetNamespace):
        tc = self.get_typeclass(msg_type, targetNamespace)
        if tc in self.__tc_to_int:
            return 'int'
        elif tc in self.__tc_to_float:
            return 'float'
        elif tc in self.__tc_to_string:
            return 'str'
        elif tc in self.__tc_to_date:
            return 'tuple'
        elif tc in [TCcompound.Array]:
            return 'list'
        elif isinstance(tc, TypeCode):
            raise EvaluateException,\
               'failed to map zsi typecode to a python type'
        return None

<|endoftext|>


<commit_before>from django.test import TestCase
from corehq.apps.domain.shortcuts import create_domain

from ..models import Location, LocationType
from .test_locations import LocationTestBase
from .util import make_loc, delete_all_locations

class TestPathLineageAndHierarchy(LocationTestBase):

    def setUp(self):
        super(TestPathLineageAndHierarchy, self).setUp()
        locs = [
            ('Mass', 'state'),
            ('Suffolk', 'district'),
            ('Boston', 'block'),
        ]
        parent = None
        self.all_locs = []
        for name, type_ in locs:
            parent = make_loc(name, type=type_, parent=parent)
            self.all_locs.append(parent)
        self.all_loc_ids = [l._id for l in self.all_locs]

    def test_path(self):
        for i in range(len(self.all_locs)):
            self.assertEqual(self.all_loc_ids[:i+1], self.all_locs[i].path)

    def test_lineage(self):
        for i in range(len(self.all_locs)):
            # a location should not be included in its own lineage
            expected_lineage = list(reversed(self.all_loc_ids[:i]))
            self.assertEqual(expected_lineage, self.all_locs[i].lineage)

    def test_move(self):
        original_parent = self.all_locs[1].sql_location
        new_parent = make_loc('New York', type='state').sql_location
        district = make_loc('NYC', type='block', parent=original_parent.couch_location).sql_location
        self.assertEqual(original_parent.site_code, district.couch_location.parent.site_code)

        district.parent = new_parent
        district.save()
        self.assertEqual(new_parent.site_code, district.couch_location.parent.site_code)

    def test_move_to_root(self):
        original_parent = self.all_locs[1].sql_location
        district = make_loc('NYC', type='block', parent=original_parent.couch_location).sql_location
        self.assertEqual(original_parent.site_code, district.couch_location.parent.site_code)

        district.parent = None
        district.save()
        self.assertEqual(None, district.couch_location.parent)

class TestNoCouchLocationTypes(TestCase):

    @classmethod
    def setUpClass(cls):
        create_domain('test-domain')
        LocationType.objects.create(domain='test-domain', name='test-type')

    @classmethod
    def tearDownClass(cls):
        LocationType.objects.all().delete()

    def setUp(self):
        self.loc = Location(
            domain='test-domain',
            name='test-type-location-name',
            location_type='test-type',
        )
        self.loc.save()

    def tearDown(self):
        delete_all_locations()

    def test_change_location_type_name(self):
        loc_type = LocationType.objects.create(domain='test-domain',
                                               name='old-name')
        loc = Location(
            domain='test-domain',
            name='Somewhere',
            location_type='old-name'
        )
        loc.save()
        loc_type.name = 'new-name'
        loc_type.save()
        # You need to look up the location from the db again, because the
        # in-memory version stores the location_type it was created with
        self.assertEqual(Location.get(loc.location_id).location_type_name, 'new-name')

    def test_no_location_type(self):
        with self.assertRaises(LocationType.DoesNotExist):
            loc = Location(name="Something")
            loc.save()

    def test_get_and_save(self):
        # Get a location from the db, wrap it, access location_type, and save
        loc = Location.get(self.loc.location_id)
        self.assertEqual(loc.location_type_name, 'test-type')
        loc.save()

    def test_change_type_later(self):
        new_type = LocationType.objects.create(domain='test-domain',
                                               name='new-type')
        self.loc.set_location_type('new-type')
        self.loc.save()
        self.assertEqual(self.loc.location_type_name, 'new-type')
        self.assertEqual(self.loc.sql_location.location_type, new_type)
        # pull the loc from the db again
        self.assertEqual(Location.get(self.loc.location_id).location_type_name, 'new-type')
        new_type.delete()

    def test_change_to_nonexistent_type(self):
        with self.assertRaises(LocationType.DoesNotExist):
            self.loc.set_location_type('nonexistent-type')
            self.loc.save()
        self.assertEqual(self.loc.location_type_name, 'test-type')
        self.assertEqual(self.loc.sql_location.location_type.name, 'test-type')
<commit_msg>Add lineage tests and drop unneeded parent class<commit_after>from django.test import TestCase
from corehq.apps.commtrack.tests.util import bootstrap_location_types
from corehq.apps.domain.shortcuts import create_domain

from ..models import Location, LocationType
from .util import make_loc, delete_all_locations

class TestPathLineageAndHierarchy(TestCase):

    def setUp(self):
        super(TestPathLineageAndHierarchy, self).setUp()
        self.domain = create_domain('locations-test')
        bootstrap_location_types(self.domain.name)
        locs = [
            ('Mass', 'state'),
            ('Suffolk', 'district'),
            ('Boston', 'block'),
        ]
        parent = None
        self.all_locs = []
        for name, type_ in locs:
            parent = make_loc(name, type=type_, parent=parent)
            self.all_locs.append(parent)
        self.all_loc_ids = [l.location_id for l in self.all_locs]
        self.loc_id_by_name = {l.name: l.location_id for l in self.all_locs}

    def tearDown(self):
        self.domain.delete()

    def test_path(self):
        for i in range(len(self.all_locs)):
            self.assertEqual(self.all_loc_ids[:i+1], self.all_locs[i].path)

    def test_lineage(self):
        for i in range(len(self.all_locs)):
            # a location should not be included in its own lineage
            expected_lineage = list(reversed(self.all_loc_ids[:i]))
            self.assertEqual(expected_lineage, self.all_locs[i].lineage)

    def test_move(self):
        original_parent = self.all_locs[1].sql_location
        district = make_loc('NYC', type='block', parent=original_parent.couch_location).sql_location
        self.assertEqual(original_parent.site_code, district.couch_location.parent.site_code)
        self.assertEqual([self.loc_id_by_name['Suffolk'], self.loc_id_by_name['Mass']],
                         district.couch_location.lineage)

        new_parent = make_loc('New York', type='state').sql_location
        district.parent = new_parent
        district.save()
        self.assertEqual(new_parent.site_code, district.couch_location.parent.site_code)
        self.assertEqual([new_parent.location_id], district.couch_location.lineage)

    def test_move_to_root(self):
        original_parent = self.all_locs[1].sql_location
        district = make_loc('NYC', type='block', parent=original_parent.couch_location).sql_location
        self.assertEqual(original_parent.site_code, district.couch_location.parent.site_code)
        self.assertEqual([self.loc_id_by_name['Suffolk'], self.loc_id_by_name['Mass']],
                         district.couch_location.lineage)

        district.parent = None
        district.save()
        self.assertEqual(None, district.couch_location.parent)
        self.assertEqual([], district.couch_location.lineage)

class TestNoCouchLocationTypes(TestCase):

    @classmethod
    def setUpClass(cls):
        create_domain('test-domain')
        LocationType.objects.create(domain='test-domain', name='test-type')

    @classmethod
    def tearDownClass(cls):
        LocationType.objects.all().delete()

    def setUp(self):
        self.loc = Location(
            domain='test-domain',
            name='test-type-location-name',
            location_type='test-type',
        )
        self.loc.save()

    def tearDown(self):
        delete_all_locations()

    def test_change_location_type_name(self):
        loc_type = LocationType.objects.create(domain='test-domain',
                                               name='old-name')
        loc = Location(
            domain='test-domain',
            name='Somewhere',
            location_type='old-name'
        )
        loc.save()
        loc_type.name = 'new-name'
        loc_type.save()
        # You need to look up the location from the db again, because the
        # in-memory version stores the location_type it was created with
        self.assertEqual(Location.get(loc.location_id).location_type_name, 'new-name')

    def test_no_location_type(self):
        with self.assertRaises(LocationType.DoesNotExist):
            loc = Location(name="Something")
            loc.save()

    def test_get_and_save(self):
        # Get a location from the db, wrap it, access location_type, and save
        loc = Location.get(self.loc.location_id)
        self.assertEqual(loc.location_type_name, 'test-type')
        loc.save()

    def test_change_type_later(self):
        new_type = LocationType.objects.create(domain='test-domain',
                                               name='new-type')
        self.loc.set_location_type('new-type')
        self.loc.save()
        self.assertEqual(self.loc.location_type_name, 'new-type')
        self.assertEqual(self.loc.sql_location.location_type, new_type)
        # pull the loc from the db again
        self.assertEqual(Location.get(self.loc.location_id).location_type_name, 'new-type')
        new_type.delete()

    def test_change_to_nonexistent_type(self):
        with self.assertRaises(LocationType.DoesNotExist):
            self.loc.set_location_type('nonexistent-type')
            self.loc.save()
        self.assertEqual(self.loc.location_type_name, 'test-type')
        self.assertEqual(self.loc.sql_location.location_type.name, 'test-type')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import collections
import re
import urllib
import logging
import urlparse
from contextlib import contextmanager

import furl

from flask import request, url_for

from website import settings as website_settings
from api.base import settings as api_settings
from modularodm import Q
from modularodm.exceptions import NoResultsFound

# Keep me: Makes rubeus importable from website.util
from . import rubeus  # noqa

logger = logging.getLogger(__name__)

guid_url_node_pattern = re.compile('^/project/[a-zA-Z0-9]{5,}/node(?=/[a-zA-Z0-9]{5,})')
guid_url_project_pattern = re.compile('^/project(?=/[a-zA-Z0-9]{5,})')
guid_url_profile_pattern = re.compile('^/profile(?=/[a-zA-Z0-9]{5,})')

waterbutler_action_map = {
    'upload': 'file',
    'delete': 'file',
    'download': 'file',
    'metadata': 'data',
    'create_folder': 'file',
}

# Function courtesy of @brianjgeiger and @abought, moved from API utils
def rapply(data, func, *args, **kwargs):
    """Recursively apply a function to all values in an iterable
    :param dict | list | basestring data: iterable to apply func to
    :param function func:
    """
    if isinstance(data, collections.Mapping):
        return {
            key: rapply(value, func, *args, **kwargs)
            for key, value in data.iteritems()
        }
    elif isinstance(data, collections.Iterable) and not isinstance(data, basestring):
        desired_type = type(data)
        return desired_type(
            rapply(item, func, *args, **kwargs) for item in data
        )
    else:
        return func(data, *args, **kwargs)

def conjunct(words, conj='and'):
    words = list(words)
    num_words = len(words)
    if num_words == 0:
        return ''
    elif num_words == 1:
        return words[0]
    elif num_words == 2:
        return ' {0} '.format(conj).join(words)
    elif num_words > 2:
        return ', '.join(words[:-1]) + ', {0} {1}'.format(conj, words[-1])

def _get_guid_url_for(url):
    """URL Post-processor transforms specific `/project/<pid>` or `/project/<pid>/node/<nid>`
    urls into guid urls. Ex: `<nid>/wiki/home`.
    """
    guid_url = guid_url_node_pattern.sub('', url, count=1)
    guid_url = guid_url_project_pattern.sub('', guid_url, count=1)
    guid_url = guid_url_profile_pattern.sub('', guid_url, count=1)
    return guid_url

def api_url_for(view_name, _absolute=False, _xml=False, *args, **kwargs):
    """Reverse URL lookup for API routes (that use the JSONRenderer or XMLRenderer).
    Takes the same arguments as Flask's url_for, with the addition of
    `_absolute`, which will make an absolute URL with the correct HTTP scheme
    based on whether the app is in debug mode. The _xml flag sets the renderer to use.
    """
    renderer = 'XMLRenderer' if _xml else 'JSONRenderer'

    url = url_for('{0}__{1}'.format(renderer, view_name), *args, **kwargs)

    if _absolute:
        # We do NOT use the url_for's _external kwarg because app.config['SERVER_NAME'] alters
        # behavior in an unknown way (currently breaks tests). /sloria /jspies
        return urlparse.urljoin(website_settings.DOMAIN, url)
    return url

def api_v2_url(path_str,
               params=None,
               base_route=website_settings.API_DOMAIN,
               base_prefix=api_settings.API_BASE,
               **kwargs):
    """
    Convenience function for APIv2 usage: Concatenates parts of the absolute API url based on arguments provided

    For example: given path_str = '/nodes/abcd3/contributors/' and params {'filter[fullname]': 'bob'},
        this function would return the following on the local staging environment:
        'http://localhost:8000/nodes/abcd3/contributors/?filter%5Bfullname%5D=bob'

    This is NOT a full lookup function. It does not verify that a route actually exists to match the path_str given.
    """
    params = params or {}  # Optional params dict for special-character param names, eg filter[fullname]

    base_url = furl.furl(base_route + base_prefix)

    base_url.path.add([x for x in path_str.split('/') if x] + [''])

    base_url.args.update(params)
    base_url.args.update(kwargs)
    return str(base_url)

def web_url_for(view_name, _absolute=False, _guid=False, *args, **kwargs):
    """Reverse URL lookup for web routes (those that use the OsfWebRenderer).
    Takes the same arguments as Flask's url_for, with the addition of
    `_absolute`, which will make an absolute URL with the correct HTTP scheme
    based on whether the app is in debug mode.
    """
    url = url_for('OsfWebRenderer__{0}'.format(view_name), *args, **kwargs)
    if _guid:
        url = _get_guid_url_for(url)

    if _absolute:
        # We do NOT use the url_for's _external kwarg because app.config['SERVER_NAME'] alters
        # behavior in an unknown way (currently breaks tests). /sloria /jspies
        return urlparse.urljoin(website_settings.DOMAIN, url)
    return url

def is_json_request():
    """Return True if the current request is a JSON/AJAX request."""
    content_type = request.content_type
    return content_type and ('application/json' in content_type)

def waterbutler_url_for(route, provider, path, node, user=None, **kwargs):
    """DEPRECATED Use waterbutler_api_url_for
    Reverse URL lookup for WaterButler routes
    :param str route: The action to preform, upload, download, delete...
    :param str provider: The name of the requested provider
    :param str path: The path of the requested file or folder
    :param Node node: The node being accessed
    :param User user: The user whos cookie will be used or None
    :param dict kwargs: Addition query parameters to be appended
    """
    url = furl.furl(website_settings.WATERBUTLER_URL)
    url.path.segments.append(waterbutler_action_map[route])

    url.args.update({
        'path': path,
        'nid': node._id,
        'provider': provider,
    })

    if user:
        url.args['cookie'] = user.get_or_create_cookie()
    elif website_settings.COOKIE_NAME in request.cookies:
        url.args['cookie'] = request.cookies[website_settings.COOKIE_NAME]

    view_only = False
    if 'view_only' in kwargs:
        view_only = kwargs.get('view_only')
    else:
        view_only = request.args.get('view_only')

    url.args['view_only'] = view_only

    url.args.update(kwargs)
    return url.url

def waterbutler_api_url_for(node_id, provider, path='/', **kwargs):
    assert path.startswith('/'), 'Path must always start with /'
    url = furl.furl(website_settings.WATERBUTLER_URL)
    segments = ['v1', 'resources', node_id, 'providers', provider] + path.split('/')[1:]
    url.path.segments.extend([urllib.quote(x.encode('utf-8')) for x in segments])
    url.args.update(kwargs)
    return url.url

@contextmanager
def disconnected_from(signal, listener):
    """Temporarily disconnect a Blinker signal."""
    signal.disconnect(listener)
    yield
    signal.connect(listener)

def check_private_key_for_anonymized_link(private_key):
    from website.project.model import PrivateLink

    is_anonymous = False
    if private_key is not None:
        try:
            link = PrivateLink.find_one(Q('key', 'eq', private_key))
        except NoResultsFound:
            link = None
        if link is not None:
            is_anonymous = link.anonymous
    return is_anonymous

def get_headers_from_request(req):
    """ Get and normalize DRF and Flask request headers
    """
    headers = getattr(req, 'META', {})
    if headers:
        headers = {
            '-'.join([part.capitalize() for part in k.split('_')]).replace('Http-', ''): v
            for k, v in headers.items()
        }
        headers['Remote-Addr'] = (headers.get('X-Forwarded-For') or headers.get('Remote-Addr')).split(',')[0].strip()
    else:
        headers = getattr(req, 'headers', {})
        headers = {
            k: v
            for k, v in headers.items()
        }
        headers['Remote-Addr'] = req.remote_addr
    return headers
<commit_msg>allow non remote-addr for tests<commit_after># -*- coding: utf-8 -*-

import collections
import re
import urllib
import logging
import urlparse
from contextlib import contextmanager

import furl

from flask import request, url_for

from website import settings as website_settings
from api.base import settings as api_settings
from modularodm import Q
from modularodm.exceptions import NoResultsFound

# Keep me: Makes rubeus importable from website.util
from . import rubeus  # noqa

logger = logging.getLogger(__name__)

guid_url_node_pattern = re.compile('^/project/[a-zA-Z0-9]{5,}/node(?=/[a-zA-Z0-9]{5,})')
guid_url_project_pattern = re.compile('^/project(?=/[a-zA-Z0-9]{5,})')
guid_url_profile_pattern = re.compile('^/profile(?=/[a-zA-Z0-9]{5,})')

waterbutler_action_map = {
    'upload': 'file',
    'delete': 'file',
    'download': 'file',
    'metadata': 'data',
    'create_folder': 'file',
}

# Function courtesy of @brianjgeiger and @abought, moved from API utils
def rapply(data, func, *args, **kwargs):
    """Recursively apply a function to all values in an iterable
    :param dict | list | basestring data: iterable to apply func to
    :param function func:
    """
    if isinstance(data, collections.Mapping):
        return {
            key: rapply(value, func, *args, **kwargs)
            for key, value in data.iteritems()
        }
    elif isinstance(data, collections.Iterable) and not isinstance(data, basestring):
        desired_type = type(data)
        return desired_type(
            rapply(item, func, *args, **kwargs) for item in data
        )
    else:
        return func(data, *args, **kwargs)

def conjunct(words, conj='and'):
    words = list(words)
    num_words = len(words)
    if num_words == 0:
        return ''
    elif num_words == 1:
        return words[0]
    elif num_words == 2:
        return ' {0} '.format(conj).join(words)
    elif num_words > 2:
        return ', '.join(words[:-1]) + ', {0} {1}'.format(conj, words[-1])

def _get_guid_url_for(url):
    """URL Post-processor transforms specific `/project/<pid>` or `/project/<pid>/node/<nid>`
    urls into guid urls. Ex: `<nid>/wiki/home`.
    """
    guid_url = guid_url_node_pattern.sub('', url, count=1)
    guid_url = guid_url_project_pattern.sub('', guid_url, count=1)
    guid_url = guid_url_profile_pattern.sub('', guid_url, count=1)
    return guid_url

def api_url_for(view_name, _absolute=False, _xml=False, *args, **kwargs):
    """Reverse URL lookup for API routes (that use the JSONRenderer or XMLRenderer).
    Takes the same arguments as Flask's url_for, with the addition of
    `_absolute`, which will make an absolute URL with the correct HTTP scheme
    based on whether the app is in debug mode. The _xml flag sets the renderer to use.
    """
    renderer = 'XMLRenderer' if _xml else 'JSONRenderer'

    url = url_for('{0}__{1}'.format(renderer, view_name), *args, **kwargs)

    if _absolute:
        # We do NOT use the url_for's _external kwarg because app.config['SERVER_NAME'] alters
        # behavior in an unknown way (currently breaks tests). /sloria /jspies
        return urlparse.urljoin(website_settings.DOMAIN, url)
    return url

def api_v2_url(path_str,
               params=None,
               base_route=website_settings.API_DOMAIN,
               base_prefix=api_settings.API_BASE,
               **kwargs):
    """
    Convenience function for APIv2 usage: Concatenates parts of the absolute API url based on arguments provided

    For example: given path_str = '/nodes/abcd3/contributors/' and params {'filter[fullname]': 'bob'},
        this function would return the following on the local staging environment:
        'http://localhost:8000/nodes/abcd3/contributors/?filter%5Bfullname%5D=bob'

    This is NOT a full lookup function. It does not verify that a route actually exists to match the path_str given.
    """
    params = params or {}  # Optional params dict for special-character param names, eg filter[fullname]

    base_url = furl.furl(base_route + base_prefix)

    base_url.path.add([x for x in path_str.split('/') if x] + [''])

    base_url.args.update(params)
    base_url.args.update(kwargs)
    return str(base_url)

def web_url_for(view_name, _absolute=False, _guid=False, *args, **kwargs):
    """Reverse URL lookup for web routes (those that use the OsfWebRenderer).
    Takes the same arguments as Flask's url_for, with the addition of
    `_absolute`, which will make an absolute URL with the correct HTTP scheme
    based on whether the app is in debug mode.
    """
    url = url_for('OsfWebRenderer__{0}'.format(view_name), *args, **kwargs)
    if _guid:
        url = _get_guid_url_for(url)

    if _absolute:
        # We do NOT use the url_for's _external kwarg because app.config['SERVER_NAME'] alters
        # behavior in an unknown way (currently breaks tests). /sloria /jspies
        return urlparse.urljoin(website_settings.DOMAIN, url)
    return url

def is_json_request():
    """Return True if the current request is a JSON/AJAX request."""
    content_type = request.content_type
    return content_type and ('application/json' in content_type)

def waterbutler_url_for(route, provider, path, node, user=None, **kwargs):
    """DEPRECATED Use waterbutler_api_url_for
    Reverse URL lookup for WaterButler routes
    :param str route: The action to preform, upload, download, delete...
    :param str provider: The name of the requested provider
    :param str path: The path of the requested file or folder
    :param Node node: The node being accessed
    :param User user: The user whos cookie will be used or None
    :param dict kwargs: Addition query parameters to be appended
    """
    url = furl.furl(website_settings.WATERBUTLER_URL)
    url.path.segments.append(waterbutler_action_map[route])

    url.args.update({
        'path': path,
        'nid': node._id,
        'provider': provider,
    })

    if user:
        url.args['cookie'] = user.get_or_create_cookie()
    elif website_settings.COOKIE_NAME in request.cookies:
        url.args['cookie'] = request.cookies[website_settings.COOKIE_NAME]

    view_only = False
    if 'view_only' in kwargs:
        view_only = kwargs.get('view_only')
    else:
        view_only = request.args.get('view_only')

    url.args['view_only'] = view_only

    url.args.update(kwargs)
    return url.url

def waterbutler_api_url_for(node_id, provider, path='/', **kwargs):
    assert path.startswith('/'), 'Path must always start with /'
    url = furl.furl(website_settings.WATERBUTLER_URL)
    segments = ['v1', 'resources', node_id, 'providers', provider] + path.split('/')[1:]
    url.path.segments.extend([urllib.quote(x.encode('utf-8')) for x in segments])
    url.args.update(kwargs)
    return url.url

@contextmanager
def disconnected_from(signal, listener):
    """Temporarily disconnect a Blinker signal."""
    signal.disconnect(listener)
    yield
    signal.connect(listener)

def check_private_key_for_anonymized_link(private_key):
    from website.project.model import PrivateLink

    is_anonymous = False
    if private_key is not None:
        try:
            link = PrivateLink.find_one(Q('key', 'eq', private_key))
        except NoResultsFound:
            link = None
        if link is not None:
            is_anonymous = link.anonymous
    return is_anonymous

def get_headers_from_request(req):
    """ Get and normalize DRF and Flask request headers
    """
    headers = getattr(req, 'META', {})
    if headers:
        headers = {
            '-'.join([part.capitalize() for part in k.split('_')]).replace('Http-', ''): v
            for k, v in headers.items()
        }
        remote_addr = (headers.get('X-Forwarded-For') or headers.get('Remote-Addr'))
        headers['Remote-Addr'] = remote_addr.split(',')[0].strip() if remote_addr else None
    else:
        headers = getattr(req, 'headers', {})
        headers = {
            k: v
            for k, v in headers.items()
        }
        headers['Remote-Addr'] = req.remote_addr
    return headers
<|endoftext|>


<commit_before>from celery import task
from celery.execute import send_task

from rockit.plugins.razberry import actions
from rockit.plugins.razberry import executors
from rockit.plugins.razberry import models
from rockit.plugins.razberry import resolvers
from rockit.plugins.razberry import services

@task(name='razberry.node.commands')
def node_commands(identifier, holder):
    instances = services.RazberryService().retrieve_instances(identifier)

    if instances:
        for key, instance in instances.items():
            builder = actions.ActionBuilder(instance['commandClasses'])
            builder.filter_actions_by_command_classes(identifier, holder)

    return holder

@task(name='razberry.node.command.value')
def node_command_value(identifier, command_id):
    service = services.RazberryService()
    command = service.retrieve(command_id)

    if command:
        data = resolvers.ValueResolver().resolve_command_value(command)

        if data:
            return service.retrieve("%s.%s" % (command_id, data))

    return None

@task(name='razberry.node.command.update.value')
def node_command_update_value(identifier, command_id, value):
    service = services.RazberryService()
    command = service.retrieve(command_id)

    if command:
        updateTime = command['data']['updateTime']
        updateValue = value

        if updateValue == '$toggle' and command['data']['level']['type'] == 'bool':
            updateValue = str(not bool(command['data']['level']['value']))

        print updateValue

        service.update(command_id, updateValue)

        data = service.data(updateTime)

        return node_command_value(identifier, command_id)

    return None

@task(name='razberry.node.detailed')
def node_detailed(identifier, holder):
    node = models.Node.objects.get(device_id=identifier)
    version = models.NodeVersion.objects.get(node=node)

    holder.add(**to_kwargs('device_type', node.device_type))
    holder.add(**to_kwargs('manufacturer_name', node.manufacturer_name))
    holder.add(**to_kwargs('manufacturer_id', node.manufacturer_id))

    holder.add(**to_kwargs('isListening', node.listening))
    holder.add(**to_kwargs('routing', node.routing))
    holder.add(**to_kwargs('beaming', node.beaming))
    holder.add(**to_kwargs('awaken', node.awaken))
    holder.add(**to_kwargs('failed', node.failed))

    holder.add(**to_kwargs('sdk', version.sdk))
    holder.add(**to_kwargs('application', version.application))
    holder.add(**to_kwargs('zw_library', version.zw_library))
    holder.add(**to_kwargs('zw_protocol', version.zw_protocol))

    return holder

@task(name='razberry.settings')
def settings(holder):

    for setting in models.Setting.objects.all():
        holder.add(**{
            'key': setting.id,
            'name': setting.name,
            'value': setting.value,
            'readonly': setting.readonly
            })
    return holder

@task(name='razberry.mixes')
def mixes(holder):
    return executors.MixesExecutor().collect(holder)

@task(name='razberry.mixes.then.create')
def mixes_then_create(uuid, criterias):

    if criterias:

        for key, criteria in criterias.iteritems():

            node = models.Node.objects.get(uuid=uuid)
            then = models.ActionThen.objects.create(target=node, command=criteria['id'], value=criteria['value'])

            return then.id
    return None

@task(name='razberry.mixes.then.destroy')
def mixes_then_destroy(identifier):
    logger = mixes_then_destroy.get_logger()

    try:
        logger.debug("Trying to delete instance %s" % identifier)

        instance = models.ActionThen.objects.get(id=identifier)
        instance.delete()

    except models.ActionThen.DoesNotExist:
        logger.warn("Instance does not exist anymore %s" % identifier)

    return True

@task(name='razberry.mixes.then.run')
def mixes_then_run(identifier):
    logger = mixes_then_create.get_logger()

    try:
        logger.debug("Trying to run action %s" % instance)

        instance = models.ActionThen.objects.get(id=identifier)

        logger.info("Run command %s with value %s" % (then.command, then.value))

        node_command_update_value(then.target.device_id, then.command, then.value)

    except models.ActionThen.DoesNotExist:
        logger.warn("Instance does not exist %s" % identifier)

    return True

@task(name='razberry.mixes.then.validate')
def mixes_then_validate(identifier, criterias, holder):
    return executors.MixesExecutor().validate(criterias, holder)

@task(name='razberry.mixes.details')
def mixes_details(identifier, holder):
    return executors.MixesExecutor().collect_details(identifier, holder)

def to_kwargs(title, value):
    return {
        'title': title,
        'value': value
    }<commit_msg>Fixed task<commit_after>from celery import task
from celery.execute import send_task

from rockit.plugins.razberry import actions
from rockit.plugins.razberry import executors
from rockit.plugins.razberry import models
from rockit.plugins.razberry import resolvers
from rockit.plugins.razberry import services

@task(name='razberry.node.commands')
def node_commands(identifier, holder):
    instances = services.RazberryService().retrieve_instances(identifier)

    if instances:
        for key, instance in instances.items():
            builder = actions.ActionBuilder(instance['commandClasses'])
            builder.filter_actions_by_command_classes(identifier, holder)

    return holder

@task(name='razberry.node.command.value')
def node_command_value(identifier, command_id):
    service = services.RazberryService()
    command = service.retrieve(command_id)

    if command:
        data = resolvers.ValueResolver().resolve_command_value(command)

        if data:
            return service.retrieve("%s.%s" % (command_id, data))

    return None

@task(name='razberry.node.command.update.value')
def node_command_update_value(identifier, command_id, value):
    service = services.RazberryService()
    command = service.retrieve(command_id)

    if command:
        updateTime = command['data']['updateTime']
        updateValue = value

        if updateValue == '$toggle' and command['data']['level']['type'] == 'bool':
            updateValue = str(not bool(command['data']['level']['value']))

        print updateValue

        service.update(command_id, updateValue)

        data = service.data(updateTime)

        return node_command_value(identifier, command_id)

    return None

@task(name='razberry.node.detailed')
def node_detailed(identifier, holder):
    node = models.Node.objects.get(device_id=identifier)
    version = models.NodeVersion.objects.get(node=node)

    holder.add(**to_kwargs('device_type', node.device_type))
    holder.add(**to_kwargs('manufacturer_name', node.manufacturer_name))
    holder.add(**to_kwargs('manufacturer_id', node.manufacturer_id))

    holder.add(**to_kwargs('isListening', node.listening))
    holder.add(**to_kwargs('routing', node.routing))
    holder.add(**to_kwargs('beaming', node.beaming))
    holder.add(**to_kwargs('awaken', node.awaken))
    holder.add(**to_kwargs('failed', node.failed))

    holder.add(**to_kwargs('sdk', version.sdk))
    holder.add(**to_kwargs('application', version.application))
    holder.add(**to_kwargs('zw_library', version.zw_library))
    holder.add(**to_kwargs('zw_protocol', version.zw_protocol))

    return holder

@task(name='razberry.settings')
def settings(holder):

    for setting in models.Setting.objects.all():
        holder.add(**{
            'key': setting.id,
            'name': setting.name,
            'value': setting.value,
            'readonly': setting.readonly
            })
    return holder

@task(name='razberry.mixes')
def mixes(holder):
    return executors.MixesExecutor().collect(holder)

@task(name='razberry.mixes.then.create')
def mixes_then_create(uuid, criterias):

    if criterias:

        for key, criteria in criterias.iteritems():

            node = models.Node.objects.get(uuid=uuid)
            then = models.ActionThen.objects.create(target=node, command=criteria['id'], value=criteria['value'])

            return then.id
    return None

@task(name='razberry.mixes.then.destroy')
def mixes_then_destroy(identifier):
    logger = mixes_then_destroy.get_logger()

    try:
        logger.debug("Trying to delete instance %s" % identifier)

        instance = models.ActionThen.objects.get(id=identifier)
        instance.delete()

    except models.ActionThen.DoesNotExist:
        logger.warn("Instance does not exist anymore %s" % identifier)

    return True

@task(name='razberry.mixes.then.run')
def mixes_then_run(identifier):
    logger = mixes_then_create.get_logger()

    try:
        logger.debug("Trying to run action %s" % identifier)

        instance = models.ActionThen.objects.get(id=identifier)

        logger.info("Run command %s with value %s" % (then.command, then.value))

        node_command_update_value(then.target.device_id, then.command, then.value)

    except models.ActionThen.DoesNotExist:
        logger.warn("Instance does not exist %s" % identifier)

    return True

@task(name='razberry.mixes.then.validate')
def mixes_then_validate(identifier, criterias, holder):
    return executors.MixesExecutor().validate(criterias, holder)

@task(name='razberry.mixes.details')
def mixes_details(identifier, holder):
    return executors.MixesExecutor().collect_details(identifier, holder)

def to_kwargs(title, value):
    return {
        'title': title,
        'value': value
    }<|endoftext|>


<commit_before>import contextlib
import os
import re
from datetime import datetime
from pathlib import Path
from typing import List, Tuple, Dict, Optional, Sequence

import pywikibot
from git import Repo
from github import Github

from scripts.service.ws_re.register.author_crawler import AuthorCrawler
from scripts.service.ws_re.register.base import RegisterException
from scripts.service.ws_re.register.lemma import LemmaDict, ChapterDict
from scripts.service.ws_re.register.registers import Registers
from scripts.service.ws_re.register.updater import Updater, RemoveList
from scripts.service.ws_re.scanner.tasks.base_task import ReScannerTask
from scripts.service.ws_re.template.article import Article
from tools.bots import WikiLogger

class SCANTask(ReScannerTask):
    def __init__(self, wiki: pywikibot.Site, logger: WikiLogger, debug: bool = True):
        super().__init__(wiki, logger, debug)
        self.registers = Registers()
        self._strategies: Dict[str, List[str]] = {}

    def task(self):
        self._process_from_article_list()

    def finish_task(self):
        super().finish_task()
        for strategy in self._strategies:
            self.logger.info(f"STRATEGY_{strategy}: {len(self._strategies[strategy])}")
            if strategy != "update_lemma_by_name":
                self.logger.info(f"{self._strategies[strategy]}")
        self.logger.info("Fetch changes for the authors.")
        authors = self.registers.authors
        authors.set_mappings(AuthorCrawler.get_author_mapping(self.wiki))
        authors.set_author(AuthorCrawler.process_author_infos(self.wiki))
        self.logger.info("Persist the author data.")
        authors.persist()
        self.logger.info("Persist the register data.")
        self.registers.persist()
        self._push_changes()

    def _fetch_wp_link(self, article_list: List[Article]) -> Tuple[LemmaDict, RemoveList]:
        article = article_list[0]
        if article['WIKIPEDIA'].value:
            wp_link: Optional[str] = f"w:de:{article['WIKIPEDIA'].value}"
        else:
            wp_link = self._get_link_from_wd(("dewiki", "enwiki", "frwiki", "itwiki", "arwiki", "elwiki", "lawiki"))
        if wp_link:
            return {"wp_link": wp_link}, []
        return {}, ["wp_link"]

    def _fetch_ws_link(self, article_list: List[Article]) -> Tuple[LemmaDict, RemoveList]:
        article = article_list[0]
        if article['WIKISOURCE'].value:
            ws_link: Optional[str] = f"s:de:{article['WIKISOURCE'].value}"
        else:
            ws_link = self._get_link_from_wd(("dewikisource", "enwikisource", "frwikisource", "itwikisource",
                                              "arwikisource", "elwikisource", "lawikisource"))
        if ws_link:
            return {"ws_link": ws_link}, []
        return {}, ["ws_link"]

    def _get_link_from_wd(self, possible_source_wikis: Sequence) -> Optional[str]:
        with contextlib.suppress(pywikibot.exceptions.NoPage):
            wp_item = self.re_page.page.data_item()
            with contextlib.suppress(KeyError):
                target = wp_item.claims["P921"][0].target
                for sitelink in possible_source_wikis:
                    with contextlib.suppress(pywikibot.exceptions.NoPage):
                        wiki_prefix = "s" if sitelink.find("wikisource") > 0 else "w"
                        link = f"{wiki_prefix}:{sitelink[0:2]}:{target.getSitelink(sitelink)}"
                        self.logger.info(f"Found link {link}")
                        return link
        return None

    @staticmethod
    def _fetch_sort_key(article_list: List[Article]) -> Tuple[LemmaDict, RemoveList]:
        article = article_list[0]
        sort_key = str(article["SORTIERUNG"].value)
        if sort_key:
            return {"sort_key": sort_key}, []
        return {}, ["sort_key"]

    def _fetch_lemma(self, _) -> Tuple[LemmaDict, RemoveList]:  # pylint: disable=unused-argument
        return {"lemma": self.re_page.lemma_without_prefix}, []

    _REGEX_REDIRECT = re.compile(r"[sS]\..*?(?:\[\[RE:|\{\{RE siehe\|)([^\|\}]+)")

    def _fetch_redirect(self, article_list: List[Article]) -> Tuple[LemmaDict, RemoveList]:
        article = article_list[0]
        redirect = article["VERWEIS"].value
        if redirect:
            match = self._REGEX_REDIRECT.search(article.text)
            if match:
                redirect = match.group(1)
            return {"redirect": redirect}, []
        return {}, ["redirect"]

    @staticmethod
    def _fetch_previous(article_list: List[Article]) -> Tuple[LemmaDict, RemoveList]:
        article = article_list[0]
        previous = str(article["VORGÄNGER"].value)
        if previous and previous != "OFF":
            return {"previous": previous}, []
        return {}, ["previous"]

    @staticmethod
    def _fetch_next(article_list: List[Article]) -> Tuple[LemmaDict, RemoveList]:
        article = article_list[0]
        next_lemma = str(article["NACHFOLGER"].value)
        if next_lemma and next_lemma != "OFF":
            return {"next": next_lemma}, []
        return {}, ["next"]

    def _fetch_pages(self, article_list: List[Article]) -> Tuple[LemmaDict, RemoveList]:
        # if there is something outside an article ignore it
        article_list = [article for article in article_list if isinstance(article, Article)]
        if self.re_page.complex_construction:
            self.logger.error(f"The construct of {self.re_page.lemma_without_prefix} is too complex, can't analyse.")
            return {}, []
        if len(article_list) == 1:
            chapter_dict = self._analyse_simple_article_list(article_list)
            if chapter_dict:
                return {"chapters": [chapter_dict]}, []
            return {}, []
        return {"chapters": self._analyse_complex_article_list(article_list)}, []

    def _analyse_simple_article_list(self, article_list: List[Article]) -> ChapterDict:
        article = article_list[0]
        try:
            spalte_start = int(article["SPALTE_START"].value)
        except ValueError:
            self.logger.error(f"{self.re_page.lemma_without_prefix} has no correct start column.")
            return {}
        spalte_end_raw = article["SPALTE_END"].value
        if spalte_end_raw and spalte_end_raw != "OFF":
            spalte_end = int(spalte_end_raw)
        else:
            spalte_end = spalte_start
        single_article_dict = self._create_chapter_dict(article, spalte_end, spalte_start)
        return single_article_dict

    def _analyse_complex_article_list(self, article_list: List[Article]) -> List[ChapterDict]:
        simple_dict = self._analyse_simple_article_list(article_list)
        article_start = int(simple_dict["start"])
        chapter_list: List[ChapterDict] = []
        for article in article_list:
            # if there will be no findings of the regex, the article continues on the next page as the predecessor
            spalte_start: int = article_start
            spalte_end: int = article_start
            findings = list(re.finditer(r"\{\{Seite\|(\d{1,4})", article.text))
            if findings:
                first_finding = findings[0]
                spalte_start = int(first_finding.group(1))
                if first_finding.start(0) > 0:
                    spalte_start -= 1
                spalte_end = int(findings[-1].group(1))
            if article is article_list[-1]:
                spalte_end = int(simple_dict["end"])
            single_article_dict = self._create_chapter_dict(article, spalte_end, spalte_start)
            chapter_list.append(single_article_dict)
            article_start = spalte_end
        return chapter_list

    @staticmethod
    def _create_chapter_dict(article: Article, spalte_end: int, spalte_start: int) -> ChapterDict:
        single_article_dict: ChapterDict = {"start": spalte_start, "end": spalte_end}
        author = article.author[0]
        if author.lower().strip() != "off":
            single_article_dict["author"] = author
        return single_article_dict

    @staticmethod
    def _fetch_proof_read(article_list: List[Article]) -> Tuple[LemmaDict, RemoveList]:
        article = article_list[0]
        proof_read = str(article["KORREKTURSTAND"].value).lower().strip()
        if article.common_free:
            if proof_read:
                if proof_read == "fertig":
                    return {"proof_read": 3}, []
                if proof_read == "korrigiert":
                    return {"proof_read": 2}, []
        return {}, ["proof_read"]

    def _process_from_article_list(self):
        function_list_properties = []
        for item in dir(self):
            if "_fetch" in item:
                function_list_properties.append(getattr(self, item))
        issues_in_articles = set()
        for article_list in self.re_page.splitted_article_list:
            # fetch from properties
            if isinstance(article_list[0], Article):
                update_dict = {}
                delete_list = []
                for fetch_function in function_list_properties:
                    function_dict, function_list = fetch_function(article_list)
                    update_dict.update(function_dict)
                    delete_list += function_list
                band_info = article_list[0]["BAND"].value
                self_supplement = band_info in issues_in_articles
                issues_in_articles.add(band_info)
                self._update_lemma(band_info, delete_list, self_supplement, update_dict)

    def _update_lemma(self, band_info: str, delete_list: RemoveList, self_supplement: bool, update_dict: LemmaDict):
        register = self.registers.volumes[band_info]
        if register:
            try:
                with Updater(register) as updater:
                    strategy = updater.update_lemma(update_dict, delete_list, self_supplement)
                self._write_strategy_statistic(strategy, update_dict, band_info)
            except RegisterException as error:
                self.logger.error(f"No available Lemma in Registers for issue {band_info} "
                                  f"and lemma {self.re_page.lemma_as_link}. "
                                  f"Reason is: {error.args[0]}")

    def _write_strategy_statistic(self, strategy: str, update_dict: LemmaDict, issue_no: str):
        entry = f"{update_dict['lemma']}/{issue_no}"
        if strategy in self._strategies:
            self._strategies[strategy].append(entry)
        else:
            self._strategies[strategy] = [entry]

    def _push_changes(self):
        repo = Repo(search_parent_directories=True)
        if repo.index.diff(None):
            master = repo.active_branch
            now = datetime.now().strftime("%y%m%d_%H%M%S")
            branch_name = f"{now}_updating_registers"
            self.logger.info(f"Pushing changes to \"{branch_name}\"")
            repo.git.checkout("-b", branch_name)
            repo.git.add(str(Path(__file__).parent.parent.parent.joinpath("register").joinpath("data")))
            repo.index.commit(f"Updating the register at {now}")
            repo.git.push("origin", repo.active_branch.name)
            repo.git.checkout(master.name)
            if "GITHUB_USER" in os.environ:  # pragma: no cover
                self.logger.info(f"Opening Pullrequest for \"{branch_name}\"")
                self._open_pullrequest(branch_name)
            else:
                self.logger.error("No env variable GITHUB_USER")
        else:
            self.logger.info("No Changes to push today.")

    @staticmethod
    def _open_pullrequest(branch_name: str):  # pragma: no cover
        github = Github(os.environ["GITHUB_USER"], os.environ["GITHUB_PASSWORD"])
        github_repo = github.get_repo("the-it/WS_THEbotIT")
        github_repo.create_pull(title=branch_name,
                                head=branch_name,
                                base="master",
                                body="Update registers",
                                maintainer_can_modify=True)
<commit_msg>change language links<commit_after>import contextlib
import os
import re
from datetime import datetime
from pathlib import Path
from typing import List, Tuple, Dict, Optional, Sequence

import pywikibot
from git import Repo
from github import Github

from scripts.service.ws_re.register.author_crawler import AuthorCrawler
from scripts.service.ws_re.register.base import RegisterException
from scripts.service.ws_re.register.lemma import LemmaDict, ChapterDict
from scripts.service.ws_re.register.registers import Registers
from scripts.service.ws_re.register.updater import Updater, RemoveList
from scripts.service.ws_re.scanner.tasks.base_task import ReScannerTask
from scripts.service.ws_re.template.article import Article
from tools.bots import WikiLogger

class SCANTask(ReScannerTask):
    def __init__(self, wiki: pywikibot.Site, logger: WikiLogger, debug: bool = True):
        super().__init__(wiki, logger, debug)
        self.registers = Registers()
        self._strategies: Dict[str, List[str]] = {}

    def task(self):
        self._process_from_article_list()

    def finish_task(self):
        super().finish_task()
        for strategy in self._strategies:
            self.logger.info(f"STRATEGY_{strategy}: {len(self._strategies[strategy])}")
            if strategy != "update_lemma_by_name":
                self.logger.info(f"{self._strategies[strategy]}")
        self.logger.info("Fetch changes for the authors.")
        authors = self.registers.authors
        authors.set_mappings(AuthorCrawler.get_author_mapping(self.wiki))
        authors.set_author(AuthorCrawler.process_author_infos(self.wiki))
        self.logger.info("Persist the author data.")
        authors.persist()
        self.logger.info("Persist the register data.")
        self.registers.persist()
        self._push_changes()

    def _fetch_wp_link(self, article_list: List[Article]) -> Tuple[LemmaDict, RemoveList]:
        article = article_list[0]
        if article['WIKIPEDIA'].value:
            wp_link: Optional[str] = f"w:de:{article['WIKIPEDIA'].value}"
        else:
            wp_link = self._get_link_from_wd(("dewiki", "enwiki", "frwiki", "itwiki", "eswiki", "ptwiki", "sewiki",
                                              "cawiki", "lawiki", "arwiki", "trwiki", "elwiki"))
        if wp_link:
            return {"wp_link": wp_link}, []
        return {}, ["wp_link"]

    def _fetch_ws_link(self, article_list: List[Article]) -> Tuple[LemmaDict, RemoveList]:
        article = article_list[0]
        if article['WIKISOURCE'].value:
            ws_link: Optional[str] = f"s:de:{article['WIKISOURCE'].value}"
        else:
            ws_link = self._get_link_from_wd(("dewikisource", "enwikisource", "frwikisource", "itwikisource",
                                              "eswikisource", "ptwikisource", "sewikisource", "cawikisource",
                                              "lawikisource", "arwikisource", "trwikisource", "elwikisource"))
        if ws_link:
            return {"ws_link": ws_link}, []
        return {}, ["ws_link"]

    def _get_link_from_wd(self, possible_source_wikis: Sequence) -> Optional[str]:
        with contextlib.suppress(pywikibot.exceptions.NoPage):
            wp_item = self.re_page.page.data_item()
            with contextlib.suppress(KeyError):
                target = wp_item.claims["P921"][0].target
                for sitelink in possible_source_wikis:
                    with contextlib.suppress(pywikibot.exceptions.NoPage):
                        wiki_prefix = "s" if sitelink.find("wikisource") > 0 else "w"
                        link = f"{wiki_prefix}:{sitelink[0:2]}:{target.getSitelink(sitelink)}"
                        self.logger.info(f"Found link {link}")
                        return link
        return None

    @staticmethod
    def _fetch_sort_key(article_list: List[Article]) -> Tuple[LemmaDict, RemoveList]:
        article = article_list[0]
        sort_key = str(article["SORTIERUNG"].value)
        if sort_key:
            return {"sort_key": sort_key}, []
        return {}, ["sort_key"]

    def _fetch_lemma(self, _) -> Tuple[LemmaDict, RemoveList]:  # pylint: disable=unused-argument
        return {"lemma": self.re_page.lemma_without_prefix}, []

    _REGEX_REDIRECT = re.compile(r"[sS]\..*?(?:\[\[RE:|\{\{RE siehe\|)([^\|\}]+)")

    def _fetch_redirect(self, article_list: List[Article]) -> Tuple[LemmaDict, RemoveList]:
        article = article_list[0]
        redirect = article["VERWEIS"].value
        if redirect:
            match = self._REGEX_REDIRECT.search(article.text)
            if match:
                redirect = match.group(1)
            return {"redirect": redirect}, []
        return {}, ["redirect"]

    @staticmethod
    def _fetch_previous(article_list: List[Article]) -> Tuple[LemmaDict, RemoveList]:
        article = article_list[0]
        previous = str(article["VORGÄNGER"].value)
        if previous and previous != "OFF":
            return {"previous": previous}, []
        return {}, ["previous"]

    @staticmethod
    def _fetch_next(article_list: List[Article]) -> Tuple[LemmaDict, RemoveList]:
        article = article_list[0]
        next_lemma = str(article["NACHFOLGER"].value)
        if next_lemma and next_lemma != "OFF":
            return {"next": next_lemma}, []
        return {}, ["next"]

    def _fetch_pages(self, article_list: List[Article]) -> Tuple[LemmaDict, RemoveList]:
        # if there is something outside an article ignore it
        article_list = [article for article in article_list if isinstance(article, Article)]
        if self.re_page.complex_construction:
            self.logger.error(f"The construct of {self.re_page.lemma_without_prefix} is too complex, can't analyse.")
            return {}, []
        if len(article_list) == 1:
            chapter_dict = self._analyse_simple_article_list(article_list)
            if chapter_dict:
                return {"chapters": [chapter_dict]}, []
            return {}, []
        return {"chapters": self._analyse_complex_article_list(article_list)}, []

    def _analyse_simple_article_list(self, article_list: List[Article]) -> ChapterDict:
        article = article_list[0]
        try:
            spalte_start = int(article["SPALTE_START"].value)
        except ValueError:
            self.logger.error(f"{self.re_page.lemma_without_prefix} has no correct start column.")
            return {}
        spalte_end_raw = article["SPALTE_END"].value
        if spalte_end_raw and spalte_end_raw != "OFF":
            spalte_end = int(spalte_end_raw)
        else:
            spalte_end = spalte_start
        single_article_dict = self._create_chapter_dict(article, spalte_end, spalte_start)
        return single_article_dict

    def _analyse_complex_article_list(self, article_list: List[Article]) -> List[ChapterDict]:
        simple_dict = self._analyse_simple_article_list(article_list)
        article_start = int(simple_dict["start"])
        chapter_list: List[ChapterDict] = []
        for article in article_list:
            # if there will be no findings of the regex, the article continues on the next page as the predecessor
            spalte_start: int = article_start
            spalte_end: int = article_start
            findings = list(re.finditer(r"\{\{Seite\|(\d{1,4})", article.text))
            if findings:
                first_finding = findings[0]
                spalte_start = int(first_finding.group(1))
                if first_finding.start(0) > 0:
                    spalte_start -= 1
                spalte_end = int(findings[-1].group(1))
            if article is article_list[-1]:
                spalte_end = int(simple_dict["end"])
            single_article_dict = self._create_chapter_dict(article, spalte_end, spalte_start)
            chapter_list.append(single_article_dict)
            article_start = spalte_end
        return chapter_list

    @staticmethod
    def _create_chapter_dict(article: Article, spalte_end: int, spalte_start: int) -> ChapterDict:
        single_article_dict: ChapterDict = {"start": spalte_start, "end": spalte_end}
        author = article.author[0]
        if author.lower().strip() != "off":
            single_article_dict["author"] = author
        return single_article_dict

    @staticmethod
    def _fetch_proof_read(article_list: List[Article]) -> Tuple[LemmaDict, RemoveList]:
        article = article_list[0]
        proof_read = str(article["KORREKTURSTAND"].value).lower().strip()
        if article.common_free:
            if proof_read:
                if proof_read == "fertig":
                    return {"proof_read": 3}, []
                if proof_read == "korrigiert":
                    return {"proof_read": 2}, []
        return {}, ["proof_read"]

    def _process_from_article_list(self):
        function_list_properties = []
        for item in dir(self):
            if "_fetch" in item:
                function_list_properties.append(getattr(self, item))
        issues_in_articles = set()
        for article_list in self.re_page.splitted_article_list:
            # fetch from properties
            if isinstance(article_list[0], Article):
                update_dict = {}
                delete_list = []
                for fetch_function in function_list_properties:
                    function_dict, function_list = fetch_function(article_list)
                    update_dict.update(function_dict)
                    delete_list += function_list
                band_info = article_list[0]["BAND"].value
                self_supplement = band_info in issues_in_articles
                issues_in_articles.add(band_info)
                self._update_lemma(band_info, delete_list, self_supplement, update_dict)

    def _update_lemma(self, band_info: str, delete_list: RemoveList, self_supplement: bool, update_dict: LemmaDict):
        register = self.registers.volumes[band_info]
        if register:
            try:
                with Updater(register) as updater:
                    strategy = updater.update_lemma(update_dict, delete_list, self_supplement)
                self._write_strategy_statistic(strategy, update_dict, band_info)
            except RegisterException as error:
                self.logger.error(f"No available Lemma in Registers for issue {band_info} "
                                  f"and lemma {self.re_page.lemma_as_link}. "
                                  f"Reason is: {error.args[0]}")

    def _write_strategy_statistic(self, strategy: str, update_dict: LemmaDict, issue_no: str):
        entry = f"{update_dict['lemma']}/{issue_no}"
        if strategy in self._strategies:
            self._strategies[strategy].append(entry)
        else:
            self._strategies[strategy] = [entry]

    def _push_changes(self):
        repo = Repo(search_parent_directories=True)
        if repo.index.diff(None):
            master = repo.active_branch
            now = datetime.now().strftime("%y%m%d_%H%M%S")
            branch_name = f"{now}_updating_registers"
            self.logger.info(f"Pushing changes to \"{branch_name}\"")
            repo.git.checkout("-b", branch_name)
            repo.git.add(str(Path(__file__).parent.parent.parent.joinpath("register").joinpath("data")))
            repo.index.commit(f"Updating the register at {now}")
            repo.git.push("origin", repo.active_branch.name)
            repo.git.checkout(master.name)
            if "GITHUB_USER" in os.environ:  # pragma: no cover
                self.logger.info(f"Opening Pullrequest for \"{branch_name}\"")
                self._open_pullrequest(branch_name)
            else:
                self.logger.error("No env variable GITHUB_USER")
        else:
            self.logger.info("No Changes to push today.")

    @staticmethod
    def _open_pullrequest(branch_name: str):  # pragma: no cover
        github = Github(os.environ["GITHUB_USER"], os.environ["GITHUB_PASSWORD"])
        github_repo = github.get_repo("the-it/WS_THEbotIT")
        github_repo.create_pull(title=branch_name,
                                head=branch_name,
                                base="master",
                                body="Update registers",
                                maintainer_can_modify=True)
<|endoftext|>


<commit_before># Copyright 2011-2016 Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

version = "1.2.2"
<commit_msg>version: bump to 1.2.3<commit_after># Copyright 2011-2016 Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

version = "1.2.3"
<|endoftext|>


<commit_before>from context import nikola
import os
import unittest
import mock

class CommandImportWordpressTest(unittest.TestCase):
    def setUp(self):
        self.import_command = nikola.plugins.command_import_wordpress.CommandImportWordpress()
        self.import_filename = os.path.abspath(
            os.path.join(os.path.dirname(__file__),
                         'wordpress_export_example.xml'))

    def tearDown(self):
        del self.import_command
        del self.import_filename

    def test_create_import_work_without_argument(self):
        # Running this without an argument must not fail.
        # It should show the proper usage of the command.
        self.import_command.run()

    def test_create_import(self):
        data_import = mock.MagicMock()
        site_generation = mock.MagicMock()
        write_urlmap = mock.MagicMock()
        write_configuration = mock.MagicMock()

        with mock.patch('nikola.plugins.command_import_wordpress.CommandImportWordpress.generate_base_site', site_generation):
            with mock.patch('nikola.plugins.command_import_wordpress.CommandImportWordpress.import_posts', data_import):
                with mock.patch('nikola.plugins.command_import_wordpress.CommandImportWordpress.write_urlmap_csv', write_urlmap):
                    with mock.patch('nikola.plugins.command_import_wordpress.CommandImportWordpress.write_configuration', write_configuration):
                        self.import_command.run(self.import_filename)

        self.assertTrue(site_generation.called)
        self.assertTrue(data_import.called)

    def test_populate_context(self):
        channel = self.import_command.get_channel_from_file(
            self.import_filename)
        context = self.import_command.populate_context(channel)

        for required_key in ('POST_PAGES', 'POST_COMPILERS'):
            self.assertTrue(required_key in context)

        self.assertEqual('de', context['DEFAULT_LANG'])
        self.assertEqual('Wordpress blog title', context['BLOG_TITLE'])
        self.assertEqual('Nikola test blog ;)', context['BLOG_DESCRIPTION'])
        self.assertEqual('http://some.blog', context['BLOG_URL'])
        self.assertEqual('mail@some.blog', context['BLOG_EMAIL'])
        self.assertEqual('Niko', context['BLOG_AUTHOR'])

    def test_importing_posts_and_attachments(self):
        channel = self.import_command.get_channel_from_file(
            self.import_filename)
        self.import_command.context = self.import_command.populate_context(channel)
        self.import_command.url_map = {} # For testing we use an empty one.

        write_metadata = mock.MagicMock()
        write_content = mock.MagicMock()
        download_mock = mock.MagicMock()

        with mock.patch('nikola.plugins.command_import_wordpress.write_content', write_content):
            with mock.patch('nikola.plugins.command_import_wordpress.write_metadata', write_metadata):
                with mock.patch('nikola.plugins.command_import_wordpress.download_url_content_to_file', download_mock):
                    with mock.patch('nikola.plugins.command_import_wordpress.os.makedirs'):
                        self.import_command.import_posts(channel)

        self.assertTrue(download_mock.called)
        download_mock.assert_any_call(u'http://some.blog/wp-content/uploads/2008/07/arzt_und_pfusch-sick-cover.png', u'new_site/files/wp-content/uploads/2008/07/arzt_und_pfusch-sick-cover.png')

        self.assertTrue(write_metadata.called)
        write_metadata.assert_any_call(u'new_site/stories/kontakt.meta', 'Kontakt', u'kontakt', '2009-07-16 20:20:32', None, [])

        self.assertTrue(write_content.called)
        write_content.assert_any_call(u'new_site/posts/200704hoert.wp', '...!\n\n\n\n[caption id="attachment_16" align="alignnone" width="739" caption="caption test"][/caption]\n\n\n\nNicht, dass daran jemals Zweifel bestanden.')
        write_content.assert_any_call(u'new_site/posts/200807arzt-und-pfusch-s-i-c-k.wp', u'Arzt+Pfusch - S.I.C.K.Gerade bin ich \xfcber das Album S.I.C.K von Arzt+Pfusch gestolpert, welches Arzt+Pfusch zum Download f\xfcr lau anbieten. Das Album steht unter einer Creative Commons BY-NC-ND-Lizenz.\n\nDie Ladung noisebmstupidevildustrial gibts als MP3s mit 64kbps und VBR, als Ogg Vorbis und als FLAC (letztere hier). Artwork und Lyrics gibts nochmal einzeln zum Download.')
        write_content.assert_any_call(u'new_site/stories/kontakt.wp', u'
Datenschutz
\n\nIch erhebe und speichere automatisch in meine Server Log Files Informationen, die dein Browser an mich \xfcbermittelt. Dies sind:\n\n
\n\n
Browsertyp und -version
\n\n
verwendetes Betriebssystem
\n\n
Referrer URL (die zuvor besuchte Seite)
\n\n
IP Adresse des zugreifenden Rechners
\n\n
Uhrzeit der Serveranfrage.
\n\n
\n\nDiese Daten sind f\xfcr mich nicht bestimmten Personen zuordenbar. Eine Zusammenf\xfchrung dieser Daten mit anderen Datenquellen wird nicht vorgenommen, die Daten werden einzig zu statistischen Zwecken erhoben.')

        self.assertTrue(len(self.import_command.url_map) > 0)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Adjusted tests.<commit_after>from context import nikola
import os
import unittest
import mock

class CommandImportWordpressTest(unittest.TestCase):
    def setUp(self):
        self.import_command = nikola.plugins.command_import_wordpress.CommandImportWordpress()
        self.import_filename = os.path.abspath(
            os.path.join(os.path.dirname(__file__),
                         'wordpress_export_example.xml'))

    def tearDown(self):
        del self.import_command
        del self.import_filename

    def test_create_import_work_without_argument(self):
        # Running this without an argument must not fail.
        # It should show the proper usage of the command.
        self.import_command.run()

    def test_create_import(self):
        data_import = mock.MagicMock()
        site_generation = mock.MagicMock()
        write_urlmap = mock.MagicMock()
        write_configuration = mock.MagicMock()

        with mock.patch('nikola.plugins.command_import_wordpress.CommandImportWordpress.generate_base_site', site_generation):
            with mock.patch('nikola.plugins.command_import_wordpress.CommandImportWordpress.import_posts', data_import):
                with mock.patch('nikola.plugins.command_import_wordpress.CommandImportWordpress.write_urlmap_csv', write_urlmap):
                    with mock.patch('nikola.plugins.command_import_wordpress.CommandImportWordpress.write_configuration', write_configuration):
                        self.import_command.run(self.import_filename)

        self.assertTrue(site_generation.called)
        self.assertTrue(data_import.called)

    def test_populate_context(self):
        channel = self.import_command.get_channel_from_file(
            self.import_filename)
        context = self.import_command.populate_context(channel)

        for required_key in ('POST_PAGES', 'POST_COMPILERS'):
            self.assertTrue(required_key in context)

        self.assertEqual('de', context['DEFAULT_LANG'])
        self.assertEqual('Wordpress blog title', context['BLOG_TITLE'])
        self.assertEqual('Nikola test blog ;)', context['BLOG_DESCRIPTION'])
        self.assertEqual('http://some.blog', context['BLOG_URL'])
        self.assertEqual('mail@some.blog', context['BLOG_EMAIL'])
        self.assertEqual('Niko', context['BLOG_AUTHOR'])

    def test_importing_posts_and_attachments(self):
        channel = self.import_command.get_channel_from_file(
            self.import_filename)
        self.import_command.context = self.import_command.populate_context(channel)
        self.import_command.url_map = {} # For testing we use an empty one.

        write_metadata = mock.MagicMock()
        write_content = mock.MagicMock()
        download_mock = mock.MagicMock()

        with mock.patch('nikola.plugins.command_import_wordpress.CommandImportWordpress.write_content', write_content):
            with mock.patch('nikola.plugins.command_import_wordpress.CommandImportWordpress.write_metadata', write_metadata):
                with mock.patch('nikola.plugins.command_import_wordpress.CommandImportWordpress.download_url_content_to_file', download_mock):
                    with mock.patch('nikola.plugins.command_import_wordpress.os.makedirs'):
                        self.import_command.import_posts(channel)

        self.assertTrue(download_mock.called)
        download_mock.assert_any_call(u'http://some.blog/wp-content/uploads/2008/07/arzt_und_pfusch-sick-cover.png', u'new_site/files/wp-content/uploads/2008/07/arzt_und_pfusch-sick-cover.png')

        self.assertTrue(write_metadata.called)
        write_metadata.assert_any_call(u'new_site/stories/kontakt.meta', 'Kontakt', u'kontakt', '2009-07-16 20:20:32', None, [])

        self.assertTrue(write_content.called)
        write_content.assert_any_call(u'new_site/posts/200704hoert.wp', '...!\n\n\n\n[caption id="attachment_16" align="alignnone" width="739" caption="caption test"][/caption]\n\n\n\nNicht, dass daran jemals Zweifel bestanden.')
        write_content.assert_any_call(u'new_site/posts/200807arzt-und-pfusch-s-i-c-k.wp', u'Arzt+Pfusch - S.I.C.K.Gerade bin ich \xfcber das Album S.I.C.K von Arzt+Pfusch gestolpert, welches Arzt+Pfusch zum Download f\xfcr lau anbieten. Das Album steht unter einer Creative Commons BY-NC-ND-Lizenz.\n\nDie Ladung noisebmstupidevildustrial gibts als MP3s mit 64kbps und VBR, als Ogg Vorbis und als FLAC (letztere hier). Artwork und Lyrics gibts nochmal einzeln zum Download.')
        write_content.assert_any_call(u'new_site/stories/kontakt.wp', u'
Datenschutz
\n\nIch erhebe und speichere automatisch in meine Server Log Files Informationen, die dein Browser an mich \xfcbermittelt. Dies sind:\n\n
\n\n
Browsertyp und -version
\n\n
verwendetes Betriebssystem
\n\n
Referrer URL (die zuvor besuchte Seite)
\n\n
IP Adresse des zugreifenden Rechners
\n\n
Uhrzeit der Serveranfrage.
\n\n
\n\nDiese Daten sind f\xfcr mich nicht bestimmten Personen zuordenbar. Eine Zusammenf\xfchrung dieser Daten mit anderen Datenquellen wird nicht vorgenommen, die Daten werden einzig zu statistischen Zwecken erhoben.')

        self.assertTrue(len(self.import_command.url_map) > 0)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from flask import request, make_response
from . import main
from datetime import datetime, timedelta
import twilio.twiml
# import json

@main.route("/report_incident", methods=['GET', 'POST'])
def handle_message():
    message = str(request.values.get('Body'))  # noqa
    twiml = twilio.twiml.Response()
    step = int(request.cookies.get('messagecount', 0))
    # incident_report = json.loads(request.cookies.get('report', "{}"))
    if step is 0 and "report" in message.lower():
        twiml.message("Which Agency Owns the Vehicle? A)SEPTA Bus, B)SEPTA CCT, C)SEPTA, D)PWD, E)PECO, F)Streets, G)Others")  # noqa
    elif step is 1:
        # incident_report["agency"] = "SEPTOR"
        twiml.message("What is the License Plate Number? (eg.MG-1234E)")
    elif step is 2:
        # incident_report["licensePlate"] = "TRAP"
        twiml.message("What is the Vehicle ID? (eg.105014)")
    elif step is 3:
        # incident_report["ID"] = "COOKOOFORCOCOAPUFFS"
        twiml.message("How many minutes has it been Idling for? (eg. 10)")
    elif step is 4:
        twiml.message("Please describe the situation (eg. The driver is sleeping)")  # noqa
    else:
        twiml.message("Thanks!")
        step = -1
    step += 1
    expires = datetime.utcnow() + timedelta(hours=4)
    expires_str = expires.strftime('%a, %d %b %Y %H:%M:%S GMT')
    response = make_response(str(twiml))
    response.set_cookie('messagecount', value=str(step), expires=expires_str)
    # response.set_cookie('report', value=json.dumps(incident_report), expires=expires_str)  # noqa
    return response
<commit_msg>Add cookie storing for incident report data<commit_after>from flask import request, make_response
from . import main
from ..models import Agency
from datetime import datetime, timedelta
import twilio.twiml
# import json

@main.route("/report_incident", methods=['GET', 'POST'])
def handle_message():
    message = str(request.values.get('Body'))  # noqa
    twiml = twilio.twiml.Response()

    # Retrieve incident cookies
    step = int(request.cookies.get('messagecount', 0))
    vehicle_id = int(request.cookies.get('vehicle_id', 0))
    agency_id = int(request.cookies.get('agency_id', 0))
    license_plate = str(request.cookies.get('license_plate', ""))
    duration = int(request.cookies.get('duration', 0))
    description = str(request.cookies.get('description'))
    body = request.values.get('Body')

    if step is 0 and "report" in message.lower():
        twiml.message("Which Agency Owns the Vehicle? A)SEPTA Bus, B)SEPTA CCT, C)SEPTA, D)PWD, E)PECO, F)Streets, G)Others")  # noqa
        agency_name = agency_letter_to_name(body)
        agency_id = Agency.query.filter_by(name=agency_name).first()
    elif step is 1:
        twiml.message("What is the License Plate Number? (eg.MG-1234E)")
        license_plate = body
    elif step is 2:
        twiml.message("What is the Vehicle ID? (eg.105014)")
        vehicle_id = int(body)
    elif step is 3:
        twiml.message("How many minutes has it been Idling for? (eg. 10)")
        duration = int(body)
    elif step is 4:
        twiml.message("Please describe the situation (eg. The driver is sleeping)")  # noqa
        description = body
    else:
        print(vehicle_id)
        print(agency_id)
        print(license_plate)
        print(duration)
        print(description)
        twiml.message("Thanks!")
        step = -1
    step += 1
    response = make_response(str(twiml))
    set_cookie(response, 'messagecount', str(step))
    set_cookie(response, 'agency_id', str(agency_id))
    set_cookie(response, 'vehicle_id', str(vehicle_id))
    set_cookie(response, 'license_plate', license_plate)
    set_cookie(response, 'duration', str(duration))
    set_cookie(response, 'description', description)
    return response

def set_cookie(resp, key, val):
    expires = datetime.utcnow() + timedelta(hours=4)
    expires_str = expires.strftime('%a, %d %b %Y %H:%M:%S GMT')
    resp.set_cookie(key, value=val, expires=expires_str)

def agency_letter_to_name(letter):
    if letter is 'A':
        return "SEPTA Bus"
    elif letter is 'B':
        return "SEPTA CCT"
    elif letter is 'C':
        return "SEPTA"
    elif letter is 'D':
        return "PWD"
    elif letter is 'E':
        return "PECO"
    elif letter is 'F':
        return "Streets"
    else:
        return "TESTESTESTESTESTESTESt"
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
import json
import mock
import unittest
from nose.tools import eq_, ok_
from ..constants import CONST
from ..test_base import TestCase

from pywechat.services.basic import Basic
from pywechat.services.wechat_shake import ShakeService

class ShakeServiceTest(TestCase):

    '''Creates a TestCase for Shake Service.'''

    def setUp(self):
        TestCase.setUp(self)
        with mock.patch.object(Basic, 'access_token', autospec=True):
            self.shakeservice = self.service.init_service('Shake')

    def test_apply_devices(self):
        with mock.patch.object(ShakeService, 'apply_devices') as mock_method:
            apply_id = CONST.NUMBER
            device_id = CONST.NUMBER
            uuid = CONST.STRING
            major = CONST.NUMBER
            minor = CONST.NUMBER
            mock_method.return_value = {
                "data": {
                    "apply_id": apply_id,
                    "device_identifiers": [
                        {
                            "device_id": device_id,
                            "uuid": uuid,
                            "major": major,
                            "minor": minor
                        }
                    ]
                },
                "errcode": 0,
                "errmsg": "success."
            }
            apply_reason = CONST.STRING
            comment = CONST.STRING
            data = self.shakeservice.apply_devices(1,
                                                   apply_reason, comment)
            eq_(data["data"]["apply_id"], apply_id)
            eq_(data["data"]["device_identifiers"][0]["device_id"], device_id)
            eq_(data["data"]["device_identifiers"][0]["uuid"], uuid)
            eq_(data["data"]["device_identifiers"][0]["major"], major)
            eq_(data["data"]["device_identifiers"][0]["minor"], minor)

    def test_update_device(self):
        comment = CONST.STRING
        device_id = CONST.NUMBER
        uuid = CONST.NUMBER
        major = CONST.NUMBER
        minor = CONST.NUMBER
        with mock.patch.object(ShakeService, 'update_device') as mock_method:
            mock_method.return_value = {
                "data": {
                },
                "errcode": 0,
                "errmsg": "success."
            }
            data = self.shakeservice.update_device(comment, device_id)
            self.assertIsNotNone(data)
            data = self.shakeservice.update_device(comment, uuid=uuid,
                                                   major=major, minor=minor)
            self.assertIsNotNone(data)

    def test_bind_location(self):
        poi_id = CONST.NUMBER
        device_id = CONST.NUMBER
        uuid = CONST.NUMBER
        major = CONST.NUMBER
        minor = CONST.NUMBER
        with mock.patch.object(ShakeService, 'bind_location') as mock_method:
            mock_method.return_value = {
                "data": {
                },
                "errcode": 0,
                "errmsg": "success."
            }
            data = self.shakeservice.bind_location(poi_id, device_id)
            self.assertIsNotNone(data)
            data = self.shakeservice.bind_location(poi_id, uuid=uuid,
                                                   major=major, minor=minor)
            self.assertIsNotNone(data)

    def test_search_device(self):
        device_id = CONST.NUMBER
        uuid = CONST.NUMBER
        major = CONST.NUMBER
        minor = CONST.NUMBER
        with mock.patch.object(ShakeService, 'search_device') as mock_method:
            mock_method.return_value = {
                "data": {
                    "devices": [
                        {
                            "comment": "",
                            "device_id": device_id,
                            "major": major,
                            "minor": minor,
                            "page_ids": "15369",
                            "status": 1,
                            "poi_id": 0,
                            "uuid": uuid
                        }
                    ],
                    "total_count": 1
                },
                "errcode": 0,
                "errmsg": "success."
            }
            data = self.shakeservice.search_device(device_id)
            eq_(data["data"]["devices"][0]["device_id"], device_id)
            eq_(data["data"]["devices"][0]["uuid"], uuid)
            eq_(data["data"]["devices"][0]["major"], major)
            eq_(data["data"]["devices"][0]["minor"], minor)
            data = self.shakeservice.search_device(uuid=uuid,
                                                   major=major, minor=minor)
            eq_(data["data"]["devices"][0]["device_id"], device_id)
            eq_(data["data"]["devices"][0]["uuid"], uuid)
            eq_(data["data"]["devices"][0]["major"], major)
            eq_(data["data"]["devices"][0]["minor"], minor)

    def test_search_devices(self):
        apply_id = CONST.NUMBER
        device_id = CONST.NUMBER
        uuid = CONST.NUMBER
        major = CONST.NUMBER
        minor = CONST.NUMBER
        with mock.patch.object(ShakeService, 'search_devices') as mock_method:
            mock_method.return_value = {
                "data": {
                    "devices": [
                        {
                            "comment": "",
                            "device_id": device_id,
                            "major": major,
                            "minor": minor,
                            "page_ids": "15369",
                            "status": 1,
                            "poi_id": 0,
                            "uuid": uuid
                        }
                    ],
                    "total_count": 1
                },
                "errcode": 0,
                "errmsg": "success."
            }
            data = self.shakeservice.search_devices(0, 1, apply_id)
            eq_(data["data"]["devices"][0]["device_id"], device_id)
            eq_(data["data"]["devices"][0]["uuid"], uuid)
            eq_(data["data"]["devices"][0]["major"], major)
            eq_(data["data"]["devices"][0]["minor"], minor)
            eq_(data["data"]["total_count"], 1)

    def test_upload_material(self):
        image = CONST.STRING
        pic_url = CONST.STRING
        with mock.patch.object(ShakeService, 'upload_material') as mock_method:
            mock_method.return_value = {
                "data": {
                    "pic_url": pic_url
                },
                "errcode": 0,
                "errmsg": "success."
            }
            data = self.shakeservice.upload_material(image)
            eq_(data["data"]["pic_url"], pic_url)
<commit_msg>Adds a test<commit_after>#-*- coding: utf-8 -*-
import json
import mock
import unittest
from nose.tools import eq_, ok_
from ..constants import CONST
from ..test_base import TestCase

from pywechat.services.basic import Basic
from pywechat.services.wechat_shake import ShakeService

class ShakeServiceTest(TestCase):

    '''Creates a TestCase for Shake Service.'''

    def setUp(self):
        TestCase.setUp(self)
        with mock.patch.object(Basic, 'access_token', autospec=True):
            self.shakeservice = self.service.init_service('Shake')

    def test_apply_devices(self):
        with mock.patch.object(ShakeService, 'apply_devices') as mock_method:
            apply_id = CONST.NUMBER
            device_id = CONST.NUMBER
            uuid = CONST.STRING
            major = CONST.NUMBER
            minor = CONST.NUMBER
            mock_method.return_value = {
                "data": {
                    "apply_id": apply_id,
                    "device_identifiers": [
                        {
                            "device_id": device_id,
                            "uuid": uuid,
                            "major": major,
                            "minor": minor
                        }
                    ]
                },
                "errcode": 0,
                "errmsg": "success."
            }
            apply_reason = CONST.STRING
            comment = CONST.STRING
            data = self.shakeservice.apply_devices(1,
                                                   apply_reason, comment)
            eq_(data["data"]["apply_id"], apply_id)
            eq_(data["data"]["device_identifiers"][0]["device_id"], device_id)
            eq_(data["data"]["device_identifiers"][0]["uuid"], uuid)
            eq_(data["data"]["device_identifiers"][0]["major"], major)
            eq_(data["data"]["device_identifiers"][0]["minor"], minor)

    def test_update_device(self):
        comment = CONST.STRING
        device_id = CONST.NUMBER
        uuid = CONST.NUMBER
        major = CONST.NUMBER
        minor = CONST.NUMBER
        with mock.patch.object(ShakeService, 'update_device') as mock_method:
            mock_method.return_value = {
                "data": {
                },
                "errcode": 0,
                "errmsg": "success."
            }
            data = self.shakeservice.update_device(comment, device_id)
            self.assertIsNotNone(data)
            data = self.shakeservice.update_device(comment, uuid=uuid,
                                                   major=major, minor=minor)
            self.assertIsNotNone(data)

    def test_bind_location(self):
        poi_id = CONST.NUMBER
        device_id = CONST.NUMBER
        uuid = CONST.NUMBER
        major = CONST.NUMBER
        minor = CONST.NUMBER
        with mock.patch.object(ShakeService, 'bind_location') as mock_method:
            mock_method.return_value = {
                "data": {
                },
                "errcode": 0,
                "errmsg": "success."
            }
            data = self.shakeservice.bind_location(poi_id, device_id)
            self.assertIsNotNone(data)
            data = self.shakeservice.bind_location(poi_id, uuid=uuid,
                                                   major=major, minor=minor)
            self.assertIsNotNone(data)

    def test_search_device(self):
        device_id = CONST.NUMBER
        uuid = CONST.NUMBER
        major = CONST.NUMBER
        minor = CONST.NUMBER
        with mock.patch.object(ShakeService, 'search_device') as mock_method:
            mock_method.return_value = {
                "data": {
                    "devices": [
                        {
                            "comment": "",
                            "device_id": device_id,
                            "major": major,
                            "minor": minor,
                            "page_ids": "15369",
                            "status": 1,
                            "poi_id": 0,
                            "uuid": uuid
                        }
                    ],
                    "total_count": 1
                },
                "errcode": 0,
                "errmsg": "success."
            }
            data = self.shakeservice.search_device(device_id)
            eq_(data["data"]["devices"][0]["device_id"], device_id)
            eq_(data["data"]["devices"][0]["uuid"], uuid)
            eq_(data["data"]["devices"][0]["major"], major)
            eq_(data["data"]["devices"][0]["minor"], minor)
            data = self.shakeservice.search_device(uuid=uuid,
                                                   major=major, minor=minor)
            eq_(data["data"]["devices"][0]["device_id"], device_id)
            eq_(data["data"]["devices"][0]["uuid"], uuid)
            eq_(data["data"]["devices"][0]["major"], major)
            eq_(data["data"]["devices"][0]["minor"], minor)

    def test_search_devices(self):
        apply_id = CONST.NUMBER
        device_id = CONST.NUMBER
        uuid = CONST.NUMBER
        major = CONST.NUMBER
        minor = CONST.NUMBER
        with mock.patch.object(ShakeService, 'search_devices') as mock_method:
            mock_method.return_value = {
                "data": {
                    "devices": [
                        {
                            "comment": "",
                            "device_id": device_id,
                            "major": major,
                            "minor": minor,
                            "page_ids": "15369",
                            "status": 1,
                            "poi_id": 0,
                            "uuid": uuid
                        }
                    ],
                    "total_count": 1
                },
                "errcode": 0,
                "errmsg": "success."
            }
            data = self.shakeservice.search_devices(0, 1, apply_id)
            eq_(data["data"]["devices"][0]["device_id"], device_id)
            eq_(data["data"]["devices"][0]["uuid"], uuid)
            eq_(data["data"]["devices"][0]["major"], major)
            eq_(data["data"]["devices"][0]["minor"], minor)
            eq_(data["data"]["total_count"], 1)

    def test_upload_material(self):
        image = CONST.STRING
        pic_url = CONST.STRING
        with mock.patch.object(ShakeService, 'upload_material') as mock_method:
            mock_method.return_value = {
                "data": {
                    "pic_url": pic_url
                },
                "errcode": 0,
                "errmsg": "success."
            }
            data = self.shakeservice.upload_material(image)
            eq_(data["data"]["pic_url"], pic_url)

    def test_add_page(self):
        title = CONST.STRING
        description = CONST.STRING
        page_url = CONST.STRING
        icon_url = CONST.STRING
        comment = CONST.STRING
        page_id = CONST.NUMBER
        with mock.patch.object(ShakeService, 'add_page') as mock_method:
            mock_method.return_value = {
                "data": {
                    "page_id": page_id
                },
                "errcode": 0,
                "errmsg": "success."
            }
            data = self.shakeservice.add_page(title, description, page_url,
                                              icon_url, comment)
            eq_(data["data"]["page_id"], page_id)
<|endoftext|>