<commit_before># Copyright 2017-2019 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import os
import subprocess

__version__ = "0.6.1.dev3"

# Used by bootstrap_main to ensure minimum required set of modules
# before continuing.
__requires__ = [
    # (<required module>, <distutils package req>)
    ("pip", "pip"),
    ("yaml", "PyYAML"),
    ("setuptools", "setuptools"),
    ("six", "six"),
    ("tabview", "tabview"),
    ("twine", "twine"),
    ("werkzeug", "Werkzeug"),
]

__pkgdir__ = os.path.dirname(os.path.dirname(__file__))

def _try_init_git_attrs():
    try:
        _init_git_commit()
    except (OSError, subprocess.CalledProcessError):
        pass
    else:
        try:
            _init_git_status()
        except (OSError, subprocess.CalledProcessError):
            pass

def _init_git_commit():
    commit = _git_cmd("git -C \"%(repo)s\" log -1 --oneline | cut -d' ' -f1")
    globals()["__git_commit__"] = commit

def _init_git_status():
    raw = _git_cmd("git -C \"%(repo)s\" status -s")
    globals()["__git_status__"] = raw.split("\n") if raw else []

def _git_cmd(cmd, **kw):
    repo = os.path.dirname(__file__)
    cmd = cmd % dict(repo=repo, **kw)
    null = open(os.devnull, "w")
    out = subprocess.check_output(cmd, stderr=null, shell=True)
    return out.decode("utf-8").strip()

def version():
    git_commit = globals().get("__git_commit__")
    if git_commit:
        git_status = globals().get("__git_status__", [])
        workspace_changed_marker = "*" if git_status else ""
        return "%s (dev %s%s)" % (__version__, git_commit,
                                 workspace_changed_marker)
    else:
        return __version__

_try_init_git_attrs()
<commit_msg>0.6.1.dev4<commit_after># Copyright 2017-2019 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import os
import subprocess

__version__ = "0.6.1.dev4"

# Used by bootstrap_main to ensure minimum required set of modules
# before continuing.
__requires__ = [
    # (<required module>, <distutils package req>)
    ("pip", "pip"),
    ("yaml", "PyYAML"),
    ("setuptools", "setuptools"),
    ("six", "six"),
    ("tabview", "tabview"),
    ("twine", "twine"),
    ("werkzeug", "Werkzeug"),
]

__pkgdir__ = os.path.dirname(os.path.dirname(__file__))

def _try_init_git_attrs():
    try:
        _init_git_commit()
    except (OSError, subprocess.CalledProcessError):
        pass
    else:
        try:
            _init_git_status()
        except (OSError, subprocess.CalledProcessError):
            pass

def _init_git_commit():
    commit = _git_cmd("git -C \"%(repo)s\" log -1 --oneline | cut -d' ' -f1")
    globals()["__git_commit__"] = commit

def _init_git_status():
    raw = _git_cmd("git -C \"%(repo)s\" status -s")
    globals()["__git_status__"] = raw.split("\n") if raw else []

def _git_cmd(cmd, **kw):
    repo = os.path.dirname(__file__)
    cmd = cmd % dict(repo=repo, **kw)
    null = open(os.devnull, "w")
    out = subprocess.check_output(cmd, stderr=null, shell=True)
    return out.decode("utf-8").strip()

def version():
    git_commit = globals().get("__git_commit__")
    if git_commit:
        git_status = globals().get("__git_status__", [])
        workspace_changed_marker = "*" if git_status else ""
        return "%s (dev %s%s)" % (__version__, git_commit,
                                 workspace_changed_marker)
    else:
        return __version__

_try_init_git_attrs()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function
from time import sleep
from threading import Thread

__version__ = '0.2.7'

class Config(object):
    __verbose = True
    __test_mode = False
    __pin_numbering = 'BOARD'

    @staticmethod
    def set_verbose(value):
        if type(value) == bool:
            Config.__verbose = value
        else:
            raise TypeError('verbose must be either True or False')

    @staticmethod
    def get_verbose():
        return Config.__verbose

    @staticmethod
    def set_test_mode(value):
        if type(value) == bool:
            Config.__test_mode = value
        else:
            raise TypeError('test_mode must be either True or False')

    @staticmethod
    def get_test_mode():
        return Config.__test_mode

    @staticmethod
    def set_pin_numbering(value):
        if type(value) != str:
            raise TypeError('pin_numbering must be a string:'
                            '\'BOARD\' or \'BCM\'')
        value = str(value).upper()
        if pins_in_use:
            raise ValueError('Pin numbering format cannot be changed '
                             'if motors already exist. Set this at '
                             'the start of your script.')
        if not (value == 'BOARD' or value == 'BCM'):
            raise ValueError(
                'Pin numbering format must be \'BOARD\' or \'BCM\'')
        Config.__pin_numbering = value
        print("Pin numbering format set: " + value)
        return value

    @staticmethod
    def get_pin_numbering():
        return Config.__pin_numbering

# Print version
if Config.get_verbose():
    print('L293D driver version ' + __version__)

# Import GPIO
try:
    import RPi.GPIO as GPIO
except ImportError:
    GPIO = None
    print("Can't import RPi.GPIO. Please (re)install.")
    Config.set_test_mode(True)
    print('Test mode has been enabled. Please view README for more info.')

# Set GPIO warnings based on verbose value
if not Config.get_test_mode():
    if Config.get_verbose():
        GPIO.setwarnings(True)
    else:
        GPIO.setwarnings(False)

# Set GPIO mode
if not Config.get_test_mode():
    if Config.get_pin_numbering() == 'BOARD':
        if Config.get_verbose():
            print('Setting GPIO mode: BOARD')
        GPIO.setmode(GPIO.BOARD)
    elif Config.get_pin_numbering() == 'BCM':
        if Config.get_verbose():
            print('Setting GPIO mode: BCM')
        GPIO.setmode(GPIO.BCM)
    else:
        print("pin_numbering must be either 'BOARD' or 'BCM'.")
        raise ValueError("pin_numbering must be either 'BOARD' or 'BCM'.")

pins_in_use = []  # Lists pins in use (all motors)

class DC(object):
    """
    A class for a motor wired to the L293D chip where
    motor_pins[0] is pinA is L293D pin1 or pin9  : On or off
    motor_pins[1] is pinB is L293D pin2 or pin10 : Anticlockwise positive
    motor_pins[2] is pinC is L293D pin7 or pin15 : Clockwise positive
    """

    def __init__(self, pin_a=0, pin_b=0, pin_c=0):
        # Assign parameters to list
        self.motor_pins = [0 for x in range(3)]
        self.motor_pins[0] = pin_a
        self.motor_pins[1] = pin_b
        self.motor_pins[2] = pin_c

        self.pin_numbering = Config.get_pin_numbering()

        # Check pins are valid
        if pins_are_valid(self.motor_pins):
            self.exists = True
        # Append to global list of pins in use
        for pin in self.motor_pins:
            pins_in_use.append(pin)
        # Set up GPIO mode for pins
        self.gpio_setup()

    def gpio_setup(self):
        """
        Set GPIO.OUT for each pin in use
        """
        for pin in self.motor_pins:
            if not Config.get_test_mode():
                GPIO.setup(pin, GPIO.OUT)

    def drive_motor(self, direction=1, duration=None, wait=True):
        """
        Method called by other functions to drive L293D via GPIO
        """
        self.check()
        if not Config.get_test_mode():
            if direction == 0:  # Then stop motor
                GPIO.output(self.motor_pins[0], GPIO.LOW)
            else:  # Spin motor
                # Set first direction GPIO level
                GPIO.output(self.motor_pins[direction], GPIO.HIGH)
                # Set second direction GPIO level
                GPIO.output(self.motor_pins[direction * -1], GPIO.LOW)
                # Turn the motor on
                GPIO.output(self.motor_pins[0], GPIO.HIGH)
        # If duration has been specified, sleep then stop
        if duration is not None and direction != 0:
            stop_thread = Thread(target=self.stop, args=(duration,))
            # Sleep in thread
            stop_thread.start()
            if wait:
                # If wait is true, the main thread is blocked
                stop_thread.join()

    def pins_string_list(self):
        """
        Return readable list of pins
        """
        return '[{}, {} and {}]'.format(*self.motor_pins)

    def clockwise(self, duration=None, wait=True, speed=100):
        """
        Uses drive_motor to spin motor clockwise
        """
        self.check()
        if Config.get_verbose():
            print('spinning motor at {0} pins {1} clockwise.'.format(
                self.pin_numbering, self.pins_string_list()))
        self.drive_motor(direction=1, duration=duration, wait=wait)

    def anticlockwise(self, duration=None, wait=True, speed=100):
        """
        Uses drive_motor to spin motor anticlockwise
        """
        self.check()
        if Config.get_verbose():
            print('spinning motor at {0} pins {1} anticlockwise.'.format(
                self.pin_numbering, self.pins_string_list()))
        self.drive_motor(direction=-1, duration=duration, wait=wait)

    def stop(self, after=0):
        """
        If 'after' is specified, sleep for amount of time
        """
        self.check()
        if after > 0:
            sleep(after)
        # Verbose output
        if Config.get_verbose():
            print('stopping motor at {0} pins {1}.'.format(
                self.pin_numbering, self.pins_string_list()))
        # Call drive_motor to stop motor after sleep
        if not Config.get_test_mode():
            self.drive_motor(direction=0, duration=after, wait=True)

    def remove(self):
        """
        Remove motor
        """
        if self.exists:
            global pins_in_use
            for m_pin in self.motor_pins:
                if m_pin in pins_in_use:
                    pins_in_use.remove(m_pin)
            self.exists = False
        else:
            if Config.get_verbose():
                print('Motor has already been removed')

    def check(self):
        """
        Check the motor exists. If not, an exception is raised
        """
        if not self.exists:
            raise ValueError('Motor has been removed. '
                             'If you wish to use this motor again, '
                             'you must redefine it.')

class Motor(object):
    def __init__(self, pin_a=0, pin_b=0, pin_c=0):
        raise DeprecationWarning('The Motor class has been deprecated. '
                                 'Please use \'DC\' or \'Stepper\' instead.')

class Stepper(object):
    def __init__(self):
        raise FutureWarning('Stepper motors are not yet supported. Go to '
                            'https://github.com/jamesevickery/l293d/issues/20 '
                            'for more info')

def pins_are_valid(pins, force_selection=False):
    """
    Check the pins specified are valid for pin numbering in use
    """
    # Pin numbering, used below, should be
    # a parameter of this function (future)
    if Config.get_pin_numbering() == 'BOARD':  # Set valid pins for BOARD
        valid_pins = [
            7, 11, 12, 13, 15, 16, 18, 22, 29, 31, 32, 33, 36, 37
        ]
    elif Config.get_pin_numbering() == 'BCM':  # Set valid pins for BCM
        valid_pins = [
            4, 5, 6, 12, 13, 16, 17, 18, 22, 23, 24, 25, 26, 27
        ]
    else:  # pin_numbering value invalid
        raise ValueError("pin_numbering must be either 'BOARD' or 'BCM'.")
    for pin in pins:
        pin_int = int(pin)
        if pin_int not in valid_pins and force_selection is False:
            err_str = (
                "GPIO pin number must be from list of valid pins: %s"
                "\nTo use selected pins anyway, set force_selection=True "
                "in function call." % str(valid_pins))
            raise ValueError(err_str)
        if pin in pins_in_use:
            raise ValueError('GPIO pin {} already in use.'.format(pin))
    return True

def cleanup():
    """
    Call GPIO cleanup method
    """
    if not Config.get_test_mode():
        try:
            GPIO.cleanup()
            if Config.get_verbose():
                print('GPIO cleanup successful.')
        except:
            if Config.get_verbose():
                print('GPIO cleanup failed.')
    else:
        # Skip GPIO cleanup if GPIO calls are not being made (test_mode)
        if Config.get_verbose():
            print('Cleanup not needed when test_mode is enabled.')
<commit_msg>Closed #66<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function
from time import sleep
from threading import Thread

__version__ = '0.2.7'

class Config(object):
    __verbose = True
    __test_mode = False
    __pin_numbering = 'BOARD'

    @staticmethod
    def set_verbose(value):
        if type(value) == bool:
            Config.__verbose = value
        else:
            raise TypeError('verbose must be either True or False')

    @staticmethod
    def get_verbose():
        return Config.__verbose

    @staticmethod
    def set_test_mode(value):
        if type(value) == bool:
            Config.__test_mode = value
        else:
            raise TypeError('test_mode must be either True or False')

    @staticmethod
    def get_test_mode():
        return Config.__test_mode

    @staticmethod
    def set_pin_numbering(value):
        if type(value) != str:
            raise TypeError('pin_numbering must be a string:'
                            '\'BOARD\' or \'BCM\'')
        value = str(value).upper()
        if pins_in_use:
            raise ValueError('Pin numbering format cannot be changed '
                             'if motors already exist. Set this at '
                             'the start of your script.')
        if not (value == 'BOARD' or value == 'BCM'):
            raise ValueError(
                'Pin numbering format must be \'BOARD\' or \'BCM\'')
        Config.__pin_numbering = value
        print("Pin numbering format set: " + value)
        return value

    @staticmethod
    def get_pin_numbering():
        return Config.__pin_numbering

# Print version
if Config.get_verbose():
    print('L293D driver version ' + __version__)

# Import GPIO
try:
    import RPi.GPIO as GPIO
except ImportError:
    GPIO = None
    print("Can't import RPi.GPIO. Please (re)install.")
    Config.set_test_mode(True)
    print('Test mode has been enabled. Please view README for more info.')

# Set GPIO warnings based on verbose value
if not Config.get_test_mode():
    if Config.get_verbose():
        GPIO.setwarnings(True)
    else:
        GPIO.setwarnings(False)

# Set GPIO mode
if not Config.get_test_mode():
    if Config.get_pin_numbering() == 'BOARD':
        if Config.get_verbose():
            print('Setting GPIO mode: BOARD')
        GPIO.setmode(GPIO.BOARD)
    elif Config.get_pin_numbering() == 'BCM':
        if Config.get_verbose():
            print('Setting GPIO mode: BCM')
        GPIO.setmode(GPIO.BCM)
    else:
        print("pin_numbering must be either 'BOARD' or 'BCM'.")
        raise ValueError("pin_numbering must be either 'BOARD' or 'BCM'.")

pins_in_use = []  # Lists pins in use (all motors)

class DC(object):
    """
    A class for a motor wired to the L293D chip where
    motor_pins[0] is pinA is L293D pin1 or pin9  : On or off
    motor_pins[1] is pinB is L293D pin2 or pin10 : Anticlockwise positive
    motor_pins[2] is pinC is L293D pin7 or pin15 : Clockwise positive
    """

    def __init__(self, pin_a=0, pin_b=0, pin_c=0):
        # Assign parameters to list
        self.motor_pins = [0 for x in range(3)]
        self.motor_pins[0] = pin_a
        self.motor_pins[1] = pin_b
        self.motor_pins[2] = pin_c

        self.pin_numbering = Config.get_pin_numbering()

        self.reversed = False

        # Check pins are valid
        if pins_are_valid(self.motor_pins):
            self.exists = True
        # Append to global list of pins in use
        for pin in self.motor_pins:
            pins_in_use.append(pin)
        # Set up GPIO mode for pins
        self.gpio_setup()

    def gpio_setup(self):
        """
        Set GPIO.OUT for each pin in use
        """
        for pin in self.motor_pins:
            if not Config.get_test_mode():
                GPIO.setup(pin, GPIO.OUT)

    def drive_motor(self, direction=1, duration=None, wait=True):
        """
        Method called by other functions to drive L293D via GPIO
        """
        self.check()
        if self.reversed:
            direction *= -1
        if not Config.get_test_mode():
            if direction == 0:  # Then stop motor
                GPIO.output(self.motor_pins[0], GPIO.LOW)
            else:  # Spin motor
                # Set first direction GPIO level
                GPIO.output(self.motor_pins[direction], GPIO.HIGH)
                # Set second direction GPIO level
                GPIO.output(self.motor_pins[direction * -1], GPIO.LOW)
                # Turn the motor on
                GPIO.output(self.motor_pins[0], GPIO.HIGH)
        # If duration has been specified, sleep then stop
        if duration is not None and direction != 0:
            stop_thread = Thread(target=self.stop, args=(duration,))
            # Sleep in thread
            stop_thread.start()
            if wait:
                # If wait is true, the main thread is blocked
                stop_thread.join()

    def pins_string_list(self):
        """
        Return readable list of pins
        """
        return '[{}, {} and {}]'.format(*self.motor_pins)

    def clockwise(self, duration=None, wait=True, speed=100):
        """
        Uses drive_motor to spin motor clockwise
        """
        self.check()
        if Config.get_verbose():
            reversed_string = ''
            if self.reversed:
                reversed_string = 'reversed '
            print('spinning {0}motor at {1} pins {2} clockwise.'.format(
                reversed_string, self.pin_numbering, self.pins_string_list()))
        self.drive_motor(direction=1, duration=duration, wait=wait)

    def anticlockwise(self, duration=None, wait=True, speed=100):
        """
        Uses drive_motor to spin motor anticlockwise
        """
        self.check()
        if Config.get_verbose():
            reversed_string = ''
            if self.reversed:
                reversed_string = 'reversed '
            print('spinning {0}motor at {1} pins {2} anticlockwise.'.format(
                reversed_string, self.pin_numbering, self.pins_string_list()))
        self.drive_motor(direction=-1, duration=duration, wait=wait)

    def stop(self, after=0):
        """
        If 'after' is specified, sleep for amount of time
        """
        self.check()
        if after > 0:
            sleep(after)
        # Verbose output
        if Config.get_verbose():
            reversed_string = ''
            if self.reversed:
                reversed_string = 'reversed '
            print('stopping {0}motor at {1} pins {2}.'.format(
                reversed_string, self.pin_numbering, self.pins_string_list()))
        # Call drive_motor to stop motor after sleep
        if not Config.get_test_mode():
            self.drive_motor(direction=0, duration=after, wait=True)

    def remove(self):
        """
        Remove motor
        """
        if self.exists:
            global pins_in_use
            for m_pin in self.motor_pins:
                if m_pin in pins_in_use:
                    pins_in_use.remove(m_pin)
            self.exists = False
        else:
            if Config.get_verbose():
                print('Motor has already been removed')

    def check(self):
        """
        Check the motor exists. If not, an exception is raised
        """
        if not self.exists:
            raise ValueError('Motor has been removed. '
                             'If you wish to use this motor again, '
                             'you must redefine it.')

class Motor(object):
    def __init__(self, pin_a=0, pin_b=0, pin_c=0):
        raise DeprecationWarning('The Motor class has been deprecated. '
                                 'Please use \'DC\' or \'Stepper\' instead.')

class Stepper(object):
    def __init__(self):
        raise FutureWarning('Stepper motors are not yet supported. Go to '
                            'https://github.com/jamesevickery/l293d/issues/20 '
                            'for more info')

def pins_are_valid(pins, force_selection=False):
    """
    Check the pins specified are valid for pin numbering in use
    """
    # Pin numbering, used below, should be
    # a parameter of this function (future)
    if Config.get_pin_numbering() == 'BOARD':  # Set valid pins for BOARD
        valid_pins = [
            7, 11, 12, 13, 15, 16, 18, 22, 29, 31, 32, 33, 36, 37
        ]
    elif Config.get_pin_numbering() == 'BCM':  # Set valid pins for BCM
        valid_pins = [
            4, 5, 6, 12, 13, 16, 17, 18, 22, 23, 24, 25, 26, 27
        ]
    else:  # pin_numbering value invalid
        raise ValueError("pin_numbering must be either 'BOARD' or 'BCM'.")
    for pin in pins:
        pin_int = int(pin)
        if pin_int not in valid_pins and force_selection is False:
            err_str = (
                "GPIO pin number must be from list of valid pins: %s"
                "\nTo use selected pins anyway, set force_selection=True "
                "in function call." % str(valid_pins))
            raise ValueError(err_str)
        if pin in pins_in_use:
            raise ValueError('GPIO pin {} already in use.'.format(pin))
    return True

def cleanup():
    """
    Call GPIO cleanup method
    """
    if not Config.get_test_mode():
        try:
            GPIO.cleanup()
            if Config.get_verbose():
                print('GPIO cleanup successful.')
        except:
            if Config.get_verbose():
                print('GPIO cleanup failed.')
    else:
        # Skip GPIO cleanup if GPIO calls are not being made (test_mode)
        if Config.get_verbose():
            print('Cleanup not needed when test_mode is enabled.')
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from django.core.exceptions import PermissionDenied
try:
    from django.conf.urls import patterns, url
except ImportError:
    from django.conf.urls.defaults import patterns, url
from django.contrib import admin
from django.contrib.admin import helpers
from django.contrib.contenttypes.models import ContentType
from django.core.urlresolvers import reverse
from django.shortcuts import get_object_or_404, render
from django.contrib.admin.util import unquote
from django.utils.text import capfirst
from django.utils.html import mark_safe
from django.utils.translation import ugettext as _
try:
    from django.utils.encoding import force_text
except ImportError:  # django 1.3 compatibility
    from django.utils.encoding import force_unicode as force_text
from django.conf import settings

try:
    USER_NATURAL_KEY = settings.AUTH_USER_MODEL
except AttributeError:
    USER_NATURAL_KEY = "auth.User"
USER_NATURAL_KEY = tuple(key.lower() for key in USER_NATURAL_KEY.split('.', 1))

class SimpleHistoryAdmin(admin.ModelAdmin):
    object_history_template = "simple_history/object_history.html"
    object_history_form_template = "simple_history/object_history_form.html"

    def get_urls(self):
        """Returns the additional urls used by the Reversion admin."""
        urls = super(SimpleHistoryAdmin, self).get_urls()
        admin_site = self.admin_site
        opts = self.model._meta
        try:
            info = opts.app_label, opts.module_name
        except AttributeError:
            info = opts.app_label, opts.model_name
        history_urls = patterns(
            "",
            url("^([^/]+)/history/([^/]+)/$",
                admin_site.admin_view(self.history_form_view),
                name='%s_%s_simple_history' % info),
        )
        return history_urls + urls

    def history_view(self, request, object_id, extra_context=None):
        "The 'history' admin view for this model."
        model = self.model
        opts = model._meta
        app_label = opts.app_label
        pk_name = opts.pk.attname
        history = getattr(model, model._meta.simple_history_manager_attribute)
        object_id = unquote(object_id)
        action_list = history.filter(**{pk_name: object_id})
        # If no history was found, see whether this object even exists.
        obj = get_object_or_404(model, pk=object_id)
        content_type = ContentType.objects.get_by_natural_key(
            *USER_NATURAL_KEY)
        admin_user_view = 'admin:%s_%s_change' % (content_type.app_label,
                                                  content_type.model)
        context = {
            'title': _('Change history: %s') % force_text(obj),
            'action_list': action_list,
            'module_name': capfirst(force_text(opts.verbose_name_plural)),
            'object': obj,
            'root_path': getattr(self.admin_site, 'root_path', None),
            'app_label': app_label,
            'opts': opts,
            'admin_user_view': admin_user_view
        }
        context.update(extra_context or {})
        return render(request, template_name=self.object_history_template,
                      dictionary=context, current_app=self.admin_site.name)

    def history_form_view(self, request, object_id, version_id):
        original_model = self.model
        original_opts = original_model._meta
        history = getattr(self.model,
                          self.model._meta.simple_history_manager_attribute)
        model = history.model
        opts = model._meta
        pk_name = original_opts.pk.attname
        record = get_object_or_404(model, **{
            pk_name: object_id,
            'history_id': version_id,
        })
        obj = record.instance
        obj._state.adding = False

        if not self.has_change_permission(request, obj):
            raise PermissionDenied

        formsets = []
        form_class = self.get_form(request, obj)
        if request.method == 'POST':
            form = form_class(request.POST, request.FILES, instance=obj)
            if form.is_valid():
                form_validated = True
                new_object = self.save_form(request, form, change=True)
            else:
                form_validated = False
                new_object = obj

            if form_validated:
                self.save_model(request, new_object, form, change=True)
                form.save_m2m()

                change_message = self.construct_change_message(request, form,
                                                               formsets)
                self.log_change(request, new_object, change_message)
                return self.response_change(request, new_object)

        else:
            form = form_class(instance=obj)

        admin_form = helpers.AdminForm(
            form,
            self.get_fieldsets(request, obj),
            self.prepopulated_fields,
            self.get_readonly_fields(request, obj),
            model_admin=self,
        )
        media = self.media + admin_form.media

        try:
            model_name = original_opts.module_name
        except AttributeError:
            model_name = original_opts.model_name
        url_triplet = self.admin_site.name, original_opts.app_label, model_name
        content_type_id = ContentType.objects.get_for_model(self.model).id
        context = {
            'title': _('Revert %s') % force_text(obj),
            'adminform': admin_form,
            'object_id': object_id,
            'original': obj,
            'is_popup': False,
            'media': mark_safe(media),
            'errors': helpers.AdminErrorList(form, formsets),
            'app_label': opts.app_label,
            'original_opts': original_opts,
            'changelist_url': reverse('%s:%s_%s_changelist' % url_triplet),
            'change_url': reverse('%s:%s_%s_change' % url_triplet,
                                  args=(obj.pk,)),
            'history_url': reverse('%s:%s_%s_history' % url_triplet,
                                   args=(obj.pk,)),
            # Context variables copied from render_change_form
            'add': False,
            'change': True,
            'has_add_permission': self.has_add_permission(request),
            'has_change_permission': self.has_change_permission(request, obj),
            'has_delete_permission': self.has_delete_permission(request, obj),
            'has_file_field': True,
            'has_absolute_url': False,
            'form_url': '',
            'opts': opts,
            'content_type_id': content_type_id,
            'save_as': self.save_as,
            'save_on_top': self.save_on_top,
            'root_path': getattr(self.admin_site, 'root_path', None),
        }
        return render(request, template_name=self.object_history_form_template,
                      dictionary=context, current_app=self.admin_site.name)

    def save_model(self, request, obj, form, change):
        """Set special model attribute to user for reference after save"""
        obj._history_user = request.user
        super(SimpleHistoryAdmin, self).save_model(request, obj, form, change)
<commit_msg>Treat ModelAdmin like an old python class<commit_after>from __future__ import unicode_literals

from django.core.exceptions import PermissionDenied
try:
    from django.conf.urls import patterns, url
except ImportError:
    from django.conf.urls.defaults import patterns, url
from django.contrib import admin
from django.contrib.admin import helpers
from django.contrib.contenttypes.models import ContentType
from django.core.urlresolvers import reverse
from django.shortcuts import get_object_or_404, render
from django.contrib.admin.util import unquote
from django.utils.text import capfirst
from django.utils.html import mark_safe
from django.utils.translation import ugettext as _
try:
    from django.utils.encoding import force_text
except ImportError:  # django 1.3 compatibility
    from django.utils.encoding import force_unicode as force_text
from django.conf import settings

try:
    USER_NATURAL_KEY = settings.AUTH_USER_MODEL
except AttributeError:
    USER_NATURAL_KEY = "auth.User"
USER_NATURAL_KEY = tuple(key.lower() for key in USER_NATURAL_KEY.split('.', 1))

class SimpleHistoryAdmin(admin.ModelAdmin, object):
    object_history_template = "simple_history/object_history.html"
    object_history_form_template = "simple_history/object_history_form.html"

    def get_urls(self):
        """Returns the additional urls used by the Reversion admin."""
        urls = super(SimpleHistoryAdmin, self).get_urls()
        admin_site = self.admin_site
        opts = self.model._meta
        try:
            info = opts.app_label, opts.module_name
        except AttributeError:
            info = opts.app_label, opts.model_name
        history_urls = patterns(
            "",
            url("^([^/]+)/history/([^/]+)/$",
                admin_site.admin_view(self.history_form_view),
                name='%s_%s_simple_history' % info),
        )
        return history_urls + urls

    def history_view(self, request, object_id, extra_context=None):
        "The 'history' admin view for this model."
        model = self.model
        opts = model._meta
        app_label = opts.app_label
        pk_name = opts.pk.attname
        history = getattr(model, model._meta.simple_history_manager_attribute)
        object_id = unquote(object_id)
        action_list = history.filter(**{pk_name: object_id})
        # If no history was found, see whether this object even exists.
        obj = get_object_or_404(model, pk=object_id)
        content_type = ContentType.objects.get_by_natural_key(
            *USER_NATURAL_KEY)
        admin_user_view = 'admin:%s_%s_change' % (content_type.app_label,
                                                  content_type.model)
        context = {
            'title': _('Change history: %s') % force_text(obj),
            'action_list': action_list,
            'module_name': capfirst(force_text(opts.verbose_name_plural)),
            'object': obj,
            'root_path': getattr(self.admin_site, 'root_path', None),
            'app_label': app_label,
            'opts': opts,
            'admin_user_view': admin_user_view
        }
        context.update(extra_context or {})
        return render(request, template_name=self.object_history_template,
                      dictionary=context, current_app=self.admin_site.name)

    def history_form_view(self, request, object_id, version_id):
        original_model = self.model
        original_opts = original_model._meta
        history = getattr(self.model,
                          self.model._meta.simple_history_manager_attribute)
        model = history.model
        opts = model._meta
        pk_name = original_opts.pk.attname
        record = get_object_or_404(model, **{
            pk_name: object_id,
            'history_id': version_id,
        })
        obj = record.instance
        obj._state.adding = False

        if not self.has_change_permission(request, obj):
            raise PermissionDenied

        formsets = []
        form_class = self.get_form(request, obj)
        if request.method == 'POST':
            form = form_class(request.POST, request.FILES, instance=obj)
            if form.is_valid():
                form_validated = True
                new_object = self.save_form(request, form, change=True)
            else:
                form_validated = False
                new_object = obj

            if form_validated:
                self.save_model(request, new_object, form, change=True)
                form.save_m2m()

                change_message = self.construct_change_message(request, form,
                                                               formsets)
                self.log_change(request, new_object, change_message)
                return self.response_change(request, new_object)

        else:
            form = form_class(instance=obj)

        admin_form = helpers.AdminForm(
            form,
            self.get_fieldsets(request, obj),
            self.prepopulated_fields,
            self.get_readonly_fields(request, obj),
            model_admin=self,
        )
        media = self.media + admin_form.media

        try:
            model_name = original_opts.module_name
        except AttributeError:
            model_name = original_opts.model_name
        url_triplet = self.admin_site.name, original_opts.app_label, model_name
        content_type_id = ContentType.objects.get_for_model(self.model).id
        context = {
            'title': _('Revert %s') % force_text(obj),
            'adminform': admin_form,
            'object_id': object_id,
            'original': obj,
            'is_popup': False,
            'media': mark_safe(media),
            'errors': helpers.AdminErrorList(form, formsets),
            'app_label': opts.app_label,
            'original_opts': original_opts,
            'changelist_url': reverse('%s:%s_%s_changelist' % url_triplet),
            'change_url': reverse('%s:%s_%s_change' % url_triplet,
                                  args=(obj.pk,)),
            'history_url': reverse('%s:%s_%s_history' % url_triplet,
                                   args=(obj.pk,)),
            # Context variables copied from render_change_form
            'add': False,
            'change': True,
            'has_add_permission': self.has_add_permission(request),
            'has_change_permission': self.has_change_permission(request, obj),
            'has_delete_permission': self.has_delete_permission(request, obj),
            'has_file_field': True,
            'has_absolute_url': False,
            'form_url': '',
            'opts': opts,
            'content_type_id': content_type_id,
            'save_as': self.save_as,
            'save_on_top': self.save_on_top,
            'root_path': getattr(self.admin_site, 'root_path', None),
        }
        return render(request, template_name=self.object_history_form_template,
                      dictionary=context, current_app=self.admin_site.name)

    def save_model(self, request, obj, form, change):
        """Set special model attribute to user for reference after save"""
        obj._history_user = request.user
        super(SimpleHistoryAdmin, self).save_model(request, obj, form, change)
<|endoftext|>


<commit_before># Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt

import binascii
import dis
import marshal
import struct
import sys
import time
import types

def show_pyc_file(fname):
    f = open(fname, "rb")
    magic = f.read(4)
    print("magic %s" % (binascii.hexlify(magic)))
    read_date_and_size = True
    if sys.version_info >= (3, 7):
        # 3.7 added a flags word
        flags = struct.unpack('<L', f.read(4))[0]
        hash_based = flags & 0x01
        check_source = flags & 0x02
        print("flags 0x%08x" % (flags,))
        if hash_based:
            source_hash = f.read(8)
            read_date_and_size = False
    if read_date_and_size:
        moddate = f.read(4)
        modtime = time.asctime(time.localtime(struct.unpack('<L', moddate)[0]))
        print("moddate %s (%s)" % (binascii.hexlify(moddate), modtime))
        if sys.version_info >= (3, 3):
            # 3.3 added another long to the header (size).
            size = f.read(4)
            print("pysize %s (%d)" % (binascii.hexlify(size), struct.unpack('<L', size)[0]))
    code = marshal.load(f)
    show_code(code)

def show_py_file(fname):
    text = open(fname).read().replace('\r\n', '\n')
    show_py_text(text, fname=fname)

def show_py_text(text, fname="<string>"):
    code = compile(text, fname, "exec")
    show_code(code)

CO_FLAGS = [
    ('CO_OPTIMIZED',                0x00001),
    ('CO_NEWLOCALS',                0x00002),
    ('CO_VARARGS',                  0x00004),
    ('CO_VARKEYWORDS',              0x00008),
    ('CO_NESTED',                   0x00010),
    ('CO_GENERATOR',                0x00020),
    ('CO_NOFREE',                   0x00040),
    ('CO_COROUTINE',                0x00080),
    ('CO_ITERABLE_COROUTINE',       0x00100),
    ('CO_ASYNC_GENERATOR',          0x00200),
    ('CO_GENERATOR_ALLOWED',        0x01000),
    ('CO_FUTURE_DIVISION',          0x02000),
    ('CO_FUTURE_ABSOLUTE_IMPORT',   0x04000),
    ('CO_FUTURE_WITH_STATEMENT',    0x08000),
    ('CO_FUTURE_PRINT_FUNCTION',    0x10000),
    ('CO_FUTURE_UNICODE_LITERALS',  0x20000),
    ('CO_FUTURE_BARRY_AS_BDFL',     0x40000),
    ('CO_FUTURE_GENERATOR_STOP',    0x80000),
]

def show_code(code, indent='', number=None):
    label = ""
    if number is not None:
        label = "%d: " % number
    print("%s%scode" % (indent, label))
    indent += '   '
    print("%sname %r" % (indent, code.co_name))
    print("%sargcount %d" % (indent, code.co_argcount))
    print("%snlocals %d" % (indent, code.co_nlocals))
    print("%sstacksize %d" % (indent, code.co_stacksize))
    print("%sflags %04x: %s" % (indent, code.co_flags, flag_words(code.co_flags, CO_FLAGS)))
    show_hex("code", code.co_code, indent=indent)
    dis.disassemble(code)
    print("%sconsts" % indent)
    for i, const in enumerate(code.co_consts):
        if type(const) == types.CodeType:
            show_code(const, indent+'   ', number=i)
        else:
            print("   %s%d: %r" % (indent, i, const))
    print("%snames %r" % (indent, code.co_names))
    print("%svarnames %r" % (indent, code.co_varnames))
    print("%sfreevars %r" % (indent, code.co_freevars))
    print("%scellvars %r" % (indent, code.co_cellvars))
    print("%sfilename %r" % (indent, code.co_filename))
    print("%sfirstlineno %d" % (indent, code.co_firstlineno))
    show_hex("lnotab", code.co_lnotab, indent=indent)

def show_hex(label, h, indent):
    h = binascii.hexlify(h)
    if len(h) < 60:
        print("%s%s %s" % (indent, label, h.decode('ascii')))
    else:
        print("%s%s" % (indent, label))
        for i in range(0, len(h), 60):
            print("%s   %s" % (indent, h[i:i+60].decode('ascii')))

def flag_words(flags, flag_defs):
    words = []
    for word, flag in flag_defs:
        if flag & flags:
            words.append(word)
    return ", ".join(words)

def show_file(fname):
    if fname.endswith('pyc'):
        show_pyc_file(fname)
    elif fname.endswith('py'):
        show_py_file(fname)
    else:
        print("Odd file:", fname)

def main(args):
    if args[0] == '-c':
        show_py_text(" ".join(args[1:]).replace(";", "\n"))
    else:
        for a in args:
            show_file(a)

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>show_pyc: Flags changed in 3.9<commit_after># Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt

"""
Dump the contents of a .pyc file.

The output will only be correct if run with the same version of Python that
produced the .pyc.

"""

import binascii
import dis
import marshal
import struct
import sys
import time
import types

def show_pyc_file(fname):
    f = open(fname, "rb")
    magic = f.read(4)
    print("magic %s" % (binascii.hexlify(magic)))
    read_date_and_size = True
    if sys.version_info >= (3, 7):
        # 3.7 added a flags word
        flags = struct.unpack('<L', f.read(4))[0]
        hash_based = flags & 0x01
        check_source = flags & 0x02
        print("flags 0x%08x" % (flags,))
        if hash_based:
            source_hash = f.read(8)
            read_date_and_size = False
    if read_date_and_size:
        moddate = f.read(4)
        modtime = time.asctime(time.localtime(struct.unpack('<L', moddate)[0]))
        print("moddate %s (%s)" % (binascii.hexlify(moddate), modtime))
        if sys.version_info >= (3, 3):
            # 3.3 added another long to the header (size).
            size = f.read(4)
            print("pysize %s (%d)" % (binascii.hexlify(size), struct.unpack('<L', size)[0]))
    code = marshal.load(f)
    show_code(code)

def show_py_file(fname):
    text = open(fname).read().replace('\r\n', '\n')
    show_py_text(text, fname=fname)

def show_py_text(text, fname="<string>"):
    code = compile(text, fname, "exec")
    show_code(code)

CO_FLAGS = [
    ('CO_OPTIMIZED',                0x00001),
    ('CO_NEWLOCALS',                0x00002),
    ('CO_VARARGS',                  0x00004),
    ('CO_VARKEYWORDS',              0x00008),
    ('CO_NESTED',                   0x00010),
    ('CO_GENERATOR',                0x00020),
    ('CO_NOFREE',                   0x00040),
    ('CO_COROUTINE',                0x00080),
    ('CO_ITERABLE_COROUTINE',       0x00100),
    ('CO_ASYNC_GENERATOR',          0x00200),
    ('CO_GENERATOR_ALLOWED',        0x01000),
]

if sys.version_info < (3, 9):
    CO_FLAGS += [
        ('CO_FUTURE_DIVISION',          0x02000),
        ('CO_FUTURE_ABSOLUTE_IMPORT',   0x04000),
        ('CO_FUTURE_WITH_STATEMENT',    0x08000),
        ('CO_FUTURE_PRINT_FUNCTION',    0x10000),
        ('CO_FUTURE_UNICODE_LITERALS',  0x20000),
        ('CO_FUTURE_BARRY_AS_BDFL',     0x40000),
        ('CO_FUTURE_GENERATOR_STOP',    0x80000),
    ]
else:
    CO_FLAGS += [
        ('CO_FUTURE_DIVISION',          0x0020000),
        ('CO_FUTURE_ABSOLUTE_IMPORT',   0x0040000),
        ('CO_FUTURE_WITH_STATEMENT',    0x0080000),
        ('CO_FUTURE_PRINT_FUNCTION',    0x0100000),
        ('CO_FUTURE_UNICODE_LITERALS',  0x0200000),
        ('CO_FUTURE_BARRY_AS_BDFL',     0x0400000),
        ('CO_FUTURE_GENERATOR_STOP',    0x0800000),
        ('CO_FUTURE_ANNOTATIONS',       0x1000000),
    ]

def show_code(code, indent='', number=None):
    label = ""
    if number is not None:
        label = "%d: " % number
    print("%s%scode" % (indent, label))
    indent += '   '
    print("%sname %r" % (indent, code.co_name))
    print("%sargcount %d" % (indent, code.co_argcount))
    print("%snlocals %d" % (indent, code.co_nlocals))
    print("%sstacksize %d" % (indent, code.co_stacksize))
    print("%sflags %04x: %s" % (indent, code.co_flags, flag_words(code.co_flags, CO_FLAGS)))
    show_hex("code", code.co_code, indent=indent)
    dis.disassemble(code)
    print("%sconsts" % indent)
    for i, const in enumerate(code.co_consts):
        if type(const) == types.CodeType:
            show_code(const, indent+'   ', number=i)
        else:
            print("   %s%d: %r" % (indent, i, const))
    print("%snames %r" % (indent, code.co_names))
    print("%svarnames %r" % (indent, code.co_varnames))
    print("%sfreevars %r" % (indent, code.co_freevars))
    print("%scellvars %r" % (indent, code.co_cellvars))
    print("%sfilename %r" % (indent, code.co_filename))
    print("%sfirstlineno %d" % (indent, code.co_firstlineno))
    show_hex("lnotab", code.co_lnotab, indent=indent)

def show_hex(label, h, indent):
    h = binascii.hexlify(h)
    if len(h) < 60:
        print("%s%s %s" % (indent, label, h.decode('ascii')))
    else:
        print("%s%s" % (indent, label))
        for i in range(0, len(h), 60):
            print("%s   %s" % (indent, h[i:i+60].decode('ascii')))

def flag_words(flags, flag_defs):
    words = []
    for word, flag in flag_defs:
        if flag & flags:
            words.append(word)
    return ", ".join(words)

def show_file(fname):
    if fname.endswith('pyc'):
        show_pyc_file(fname)
    elif fname.endswith('py'):
        show_py_file(fname)
    else:
        print("Odd file:", fname)

def main(args):
    if args[0] == '-c':
        show_py_text(" ".join(args[1:]).replace(";", "\n"))
    else:
        for a in args:
            show_file(a)

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before>"""
Yandex OpenID support.

This contribution adds support for Yandex.ru OpenID service in the form
openid.yandex.ru/user. Username is retrieved from the identity url.

If username is not specified, OpenID 2.0 url used for authentication.
"""
import logging
logger = logging.getLogger(__name__)

import urlparse

from urllib import urlencode, unquote
from urllib2 import Request, urlopen, HTTPError

from django.conf import settings
import xml.dom.minidom

from social_auth.backends import OpenIDBackend, OpenIdAuth, USERNAME, OAuthBackend, BaseOAuth2

# Yandex conf
YANDEX_URL = 'http://openid.yandex.ru/%s'
YANDEX_USER_FIELD = 'openid_ya_user'
YANDEX_OID_2_URL = 'http://yandex.ru'

EXPIRES_NAME = getattr(settings, 'SOCIAL_AUTH_EXPIRATION', 'expires')

class YandexBackend(OpenIDBackend):
    """Yandex OpenID authentication backend"""
    name = 'yandex'

    def get_user_details(self, response):
        """Generate username from identity url"""
        values = super(YandexBackend, self).get_user_details(response)
        values[USERNAME] = values.get(USERNAME) or \
                           urlparse.urlsplit(response.identity_url)\
                                   .path.strip('/')

        values['email'] = values.get('email') or ''

        return values

class YandexOAuth2Backend(OAuthBackend):
    """Yandex OAuth2 authentication backend"""
    name = 'yandex-oauth2'

    def get_user_id(self, details, response):
        """Return user unique id provided by Yandex"""
        return int(response['id'])

    def get_user_details(self, response):
        """Return user details from Yandex request"""

        name = unquote(response['name'])
        first_name = ''
        last_name = ''

        if ' ' in name:
            last_name, first_name = name.split(' ')
            name = first_name
        else:
            first_name = name

        values = { USERNAME: name, 'email': '',
                   'first_name': first_name, 'last_name': last_name}

        return values

class YandexAuth(OpenIdAuth):
    """Yandex OpenID authentication"""
    AUTH_BACKEND = YandexBackend

    def openid_url(self):
        """Returns Yandex authentication URL"""
        if YANDEX_USER_FIELD not in self.data:
            return YANDEX_OID_2_URL
        else:
            return YANDEX_URL % self.data[YANDEX_USER_FIELD]

class YandexOAuth2(BaseOAuth2):
    """Yandex OAuth2 support
       See http://api.yandex.ru/oauth/doc/dg/concepts/About.xml for details"""
    AUTH_BACKEND = YandexOAuth2Backend
    AUTHORIZATION_URL = 'https://oauth.yandex.ru/authorize'
    ACCESS_TOKEN_URL = 'https://oauth.yandex.ru/token'
    SETTINGS_KEY_NAME = 'YANDEX_OAUTH2_CLIENT_KEY'
    SETTINGS_SECRET_NAME = 'YANDEX_OAUTH2_CLIENT_SECRET'

    def get_scope(self):
        return [] # Yandex does not allow custom scope

    def auth_complete(self, *args, **kwargs):
        try:
            auth_result = super(YandexOAuth2, self).auth_complete(*args, **kwargs)
        except HTTPError: # Returns HTTPError 400 if cancelled
            raise ValueError('Authentication cancelled')

        return auth_result

    def user_data(self, access_token):
        """Return user data from Yandex REST API specified in settings"""
        params = urlencode({'oauth_token': access_token, 'text': 1, 'format': 'xml'})
        request = Request(settings.YANDEX_OAUTH2_API_URL + '?' + params)

        try:
            dom = xml.dom.minidom.parseString(urlopen(request).read())

            id = getNodeText(dom, "id")
            if "/" in id:
                id = id.split("/")[-1]

            name = getNodeText(dom, "name")

            links = getNodesWithAttribute(dom, "link", {"rel": "userpic"})
            userpic = links[0].getAttribute("href") if links else ""

            return {"id": id, "name": name, "userpic": userpic, "access_token": access_token}
        except (TypeError, KeyError, IOError, ValueError, IndexError):
            logger.error('Could not load data from Yandex.', exc_info=True, extra=dict(data=params))
            return None

def getNodeText(dom, nodeName):
    nodelist = dom.getElementsByTagName(nodeName)[0].childNodes

    rc = []
    for node in nodelist:
        if node.nodeType == node.TEXT_NODE:
            rc.append(node.data)
    return ''.join(rc)

def getNodesWithAttribute(dom, nodeName, attrDict):
    nodelist = dom.getElementsByTagName(nodeName)
    found = []

    for node in nodelist:
        for key, value in attrDict.items():
            if node.hasAttribute(key) and node.getAttribute(key) == value:
                found.append(node)

    return found

# Backend definition
BACKENDS = {
    'yandex': YandexAuth,
    'yandex-oauth2': YandexOAuth2
}
<commit_msg>Yandex OAuth2: Token moved to Authorization header<commit_after>"""
Yandex OpenID support.

This contribution adds support for Yandex.ru OpenID service in the form
openid.yandex.ru/user. Username is retrieved from the identity url.

If username is not specified, OpenID 2.0 url used for authentication.
"""
import logging
logger = logging.getLogger(__name__)

import urlparse

from urllib import urlencode, unquote
from urllib2 import Request, urlopen, HTTPError

from django.conf import settings
import xml.dom.minidom

from social_auth.backends import OpenIDBackend, OpenIdAuth, USERNAME, OAuthBackend, BaseOAuth2

# Yandex conf
YANDEX_URL = 'http://openid.yandex.ru/%s'
YANDEX_USER_FIELD = 'openid_ya_user'
YANDEX_OID_2_URL = 'http://yandex.ru'

EXPIRES_NAME = getattr(settings, 'SOCIAL_AUTH_EXPIRATION', 'expires')

class YandexBackend(OpenIDBackend):
    """Yandex OpenID authentication backend"""
    name = 'yandex'

    def get_user_details(self, response):
        """Generate username from identity url"""
        values = super(YandexBackend, self).get_user_details(response)
        values[USERNAME] = values.get(USERNAME) or \
                           urlparse.urlsplit(response.identity_url)\
                                   .path.strip('/')

        values['email'] = values.get('email') or ''

        return values

class YandexOAuth2Backend(OAuthBackend):
    """Yandex OAuth2 authentication backend"""
    name = 'yandex-oauth2'

    def get_user_id(self, details, response):
        """Return user unique id provided by Yandex"""
        return int(response['id'])

    def get_user_details(self, response):
        """Return user details from Yandex request"""

        name = unquote(response['name'])
        first_name = ''
        last_name = ''

        if ' ' in name:
            last_name, first_name = name.split(' ')
            name = first_name
        else:
            first_name = name

        values = { USERNAME: name, 'email': '',
                   'first_name': first_name, 'last_name': last_name}

        return values

class YandexAuth(OpenIdAuth):
    """Yandex OpenID authentication"""
    AUTH_BACKEND = YandexBackend

    def openid_url(self):
        """Returns Yandex authentication URL"""
        if YANDEX_USER_FIELD not in self.data:
            return YANDEX_OID_2_URL
        else:
            return YANDEX_URL % self.data[YANDEX_USER_FIELD]

class YandexOAuth2(BaseOAuth2):
    """Yandex OAuth2 support
       See http://api.yandex.ru/oauth/doc/dg/concepts/About.xml for details"""
    AUTH_BACKEND = YandexOAuth2Backend
    AUTHORIZATION_URL = 'https://oauth.yandex.ru/authorize'
    ACCESS_TOKEN_URL = 'https://oauth.yandex.ru/token'
    SETTINGS_KEY_NAME = 'YANDEX_OAUTH2_CLIENT_KEY'
    SETTINGS_SECRET_NAME = 'YANDEX_OAUTH2_CLIENT_SECRET'

    def get_scope(self):
        return [] # Yandex does not allow custom scope

    def auth_complete(self, *args, **kwargs):
        try:
            auth_result = super(YandexOAuth2, self).auth_complete(*args, **kwargs)
        except HTTPError: # Returns HTTPError 400 if cancelled
            raise ValueError('Authentication cancelled')

        return auth_result

    def user_data(self, access_token):
        """Return user data from Yandex REST API specified in settings"""
        params = urlencode({'text': 1, 'format': 'xml'})
        request = Request(settings.YANDEX_OAUTH2_API_URL + '?' + params, headers={'Authorization': "OAuth " + access_token })

        try:
            dom = xml.dom.minidom.parseString(urlopen(request).read())

            id = getNodeText(dom, "id")
            if "/" in id:
                id = id.split("/")[-1]

            name = getNodeText(dom, "name")

            links = getNodesWithAttribute(dom, "link", {"rel": "userpic"})
            userpic = links[0].getAttribute("href") if links else ""

            return {"id": id, "name": name, "userpic": userpic, "access_token": access_token}
        except (TypeError, KeyError, IOError, ValueError, IndexError):
            logger.error('Could not load data from Yandex.', exc_info=True, extra=dict(data=params))
            return None

def getNodeText(dom, nodeName):
    nodelist = dom.getElementsByTagName(nodeName)[0].childNodes

    rc = []
    for node in nodelist:
        if node.nodeType == node.TEXT_NODE:
            rc.append(node.data)

    return ''.join(rc)

def getNodesWithAttribute(dom, nodeName, attrDict):
    nodelist = dom.getElementsByTagName(nodeName)
    found = []

    for node in nodelist:
        for key, value in attrDict.items():
            if node.hasAttribute(key):
                if value and node.getAttribute(key) != value:
                    continue
                found.append(node)

    return found

# Backend definition
BACKENDS = {
    'yandex': YandexAuth,
    'yandex-oauth2': YandexOAuth2
}
<|endoftext|>


<commit_before>import logging
import shlex
import time

from django.db import models
from django.utils.encoding import smart_str, smart_unicode
from google.appengine.ext import db
from google.appengine.ext.deferred import defer
from django.conf import settings

"""
    REMAINING TO DO!

    1. Partial matches. These should be recorded as new Index instances, with an FK to the full term being indexes.
       Partials should only be recorded for between 4, and len(original_term) - 1 characters. Partial matches should be much more highly scored,
       the lower the match, the more the score should be
    2. Cross-join indexing  e.g. book__title on an Author.
    3. Field matches. e.g "id:1234 field1:banana". This should match any other words using indexes, but only return matches that match the field lookups
"""

QUEUE_FOR_INDEXING = getattr(settings, "QUEUE_FOR_INDEXING", "default")

def _do_index(instance, fields_to_index):
    def get_data_from_field(field_, instance_):
        lookups = field_.split("__")
        value = instance
        for lookup in lookups:
            if value is None:
                continue
            value = getattr(value, lookup)

            if "RelatedManager" in value.__class__.__name__:
                if lookup == lookups[-2]:
                    return [ getattr(x, lookups[-1]) for x in value.all() ]
                else:
                    raise TypeError("You can only index one level of related object")

            elif hasattr(value, "__iter__"):
                if lookup == lookups[-1]:
                    return value
                else:
                    raise TypeError("You can only index one level of iterable")

        return [ value ]

    for field in fields_to_index:
        texts = get_data_from_field(field, instance)
        for text in texts:
            if text is None:
                continue

            text = smart_unicode(text)
            text = text.lower() #Normalize

            words = text.split(" ") #Split on whitespace

            #Build up combinations of adjacent words
            for i in xrange(0, len(words)):
                for j in xrange(1, 5):
                    term_words = words[i:i+j]

                    if len(term_words) != j:
                        break

                    term = u" ".join(term_words)

                    if not term.strip(): continue

                    @db.transactional(xg=True)
                    def txn(term_):
                        logging.info("Indexing: '%s', %s", term_, type(term_))
                        term_count = text.count(term_)

                        Index.objects.create(
                            iexact=term_,
                            instance_db_table=instance._meta.db_table,
                            instance_pk=instance.pk,
                            occurances=term_count
                        )
                        counter, created = GlobalOccuranceCount.objects.get_or_create(pk=term_)
                        counter.count += term_count
                        counter.save()

                    while True:
                        try:
                            txn(term)
                            break
                        except db.TransactionFailedError:
                            logging.warning("Transaction collision, retrying!")
                            time.sleep(1)
                            continue

def _unindex_then_reindex(instance, fields_to_index):
    unindex_instance(instance)
    _do_index(instance, fields_to_index)

def index_instance(instance, fields_to_index, defer_index=True):
    """
        If we are in a transaction, we must defer the unindex and reindex :(
    """

    if db.is_in_transaction() or defer_index:
        defer(_unindex_then_reindex, instance, fields_to_index, _queue=QUEUE_FOR_INDEXING)
    else:
        unindex_instance(instance)
        _do_index(instance, fields_to_index)

def unindex_instance(instance):
    indexes = Index.objects.filter(instance_db_table=instance._meta.db_table, instance_pk=instance.pk).all()
    for index in indexes:

        @db.transactional(xg=True)
        def txn(_index):
            count = GlobalOccuranceCount.objects.get(pk=_index.iexact)
            count.count -= _index.occurances
            count.save()
            _index.delete()

        try:
            while True:
                try:
                    txn(index)
                    break
                except db.TransactionFailedError:
                    logging.warning("Transaction collision, retrying!")
                    time.sleep(1)
                    continue
        except GlobalOccuranceCount.DoesNotExist:
            logging.warning("A GlobalOccuranceCount for Index: %s does not exist, ignoring", index.pk)
            continue

def parse_terms(search_string):
    return shlex.split(smart_str(search_string.lower()))

def search(model_class, search_string, per_page=50, current_page=1, total_pages=10, **filters):
    terms = parse_terms(search_string)

    #Get all matching terms
    matching_terms = dict(GlobalOccuranceCount.objects.filter(pk__in=terms).values_list('pk', 'count'))
    matches = Index.objects.filter(iexact__in=terms, instance_db_table=model_class._meta.db_table).all()

    instance_weights = {}

    for match in matches:
        instance_weights.setdefault(match.instance_pk, []).append(matching_terms[match.iexact])

    final_weights = []
    for k, v in instance_weights.items():
        """
            This is where we rank the results. Lower scores are better. Scores are based
            on the commonality of the word. More matches are rewarded, but not too much so
            that rarer terms still have a chance.

            Examples for n matches:

            1 = 1 + (0 * 0.5) = 1    -> scores / 1
            2 = 2 + (1 * 0.5) = 2.5  -> scores / 2.5 (rather than 2)
            3 = 3 + (2 * 0.5) = 4    -> scores / 4 (rather than 3)
        """

        n = float(len(v))
        final_weights.append((sum(v) / (n + ((n-1) * 0.5)), k))

    final_weights.sort()

    #Restrict to the max possible
    final_weights = final_weights[:total_pages*per_page]

    #Restrict to the page
    offset = ((current_page - 1) * per_page)
    final_weights = final_weights[offset:offset + per_page]

    order = {}
    for index, (score, pk) in enumerate(final_weights):
        order[pk] = index

    sorted_results = [None] * len(order.keys())

    queryset = model_class.objects.all()
    if filters:
        queryset = queryset.filter(**filters)

    results = queryset.filter(pk__in=order.keys())
    for result in results:
        position = order[result.pk]
        sorted_results[position] = result

    return sorted_results

class GlobalOccuranceCount(models.Model):
    id = models.CharField(max_length=1024, primary_key=True)
    count = models.PositiveIntegerField(default=0)

class Index(models.Model):
    iexact = models.CharField(max_length=1024)
    instance_db_table = models.CharField(max_length=1024)
    instance_pk = models.PositiveIntegerField(default=0)
    occurances = models.PositiveIntegerField(default=0)

    class Meta:
        unique_together = [
            ('iexact', 'instance_db_table', 'instance_pk')
        ]

from django.dispatch import receiver
from django.db.models.signals import post_save, pre_delete

@receiver(post_save)
def post_save_index(sender, instance, created, raw, *args, **kwargs):
    if getattr(instance, "Search", None):
        fields_to_index = getattr(instance.Search, "fields", [])
        if fields_to_index:
            index_instance(instance, fields_to_index, defer_index=not raw) #Don't defer if we are loading from a fixture

@receiver(pre_delete)
def pre_delete_unindex(sender, instance, using, *args, **kwarg):
    if getattr(instance, "Search", None):
        unindex_instance(instance)
<commit_msg>Add an update function to the global occurrance count<commit_after>import logging
import shlex
import time

from django.db import models
from django.utils.encoding import smart_str, smart_unicode
from google.appengine.ext import db
from google.appengine.ext.deferred import defer
from django.conf import settings

"""
    REMAINING TO DO!

    1. Partial matches. These should be recorded as new Index instances, with an FK to the full term being indexes.
       Partials should only be recorded for between 4, and len(original_term) - 1 characters. Partial matches should be much more highly scored,
       the lower the match, the more the score should be
    2. Cross-join indexing  e.g. book__title on an Author.
    3. Field matches. e.g "id:1234 field1:banana". This should match any other words using indexes, but only return matches that match the field lookups
"""

QUEUE_FOR_INDEXING = getattr(settings, "QUEUE_FOR_INDEXING", "default")

def _do_index(instance, fields_to_index):
    def get_data_from_field(field_, instance_):
        lookups = field_.split("__")
        value = instance
        for lookup in lookups:
            if value is None:
                continue
            value = getattr(value, lookup)

            if "RelatedManager" in value.__class__.__name__:
                if lookup == lookups[-2]:
                    return [ getattr(x, lookups[-1]) for x in value.all() ]
                else:
                    raise TypeError("You can only index one level of related object")

            elif hasattr(value, "__iter__"):
                if lookup == lookups[-1]:
                    return value
                else:
                    raise TypeError("You can only index one level of iterable")

        return [ value ]

    for field in fields_to_index:
        texts = get_data_from_field(field, instance)
        for text in texts:
            if text is None:
                continue

            text = smart_unicode(text)
            text = text.lower() #Normalize

            words = text.split(" ") #Split on whitespace

            #Build up combinations of adjacent words
            for i in xrange(0, len(words)):
                for j in xrange(1, 5):
                    term_words = words[i:i+j]

                    if len(term_words) != j:
                        break

                    term = u" ".join(term_words)

                    if not term.strip(): continue

                    @db.transactional(xg=True)
                    def txn(term_):
                        logging.info("Indexing: '%s', %s", term_, type(term_))
                        term_count = text.count(term_)

                        Index.objects.create(
                            iexact=term_,
                            instance_db_table=instance._meta.db_table,
                            instance_pk=instance.pk,
                            occurances=term_count
                        )
                        counter, created = GlobalOccuranceCount.objects.get_or_create(pk=term_)
                        counter.count += term_count
                        counter.save()

                    while True:
                        try:
                            txn(term)
                            break
                        except db.TransactionFailedError:
                            logging.warning("Transaction collision, retrying!")
                            time.sleep(1)
                            continue

def _unindex_then_reindex(instance, fields_to_index):
    unindex_instance(instance)
    _do_index(instance, fields_to_index)

def index_instance(instance, fields_to_index, defer_index=True):
    """
        If we are in a transaction, we must defer the unindex and reindex :(
    """

    if db.is_in_transaction() or defer_index:
        defer(_unindex_then_reindex, instance, fields_to_index, _queue=QUEUE_FOR_INDEXING)
    else:
        unindex_instance(instance)
        _do_index(instance, fields_to_index)

def unindex_instance(instance):
    indexes = Index.objects.filter(instance_db_table=instance._meta.db_table, instance_pk=instance.pk).all()
    for index in indexes:

        @db.transactional(xg=True)
        def txn(_index):
            count = GlobalOccuranceCount.objects.get(pk=_index.iexact)
            count.count -= _index.occurances
            count.save()
            _index.delete()

        try:
            while True:
                try:
                    txn(index)
                    break
                except db.TransactionFailedError:
                    logging.warning("Transaction collision, retrying!")
                    time.sleep(1)
                    continue
        except GlobalOccuranceCount.DoesNotExist:
            logging.warning("A GlobalOccuranceCount for Index: %s does not exist, ignoring", index.pk)
            continue

def parse_terms(search_string):
    return shlex.split(smart_str(search_string.lower()))

def search(model_class, search_string, per_page=50, current_page=1, total_pages=10, **filters):
    terms = parse_terms(search_string)

    #Get all matching terms
    matching_terms = dict(GlobalOccuranceCount.objects.filter(pk__in=terms).values_list('pk', 'count'))
    matches = Index.objects.filter(iexact__in=terms, instance_db_table=model_class._meta.db_table).all()

    instance_weights = {}

    for match in matches:
        instance_weights.setdefault(match.instance_pk, []).append(matching_terms[match.iexact])

    final_weights = []
    for k, v in instance_weights.items():
        """
            This is where we rank the results. Lower scores are better. Scores are based
            on the commonality of the word. More matches are rewarded, but not too much so
            that rarer terms still have a chance.

            Examples for n matches:

            1 = 1 + (0 * 0.5) = 1    -> scores / 1
            2 = 2 + (1 * 0.5) = 2.5  -> scores / 2.5 (rather than 2)
            3 = 3 + (2 * 0.5) = 4    -> scores / 4 (rather than 3)
        """

        n = float(len(v))
        final_weights.append((sum(v) / (n + ((n-1) * 0.5)), k))

    final_weights.sort()

    #Restrict to the max possible
    final_weights = final_weights[:total_pages*per_page]

    #Restrict to the page
    offset = ((current_page - 1) * per_page)
    final_weights = final_weights[offset:offset + per_page]

    order = {}
    for index, (score, pk) in enumerate(final_weights):
        order[pk] = index

    sorted_results = [None] * len(order.keys())

    queryset = model_class.objects.all()
    if filters:
        queryset = queryset.filter(**filters)

    results = queryset.filter(pk__in=order.keys())
    for result in results:
        position = order[result.pk]
        sorted_results[position] = result

    return sorted_results

class GlobalOccuranceCount(models.Model):
    id = models.CharField(max_length=1024, primary_key=True)
    count = models.PositiveIntegerField(default=0)

    def update(self):
        count = sum(Index.objects.filter(iexact=self.id).values_list('occurances', flat=True))

        @db.transactional
        def txn():
            goc = GlobalOccuranceCount.objects.get(pk=self.id)
            goc.count = count
            goc.save()

        while True:
            try:
                txn()
                break
            except db.TransactionFailedError:
                time.sleep(1)
                continue

class Index(models.Model):
    iexact = models.CharField(max_length=1024)
    instance_db_table = models.CharField(max_length=1024)
    instance_pk = models.PositiveIntegerField(default=0)
    occurances = models.PositiveIntegerField(default=0)

    class Meta:
        unique_together = [
            ('iexact', 'instance_db_table', 'instance_pk')
        ]

from django.dispatch import receiver
from django.db.models.signals import post_save, pre_delete

@receiver(post_save)
def post_save_index(sender, instance, created, raw, *args, **kwargs):
    if getattr(instance, "Search", None):
        fields_to_index = getattr(instance.Search, "fields", [])
        if fields_to_index:
            index_instance(instance, fields_to_index, defer_index=not raw) #Don't defer if we are loading from a fixture

@receiver(pre_delete)
def pre_delete_unindex(sender, instance, using, *args, **kwarg):
    if getattr(instance, "Search", None):
        unindex_instance(instance)
<|endoftext|>


<commit_before># Copyright (c) 2013-2015 Unidata.
# Distributed under the terms of the MIT License.
# SPDX-License-Identifier: MIT

from __future__ import print_function
import itertools
import logging
import zlib

from collections import OrderedDict

import numpy as np

from . import ncStream_pb2 as stream  # noqa

MAGIC_HEADER = b'\xad\xec\xce\xda'
MAGIC_DATA = b'\xab\xec\xce\xba'
MAGIC_DATA2 = b'\xab\xeb\xbe\xba'
MAGIC_VDATA = b'\xab\xef\xfe\xba'
MAGIC_VEND = b'\xed\xef\xfe\xda'
MAGIC_ERR = b'\xab\xad\xba\xda'

log = logging.getLogger('siphon.ncstream')
log.addHandler(logging.StreamHandler())  # Python 2.7 needs a handler set
log.setLevel(logging.WARNING)

#
# NCStream handling
#
def read_ncstream_data(fobj):
    data = read_proto_object(fobj, stream.Data)
    if data.dataType in (stream.STRING, stream.OPAQUE) or data.vdata:
        log.debug('Reading string/opaque/vlen')
        num_obj = read_var_int(fobj)
        log.debug('Num objects: %d', num_obj)
        blocks = [read_block(fobj) for _ in range(num_obj)]
        if data.dataType == stream.STRING:
            blocks = [b.decode('utf-8', errors='ignore') for b in blocks]

        # Again endian isn't coded properly
        dt = data_type_to_numpy(data.dataType).newbyteorder('>')
        if data.vdata:
            return np.array([np.frombuffer(b, dtype=dt) for b in blocks])
        else:
            return np.array(blocks, dtype=dt)
    elif data.dataType in _dtypeLookup:
        log.debug('Reading array data')
        bin_data = read_block(fobj)
        log.debug('Binary data: %s', bin_data)

        # Hard code to big endian for now since it's not encoded correctly
        dt = data_type_to_numpy(data.dataType).newbyteorder('>')

        # Handle decompressing the bytes
        if data.compress == stream.DEFLATE:
            bin_data = zlib.decompress(bin_data)
            assert len(bin_data) == data.uncompressedSize
        elif data.compress != stream.NONE:
            raise NotImplementedError('Compression type {0} not implemented!'.format(
                data.compress))

        # Turn bytes into an array
        return reshape_array(data, np.frombuffer(bin_data, dtype=dt))
    elif data.dataType == stream.STRUCTURE:
        sd = read_proto_object(fobj, stream.StructureData)

        # Make a datatype appropriate to the rows of struct
        endian = '>' if data.bigend else '<'
        dt = np.dtype([(endian, np.void, sd.rowLength)])

        # Turn bytes into an array
        return reshape_array(data, np.frombuffer(sd.data, dtype=dt))
    elif data.dataType == stream.SEQUENCE:
        log.debug('Reading sequence')
        blocks = []
        magic = read_magic(fobj)
        while magic != MAGIC_VEND:
            if magic == MAGIC_VDATA:
                log.error('Bad magic for struct/seq data!')
            blocks.append(read_proto_object(fobj, stream.StructureData))
            magic = read_magic(fobj)
        return data, blocks
    else:
        raise NotImplementedError("Don't know how to handle data type: {0}".format(
            data.dataType))

def read_ncstream_data2(fobj):
    data = read_proto_object(fobj, stream.DataCol)
    return datacol_to_array(data)

def read_ncstream_err(fobj):
    err = read_proto_object(fobj, stream.Error)
    raise RuntimeError(err.message)

ncstream_table = {MAGIC_HEADER: lambda f: read_proto_object(f, stream.Header),
                  MAGIC_DATA: read_ncstream_data,
                  MAGIC_DATA2: read_ncstream_data2,
                  MAGIC_ERR: read_ncstream_err}

def read_ncstream_messages(fobj):
    return read_messages(fobj, ncstream_table)

def read_messages(fobj, magic_table):
    messages = []

    while True:
        magic = read_magic(fobj)
        if not magic:
            break

        func = magic_table.get(magic)
        if func is not None:
            messages.append(func(fobj))
        else:
            log.error('Unknown magic: ' + str(' '.join('%02x' % b for b in magic)))

    return messages

def read_proto_object(fobj, klass):
    log.debug('%s chunk', klass.__name__)
    obj = klass()
    obj.ParseFromString(read_block(fobj))
    log.debug('Header: %s', str(obj))
    return obj

def read_magic(fobj):
    """Read a magic bytes.

    Parameters
    ----------
    fobj : file-like object
        The file to read from.

    Returns
    -------
    bytes
        magic byte sequence read
    """
    return fobj.read(4)

def read_block(fobj):
    """Read a block.

    Reads a block from a file object by first reading the number of bytes to read, which must
    be encoded as a variable-byte length integer.

    Parameters
    ----------
    fobj : file-like object
        The file to read from.

    Returns
    -------
    bytes
        block of bytes read
    """
    num = read_var_int(fobj)
    log.debug('Next block: %d bytes', num)
    return fobj.read(num)

def process_vlen(data_header, array):
    """Process vlen coming back from NCStream v2

    This takes the array of values and slices into an object array, with entries containing
    the appropriate pieces of the original array. Sizes are controlled by the passed in
    `data_header`.

    Parameters
    ----------
    data_header : Header
    array : :class:`numpy.ndarray`

    Returns
    -------
    ndarray
        object array containing sub-sequences from the original primitive array
    """
    source = iter(array)
    return np.array([np.fromiter(itertools.islice(source, size), dtype=array.dtype)
                     for size in data_header.vlens])

def datacol_to_array(datacol):
    """Convert DataCol from NCStream v2 into an array with appropriate type

    Depending on the data type specified, this extracts data from the appropriate members
    and packs into a :class:`numpy.ndarray`, recursing as necessary for compound data types.

    Parameters
    ----------
    datacol : DataCol

    Returns
    -------
    ndarray
        array containing extracted data
    """
    if datacol.dataType == stream.STRING:
        arr = np.array(datacol.stringdata, dtype=np.object)
    elif datacol.dataType == stream.OPAQUE:
        arr = np.array(datacol.opaquedata, dtype=np.object)
    elif datacol.dataType == stream.STRUCTURE:
        members = OrderedDict((mem.name, datacol_to_array(mem))
                              for mem in datacol.structdata.memberData)
        log.debug('Struct members:\n%s', str(members))

        # str() around name necessary because protobuf gives unicode names, but dtype doesn't
        # support them on Python 2
        dt = np.dtype([(str(name), arr.dtype) for name, arr in members.items()])
        log.debug('Struct dtype: %s', str(dt))

        arr = np.empty((datacol.nelems,), dtype=dt)
        for name, arr_data in members.items():
            arr[name] = arr_data
    else:
        # Make an appropriate datatype
        endian = '>' if datacol.bigend else '<'
        dt = data_type_to_numpy(datacol.dataType).newbyteorder(endian)

        # Turn bytes into an array
        arr = np.frombuffer(datacol.primdata, dtype=dt)
        if arr.size != datacol.nelems:
            log.warning('Array size %d does not agree with nelems %d',
                        arr.size, datacol.nelems)
        if datacol.isVlen:
            arr = process_vlen(datacol, arr)
            if arr.dtype == np.object_:
                arr = reshape_array(datacol, arr)
            else:
                # In this case, the array collapsed, need different resize that
                # correctly sizes from elements
                shape = tuple(r.size for r in datacol.section.range) + (datacol.vlens[0],)
                arr = arr.reshape(*shape)
        else:
            arr = reshape_array(datacol, arr)
    return arr

def reshape_array(data_header, array):
    """Extracts the appropriate array shape from the header

    Can handle taking a data header and either bytes containing data or a StructureData
    instance, which will have binary data as well as some additional information.

    Parameters
    ----------
    array : :class:`numpy.ndarray`
    data_header : Data
    """
    shape = tuple(r.size for r in data_header.section.range)
    if shape:
        return array.reshape(*shape)
    else:
        return array

# STRUCTURE = 8;
# SEQUENCE = 9;
_dtypeLookup = {stream.CHAR: 'S1', stream.BYTE: 'b', stream.SHORT: 'i2',
                stream.INT: 'i4', stream.LONG: 'i8', stream.FLOAT: 'f4',
                stream.DOUBLE: 'f8', stream.STRING: 'O',
                stream.ENUM1: 'B', stream.ENUM2: 'u2', stream.ENUM4: 'u4',
                stream.OPAQUE: 'O', stream.UBYTE: 'B', stream.USHORT: 'u2',
                stream.UINT: 'u4', stream.ULONG: 'u8'}

def data_type_to_numpy(datatype, unsigned=False):
    basic_type = _dtypeLookup[datatype]

    if datatype in (stream.STRING, stream.OPAQUE):
        return np.dtype(basic_type)

    if unsigned:
        basic_type = basic_type.replace('i', 'u')
    return np.dtype('=' + basic_type)

def struct_to_dtype(struct):
    """Convert a Structure specification to a numpy structured dtype."""
    # str() around name necessary because protobuf gives unicode names, but dtype doesn't
    # support them on Python 2
    fields = [(str(var.name), data_type_to_numpy(var.dataType, var.unsigned))
              for var in struct.vars]
    for s in struct.structs:
        fields.append((str(s.name), struct_to_dtype(s)))

    log.debug('Structure fields: %s', fields)
    dt = np.dtype(fields)
    return dt

def unpack_variable(var):
    # If we actually get a structure instance, handle turning that into a variable
    if var.dataType == stream.STRUCTURE:
        return None, struct_to_dtype(var), 'Structure'
    elif var.dataType == stream.SEQUENCE:
        log.warning('Sequence support not implemented!')

    dt = data_type_to_numpy(var.dataType, var.unsigned)
    if var.dataType == stream.OPAQUE:
        type_name = 'opaque'
    elif var.dataType == stream.STRING:
        type_name = 'string'
    else:
        type_name = dt.name

    if var.data:
        log.debug('Storing variable data: %s %s', dt, var.data)
        if var.dataType is str:
            data = var.data
        else:
            # Always sent big endian
            data = np.fromstring(var.data, dtype=dt.newbyteorder('>'))
    else:
        data = None

    return data, dt, type_name

_attrConverters = {stream.Attribute.BYTE: np.dtype('>b'),
                   stream.Attribute.SHORT: np.dtype('>i2'),
                   stream.Attribute.INT: np.dtype('>i4'),
                   stream.Attribute.LONG: np.dtype('>i8'),
                   stream.Attribute.FLOAT: np.dtype('>f4'),
                   stream.Attribute.DOUBLE: np.dtype('>f8')}

def unpack_attribute(att):
    if att.unsigned:
        log.warning('Unsupported unsigned attribute!')

    # TDS 5.0 now has a dataType attribute that takes precedence
    if att.len == 0:  # Empty
        val = None
    elif att.dataType == stream.STRING:  # Then look for new datatype string
        val = att.sdata
    elif att.dataType:  # Then a non-zero new data type
        val = np.fromstring(att.data,
                            dtype='>' + _dtypeLookup[att.dataType], count=att.len)
    elif att.type:  # Then non-zero old-data type0
        val = np.fromstring(att.data,
                            dtype=_attrConverters[att.type], count=att.len)
    elif att.sdata:  # This leaves both 0, try old string
        val = att.sdata
    else:  # Assume new datatype is Char (0)
        val = np.array(att.data, dtype=_dtypeLookup[att.dataType])

    if att.len == 1:
        val = val[0]

    return att.name, val

def read_var_int(file_obj):
    """Read a variable-length integer

    Parameters
    ----------
    file_obj : file-like object
        The file to read from.

    Returns
    -------
    int
        the variable-length value read
    """
    # Read all bytes from here, stopping with the first one that does not have
    # the MSB set. Save the lower 7 bits, and keep stacking to the *left*.
    val = 0
    shift = 0
    while True:
        # Read next byte
        next_val = ord(file_obj.read(1))
        val |= ((next_val & 0x7F) << shift)
        shift += 7
        if not next_val & 0x80:
            break

    return val
<commit_msg>Replace old-style string interpolation with format.<commit_after># Copyright (c) 2013-2015 Unidata.
# Distributed under the terms of the MIT License.
# SPDX-License-Identifier: MIT

from __future__ import print_function
import itertools
import logging
import zlib

from collections import OrderedDict

import numpy as np

from . import ncStream_pb2 as stream  # noqa

MAGIC_HEADER = b'\xad\xec\xce\xda'
MAGIC_DATA = b'\xab\xec\xce\xba'
MAGIC_DATA2 = b'\xab\xeb\xbe\xba'
MAGIC_VDATA = b'\xab\xef\xfe\xba'
MAGIC_VEND = b'\xed\xef\xfe\xda'
MAGIC_ERR = b'\xab\xad\xba\xda'

log = logging.getLogger('siphon.ncstream')
log.addHandler(logging.StreamHandler())  # Python 2.7 needs a handler set
log.setLevel(logging.WARNING)

#
# NCStream handling
#
def read_ncstream_data(fobj):
    data = read_proto_object(fobj, stream.Data)
    if data.dataType in (stream.STRING, stream.OPAQUE) or data.vdata:
        log.debug('Reading string/opaque/vlen')
        num_obj = read_var_int(fobj)
        log.debug('Num objects: %d', num_obj)
        blocks = [read_block(fobj) for _ in range(num_obj)]
        if data.dataType == stream.STRING:
            blocks = [b.decode('utf-8', errors='ignore') for b in blocks]

        # Again endian isn't coded properly
        dt = data_type_to_numpy(data.dataType).newbyteorder('>')
        if data.vdata:
            return np.array([np.frombuffer(b, dtype=dt) for b in blocks])
        else:
            return np.array(blocks, dtype=dt)
    elif data.dataType in _dtypeLookup:
        log.debug('Reading array data')
        bin_data = read_block(fobj)
        log.debug('Binary data: %s', bin_data)

        # Hard code to big endian for now since it's not encoded correctly
        dt = data_type_to_numpy(data.dataType).newbyteorder('>')

        # Handle decompressing the bytes
        if data.compress == stream.DEFLATE:
            bin_data = zlib.decompress(bin_data)
            assert len(bin_data) == data.uncompressedSize
        elif data.compress != stream.NONE:
            raise NotImplementedError('Compression type {0} not implemented!'.format(
                data.compress))

        # Turn bytes into an array
        return reshape_array(data, np.frombuffer(bin_data, dtype=dt))
    elif data.dataType == stream.STRUCTURE:
        sd = read_proto_object(fobj, stream.StructureData)

        # Make a datatype appropriate to the rows of struct
        endian = '>' if data.bigend else '<'
        dt = np.dtype([(endian, np.void, sd.rowLength)])

        # Turn bytes into an array
        return reshape_array(data, np.frombuffer(sd.data, dtype=dt))
    elif data.dataType == stream.SEQUENCE:
        log.debug('Reading sequence')
        blocks = []
        magic = read_magic(fobj)
        while magic != MAGIC_VEND:
            if magic == MAGIC_VDATA:
                log.error('Bad magic for struct/seq data!')
            blocks.append(read_proto_object(fobj, stream.StructureData))
            magic = read_magic(fobj)
        return data, blocks
    else:
        raise NotImplementedError("Don't know how to handle data type: {0}".format(
            data.dataType))

def read_ncstream_data2(fobj):
    data = read_proto_object(fobj, stream.DataCol)
    return datacol_to_array(data)

def read_ncstream_err(fobj):
    err = read_proto_object(fobj, stream.Error)
    raise RuntimeError(err.message)

ncstream_table = {MAGIC_HEADER: lambda f: read_proto_object(f, stream.Header),
                  MAGIC_DATA: read_ncstream_data,
                  MAGIC_DATA2: read_ncstream_data2,
                  MAGIC_ERR: read_ncstream_err}

def read_ncstream_messages(fobj):
    return read_messages(fobj, ncstream_table)

def read_messages(fobj, magic_table):
    messages = []

    while True:
        magic = read_magic(fobj)
        if not magic:
            break

        func = magic_table.get(magic)
        if func is not None:
            messages.append(func(fobj))
        else:
            log.error('Unknown magic: ' + str(' '.join('{0:02x}'.format(b) for b in magic)))

    return messages

def read_proto_object(fobj, klass):
    log.debug('%s chunk', klass.__name__)
    obj = klass()
    obj.ParseFromString(read_block(fobj))
    log.debug('Header: %s', str(obj))
    return obj

def read_magic(fobj):
    """Read a magic bytes.

    Parameters
    ----------
    fobj : file-like object
        The file to read from.

    Returns
    -------
    bytes
        magic byte sequence read
    """
    return fobj.read(4)

def read_block(fobj):
    """Read a block.

    Reads a block from a file object by first reading the number of bytes to read, which must
    be encoded as a variable-byte length integer.

    Parameters
    ----------
    fobj : file-like object
        The file to read from.

    Returns
    -------
    bytes
        block of bytes read
    """
    num = read_var_int(fobj)
    log.debug('Next block: %d bytes', num)
    return fobj.read(num)

def process_vlen(data_header, array):
    """Process vlen coming back from NCStream v2

    This takes the array of values and slices into an object array, with entries containing
    the appropriate pieces of the original array. Sizes are controlled by the passed in
    `data_header`.

    Parameters
    ----------
    data_header : Header
    array : :class:`numpy.ndarray`

    Returns
    -------
    ndarray
        object array containing sub-sequences from the original primitive array
    """
    source = iter(array)
    return np.array([np.fromiter(itertools.islice(source, size), dtype=array.dtype)
                     for size in data_header.vlens])

def datacol_to_array(datacol):
    """Convert DataCol from NCStream v2 into an array with appropriate type

    Depending on the data type specified, this extracts data from the appropriate members
    and packs into a :class:`numpy.ndarray`, recursing as necessary for compound data types.

    Parameters
    ----------
    datacol : DataCol

    Returns
    -------
    ndarray
        array containing extracted data
    """
    if datacol.dataType == stream.STRING:
        arr = np.array(datacol.stringdata, dtype=np.object)
    elif datacol.dataType == stream.OPAQUE:
        arr = np.array(datacol.opaquedata, dtype=np.object)
    elif datacol.dataType == stream.STRUCTURE:
        members = OrderedDict((mem.name, datacol_to_array(mem))
                              for mem in datacol.structdata.memberData)
        log.debug('Struct members:\n%s', str(members))

        # str() around name necessary because protobuf gives unicode names, but dtype doesn't
        # support them on Python 2
        dt = np.dtype([(str(name), arr.dtype) for name, arr in members.items()])
        log.debug('Struct dtype: %s', str(dt))

        arr = np.empty((datacol.nelems,), dtype=dt)
        for name, arr_data in members.items():
            arr[name] = arr_data
    else:
        # Make an appropriate datatype
        endian = '>' if datacol.bigend else '<'
        dt = data_type_to_numpy(datacol.dataType).newbyteorder(endian)

        # Turn bytes into an array
        arr = np.frombuffer(datacol.primdata, dtype=dt)
        if arr.size != datacol.nelems:
            log.warning('Array size %d does not agree with nelems %d',
                        arr.size, datacol.nelems)
        if datacol.isVlen:
            arr = process_vlen(datacol, arr)
            if arr.dtype == np.object_:
                arr = reshape_array(datacol, arr)
            else:
                # In this case, the array collapsed, need different resize that
                # correctly sizes from elements
                shape = tuple(r.size for r in datacol.section.range) + (datacol.vlens[0],)
                arr = arr.reshape(*shape)
        else:
            arr = reshape_array(datacol, arr)
    return arr

def reshape_array(data_header, array):
    """Extracts the appropriate array shape from the header

    Can handle taking a data header and either bytes containing data or a StructureData
    instance, which will have binary data as well as some additional information.

    Parameters
    ----------
    array : :class:`numpy.ndarray`
    data_header : Data
    """
    shape = tuple(r.size for r in data_header.section.range)
    if shape:
        return array.reshape(*shape)
    else:
        return array

# STRUCTURE = 8;
# SEQUENCE = 9;
_dtypeLookup = {stream.CHAR: 'S1', stream.BYTE: 'b', stream.SHORT: 'i2',
                stream.INT: 'i4', stream.LONG: 'i8', stream.FLOAT: 'f4',
                stream.DOUBLE: 'f8', stream.STRING: 'O',
                stream.ENUM1: 'B', stream.ENUM2: 'u2', stream.ENUM4: 'u4',
                stream.OPAQUE: 'O', stream.UBYTE: 'B', stream.USHORT: 'u2',
                stream.UINT: 'u4', stream.ULONG: 'u8'}

def data_type_to_numpy(datatype, unsigned=False):
    basic_type = _dtypeLookup[datatype]

    if datatype in (stream.STRING, stream.OPAQUE):
        return np.dtype(basic_type)

    if unsigned:
        basic_type = basic_type.replace('i', 'u')
    return np.dtype('=' + basic_type)

def struct_to_dtype(struct):
    """Convert a Structure specification to a numpy structured dtype."""
    # str() around name necessary because protobuf gives unicode names, but dtype doesn't
    # support them on Python 2
    fields = [(str(var.name), data_type_to_numpy(var.dataType, var.unsigned))
              for var in struct.vars]
    for s in struct.structs:
        fields.append((str(s.name), struct_to_dtype(s)))

    log.debug('Structure fields: %s', fields)
    dt = np.dtype(fields)
    return dt

def unpack_variable(var):
    # If we actually get a structure instance, handle turning that into a variable
    if var.dataType == stream.STRUCTURE:
        return None, struct_to_dtype(var), 'Structure'
    elif var.dataType == stream.SEQUENCE:
        log.warning('Sequence support not implemented!')

    dt = data_type_to_numpy(var.dataType, var.unsigned)
    if var.dataType == stream.OPAQUE:
        type_name = 'opaque'
    elif var.dataType == stream.STRING:
        type_name = 'string'
    else:
        type_name = dt.name

    if var.data:
        log.debug('Storing variable data: %s %s', dt, var.data)
        if var.dataType is str:
            data = var.data
        else:
            # Always sent big endian
            data = np.fromstring(var.data, dtype=dt.newbyteorder('>'))
    else:
        data = None

    return data, dt, type_name

_attrConverters = {stream.Attribute.BYTE: np.dtype('>b'),
                   stream.Attribute.SHORT: np.dtype('>i2'),
                   stream.Attribute.INT: np.dtype('>i4'),
                   stream.Attribute.LONG: np.dtype('>i8'),
                   stream.Attribute.FLOAT: np.dtype('>f4'),
                   stream.Attribute.DOUBLE: np.dtype('>f8')}

def unpack_attribute(att):
    if att.unsigned:
        log.warning('Unsupported unsigned attribute!')

    # TDS 5.0 now has a dataType attribute that takes precedence
    if att.len == 0:  # Empty
        val = None
    elif att.dataType == stream.STRING:  # Then look for new datatype string
        val = att.sdata
    elif att.dataType:  # Then a non-zero new data type
        val = np.fromstring(att.data,
                            dtype='>' + _dtypeLookup[att.dataType], count=att.len)
    elif att.type:  # Then non-zero old-data type0
        val = np.fromstring(att.data,
                            dtype=_attrConverters[att.type], count=att.len)
    elif att.sdata:  # This leaves both 0, try old string
        val = att.sdata
    else:  # Assume new datatype is Char (0)
        val = np.array(att.data, dtype=_dtypeLookup[att.dataType])

    if att.len == 1:
        val = val[0]

    return att.name, val

def read_var_int(file_obj):
    """Read a variable-length integer

    Parameters
    ----------
    file_obj : file-like object
        The file to read from.

    Returns
    -------
    int
        the variable-length value read
    """
    # Read all bytes from here, stopping with the first one that does not have
    # the MSB set. Save the lower 7 bits, and keep stacking to the *left*.
    val = 0
    shift = 0
    while True:
        # Read next byte
        next_val = ord(file_obj.read(1))
        val |= ((next_val & 0x7F) << shift)
        shift += 7
        if not next_val & 0x80:
            break

    return val
<|endoftext|>