<commit_before>#!/usr/bin/python

import sys
import os.path

from PySide import QtGui, QtCore, QtUiTools

import APS

libPath = '../../../common/src/+deviceDrivers/@APS/lib/'

class LoadBitFileRunner(QtCore.QThread):

    messageSignal = QtCore.Signal(str)

    def __init__(self, bitFileName, aps, apsNum):
        super(LoadBitFileRunner, self).__init__()
        self.bitFileName = bitFileName
        self.aps = aps
        self.apsNum = apsNum

    #Overwrite the destructor to make sure it doesn't get GC'd before it is finished
    def __del__(self):
        self.wait()

    def run(self):
        self.messageSignal.emit('Programming FPGA bitfile....')
        self.aps.connect(self.apsNum)
        self.aps.loadBitFile(self.bitFileName)
        self.messageSignal.emit('Loaded firmware version {0}'.format(self.aps.readBitFileVersion()))
        self.aps.disconnect()

class APScontrol(object):
    _bitFileName = ''
    def __init__(self):

        #Dynamically load the ui file
        loader = QtUiTools.QUiLoader()
        file = QtCore.QFile(os.path.join(os.path.dirname(sys.argv[0]), 'APS.ui'))
        file.open(QtCore.QFile.ReadOnly)
        self.ui = loader.load(file)
        file.close()

        #Connect UI element to signals
        self.ui.runButton.clicked.connect(self.run)
        self.ui.stopButton.clicked.connect(self.stop)

        self.ui.ch1fileOpen.clicked.connect(lambda : self.waveformDialog(self.ui.ch1file))
        self.ui.ch2fileOpen.clicked.connect(lambda : self.waveformDialog(self.ui.ch2file))
        self.ui.ch3fileOpen.clicked.connect(lambda : self.waveformDialog(self.ui.ch3file))
        self.ui.ch4fileOpen.clicked.connect(lambda : self.waveformDialog(self.ui.ch4file))
        self.ui.chAllfileOpen.clicked.connect(lambda : self.waveformDialog(self.ui.chAllfile))

        self.ui.chAllOnOff.toggled.connect(self.update_channel_enablers)

        self.ui.actionLoad_Bit_File.triggered.connect(self.bitFileDialog)
        self.ui.actionTest_PLL_Sync.triggered.connect(self.test_PLL_sync)

        #Set some validators for the scale factor / offset values so we don't have to error check later
        for channelct in range(1,5):
            #First the scale to anything
            tmpLineEdit = getattr(self.ui, 'ch{0}scale'.format(channelct))
            tmpLineEdit.setValidator(QtGui.QDoubleValidator())

            #Then the offset to +/-1 because it is of the full scale
            tmpLineEdit = getattr(self.ui, 'ch{0}offset'.format(channelct))
            tmpLineEdit.setValidator(QtGui.QDoubleValidator())
            tmpLineEdit.validator().setRange(-1,1,4)

        #Create an APS class instance for interacting with the instrument
        self.aps = APS.APS(libPath)

        #Enumerate the number of connected APS devices and fill out the combo box
        (numAPS, deviceSerials) = self.aps.enumerate()
        self.printMessage('Found {0} APS units'.format(numAPS))

        #Fill out the device ID combo box
        self.ui.deviceIDComboBox.clear()
        if numAPS > 0:
            self.ui.deviceIDComboBox.insertItems(0,['{0} ({1})'.format(num, deviceSerials[num]) for num in range(numAPS)])
            self._bitfilename = self.aps.getDefaultBitFileName()

        self.ui.show()

    def connect(self):
        #Connect to the specified APS
        apsNum = self.ui.deviceIDComboBox.currentIndex()
        self.printMessage('Opening connection to APS {0}'.format(apsNum))
        self.aps.connect(apsNum)
        self.printMessage("Firmware version: {0}".format(self.aps.readBitFileVersion()))

    def printMessage(self, message):
        self.ui.messageLog.append(message)

    #A slot to recieve log messages from threads.
    @QtCore.Slot(str)
    def printFromThread(self, message):
        self.printMessage(message)

    def bitFileDialog(self):
        #Launch a dialog to poll the user for the file name
        self._bitFileName = QtGui.QFileDialog.getOpenFileName(self.ui, 'Open File', '', 'Bit files (*.bit)')[0]
        #Check that file exists
        if not os.path.isfile(self._bitFileName):
            self.printMessage("Error bitfile not found: %s" % self.bitFileName.text() )
            return
        #If it does then create a thread (loading the bit file takes a few seconds)
        #Have to keep a reference around otherwise it gets deleted to early
        self.bitFileLoader = LoadBitFileRunner(self._bitFileName, self.aps, self.ui.deviceIDComboBox.currentIndex())
        self.bitFileLoader.messageSignal.connect(self.printFromThread)
        #Disable the run button while we are programming
        self.ui.runButton.setEnabled(0)
        self.bitFileLoader.finished.connect(lambda : self.ui.runButton.setEnabled(1))
        self.bitFileLoader.start()
        print('Started thread...')

    def waveformDialog(self, textBox):
        fileName, fileFilter = QtGui.QFileDialog.getOpenFileName(self.ui, 'Open File', '', 'Matlab Files (*.mat);;Waveform files (*.m);;Sequence files (*.seq)')
        textBox.setText(fileName)

    def update_channel_enablers(self):
        '''
        Update the single channel entries based on whether 4-channel mode is activiated.
        '''
        fourChanMode = self.ui.chAllOnOff.isChecked()
        for ct in range(1,5):
            getattr(self.ui,'ch{0}file'.format(ct)).setEnabled(not fourChanMode)
        self.ui.chAllfile.setEnabled(fourChanMode)

    def run(self):
        self.ui.runButton.setEnabled(0)
        self.ui.stopButton.setEnabled(1)

        #Pull the settings from the gui
        settings = {}

        #Sampling frequency frequencies
        settings['frequency'] = self.aps.VALID_FREQUENCIES[self.ui.sampleRate.currentIndex()];
        self.printMessage('Set frequency to {0}'.format(settings['frequency']))

        #Get the run mode
        if self.ui.sequencerMode.currentIndex() == 0:
            self.printMessage('Continous Mode')
            settings['runMode'] = self.aps.LL_CONTINUOUS
        else:
            self.printMessage('One Shot Mode')
            settings['runMode'] = self.aps.LL_ONESHOT

        #Check to see how to trigger
        if self.ui.triggerType.currentIndex() == 0:  # Internal (aka Software Trigger)
            settings['triggerSource'] = self.aps.TRIGGER_SOFTWARE
            self.printMessage('Sofware trigger.')
        else: # External (aka Software Trigger):
            self.printMessage('Hardware trigger.')
            settings['triggerSource'] = self.aps.TRIGGER_HARDWARE

        #Get the four channel mode stuff
        settings['fourChannelMode'] = bool(self.ui.chAllOnOff.isChecked())
        if settings['fourChannelMode']:
            settings['chAll'] = {}
            settings['chAll']['seqfile'] = self.ui.chAllfile.text()

        #Pull out specific channel properties
        for ct,channelName in enumerate(self.aps.CHANNELNAMES):
            settings[channelName] = {}
            settings[channelName]['amplitude'] = float(getattr(self.ui,'ch{0}scale'.format(ct+1)).text())
            settings[channelName]['offset'] = float(getattr(self.ui,'ch{0}offset'.format(ct+1)).text())
            settings[channelName]['enabled'] = bool(getattr(self.ui,'ch{0}enable'.format(ct+1)).isChecked())
            settings[channelName]['seqfile'] = getattr(self.ui,'ch{0}file'.format(ct+1)).text()
            #Do a check whether the file exists
            if (not settings['fourChannelMode']) and settings[channelName]['enabled'] and (not os.path.isfile(settings[channelName]['seqfile'])):
                QtGui.QMessageBox.warning(self.ui, 'Oops!', 'Channel {0} is enabled with a non-existent file.'.format(ct+1))
                self.stop()
                raise

        try:
            self.connect()
            self.aps.init()
            self.aps.setAll(settings)
            self.aps.run()
            self.printMessage('Running')
        except:
            self.printMessage('WARNING: Could not get APS running!')
            self.stop()
            raise

    def stop(self):
        self.ui.stopButton.setEnabled(0)
        self.ui.runButton.setEnabled(1)

        self.aps.stop()
        self.aps.disconnect()
        self.printMessage('Stopped')

    def setScaleFactor(self,channel,value):
        value = '%.2f' % value
        if channel in range(1,5):
            textBox = getattr(self, 'ch%iscale' % channel)
            textBox.setText(value)
        else:
            print 'Unknown channel', channel

    def setOffset(self,channel,value):
        value = '%.2f' % value
        if channel in range(1,5):
            textBox = getattr(self, 'ch%ioffset' % channel)
            textBox.setText(value)
        else:
            print 'Unknown channel', channel

    def test_PLL_sync(self):
        self.connect()
        status = self.aps.test_PLL_sync(0)
        self.printMessage('PLL Sync Test for DAC {0} returned {1}'.format(0, status))
        status = self.aps.test_PLL_sync(2)
        self.printMessage('PLL Sync Test for DAC {0} returned {1}'.format(2, status))
        self.aps.disconnect()

if __name__ == '__main__':
    # create the Qt application
    app = QtGui.QApplication(sys.argv)
    frame = APScontrol()
    sys.exit(app.exec_())
<commit_msg>Added a thread for the the PLL testing. --CAR<commit_after>#!/usr/bin/python

import sys
import os.path

from PySide import QtGui, QtCore, QtUiTools

import APS

libPath = '../../../common/src/+deviceDrivers/@APS/lib/'

class LoadBitFileRunner(QtCore.QThread):

    messageSignal = QtCore.Signal(str)

    def __init__(self, bitFileName, aps, apsNum):
        super(LoadBitFileRunner, self).__init__()
        self.bitFileName = bitFileName
        self.aps = aps
        self.apsNum = apsNum

    #Overwrite the destructor to make sure it doesn't get GC'd before it is finished
    def __del__(self):
        self.wait()

    def run(self):
        self.messageSignal.emit('Programming FPGA bitfile....')
        self.aps.connect(self.apsNum)
        self.aps.loadBitFile(self.bitFileName)
        self.messageSignal.emit('Loaded firmware version {0}'.format(self.aps.readBitFileVersion()))
        self.aps.disconnect()

class PLLSyncTestRunner(QtCore.QThread):
    messageSignal = QtCore.Signal(str)

    def __init__(self, bitFileName, aps, apsNum):
        super(PLLSyncTestRunner, self).__init__()
        self.bitFileName = bitFileName
        self.aps = aps
        self.apsNum = apsNum

    def run(self):
        self.messageSignal.emit('Testing the PLL sync....')
        self.aps.connect(self.apsNum)
        self.messageSignal.emit('PLL Sync Test for DAC {0} returned {1}'.format(0, self.aps.test_PLL_sync(0)))
        self.messageSignal.emit('PLL Sync Test for DAC {0} returned {1}'.format(2, self.aps.test_PLL_sync(2)))
        self.aps.disconnect()

class APScontrol(object):
    _bitFileName = ''
    def __init__(self):

        #Dynamically load the ui file
        loader = QtUiTools.QUiLoader()
        file = QtCore.QFile(os.path.join(os.path.dirname(sys.argv[0]), 'APS.ui'))
        file.open(QtCore.QFile.ReadOnly)
        self.ui = loader.load(file)
        file.close()

        #Connect UI element to signals
        self.ui.runButton.clicked.connect(self.run)
        self.ui.stopButton.clicked.connect(self.stop)

        self.ui.ch1fileOpen.clicked.connect(lambda : self.waveformDialog(self.ui.ch1file))
        self.ui.ch2fileOpen.clicked.connect(lambda : self.waveformDialog(self.ui.ch2file))
        self.ui.ch3fileOpen.clicked.connect(lambda : self.waveformDialog(self.ui.ch3file))
        self.ui.ch4fileOpen.clicked.connect(lambda : self.waveformDialog(self.ui.ch4file))
        self.ui.chAllfileOpen.clicked.connect(lambda : self.waveformDialog(self.ui.chAllfile))

        self.ui.chAllOnOff.toggled.connect(self.update_channel_enablers)

        self.ui.actionLoad_Bit_File.triggered.connect(self.bitFileDialog)
        self.ui.actionTest_PLL_Sync.triggered.connect(self.test_PLL_sync)

        #Set some validators for the scale factor / offset values so we don't have to error check later
        for channelct in range(1,5):
            #First the scale to anything
            tmpLineEdit = getattr(self.ui, 'ch{0}scale'.format(channelct))
            tmpLineEdit.setValidator(QtGui.QDoubleValidator())

            #Then the offset to +/-1 because it is of the full scale
            tmpLineEdit = getattr(self.ui, 'ch{0}offset'.format(channelct))
            tmpLineEdit.setValidator(QtGui.QDoubleValidator())
            tmpLineEdit.validator().setRange(-1,1,4)

        #Create an APS class instance for interacting with the instrument
        self.aps = APS.APS(libPath)

        #Enumerate the number of connected APS devices and fill out the combo box
        (numAPS, deviceSerials) = self.aps.enumerate()
        self.printMessage('Found {0} APS units'.format(numAPS))

        #Fill out the device ID combo box
        self.ui.deviceIDComboBox.clear()
        if numAPS > 0:
            self.ui.deviceIDComboBox.insertItems(0,['{0} ({1})'.format(num, deviceSerials[num]) for num in range(numAPS)])
            self._bitfilename = self.aps.getDefaultBitFileName()

        self.ui.show()

    def connect(self):
        #Connect to the specified APS
        apsNum = self.ui.deviceIDComboBox.currentIndex()
        self.printMessage('Opening connection to APS {0}'.format(apsNum))
        self.aps.connect(apsNum)
        self.printMessage("Firmware version: {0}".format(self.aps.readBitFileVersion()))

    def printMessage(self, message):
        self.ui.messageLog.append(message)

    #A slot to recieve log messages from threads.
    @QtCore.Slot(str)
    def printFromThread(self, message):
        self.printMessage(message)

    def bitFileDialog(self):
        #Launch a dialog to poll the user for the file name
        self._bitFileName = QtGui.QFileDialog.getOpenFileName(self.ui, 'Open File', '', 'Bit files (*.bit)')[0]
        #Check that file exists
        if not os.path.isfile(self._bitFileName):
            self.printMessage("Error bitfile not found: %s" % self.bitFileName.text() )
            return
        #If it does then create a thread (loading the bit file takes a few seconds)
        #Have to keep a reference around otherwise it gets deleted to early
        self.bitFileLoader = LoadBitFileRunner(self._bitFileName, self.aps, self.ui.deviceIDComboBox.currentIndex())
        self.bitFileLoader.messageSignal.connect(self.printFromThread)
        #Disable the run button while we are programming
        self.ui.runButton.setEnabled(0)
        self.bitFileLoader.finished.connect(lambda : self.ui.runButton.setEnabled(1))
        self.bitFileLoader.start()

    def test_PLL_sync(self):
        #Start a background thread for testing the PLL sync
        self.PLLTester = PLLSyncTestRunner(self._bitFileName, self.aps, self.ui.deviceIDComboBox.currentIndex())
        self.PLLTester.messageSignal.connect(self.printFromThread)
        #Disable the run button while we are programming
        self.ui.runButton.setEnabled(0)
        self.PLLTester.finished.connect(lambda : self.ui.runButton.setEnabled(1))
        self.PLLTester.start()

    def waveformDialog(self, textBox):
        fileName, fileFilter = QtGui.QFileDialog.getOpenFileName(self.ui, 'Open File', '', 'Matlab Files (*.mat);;Waveform files (*.m);;Sequence files (*.seq)')
        textBox.setText(fileName)

    def update_channel_enablers(self):
        '''
        Update the single channel entries based on whether 4-channel mode is activiated.
        '''
        fourChanMode = self.ui.chAllOnOff.isChecked()
        for ct in range(1,5):
            getattr(self.ui,'ch{0}file'.format(ct)).setEnabled(not fourChanMode)
        self.ui.chAllfile.setEnabled(fourChanMode)

    def run(self):
        self.ui.runButton.setEnabled(0)
        self.ui.stopButton.setEnabled(1)

        #Pull the settings from the gui
        settings = {}

        #Sampling frequency frequencies
        settings['frequency'] = self.aps.VALID_FREQUENCIES[self.ui.sampleRate.currentIndex()];
        self.printMessage('Set frequency to {0}'.format(settings['frequency']))

        #Get the run mode
        if self.ui.sequencerMode.currentIndex() == 0:
            self.printMessage('Continous Mode')
            settings['runMode'] = self.aps.LL_CONTINUOUS
        else:
            self.printMessage('One Shot Mode')
            settings['runMode'] = self.aps.LL_ONESHOT

        #Check to see how to trigger
        if self.ui.triggerType.currentIndex() == 0:  # Internal (aka Software Trigger)
            settings['triggerSource'] = self.aps.TRIGGER_SOFTWARE
            self.printMessage('Sofware trigger.')
        else: # External (aka Software Trigger):
            self.printMessage('Hardware trigger.')
            settings['triggerSource'] = self.aps.TRIGGER_HARDWARE

        #Get the four channel mode stuff
        settings['fourChannelMode'] = bool(self.ui.chAllOnOff.isChecked())
        if settings['fourChannelMode']:
            settings['chAll'] = {}
            settings['chAll']['seqfile'] = self.ui.chAllfile.text()

        #Pull out specific channel properties
        for ct,channelName in enumerate(self.aps.CHANNELNAMES):
            settings[channelName] = {}
            settings[channelName]['amplitude'] = float(getattr(self.ui,'ch{0}scale'.format(ct+1)).text())
            settings[channelName]['offset'] = float(getattr(self.ui,'ch{0}offset'.format(ct+1)).text())
            settings[channelName]['enabled'] = bool(getattr(self.ui,'ch{0}enable'.format(ct+1)).isChecked())
            settings[channelName]['seqfile'] = getattr(self.ui,'ch{0}file'.format(ct+1)).text()
            #Do a check whether the file exists
            if (not settings['fourChannelMode']) and settings[channelName]['enabled'] and (not os.path.isfile(settings[channelName]['seqfile'])):
                QtGui.QMessageBox.warning(self.ui, 'Oops!', 'Channel {0} is enabled with a non-existent file.'.format(ct+1))
                self.stop()
                raise

        try:
            self.connect()
            self.aps.init()
            self.aps.setAll(settings)
            self.aps.run()
            self.printMessage('Running')
        except:
            self.printMessage('WARNING: Could not get APS running!')
            self.stop()
            raise

    def stop(self):
        self.ui.stopButton.setEnabled(0)
        self.ui.runButton.setEnabled(1)

        self.aps.stop()
        self.aps.disconnect()
        self.printMessage('Stopped')

if __name__ == '__main__':
    # create the Qt application
    app = QtGui.QApplication(sys.argv)
    frame = APScontrol()
    sys.exit(app.exec_())
<|endoftext|>


<commit_before>import json
import logging
from .exceptions import (
    JSONRPCInvalidParams,
    JSONRPCInvalidRequest,
    JSONRPCInvalidRequestException,
    JSONRPCMethodNotFound,
    JSONRPCParseError,
    JSONRPCServerError,
)
from .jsonrpc1 import JSONRPC10Response
from .jsonrpc2 import (
    JSONRPC20BatchRequest,
    JSONRPC20BatchResponse,
    JSONRPC20Response,
)
from .jsonrpc import JSONRPCRequest

logger = logging.getLogger(__name__)

class JSONRPCResponseManager(object):

    """ JSON-RPC response manager.

    Method brings syntactic sugar into library. Given dispatcher it handles
    request (both single and batch) and handles errors.
    Request could be handled in parallel, it is server responsibility.

    :param str request_str: json string. Will be converted into
        JSONRPC20Request, JSONRPC20BatchRequest or JSONRPC10Request

    :param dict dispather: dict<function_name:function>.

    """

    RESPONSE_CLASS_MAP = {
        "1.0": JSONRPC10Response,
        "2.0": JSONRPC20Response,
    }

    @classmethod
    def handle(cls, request_str, dispatcher):
        if isinstance(request_str, bytes):
            request_str = request_str.decode("utf-8")

        try:
            json.loads(request_str)
        except (TypeError, ValueError):
            return JSONRPC20Response(error=JSONRPCParseError()._data)

        try:
            request = JSONRPCRequest.from_json(request_str)
        except JSONRPCInvalidRequestException:
            return JSONRPC20Response(error=JSONRPCInvalidRequest()._data)

        rs = request if isinstance(request, JSONRPC20BatchRequest) \
            else [request]
        responses = [r for r in cls._get_responses(rs, dispatcher)
                     if r is not None]

        # notifications
        if not responses:
            return

        if isinstance(request, JSONRPC20BatchRequest):
            return JSONRPC20BatchResponse(*responses)
        else:
            return responses[0]

    @classmethod
    def _get_responses(cls, requests, dispatcher):
        """ Response to each single JSON-RPC Request.

        :return iterator(JSONRPC20Response):

        """
        for request in requests:
            response = lambda **kwargs: cls.RESPONSE_CLASS_MAP[
                request.JSONRPC_VERSION](_id=request._id, **kwargs)

            try:
                method = dispatcher[request.method]
            except KeyError:
                if not request.is_notification:
                    yield response(error=JSONRPCMethodNotFound()._data)
                continue

            try:
                result = method(*request.args, **request.kwargs)
            except TypeError:
                if not request.is_notification:
                    yield response(error=JSONRPCInvalidParams()._data)
            except Exception as e:
                data = {
                    "type": e.__class__.__name__,
                    "args": e.args,
                    "message": str(e),
                }
                logger.exception("API Exception: {0}".format(data))
                result = response(error=JSONRPCServerError(data=data)._data)
                if not request.is_notification:
                    yield result
            else:
                if not request.is_notification:
                    yield response(result=result)
<commit_msg>substitute continue with else block of try-except statement. Use additional finally block instead of 4 if-checks for notification.<commit_after>import json
import logging
from .exceptions import (
    JSONRPCInvalidParams,
    JSONRPCInvalidRequest,
    JSONRPCInvalidRequestException,
    JSONRPCMethodNotFound,
    JSONRPCParseError,
    JSONRPCServerError,
)
from .jsonrpc1 import JSONRPC10Response
from .jsonrpc2 import (
    JSONRPC20BatchRequest,
    JSONRPC20BatchResponse,
    JSONRPC20Response,
)
from .jsonrpc import JSONRPCRequest

logger = logging.getLogger(__name__)

class JSONRPCResponseManager(object):

    """ JSON-RPC response manager.

    Method brings syntactic sugar into library. Given dispatcher it handles
    request (both single and batch) and handles errors.
    Request could be handled in parallel, it is server responsibility.

    :param str request_str: json string. Will be converted into
        JSONRPC20Request, JSONRPC20BatchRequest or JSONRPC10Request

    :param dict dispather: dict<function_name:function>.

    """

    RESPONSE_CLASS_MAP = {
        "1.0": JSONRPC10Response,
        "2.0": JSONRPC20Response,
    }

    @classmethod
    def handle(cls, request_str, dispatcher):
        if isinstance(request_str, bytes):
            request_str = request_str.decode("utf-8")

        try:
            json.loads(request_str)
        except (TypeError, ValueError):
            return JSONRPC20Response(error=JSONRPCParseError()._data)

        try:
            request = JSONRPCRequest.from_json(request_str)
        except JSONRPCInvalidRequestException:
            return JSONRPC20Response(error=JSONRPCInvalidRequest()._data)

        rs = request if isinstance(request, JSONRPC20BatchRequest) \
            else [request]
        responses = [r for r in cls._get_responses(rs, dispatcher)
                     if r is not None]

        # notifications
        if not responses:
            return

        if isinstance(request, JSONRPC20BatchRequest):
            return JSONRPC20BatchResponse(*responses)
        else:
            return responses[0]

    @classmethod
    def _get_responses(cls, requests, dispatcher):
        """ Response to each single JSON-RPC Request.

        :return iterator(JSONRPC20Response):

        """
        for request in requests:
            response = lambda **kwargs: cls.RESPONSE_CLASS_MAP[
                request.JSONRPC_VERSION](_id=request._id, **kwargs)

            try:
                method = dispatcher[request.method]
            except KeyError:
                output = response(error=JSONRPCMethodNotFound()._data)
            else:
                try:
                    result = method(*request.args, **request.kwargs)
                except TypeError:
                    output = response(error=JSONRPCInvalidParams()._data)
                except Exception as e:
                    data = {
                        "type": e.__class__.__name__,
                        "args": e.args,
                        "message": str(e),
                    }
                    logger.exception("API Exception: {0}".format(data))
                    output = response(
                        error=JSONRPCServerError(data=data)._data)
                else:
                    output = response(result=result)
            finally:
                if not request.is_notification:
                    yield output
<|endoftext|>


<commit_before>from django.conf import settings
from django.contrib.auth.tokens import default_token_generator
from django.contrib.sites.shortcuts import get_current_site
from django.template.loader import render_to_string
from django.utils.http import urlsafe_base64_encode, urlsafe_base64_decode
from django.utils.encoding import force_bytes, force_text

def encode_uid(pk):
    return urlsafe_base64_encode(force_bytes(pk)).decode()

def decode_uid(pk):
    return force_text(urlsafe_base64_decode(pk))

class UserEmailBase(object):
    mail_subject = None
    html_body_template = None
    plaintext_body_template = None
    url = None

    def __init__(self, request, user, from_email=None):
        self.from_email = from_email if from_email \
            else getattr(settings, 'DEFAULT_FROM_EMAIL')
        self.user = user or request.user
        self.protocol = 'https' if request.is_secure() else 'http'
        self.site = get_current_site(request)

    def __iter__(self):
        context = self.get_context()
        html_message = render_to_string(self.html_body_template, context)
        message = render_to_string(self.plaintext_body_template, context)

        envelope = {
            'subject': self.mail_subject,
            'message': message,
            'html_message': html_message,
            'from_email': self.from_email,
        }
        for item in envelope.items():
            yield item

    def get_context(self):
        return {
            'user': self.user,
            'protocol': self.protocol,
            'domain': self.site.domain,
            'site': self.site,
            'uid': encode_uid(self.user.pk),
            'token': default_token_generator.make_token(self.user),
        }

class UserActivationEmail(UserEmailBase):
    mail_subject = 'Account activation'
    plaintext_body_template = 'email_activation_body.txt'
    html_body_template = 'email_activation_body.html'
    url = 'account/activate/?uid={uid}&token={token}'

    def get_context(self):
        context = super(UserActivationEmail, self).get_context()
        context['url'] = self.url.format(**context)
        return context

class UserPasswordResetEmail(UserEmailBase):
    mail_subject = 'Password reset'
    plaintext_body_template = 'email_pass_reset_body.txt'
    html_body_template = 'email_pass_reset_body.html'
    url = 'account/password/reset/confirm/?uid={uid}&token={token}'

    def get_context(self):
        context = super(UserPasswordResetEmail, self).get_context()
        context['url'] = self.url.format(**context)
        return context
<commit_msg>jobs-012: Refactored UserActivationEmail and UserPasswordResetEmail<commit_after>from django.conf import settings
from django.contrib.auth.tokens import default_token_generator
from django.contrib.sites.shortcuts import get_current_site
from django.template.loader import render_to_string
from django.utils.http import urlsafe_base64_encode, urlsafe_base64_decode
from django.utils.encoding import force_bytes, force_text

def encode_uid(pk):
    return urlsafe_base64_encode(force_bytes(pk)).decode()

def decode_uid(pk):
    return force_text(urlsafe_base64_decode(pk))

class UserEmailBase(object):
    mail_subject = None
    html_body_template = None
    plaintext_body_template = None
    url = None

    def __init__(self, request, user, from_email=None):
        self.from_email = from_email if from_email \
            else getattr(settings, 'DEFAULT_FROM_EMAIL')
        self.user = user or request.user
        self.protocol = 'https' if request.is_secure() else 'http'
        self.site = get_current_site(request)

    def __iter__(self):
        context = self.get_context()
        html_message = render_to_string(self.html_body_template, context)
        message = render_to_string(self.plaintext_body_template, context)

        envelope = {
            'subject': self.mail_subject,
            'message': message,
            'html_message': html_message,
            'from_email': self.from_email,
        }
        for item in envelope.items():
            yield item

    def get_context(self):
        return {
            'user': self.user,
            'protocol': self.protocol,
            'domain': self.site.domain,
            'site': self.site,
            'uid': encode_uid(self.user.pk),
            'token': default_token_generator.make_token(self.user),
        }

class UserEmailUrlMixin(object):
    """
    Adds formated url to base context
    """
    def get_context(self):
        context = super(UserEmailUrlMixin, self).get_context()
        context['url'] = self.url.format(**context)
        return context

class UserActivationEmail(UserEmailUrlMixin, UserEmailBase):
    mail_subject = 'Account activation'
    plaintext_body_template = 'email_activation_body.txt'
    html_body_template = 'email_activation_body.html'
    url = 'account/activate/?uid={uid}&token={token}'

class UserPasswordResetEmail(UserEmailUrlMixin, UserEmailBase):
    mail_subject = 'Password reset'
    plaintext_body_template = 'email_pass_reset_body.txt'
    html_body_template = 'email_pass_reset_body.html'
    url = 'account/password/reset/confirm/?uid={uid}&token={token}'
<|endoftext|>


<commit_before>import datetime

from django.conf import settings
from django.contrib.sites.models import Site
from django.views.generic import ListView
from haystack.query import SearchQuerySet

from localtv.search.utils import NormalizedVideoList
from localtv.search.views import SortFilterView
from localtv.search_indexes import DATETIME_NULL_PLACEHOLDER

MAX_VOTES_PER_CATEGORY = getattr(settings, 'MAX_VOTES_PER_CATEGORY', 3)

class CompatibleListingView(SortFilterView):
    """
    This is the backwards-compatible version of the :class:`SortFilterView`,
    which provides some extra context, normalizes the search results as
    :class:`.Video` instances, and provides some querystring handling.

    """
    #: Period of time within which the video was approved.
    approved_since = None

    def get_paginate_by(self, queryset):
        paginate_by = self.request.GET.get('count')
        if paginate_by:
            try:
                paginate_by = int(paginate_by)
            except ValueError:
                paginate_by = None
        if paginate_by is None:
            paginate_by = self.paginate_by
        return paginate_by

    def get_form_data(self, base_data=None, filter_value=None):
        data = super(CompatibleListingView, self).get_form_data(base_data,
                                                                filter_value)
        if 'q' not in data:
            data['q'] = data.get('query', '')
        if data.get('sort') == 'latest':
            data['sort'] = 'newest'
        return data

    def get_queryset(self):
        """Wraps the normal queryset in a :class:`.NormalizedVideoList`."""
        qs = super(CompatibleListingView, self).get_queryset()

        if self.approved_since is not None:
            if isinstance(qs, SearchQuerySet):
                qs = qs.exclude(when_approved__exact=DATETIME_NULL_PLACEHOLDER)
            else:
                qs = qs.exclude(when_approved__isnull=True)
            qs = qs.filter(when_approved__gt=(
                                datetime.datetime.now() - self.approved_since))

        return NormalizedVideoList(qs)

    def get_context_data(self, **kwargs):
        context = super(CompatibleListingView, self).get_context_data(
                                                                     **kwargs)
        form = context['form']
        context['query'] = form.cleaned_data['q']
        context['video_list'] = context['videos']
        return context

class SiteListView(ListView):
    """
    Filters the ordinary queryset according to the current site.

    """
    def get_queryset(self):
        return super(SiteListView, self).get_queryset().filter(
                                site=Site.objects.get_current())
<commit_msg>Corrected query value fetching in CompatibleListingView. Was failing if the form was invalid, because it was using cleaned_data rather than using the form API to directly fetch the value.<commit_after>import datetime

from django.conf import settings
from django.contrib.sites.models import Site
from django.views.generic import ListView
from haystack.query import SearchQuerySet

from localtv.search.utils import NormalizedVideoList
from localtv.search.views import SortFilterView
from localtv.search_indexes import DATETIME_NULL_PLACEHOLDER

MAX_VOTES_PER_CATEGORY = getattr(settings, 'MAX_VOTES_PER_CATEGORY', 3)

class CompatibleListingView(SortFilterView):
    """
    This is the backwards-compatible version of the :class:`SortFilterView`,
    which provides some extra context, normalizes the search results as
    :class:`.Video` instances, and provides some querystring handling.

    """
    #: Period of time within which the video was approved.
    approved_since = None

    def get_paginate_by(self, queryset):
        paginate_by = self.request.GET.get('count')
        if paginate_by:
            try:
                paginate_by = int(paginate_by)
            except ValueError:
                paginate_by = None
        if paginate_by is None:
            paginate_by = self.paginate_by
        return paginate_by

    def get_form_data(self, base_data=None, filter_value=None):
        data = super(CompatibleListingView, self).get_form_data(base_data,
                                                                filter_value)
        if 'q' not in data:
            data['q'] = data.get('query', '')
        if data.get('sort') == 'latest':
            data['sort'] = 'newest'
        return data

    def get_queryset(self):
        """Wraps the normal queryset in a :class:`.NormalizedVideoList`."""
        qs = super(CompatibleListingView, self).get_queryset()

        if self.approved_since is not None:
            if isinstance(qs, SearchQuerySet):
                qs = qs.exclude(when_approved__exact=DATETIME_NULL_PLACEHOLDER)
            else:
                qs = qs.exclude(when_approved__isnull=True)
            qs = qs.filter(when_approved__gt=(
                                datetime.datetime.now() - self.approved_since))

        return NormalizedVideoList(qs)

    def get_context_data(self, **kwargs):
        context = super(CompatibleListingView, self).get_context_data(
                                                                     **kwargs)
        form = context['form']
        context['query'] = form['q'].value()
        context['video_list'] = context['videos']
        return context

class SiteListView(ListView):
    """
    Filters the ordinary queryset according to the current site.

    """
    def get_queryset(self):
        return super(SiteListView, self).get_queryset().filter(
                                site=Site.objects.get_current())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from openerp import models, fields, api
from openerp.osv import osv
from openerp.osv import fields as old_fields
import openerp.addons.decimal_precision as dp
from openerp.tools.translate import _

class ResPartner(models.Model):
    _inherit = 'res.partner'

    @api.multi
    def _get_debt(self):
        old_debt_account = self.env['ir.model.data'].search([('name', '=', 'debt_account')])
        if old_debt_account:
            debt_account = old_debt_account
        else:
            debt_account_name = 'debt_account_' + self.env.user.company_id.id
            debt_account = self.env['ir.model.data'].search([('name', '=', debt_account_name)])
        debt_journal = self.env['account.journal'].search([
            ('company_id', '=', self.env.user.company_id.id),
            ('debt', '=', True)
        ])

        self._cr.execute(
            """SELECT l.partner_id, SUM(l.debit - l.credit)
            FROM account_move_line l
            WHERE l.account_id = %s AND l.partner_id IN %s
            GROUP BY l.partner_id
            """,
            (debt_account.id, tuple(self.ids)))

        res = {}
        for partner in self:
            res[partner.id] = 0
        for partner_id, val in self._cr.fetchall():
            res[partner_id] += val

        statements = self.env['account.bank.statement'].search(
            [('journal_id', '=', debt_journal.id), ('state', '=', 'open')])
        if statements:

            self._cr.execute(
                """SELECT l.partner_id, SUM(l.amount)
                FROM account_bank_statement_line l
                WHERE l.statement_id IN %s AND l.partner_id IN %s
                GROUP BY l.partner_id
                """,
                (tuple(statements.ids), tuple(self.ids)))
            for partner_id, val in self._cr.fetchall():
                res[partner_id] += val
        for partner in self:
            partner.debt = res[partner.id]

    debt = fields.Float(
        compute='_get_debt', string='Debt', readonly=True,
        digits=dp.get_precision('Account'))

class AccountJournal(models.Model):
    _inherit = 'account.journal'

    debt = fields.Boolean(string='Debt Payment Method')

class PosConfig(osv.osv):
    _inherit = 'pos.config'

    _columns = {
        'debt_dummy_product_id': old_fields.many2one(
            'product.product',
            string='Dummy Product for Debt',
            domain=[('available_in_pos', '=', True)],
            help="Dummy product used when a customer pays his debt "
                 "without ordering new products. This is a workaround to the fact "
                 "that Odoo needs to have at least one product on the order to "
                 "validate the transaction.")
    }
<commit_msg>add help<commit_after># -*- coding: utf-8 -*-
from openerp import models, fields, api
from openerp.osv import osv
from openerp.osv import fields as old_fields
import openerp.addons.decimal_precision as dp
from openerp.tools.translate import _

class ResPartner(models.Model):
    _inherit = 'res.partner'

    @api.multi
    def _get_debt(self):
        old_debt_account = self.env['ir.model.data'].search([('name', '=', 'debt_account')])
        if old_debt_account:
            debt_account = old_debt_account
        else:
            debt_account_name = 'debt_account_' + self.env.user.company_id.id
            debt_account = self.env['ir.model.data'].search([('name', '=', debt_account_name)])
        debt_journal = self.env['account.journal'].search([
            ('company_id', '=', self.env.user.company_id.id),
            ('debt', '=', True)
        ])

        self._cr.execute(
            """SELECT l.partner_id, SUM(l.debit - l.credit)
            FROM account_move_line l
            WHERE l.account_id = %s AND l.partner_id IN %s
            GROUP BY l.partner_id
            """,
            (debt_account.id, tuple(self.ids)))

        res = {}
        for partner in self:
            res[partner.id] = 0
        for partner_id, val in self._cr.fetchall():
            res[partner_id] += val

        statements = self.env['account.bank.statement'].search(
            [('journal_id', '=', debt_journal.id), ('state', '=', 'open')])
        if statements:

            self._cr.execute(
                """SELECT l.partner_id, SUM(l.amount)
                FROM account_bank_statement_line l
                WHERE l.statement_id IN %s AND l.partner_id IN %s
                GROUP BY l.partner_id
                """,
                (tuple(statements.ids), tuple(self.ids)))
            for partner_id, val in self._cr.fetchall():
                res[partner_id] += val
        for partner in self:
            partner.debt = res[partner.id]

    debt = fields.Float(
        compute='_get_debt', string='Debt', readonly=True,
        digits=dp.get_precision('Account'), help='This debt value for only current company')

class AccountJournal(models.Model):
    _inherit = 'account.journal'

    debt = fields.Boolean(string='Debt Payment Method')

class PosConfig(osv.osv):
    _inherit = 'pos.config'

    _columns = {
        'debt_dummy_product_id': old_fields.many2one(
            'product.product',
            string='Dummy Product for Debt',
            domain=[('available_in_pos', '=', True)],
            help="Dummy product used when a customer pays his debt "
                 "without ordering new products. This is a workaround to the fact "
                 "that Odoo needs to have at least one product on the order to "
                 "validate the transaction.")
    }
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import
from __future__ import print_function
from __future__ import with_statement

import functools
import json
import os
import platform

from re import sub

from .const import IS_PY2
from .const import PLUGIN_NAME
from .const import PRETTIER_CONFIG_FILES
from .const import PRETTIER_IGNORE_FILE

if IS_PY2:
    text_type = unicode
    string_types = (str, unicode)
else:
    text_type = str
    string_types = (str,)

def memoize(obj):
    cache = obj.cache = {}

    @functools.wraps(obj)
    def memoizer(*args, **kwargs):
        key = str(args) + str(kwargs)
        if key not in cache:
            cache[key] = obj(*args, **kwargs)
        return cache[key]

    return memoizer

def contains(needle, haystack):
    if not needle or not haystack:
        return False
    return needle in haystack

def find_prettier_config(start_dir, alt_dirs=None):
    if alt_dirs is None:
        alt_dirs = []
    if '~' not in alt_dirs:
        alt_dirs.append('~')
    prettier_config = None
    for config_file in PRETTIER_CONFIG_FILES:
        prettier_config = _find_file(
            start_dir=start_dir, filename=config_file, parent=False, limit=500, aux_dirs=alt_dirs)
        if prettier_config and os.path.exists(prettier_config):
            # check for prettier key defined package.json
            if os.path.basename(prettier_config) == 'package.json' and not \
                    _prettier_opts_in_package_json(prettier_config):
                # no prettier key... reset found confile to None
                prettier_config = None
                continue
            break

    return prettier_config

def _climb_dirs(start_dir, limit=None):
    """
    Generate directories, starting from start_dir.

    Hat tip goes to SublimeLinter 3.

    :param start_dir: The search start path.
    :param limit: If limit is None, stop at the root directory. Otherwise return a maximum of limit directories.
    """
    right = True

    while right and (limit is None or limit > 0):
        yield start_dir
        start_dir, right = os.path.split(start_dir)

        if limit is not None:
            limit -= 1

def _find_file(start_dir, filename, parent=False, limit=None, aux_dirs=None):
    """
    Find the given file by searching up the file hierarchy from start_dir.

    Hat tip goes to SublimeLinter 3.

    :param start_dir: The search start path.
    :param filename: The file name to search for.
    :param parent: If the file is found and parent is False, returns the path to the file.
        If parent is True the path to the file's parent directory is returned.
    :param limit: If limit is None, the search will continue up to the root directory.
        Otherwise a maximum of limit directories will be checked.
    :param aux_dirs: If aux_dirs is not empty and the file hierarchy search failed,
        those directories are also checked.
    """
    if aux_dirs is None:
        aux_dirs = []
    for d in _climb_dirs(start_dir, limit=limit):
        target = os.path.join(d, filename)

        if os.path.exists(target):
            if parent:
                return d

            return target

    for d in aux_dirs:
        d = os.path.expanduser(d)
        target = os.path.join(d, filename)

        if os.path.exists(target):
            if parent:
                return d

            return target

def _prettier_opts_in_package_json(package_json_file):
    try:
        with open(package_json_file) as package_file:
            json_data = json.load(package_file)
    except Exception:
        from .sthelper import log_warn
        log_warn("Cannot parse '{0}' file. Prettier options "
                 "defined in this file will be ignored.".format(package_json_file), True)
        return False

    try:
        if json_data['prettier']:
            return True
        return False
    except KeyError:
        return False

def is_mac_os():
    return platform.system() == 'Darwin'

def is_windows():
    return platform.system() == 'Windows' or os.name == 'nt'

def to_str(value):
    if value is None:
        return ''
    if value is True:
        return 'true'
    if value is False:
        return 'false'
    return text_type(value)

def is_bool_str(val):
    """Determine if the specified string :val is 'true' or 'false'.

    :param val: The value to check.
    :return: True if if val: is a boolean string, otherwise False.
    :rtype: bool
    """
    if val is None:
        return False
    if isinstance(val, string_types):
        val = val.lower().strip()
        if val == 'true' or val == 'false':
            return True
    return False

def trim_trailing_ws_and_lines(val):
    """Trim trailing whitespace and line-breaks at the end of a string.

    :param val: The value to trim.
    :return: The val with trailing whitespace and line-breaks removed.
    """
    if val is None:
        return val
    val = sub(r'\s+\Z', '', val)
    return val

def repeat_str(str_to_repeat, repeat_length):
    """Repeat a string to a certain length.

    :param str_to_repeat: The string to repeat. Normally a single char.
    :param repeat_length: The amount of times to repeat the string.
    :return: The repeated string.
    """
    quotient, remainder = divmod(repeat_length, len(str_to_repeat))
    return str_to_repeat * quotient + str_to_repeat[:remainder]

def list_to_str(list_to_convert):
    """Convert a list of values into string.

    Each value will be separated by a single space.

    :param list_to_convert: The list to convert to a string.
    :return: The list converted into a string.
    """
    return ' '.join(to_str(l) for l in list_to_convert)

def is_str_empty_or_whitespace_only(txt):
    if not txt or len(txt) == 0:
        return True
    # strip all whitespace/invisible chars to determine textual content:
    txt = sub(r'\s+', '', txt)
    if not txt or len(txt) == 0:
        return True
    return False

def is_str_none_or_empty(val):
    """Determine if the specified str val is None or an empty.

    :param val: The str to check.
    :return: True if if val: is None or an empty, otherwise False.
    :rtype: bool
    """
    if val is None:
        return True
    if isinstance(val, string_types):
        val = val.strip()
    if not val:
        return True
    return False

def get_file_abs_dir(filepath):
    return os.path.abspath(os.path.dirname(filepath))

def env_path_contains(path_to_look_for, env_path=None):
    """Check if the specified path is listed in OS environment path.

    :param path_to_look_for: The path the search for.
    :param env_path: The environment path str.
    :return: True if the find_path exists in the env_path.
    :rtype: bool
    """
    if not path_to_look_for:
        return False
    if not env_path:
        env_path = os.environ['PATH']
    path_to_look_for = str.replace(path_to_look_for, os.pathsep, '')
    paths = env_path.split(os.pathsep)
    for path in paths:
        if path == path_to_look_for:
            return True
    return False

def env_path_exists(path):
    if not path:
        return False
    if os.path.exists(str.replace(path, os.pathsep, '')):
        return True
    return False

def which(executable, path=None):
    if is_str_none_or_empty(executable):
        return None

    executable = os.path.normpath(executable)
    if os.path.isfile(executable) and os.access(executable, os.X_OK):
        return executable

    if is_str_none_or_empty(path):
        path = os.environ.get("PATH", os.defpath)
        if not path:
            return None

        if not is_windows():
            # add '/usr/local/bin' on macos/linux if not already in path.
            usr_local_bin = ':/usr/local/bin'
            if not env_path_contains(usr_local_bin, path) \
                    and env_path_exists(usr_local_bin):
                path += usr_local_bin

    search_paths = path.split(os.pathsep)

    if is_windows():
        # The current directory takes precedence on Windows.
        if os.curdir not in search_paths:
            search_paths.insert(0, os.curdir)

        # PATHEXT is necessary to check on Windows.
        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
        # See if the given file matches any of the expected path
        # extensions. This will allow us to short circuit when given
        # "python.exe". If it does match, only test that one, otherwise we
        # have to try others.
        # hat tip: https://github.com/pydanny/whichcraft/blob/master/whichcraft.py
        if any(executable.lower().endswith(ext.lower()) for ext in pathext):
            executable_files = [executable]
        else:
            executable_files = [executable + ext for ext in pathext]
    else:
        # On other platforms you don't have things like PATHEXT to tell you
        # what file suffixes are executable, so just pass on cmd as-is.
        executable_files = [executable]

    dirs_seen = set()
    for directory in search_paths:
        dir_normalized = os.path.normcase(directory)
        if dir_normalized not in dirs_seen:
            dirs_seen.add(dir_normalized)
            for exec_file in executable_files:
                exec_file_path = os.path.normpath(os.path.join(directory, exec_file))
                if os.path.isfile(exec_file_path) and os.access(exec_file_path, os.X_OK):
                    return exec_file_path

    return None

def get_proc_env():
    env = None
    if not is_windows():
        env = os.environ.copy()
        usr_path = ':/usr/local/bin'
        if not env_path_contains(usr_path) \
                and env_path_exists(usr_path):
            env['PATH'] += usr_path
    return env

def in_source_file_path_or_project_root(source_file_dir, st_project_path, filename):
    # check in source file dir:
    source_file_dir_ignore_path = os.path.join(source_file_dir, filename)
    if os.path.exists(source_file_dir_ignore_path):
        return source_file_dir_ignore_path

    # check in sublime text project root dir:
    sublime_text_project_dir_path = os.path.join(st_project_path, filename)
    if os.path.exists(sublime_text_project_dir_path):
        return sublime_text_project_dir_path

    return None

def resolve_prettier_ignore_path(source_file_dir, st_project_path):
    """Look for a '.prettierignore'

    Try to resolve a '.prettieringore' file in source file dir, or ST project root (#97).

    :return: The path (str) to a '.prettierignore' file (if one exists) in the active Sublime Text Project Window.
    """

    return in_source_file_path_or_project_root(source_file_dir, st_project_path, PRETTIER_IGNORE_FILE)

def format_error_message(error_message, error_code):
    # inject a line break between the error message, and debug output (legibility purposes):
    error_message = error_message.replace('[error] stdin: ', '\n[error] stdin: ')

    return '\nPrettier reported the following output:\n\n' \
           '{0}\n' \
           '\nPrettier process finished with exit code {1}.\n' \
        .format(error_message, '{0}'.format(error_code))

def format_debug_message(label, message, debug_enabled=False):
    if not debug_enabled:
        return
    header = ' {0} DEBUG - {1} '.format(PLUGIN_NAME, label)
    horizontal_rule = repeat_str('-', len(header))
    print('\n{0}\n{1}\n{2}\n\n''{3}'.format(
        horizontal_rule, header, horizontal_rule, message))

def get_cli_arg_value(additional_cli_args, arg_key, arg_val_can_be_empty=False, default=None):
    if not additional_cli_args or not arg_key:
        return default
    if not isinstance(additional_cli_args, dict):
        return default
    result = None
    for key, val in additional_cli_args.items():
        if key == arg_key:
            if arg_val_can_be_empty:
                result = key
            else:
                result = val
        break
    if result is None:
        return default
    return result

def ensure_file_has_ext(file_name, file_ext):
    if not file_name.endswith(file_ext):
        return '{0}{1}'.format(file_name, file_ext)
    return file_name
<commit_msg>Search all config types before moving up path hierarchy<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import
from __future__ import print_function
from __future__ import with_statement

import functools
import json
import os
import platform

from re import sub

from .const import IS_PY2
from .const import PLUGIN_NAME
from .const import PRETTIER_CONFIG_FILES
from .const import PRETTIER_IGNORE_FILE

if IS_PY2:
    text_type = unicode
    string_types = (str, unicode)
else:
    text_type = str
    string_types = (str,)

def memoize(obj):
    cache = obj.cache = {}

    @functools.wraps(obj)
    def memoizer(*args, **kwargs):
        key = str(args) + str(kwargs)
        if key not in cache:
            cache[key] = obj(*args, **kwargs)
        return cache[key]

    return memoizer

def contains(needle, haystack):
    if not needle or not haystack:
        return False
    return needle in haystack

@memoize
def find_prettier_config(start_dir, alt_dirs=None):
    """
    Find a prettier config file by searching up the file hierarchy.

    Hat tip to SublimeLinter 3!

    :param start_dir: The search start path.
    :param alt_dirs: If aux_dirs is not empty and the file hierarchy search failed,
        those directories are also checked.
    """
    dirs = _climb_dirs(start_dir, limit=500)
    for d in dirs:
        for config_file in PRETTIER_CONFIG_FILES:
            target = os.path.join(d, config_file)
            if os.path.exists(target):
                if config_file == 'package.json' and not _prettier_opts_in_package_json(prettier_config):
                    continue
                return target

    if alt_dirs is None:
        alt_dirs = []
    if '~' not in alt_dirs:
        alt_dirs.append('~')

    for d in alt_dirs:
        d = os.path.expanduser(d)
        for config_file in PRETTIER_CONFIG_FILES:
            target = os.path.join(d, config_file)
            if os.path.exists(target):
                if config_file == 'package.json' and not _prettier_opts_in_package_json(prettier_config):
                    continue
                return target

    return None

def _climb_dirs(start_dir, limit=None):
    """
    Generate directories, starting from start_dir.

    Hat tip goes to SublimeLinter 3.

    :param start_dir: The search start path.
    :param limit: If limit is None, the search will continue up to the root directory.
        Otherwise a maximum of limit directories will be checked.
    """
    right = True

    while right and (limit is None or limit > 0):
        yield start_dir
        start_dir, right = os.path.split(start_dir)

        if limit is not None:
            limit -= 1

def _prettier_opts_in_package_json(package_json_file):
    try:
        with open(package_json_file) as package_file:
            json_data = json.load(package_file)
    except Exception:
        from .sthelper import log_warn
        log_warn("Cannot parse '{0}' file. Prettier options "
                 "defined in this file will be ignored.".format(package_json_file), True)
        return False

    try:
        if json_data['prettier']:
            return True
        return False
    except KeyError:
        return False

def is_mac_os():
    return platform.system() == 'Darwin'

def is_windows():
    return platform.system() == 'Windows' or os.name == 'nt'

def to_str(value):
    if value is None:
        return ''
    if value is True:
        return 'true'
    if value is False:
        return 'false'
    return text_type(value)

def is_bool_str(val):
    """Determine if the specified string :val is 'true' or 'false'.

    :param val: The value to check.
    :return: True if if val: is a boolean string, otherwise False.
    :rtype: bool
    """
    if val is None:
        return False
    if isinstance(val, string_types):
        val = val.lower().strip()
        if val == 'true' or val == 'false':
            return True
    return False

def trim_trailing_ws_and_lines(val):
    """Trim trailing whitespace and line-breaks at the end of a string.

    :param val: The value to trim.
    :return: The val with trailing whitespace and line-breaks removed.
    """
    if val is None:
        return val
    val = sub(r'\s+\Z', '', val)
    return val

def repeat_str(str_to_repeat, repeat_length):
    """Repeat a string to a certain length.

    :param str_to_repeat: The string to repeat. Normally a single char.
    :param repeat_length: The amount of times to repeat the string.
    :return: The repeated string.
    """
    quotient, remainder = divmod(repeat_length, len(str_to_repeat))
    return str_to_repeat * quotient + str_to_repeat[:remainder]

def list_to_str(list_to_convert):
    """Convert a list of values into string.

    Each value will be separated by a single space.

    :param list_to_convert: The list to convert to a string.
    :return: The list converted into a string.
    """
    return ' '.join(to_str(l) for l in list_to_convert)

def is_str_empty_or_whitespace_only(txt):
    if not txt or len(txt) == 0:
        return True
    # strip all whitespace/invisible chars to determine textual content:
    txt = sub(r'\s+', '', txt)
    if not txt or len(txt) == 0:
        return True
    return False

def is_str_none_or_empty(val):
    """Determine if the specified str val is None or an empty.

    :param val: The str to check.
    :return: True if if val: is None or an empty, otherwise False.
    :rtype: bool
    """
    if val is None:
        return True
    if isinstance(val, string_types):
        val = val.strip()
    if not val:
        return True
    return False

def get_file_abs_dir(filepath):
    return os.path.abspath(os.path.dirname(filepath))

def env_path_contains(path_to_look_for, env_path=None):
    """Check if the specified path is listed in OS environment path.

    :param path_to_look_for: The path the search for.
    :param env_path: The environment path str.
    :return: True if the find_path exists in the env_path.
    :rtype: bool
    """
    if not path_to_look_for:
        return False
    if not env_path:
        env_path = os.environ['PATH']
    path_to_look_for = str.replace(path_to_look_for, os.pathsep, '')
    paths = env_path.split(os.pathsep)
    for path in paths:
        if path == path_to_look_for:
            return True
    return False

def env_path_exists(path):
    if not path:
        return False
    if os.path.exists(str.replace(path, os.pathsep, '')):
        return True
    return False

def which(executable, path=None):
    if is_str_none_or_empty(executable):
        return None

    executable = os.path.normpath(executable)
    if os.path.isfile(executable) and os.access(executable, os.X_OK):
        return executable

    if is_str_none_or_empty(path):
        path = os.environ.get("PATH", os.defpath)
        if not path:
            return None

        if not is_windows():
            # add '/usr/local/bin' on macos/linux if not already in path.
            usr_local_bin = ':/usr/local/bin'
            if not env_path_contains(usr_local_bin, path) \
                    and env_path_exists(usr_local_bin):
                path += usr_local_bin

    search_paths = path.split(os.pathsep)

    if is_windows():
        # The current directory takes precedence on Windows.
        if os.curdir not in search_paths:
            search_paths.insert(0, os.curdir)

        # PATHEXT is necessary to check on Windows.
        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
        # See if the given file matches any of the expected path
        # extensions. This will allow us to short circuit when given
        # "python.exe". If it does match, only test that one, otherwise we
        # have to try others.
        # hat tip: https://github.com/pydanny/whichcraft/blob/master/whichcraft.py
        if any(executable.lower().endswith(ext.lower()) for ext in pathext):
            executable_files = [executable]
        else:
            executable_files = [executable + ext for ext in pathext]
    else:
        # On other platforms you don't have things like PATHEXT to tell you
        # what file suffixes are executable, so just pass on cmd as-is.
        executable_files = [executable]

    dirs_seen = set()
    for directory in search_paths:
        dir_normalized = os.path.normcase(directory)
        if dir_normalized not in dirs_seen:
            dirs_seen.add(dir_normalized)
            for exec_file in executable_files:
                exec_file_path = os.path.normpath(os.path.join(directory, exec_file))
                if os.path.isfile(exec_file_path) and os.access(exec_file_path, os.X_OK):
                    return exec_file_path

    return None

def get_proc_env():
    env = None
    if not is_windows():
        env = os.environ.copy()
        usr_path = ':/usr/local/bin'
        if not env_path_contains(usr_path) and env_path_exists(usr_path):
            env['PATH'] += usr_path
    return env

def in_source_file_path_or_project_root(source_file_dir, st_project_path, filename):
    # check in source file dir:
    source_file_dir_ignore_path = os.path.join(source_file_dir, filename)
    if os.path.exists(source_file_dir_ignore_path):
        return source_file_dir_ignore_path

    # check in sublime text project root dir:
    sublime_text_project_dir_path = os.path.join(st_project_path, filename)
    if os.path.exists(sublime_text_project_dir_path):
        return sublime_text_project_dir_path

    return None

def resolve_prettier_ignore_path(source_file_dir, st_project_path):
    """Look for a '.prettierignore'

    Try to resolve a '.prettieringore' file in source file dir, or ST project root (#97).

    :return: The path (str) to a '.prettierignore' file (if one exists) in the active Sublime Text Project Window.
    """

    return in_source_file_path_or_project_root(source_file_dir, st_project_path, PRETTIER_IGNORE_FILE)

def format_error_message(error_message, error_code):
    # inject a line break between the error message, and debug output (legibility purposes):
    error_message = error_message.replace('[error] stdin: ', '\n[error] stdin: ')

    return '\nPrettier reported the following output:\n\n' \
           '{0}\n' \
           '\nPrettier process finished with exit code {1}.\n' \
        .format(error_message, '{0}'.format(error_code))

def format_debug_message(label, message, debug_enabled=False):
    if not debug_enabled:
        return
    header = ' {0} DEBUG - {1} '.format(PLUGIN_NAME, label)
    horizontal_rule = repeat_str('-', len(header))
    print('\n{0}\n{1}\n{2}\n\n''{3}'.format(
        horizontal_rule, header, horizontal_rule, message))

def get_cli_arg_value(additional_cli_args, arg_key, arg_val_can_be_empty=False, default=None):
    if not additional_cli_args or not arg_key:
        return default
    if not isinstance(additional_cli_args, dict):
        return default
    result = None
    for key, val in additional_cli_args.items():
        if key == arg_key:
            if arg_val_can_be_empty:
                result = key
            else:
                result = val
        break
    if result is None:
        return default
    return result

def ensure_file_has_ext(file_name, file_ext):
    if not file_name.endswith(file_ext):
        return '{0}{1}'.format(file_name, file_ext)
    return file_name
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import scrapy
import re
from scrapy.utils.gz import gunzip, is_gzipped

from locations.items import GeojsonPointItem

class PDQSpider(scrapy.Spider):
    name = "pdq"
    download_delay = 1.5
    allowed_domains = ['eatpdq.qatserver.com']
    start_urls = (
        'http://eatpdq.qatserver.com/sitemap/sitemap.gz',
    )

    def parse(self, response):

        sitemap = gunzip(response.body)
        regex = re.compile(r'http://eatpdq.qatserver.com/locations/\S+(?=</loc>)')
        city_urls = re.findall(regex, str(sitemap))

        for path in city_urls:
            yield scrapy.Request(
                path.strip(),
                callback=self.parse_store,
            )
        else:
            pass

    def parse_store(self, response):

        if response.xpath('//div[@class="hours"]/div').extract():
            storeHoursHTML = str(response.xpath('//div[@class="hours"]/div').extract()).replace("'",'').replace('[','').replace(']','')
            p = re.compile(r'<.*?>')
            storeHours = p.sub('', storeHoursHTML)
            storeHours = storeHours.replace('\\n', ' - ')
            storeHours = "".join(storeHours.strip())
        else:
            storeHours = response.xpath('//div[@class="hours"]/div').extract()

        properties = {
        'name': response.xpath('//div[@class="name"]/h1/text()').extract_first(),
        'ref': response.xpath('//div[@class="name"]/h1/text()').extract_first(),
        'addr_full': response.xpath('//div[@class="address"]/text()').extract_first().strip(),
        'city': response.xpath('//div[@class="address"]/text()[2]').extract_first().split(',')[0],
        'state': response.xpath('//div[@class="address"]/text()[2]').extract_first().split()[1],
        'postcode': response.xpath('//div[@class="address"]/text()[2]').extract_first().split()[-1],
        'phone': response.xpath('//tel/a[@href]/text()').extract_first(),
        'website': response.request.url,
        'opening_hours': storeHours,
        'lat': float(response.xpath('//div[@class="address"]/a/@href').extract_first().split('@')[1].split(',')[0]),
        'lon': float(response.xpath('//div[@class="address"]/a/@href').extract_first().split('@')[1].split(',')[1]),
        }

        yield GeojsonPointItem(**properties)<commit_msg>Add files via upload<commit_after># -*- coding: utf-8 -*-
import scrapy
import re
from scrapy.utils.gz import gunzip, is_gzipped

from locations.items import GeojsonPointItem

class PDQSpider(scrapy.Spider):
    name = "pdq"
    allowed_domains = ['eatpdq.qatserver.com']
    start_urls = (
        'http://eatpdq.qatserver.com/sitemap/sitemap.gz',
    )

    def parse(self, response):

        sitemap = gunzip(response.body)
        regex = re.compile(r'http://eatpdq.qatserver.com/locations/\S+(?=</loc>)')
        city_urls = re.findall(regex, str(sitemap))

        for path in city_urls:
            yield scrapy.Request(
                path.strip(),
                callback=self.parse_store,
            )
        else:
            pass

    def parse_store(self, response):

        if response.xpath('//div[@class="hours"]/div').extract():
            storeHoursHTML = str(response.xpath('//div[@class="hours"]/div').extract()).replace("'",'').replace('[','').replace(']','')
            p = re.compile(r'<.*?>')
            storeHours = p.sub('', storeHoursHTML)
            storeHours = storeHours.replace('\\n', ' - ')
            storeHours = "".join(storeHours.strip())
        else:
            storeHours = response.xpath('//div[@class="hours"]/div').extract()

        properties = {
        'name': response.xpath('//div[@class="name"]/h1/text()').extract_first(),
        'ref': response.xpath('//div[@class="name"]/h1/text()').extract_first(),
        'addr_full': response.xpath('//div[@class="address"]/text()').extract_first(),
        'city': response.xpath('//div[@class="address"]/text()[2]').extract_first().split(',')[0],
        'state': response.xpath('//div[@class="address"]/text()[2]').extract_first().split()[1],
        'postcode': response.xpath('//div[@class="address"]/text()[2]').extract_first().split()[-1],
        'phone': response.xpath('//tel/a[@href]/text()').extract_first(),
        'website': response.request.url,
        'opening_hours': storeHours,
        'lat': float(response.xpath('//div[@class="address"]/a/@href').extract_first().split('@')[1].split(',')[0]),
        'lon': float(response.xpath('//div[@class="address"]/a/@href').extract_first().split('@')[1].split(',')[1]),
        }

        yield GeojsonPointItem(**properties)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from types import NoneType

"""

"""
import datetime as dt
import collections

import pandas as pd
import geopandas as gp
import numpy as np
import shapely.geometry
import os
from hub.exceptions import warn

from hub.odhql.exceptions import OdhQLExecutionException

from opendatahub.utils.plugins import RegistrationMixin

import shapely.speedups

if shapely.speedups.available:
    shapely.speedups.enable()

class OdhType(RegistrationMixin):
    dtypes = ()
    ptypes = ()
    name = ''

    by_name = {}
    by_dtype = collections.defaultdict(list)
    by_ptype = collections.defaultdict(list)

    _is_abstract = True

    @classmethod
    def register_child(cls, name, bases, own_dict):
        if not own_dict.get('_is_abstract'):
            instance = cls()

            instance.ptype = cls.ptypes[0]
            instance.dtype = cls.dtypes[0]
            instance.name = cls.name or name.upper()

            setattr(OdhType, instance.name, instance)
            instance.by_name[instance.name] = instance
            for dtype in instance.dtypes:
                OdhType.by_dtype[dtype].append(instance)

            for ptype in instance.ptypes:
                OdhType.by_ptype[ptype].append(instance)

    @classmethod
    def identify_value(cls, value):
        type_ = type(value)
        if issubclass(type_, np.generic):
            return cls.by_dtype[type_]
        else:
            try:
                return cls.by_ptype[type_][0]
            except:
                pass

    @classmethod
    def identify_series(cls, s):
        by_dtype = cls.by_dtype[s.dtype.type]
        if len(by_dtype) == 1:
            return by_dtype[0]

        first = s.first_valid_index()
        if first is not None:
            value = s.iat[first]
            return cls.identify_value(value)
        else:
            return cls.TEXT

    def convert(self, series):
        return series._constructor(series.values.astype(self.dtypes[0]), index=series.index).__finalize__(series)

    def __repr__(self):
        return 'OdhType({})'.format(self.name)

    def __str__(self):
        return self.name

    def __eq__(self, other):
        if not isinstance(other, self.__class__):
            return False
        return self.name == other.name

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(self.name)

class IntegerType(OdhType):
    name = 'INTEGER'
    dtypes = np.int32,
    ptypes = int,

    def convert(self, series):
        return series._constructor(series.values.astype(np.float_).astype(self.dtypes[0]),
                                   index=series.index).__finalize__(series)

class BigIntType(IntegerType):
    name = 'BIGINT'
    dtypes = np.int64,
    ptypes = int,

class SmallIntType(IntegerType):
    name = 'SMALLINT'
    dtypes = np.int16,
    ptypes = int,

class FloatType(OdhType):
    name = 'FLOAT'
    dtypes = np.float64,
    ptypes = float,

class DateTimeType(OdhType):
    name = 'DATETIME'
    dtypes = np.datetime64,
    ptypes = pd.Timestamp,

    def convert(self, series):
        if series.odh_type.ptype not in (int, float):
            raise OdhQLExecutionException('Unable to convert from {} to {}'.format(str(series.odh_type), str(self)))
        return series._constructor(pd.to_datetime(series, unit='s'), index=series.index).__finalize__(series)

class IntervalType(OdhType):
    name = 'INTERVAL'
    dtypes = pd.Timedelta,
    ptypes = dt.timedelta,

class BooleanType(OdhType):
    name = 'BOOLEAN'
    dtypes = np.bool_,
    ptypes = bool,

class TextType(OdhType):
    name = 'TEXT'
    dtypes = np.object_,
    ptypes = unicode, str, NoneType

    def convert(self, series):
        try:
            return series._constructor(series.values.astype(unicode).astype(object), index=series.index).__finalize__(
                series)
        except:  # fallback, slower
            return series._constructor(series.astype(unicode).astype(object), index=series.index).__finalize__(series)

class GeometryType(OdhType):
    name = 'GEOMETRY'
    dtypes = np.object_,
    ptypes = (
        shapely.geometry.Point, shapely.geometry.LineString, shapely.geometry.Polygon, shapely.geometry.LinearRing,
        shapely.geometry.MultiLineString, shapely.geometry.MultiPoint, shapely.geometry.MultiPolygon
    )

from shapely.geometry.base import GEOMETRY_TYPES

class OdhFrame(pd.DataFrame):
    _metadata = ['name']

    @classmethod
    def from_df(cls, df, name=None):
        odf = cls(df, index=df.index).__finalize__(df)
        odf.name = name or getattr(df, 'name', None)
        return odf

    def rename(self, *args, **kwargs):
        old = self.copy() if kwargs.get('inplace', False) else self
        df = super(OdhFrame, self).rename(*args, **kwargs)
        df = df or self
        return df.__finalize__(old, method='rename')

    def __finalize__(self, other, method=None, **kwargs):
        super(OdhFrame, self).__finalize__(other, method=method, **kwargs)
        if method == 'concat':
            other = other.objs[0]
        elif method == 'merge':
            other = other.left

        if method == 'rename':
            for c0, c1 in zip(self.columns, other.columns):
                self[c0].__finalize__(other[c1], method='rename')
        else:
            for c in self.columns:
                if c in other:
                    self[c].__finalize__(other[c])
        return self

    @property
    def has_geoms(self):
        return OdhType.GEOMETRY in [s.odh_type for c, s in self.iteritems()]

    def to_gdf(self, supported_geoms=set(GEOMETRY_TYPES)):
        supported_geom_types = set(supported_geoms)

        # if we have multiple geometry columns and none of them is called "geometry" we pick the first
        # the formatters require a single geometry column as most file formats only support one
        df = self.copy()
        df.__class__ = gp.GeoDataFrame
        geoseries = collections.OrderedDict([(c, s) for c, s in df.iteritems() if s.odh_type == OdhType.GEOMETRY])
        geometry = geoseries.get('geometry', geoseries.values()[0]).copy()
        if geometry.name != 'geometry':
            warn('Column "{}" containing geometry was renamed to "geometry".'.format(geometry.name))
            df.rename(columns={geometry.name: 'geometry'}, inplace=True)

        # remove additional geometry series, not supported
        other_geoms = set(geoseries.keys()) - {geometry.name}
        if other_geoms:
            warn('Multiple geometries not supported. Dropped column(s): {}.'.format(', '.join(other_geoms)))
            df.drop(other_geoms, axis=1, inplace=True)

        df.crs = geometry.crs

        geom_type = geometry.first_valid_entry.geom_type if geometry.first_valid_entry else 'Point'
        geometry = gp.GeoSeries(geometry, crs=geometry.crs)
        geometry.iloc[~np.in1d(geometry[~pd.isnull(geometry)].geom_type, list(supported_geom_types))] = None
        geometry.fillna(EmptyGeometryMarker(geom_type), inplace=True)

        geom_types = geometry.geom_type
        unique_geom_types = geom_types.unique()

        if len(unique_geom_types) > 1:
            common_type = os.path.commonprefix([gt[::-1] for gt in geom_types.unique()])[::-1]
            multi_type = 'Multi' + common_type
            mask = None
            if common_type in supported_geom_types and multi_type in supported_geom_types:
                constructor = getattr(shapely.geometry, 'Multi' + common_type)
                mask = geom_types == common_type
            elif shapely.geometry.GeometryCollection.__name__ in supported_geom_types:
                constructor = shapely.geometry.GeometryCollection
                mask = Ellipsis
            else:
                geometry[geometry.geom_type != geom_type] = EmptyGeometryMarker(geom_type)

            if mask is not None:
                geometry[mask] = geometry[mask].apply(lambda g: constructor([g]))

        df['geometry'] = geometry
        return df

    @property
    def _constructor(self):
        return OdhFrame

    @property
    def _constructor_sliced(self):
        return OdhSeries

    def __getstate__(self):
        d = dict(_data=super(OdhFrame, self).__getstate__())
        d.update({attr: getattr(self, attr, None) for attr in self._metadata})
        d['series_metadata'] = {c: s._get_metadata() for c, s in self.iteritems()}
        return d

    def __setstate__(self, state):
        super(OdhFrame, self).__setstate__(state.pop('_data', None))
        for attr in self._metadata:
            setattr(self, attr, state.get(attr, None))

        series_metadata = state.pop('series_metadata', None)
        for c, s in self.iteritems():
            meta = series_metadata.get(c, {})
            for attr in self._constructor_sliced._metadata:
                setattr(s, attr, meta.get(attr, None))

    def as_safe_serializable(self):
        """
        :return: DataFrame which contains only exportable data (no objects)
        """
        df = self.copy()
        for col in df.columns:
            df[col] = df[col].as_safe_serializable()

        return df

class OdhSeries(pd.Series):
    _metadata = ['name', '_odh_type', 'crs']

    def __init__(self, data, *args, **kwargs):
        if isinstance(data, pd.Series):
            self.__finalize__(data)
        super(OdhSeries, self).__init__(data, *args, **kwargs)
        self._odh_type = None
        self._first_valid_entry = None
        if not getattr(self, 'crs', None):
            self.crs = kwargs.get('crs', {})

    @property
    def odh_type(self):
        if not self._odh_type:
            self._odh_type = OdhType.identify_series(self)
        return self._odh_type

    def __finalize__(self, other, method=None, **kwargs):
        name = self.name if method == 'rename' else other.name
        super(OdhSeries, self).__finalize__(other, method=method, **kwargs)
        self.name = name
        return self

    @property
    def _constructor(self):
        return OdhSeries

    def _get_metadata(self):
        return {attr: getattr(self, attr, None) for attr in self._metadata}

    def __getstate__(self):
        d = super(OdhSeries, self).__getstate__()
        d.update(self._get_metadata())
        return d

    def __setstate__(self, state):
        super(OdhSeries, self).__setstate__(state)
        for attr in self._metadata:
            setattr(self, attr, state.get(attr, None))

    @property
    def first_valid_entry(self):
        if self._first_valid_entry is None:
            ix = self.first_valid_index()
            if ix is not None:
                self._first_valid_entry = self.iat[ix]
        return self._first_valid_entry

    @classmethod
    def concat(cls, series, *args, **kwargs):
        df = OdhFrame(pd.concat(series, *args, **kwargs))
        for i, (name, s) in enumerate(df.iteritems()):
            s.__finalize__(series[i])
        return df

    def to_crs(self, crs):
        if not self.crs:
            raise OdhQLExecutionException('Missing SRID on source column')
        return self.geom_op('to_crs', crs)

    def geom_op(self, op, *args, **kwargs):
        assert self.odh_type == OdhType.GEOMETRY, 'Cannot execute geometry function on non-geometry column'
        s = self.copy()
        mask = ~pd.isnull(s)
        gs = gp.GeoSeries(s[mask], crs=self.crs)
        attr = getattr(gs, op)
        result = attr(*args, **kwargs) if callable(attr) else attr
        if isinstance(result, pd.Series):
            s[mask] = result
            s.crs = getattr(result, 'crs', self.crs)
            result = s
        elif isinstance(result, pd.DataFrame):
            result = result.reindex(self.index)

        return result

    def as_safe_serializable(self):
        s = self
        if s.odh_type is not OdhType.TEXT:
            s = OdhType.TEXT.convert(s)

        return s

class EmptyGeometryMarker(shapely.geometry.Point):
    def __init__(self, geom_type='Point'):
        self.__geom_type = geom_type
        super(EmptyGeometryMarker, self).__init__()

    def geometryType(self):
        return self.__geom_type

def monkey_patch_geopandas():
    import geopandas.geodataframe as geodataframe

    original_mapping = geodataframe.mapping

    # https://github.com/Toblerity/Shapely/issues/250
    def mapping(geometry, *a, **kw):
        if geometry.__class__ == EmptyGeometryMarker:
            return None
        return original_mapping(geometry, *a, **kw)

    geodataframe.mapping = mapping

monkey_patch_geopandas()
<commit_msg>class/module documentation<commit_after># -*- coding: utf-8 -*-

"""
Custom extensions to Series/GeoSeries and DataFrame/GeoDataFrame from pandas/geopandas.
"""

from __future__ import unicode_literals
from types import NoneType

import datetime as dt
import collections

import pandas as pd
import geopandas as gp
import numpy as np
import shapely.geometry
import os
from hub.exceptions import warn

from hub.odhql.exceptions import OdhQLExecutionException

from opendatahub.utils.plugins import RegistrationMixin

import shapely.speedups

if shapely.speedups.available:
    shapely.speedups.enable()

class OdhType(RegistrationMixin):
    """ Base class for the supported data types. """
    dtypes = ()
    ptypes = ()
    name = ''

    by_name = {}
    by_dtype = collections.defaultdict(list)
    by_ptype = collections.defaultdict(list)

    _is_abstract = True

    @classmethod
    def register_child(cls, name, bases, own_dict):
        """ Implementation for RegistrationMixin """
        if not own_dict.get('_is_abstract'):
            instance = cls()

            instance.ptype = cls.ptypes[0]
            instance.dtype = cls.dtypes[0]
            instance.name = cls.name or name.upper()

            setattr(OdhType, instance.name, instance)
            instance.by_name[instance.name] = instance
            for dtype in instance.dtypes:
                OdhType.by_dtype[dtype].append(instance)

            for ptype in instance.ptypes:
                OdhType.by_ptype[ptype].append(instance)

    @classmethod
    def identify_value(cls, value):
        """ Tries to guess the data type of a single value.
        :param value: Value to examine.
        :return: Data type, if one was detected. Otherwise, None is returned.
        """
        type_ = type(value)
        if issubclass(type_, np.generic):
            return cls.by_dtype[type_]
        else:
            try:
                return cls.by_ptype[type_][0]
            except:
                pass

    @classmethod
    def identify_series(cls, s):
        """ Tries to guess the data type of a series.
        :param s: Series to examine.
        :return: Data type, if one was detected. If the series contained no not-None values, TEXT is returned.
        """
        by_dtype = cls.by_dtype[s.dtype.type]
        if len(by_dtype) == 1:
            return by_dtype[0]

        first = s.first_valid_index()
        if first is not None:
            value = s.iat[first]
            return cls.identify_value(value)
        else:
            return cls.TEXT

    def convert(self, series):
        """ Convert a series to a different data type.
        :param series: Series to convert.
        :return: Converted series.
        """
        return series._constructor(series.values.astype(self.dtypes[0]), index=series.index).__finalize__(series)

    def __repr__(self):
        return 'OdhType({})'.format(self.name)

    def __str__(self):
        return self.name

    def __eq__(self, other):
        if not isinstance(other, self.__class__):
            return False
        return self.name == other.name

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(self.name)

class IntegerType(OdhType):
    """ 32-bit integer type and base for other integer types. """
    name = 'INTEGER'
    dtypes = np.int32,
    ptypes = int,

    def convert(self, series):
        """ Converts the series to integer.
        :type series: hub.structures.frame.OdhSeries
        :return: Converted series.
        """
        return series._constructor(series.values.astype(np.float_).astype(self.dtypes[0]),
                                   index=series.index).__finalize__(series)

class BigIntType(IntegerType):
    """ 64-bit integer type. """
    name = 'BIGINT'
    dtypes = np.int64,
    ptypes = int,

class SmallIntType(IntegerType):
    """ 16-bit integer type. """
    name = 'SMALLINT'
    dtypes = np.int16,
    ptypes = int,

class FloatType(OdhType):
    """ 64-bit floating point type. """
    name = 'FLOAT'
    dtypes = np.float64,
    ptypes = float,

class DateTimeType(OdhType):
    """ Date/time type. """
    name = 'DATETIME'
    dtypes = np.datetime64,
    ptypes = pd.Timestamp,

    def convert(self, series):
        """ Converts the series to datetime. This interprets the series as unix epoch (seconds from
        1970-01-01T00:00:00), so this only works for integer/float types.
        :type series: hub.structures.frame.OdhSeries
        :return: converted series.
        """
        if series.odh_type.ptype not in (int, float):
            raise OdhQLExecutionException('Unable to convert from {} to {}'.format(str(series.odh_type), str(self)))
        return series._constructor(pd.to_datetime(series, unit='s'), index=series.index).__finalize__(series)

class IntervalType(OdhType):
    """ Date/time interval type. """
    name = 'INTERVAL'
    dtypes = pd.Timedelta,
    ptypes = dt.timedelta,

class BooleanType(OdhType):
    """ Boolean type. """
    name = 'BOOLEAN'
    dtypes = np.bool_,
    ptypes = bool,

class TextType(OdhType):
    """ Text type. This is also used for all-None series.. """
    name = 'TEXT'
    dtypes = np.object_,
    ptypes = unicode, str, NoneType

    def convert(self, series):
        """ Convert the series to text.
        :type series: hub.structures.frame.OdhSeries
        :return: Converted series.
        """
        try:
            return series._constructor(series.values.astype(unicode).astype(object), index=series.index).__finalize__(
                series)
        except:  # fallback, slower
            return series._constructor(series.astype(unicode).astype(object), index=series.index).__finalize__(series)

class GeometryType(OdhType):
    """ Geometry type. Supported are Point, LineString, Polygon, LinearRing and their Multi* versions. """
    name = 'GEOMETRY'
    dtypes = np.object_,
    ptypes = (
        shapely.geometry.Point, shapely.geometry.LineString, shapely.geometry.Polygon, shapely.geometry.LinearRing,
        shapely.geometry.MultiLineString, shapely.geometry.MultiPoint, shapely.geometry.MultiPolygon
    )

from shapely.geometry.base import GEOMETRY_TYPES

class OdhFrame(pd.DataFrame):
    """ Custom extensions for pandas' DataFrame. """
    _metadata = ['name']

    @classmethod
    def from_df(cls, df, name=None):
        """ Wrap a data frame.
        :param df: Data frame to wrap.
        :param name: The data frame's name.
        :return: Newly created OdhFrame instance.
        """
        odf = cls(df, index=df.index).__finalize__(df)
        odf.name = name or getattr(df, 'name', None)
        return odf

    def rename(self, *args, **kwargs):
        """ Rename a data frame.
        :param args:
        :param kwargs:
        :return: Renamed data frame.
        """
        old = self.copy() if kwargs.get('inplace', False) else self
        df = super(OdhFrame, self).rename(*args, **kwargs)
        df = df or self
        return df.__finalize__(old, method='rename')

    def __finalize__(self, other, method=None, **kwargs):
        super(OdhFrame, self).__finalize__(other, method=method, **kwargs)
        if method == 'concat':
            other = other.objs[0]
        elif method == 'merge':
            other = other.left

        if method == 'rename':
            for c0, c1 in zip(self.columns, other.columns):
                self[c0].__finalize__(other[c1], method='rename')
        else:
            for c in self.columns:
                if c in other:
                    self[c].__finalize__(other[c])
        return self

    @property
    def has_geoms(self):
        """
        :return: True if this data frame contains geometry columns.
        """
        return OdhType.GEOMETRY in [s.odh_type for c, s in self.iteritems()]

    def to_gdf(self, supported_geoms=set(GEOMETRY_TYPES)):
        supported_geom_types = set(supported_geoms)

        # if we have multiple geometry columns and none of them is called "geometry" we pick the first
        # the formatters require a single geometry column as most file formats only support one
        df = self.copy()
        df.__class__ = gp.GeoDataFrame
        geoseries = collections.OrderedDict([(c, s) for c, s in df.iteritems() if s.odh_type == OdhType.GEOMETRY])
        geometry = geoseries.get('geometry', geoseries.values()[0]).copy()
        if geometry.name != 'geometry':
            warn('Column "{}" containing geometry was renamed to "geometry".'.format(geometry.name))
            df.rename(columns={geometry.name: 'geometry'}, inplace=True)

        # remove additional geometry series, not supported
        other_geoms = set(geoseries.keys()) - {geometry.name}
        if other_geoms:
            warn('Multiple geometries not supported. Dropped column(s): {}.'.format(', '.join(other_geoms)))
            df.drop(other_geoms, axis=1, inplace=True)

        df.crs = geometry.crs

        geom_type = geometry.first_valid_entry.geom_type if geometry.first_valid_entry else 'Point'
        geometry = gp.GeoSeries(geometry, crs=geometry.crs)
        geometry.iloc[~np.in1d(geometry[~pd.isnull(geometry)].geom_type, list(supported_geom_types))] = None
        geometry.fillna(EmptyGeometryMarker(geom_type), inplace=True)

        geom_types = geometry.geom_type
        unique_geom_types = geom_types.unique()

        if len(unique_geom_types) > 1:
            common_type = os.path.commonprefix([gt[::-1] for gt in geom_types.unique()])[::-1]
            multi_type = 'Multi' + common_type
            mask = None
            if common_type in supported_geom_types and multi_type in supported_geom_types:
                constructor = getattr(shapely.geometry, 'Multi' + common_type)
                mask = geom_types == common_type
            elif shapely.geometry.GeometryCollection.__name__ in supported_geom_types:
                constructor = shapely.geometry.GeometryCollection
                mask = Ellipsis
            else:
                geometry[geometry.geom_type != geom_type] = EmptyGeometryMarker(geom_type)

            if mask is not None:
                geometry[mask] = geometry[mask].apply(lambda g: constructor([g]))

        df['geometry'] = geometry
        return df

    @property
    def _constructor(self):
        return OdhFrame

    @property
    def _constructor_sliced(self):
        return OdhSeries

    def __getstate__(self):
        d = dict(_data=super(OdhFrame, self).__getstate__())
        d.update({attr: getattr(self, attr, None) for attr in self._metadata})
        d['series_metadata'] = {c: s._get_metadata() for c, s in self.iteritems()}
        return d

    def __setstate__(self, state):
        super(OdhFrame, self).__setstate__(state.pop('_data', None))
        for attr in self._metadata:
            setattr(self, attr, state.get(attr, None))

        series_metadata = state.pop('series_metadata', None)
        for c, s in self.iteritems():
            meta = series_metadata.get(c, {})
            for attr in self._constructor_sliced._metadata:
                setattr(s, attr, meta.get(attr, None))

    def as_safe_serializable(self):
        """
        :return: DataFrame which contains only exportable data (no objects)
        """
        df = self.copy()
        for col in df.columns:
            df[col] = df[col].as_safe_serializable()

        return df

class OdhSeries(pd.Series):
    """ Custom extensions for pandas' Series. """
    _metadata = ['name', '_odh_type', 'crs']

    def __init__(self, data, *args, **kwargs):
        if isinstance(data, pd.Series):
            self.__finalize__(data)
        super(OdhSeries, self).__init__(data, *args, **kwargs)
        self._odh_type = None
        self._first_valid_entry = None
        if not getattr(self, 'crs', None):
            self.crs = kwargs.get('crs', {})

    @property
    def odh_type(self):
        if not self._odh_type:
            self._odh_type = OdhType.identify_series(self)
        return self._odh_type

    def __finalize__(self, other, method=None, **kwargs):
        name = self.name if method == 'rename' else other.name
        super(OdhSeries, self).__finalize__(other, method=method, **kwargs)
        self.name = name
        return self

    @property
    def _constructor(self):
        return OdhSeries

    def _get_metadata(self):
        return {attr: getattr(self, attr, None) for attr in self._metadata}

    def __getstate__(self):
        d = super(OdhSeries, self).__getstate__()
        d.update(self._get_metadata())
        return d

    def __setstate__(self, state):
        super(OdhSeries, self).__setstate__(state)
        for attr in self._metadata:
            setattr(self, attr, state.get(attr, None))

    @property
    def first_valid_entry(self):
        if self._first_valid_entry is None:
            ix = self.first_valid_index()
            if ix is not None:
                self._first_valid_entry = self.iat[ix]
        return self._first_valid_entry

    @classmethod
    def concat(cls, series, *args, **kwargs):
        df = OdhFrame(pd.concat(series, *args, **kwargs))
        for i, (name, s) in enumerate(df.iteritems()):
            s.__finalize__(series[i])
        return df

    def to_crs(self, crs):
        if not self.crs:
            raise OdhQLExecutionException('Missing SRID on source column')
        return self.geom_op('to_crs', crs)

    def geom_op(self, op, *args, **kwargs):
        assert self.odh_type == OdhType.GEOMETRY, 'Cannot execute geometry function on non-geometry column'
        s = self.copy()
        mask = ~pd.isnull(s)
        gs = gp.GeoSeries(s[mask], crs=self.crs)
        attr = getattr(gs, op)
        result = attr(*args, **kwargs) if callable(attr) else attr
        if isinstance(result, pd.Series):
            s[mask] = result
            s.crs = getattr(result, 'crs', self.crs)
            result = s
        elif isinstance(result, pd.DataFrame):
            result = result.reindex(self.index)

        return result

    def as_safe_serializable(self):
        s = self
        if s.odh_type is not OdhType.TEXT:
            s = OdhType.TEXT.convert(s)

        return s

class EmptyGeometryMarker(shapely.geometry.Point):
    """ Replacement for empty geometry values - certain file formats don't handle those well at all. """
    def __init__(self, geom_type='Point'):
        self.__geom_type = geom_type
        super(EmptyGeometryMarker, self).__init__()

    def geometryType(self):
        return self.__geom_type

def monkey_patch_geopandas():
    """ Fix None handling in geopandas/shapely. """
    import geopandas.geodataframe as geodataframe

    original_mapping = geodataframe.mapping

    # https://github.com/Toblerity/Shapely/issues/250
    def mapping(geometry, *a, **kw):
        if geometry.__class__ == EmptyGeometryMarker:
            return None
        return original_mapping(geometry, *a, **kw)

    geodataframe.mapping = mapping

monkey_patch_geopandas()
<|endoftext|>