<commit_before>from django.core.mail import EmailMessage

from plata.shop import signals

class BaseHandler(object):
    @classmethod
    def register(cls, **kwargs):
        instance = cls(**kwargs)

        dispatch_uid = kwargs.get('dispatch_uid', cls.__name__)

        signals.contact_created.connect(instance.on_contact_created,
            dispatch_uid=dispatch_uid)
        signals.order_confirmed.connect(instance.on_order_confirmed,
            dispatch_uid=dispatch_uid)
        signals.order_completed.connect(instance.on_order_completed,
            dispatch_uid=dispatch_uid)

        return instance

    def on_contact_created(self, sender, **kwargs): pass
    def on_order_confirmed(self, sender, **kwargs): pass
    def on_order_completed(self, sender, **kwargs): pass

class ConsoleHandler(BaseHandler):
    def __init__(self, stream):
        self.stream = stream

    def on_contact_created(self, sender, **kwargs):
        print >>self.stream, 'Contact created: %s' % kwargs.get('contact')

    def on_order_confirmed(self, sender, **kwargs):
        print >>self.stream, 'Order confirmed: %s' % kwargs.get('order')

    def on_order_completed(self, sender, **kwargs):
        print >>self.stream, 'Order completed: %s, payment %s, new discount %s' % (
            kwargs.get('order'),
            kwargs.get('payment'),
            kwargs.get('remaining_discount'))

class EmailHandler(BaseHandler):
    def on_contact_created(self, sender, **kwargs):
        contact = kwargs.get('contact')
        if not contact:
            return

        message = EmailMessage(
            subject='Account created',
            body='Your account has been created.',
            to=[contact.user.email],
            )
        message.send()

    def on_order_confirmed(self, sender, **kwargs):
        pass

    def on_order_completed(self, sender, **kwargs):
        order = kwargs.get('order')
        if not order:
            return

        message = EmailMessage(
            subject='Order completed',
            body='Your order has been successfully paid.',
            to=[order.email],
            )
        message.send()
<commit_msg>Remove unused code<commit_after>from django.core.mail import EmailMessage

from plata.shop import signals

class BaseHandler(object):
    @classmethod
    def register(cls, **kwargs):
        instance = cls(**kwargs)

        dispatch_uid = kwargs.get('dispatch_uid', cls.__name__)

        signals.contact_created.connect(instance.on_contact_created,
            dispatch_uid=dispatch_uid)
        signals.order_confirmed.connect(instance.on_order_confirmed,
            dispatch_uid=dispatch_uid)
        signals.order_completed.connect(instance.on_order_completed,
            dispatch_uid=dispatch_uid)

        return instance

    def on_contact_created(self, sender, **kwargs): pass
    def on_order_confirmed(self, sender, **kwargs): pass
    def on_order_completed(self, sender, **kwargs): pass

class ConsoleHandler(BaseHandler):
    def __init__(self, stream):
        self.stream = stream

    def on_contact_created(self, sender, **kwargs):
        print >>self.stream, 'Contact created: %s' % kwargs.get('contact')

    def on_order_confirmed(self, sender, **kwargs):
        print >>self.stream, 'Order confirmed: %s' % kwargs.get('order')

    def on_order_completed(self, sender, **kwargs):
        print >>self.stream, 'Order completed: %s, payment %s, new discount %s' % (
            kwargs.get('order'),
            kwargs.get('payment'),
            kwargs.get('remaining_discount'))

class EmailHandler(BaseHandler):
    def on_contact_created(self, sender, **kwargs):
        contact = kwargs.get('contact')
        if not contact:
            return

        message = EmailMessage(
            subject='Account created',
            body='Your account has been created.',
            to=[contact.user.email],
            )
        message.send()

    def on_order_completed(self, sender, **kwargs):
        order = kwargs.get('order')
        if not order:
            return

        message = EmailMessage(
            subject='Order completed',
            body='Your order has been successfully paid.',
            to=[order.email],
            )
        message.send()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
$Id$

Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
See the file 'doc/COPYING' for copying permission
"""

from lib.core.common import randomStr
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.enums import PLACE
from lib.core.exception import sqlmapNoneDataException
from lib.request import inject
from lib.techniques.inband.union.use import unionUse

from plugins.generic.filesystem import Filesystem as GenericFilesystem

class Filesystem(GenericFilesystem):
    def __init__(self):
        GenericFilesystem.__init__(self)

    def unionReadFile(self, rFile):
        infoMsg = "fetching file: '%s'" % rFile
        logger.info(infoMsg)

        result = inject.getValue("SELECT HEX(LOAD_FILE('%s'))" % rFile)

        return result

    def stackedReadFile(self, rFile):
        infoMsg = "fetching file: '%s'" % rFile
        logger.info(infoMsg)

        self.createSupportTbl(self.fileTblName, self.tblField, "longtext")
        self.getRemoteTempPath()

        tmpFile = "%s/tmpf%s" % (conf.tmpPath, randomStr(lowercase=True))

        debugMsg  = "saving hexadecimal encoded content of file '%s' " % rFile
        debugMsg += "into temporary file '%s'" % tmpFile
        logger.debug(debugMsg)
        inject.goStacked("SELECT HEX(LOAD_FILE('%s')) INTO DUMPFILE '%s'" % (rFile, tmpFile))

        debugMsg  = "loading the content of hexadecimal encoded file "
        debugMsg += "'%s' into support table" % rFile
        logger.debug(debugMsg)
        inject.goStacked("LOAD DATA INFILE '%s' INTO TABLE %s FIELDS TERMINATED BY '%s' (%s)" % (tmpFile, self.fileTblName, randomStr(10), self.tblField))

        length = inject.getValue("SELECT LENGTH(%s) FROM %s" % (self.tblField, self.fileTblName), sort=False, resumeValue=False, charsetType=2)

        if length is None or not length.isdigit() or not len(length) or length in ( "0", "1" ):
            errMsg  = "unable to retrieve the content of the "
            errMsg += "file '%s'" % rFile
            raise sqlmapNoneDataException, errMsg

        length   = int(length)
        sustrLen = 1024

        if length > sustrLen:
            result = []

            for i in range(1, length, sustrLen):
                chunk = inject.getValue("SELECT MID(%s, %d, %d) FROM %s" % (self.tblField, i, sustrLen, self.fileTblName), unpack=False, sort=False, resumeValue=False, charsetType=3)

                result.append(chunk)
        else:
            result = inject.getValue("SELECT %s FROM %s" % (self.tblField, self.fileTblName), sort=False, resumeValue=False, charsetType=3)

        return result

    def unionWriteFile(self, wFile, dFile, fileType, confirm=True):
        logger.debug("encoding file to its hexadecimal string value")

        fcEncodedList   = self.fileEncode(wFile, "hex", True)
        fcEncodedStr    = fcEncodedList[0]
        fcEncodedStrLen = len(fcEncodedStr)

        if kb.injection.place == PLACE.GET and fcEncodedStrLen > 8000:
            warnMsg  = "the injection is on a GET parameter and the file "
            warnMsg += "to be written hexadecimal value is %d " % fcEncodedStrLen
            warnMsg += "bytes, this might cause errors in the file "
            warnMsg += "writing process"
            logger.warn(warnMsg)

        debugMsg = "exporting the %s file content to file '%s'" % (fileType, dFile)
        logger.debug(debugMsg)

        sqlQuery = "%s INTO DUMPFILE '%s'" % (fcEncodedStr, dFile)
        unionUse(sqlQuery, unescape=False)

        if confirm:
            self.askCheckWrittenFile(wFile, dFile, fileType)

    def stackedWriteFile(self, wFile, dFile, fileType, confirm=True):
        debugMsg  = "creating a support table to write the hexadecimal "
        debugMsg += "encoded file to"
        logger.debug(debugMsg)

        self.createSupportTbl(self.fileTblName, self.tblField, "longblob")

        logger.debug("encoding file to its hexadecimal string value")
        fcEncodedList = self.fileEncode(wFile, "hex", False)

        debugMsg  = "forging SQL statements to write the hexadecimal "
        debugMsg += "encoded file to the support table"
        logger.debug(debugMsg)

        sqlQueries = self.fileToSqlQueries(fcEncodedList)

        logger.debug("inserting the hexadecimal encoded file to the support table")

        for sqlQuery in sqlQueries:
            inject.goStacked(sqlQuery)

        debugMsg = "exporting the %s file content to file '%s'" % (fileType, dFile)
        logger.debug(debugMsg)

        # Reference: http://dev.mysql.com/doc/refman/5.1/en/select.html
        inject.goStacked("SELECT %s FROM %s INTO DUMPFILE '%s'" % (self.tblField, self.fileTblName, dFile), silent=True)

        if confirm:
            self.askCheckWrittenFile(wFile, dFile, fileType)
<commit_msg>Code cleanup<commit_after>#!/usr/bin/env python

"""
$Id$

Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
See the file 'doc/COPYING' for copying permission
"""

from lib.core.common import randomStr
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.enums import PLACE
from lib.core.exception import sqlmapNoneDataException
from lib.request import inject
from lib.techniques.inband.union.use import unionUse

from plugins.generic.filesystem import Filesystem as GenericFilesystem

class Filesystem(GenericFilesystem):
    def __init__(self):
        GenericFilesystem.__init__(self)

    def unionReadFile(self, rFile):
        infoMsg = "fetching file: '%s'" % rFile
        logger.info(infoMsg)

        result = inject.getValue("SELECT HEX(LOAD_FILE('%s'))" % rFile)

        return result

    def stackedReadFile(self, rFile):
        infoMsg = "fetching file: '%s'" % rFile
        logger.info(infoMsg)

        self.createSupportTbl(self.fileTblName, self.tblField, "longtext")
        self.getRemoteTempPath()

        tmpFile = "%s/tmpf%s" % (conf.tmpPath, randomStr(lowercase=True))

        debugMsg  = "saving hexadecimal encoded content of file '%s' " % rFile
        debugMsg += "into temporary file '%s'" % tmpFile
        logger.debug(debugMsg)
        inject.goStacked("SELECT HEX(LOAD_FILE('%s')) INTO DUMPFILE '%s'" % (rFile, tmpFile))

        debugMsg  = "loading the content of hexadecimal encoded file "
        debugMsg += "'%s' into support table" % rFile
        logger.debug(debugMsg)
        inject.goStacked("LOAD DATA INFILE '%s' INTO TABLE %s FIELDS TERMINATED BY '%s' (%s)" % (tmpFile, self.fileTblName, randomStr(10), self.tblField))

        length = inject.getValue("SELECT LENGTH(%s) FROM %s" % (self.tblField, self.fileTblName), sort=False, resumeValue=False, charsetType=2)

        if length is None or not length.isdigit() or not len(length) or length in ( "0", "1" ):
            errMsg  = "unable to retrieve the content of the "
            errMsg += "file '%s'" % rFile
            raise sqlmapNoneDataException, errMsg

        length   = int(length)
        sustrLen = 1024

        if length > sustrLen:
            result = []

            for i in range(1, length, sustrLen):
                chunk = inject.getValue("SELECT MID(%s, %d, %d) FROM %s" % (self.tblField, i, sustrLen, self.fileTblName), unpack=False, sort=False, resumeValue=False, charsetType=3)

                result.append(chunk)
        else:
            result = inject.getValue("SELECT %s FROM %s" % (self.tblField, self.fileTblName), sort=False, resumeValue=False, charsetType=3)

        return result

    def unionWriteFile(self, wFile, dFile, fileType, confirm=True):
        logger.debug("encoding file to its hexadecimal string value")

        fcEncodedList   = self.fileEncode(wFile, "hex", True)
        fcEncodedStr    = fcEncodedList[0]
        fcEncodedStrLen = len(fcEncodedStr)

        if kb.injection.place == PLACE.GET and fcEncodedStrLen > 8000:
            warnMsg  = "the injection is on a GET parameter and the file "
            warnMsg += "to be written hexadecimal value is %d " % fcEncodedStrLen
            warnMsg += "bytes, this might cause errors in the file "
            warnMsg += "writing process"
            logger.warn(warnMsg)

        debugMsg = "exporting the %s file content to file '%s'" % (fileType, dFile)
        logger.debug(debugMsg)

        sqlQuery = "%s INTO DUMPFILE '%s'" % (fcEncodedStr, dFile)
        unionUse(sqlQuery)

        if confirm:
            self.askCheckWrittenFile(wFile, dFile, fileType)

    def stackedWriteFile(self, wFile, dFile, fileType, confirm=True):
        debugMsg  = "creating a support table to write the hexadecimal "
        debugMsg += "encoded file to"
        logger.debug(debugMsg)

        self.createSupportTbl(self.fileTblName, self.tblField, "longblob")

        logger.debug("encoding file to its hexadecimal string value")
        fcEncodedList = self.fileEncode(wFile, "hex", False)

        debugMsg  = "forging SQL statements to write the hexadecimal "
        debugMsg += "encoded file to the support table"
        logger.debug(debugMsg)

        sqlQueries = self.fileToSqlQueries(fcEncodedList)

        logger.debug("inserting the hexadecimal encoded file to the support table")

        for sqlQuery in sqlQueries:
            inject.goStacked(sqlQuery)

        debugMsg = "exporting the %s file content to file '%s'" % (fileType, dFile)
        logger.debug(debugMsg)

        # Reference: http://dev.mysql.com/doc/refman/5.1/en/select.html
        inject.goStacked("SELECT %s FROM %s INTO DUMPFILE '%s'" % (self.tblField, self.fileTblName, dFile), silent=True)

        if confirm:
            self.askCheckWrittenFile(wFile, dFile, fileType)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

from girder import events
from girder.models.notification import ProgressState

JOB_HANDLER_LOCAL = 'jobs._local'

# Constants representing the setting keys for this plugin
class JobStatus(object):
    INACTIVE = 0
    QUEUED = 1
    RUNNING = 2
    SUCCESS = 3
    ERROR = 4
    CANCELED = 5

    @staticmethod
    def isValid(status):
        event = events.trigger('jobs.status.validate', info=status)

        if event.defaultPrevented and len(event.responses):
            return event.responses[-1]

        return status in (JobStatus.INACTIVE, JobStatus.QUEUED,
                          JobStatus.RUNNING, JobStatus.SUCCESS, JobStatus.ERROR,
                          JobStatus.CANCELED)

    @staticmethod
    def toNotificationStatus(status):
        if status in (JobStatus.INACTIVE, JobStatus.QUEUED):
            return ProgressState.QUEUED
        if status == JobStatus.RUNNING:
            return ProgressState.ACTIVE
        if status == JobStatus.SUCCESS:
            return ProgressState.SUCCESS
        else:
            return ProgressState.ERROR
<commit_msg>Add comment to JobStatus classes<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

from girder import events
from girder.models.notification import ProgressState

JOB_HANDLER_LOCAL = 'jobs._local'

# integer enum describing job states. Note, no order is implied.
class JobStatus(object):
    INACTIVE = 0
    QUEUED = 1
    RUNNING = 2
    SUCCESS = 3
    ERROR = 4
    CANCELED = 5

    @staticmethod
    def isValid(status):
        event = events.trigger('jobs.status.validate', info=status)

        if event.defaultPrevented and len(event.responses):
            return event.responses[-1]

        return status in (JobStatus.INACTIVE, JobStatus.QUEUED,
                          JobStatus.RUNNING, JobStatus.SUCCESS, JobStatus.ERROR,
                          JobStatus.CANCELED)

    @staticmethod
    def toNotificationStatus(status):
        if status in (JobStatus.INACTIVE, JobStatus.QUEUED):
            return ProgressState.QUEUED
        if status == JobStatus.RUNNING:
            return ProgressState.ACTIVE
        if status == JobStatus.SUCCESS:
            return ProgressState.SUCCESS
        else:
            return ProgressState.ERROR
<|endoftext|>


<commit_before>import logging
import sys
import traceback

import numpy as np
from PyQt5.QtGui import QPixmap, QPainter, QMouseEvent
from PyQt5.QtWidgets import QApplication, QWidget, QVBoxLayout, QMenu

import ImageProcessor

sys._excepthook = sys.excepthook

def excepthook(excType, excValue, tracebackobj):
    traceback.print_tb(tracebackobj)
sys.excepthook = excepthook

FILL_COLOR = np.array([[[0, 255, 255]]], dtype=np.uint8)
BORDER_COLOR = np.array([[[0, 0, 255]]], dtype=np.uint8)
BORDER_WIDTH = 5
MAX_SIZE = 800

class Label(QWidget):
    def __init__(self, parent=None):
        QWidget.__init__(self, parent=parent)
        self.p = None

    def setPixmap(self, p):
        self.p = p

    def paintEvent(self, event):
        if self.p:
            painter = QPainter(self)
            painter.setRenderHint(QPainter.SmoothPixmapTransform)
            painter.drawPixmap(self.rect(), self.p)

    def contextMenuEvent(self, QContextMenuEvent):
        cmenu = QMenu(self)
        page = self.parentWidget()
        positionx = QContextMenuEvent.x()
        positiony = QContextMenuEvent.y()
        logging.info(str(positionx) + ' ' + str(positiony))
        im_pos_x, im_pos_y = page.image.window_coords_to_image_coords(positionx, positiony, self.width(), self.height())
        logging.info(str(positionx) + ' ' + str(positiony) + ' -> ' + str(im_pos_x) + ' ' + str(im_pos_y))
        min_vline_dist = min(abs(im_pos_x - vl) for vl in page.image.coords_of_vert_lns)
        min_hline_dist = min(abs(im_pos_y - vl) for vl in page.image.coords_of_horiz_lns)
        self._actions = []
        self._actions_objects = []
        if min_hline_dist <= BORDER_WIDTH * 3:
            DelHorAction = cmenu.addAction('Delete Horizontal line here')
            self._actions.append('DelHorAction')
            self._actions_objects.append(DelHorAction )
        else:
            AddHorAction = cmenu.addAction('Add Horizontal line here')
            self._actions.append('AddHorAction')
            self._actions_objects.append(AddHorAction )
        if min_vline_dist <= BORDER_WIDTH * 3:
            DelVertAction = cmenu.addAction('Delete Vertical line here')
            self._actions.append('DelVertAction')
            self._actions_objects.append(DelVertAction )
        else:
            AddVertAction = cmenu.addAction('Add Vertical line here')
            self._actions.append('AddVertAction')
            self._actions_objects.append(AddVertAction )
        action = cmenu.exec_(self.mapToGlobal(QContextMenuEvent.pos()))
        if action:
            selected_action_index = self._actions_objects.index(action)
            selected_action = self._actions[selected_action_index]
            print(selected_action )

        print(self.pa(action))
        # TODO Поправить код на точное орпределение действия action
        # TODO Сейчас обрабатываются все сразу которые есть в меню
        method_name = str(self._actions[0])
        method = getattr(self, method_name)
        method((im_pos_x, im_pos_y))
        method_name = str(self._actions[1])
        method = getattr(self, method_name)
        method((im_pos_x, im_pos_y))
    # TODO Дебажный кусок кода( удалению не подлежит :-))
    def pa(self, obj):
        print('*' * 100)
        print(type(obj), obj)
        dr = [x for x in dir(obj) if not x.startswith('__')]
        cur = {}
        for atr in dr:
            if atr == 'label':
                pass
            txt = str(obj.__getattribute__(atr)).replace('\n', '')
            if txt in ['None']:
                continue
            if txt.startswith('<') and txt.endswith('>') and ' at ' in txt:
                continue
            cur[atr] = txt
        return cur

    def AddHorAction(self, coords):
        logging.info('ДОБАВИТЬ ГОРИЗОНТАЛЬ')
        pass

    def DelHorAction(self, coords):
        logging.info('УДАЛИТЬ ГОРИЗОНТАЛЬ')
        pass

    def DelVertAction(self, coords):
        logging.info('УДАЛИТЬ ВЕРТИКАЛЬ')
        pass

    def AddVertAction(self, coords):
        logging.info('ДОБАВИТЬ ВЕРТИКАЛЬ')
        pass

    def mousePressEvent(self, a0: QMouseEvent):
        button_pressed = a0.button()
        page = self.parentWidget()
        cursor_pos_x = int(a0.x())
        cursor_pos_y = int(a0.y())
        logging.info(str(cursor_pos_x) + ' ' + str(cursor_pos_y))
        if button_pressed == 1:
            cell_pos = page.image.coord_to_cell(cursor_pos_x, cursor_pos_y, self.width(), self.height())
            if cell_pos:
                page.image.toggle_highlight_cell(*cell_pos)
            page.qp.loadFromData(page.image.to_bin())
            page.lb.setPixmap(page.qp)
            page.lb.update()

class ScannedPageWidget(QWidget):
    def __init__(self, image, parent=None):
        QWidget.__init__(self, parent=parent)
        self.image = image
        self.lay = QVBoxLayout(self)
        self.lay.setContentsMargins(0, 0, 0, 0)
        self.lb = Label(self)
        self.qp = QPixmap()
        self.qp.loadFromData(self.image.to_bin())
        self.lb.setPixmap(self.qp)
        self.lay.addWidget(self.lb)

def show(image):
    mx = max(image.H, image.W)
    w_height, w_width = int(image.H/mx*MAX_SIZE), int(image.W/mx*MAX_SIZE),
    app = QApplication(sys.argv)
    w = ScannedPageWidget(image)
    w.resize(w_width, w_height)
    # w.setFixedSize(w_width, w_height)
    w.show()
    app.exec_()

def feature_qt(gray_np_image, filled_cells, coords_of_horiz_lns, coords_of_vert_lns):
    global image
    image = ImageProcessor.ImageProcessor(gray_np_image, filled_cells, coords_of_horiz_lns, coords_of_vert_lns)
    show(image)
    return filled_cells

if __name__ == '__main__':
    pass<commit_msg>Continuing developing contex menu event<commit_after>import logging
import sys
import traceback

import numpy as np
from PyQt5.QtGui import QPixmap, QPainter, QMouseEvent
from PyQt5.QtWidgets import QApplication, QWidget, QVBoxLayout, QMenu

import ImageProcessor
from cell_recognizer import find_filled_cells

sys._excepthook = sys.excepthook

def excepthook(excType, excValue, tracebackobj):
    traceback.print_tb(tracebackobj)
sys.excepthook = excepthook

FILL_COLOR = np.array([[[0, 255, 255]]], dtype=np.uint8)
BORDER_COLOR = np.array([[[0, 0, 255]]], dtype=np.uint8)
BORDER_WIDTH = 5
MAX_SIZE = 800

class Label(QWidget):
    def __init__(self, parent=None):
        QWidget.__init__(self, parent=parent)
        self.p = None

    def setPixmap(self, p):
        self.p = p

    def paintEvent(self, event):
        if self.p:
            painter = QPainter(self)
            painter.setRenderHint(QPainter.SmoothPixmapTransform)
            painter.drawPixmap(self.rect(), self.p)

    def contextMenuEvent(self, QContextMenuEvent):
        cmenu = QMenu(self)
        page = self.parentWidget()
        positionx = QContextMenuEvent.x()
        positiony = QContextMenuEvent.y()
        logging.info(str(positionx) + ' ' + str(positiony))
        im_pos_x, im_pos_y = page.image.window_coords_to_image_coords(positionx, positiony, self.width(), self.height())
        logging.info(str(positionx) + ' ' + str(positiony) + ' -> ' + str(im_pos_x) + ' ' + str(im_pos_y))
        min_vline_dist = min(abs(im_pos_x - vl) for vl in page.image.coords_of_vert_lns)
        min_hline_dist = min(abs(im_pos_y - vl) for vl in page.image.coords_of_horiz_lns)
        self._actions = []
        self._actions_objects = []
        if min_hline_dist <= BORDER_WIDTH:
            DelHorAction = cmenu.addAction('Delete Horizontal line here')
            self._actions.append('DelHorAction')
            self._actions_objects.append(DelHorAction )
        else:
            AddHorAction = cmenu.addAction('Add Horizontal line here')
            self._actions.append('AddHorAction')
            self._actions_objects.append(AddHorAction )
        if min_vline_dist <= BORDER_WIDTH:
            DelVertAction = cmenu.addAction('Delete Vertical line here')
            self._actions.append('DelVertAction')
            self._actions_objects.append(DelVertAction )
        else:
            AddVertAction = cmenu.addAction('Add Vertical line here')
            self._actions.append('AddVertAction')
            self._actions_objects.append(AddVertAction )
        action = cmenu.exec_(self.mapToGlobal(QContextMenuEvent.pos()))
        if action:
            selected_action_index = self._actions_objects.index(action)
            selected_action = self._actions[selected_action_index]
            logging.info(str(selected_action))
            # TODO работающих методов ещё нет поэтому этот кусок пока не нужен
            # method = getattr(self, selected_action)
            # method((im_pos_x, im_pos_y))

    # TODO Дальше написан неработающий код методов
    # TODO Надо исправлять
    def AddHorAction(self, coords):
        logging.info('ДОБАВИТЬ ГОРИЗОНТАЛЬ')
        page = self.parentWidget()
        page.image.coords_of_horiz_lns.append(coords[0])
        # TODO Вот тут ошибка TypeError: slice indices must be integers or None or have an __index__ method
        # page.image.filled_cells = find_filled_cells(page.image.image, page.image.coords_of_horiz_lns, page.image.coords_of_vert_lns)
        # page.image.initial_mark_filled_cells()
        # page.qp.loadFromData(page.image.to_bin())
        # page.lb.setPixmap(page.qp)
        # page.lb.update()

    def DelHorAction(self, coords):
        logging.info('УДАЛИТЬ ГОРИЗОНТАЛЬ')
        page = self.parentWidget()
        # TODO Тут надо точное определение линии по приблизительным координатам
        # page.image.coords_of_horiz_lns.remove(coords[1])

    def DelVertAction(self, coords):
        logging.info('УДАЛИТЬ ВЕРТИКАЛЬ')
        page = self.parentWidget()
        # TODO Тут надо точное определение линии по приблизительным координатам
        # page.image.coords_of_vert_lns.remove(coords[0])

    def AddVertAction(self, coords):
        logging.info('ДОБАВИТЬ ВЕРТИКАЛЬ')
        page = self.parentWidget()
        # TODO Добавить переобработку изображения
        page.image.coords_of_vert_lns.append(coords[1])

    def mousePressEvent(self, a0: QMouseEvent):
        button_pressed = a0.button()
        page = self.parentWidget()
        cursor_pos_x = int(a0.x())
        cursor_pos_y = int(a0.y())
        logging.info(str(cursor_pos_x) + ' ' + str(cursor_pos_y))
        if button_pressed == 1:
            cell_pos = page.image.coord_to_cell(cursor_pos_x, cursor_pos_y, self.width(), self.height())
            if cell_pos:
                page.image.toggle_highlight_cell(*cell_pos)
            page.qp.loadFromData(page.image.to_bin())
            page.lb.setPixmap(page.qp)
            page.lb.update()

class ScannedPageWidget(QWidget):
    def __init__(self, image, parent=None):
        QWidget.__init__(self, parent=parent)
        self.image = image
        self.lay = QVBoxLayout(self)
        self.lay.setContentsMargins(0, 0, 0, 0)
        self.lb = Label(self)
        self.qp = QPixmap()
        self.qp.loadFromData(self.image.to_bin())
        self.lb.setPixmap(self.qp)
        self.lay.addWidget(self.lb)

def show(image):
    mx = max(image.H, image.W)
    w_height, w_width = int(image.H/mx*MAX_SIZE), int(image.W/mx*MAX_SIZE),
    app = QApplication(sys.argv)
    w = ScannedPageWidget(image)
    w.resize(w_width, w_height)
    # w.setFixedSize(w_width, w_height)
    w.show()
    app.exec_()

def feature_qt(gray_np_image, filled_cells, coords_of_horiz_lns, coords_of_vert_lns):
    image = ImageProcessor.ImageProcessor(gray_np_image, filled_cells, coords_of_horiz_lns, coords_of_vert_lns)
    show(image)
    return filled_cells

if __name__ == '__main__':
    pass<|endoftext|>


<commit_before># -*- coding: utf-8 -*- {{{
# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et:

# Copyright (c) 2016, Battelle Memorial Institute
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and documentation
# are those of the authors and should not be interpreted as representing
# official policies, either expressed or implied, of the FreeBSD
# Project.
#
# This material was prepared as an account of work sponsored by an
# agency of the United States Government.  Neither the United States
# Government nor the United States Department of Energy, nor Battelle,
# nor any of their employees, nor any jurisdiction or organization that
# has cooperated in the development of these materials, makes any
# warranty, express or implied, or assumes any legal liability or
# responsibility for the accuracy, completeness, or usefulness or any
# information, apparatus, product, software, or process disclosed, or
# represents that its use would not infringe privately owned rights.
#
# Reference herein to any specific commercial product, process, or
# service by trade name, trademark, manufacturer, or otherwise does not
# necessarily constitute or imply its endorsement, recommendation, or
# favoring by the United States Government or any agency thereof, or
# Battelle Memorial Institute. The views and opinions of authors
# expressed herein do not necessarily state or reflect those of the
# United States Government or any agency thereof.
#
# PACIFIC NORTHWEST NATIONAL LABORATORY
# operated by BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY
# under Contract DE-AC05-76RL01830

# }}}
import os
import sys
import logging
import datetime
import pytz
from dateutil import parser

from volttron.platform.vip.agent import Agent, Core, PubSub, RPC, compat
from volttron.platform.agent import utils
from volttron.platform.agent.utils import (get_aware_utc_now,
                                           format_timestamp)

import pandas as pd
import statsmodels.formula.api as sm

utils.setup_logging()
_log = logging.getLogger(__name__)

class TargetAgent(Agent):
    def __init__(self, config_path, **kwargs):
        super(TargetAgent, self).__init__(**kwargs)
        self.config = utils.load_config(config_path)
        self.site = self.config.get('campus')
        self.building = self.config.get('building')
        self.temp_unit = self.config.get('temp_unit')
        self.power_unit = self.config.get('power_unit')
        self.out_temp_name = self.config.get('out_temp_name')
        self.power_name = self.config.get('power_name')
        self.aggregate_in_min = self.config.get('aggregate_in_min')
        self.aggregate_freq = str(self.aggregate_in_min) + 'Min'
        self.ts_name = self.config.get('ts_name')

        self.window_size_in_day = int(self.config.get('window_size_in_day'))
        self.min_required_window_size_in_percent = float(self.config.get('min_required_window_size_in_percent'))
        self.interval_in_min = int(self.config.get('interval_in_min'))
        self.no_of_recs_needed = self.window_size_in_day * 24 * (60 / self.interval_in_min)
        self.min_no_of_records_needed_after_aggr = \
            int(self.min_required_window_size_in_percent/100 *
                self.no_of_recs_needed /
                (self.aggregate_in_min/self.interval_in_min))
        self.schedule_run_in_sec = int(self.config.get('schedule_run_in_hr')) * 3600
        self.tz = self.config.get('tz')
        #Debug
        self.debug_folder = self.config.get('debug_folder')

        # Testing
        #self.no_of_recs_needed = 200
        #self.min_no_of_records_needed_after_aggr = self.no_of_recs_needed/self.aggregate_in_min

    @Core.receiver('onstart')
    def onstart(self, sender, **kwargs):
        self.core.periodic(self.schedule_run_in_sec, self.publish_target_info)

    def get_event_info(self):
        """
        Get event start and end datetime
        Returns:
            A dictionary that has start & end time for event day
        """
        tz = pytz.timezone(self.tz)
        start_time = datetime.datetime(2016, 5, 1, 13, 0, 0, tzinfo=tz)
        end_time = datetime.datetime(2016, 5, 1, 17, 0, 0, tzinfo=tz)
        return {
            'start': format_timestamp(start_time),
            'end': format_timestamp(end_time)
        }

    def get_baseline_target(self):
        """
        Get baseline value from PGNE agent
        Returns:
            Average value of the next 2 baseline prediction
        """
        prediction1 = 90
        prediction2 = 95
        baseline_target = (prediction1+prediction2)/2.0
        return baseline_target

    @RPC.export('get_target_info')
    def get_target_info(self):
        """
        Combine event start, end, and baseline target
        Returns:
            A dictionary of start, end datetime and baseline target
        """
        target_info = self.get_event_info()
        baseline_target = self.get_baseline_target()
        target_info['target'] = baseline_target
        return target_info

    def publish_target_info(self):
        target_info = self.get_target_info()
        headers = {'Date': format_timestamp(get_aware_utc_now())}
        tz = self.tz
        time_meta = {'type': 'datetime', 'tz': tz, 'units': 'ISO-8601'}
        target_meta = {'type': 'float', 'tz': tz, 'units': 'kW'}
        #start
        start_topic = 'target/start'
        start_msg = [{"value": target_info['start']}, time_meta]
        #end
        end_topic = 'target/end'
        end_msg = [{"value": target_info['end']}, time_meta]
        #target
        target_topic = 'target/target'
        target_msg = [{"value": target_info['target']}, target_meta]
        self.vip.pubsub.publish(
            'pubsub', start_topic, headers, start_msg).get(timeout=10)
        self.vip.pubsub.publish(
            'pubsub', end_topic, headers, end_msg).get(timeout=10)
        self.vip.pubsub.publish(
            'pubsub', target_topic, headers, target_msg).get(timeout=10)
        _log.debug("{topic}: {value}".format(
            topic=start_topic,
            value=start_msg))
        _log.debug("{topic}: {value}".format(
            topic=end_topic,
            value=end_msg))
        _log.debug("{topic}: {value}".format(
            topic=target_topic,
            value=target_msg))

def main(argv=sys.argv):
    '''Main method called by the eggsecutable.'''
    try:
        utils.vip_main(TargetAgent, identity='target_agent')
    except Exception as e:
        _log.exception('unhandled exception')

if __name__ == '__main__':
    # Entry point for script
    sys.exit(main())
<commit_msg>Updated target agent<commit_after># -*- coding: utf-8 -*- {{{
# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et:

# Copyright (c) 2016, Battelle Memorial Institute
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and documentation
# are those of the authors and should not be interpreted as representing
# official policies, either expressed or implied, of the FreeBSD
# Project.
#
# This material was prepared as an account of work sponsored by an
# agency of the United States Government.  Neither the United States
# Government nor the United States Department of Energy, nor Battelle,
# nor any of their employees, nor any jurisdiction or organization that
# has cooperated in the development of these materials, makes any
# warranty, express or implied, or assumes any legal liability or
# responsibility for the accuracy, completeness, or usefulness or any
# information, apparatus, product, software, or process disclosed, or
# represents that its use would not infringe privately owned rights.
#
# Reference herein to any specific commercial product, process, or
# service by trade name, trademark, manufacturer, or otherwise does not
# necessarily constitute or imply its endorsement, recommendation, or
# favoring by the United States Government or any agency thereof, or
# Battelle Memorial Institute. The views and opinions of authors
# expressed herein do not necessarily state or reflect those of the
# United States Government or any agency thereof.
#
# PACIFIC NORTHWEST NATIONAL LABORATORY
# operated by BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY
# under Contract DE-AC05-76RL01830

# }}}
import os
import sys
import logging
import datetime
import pytz
from dateutil import parser

from volttron.platform.vip.agent import Agent, Core, PubSub, RPC, compat
from volttron.platform.agent import utils
from volttron.platform.agent.utils import (get_aware_utc_now,
                                           format_timestamp)

import pandas as pd
import statsmodels.formula.api as sm

utils.setup_logging()
_log = logging.getLogger(__name__)

class TargetAgent(Agent):
    def __init__(self, config_path, **kwargs):
        super(TargetAgent, self).__init__(**kwargs)
        self.config = utils.load_config(config_path)
        self.site = self.config.get('campus')
        self.building = self.config.get('building')
        self.temp_unit = self.config.get('temp_unit')
        self.power_unit = self.config.get('power_unit')
        self.out_temp_name = self.config.get('out_temp_name')
        self.power_name = self.config.get('power_name')
        self.aggregate_in_min = self.config.get('aggregate_in_min')
        self.aggregate_freq = str(self.aggregate_in_min) + 'Min'
        self.ts_name = self.config.get('ts_name')

        self.window_size_in_day = int(self.config.get('window_size_in_day'))
        self.min_required_window_size_in_percent = float(self.config.get('min_required_window_size_in_percent'))
        self.interval_in_min = int(self.config.get('interval_in_min'))
        self.no_of_recs_needed = self.window_size_in_day * 24 * (60 / self.interval_in_min)
        self.min_no_of_records_needed_after_aggr = \
            int(self.min_required_window_size_in_percent/100 *
                self.no_of_recs_needed /
                (self.aggregate_in_min/self.interval_in_min))
        self.schedule_run_in_sec = int(self.config.get('schedule_run_in_hr')) * 3600
        self.tz = self.config.get('tz')
        #Debug
        self.debug_folder = self.config.get('debug_folder')

        # Testing
        #self.no_of_recs_needed = 200
        #self.min_no_of_records_needed_after_aggr = self.no_of_recs_needed/self.aggregate_in_min

    @Core.receiver('onstart')
    def onstart(self, sender, **kwargs):
        #self.core.periodic(self.schedule_run_in_sec, self.publish_target_info)
        self.publish_target_info()

    def get_event_info(self):
        """
        Get event start and end datetime
        Returns:
            A dictionary that has start & end time for event day
        """
        event_info = {}
        tz = pytz.timezone(self.tz)
        event_info['start'] = datetime.datetime(2017, 4, 14, 13, 0, 0, tzinfo=tz)
        event_info['end'] = datetime.datetime(2017, 4, 30, 17, 0, 0, tzinfo=tz)
        return event_info

    def get_baseline_target(self, dt1, dt2):
        """
        Get baseline value from PGNE agent
        Returns:
            Average value of the next 2 baseline prediction
        """
        # prediction for dt1
        prediction1 = 90
        # prediction for dt2
        prediction2 = 95
        # avg
        baseline_target = (prediction1+prediction2)/2.0

        return baseline_target

    @RPC.export('get_target_info')
    def get_target_info(self):
        """
        Combine event start, end, and baseline target
        Returns:
            A dictionary of start, end datetime and baseline target
        """
        target_info = {}
        event_info = self.get_event_info()
        if len(event_info.keys()) > 0:
            start = event_info['start']
            end = event_info['end']
            cur_time = datetime.datetime.now(pytz.timezone(self.tz))
            if start <= cur_time < end:
                one_hour = datetime.timedelta(hours=1)
                next_hour = \
                    cur_time.replace(minute=0, second=0, microsecond=0) + one_hour
                next_hour_end = next_hour.replace(minute=59, second=59)
                baseline_target = \
                    self.get_baseline_target(next_hour, next_hour+one_hour)
                target_info['id'] = format_timestamp(next_hour)
                target_info['start'] = format_timestamp(next_hour)
                target_info['end'] = format_timestamp(next_hour_end)
                target_info['target'] = baseline_target

        return target_info

    def publish_target_info(self):
        target_info = self.get_target_info()
        if len(target_info.keys()) > 0:
            headers = {'Date': format_timestamp(get_aware_utc_now())}
            tz = self.tz
            target_meta = {'type': 'float', 'tz': tz, 'units': 'kW'}
            target_topic = '/'.join([self.site, self.building, 'target'])
            target_msg = [{
                "id": target_info['id'],
                "start": target_info['start'],
                "end": target_info['end'],
                "target": target_info['target']
            }, target_meta]
            self.vip.pubsub.publish(
                'pubsub', target_topic, headers, target_msg).get(timeout=10)
            _log.debug("{topic}: {value}".format(
                topic=target_topic,
                value=target_msg))
            # Schedule the next run at minute 30 of next hour
            cur_time = datetime.datetime.now(pytz.timezone(self.tz))
            one_hour = datetime.timedelta(hours=1)
            next_update_time = \
                cur_time.replace(minute=30, second=0, microsecond=0) + one_hour
            self._still_connected_event = self.core.schedule(
                next_update_time, self.publish_target_info)

def main(argv=sys.argv):
    '''Main method called by the eggsecutable.'''
    try:
        utils.vip_main(TargetAgent, identity='target_agent')
    except Exception as e:
        _log.exception('unhandled exception')

if __name__ == '__main__':
    # Entry point for script
    sys.exit(main())
<|endoftext|>


<commit_before>import httplib, os, random, struct, time, socket, base64
from cStringIO import StringIO

from disco.error import CommError
from disco.settings import DiscoSettings
from disco.util import iterify, urlresolve, urlsplit

BUFFER_SIZE = int(1024**2)
CHUNK_SIZE = int(10 * 1024**2)

settings = DiscoSettings()
nocurl = 'nocurl' in settings['DISCO_FLAGS'].lower().split()

try:
    import pycurl
except ImportError:
    nocurl = True

if nocurl:
    from httplib import HTTPConnection
else:
    from disco import comm_pycurl
    from disco.comm_pycurl import HTTPConnection

# get rid of this for python2.6+
try:
    import json
except ImportError:
    try:
        import simplejson as json
    except ImportError:
        import cjson
        class Dummy(object):
            pass
        json = Dummy()
        json.loads = cjson.decode
        json.dumps = cjson.encode

def isredirection(status):
    return str(status).startswith('3')

def issuccessful(status):
    return str(status).startswith('2')

def isunavailable(status):
    return status == httplib.SERVICE_UNAVAILABLE

def range_header(offset):
    def httprange(start='', end=''):
        return '%s-%s' % (start, end)
    if offset:
        return {'Range': 'bytes=%s' % httprange(*tuple(iterify(offset)))}
    return {}

def auth_header(token):
    if token:
        return {'Authorization': 'Basic ' + base64.b64encode("token:" + token)}
    return {}

def resolveuri(baseuri, uri):
    if uri.startswith('/'):
        scheme, netloc, _path = urlsplit(baseuri)
        return '%s://%s%s' % (scheme, netloc, uri)
    return '%s/%s' % (baseuri, uri)

def request(method, url, data=None, headers={}, sleep=0):
    scheme, netloc, path = urlsplit(urlresolve(url))

    try:
        conn = HTTPConnection(str(netloc))
        conn.request(method, '/%s' % path, body=data, headers=headers)
        response = conn.getresponse()
        status = response.status
        errmsg = response.reason
    except httplib.HTTPException, e:
        status = None
        errmsg = str(e) or repr(e)
    except (httplib.socket.error, socket.error), e:
        status = None
        errmsg = e if isinstance(e, basestring) else e[1]

    if not status or isunavailable(status):
        if sleep == 9:
            raise CommError(errmsg, url, status)
        time.sleep(random.randint(1, 2**sleep))
        return request(method, url, data=data, headers=headers, sleep=sleep + 1)
    elif isredirection(status):
        loc = response.getheader('location')
        return request(method,
                       loc if loc.startswith('http:') else resolveuri(url, loc),
                       data=data,
                       headers=headers,
                       sleep=sleep)
    elif not issuccessful(status):
        raise CommError(response.read(), url, status)
    return response

def download(url, method='GET', data=None, offset=(), token=None):
    headers = range_header(offset)
    headers.update(auth_header(token))
    return request(method if data is None else 'POST',
                   url,
                   data=data,
                   headers=headers).read()

def upload(urls, source, token=None, **kwargs):
    source = FileSource(source)
    if nocurl:
        return [request('PUT',
                        url,
                        data=source.read(),
                        headers=auth_header(token)).read() for url in urls]
    return list(comm_pycurl.upload(urls, source, token, **kwargs))

def open_local(path):
    fd = open(path, 'r', BUFFER_SIZE)
    size = os.stat(path).st_size
    return fd, size, 'file://%s' % path

def open_remote(url):
    conn = Connection(urlresolve(url))
    return conn, len(conn), conn.url

class FileSource(object):
    def __init__(self, source):
        self.isopen = hasattr(source, 'read')
        self.source = source.read() if self.isopen else source

    def __len__(self):
        if self.isopen:
            return len(self.source)
        return os.stat(self.source).st_size

    @property
    def read(self):
        if self.isopen:
            return StringIO(self.source).read
        return open(self.source, 'r').read

class Connection(object):
    def __init__(self, url):
        self.url = url
        self.buf = None
        self.offset = 0
        self.orig_offset = 0
        self.eof = False
        self.headers = {}
        self.read(1)
        self.i = 0

    def __iter__(self):
        chunk = self._read_chunk(CHUNK_SIZE)
        while chunk:
            next_chunk = self._read_chunk(CHUNK_SIZE)
            lines = list(StringIO(chunk))
            last  = lines.pop() if next_chunk else ''
            for line in lines:
                yield line
            chunk = last + next_chunk

    def __len__(self):
        if 'content-range' in self.headers:
            return int(self.headers['content-range'].split('/')[1])
        return int(self.headers.get('content-length', 0))

    def close(self):
        pass

    def read(self, size=-1):
        buf = StringIO()
        while size:
            bytes = self._read_chunk(size if size > 0 else CHUNK_SIZE)
            if not bytes:
                break
            size -= len(bytes)
            buf.write(bytes)
        return buf.getvalue()

    def _read_chunk(self, n):
        if self.buf is None or self.i >= len(self.buf):
            if self.eof:
                return ''
            self.i = 0
            if len(self):
                end = min(len(self), self.offset + CHUNK_SIZE) - 1
            else:
                end = self.offset + CHUNK_SIZE - 1
            response = request('GET',
                               self.url,
                               headers=range_header((self.offset, end)))
            self.buf = response.read()
            self.headers = dict(response.getheaders())
            self.orig_offset = self.offset
            self.offset += len(self.buf)
            if len(self) and self.offset >= len(self):
                self.eof = True
            elif self.buf == '':
                self.eof = True
        ret = self.buf[self.i:self.i + n]
        self.i += len(ret)
        return ret

    def tell(self):
        return self.orig_offset + self.i

    def seek(self, pos, mode=0):
        if mode == 0:
            self.offset = pos
        elif mode == 1:
            self.offset = self.tell() + pos
        else:
            self.offset = len(self) - pos
        self.eof = False
        self.buf = None
        self.orig_offset = self.offset
        self.i = 0
<commit_msg>Add token support to open_remote.<commit_after>import httplib, os, random, struct, time, socket, base64
from cStringIO import StringIO

from disco.error import CommError
from disco.settings import DiscoSettings
from disco.util import iterify, urlresolve, urlsplit

BUFFER_SIZE = int(1024**2)
CHUNK_SIZE = int(10 * 1024**2)

settings = DiscoSettings()
nocurl = 'nocurl' in settings['DISCO_FLAGS'].lower().split()

try:
    import pycurl
except ImportError:
    nocurl = True

if nocurl:
    from httplib import HTTPConnection
else:
    from disco import comm_pycurl
    from disco.comm_pycurl import HTTPConnection

# get rid of this for python2.6+
try:
    import json
except ImportError:
    try:
        import simplejson as json
    except ImportError:
        import cjson
        class Dummy(object):
            pass
        json = Dummy()
        json.loads = cjson.decode
        json.dumps = cjson.encode

def isredirection(status):
    return str(status).startswith('3')

def issuccessful(status):
    return str(status).startswith('2')

def isunavailable(status):
    return status == httplib.SERVICE_UNAVAILABLE

def range_header(offset):
    def httprange(start='', end=''):
        return '%s-%s' % (start, end)
    if offset:
        return {'Range': 'bytes=%s' % httprange(*tuple(iterify(offset)))}
    return {}

def auth_header(token):
    if token:
        return {'Authorization': 'Basic ' + base64.b64encode("token:" + token)}
    return {}

def resolveuri(baseuri, uri):
    if uri.startswith('/'):
        scheme, netloc, _path = urlsplit(baseuri)
        return '%s://%s%s' % (scheme, netloc, uri)
    return '%s/%s' % (baseuri, uri)

def request(method, url, data=None, headers={}, sleep=0):
    scheme, netloc, path = urlsplit(urlresolve(url))

    try:
        conn = HTTPConnection(str(netloc))
        conn.request(method, '/%s' % path, body=data, headers=headers)
        response = conn.getresponse()
        status = response.status
        errmsg = response.reason
    except httplib.HTTPException, e:
        status = None
        errmsg = str(e) or repr(e)
    except (httplib.socket.error, socket.error), e:
        status = None
        errmsg = e if isinstance(e, basestring) else e[1]

    if not status or isunavailable(status):
        if sleep == 9:
            raise CommError(errmsg, url, status)
        time.sleep(random.randint(1, 2**sleep))
        return request(method, url, data=data, headers=headers, sleep=sleep + 1)
    elif isredirection(status):
        loc = response.getheader('location')
        return request(method,
                       loc if loc.startswith('http:') else resolveuri(url, loc),
                       data=data,
                       headers=headers,
                       sleep=sleep)
    elif not issuccessful(status):
        raise CommError(response.read(), url, status)
    return response

def download(url, method='GET', data=None, offset=(), token=None):
    headers = range_header(offset)
    headers.update(auth_header(token))
    return request(method if data is None else 'POST',
                   url,
                   data=data,
                   headers=headers).read()

def upload(urls, source, token=None, **kwargs):
    source = FileSource(source)
    if nocurl:
        return [request('PUT',
                        url,
                        data=source.read(),
                        headers=auth_header(token)).read() for url in urls]
    return list(comm_pycurl.upload(urls, source, token, **kwargs))

def open_local(path):
    fd = open(path, 'r', BUFFER_SIZE)
    size = os.stat(path).st_size
    return fd, size, 'file://%s' % path

def open_remote(url, token=None):
    conn = Connection(urlresolve(url), token)
    return conn, len(conn), conn.url

class FileSource(object):
    def __init__(self, source):
        self.isopen = hasattr(source, 'read')
        self.source = source.read() if self.isopen else source

    def __len__(self):
        if self.isopen:
            return len(self.source)
        return os.stat(self.source).st_size

    @property
    def read(self):
        if self.isopen:
            return StringIO(self.source).read
        return open(self.source, 'r').read

class Connection(object):
    def __init__(self, url, token=None):
        self.url = url
        self.token = token
        self.buf = None
        self.offset = 0
        self.orig_offset = 0
        self.eof = False
        self.headers = {}
        self.read(1)
        self.i = 0

    def __iter__(self):
        chunk = self._read_chunk(CHUNK_SIZE)
        while chunk:
            next_chunk = self._read_chunk(CHUNK_SIZE)
            lines = list(StringIO(chunk))
            last  = lines.pop() if next_chunk else ''
            for line in lines:
                yield line
            chunk = last + next_chunk

    def __len__(self):
        if 'content-range' in self.headers:
            return int(self.headers['content-range'].split('/')[1])
        return int(self.headers.get('content-length', 0))

    def close(self):
        pass

    def read(self, size=-1):
        buf = StringIO()
        while size:
            bytes = self._read_chunk(size if size > 0 else CHUNK_SIZE)
            if not bytes:
                break
            size -= len(bytes)
            buf.write(bytes)
        return buf.getvalue()

    def _read_chunk(self, n):
        if self.buf is None or self.i >= len(self.buf):
            if self.eof:
                return ''
            self.i = 0
            if len(self):
                end = min(len(self), self.offset + CHUNK_SIZE) - 1
            else:
                end = self.offset + CHUNK_SIZE - 1
            headers = auth_header(self.token)
            headers.update(range_header((self.offset, end)))
            response = request('GET',
                               self.url,
                               headers=headers)
            self.buf = response.read()
            self.headers = dict(response.getheaders())
            self.orig_offset = self.offset
            self.offset += len(self.buf)
            if len(self) and self.offset >= len(self):
                self.eof = True
            elif self.buf == '':
                self.eof = True
        ret = self.buf[self.i:self.i + n]
        self.i += len(ret)
        return ret

    def tell(self):
        return self.orig_offset + self.i

    def seek(self, pos, mode=0):
        if mode == 0:
            self.offset = pos
        elif mode == 1:
            self.offset = self.tell() + pos
        else:
            self.offset = len(self) - pos
        self.eof = False
        self.buf = None
        self.orig_offset = self.offset
        self.i = 0
<|endoftext|>


<commit_before>#
#

# Copyright (C) 2013 Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Module for generic RPC clients.

"""

import logging

import ganeti.rpc.transport as t

from ganeti import constants
from ganeti import errors
from ganeti.rpc.errors import (ProtocolError, RequestError, LuxiError)
from ganeti import serializer

KEY_METHOD = constants.LUXI_KEY_METHOD
KEY_ARGS = constants.LUXI_KEY_ARGS
KEY_SUCCESS = constants.LUXI_KEY_SUCCESS
KEY_RESULT = constants.LUXI_KEY_RESULT
KEY_VERSION = constants.LUXI_KEY_VERSION

def ParseRequest(msg):
  """Parses a request message.

  """
  try:
    request = serializer.LoadJson(msg)
  except ValueError, err:
    raise ProtocolError("Invalid RPC request (parsing error): %s" % err)

  logging.debug("RPC request: %s", request)

  if not isinstance(request, dict):
    logging.error("RPC request not a dict: %r", msg)
    raise ProtocolError("Invalid RPC request (not a dict)")

  method = request.get(KEY_METHOD, None) # pylint: disable=E1103
  args = request.get(KEY_ARGS, None) # pylint: disable=E1103
  version = request.get(KEY_VERSION, None) # pylint: disable=E1103

  if method is None or args is None:
    logging.error("RPC request missing method or arguments: %r", msg)
    raise ProtocolError(("Invalid RPC request (no method or arguments"
                         " in request): %r") % msg)

  return (method, args, version)

def ParseResponse(msg):
  """Parses a response message.

  """
  # Parse the result
  try:
    data = serializer.LoadJson(msg)
  except KeyboardInterrupt:
    raise
  except Exception, err:
    raise ProtocolError("Error while deserializing response: %s" % str(err))

  # Validate response
  if not (isinstance(data, dict) and
          KEY_SUCCESS in data and
          KEY_RESULT in data):
    raise ProtocolError("Invalid response from server: %r" % data)

  return (data[KEY_SUCCESS], data[KEY_RESULT],
          data.get(KEY_VERSION, None)) # pylint: disable=E1103

def FormatResponse(success, result, version=None):
  """Formats a response message.

  """
  response = {
    KEY_SUCCESS: success,
    KEY_RESULT: result,
    }

  if version is not None:
    response[KEY_VERSION] = version

  logging.debug("RPC response: %s", response)

  return serializer.DumpJson(response)

def FormatRequest(method, args, version=None):
  """Formats a request message.

  """
  # Build request
  request = {
    KEY_METHOD: method,
    KEY_ARGS: args,
    }

  if version is not None:
    request[KEY_VERSION] = version

  # Serialize the request
  return serializer.DumpJson(request,
                             private_encoder=serializer.EncodeWithPrivateFields)

def CallRPCMethod(transport_cb, method, args, version=None):
  """Send a RPC request via a transport and return the response.

  """
  assert callable(transport_cb)

  request_msg = FormatRequest(method, args, version=version)

  # Send request and wait for response
  response_msg = transport_cb(request_msg)

  (success, result, resp_version) = ParseResponse(response_msg)

  # Verify version if there was one in the response
  if resp_version is not None and resp_version != version:
    raise LuxiError("RPC version mismatch, client %s, response %s" %
                    (version, resp_version))

  if success:
    return result

  errors.MaybeRaise(result)
  raise RequestError(result)

class AbstractClient(object):
  """High-level client abstraction.

  This uses a backing Transport-like class on top of which it
  implements data serialization/deserialization.

  """

  def __init__(self, timeouts=None, transport=t.Transport,
               allow_non_master=False):
    """Constructor for the Client class.

    If timeout is not passed, the default timeouts of the transport
    class are used.

    @type timeouts: list of ints
    @param timeouts: timeouts to be used on connect and read/write
    @type transport: L{Transport} or another compatible class
    @param transport: the underlying transport to use for the RPC calls
    @type allow_non_master: bool
    @param allow_non_master: skip checks for the master node on errors

    """
    self.timeouts = timeouts
    self.transport_class = transport
    self.allow_non_master = allow_non_master
    self.transport = None
    # The version used in RPC communication, by default unused:
    self.version = None

  def _GetAddress(self):
    """Returns the socket address

    """
    raise NotImplementedError

  def _InitTransport(self):
    """(Re)initialize the transport if needed.

    """
    if self.transport is None:
      self.transport = \
        self.transport_class(self._GetAddress(),
                             timeouts=self.timeouts,
                             allow_non_master=self.allow_non_master)

  def _CloseTransport(self):
    """Close the transport, ignoring errors.

    """
    if self.transport is None:
      return
    try:
      old_transp = self.transport
      self.transport = None
      old_transp.Close()
    except Exception: # pylint: disable=W0703
      pass

  def _SendMethodCall(self, data):
    # Send request and wait for response
    def send(try_no):
      if try_no:
        logging.debug("RPC peer disconnected, retrying")
      self._InitTransport()
      return self.transport.Call(data)
    return t.Transport.RetryOnNetworkError(send,
                                           lambda _: self._CloseTransport())

  def Close(self):
    """Close the underlying connection.

    """
    self._CloseTransport()

  def close(self):
    """Same as L{Close}, to be used with contextlib.closing(...).

    """
    self.Close()

  def CallMethod(self, method, args):
    """Send a generic request and return the response.

    """
    if not isinstance(args, (list, tuple)):
      raise errors.ProgrammerError("Invalid parameter passed to CallMethod:"
                                   " expected list, got %s" % type(args))
    return CallRPCMethod(self._SendMethodCall, method, args,
                         version=self.version)

class AbstractStubClient(AbstractClient):
  """An abstract Client that connects a generated stub client to a L{Transport}.

  Subclasses should inherit from this class (first) as well and a designated
  stub (second).
  """

  def __init__(self, timeouts=None, transport=t.Transport,
               allow_non_master=None):
    """Constructor for the class.

    Arguments are the same as for L{AbstractClient}. Checks that SOCKET_PATH
    attribute is defined (in the stub class).

    @type timeouts: list of ints
    @param timeouts: timeouts to be used on connect and read/write
    @type transport: L{Transport} or another compatible class
    @param transport: the underlying transport to use for the RPC calls
    @type allow_non_master: bool
    @param allow_non_master: skip checks for the master node on errors
    """

    super(AbstractStubClient, self).__init__(timeouts=timeouts,
                                             transport=transport,
                                             allow_non_master=allow_non_master)

  def _GenericInvoke(self, method, *args):
    return self.CallMethod(method, args)

  def _GetAddress(self):
    return self._GetSocketPath() # pylint: disable=E1101
<commit_msg>Add debug logging to profile python CallRPCMethod<commit_after>#
#

# Copyright (C) 2013 Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Module for generic RPC clients.

"""

import logging
import time

import ganeti.rpc.transport as t

from ganeti import constants
from ganeti import errors
from ganeti.rpc.errors import (ProtocolError, RequestError, LuxiError)
from ganeti import serializer

KEY_METHOD = constants.LUXI_KEY_METHOD
KEY_ARGS = constants.LUXI_KEY_ARGS
KEY_SUCCESS = constants.LUXI_KEY_SUCCESS
KEY_RESULT = constants.LUXI_KEY_RESULT
KEY_VERSION = constants.LUXI_KEY_VERSION

def ParseRequest(msg):
  """Parses a request message.

  """
  try:
    request = serializer.LoadJson(msg)
  except ValueError, err:
    raise ProtocolError("Invalid RPC request (parsing error): %s" % err)

  logging.debug("RPC request: %s", request)

  if not isinstance(request, dict):
    logging.error("RPC request not a dict: %r", msg)
    raise ProtocolError("Invalid RPC request (not a dict)")

  method = request.get(KEY_METHOD, None) # pylint: disable=E1103
  args = request.get(KEY_ARGS, None) # pylint: disable=E1103
  version = request.get(KEY_VERSION, None) # pylint: disable=E1103

  if method is None or args is None:
    logging.error("RPC request missing method or arguments: %r", msg)
    raise ProtocolError(("Invalid RPC request (no method or arguments"
                         " in request): %r") % msg)

  return (method, args, version)

def ParseResponse(msg):
  """Parses a response message.

  """
  # Parse the result
  try:
    data = serializer.LoadJson(msg)
  except KeyboardInterrupt:
    raise
  except Exception, err:
    raise ProtocolError("Error while deserializing response: %s" % str(err))

  # Validate response
  if not (isinstance(data, dict) and
          KEY_SUCCESS in data and
          KEY_RESULT in data):
    raise ProtocolError("Invalid response from server: %r" % data)

  return (data[KEY_SUCCESS], data[KEY_RESULT],
          data.get(KEY_VERSION, None)) # pylint: disable=E1103

def FormatResponse(success, result, version=None):
  """Formats a response message.

  """
  response = {
    KEY_SUCCESS: success,
    KEY_RESULT: result,
    }

  if version is not None:
    response[KEY_VERSION] = version

  logging.debug("RPC response: %s", response)

  return serializer.DumpJson(response)

def FormatRequest(method, args, version=None):
  """Formats a request message.

  """
  # Build request
  request = {
    KEY_METHOD: method,
    KEY_ARGS: args,
    }

  if version is not None:
    request[KEY_VERSION] = version

  # Serialize the request
  return serializer.DumpJson(request,
                             private_encoder=serializer.EncodeWithPrivateFields)

def CallRPCMethod(transport_cb, method, args, version=None):
  """Send a RPC request via a transport and return the response.

  """
  assert callable(transport_cb)
  t1 = time.time()*1000
  request_msg = FormatRequest(method, args, version=version)
  t2 = time.time()*1000
  # Send request and wait for response
  response_msg = transport_cb(request_msg)
  t3 = time.time()*1000
  (success, result, resp_version) = ParseResponse(response_msg)
  t4 = time.time()*1000
  logging.info("CallRPCMethod %s: format: %dms, sock: %dms, "
      "parse: %dms", method, int(t2-t1), int(t3-t2), int(t4-t3))
  # Verify version if there was one in the response
  if resp_version is not None and resp_version != version:
    raise LuxiError("RPC version mismatch, client %s, response %s" %
                    (version, resp_version))

  if success:
    return result

  errors.MaybeRaise(result)
  raise RequestError(result)

class AbstractClient(object):
  """High-level client abstraction.

  This uses a backing Transport-like class on top of which it
  implements data serialization/deserialization.

  """

  def __init__(self, timeouts=None, transport=t.Transport,
               allow_non_master=False):
    """Constructor for the Client class.

    If timeout is not passed, the default timeouts of the transport
    class are used.

    @type timeouts: list of ints
    @param timeouts: timeouts to be used on connect and read/write
    @type transport: L{Transport} or another compatible class
    @param transport: the underlying transport to use for the RPC calls
    @type allow_non_master: bool
    @param allow_non_master: skip checks for the master node on errors

    """
    self.timeouts = timeouts
    self.transport_class = transport
    self.allow_non_master = allow_non_master
    self.transport = None
    # The version used in RPC communication, by default unused:
    self.version = None

  def _GetAddress(self):
    """Returns the socket address

    """
    raise NotImplementedError

  def _InitTransport(self):
    """(Re)initialize the transport if needed.

    """
    if self.transport is None:
      self.transport = \
        self.transport_class(self._GetAddress(),
                             timeouts=self.timeouts,
                             allow_non_master=self.allow_non_master)

  def _CloseTransport(self):
    """Close the transport, ignoring errors.

    """
    if self.transport is None:
      return
    try:
      old_transp = self.transport
      self.transport = None
      old_transp.Close()
    except Exception: # pylint: disable=W0703
      pass

  def _SendMethodCall(self, data):
    # Send request and wait for response
    def send(try_no):
      if try_no:
        logging.debug("RPC peer disconnected, retrying")
      self._InitTransport()
      return self.transport.Call(data)
    return t.Transport.RetryOnNetworkError(send,
                                           lambda _: self._CloseTransport())

  def Close(self):
    """Close the underlying connection.

    """
    self._CloseTransport()

  def close(self):
    """Same as L{Close}, to be used with contextlib.closing(...).

    """
    self.Close()

  def CallMethod(self, method, args):
    """Send a generic request and return the response.

    """
    if not isinstance(args, (list, tuple)):
      raise errors.ProgrammerError("Invalid parameter passed to CallMethod:"
                                   " expected list, got %s" % type(args))
    return CallRPCMethod(self._SendMethodCall, method, args,
                         version=self.version)

class AbstractStubClient(AbstractClient):
  """An abstract Client that connects a generated stub client to a L{Transport}.

  Subclasses should inherit from this class (first) as well and a designated
  stub (second).
  """

  def __init__(self, timeouts=None, transport=t.Transport,
               allow_non_master=None):
    """Constructor for the class.

    Arguments are the same as for L{AbstractClient}. Checks that SOCKET_PATH
    attribute is defined (in the stub class).

    @type timeouts: list of ints
    @param timeouts: timeouts to be used on connect and read/write
    @type transport: L{Transport} or another compatible class
    @param transport: the underlying transport to use for the RPC calls
    @type allow_non_master: bool
    @param allow_non_master: skip checks for the master node on errors
    """

    super(AbstractStubClient, self).__init__(timeouts=timeouts,
                                             transport=transport,
                                             allow_non_master=allow_non_master)

  def _GenericInvoke(self, method, *args):
    return self.CallMethod(method, args)

  def _GetAddress(self):
    return self._GetSocketPath() # pylint: disable=E1101
<|endoftext|>


<commit_before>'''
Build and return SQL statements to select Amazon EC2 information.
'''

from cloudscheduler.lib.view_utils import qt

#-------------------------------------------------------------------------------

def select_ec2_images(config, group_name, cloud_name):
    """
    Select EC2 instance_types.
    """

    rc, msg, close_db_on_exit, region, my_ec2_id = _get_ec2_region_and_owner_id(config, group_name, cloud_name)
    if rc != 0:
        return 1, msg, None

    sql = ['select * from view_ec2_images where region="%s"' % region]

    ec2_filter = qt(config.db_connection.execute('select * from ec2_image_filters where group_name="%s" and cloud_name="%s"' % (group_name, cloud_name)))
    if len(ec2_filter) == 1:
        if ec2_filter[0]['owner_aliases']:
            sql.append(_get_ec2_equal('owner_alias', ec2_filter[0]['owner_aliases']))
            sql[-1] = sql[-1].replace('owner_alias="self"', 'owner_id="%s"' % my_ec2_id).replace('owner_alias="shared"', 'borrower_id="%s"' % my_ec2_id)

        if ec2_filter[0]['owner_ids']:
            sql.append(_get_ec2_equal('owner_id', ec2_filter[0]['owner_ids']))

        _bracket_and_bracket_to_or(sql)

        if ec2_filter[0]['like']:
            sql.append(_get_ec2_like('image_location', ec2_filter[0]['like']))

        if ec2_filter[0]['not_like']:
            sql.append(_get_ec2_not_like('image_location', ec2_filter[0]['not_like']))

        if ec2_filter[0]['operating_systems']:
            sql.append(_get_ec2_equal('opsys', ec2_filter[0]['operating_systems']))

        if ec2_filter[0]['architectures']:
            sql.append(_get_ec2_equal('arch', ec2_filter[0]['architectures']))

    if close_db_on_exit:
        config.db_close()

    return 0, None, ' '.join(sql)

#-------------------------------------------------------------------------------

def select_ec2_instance_types(config, group_name, cloud_name):
    """
    Select EC2 instance_types.
    """

    rc, msg, close_db_on_exit, region, my_ec2_id = _get_ec2_region_and_owner_id(config, group_name, cloud_name)
    if rc != 0:
        return 1, msg, None

    sql = ['select * from view_ec2_instance_types where region="%s"' % region]

    ec2_filter = qt(config.db_connection.execute('select * from ec2_instance_type_filters where group_name="%s" and cloud_name="%s"' % (group_name, cloud_name)))
    if len(ec2_filter) == 1:
        if ec2_filter[0]['families']:
            sql.append(_get_ec2_equal('instance_family', ec2_filter[0]['families']))

        if ec2_filter[0]['operating_systems']:
            sql.append(_get_ec2_equal('operating_system', ec2_filter[0]['operating_systems']))

        if ec2_filter[0]['processors']:
            sql.append(_get_ec2_equal('processor', ec2_filter[0]['processors']))

        if ec2_filter[0]['processor_manufacturers']:
            sql.append(_get_ec2_like('processor', ec2_filter[0]['processor_manufacturers']))

        if ec2_filter[0]['cores']:
            sql.append(_get_ec2_equal('cores', ec2_filter[0]['cores'], quotes=''))

        if ec2_filter[0]['memory_min_gigabytes_per_core'] or ec2_filter[0]['memory_max_gigabytes_per_core']:
            sql.append(_get_ec2_range('memory_per_core', ec2_filter[0]['memory_min_gigabytes_per_core'], ec2_filter[0]['memory_max_gigabytes_per_core'], quotes=''))

    if close_db_on_exit:
        config.db_close()

    return 0, None, ' '.join(sql)

#-------------------------------------------------------------------------------

def _bracket_and_bracket_to_or(sql):
    """
    Return a clause to select where a column is equal to one or more values.
    """

    if sql[-2][-1] == ')' and sql[-1][:5] == 'and (':
        sql[-2] = sql[-2][:-1] + ' or ' + sql[-1][5:]
        del sql[-1]

    return

#-------------------------------------------------------------------------------

def _get_ec2_equal(column_name, values, delimiter=',', quotes='"'):
    """
    Return a clause to select where a column is equal to one or more values.
    """
    value_list = values.split(delimiter)

    phrases = []
    for value in value_list:
        phrases.append('%s=%s' % (column_name, '%s%s%s' % (quotes, value, quotes)))

    return 'and (%s)' % ' or '.join(phrases)

#-------------------------------------------------------------------------------

def _get_ec2_like(column_name, values, delimiter=','):
    """
    Return a clause to select where a column is equal to one or more values.
    """
    value_list = values.split(delimiter)

    phrases = []
    for value in value_list:
        phrases.append('%s like "%%%%%s%%%%"' % (column_name, value))

    return 'and (%s)' % ' or '.join(phrases)

#-------------------------------------------------------------------------------

def _get_ec2_not_equal(column_name, values, delimiter=',', quotes='"'):
    """
    Return a clause to select where a column is equal to one or more values.
    """
    value_list = values.split(delimiter)

    phrases = []
    for value in value_list:
        phrases.append('%s!=%s' % (column_name, '%s%s%s' % (quotes, value, quotes)))

    return 'and %s' % ' and '.join(phrases)

#-------------------------------------------------------------------------------

def _get_ec2_not_like(column_name, values, delimiter=','):
    """
    Return a clause to select where a column is equal to one or more values.
    """
    value_list = values.split(delimiter)

    phrases = []
    for value in value_list:
        phrases.append('%s not like "%%%%%s%%%%"' % (column_name, value))

    return 'and %s' % ' and '.join(phrases)

#-------------------------------------------------------------------------------

def _get_ec2_range(column_name, min_value, max_value, quotes='"'):
    """
    Return a clause to select where a column is equal to one or more values.
    """
    if min_value and max_value:
        return('and %s>=%s and %s<=%s' % (column_name, '%s%s%s' % (quotes, min_value, quotes), column_name, '%s%s%s' % (quotes, max_value, quotes)))

    if min_value:
        return('and %s>=%s' % (column_name, '%s%s%s' % (quotes, min_value, quotes)))

    if max_value:
        return('and %s<=%s' % (column_name, '%s%s%s' % (quotes, max_value, quotes)))

#-------------------------------------------------------------------------------

def _get_ec2_region_and_owner_id(config, group_name, cloud_name):
    """
    Select EC2 images.
    """

    if config.db_connection:
        close_db_on_exit = False
    else:
        close_db_on_exit = True
        config.db_open()

    cloud = qt(config.db_connection.execute('select * from csv2_clouds where group_name="%s" and cloud_name="%s"' % (group_name, cloud_name)))

    if len(cloud) != 1:
        if close_db_on_exit:
            config.db_close()
        return 1, 'specified cloud "%s::%s" does not exist.' % (group_name, cloud_name), None, None, None

    if cloud[0]['cloud_type'] != 'amazon':
        if close_db_on_exit:
            config.db_close()
        return 1, 'specified cloud "%s::%s" is not an "amazon" cloud.' % (group_name, cloud_name), None, None, None

    return 0,  None, close_db_on_exit, cloud[0]['region'], cloud[0]['ec2_owner_id']

<commit_msg>fix index error<commit_after>'''
Build and return SQL statements to select Amazon EC2 information.
'''

from cloudscheduler.lib.view_utils import qt

#-------------------------------------------------------------------------------

def select_ec2_images(config, group_name, cloud_name):
    """
    Select EC2 instance_types.
    """

    rc, msg, close_db_on_exit, region, my_ec2_id = _get_ec2_region_and_owner_id(config, group_name, cloud_name)
    if rc != 0:
        return 1, msg, None

    sql = ['select * from view_ec2_images where region="%s"' % region]

    ec2_filter = qt(config.db_connection.execute('select * from ec2_image_filters where group_name="%s" and cloud_name="%s"' % (group_name, cloud_name)))
    if len(ec2_filter) == 1:
        if ec2_filter[0]['owner_aliases']:
            sql.append(_get_ec2_equal('owner_alias', ec2_filter[0]['owner_aliases']))
            sql[-1] = sql[-1].replace('owner_alias="self"', 'owner_id="%s"' % my_ec2_id).replace('owner_alias="shared"', 'borrower_id="%s"' % my_ec2_id)

        if ec2_filter[0]['owner_ids']:
            sql.append(_get_ec2_equal('owner_id', ec2_filter[0]['owner_ids']))

        _bracket_and_bracket_to_or(sql)

        if ec2_filter[0]['like']:
            sql.append(_get_ec2_like('image_location', ec2_filter[0]['like']))

        if ec2_filter[0]['not_like']:
            sql.append(_get_ec2_not_like('image_location', ec2_filter[0]['not_like']))

        if ec2_filter[0]['operating_systems']:
            sql.append(_get_ec2_equal('opsys', ec2_filter[0]['operating_systems']))

        if ec2_filter[0]['architectures']:
            sql.append(_get_ec2_equal('arch', ec2_filter[0]['architectures']))

    if close_db_on_exit:
        config.db_close()

    return 0, None, ' '.join(sql)

#-------------------------------------------------------------------------------

def select_ec2_instance_types(config, group_name, cloud_name):
    """
    Select EC2 instance_types.
    """

    rc, msg, close_db_on_exit, region, my_ec2_id = _get_ec2_region_and_owner_id(config, group_name, cloud_name)
    if rc != 0:
        return 1, msg, None

    sql = ['select * from view_ec2_instance_types where region="%s"' % region]

    ec2_filter = qt(config.db_connection.execute('select * from ec2_instance_type_filters where group_name="%s" and cloud_name="%s"' % (group_name, cloud_name)))
    if len(ec2_filter) == 1:
        if ec2_filter[0]['families']:
            sql.append(_get_ec2_equal('instance_family', ec2_filter[0]['families']))

        if ec2_filter[0]['operating_systems']:
            sql.append(_get_ec2_equal('operating_system', ec2_filter[0]['operating_systems']))

        if ec2_filter[0]['processors']:
            sql.append(_get_ec2_equal('processor', ec2_filter[0]['processors']))

        if ec2_filter[0]['processor_manufacturers']:
            sql.append(_get_ec2_like('processor', ec2_filter[0]['processor_manufacturers']))

        if ec2_filter[0]['cores']:
            sql.append(_get_ec2_equal('cores', ec2_filter[0]['cores'], quotes=''))

        if ec2_filter[0]['memory_min_gigabytes_per_core'] or ec2_filter[0]['memory_max_gigabytes_per_core']:
            sql.append(_get_ec2_range('memory_per_core', ec2_filter[0]['memory_min_gigabytes_per_core'], ec2_filter[0]['memory_max_gigabytes_per_core'], quotes=''))

    if close_db_on_exit:
        config.db_close()

    return 0, None, ' '.join(sql)

#-------------------------------------------------------------------------------

def _bracket_and_bracket_to_or(sql):
    """
    Return a clause to select where a column is equal to one or more values.
    """

    if len(sql) > 1 and len(sql[-2]) > 0 and len(sql[-1]) > 5 and sql[-2][-1] == ')' and sql[-1][:5] == 'and (':
        sql[-2] = sql[-2][:-1] + ' or ' + sql[-1][5:]
        del sql[-1]

    return

#-------------------------------------------------------------------------------

def _get_ec2_equal(column_name, values, delimiter=',', quotes='"'):
    """
    Return a clause to select where a column is equal to one or more values.
    """
    value_list = values.split(delimiter)

    phrases = []
    for value in value_list:
        phrases.append('%s=%s' % (column_name, '%s%s%s' % (quotes, value, quotes)))

    return 'and (%s)' % ' or '.join(phrases)

#-------------------------------------------------------------------------------

def _get_ec2_like(column_name, values, delimiter=','):
    """
    Return a clause to select where a column is equal to one or more values.
    """
    value_list = values.split(delimiter)

    phrases = []
    for value in value_list:
        phrases.append('%s like "%%%%%s%%%%"' % (column_name, value))

    return 'and (%s)' % ' or '.join(phrases)

#-------------------------------------------------------------------------------

def _get_ec2_not_equal(column_name, values, delimiter=',', quotes='"'):
    """
    Return a clause to select where a column is equal to one or more values.
    """
    value_list = values.split(delimiter)

    phrases = []
    for value in value_list:
        phrases.append('%s!=%s' % (column_name, '%s%s%s' % (quotes, value, quotes)))

    return 'and %s' % ' and '.join(phrases)

#-------------------------------------------------------------------------------

def _get_ec2_not_like(column_name, values, delimiter=','):
    """
    Return a clause to select where a column is equal to one or more values.
    """
    value_list = values.split(delimiter)

    phrases = []
    for value in value_list:
        phrases.append('%s not like "%%%%%s%%%%"' % (column_name, value))

    return 'and %s' % ' and '.join(phrases)

#-------------------------------------------------------------------------------

def _get_ec2_range(column_name, min_value, max_value, quotes='"'):
    """
    Return a clause to select where a column is equal to one or more values.
    """
    if min_value and max_value:
        return('and %s>=%s and %s<=%s' % (column_name, '%s%s%s' % (quotes, min_value, quotes), column_name, '%s%s%s' % (quotes, max_value, quotes)))

    if min_value:
        return('and %s>=%s' % (column_name, '%s%s%s' % (quotes, min_value, quotes)))

    if max_value:
        return('and %s<=%s' % (column_name, '%s%s%s' % (quotes, max_value, quotes)))

#-------------------------------------------------------------------------------

def _get_ec2_region_and_owner_id(config, group_name, cloud_name):
    """
    Select EC2 images.
    """

    if config.db_connection:
        close_db_on_exit = False
    else:
        close_db_on_exit = True
        config.db_open()

    cloud = qt(config.db_connection.execute('select * from csv2_clouds where group_name="%s" and cloud_name="%s"' % (group_name, cloud_name)))

    if len(cloud) != 1:
        if close_db_on_exit:
            config.db_close()
        return 1, 'specified cloud "%s::%s" does not exist.' % (group_name, cloud_name), None, None, None

    if cloud[0]['cloud_type'] != 'amazon':
        if close_db_on_exit:
            config.db_close()
        return 1, 'specified cloud "%s::%s" is not an "amazon" cloud.' % (group_name, cloud_name), None, None, None

    return 0,  None, close_db_on_exit, cloud[0]['region'], cloud[0]['ec2_owner_id']

<|endoftext|>


<commit_before>#!/usr/bin/env python

import errno
from xapi.storage import log
import os
import xapi
import subprocess

# Attempts to create an empty regular file with the given prefix and suffix
# If the name "prefix.suffix" exists, this will try "prefix.%d.suffix" until
# a unique name has been found. Suffix may be an empty string.

def touch_file_unique(dbg, prefix, suffix, mode=0644):
    if suffix.startswith("."):
        suffix = suffix[1:]
    counter = 0
    unique = prefix
    while True:
        try:
            if counter > 0:
                unique = "%s.%d" % (prefix, counter)
            if suffix != "":
                unique = "%s.%s" % (unique, suffix)
            fd = os.open(unique, os.O_CREAT | os.O_EXCL | os.O_WRONLY, mode)
            os.close(fd)
            break
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise
            counter = counter + 1
    return unique
<commit_msg>Remove unused imports<commit_after>#!/usr/bin/env python

import errno
import os

# Attempts to create an empty regular file with the given prefix and suffix
# If the name "prefix.suffix" exists, this will try "prefix.%d.suffix" until
# a unique name has been found. Suffix may be an empty string.

def touch_file_unique(dbg, prefix, suffix, mode=0644):
    if suffix.startswith("."):
        suffix = suffix[1:]
    counter = 0
    unique = prefix
    while True:
        try:
            if counter > 0:
                unique = "%s.%d" % (prefix, counter)
            if suffix != "":
                unique = "%s.%s" % (unique, suffix)
            fd = os.open(unique, os.O_CREAT | os.O_EXCL | os.O_WRONLY, mode)
            os.close(fd)
            break
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise
            counter = counter + 1
    return unique
<|endoftext|>