<commit_before>#-*- coding: utf-8 -*-

import cv2
import utils
import object_detector.file_io as file_io
import numpy as np
import random

class FeatureExtractor():

    def __init__(self, descriptor, patch_size, data_file):
        self._desc = descriptor
        self._patch_size = patch_size

        if data_file is None:
            self._features = None   # (N, n, m, ...)
            self._labels = None     # (N, 1)
        else:
            self._features = file_io.FileHDF5().read(data_file, "features")
            self._labels = file_io.FileHDF5().read(data_file, "labels")

    def add_positive_sets(self, annotation_file, sample_ratio=1.0, padding=0, label=1):

        features_set = []
        labels_set = []
        image_path = os.path.split(annotation_file)[0]
        annotations = file_io.FileJson().read(annotation_file)
        annotations = annotations[:int(len(annotations)*sample_ratio)]

        for annotation in annotations:
            image = cv2.imread(os.path.join(image_path, annotation["filename"]))
            image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

            patches = []
            labels = []
            for box in annotation["boxes"]:
                x1 = int(box["left"])
                y1 = int(box["top"])
                w = int(box["width"])
                h = int(box["height"])

                bb = (y1, y1+h, x1, x1+w)
                label = int(box["label"])

                roi = utils.crop_bb(image, bb, padding=padding, dst_size=self._patch_size)
                patches.append(roi)
                labels.append(label)

            # Todo : augment modulization
            features = self._desc.describe(patches)
            features_set += features.tolist()
            labels_set += labels

        features = np.array(features_set)
        labels = np.array(labels_set).reshape(-1, 1)

        if self._features is None:
            self._features = features
            self._labels = labels
        else:
            self._features = np.concatenate([self._features, features], axis = 0) if self._features is not None else features
            self._labels = np.concatenate([self._labels, labels], axis = 0) if self._labels is not None else labels

    def add_negative_sets(self, image_dir, pattern, n_samples_per_img, sample_ratio=1.0):
        # Todo : progressbar
        # Todo : 한꺼번에 write 하지말고, 10개 이미지를 모아서 write 를 자주하는 방식으로 수정
        features_set = []
        image_files = self._get_image_files(image_dir, pattern, sample_ratio)

        for image_file in image_files:
            image = cv2.imread(image_file)
            image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

            patches = utils.crop_random(image, self._patch_size, n_samples_per_img)

            features = self._desc.describe(patches)
            features_set += features.tolist()

        features = np.array(features_set)
        labels = np.zeros((len(features_set), 1))

        if self._features is None:
            self._features = features
            self._labels = labels
        else:
            self._features = np.concatenate([self._features, features], axis = 0) if self._features is not None else features
            self._labels = np.concatenate([self._labels, labels], axis = 0) if self._labels is not None else labels

    def add_data(self, features, label):
        labels = np.zeros((len(features), 1)) + label

        if self._features is None:
            self._features = features
            self._labels = label
        else:
            self._features = np.concatenate([self._features, features], axis=0)
            self._labels = np.concatenate([self._labels, labels], axis=0)

    def save(self, data_file):
        file_io.FileHDF5().write(self._features, data_file, "features")
        file_io.FileHDF5().write(self._labels, data_file, "labels")

    def summary(self):

        labels = self._labels
        feature_shape = self._features.shape

        n_positive_samples = len(labels[labels > 0])
        n_negative_samples = len(labels[labels == 0])
        n_hard_negative_samples = len(labels[labels == -1])

        print "[FeatureGetter INFO] Positive samples: {}, Negative samples: {}, Hard Negative Mined samples: {}".format(n_positive_samples, n_negative_samples, n_hard_negative_samples)
        print "[FeatureGetter INFO] Feature Dimension: {}".format(feature_shape[1])

    def get_dataset(self, include_hard_negative=True):

        if self._features is None:
            raise ValueError('There is no dataset in this instance')
        else:
            labels = self._labels
            if include_hard_negative:
                features = self._features
                labels[labels < 0] = 0
            else:
                features = self._features[self._labels >= 0]
                labels = self._labels[self._labels >= 0]
            return features, labels

    def _get_image_files(self, directory, pattern, sample_ratio):
        image_files = file_io.list_files(directory, pattern)
        image_files = random.sample(image_files, int(len(image_files) * sample_ratio))
        return image_files

import os
def calc_average_patch_size(annotation_file):
    widths = []
    heights = []

    annotations = file_io.FileJson().read(annotation_file)
    """
    {
        u'boxes':
            [
                {u'height': 219.0, u'width': 81.0, u'top': 77.0, u'left': 246.0, u'label': 1.0},
                {u'height': 219.0, u'width': 96.0, u'top': 81.0, u'left': 323.0, u'label': 9.0}
            ],
        u'filename': u'1.png'
    }
    """
    image_path = os.path.split(annotation_file)[0]

    for annotation in annotations:
        image = cv2.imread(os.path.join(image_path, annotation["filename"]))

        for box in annotation["boxes"]:
            x1 = int(box["left"])
            y1 = int(box["top"])
            w = int(box["width"])
            h = int(box["height"])
            label = int(box["label"])
            cv2.rectangle(image, (x1, y1), (x1+w, y1+h), (0, 0, 255), 2)
            cv2.putText(image, str(label), (x1, y1), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), thickness=2)

            widths.append(w)
            heights.append(h)

    height = np.mean(heights)
    width = np.mean(widths)

    print "h_min", np.min(heights)
    print "h_max", np.max(heights)
    print "w_min", np.min(widths)
    print "w_max", np.max(widths)

    data = np.array([(h, w, h+w) for h, w in zip(heights, widths)])
    print data.shape

    data = data[data[:, 0].argsort()]
    print data[:20]

    data = data[data[:, 1].argsort()]
    print data[:20]

    data = data[data[:, 2].argsort()]
    print data[:20]
    return height, width

<commit_msg>Extractor.summary() 버그 수정<commit_after>#-*- coding: utf-8 -*-

import cv2
import utils
import object_detector.file_io as file_io
import numpy as np
import random

class FeatureExtractor():

    def __init__(self, descriptor, patch_size, data_file):
        self._desc = descriptor
        self._patch_size = patch_size

        if data_file is None:
            self._features = None   # (N, n, m, ...)
            self._labels = None     # (N, 1)
        else:
            self._features = file_io.FileHDF5().read(data_file, "features")
            self._labels = file_io.FileHDF5().read(data_file, "labels")

    def add_positive_sets(self, annotation_file, sample_ratio=1.0, padding=0, label=1):

        features_set = []
        labels_set = []
        image_path = os.path.split(annotation_file)[0]
        annotations = file_io.FileJson().read(annotation_file)
        annotations = annotations[:int(len(annotations)*sample_ratio)]

        for annotation in annotations:
            image = cv2.imread(os.path.join(image_path, annotation["filename"]))
            image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

            patches = []
            labels = []
            for box in annotation["boxes"]:
                x1 = int(box["left"])
                y1 = int(box["top"])
                w = int(box["width"])
                h = int(box["height"])

                bb = (y1, y1+h, x1, x1+w)
                label = int(box["label"])

                roi = utils.crop_bb(image, bb, padding=padding, dst_size=self._patch_size)
                patches.append(roi)
                labels.append(label)

            # Todo : augment modulization
            features = self._desc.describe(patches)
            features_set += features.tolist()
            labels_set += labels

        features = np.array(features_set)
        labels = np.array(labels_set).reshape(-1, 1)

        if self._features is None:
            self._features = features
            self._labels = labels
        else:
            self._features = np.concatenate([self._features, features], axis = 0) if self._features is not None else features
            self._labels = np.concatenate([self._labels, labels], axis = 0) if self._labels is not None else labels

    def add_negative_sets(self, image_dir, pattern, n_samples_per_img, sample_ratio=1.0):
        # Todo : progressbar
        # Todo : 한꺼번에 write 하지말고, 10개 이미지를 모아서 write 를 자주하는 방식으로 수정
        features_set = []
        image_files = self._get_image_files(image_dir, pattern, sample_ratio)

        for image_file in image_files:
            image = cv2.imread(image_file)
            image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

            patches = utils.crop_random(image, self._patch_size, n_samples_per_img)

            features = self._desc.describe(patches)
            features_set += features.tolist()

        features = np.array(features_set)
        labels = np.zeros((len(features_set), 1))

        if self._features is None:
            self._features = features
            self._labels = labels
        else:
            self._features = np.concatenate([self._features, features], axis = 0) if self._features is not None else features
            self._labels = np.concatenate([self._labels, labels], axis = 0) if self._labels is not None else labels

    def add_data(self, features, label):
        labels = np.zeros((len(features), 1)) + label

        if self._features is None:
            self._features = features
            self._labels = label
        else:
            self._features = np.concatenate([self._features, features], axis=0)
            self._labels = np.concatenate([self._labels, labels], axis=0)

    def save(self, data_file):
        file_io.FileHDF5().write(self._features, data_file, "features")
        file_io.FileHDF5().write(self._labels, data_file, "labels")

    def summary(self):

        labels = self._labels
        feature_shape = self._features.shape

        n_positive_samples = len(labels[labels > 0])
        n_negative_samples = len(labels[labels == 0])
        n_hard_negative_samples = len(labels[labels == -1])

        print "[FeatureGetter INFO] Positive samples: {}, Negative samples: {}, Hard Negative Mined samples: {}".format(n_positive_samples, n_negative_samples, n_hard_negative_samples)
        print "[FeatureGetter INFO] Feature Dimension: {}".format(feature_shape[1:])

    def get_dataset(self, include_hard_negative=True):

        if self._features is None:
            raise ValueError('There is no dataset in this instance')
        else:
            labels = self._labels
            if include_hard_negative:
                features = self._features
                labels[labels < 0] = 0
            else:
                features = self._features[self._labels >= 0]
                labels = self._labels[self._labels >= 0]
            return features, labels

    def _get_image_files(self, directory, pattern, sample_ratio):
        image_files = file_io.list_files(directory, pattern)
        image_files = random.sample(image_files, int(len(image_files) * sample_ratio))
        return image_files

import os
def calc_average_patch_size(annotation_file):
    widths = []
    heights = []

    annotations = file_io.FileJson().read(annotation_file)
    """
    {
        u'boxes':
            [
                {u'height': 219.0, u'width': 81.0, u'top': 77.0, u'left': 246.0, u'label': 1.0},
                {u'height': 219.0, u'width': 96.0, u'top': 81.0, u'left': 323.0, u'label': 9.0}
            ],
        u'filename': u'1.png'
    }
    """
    image_path = os.path.split(annotation_file)[0]

    for annotation in annotations:
        image = cv2.imread(os.path.join(image_path, annotation["filename"]))

        for box in annotation["boxes"]:
            x1 = int(box["left"])
            y1 = int(box["top"])
            w = int(box["width"])
            h = int(box["height"])
            label = int(box["label"])
            cv2.rectangle(image, (x1, y1), (x1+w, y1+h), (0, 0, 255), 2)
            cv2.putText(image, str(label), (x1, y1), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), thickness=2)

            widths.append(w)
            heights.append(h)

    height = np.mean(heights)
    width = np.mean(widths)

    print "h_min", np.min(heights)
    print "h_max", np.max(heights)
    print "w_min", np.min(widths)
    print "w_max", np.max(widths)

    data = np.array([(h, w, h+w) for h, w in zip(heights, widths)])
    print data.shape

    data = data[data[:, 0].argsort()]
    print data[:20]

    data = data[data[:, 1].argsort()]
    print data[:20]

    data = data[data[:, 2].argsort()]
    print data[:20]
    return height, width

<|endoftext|>


<commit_before># Copyright 2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#

import datetime
import sys
import threading
import time

from oslo_config import cfg
from oslo_log import log as logging
from oslo_reports import guru_meditation_report as gmr

from octavia.common import service
from octavia.controller.housekeeping import house_keeping
from octavia.i18n import _LI
from octavia import version

LOG = logging.getLogger(__name__)
CONF = cfg.CONF
CONF.import_group('house_keeping', 'octavia.common.config')

spare_amp_thread_event = threading.Event()
db_cleanup_thread_event = threading.Event()
cert_rotate_thread_event = threading.Event()

def spare_amphora_check():
    """Initiates spare amp check with respect to configured interval."""

    # Read the interval from CONF
    interval = CONF.house_keeping.spare_check_interval
    LOG.info(_LI("Spare check interval is set to %d sec"), interval)

    spare_amp = house_keeping.SpareAmphora()
    while spare_amp_thread_event.is_set():
        LOG.debug("Initiating spare amphora check...")
        spare_amp.spare_check()
        time.sleep(interval)

def db_cleanup():
    """Perform db cleanup for old resources."""
    # Read the interval from CONF
    interval = CONF.house_keeping.cleanup_interval
    LOG.info(_LI("DB cleanup interval is set to %d sec"), interval)
    LOG.info(_LI('Amphora expiry age is %s seconds'),
             CONF.house_keeping.amphora_expiry_age)
    LOG.info(_LI('Load balancer expiry age is %s seconds'),
             CONF.house_keeping.load_balancer_expiry_age)

    db_cleanup = house_keeping.DatabaseCleanup()
    while db_cleanup_thread_event.is_set():
        LOG.debug("Initiating the cleanup of old resources...")
        db_cleanup.delete_old_amphorae()
        db_cleanup.cleanup_load_balancers()
        time.sleep(interval)

def cert_rotation():
    """Perform certificate rotation."""
    interval = CONF.house_keeping.cert_interval
    LOG.info(
        _LI("Expiring certificate check interval is set to %d sec") % interval)
    cert_rotate = house_keeping.CertRotation()
    while cert_rotate_thread_event.is_set():
        LOG.debug("Initiating certification rotation ...")
        cert_rotate.rotate()
        time.sleep(interval)

def main():
    service.prepare_service(sys.argv)

    gmr.TextGuruMeditation.setup_autorun(version)

    timestamp = str(datetime.datetime.utcnow())
    LOG.info(_LI("Starting house keeping at %s"), timestamp)

    # Thread to perform spare amphora check
    spare_amp_thread = threading.Thread(target=spare_amphora_check)
    spare_amp_thread.daemon = True
    spare_amp_thread_event.set()
    spare_amp_thread.start()

    # Thread to perform db cleanup
    db_cleanup_thread = threading.Thread(target=db_cleanup)
    db_cleanup_thread.daemon = True
    db_cleanup_thread_event.set()
    db_cleanup_thread.start()

    # Thread to perform certificate rotation
    cert_rotate_thread = threading.Thread(target=cert_rotation)
    cert_rotate_thread.daemon = True
    cert_rotate_thread_event.set()
    cert_rotate_thread.start()

    # Try-Exception block should be at the end to gracefully exit threads
    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        LOG.info(_LI("Attempting to gracefully terminate House-Keeping"))
        spare_amp_thread_event.clear()
        db_cleanup_thread_event.clear()
        cert_rotate_thread_event.clear()
        spare_amp_thread.join()
        db_cleanup_thread.join()
        cert_rotate_thread.join()
        LOG.info(_LI("House-Keeping process terminated"))
<commit_msg>Modify variable's using method in Log Messages<commit_after># Copyright 2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#

import datetime
import sys
import threading
import time

from oslo_config import cfg
from oslo_log import log as logging
from oslo_reports import guru_meditation_report as gmr

from octavia.common import service
from octavia.controller.housekeeping import house_keeping
from octavia.i18n import _LI
from octavia import version

LOG = logging.getLogger(__name__)
CONF = cfg.CONF
CONF.import_group('house_keeping', 'octavia.common.config')

spare_amp_thread_event = threading.Event()
db_cleanup_thread_event = threading.Event()
cert_rotate_thread_event = threading.Event()

def spare_amphora_check():
    """Initiates spare amp check with respect to configured interval."""

    # Read the interval from CONF
    interval = CONF.house_keeping.spare_check_interval
    LOG.info(_LI("Spare check interval is set to %d sec"), interval)

    spare_amp = house_keeping.SpareAmphora()
    while spare_amp_thread_event.is_set():
        LOG.debug("Initiating spare amphora check...")
        spare_amp.spare_check()
        time.sleep(interval)

def db_cleanup():
    """Perform db cleanup for old resources."""
    # Read the interval from CONF
    interval = CONF.house_keeping.cleanup_interval
    LOG.info(_LI("DB cleanup interval is set to %d sec"), interval)
    LOG.info(_LI('Amphora expiry age is %s seconds'),
             CONF.house_keeping.amphora_expiry_age)
    LOG.info(_LI('Load balancer expiry age is %s seconds'),
             CONF.house_keeping.load_balancer_expiry_age)

    db_cleanup = house_keeping.DatabaseCleanup()
    while db_cleanup_thread_event.is_set():
        LOG.debug("Initiating the cleanup of old resources...")
        db_cleanup.delete_old_amphorae()
        db_cleanup.cleanup_load_balancers()
        time.sleep(interval)

def cert_rotation():
    """Perform certificate rotation."""
    interval = CONF.house_keeping.cert_interval
    LOG.info(
        _LI("Expiring certificate check interval is set to %d sec"), interval)
    cert_rotate = house_keeping.CertRotation()
    while cert_rotate_thread_event.is_set():
        LOG.debug("Initiating certification rotation ...")
        cert_rotate.rotate()
        time.sleep(interval)

def main():
    service.prepare_service(sys.argv)

    gmr.TextGuruMeditation.setup_autorun(version)

    timestamp = str(datetime.datetime.utcnow())
    LOG.info(_LI("Starting house keeping at %s"), timestamp)

    # Thread to perform spare amphora check
    spare_amp_thread = threading.Thread(target=spare_amphora_check)
    spare_amp_thread.daemon = True
    spare_amp_thread_event.set()
    spare_amp_thread.start()

    # Thread to perform db cleanup
    db_cleanup_thread = threading.Thread(target=db_cleanup)
    db_cleanup_thread.daemon = True
    db_cleanup_thread_event.set()
    db_cleanup_thread.start()

    # Thread to perform certificate rotation
    cert_rotate_thread = threading.Thread(target=cert_rotation)
    cert_rotate_thread.daemon = True
    cert_rotate_thread_event.set()
    cert_rotate_thread.start()

    # Try-Exception block should be at the end to gracefully exit threads
    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        LOG.info(_LI("Attempting to gracefully terminate House-Keeping"))
        spare_amp_thread_event.clear()
        db_cleanup_thread_event.clear()
        cert_rotate_thread_event.clear()
        spare_amp_thread.join()
        db_cleanup_thread.join()
        cert_rotate_thread.join()
        LOG.info(_LI("House-Keeping process terminated"))
<|endoftext|>


<commit_before>import re
import logging
import serial

log = logging.getLogger(__name__)

class SmartMeter(object):

    def __init__(self, port, *args, **kwargs):
        try:
            self.serial = serial.Serial(
                port,
                9600,
                timeout=2,
                bytesize=serial.SEVENBITS,
                parity=serial.PARITY_EVEN,
                stopbits=serial.STOPBITS_ONE
            )
        except serial.SerialException as e:
            raise SmartMeterError(e)
        else:
            self.serial.setRTS(False)
            self.port = self.serial.name

        log.info('New serial connection opened to %s', self.port)

    def connect(self):
        if not self.serial.isOpen():
            log.info('Opening connection to `%s`', self.serial.name)
            self.serial.open()
            self.serial.setRTS(False)
        else:
            log.debug('`%s` was already open.', self.serial.name)

    def disconnect(self):
        if self.serial.isOpen():
            log.info('Closing connection to `%s`.', self.serial.name)
            self.serial.close()
        else:
            log.debug('`%s` was already closed.', self.serial.name)

    def connected(self):
        return self.serial.isOpen()

    def read_one_packet(self):
        lines = []
        lines_read = 0
        complete_packet = False

        log.info('Start reading lines')

        while not complete_packet:
            line = ''
            try:
                line = self.serial.readline().strip()
            except Exception as e:
                log.error(e)
                log.error('Read a total of %d lines', lines_read)
                raise SmartMeterError(e)
            else:
                lines_read += 1
                if line.startswith('/ISk5'):
                    lines = [line]
                else:
                    lines.append(line)
                if line == '!' and len(lines) > 19:
                    complete_packet = True
            finally:
                log.debug('>> %s', line)

        log.info('Done reading one packet (containing %d lines)' % len(lines))
        log.debug('Total lines read from serial port: %d', lines_read)
        log.debug('Constructing P1Packet from raw data')

        return P1Packet('\n'.join(lines))

class SmartMeterError(Exception):
    pass

class P1Packet(object):
    _raw = ''

    def __init__(self, data):
        if type(data) == list:
            self._raw = '\n'.join(data)
        else:
            self._raw = data

        keys = {}
        keys['header'] = self.get(r'^(/ISk5.*)$', '')

        keys['kwh'] = {}
        keys['kwh']['eid'] = self.get(r'^0-0:96\.1\.1\(([^)]+)\)$')
        keys['kwh']['tariff'] = self.get_int(r'^0-0:96\.14\.0\(([0-9]+)\)$')
        keys['kwh']['switch'] = self.get_int(r'^0-0:96\.3\.10\((\d)\)$')
        keys['kwh']['treshold'] = self.get_float(r'^0-0:17\.0\.0\(([0-9]{4}\.[0-9]{2})\*kW\)$')

        keys['kwh']['high'] = {}
        keys['kwh']['high']['consumed'] = self.get_float(r'^1-0:1\.8\.1\(([0-9]{5}\.[0-9]{3})\*kWh\)$')
        keys['kwh']['high']['produced'] = self.get_float(r'^1-0:2\.8\.1\(([0-9]{5}\.[0-9]{3})\*kWh\)$')

        keys['kwh']['low'] = {}
        keys['kwh']['low']['consumed'] = self.get_float(r'^1-0:1\.8\.2\(([0-9]{5}\.[0-9]{3})\*kWh\)$')
        keys['kwh']['low']['produced'] = self.get_float(r'^1-0:2\.8\.2\(([0-9]{5}\.[0-9]{3})\*kWh\)$')

        keys['kwh']['current_consumed'] = self.get_float(r'^1-0:1\.7\.0\(([0-9]{4}\.[0-9]{2})\*kW\)$')
        keys['kwh']['current_produced'] = self.get_float(r'^1-0:2\.7\.0\(([0-9]{4}\.[0-9]{2})\*kW\)$')

        keys['gas'] = {}
        keys['gas']['eid'] = self.get(r'^0-1:96\.1\.0\(([^)]+)\)$')
        keys['gas']['device_type'] = self.get_int(r'^0-1:24\.1\.0\((\d)\)$')
        keys['gas']['total'] = self.get_float(r'^\(([0-9]{5}\.[0-9]{3})\)$')
        keys['gas']['valve'] = self.get_int(r'^0-1:24\.4\.0\((\d)\)$')

        keys['msg'] = {}
        keys['msg']['code'] = self.get(r'^0-0:96\.13\.1\((\d+)\)$')
        keys['msg']['text'] = self.get(r'^0-0:96\.13\.0\((.+)\)$')

        self._keys = keys

    def __getitem__(self, key):
        return self._keys[key]

    def get_float(self, regex, default=None):
        result = self.get(regex, None)
        if not result:
            return default
        return float(self.get(regex, default))

    def get_int(self, regex, default=None):
        result = self.get(regex, None)
        if not result:
            return default
        return int(result)

    def get(self, regex, default=None):
        results = re.search(regex, self._raw, re.MULTILINE)
        if not results:
            return default
        return results.group(1)

    def __str__(self):
        return self._raw
<commit_msg>Increase timeout to 10 seconds<commit_after>import re
import logging
import serial

log = logging.getLogger(__name__)

class SmartMeter(object):

    def __init__(self, port, *args, **kwargs):
        try:
            self.serial = serial.Serial(
                port,
                9600,
                timeout=10,
                bytesize=serial.SEVENBITS,
                parity=serial.PARITY_EVEN,
                stopbits=serial.STOPBITS_ONE
            )
        except serial.SerialException as e:
            raise SmartMeterError(e)
        else:
            self.serial.setRTS(False)
            self.port = self.serial.name

        log.info('New serial connection opened to %s', self.port)

    def connect(self):
        if not self.serial.isOpen():
            log.info('Opening connection to `%s`', self.serial.name)
            self.serial.open()
            self.serial.setRTS(False)
        else:
            log.debug('`%s` was already open.', self.serial.name)

    def disconnect(self):
        if self.serial.isOpen():
            log.info('Closing connection to `%s`.', self.serial.name)
            self.serial.close()
        else:
            log.debug('`%s` was already closed.', self.serial.name)

    def connected(self):
        return self.serial.isOpen()

    def read_one_packet(self):
        lines = []
        lines_read = 0
        complete_packet = False

        log.info('Start reading lines')

        while not complete_packet:
            line = ''
            try:
                line = self.serial.readline().strip()
            except Exception as e:
                log.error(e)
                log.error('Read a total of %d lines', lines_read)
                raise SmartMeterError(e)
            else:
                lines_read += 1
                if line.startswith('/ISk5'):
                    lines = [line]
                else:
                    lines.append(line)
                if line == '!' and len(lines) > 19:
                    complete_packet = True
            finally:
                log.debug('>> %s', line)

        log.info('Done reading one packet (containing %d lines)' % len(lines))
        log.debug('Total lines read from serial port: %d', lines_read)
        log.debug('Constructing P1Packet from raw data')

        return P1Packet('\n'.join(lines))

class SmartMeterError(Exception):
    pass

class P1Packet(object):
    _raw = ''

    def __init__(self, data):
        if type(data) == list:
            self._raw = '\n'.join(data)
        else:
            self._raw = data

        keys = {}
        keys['header'] = self.get(r'^(/ISk5.*)$', '')

        keys['kwh'] = {}
        keys['kwh']['eid'] = self.get(r'^0-0:96\.1\.1\(([^)]+)\)$')
        keys['kwh']['tariff'] = self.get_int(r'^0-0:96\.14\.0\(([0-9]+)\)$')
        keys['kwh']['switch'] = self.get_int(r'^0-0:96\.3\.10\((\d)\)$')
        keys['kwh']['treshold'] = self.get_float(r'^0-0:17\.0\.0\(([0-9]{4}\.[0-9]{2})\*kW\)$')

        keys['kwh']['high'] = {}
        keys['kwh']['high']['consumed'] = self.get_float(r'^1-0:1\.8\.1\(([0-9]{5}\.[0-9]{3})\*kWh\)$')
        keys['kwh']['high']['produced'] = self.get_float(r'^1-0:2\.8\.1\(([0-9]{5}\.[0-9]{3})\*kWh\)$')

        keys['kwh']['low'] = {}
        keys['kwh']['low']['consumed'] = self.get_float(r'^1-0:1\.8\.2\(([0-9]{5}\.[0-9]{3})\*kWh\)$')
        keys['kwh']['low']['produced'] = self.get_float(r'^1-0:2\.8\.2\(([0-9]{5}\.[0-9]{3})\*kWh\)$')

        keys['kwh']['current_consumed'] = self.get_float(r'^1-0:1\.7\.0\(([0-9]{4}\.[0-9]{2})\*kW\)$')
        keys['kwh']['current_produced'] = self.get_float(r'^1-0:2\.7\.0\(([0-9]{4}\.[0-9]{2})\*kW\)$')

        keys['gas'] = {}
        keys['gas']['eid'] = self.get(r'^0-1:96\.1\.0\(([^)]+)\)$')
        keys['gas']['device_type'] = self.get_int(r'^0-1:24\.1\.0\((\d)\)$')
        keys['gas']['total'] = self.get_float(r'^\(([0-9]{5}\.[0-9]{3})\)$')
        keys['gas']['valve'] = self.get_int(r'^0-1:24\.4\.0\((\d)\)$')

        keys['msg'] = {}
        keys['msg']['code'] = self.get(r'^0-0:96\.13\.1\((\d+)\)$')
        keys['msg']['text'] = self.get(r'^0-0:96\.13\.0\((.+)\)$')

        self._keys = keys

    def __getitem__(self, key):
        return self._keys[key]

    def get_float(self, regex, default=None):
        result = self.get(regex, None)
        if not result:
            return default
        return float(self.get(regex, default))

    def get_int(self, regex, default=None):
        result = self.get(regex, None)
        if not result:
            return default
        return int(result)

    def get(self, regex, default=None):
        results = re.search(regex, self._raw, re.MULTILINE)
        if not results:
            return default
        return results.group(1)

    def __str__(self):
        return self._raw
<|endoftext|>


<commit_before>__version__ = "0.14.1rc0"

# Void cython.* directives (for case insensitive operating systems).
from Cython.Shadow import *
<commit_msg>Version number.<commit_after>__version__ = "0.14.1rc1"

# Void cython.* directives (for case insensitive operating systems).
from Cython.Shadow import *
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2016, 2017 dpa-infocom GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import aiobotocore
import asyncio
import boto3
import json
import logging
import os
import yaml
from botocore.client import Config
from botocore.exceptions import ClientError
from livebridge.config import AWS

logger = logging.getLogger(__name__)

class ControlFile(object):

    def __init__(self):
        self.control_data = {}
        self._sqs_client = None
        self.config = AWS

    def __del__(self):
        if self._sqs_client:
            self._sqs_client.close()

    @property
    async def sqs_client(self):
        if self._sqs_client:
            return self._sqs_client
        session = aiobotocore.get_session()
        self._sqs_client = session.create_client('sqs',
             region_name=self.config["region"],
             aws_secret_access_key=self.config["secret_key"] or None,
             aws_access_key_id=self.config["access_key"] or None)

        # purge queue before starting watching
        try:
            await self._sqs_client.purge_queue(
                QueueUrl=self.config["sqs_s3_queue"]
            )
            logger.info("Purged SQS queue {}".format(self.config["sqs_s3_queue"]))
        except ClientError as exc:
            logger.warning("Purging SQS queue failed with: {}".format(exc))

        return self._sqs_client

    async def check_control_change(self):
        client = await self.sqs_client
        # check for update events
        try:
            response = await client.receive_message(
                QueueUrl=self.config["sqs_s3_queue"]
            )
            for msg in response.get("Messages", []):
                logger.debug("SQS {}".format(msg.get("MessageId")))
                body = msg.get("Body")
                data = json.loads(body) if body else None
                await client.delete_message(
                    QueueUrl=self.config["sqs_s3_queue"],
                    ReceiptHandle=msg.get("ReceiptHandle")
                )
                if data:
                    for rec in data.get("Records", []):
                        logger.debug("EVENT: {} {}".format(
                            rec.get("s3", {}).get("object", {}).get("key"), rec.get("eventName")))
                        return True
        except Exception as exc:
            logger.error("Error fetching SQS messages with: {}".format(exc))
        return False

    def load(self, path, *, resolve_auth=False):
        if not path.startswith("s3://"):
            body = self.__del__load_from_file(path)
        else:
            body = self._load_from_s3(path)
        return yaml.load(body)

    def _load_from_file(self, path):
        logger.info("Loading control file from disk: {}".format(path))
        if not os.path.exists(path):
            raise IOError("Path for control file not found.")

        file = open(path, "r")
        body = file.read()
        file.close()

        return body

    def _load_from_s3(self, url):
        bucket, key = url.split('/', 2)[-1].split('/', 1)
        logger.info("Loading control file from s3: {} - {}".format(bucket, key))
        config = Config(signature_version="s3v4") if self.config["region"] in ["eu-central-1"] else None
        client = boto3.client(
            's3',
            region_name=self.config["region"],
            aws_access_key_id=self.config["access_key"] or None,
            aws_secret_access_key=self.config["secret_key"] or None,
            config=config,
        )
        control_file = client.get_object(Bucket=bucket, Key=key)
        return control_file["Body"].read()
<commit_msg>Fixed typo.<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2016, 2017 dpa-infocom GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import aiobotocore
import asyncio
import boto3
import json
import logging
import os
import yaml
from botocore.client import Config
from botocore.exceptions import ClientError
from livebridge.config import AWS

logger = logging.getLogger(__name__)

class ControlFile(object):

    def __init__(self):
        self.control_data = {}
        self._sqs_client = None
        self.config = AWS

    def __del__(self):
        if self._sqs_client:
            self._sqs_client.close()

    @property
    async def sqs_client(self):
        if self._sqs_client:
            return self._sqs_client
        session = aiobotocore.get_session()
        self._sqs_client = session.create_client('sqs',
             region_name=self.config["region"],
             aws_secret_access_key=self.config["secret_key"] or None,
             aws_access_key_id=self.config["access_key"] or None)

        # purge queue before starting watching
        try:
            await self._sqs_client.purge_queue(
                QueueUrl=self.config["sqs_s3_queue"]
            )
            logger.info("Purged SQS queue {}".format(self.config["sqs_s3_queue"]))
        except ClientError as exc:
            logger.warning("Purging SQS queue failed with: {}".format(exc))

        return self._sqs_client

    async def check_control_change(self):
        client = await self.sqs_client
        # check for update events
        try:
            response = await client.receive_message(
                QueueUrl=self.config["sqs_s3_queue"]
            )
            for msg in response.get("Messages", []):
                logger.debug("SQS {}".format(msg.get("MessageId")))
                body = msg.get("Body")
                data = json.loads(body) if body else None
                await client.delete_message(
                    QueueUrl=self.config["sqs_s3_queue"],
                    ReceiptHandle=msg.get("ReceiptHandle")
                )
                if data:
                    for rec in data.get("Records", []):
                        logger.debug("EVENT: {} {}".format(
                            rec.get("s3", {}).get("object", {}).get("key"), rec.get("eventName")))
                        return True
        except Exception as exc:
            logger.error("Error fetching SQS messages with: {}".format(exc))
        return False

    def load(self, path, *, resolve_auth=False):
        if not path.startswith("s3://"):
            body = self._load_from_file(path)
        else:
            body = self._load_from_s3(path)
        return yaml.load(body)

    def _load_from_file(self, path):
        logger.info("Loading control file from disk: {}".format(path))
        if not os.path.exists(path):
            raise IOError("Path for control file not found.")

        file = open(path, "r")
        body = file.read()
        file.close()

        return body

    def _load_from_s3(self, url):
        bucket, key = url.split('/', 2)[-1].split('/', 1)
        logger.info("Loading control file from s3: {} - {}".format(bucket, key))
        config = Config(signature_version="s3v4") if self.config["region"] in ["eu-central-1"] else None
        client = boto3.client(
            's3',
            region_name=self.config["region"],
            aws_access_key_id=self.config["access_key"] or None,
            aws_secret_access_key=self.config["secret_key"] or None,
            config=config,
        )
        control_file = client.get_object(Bucket=bucket, Key=key)
        return control_file["Body"].read()
<|endoftext|>


<commit_before>#!/usr/bin/python

import unittest
import math

class result:
	def __init__(self, histogram, frequencies, passed):
		self.histogram = histogram
		self.frequencies = frequencies
		self.passed = passed

	def __str__(self):
		output = ""
		bfd = benfords_frequncy_distribution()
		for digit in range(1, 10):
			output += str(digit) + ":\t" + str(self.histogram[digit]) + "\t"
			output += str(self.frequencies[digit]) + "\t" + str(bfd[digit]) + "\t" + str(self.passed[digit]) + "\n"
		return output

def digits():
	return {x: 0 for x in range(1, 10)}

def benfords_frequncy_distribution():
	return {x: math.log(1 + (1/x), 10) for x in range (1, 10)}

def calculate_frequncies_from_histogram(histogram):
	count = 0
	for c in histogram.itervalues():
		count += c
	frequencies  = dict()
	for k,v in histogram.iteritems():
		frequencies[k] = v * 100 / count
	return frequencies

def calculate_benfords_histogram_from_text(text):
	digits_histogram = digits()
	for word in text.split():
		if word[0].isdigit(): # TODO is not a date
			digits_histogram[int(word[0])] += 1
	return digits_histogram

def benfords_law_test_for_digits_frequencies(frequencies):
	bfd = benfords_frequncy_distribution()
	return {x: frequencies[x] == bfd[x] for x in range(1, 10)}

class benfords_law:

	def __init__(self, text):
		self.text = text

	def calculate_histogram(self):
		return calculate_benfords_histogram_from_text(self.text)

	def get_result(self):
		histogram = self.calculate_histogram()
		return format_result(result(histogram, calculate_frequncies_from_histogram(histogram), {x: False for x in range (1, 10)}))

class test_calculate_benfords_histogram_from_text(unittest.TestCase):
	def test_one_number(self):
		text = "12345"
		digits_histogram = {1:1, 2:0, 3:0, 4:0, 5:0, 6:0, 7:0, 8:0, 9:0}
		self.assertEqual(calculate_benfords_histogram_from_text(text), digits_histogram)

	def test_two_numbers_separated_by_space(self):
		text = "12345 234"
		digits_histogram = {1:1, 2:1, 3:0, 4:0, 5:0, 6:0, 7:0, 8:0, 9:0}
		self.assertEqual(calculate_benfords_histogram_from_text(text), digits_histogram)

class test_result_str(unittest.TestCase):
	def test_print(self):
		digits_histogram = {x: x for x in range(1, 10)}
		digits_frequencies = {x: 10 - x for x in range(1, 10)}
		digits_passed = {x: (x % 2) == 0 for x in range(1, 10)}
		r = result(digits_histogram, digits_frequencies, digits_passed)
		bfd = benfords_frequncy_distribution()
		output = ""
		for digit in range(1, 10):
			output += str(digit) + ":\t" + str(digits_histogram[digit]) + "\t"
			output += str(digits_frequencies[digit]) + "\t" + str(bfd[digit]) + "\t" + str(digits_passed[digit]) + "\n"
		self.assertEqual(output, str(r))

class test_calculate_frequncies_from_histogram(unittest.TestCase):
	def test_two_positions(self):
		histogram = {1: 2, 2: 18}
		frequencies = {1: 10, 2: 90}
		self.assertEqual(frequencies, calculate_frequncies_from_histogram(histogram))

class test_benfords_law_test_for_digits_frequencies(unittest.TestCase):
	def test_all_failed(self):
		frequencies = {x: 10 for x in range(1, 10)}
		marks = {x: False for x in range(1, 10)}
		self.assertEqual(marks, benfords_law_test_for_digits_frequencies(frequencies))

	def test_two_passed(self):
		frequencies = {x: 10 for x in range(1, 10)}
		marks = {x: False for x in range(1, 10)}
		bfd = benfords_frequncy_distribution()
		for i in [2, 7]:
			frequencies[i] = bfd[i]
			marks[i] = True
		self.assertEqual(marks, benfords_law_test_for_digits_frequencies(frequencies))

unittest.main()
<commit_msg>benfords_law_test_for_digits_frequencies with treshold<commit_after>#!/usr/bin/python

import unittest
import math

class result:
	def __init__(self, histogram, frequencies, passed):
		self.histogram = histogram
		self.frequencies = frequencies
		self.passed = passed

	def __str__(self):
		output = ""
		bfd = benfords_frequncy_distribution()
		for digit in range(1, 10):
			output += str(digit) + ":\t" + str(self.histogram[digit]) + "\t"
			output += str(self.frequencies[digit]) + "\t" + str(bfd[digit]) + "\t" + str(self.passed[digit]) + "\n"
		return output

def digits():
	return {x: 0 for x in range(1, 10)}

def benfords_frequncy_distribution():
	return {x: math.log(1 + (1/x), 10) for x in range (1, 10)}

def calculate_frequncies_from_histogram(histogram):
	count = 0
	for c in histogram.itervalues():
		count += c
	frequencies  = dict()
	for k,v in histogram.iteritems():
		frequencies[k] = v * 100 / count
	return frequencies

def calculate_benfords_histogram_from_text(text):
	digits_histogram = digits()
	for word in text.split():
		if word[0].isdigit(): # TODO is not a date
			digits_histogram[int(word[0])] += 1
	return digits_histogram

def benfords_law_test_for_digits_frequencies(frequencies, treshold = 0):
	bfd = benfords_frequncy_distribution()
	return {x: math.fabs(frequencies[x] - bfd[x]) <= treshold for x in range(1, 10)}

class benfords_law:

	def __init__(self, text):
		self.text = text

	def calculate_histogram(self):
		return calculate_benfords_histogram_from_text(self.text)

	def get_result(self):
		histogram = self.calculate_histogram()
		return format_result(result(histogram, calculate_frequncies_from_histogram(histogram), {x: False for x in range (1, 10)}))

class test_calculate_benfords_histogram_from_text(unittest.TestCase):
	def test_one_number(self):
		text = "12345"
		digits_histogram = {1:1, 2:0, 3:0, 4:0, 5:0, 6:0, 7:0, 8:0, 9:0}
		self.assertEqual(calculate_benfords_histogram_from_text(text), digits_histogram)

	def test_two_numbers_separated_by_space(self):
		text = "12345 234"
		digits_histogram = {1:1, 2:1, 3:0, 4:0, 5:0, 6:0, 7:0, 8:0, 9:0}
		self.assertEqual(calculate_benfords_histogram_from_text(text), digits_histogram)

class test_result_str(unittest.TestCase):
	def test_print(self):
		digits_histogram = {x: x for x in range(1, 10)}
		digits_frequencies = {x: 10 - x for x in range(1, 10)}
		digits_passed = {x: (x % 2) == 0 for x in range(1, 10)}
		r = result(digits_histogram, digits_frequencies, digits_passed)
		bfd = benfords_frequncy_distribution()
		output = ""
		for digit in range(1, 10):
			output += str(digit) + ":\t" + str(digits_histogram[digit]) + "\t"
			output += str(digits_frequencies[digit]) + "\t" + str(bfd[digit]) + "\t" + str(digits_passed[digit]) + "\n"
		self.assertEqual(output, str(r))

class test_calculate_frequncies_from_histogram(unittest.TestCase):
	def test_two_positions(self):
		histogram = {1: 2, 2: 18}
		frequencies = {1: 10, 2: 90}
		self.assertEqual(frequencies, calculate_frequncies_from_histogram(histogram))

class test_benfords_law_test_for_digits_frequencies(unittest.TestCase):
	def test_all_failed(self):
		frequencies = {x: 10 for x in range(1, 10)}
		marks = {x: False for x in range(1, 10)}
		self.assertEqual(marks, benfords_law_test_for_digits_frequencies(frequencies))

	def test_two_passed(self):
		frequencies = {x: 10 for x in range(1, 10)}
		marks = {x: False for x in range(1, 10)}
		bfd = benfords_frequncy_distribution()
		for i in [2, 7]:
			frequencies[i] = bfd[i]
			marks[i] = True
		self.assertEqual(marks, benfords_law_test_for_digits_frequencies(frequencies))

	def test_two_passed_with_treshold(self):
		frequencies = {x: 10 for x in range(1, 10)}
		marks = {x: False for x in range(1, 10)}
		bfd = benfords_frequncy_distribution()
		frequencies[2] = bfd[2] + 1
		marks[2] = True
		frequencies[7] = bfd[7] - 1
		marks[7] = True
		self.assertEqual(marks, benfords_law_test_for_digits_frequencies(frequencies, 1))

unittest.main()
<|endoftext|>


<commit_before>""" Defines the visual part of core: NodePanel, Group and Arrow classes.

Here we're using OpenGL to draw nodes and connections between them. Note the source code is completely PEP8-incompatible;
there's a desperate need to refactor the function names et al...

"""

from core import node
import core.node_draw as node_draw

from core.utils import *
from core.shared import *

import wx

try:
	from OpenGL.GL import *
except ImportError:
	print "The OpenGL extensions do not appear to be installed."
	print "This application cannot run."
	sys.exit(1)

imageCache = {}

def clearImageCache():
	imageCache.clear()

class NodePanel(object):
	def __init__(self, parent, x=100, y=100, showParameters = True, showPreview = False, iconicMode = False):
		self.x = x
		self.y = y

		self.width = 100
		self.height = 150

		self.delta = (0,0)
		self.originalClick = (0, 0)
		self.node = None
		self.owner = parent

		self.showParameters = showParameters # this and following 3 might be a bit field; we'll see about it...
		self.showPreview = False
		self.iconicMode = iconicMode

	def assignNode(self, Node):
		self.node = Node
		Node.panel = self

		width, height, headerh, dh, col1, col2 = node_draw.CalcMinMaxSize(self.node, wx.ClientDC(self.owner))

		self.width = width
		self.height = height

	def __repr__(self):
		return """pnl = NodePanel(self, %d, %d, %s, %s, %s)\npanels.append(pnl)\npnl.assignNode(node%s)\n""" % (self.x, self.y,  str(self.showParameters), str(self.showPreview), str(self.iconicMode), self.node.id)

	def refreshFont(self):
		width, height, headerh, dh, col1, col2 = node_draw.CalcMinMaxSize(self.node, wx.ClientDC(self.owner))

		self.width = width
		self.height = height

		self.ClearCache() # is this enough? no.

	def inside(self, ax, ay):
		return (ax in range(self.x, self.x+self.width)) and ((ay in range(self.y, self.y+self.height)))

	def ClearCache(self):
		imageCache[self.node.id] = None

	def paint(self):
		GL_TEXTURE_RECTANGLE_ARB = 0x84F5
		mode = GL_TEXTURE_RECTANGLE_ARB # GL_TEXTURE_2D

		if self in self.owner.c.markedPanels:
		 	#glDisable(GL_TEXTURE_2D) # to ensure the image isn't "color corrected" by texture :)
			glDisable(mode)
			glColor4f( 248/255.0, 206/255.0, 36/255.0, 0.5 ) # hardcoded selection color

			selectionBorder = 5 # ... and size
			glBegin( GL_QUADS )

			glVertex2i( self.x-selectionBorder, self.y-selectionBorder )
			glVertex2i( self.x+self.width+selectionBorder, self.y-selectionBorder )
			glVertex2i( self.x+self.width+selectionBorder, self.y+self.height+selectionBorder )
			glVertex2i( self.x-selectionBorder, self.y+self.height+selectionBorder )

			glEnd()

	 	glEnable(mode)

		if imageCache.get(self.node.id, None) == None:
			#print "miss"
			TextureBitmap = node_draw.PaintForm(self.node, 0, 0, wx.ClientDC(self.owner))
			image = wx.ImageFromBitmap(TextureBitmap) #.GetData()
			imageCache[self.node.id] = image
		else:
			#print "cached"
			image = imageCache[self.node.id]

		#image = wx.ImageFromBitmap(TextureBitmap).GetData()

		# Create Texture
		_textureName = 0
		_textureName = glGenTextures(1)

		glBindTexture(mode, _textureName)

		glPixelStorei(GL_UNPACK_ALIGNMENT,1)
		glTexImage2D(mode, 0, 3, image.GetWidth(), image.GetHeight(), 0, GL_RGB, GL_UNSIGNED_BYTE, image.GetData())
		glTexParameterf(mode, GL_TEXTURE_WRAP_S, GL_CLAMP)
		glTexParameterf(mode, GL_TEXTURE_WRAP_T, GL_CLAMP)
		glTexParameterf(mode, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
		glTexParameterf(mode, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
		glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_DECAL)

		glColor3f( 1, 1, 1 ) # to ensure the image isn't "color corrected" :)

		glBegin( GL_QUADS )

		glTexCoord2i(0, 0)
		glVertex2i( self.x, self.y )

		#glTexCoord2i(1, 0)
		glTexCoord2i(self.width, 0)
		glVertex2i( self.x+self.width, self.y )

		#glTexCoord2i(1, 1)
		glTexCoord2i(self.width, self.height)
		glVertex2i( self.x+self.width, self.y+self.height )

		#glTexCoord2i(0, 1)
		glTexCoord2i(0, self.height)
		glVertex2i( self.x, self.y+self.height )

		glEnd()

		glDeleteTextures(_textureName)

class Arrow(object):
	def __init__(self, parent):
		self.x1 = 0
		self.y1 = 0
		self.x2 = 0
		self.y2 = 0
		self.connection = None
		self.idx1 = -1
		self.idx2 = -1
		self.owner = parent

	def assignConnection(self, Connection):
		self.connection = Connection
		Connection.arrow = self

		inpn = self.connection.inputNode
		outn = self.connection.outputNode

		inname = self.connection.inputName
		outname = self.connection.outputName

		for i in range(len(outn.in_params)):
			param = outn.in_params[i]
			if param["name"] == outname:
				self.idx2 = i

		for i in range(len(inpn.out_params)):
			param = inpn.out_params[i]
			if param["name"] == inname:
				self.idx1 = i

		if ((self.idx1 == -1) or (self.idx2 == -1)):
			print "connection not found! something fatal!"

		self.refreshFont()

	def __repr__(self):
		return """arr = Arrow(self)\narrows.append(arr)\narr.assignConnection(connection%s)\n""" % (self.connection.id)

	def refreshFont(self):
		self.hoffset1 = node_draw.CalcArrowPosition(self.idx1, wx.ClientDC(self.owner), self.connection.inputNode)
		self.hoffset2 = node_draw.CalcArrowPosition(self.idx2, wx.ClientDC(self.owner), self.connection.outputNode)

	def paint(self):
		if self.connection != None:
			x1 = self.connection.inputNode.panel.x+self.connection.inputNode.panel.width
			y1 = self.connection.inputNode.panel.y+self.hoffset1

			x2 = self.connection.outputNode.panel.x
			y2 = self.connection.outputNode.panel.y+self.hoffset2
		else:
			x1 = self.owner.tax
			y1 = self.owner.tay
			x2 = self.owner.tax2
			y2 = self.owner.tay2

		bezscale = 20
		bezmiddle = abs(x2-x1)/1.5

		glEnable(GL_MAP1_VERTEX_3)
		ctrlpoints=[[x1, y1, 0.01], [x1+bezscale, y1, 0.01], [x1+bezmiddle, y1, 0.01], [x2-bezmiddle, y2, 0.01],[x2-bezscale, y2, 0.01], [x2, y2, 0.01]]
		glMap1f(GL_MAP1_VERTEX_3, 0.0, 1.0, ctrlpoints)
		glColor3f(0.4, 0.4, 0.4)        # grey
		glBegin(GL_LINE_STRIP)
		SEGS=20
		for p in range(0, SEGS+1):
			glEvalCoord1f(float(p)/float(SEGS))
		glEnd()

		arrow_size = 5

		glBegin(GL_POLYGON)
		glVertex2i( x2, y2 )
		glVertex2i( x2-arrow_size, y2-arrow_size )
		glVertex2i( x2-arrow_size, y2+arrow_size )
		glEnd()

class Group(object):
	""" Group of panels """

	_instance_count = 0

	def __init__(self, parent):
		Group._instance_count += 1
		self.id = str(Group._instance_count)
		self.owner = parent

		self.panels = []

	def AddPanel(self, apanel):
		self.panels.append(apanel)

	def __repr__(self): # serialize the state into file
		s = """group%s = Group(self)\ngroups.append(group%s)\n""" % (self.id, self.id)
		for p in self.panels:
			s += "group%s.AddPanel(node%s.panel)\n" % (self.id, p.node.id)
		return s

	def paint(self):
		if len(self.panels):
			w, h, dh = node_draw.CalcGroupSize(self, wx.ClientDC(self.owner))
			#print gs

			minx = min([p.x for p in self.panels])
			maxx = max([(p.x+p.width) for p in self.panels])
			miny = min([p.y for p in self.panels])
			maxy = max([(p.y+p.height) for p in self.panels])

			GL_TEXTURE_RECTANGLE_ARB = 0x84F5
			mode = GL_TEXTURE_RECTANGLE_ARB # GL_TEXTURE_2D

			glDisable(mode)
			glColor4f( 161/255.0, 223/255.0, 149/255.0, 1 ) # hardcoded selection color

			selectionBorder = 5 # ... and size
			glBegin( GL_QUADS )

			# top
			glVertex2i( minx-selectionBorder, miny-h )
			glVertex2i( maxx+selectionBorder, miny-h )
			glVertex2i( maxx+selectionBorder, miny )
			glVertex2i( minx-selectionBorder, miny )

			# left
			glVertex2i( minx-selectionBorder, miny )
			glVertex2i( minx, miny )
			glVertex2i( minx, maxy+selectionBorder )
			glVertex2i( minx-selectionBorder, maxy+selectionBorder )

			# right
			glVertex2i( maxx, miny )
			glVertex2i( maxx+selectionBorder, miny )
			glVertex2i( maxx+selectionBorder, maxy+selectionBorder )
			glVertex2i( maxx, maxy+selectionBorder )

			# bottom
			glVertex2i( minx, maxy )
			glVertex2i( maxx, maxy )
			glVertex2i( maxx, maxy+selectionBorder )
			glVertex2i( minx, maxy+selectionBorder )

			glEnd()

		 	glEnable(mode)
			TextureBitmap = node_draw.PaintGroup(self, wx.ClientDC(self.owner))
			image = wx.ImageFromBitmap(TextureBitmap).GetData()

			# Create Texture
			_textureName = 0
			_textureName = glGenTextures(1)

			glBindTexture(mode, _textureName)

			glPixelStorei(GL_UNPACK_ALIGNMENT,1)
			glTexImage2D(mode, 0, 3, TextureBitmap.GetWidth(), TextureBitmap.GetHeight(), 0, GL_RGB, GL_UNSIGNED_BYTE, image)
			glTexParameterf(mode, GL_TEXTURE_WRAP_S, GL_CLAMP)
			glTexParameterf(mode, GL_TEXTURE_WRAP_T, GL_CLAMP)
			glTexParameterf(mode, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
			glTexParameterf(mode, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
			glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_DECAL)

			glColor4f( 1, 1, 1, 1 ) # to ensure the image isn't "color corrected" :)

			glBegin( GL_QUADS )

			glTexCoord2i(0, 0)
			glVertex2i( minx, miny-h )

			glTexCoord2i(w, 0)
			glVertex2i( minx+w, miny-h )

			glTexCoord2i(w, h)
			glVertex2i( minx+w, miny )

			glTexCoord2i(0, h)
			glVertex2i( minx, miny )

			glEnd()

			glDeleteTextures(_textureName)

<commit_msg>Texture_mode refactoring; support for groups movement on canvas<commit_after>""" Defines the visual part of core: NodePanel, Group and Arrow classes.

Here we're using OpenGL to draw nodes and connections between them. Note the source code is completely PEP8-incompatible;
there's a desperate need to refactor the function names et al...

"""

from core import node
import core.node_draw as node_draw

from core.utils import *
from core.shared import *

import wx

try:
	from OpenGL.GL import *
except ImportError:
	print "The OpenGL extensions do not appear to be installed."
	print "This application cannot run."
	sys.exit(1)

imageCache = {}

def clearImageCache():
	imageCache.clear()

GL_TEXTURE_RECTANGLE_ARB = 0x84F5
texture_mode = GL_TEXTURE_RECTANGLE_ARB # GL_TEXTURE_2D

selectionBorder = 5

class NodePanel(object):
	def __init__(self, parent, x=100, y=100, showParameters = True, showPreview = False, iconicMode = False):
		self.x = x
		self.y = y

		self.width = 100
		self.height = 150

		self.delta = (0,0)
		#self.originalClick = (0, 0)
		self.node = None
		self.owner = parent

		self.showParameters = showParameters # this and following 3 might be a bit field; we'll see about it...
		self.showPreview = False
		self.iconicMode = iconicMode

	def assignNode(self, Node):
		self.node = Node
		Node.panel = self

		width, height, headerh, dh, col1, col2 = node_draw.CalcMinMaxSize(self.node, wx.ClientDC(self.owner))

		self.width = width
		self.height = height

	def __repr__(self):
		return """pnl = NodePanel(self, %d, %d, %s, %s, %s)\npanels.append(pnl)\npnl.assignNode(node%s)\n""" % (self.x, self.y,  str(self.showParameters), str(self.showPreview), str(self.iconicMode), self.node.id)

	def refreshFont(self):
		width, height, headerh, dh, col1, col2 = node_draw.CalcMinMaxSize(self.node, wx.ClientDC(self.owner))

		self.width = width
		self.height = height

		self.ClearCache() # is this enough? no.

	def inside(self, ax, ay):
		return (ax in range(self.x, self.x+self.width)) and ((ay in range(self.y, self.y+self.height)))

	def ClearCache(self):
		imageCache[self.node.id] = None

	def paint(self):
		if self in self.owner.c.markedPanels:
		 	#glDisable(GL_TEXTURE_2D) # to ensure the image isn't "color corrected" by texture :)
			glDisable(texture_mode)
			glColor4f( 248/255.0, 206/255.0, 36/255.0, 0.5 ) # hardcoded selection color

			selectionBorder = 5 # ... and size
			glBegin( GL_QUADS )

			glVertex2i( self.x-selectionBorder, self.y-selectionBorder )
			glVertex2i( self.x+self.width+selectionBorder, self.y-selectionBorder )
			glVertex2i( self.x+self.width+selectionBorder, self.y+self.height+selectionBorder )
			glVertex2i( self.x-selectionBorder, self.y+self.height+selectionBorder )

			glEnd()

	 	glEnable(texture_mode)

		if imageCache.get(self.node.id, None) == None:
			#print "miss"
			TextureBitmap = node_draw.PaintForm(self.node, 0, 0, wx.ClientDC(self.owner))
			image = wx.ImageFromBitmap(TextureBitmap) #.GetData()
			imageCache[self.node.id] = image
		else:
			#print "cached"
			image = imageCache[self.node.id]

		#image = wx.ImageFromBitmap(TextureBitmap).GetData()

		# Create Texture
		_textureName = 0
		_textureName = glGenTextures(1)

		glBindTexture(texture_mode, _textureName)

		glPixelStorei(GL_UNPACK_ALIGNMENT,1)
		glTexImage2D(texture_mode, 0, 3, image.GetWidth(), image.GetHeight(), 0, GL_RGB, GL_UNSIGNED_BYTE, image.GetData())
		glTexParameterf(texture_mode, GL_TEXTURE_WRAP_S, GL_CLAMP)
		glTexParameterf(texture_mode, GL_TEXTURE_WRAP_T, GL_CLAMP)
		glTexParameterf(texture_mode, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
		glTexParameterf(texture_mode, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
		glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_DECAL)

		glColor3f( 1, 1, 1 ) # to ensure the image isn't "color corrected" :)

		glBegin( GL_QUADS )

		glTexCoord2i(0, 0)
		glVertex2i( self.x, self.y )

		#glTexCoord2i(1, 0)
		glTexCoord2i(self.width, 0)
		glVertex2i( self.x+self.width, self.y )

		#glTexCoord2i(1, 1)
		glTexCoord2i(self.width, self.height)
		glVertex2i( self.x+self.width, self.y+self.height )

		#glTexCoord2i(0, 1)
		glTexCoord2i(0, self.height)
		glVertex2i( self.x, self.y+self.height )

		glEnd()

		glDeleteTextures(_textureName)

class Arrow(object):
	def __init__(self, parent):
		self.x1 = 0
		self.y1 = 0
		self.x2 = 0
		self.y2 = 0
		self.connection = None
		self.idx1 = -1
		self.idx2 = -1
		self.owner = parent

	def assignConnection(self, Connection):
		self.connection = Connection
		Connection.arrow = self

		inpn = self.connection.inputNode
		outn = self.connection.outputNode

		inname = self.connection.inputName
		outname = self.connection.outputName

		for i in range(len(outn.in_params)):
			param = outn.in_params[i]
			if param["name"] == outname:
				self.idx2 = i

		for i in range(len(inpn.out_params)):
			param = inpn.out_params[i]
			if param["name"] == inname:
				self.idx1 = i

		if ((self.idx1 == -1) or (self.idx2 == -1)):
			print "connection not found! something fatal!"

		self.refreshFont()

	def __repr__(self):
		return """arr = Arrow(self)\narrows.append(arr)\narr.assignConnection(connection%s)\n""" % (self.connection.id)

	def refreshFont(self):
		self.hoffset1 = node_draw.CalcArrowPosition(self.idx1, wx.ClientDC(self.owner), self.connection.inputNode)
		self.hoffset2 = node_draw.CalcArrowPosition(self.idx2, wx.ClientDC(self.owner), self.connection.outputNode)

	def paint(self):
		if self.connection != None:
			x1 = self.connection.inputNode.panel.x+self.connection.inputNode.panel.width
			y1 = self.connection.inputNode.panel.y+self.hoffset1

			x2 = self.connection.outputNode.panel.x
			y2 = self.connection.outputNode.panel.y+self.hoffset2
		else:
			x1 = self.owner.tax
			y1 = self.owner.tay
			x2 = self.owner.tax2
			y2 = self.owner.tay2

		bezscale = 20
		bezmiddle = abs(x2-x1)/1.5

		glEnable(GL_MAP1_VERTEX_3)
		ctrlpoints=[[x1, y1, 0.01], [x1+bezscale, y1, 0.01], [x1+bezmiddle, y1, 0.01], [x2-bezmiddle, y2, 0.01],[x2-bezscale, y2, 0.01], [x2, y2, 0.01]]
		glMap1f(GL_MAP1_VERTEX_3, 0.0, 1.0, ctrlpoints)
		glColor3f(0.4, 0.4, 0.4)        # grey
		glBegin(GL_LINE_STRIP)
		SEGS=20
		for p in range(0, SEGS+1):
			glEvalCoord1f(float(p)/float(SEGS))
		glEnd()

		arrow_size = 5

		glBegin(GL_POLYGON)
		glVertex2i( x2, y2 )
		glVertex2i( x2-arrow_size, y2-arrow_size )
		glVertex2i( x2-arrow_size, y2+arrow_size )
		glEnd()

class Group(object):
	""" Group of panels """

	_instance_count = 0

	def __init__(self, parent):
		Group._instance_count += 1
		self.id = str(Group._instance_count)
		self.owner = parent

		self.panels = []

		self.delta = (0,0)

	def AddPanel(self, apanel):
		self.panels.append(apanel)

	def __repr__(self): # serialize the state into file
		s = """group%s = Group(self)\ngroups.append(group%s)\n""" % (self.id, self.id)
		for p in self.panels:
			s += "group%s.AddPanel(node%s.panel)\n" % (self.id, p.node.id)
		return s

	def insideHeader(self, ax, ay):
		#print "checking..."
		if len(self.panels):
			w, h, dh = node_draw.CalcGroupSize(self, wx.ClientDC(self.owner))

			selectionBorder = 5 # ... and size

			minx = min([p.x for p in self.panels])-selectionBorder
			maxx = max([(p.x+p.width) for p in self.panels])+selectionBorder
			maxy = min([p.y for p in self.panels])
			miny = maxy-h

			#print ax, ay, minx, maxx, miny, maxy

			return (ax in range(minx, maxx)) and ((ay in range(miny, maxy)))
		else:
			return False

	def getLeft(self):
		return min([p.x for p in self.panels])-selectionBorder

	def getTop(self):
		w, h, dh = node_draw.CalcGroupSize(self, wx.ClientDC(self.owner))
		return min([p.y for p in self.panels])-h

	def paint(self):
		if len(self.panels):
			w, h, dh = node_draw.CalcGroupSize(self, wx.ClientDC(self.owner))

			minx = min([p.x for p in self.panels])
			maxx = max([(p.x+p.width) for p in self.panels])
			miny = min([p.y for p in self.panels])
			maxy = max([(p.y+p.height) for p in self.panels])

			glDisable(texture_mode)
			glColor4f( 161/255.0, 223/255.0, 149/255.0, 1 ) # hardcoded selection color

			selectionBorder = 5 # ... and size
			glBegin( GL_QUADS )

			# top
			glVertex2i( minx-selectionBorder, miny-h )
			glVertex2i( maxx+selectionBorder, miny-h )
			glVertex2i( maxx+selectionBorder, miny )
			glVertex2i( minx-selectionBorder, miny )

			# left
			glVertex2i( minx-selectionBorder, miny )
			glVertex2i( minx, miny )
			glVertex2i( minx, maxy+selectionBorder )
			glVertex2i( minx-selectionBorder, maxy+selectionBorder )

			# right
			glVertex2i( maxx, miny )
			glVertex2i( maxx+selectionBorder, miny )
			glVertex2i( maxx+selectionBorder, maxy+selectionBorder )
			glVertex2i( maxx, maxy+selectionBorder )

			# bottom
			glVertex2i( minx, maxy )
			glVertex2i( maxx, maxy )
			glVertex2i( maxx, maxy+selectionBorder )
			glVertex2i( minx, maxy+selectionBorder )

			glEnd()

		 	glEnable(texture_mode)
			TextureBitmap = node_draw.PaintGroup(self, wx.ClientDC(self.owner))
			image = wx.ImageFromBitmap(TextureBitmap).GetData()

			# Create Texture
			_textureName = 0
			_textureName = glGenTextures(1)

			glBindTexture(texture_mode, _textureName)

			glPixelStorei(GL_UNPACK_ALIGNMENT,1)
			glTexImage2D(texture_mode, 0, 3, TextureBitmap.GetWidth(), TextureBitmap.GetHeight(), 0, GL_RGB, GL_UNSIGNED_BYTE, image)
			glTexParameterf(texture_mode, GL_TEXTURE_WRAP_S, GL_CLAMP)
			glTexParameterf(texture_mode, GL_TEXTURE_WRAP_T, GL_CLAMP)
			glTexParameterf(texture_mode, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
			glTexParameterf(texture_mode, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
			glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_DECAL)

			glColor4f( 1, 1, 1, 1 ) # to ensure the image isn't "color corrected" :)

			glBegin( GL_QUADS )

			glTexCoord2i(0, 0)
			glVertex2i( minx, miny-h )

			glTexCoord2i(w, 0)
			glVertex2i( minx+w, miny-h )

			glTexCoord2i(w, h)
			glVertex2i( minx+w, miny )

			glTexCoord2i(0, h)
			glVertex2i( minx, miny )

			glEnd()

			glDeleteTextures(_textureName)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# HFacer.py - regenerate the Scintilla.h and SciLexer.h files from the Scintilla.iface interface
# definition file.
# Implemented 2000 by Neil Hodgson neilh@scintilla.org
# Requires Python 2.5 or later

import sys
import os
import Face

from FileGenerator import UpdateFile, Generate, Regenerate, UpdateLineInFile, lineEnd

def printLexHFile(f):
	out = []
	for name in f.order:
		v = f.features[name]
		if v["FeatureType"] in ["val"]:
			if "SCE_" in name or "SCLEX_" in name:
				out.append("#define " + name + " " + v["Value"])
	return out

def printHFile(f):
	out = []
	previousCategory = ""
	for name in f.order:
		v = f.features[name]
		if v["Category"] != "Deprecated":
			if v["Category"] == "Provisional" and previousCategory != "Provisional":
				out.append("#ifndef SCI_DISABLE_PROVISIONAL")
			previousCategory = v["Category"]
			if v["FeatureType"] in ["fun", "get", "set"]:
				featureDefineName = "SCI_" + name.upper()
				out.append("#define " + featureDefineName + " " + v["Value"])
			elif v["FeatureType"] in ["evt"]:
				featureDefineName = "SCN_" + name.upper()
				out.append("#define " + featureDefineName + " " + v["Value"])
			elif v["FeatureType"] in ["val"]:
				if not ("SCE_" in name or "SCLEX_" in name):
					out.append("#define " + name + " " + v["Value"])
	out.append("#endif")
	return out

f = Face.Face()
try:
	f.ReadFromFile("../include/Scintilla.iface")
	Regenerate("../include/Scintilla.h", "/* ", printHFile(f))
	Regenerate("../include/SciLexer.h", "/* ", printLexHFile(f))
	print("Maximum ID is %s" % max([x for x in f.values if int(x) < 3000]))
except:
	raise
<commit_msg>Add code for finding unsued values but leave commented out.<commit_after>#!/usr/bin/env python
# HFacer.py - regenerate the Scintilla.h and SciLexer.h files from the Scintilla.iface interface
# definition file.
# Implemented 2000 by Neil Hodgson neilh@scintilla.org
# Requires Python 2.5 or later

import sys
import os
import Face

from FileGenerator import UpdateFile, Generate, Regenerate, UpdateLineInFile, lineEnd

def printLexHFile(f):
	out = []
	for name in f.order:
		v = f.features[name]
		if v["FeatureType"] in ["val"]:
			if "SCE_" in name or "SCLEX_" in name:
				out.append("#define " + name + " " + v["Value"])
	return out

def printHFile(f):
	out = []
	previousCategory = ""
	for name in f.order:
		v = f.features[name]
		if v["Category"] != "Deprecated":
			if v["Category"] == "Provisional" and previousCategory != "Provisional":
				out.append("#ifndef SCI_DISABLE_PROVISIONAL")
			previousCategory = v["Category"]
			if v["FeatureType"] in ["fun", "get", "set"]:
				featureDefineName = "SCI_" + name.upper()
				out.append("#define " + featureDefineName + " " + v["Value"])
			elif v["FeatureType"] in ["evt"]:
				featureDefineName = "SCN_" + name.upper()
				out.append("#define " + featureDefineName + " " + v["Value"])
			elif v["FeatureType"] in ["val"]:
				if not ("SCE_" in name or "SCLEX_" in name):
					out.append("#define " + name + " " + v["Value"])
	out.append("#endif")
	return out

f = Face.Face()
try:
	f.ReadFromFile("../include/Scintilla.iface")
	Regenerate("../include/Scintilla.h", "/* ", printHFile(f))
	Regenerate("../include/SciLexer.h", "/* ", printLexHFile(f))
	valueSet = set(int(x) for x in f.values if int(x) < 3000)
	maximumID = max(valueSet)
	print("Maximum ID is %d" % maximumID)
	#~ valuesUnused = sorted(x for x in range(2001,maximumID) if x not in valueSet)
	#~ print("\nUnused values")
	#~ for v in valuesUnused:
		#~ print(v)
except:
	raise
<|endoftext|>


<commit_before>"""
Try to run a tf kernel
Check does/doesnt work
"""
import numpy as np
import pyopencl as cl
import subprocess
from test import test_common
from test.test_common import compile_code

def test_cwise_sqrt(context, q, float_data, float_data_gpu):
    options = test_common.cocl_options()
    i = 0
    opt_options = []
    iropencl_options = []
    while i < len(options):
        if options[i] == '--devicell-opt':
            opt_options.append('-' + options[i + 1])
            i += 2
            continue
        if options[i] in ['--run_branching_transforms', '--branches_as_switch']:
            iropencl_options.append(options[i])
            i += 1
            continue
        raise Exception('unknown option ', options[i])
        i += 1
    res = subprocess.run([
        'opt-3.8'
    ] + opt_options + [
        '-S',
        'test/tf/samples/cwise_op_gpu_sqrt-device-noopt.ll',
        '-o', '/tmp/test-opt.ll'
    ], stdout=subprocess.PIPE)
    print(' '.join(res.args))
    assert res.returncode == 0

    res = subprocess.run([
        'build/ir-to-opencl'
    ] + iropencl_options + [
        '--inputfile', '/tmp/test-opt.ll',
        '--outputfile', '/tmp/test-device.cl'
    ], stdout=subprocess.PIPE)
    print(' '.join(res.args))
    assert res.returncode == 0

    with open('/tmp/test-device.cl', 'r') as f:
        cl_sourcecode = f.read()

    prog = cl.Program(context, cl_sourcecode).build()

    N = 10

    # global struct Eigen__TensorEvaluator_7_nopointers* eval_nopointers, global float* eval_ptr0, long eval_ptr0_offset, global float* eval_ptr1, long eval_ptr1_offset, global float* eval_ptr2, long eval_ptr2_offset, long size, local int *scratch

    # what we need:
    # struct Eigen__TensorEvaluator_7_nopointers   Note that none of the values we copy across are actually use, so we can just create a sufficiently large buffer...
    # global float *eval_ptr0  => this will receive the result.  just create a sufficiently large buffer
    # ptr0_offset => 0
    # eval_ptr1 => will contian the data we want to reduce
    # eval_ptr1_offset=> 0
    # eval_ptr2 => just send in a tiny buffer, size 4
    # eval_ptr2_offset => set to -1, means 'null ointer'
    # size =>  eg 10, to reduce 10 values
    # scratch => set to workgroupsize * sizeof(float)

    eval_nopointers_gpu = cl.Buffer(context, cl.mem_flags.READ_WRITE, size=4096)

    eval_ptr0 = np.random.randn(1024).astype(np.float32)
    eval_ptr0_gpu = cl.Buffer(context, cl.mem_flags.READ_WRITE | cl.mem_flags.COPY_HOST_PTR, hostbuf=eval_ptr0)
    eval_ptr0_offset = 0

    eval_ptr1 = np.random.randn(1024).astype(np.float32)
    eval_ptr1_gpu = cl.Buffer(context, cl.mem_flags.READ_WRITE | cl.mem_flags.COPY_HOST_PTR, hostbuf=eval_ptr1)
    eval_ptr1_offset = 0

    eval_ptr2_gpu = cl.Buffer(context, cl.mem_flags.READ_WRITE, size=128)
    eval_ptr2_offset = -1

    size = N

    global_size = 256
    workgroup_size = 256
    scratch = workgroup_size * 4

    prog.__getattr__('_ZN5Eigen8internal15EigenMetaKernelINS_15TensorEvaluatorIKNS_14TensorAssignOpINS_9TensorMapINS_6TensorIfLi1ELi1EiEELi16ENS_11MakePointerEEEKNS_19TensorCwiseBinaryOpINS0_23scalar_sqrt_gradient_opIfEEKNS4_INS5_IKfLi1ELi1EiEELi16ES7_EEKNS4_INS5_ISC_Li1ELi1ElEELi16ES7_EEEEEENS_9GpuDeviceEEElEEvT_T0_')(
        q, (global_size,), (workgroup_size,),
        eval_nopointers_gpu,
        eval_ptr0_gpu, np.int64(eval_ptr0_offset),
        eval_ptr1_gpu, np.int64(eval_ptr1_offset),
        eval_ptr2_gpu, np.int64(eval_ptr2_offset),
        np.int64(size),
        cl.LocalMemory(scratch)
    )
    # copy eval_ptr0 back, and check the results...
    cl.enqueue_copy(q, eval_ptr0, eval_ptr0_gpu)
    q.finish()
    print('eval_ptr0', eval_ptr0[0])

    assert eval_ptr0[0] != 0

    expected_sum = 0
    for i in range(N):
        expected_sum += eval_ptr1[i]
    print('epxected_sum', expected_sum)
    assert abs(expected_sum - eval_ptr0[0]) < 1e-4
<commit_msg>test_tf this test case passes<commit_after>"""
Try to run a tf kernel
Check does/doesnt work
"""
import numpy as np
import pyopencl as cl
import subprocess
import math
from test import test_common
from test.test_common import compile_code

def test_cwise_sqrt(context, q, float_data, float_data_gpu):
    options = test_common.cocl_options()
    i = 0
    opt_options = []
    iropencl_options = []
    while i < len(options):
        if options[i] == '--devicell-opt':
            opt_options.append('-' + options[i + 1])
            i += 2
            continue
        if options[i] in ['--run_branching_transforms', '--branches_as_switch']:
            iropencl_options.append(options[i])
            i += 1
            continue
        raise Exception('unknown option ', options[i])
        i += 1
    print('opt_options', opt_options)
    print('iropencl_options', iropencl_options)
    res = subprocess.run([
        'opt-3.8'
    ] + opt_options + [
        '-S',
        'test/tf/samples/cwise_op_gpu_sqrt-device-noopt.ll',
        '-o', '/tmp/test-opt.ll'
    ], stdout=subprocess.PIPE)
    print(' '.join(res.args))
    assert res.returncode == 0

    res = subprocess.run([
        'build/ir-to-opencl'
    ] + iropencl_options + [
        '--inputfile', '/tmp/test-opt.ll',
        '--outputfile', '/tmp/test-device.cl'
    ], stdout=subprocess.PIPE)
    print(' '.join(res.args))
    assert res.returncode == 0

    with open('/tmp/test-device.cl', 'r') as f:
        cl_sourcecode = f.read()

    prog = cl.Program(context, cl_sourcecode).build()

    N = 10

    # global struct Eigen__TensorEvaluator_nopointers* eval_nopointers, global float* eval_ptr0, long eval_ptr0_offset, global float* eval_ptr1, long eval_ptr1_offset, int size, local int *scratch

    # what we need:
    # struct Eigen__TensorEvaluator_nopointers   Note that none of the values we copy across are actually use, so we can just create a sufficiently large buffer...
    # global float *eval_ptr0  => this will receive the result.  just create a sufficiently large buffer
    # ptr0_offset => 0
    # eval_ptr1 => will contian the data we want to reduce
    # eval_ptr1_offset=> 0
    # size =>  eg 10, to reduce 10 values
    # scratch => set to workgroupsize * sizeof(float)

    eval_nopointers_gpu = cl.Buffer(context, cl.mem_flags.READ_WRITE, size=4096)

    eval_ptr0 = np.zeros(1024, dtype=np.float32)
    eval_ptr0_gpu = cl.Buffer(context, cl.mem_flags.READ_WRITE | cl.mem_flags.COPY_HOST_PTR, hostbuf=eval_ptr0)
    eval_ptr0_offset = 0

    eval_ptr1 = np.random.uniform(0, 1, size=(1024,)).astype(np.float32) + 1.0
    eval_ptr1_gpu = cl.Buffer(context, cl.mem_flags.READ_WRITE | cl.mem_flags.COPY_HOST_PTR, hostbuf=eval_ptr1)
    eval_ptr1_offset = 0

    size = N

    global_size = 256
    workgroup_size = 256
    scratch = workgroup_size * 4

    prog.__getattr__('_ZN5Eigen8internal15EigenMetaKernelINS_15TensorEvaluatorIKNS_14TensorAssignOpINS_9TensorMapINS_6TensorIfLi1ELi1EiEELi16ENS_11MakePointerEEEKNS_18TensorCwiseUnaryOpINS0_14scalar_sqrt_opIfEEKNS4_INS5_IKfLi1ELi1EiEELi16ES7_EEEEEENS_9GpuDeviceEEEiEEvT_T0_')(
        q, (global_size,), (workgroup_size,),
        eval_nopointers_gpu,
        eval_ptr0_gpu, np.int64(eval_ptr0_offset),
        eval_ptr1_gpu, np.int64(eval_ptr1_offset),
        np.int32(size),
        cl.LocalMemory(scratch)
    )
    # check for errors
    q.finish()
    # copy eval_ptr0 back, and check the results...
    cl.enqueue_copy(q, eval_ptr0, eval_ptr0_gpu)
    q.finish()
    print('eval_ptr0[:N]', eval_ptr0[:N])

    expected = np.sqrt(eval_ptr1)
    print('expected[:10]', expected[:N])
    assert np.abs(expected[:N] - eval_ptr0[:N]).max() < 1e-4
<|endoftext|>