<commit_before>from django.dispatch import receiver
from django.db.models.signals import post_save

from api.caching.tasks import ban_url
from framework.postcommit_tasks.handlers import enqueue_postcommit_task

@receiver(post_save)
def ban_object_from_cache(sender, instance, created, **kwargs):
    if hasattr(instance, 'absolute_api_v2_url'):
        enqueue_postcommit_task(ban_url, (instance, ), {}, celery=False, once_per_request=True)
<commit_msg>Don't call ban_object_from_cache<commit_after>from api.caching.tasks import ban_url
from framework.postcommit_tasks.handlers import enqueue_postcommit_task

# unused for now
# from django.dispatch import receiver
# from django.db.models.signals import post_save
# @receiver(post_save)
def ban_object_from_cache(sender, instance, **kwargs):
    if hasattr(instance, 'absolute_api_v2_url'):
        enqueue_postcommit_task(ban_url, (instance, ), {}, celery=False, once_per_request=True)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

__LIBRARY_VERSION__ = '1.7.4'
__SII_VERSION__ = '1.0'
<commit_msg>Bump to v1.7.5<commit_after># -*- coding: utf-8 -*-

__LIBRARY_VERSION__ = '1.7.5'
__SII_VERSION__ = '1.0'
<|endoftext|>


<commit_before>#!/usr/bin/env python
import socket
import sys
from collections import namedtuple
from lib.frostbite_wire.packet import Packet
import argparse
import urllib
import json
import os
import socket
import sys
import time
from django.template import Template, Context
from django.conf import settings
from django.utils.datastructures import SortedDict
settings.configure() # We have to do this to use django templates standalone - see
# http://stackoverflow.com/questions/98135/how-do-i-use-django-templates-without-the-rest-of-django

# Our template. Could just as easily be stored in a separate file
template = """

<meta http-equiv="refresh" content="{{refresh}}" >
{{player_count}} player(s) on {{current_map}} {{current_mode}}.

        Player
        Cheat Score

    {% for key, value in player_data.items %}

        {{key}}
        {% if value.cheatscore < 10 or value.cheatscore == None %}
            {{value.cheatscore}}
        {% else %}
            {{value.cheatscore}}
        {% endif %}

    {% endfor %}

Last updated at {{update_time}} UTC.
"""

# Mapping engine map names to human-readable names
map_names = {'MP_Abandoned': 'Zavod 311',
             'MP_Damage': 'Lancang Dam',
             'MP_Flooded': 'Flood Zone',
             'MP_Journey': 'Golmud Railway',
             'MP_Naval': 'Paracel Storm',
             'MP_Prison': 'Operation Locker',
             'MP_Resort': 'Hainan Resort',
             'MP_Siege': 'Siege of Shanghai',
             'MP_TheDish': 'Rogue Transmission',
             'MP_Tremors': 'Dawnbreaker',
             'XP1_001': 'Silk Road',
             'XP1_002': 'Altai Range',
             'XP1_003': 'Guilin Peaks',
             'XP1_004': 'Dragon Pass',
             'XP0_Caspian': 'Caspian Border',
             'XP0_Firestorm': 'Operation Firestorm',
             'XP0_Metro': 'Operation Metro',
             'XP0_Oman': 'Gulf of Oman',
             'XP2_001': 'Lost Islands',
             'XP2_002': 'Nansha strike',
             'XP2_003': 'WaveBreaker',
             'XP2_004': 'Operation Mortar',
             'XP3_MarketPl': 'Pearl Market',
             'XP3_Prpganda': 'Propaganda',
             'XP3_UrbanGdn': 'Lumphini Garden',
             'XP3_WtrFront': 'Sunken Dragon'}

# Mapping engine map modes to human-readable names
game_modes = {'AirSuperiority0': 'Air Superiority',
              'CaptureTheFlag0': 'Capture the Flag',
              'CarrierAssaultSmall0': 'Carrier Assault',
              'CarrierAssaultLarge0': 'Carrier Assault Large',
              'Chainlink0': 'Chain Link',
              'ConquestSmall0': 'Conquest Small',
              'ConquestLarge0': 'Conquest Large',
              'Elimination0': 'Defuse',
              'Domination0': 'Domination',
              'Obliteration': 'Obliteration',
              'RushLarge0': 'Rush',
              'SquadDeathMatch0': 'Squad DM',
              'TeamDeathMatch0': 'Team DM'}

# Generic way to write our files
def write_file(filename, text):
    with open(filename, 'w') as f:
        f.write(text)

# http://stackoverflow.com/questions/788411/check-to-see-if-python-script-is-running
class ProcessLock:
    def __init__(self):
        self.lock_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
    def get_lock(self, process_name):
        try:
            self.lock_socket.bind('\0' + process_name)
        except socket.error:
            print 'already running.  exiting.'
            sys.exit()

def json_query(json_url):
    retry_limit = range(1,6)
    for x in retry_limit:
        try:
            result_json = json.load(urllib.urlopen(json_url))
            return result_json
        except:
            print 'query failed - URL was ' + json_url
            print 'attempting retry ' + str(x) + ' of ' + str(retry_limit[-1])
            time.sleep(1)
            if x >= retry_limit[-1]:
                print 'giving up.  exiting.'
                sys.exit(1)
            else:
                continue

def write_template(player_count, current_map, current_mode, player_data):
    write_file(os.path.join(cmdline.file_dir + '/player_count.html'), player_count)
    update_time = time.strftime('%H:%M:%S %m/%d/%Y')
    t = Template(template)
    c = Context({"player_count": player_count,
                 "current_map": current_map,
                 "current_mode": current_mode,
                 "refresh": refresh,
                 "update_time": update_time,
                 "player_data": player_data})
    write_file(os.path.join(cmdline.file_dir + '/index.html'), t.render(c))

def server_status(address, server_port=None):
    def recv(sock):
        # Pull enough to get the int headers and instantiate a Packet
        out = sock.recv(12)
        p = Packet.from_buffer(out)
        packet_size = len(p)
        # Pull one character at a time until we've recv'd
        # up to the reported size
        while len(out) < packet_size:
            out += sock.recv(1)
        return out

    try:
        port = int(server_port)
    except TypeError:
        port = 47200

    server = address, port

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect(server)

    serverinfo = Packet(1, False, True, 'serverinfo')
    sock.sendall(serverinfo.to_buffer())

    response = Packet.from_buffer(recv(sock))
    serverinfo = response.words

    listplayers = Packet(2, False, True, 'listPlayers all')
    sock.sendall(listplayers.to_buffer())
    response = Packet.from_buffer(recv(sock))
    listplayers = response.words

    sock.close()

    # Need both of these to go on
    assert serverinfo[0] == 'OK'
    assert listplayers[0] == 'OK'

    # Chomp on the listplayers output and loop out some namedtuples
    num_fields, the_rest = int(listplayers[1]), listplayers[2:]
    fields, num_players, players = (
        the_rest[:num_fields],
        the_rest[num_fields],
        the_rest[num_fields + 1:]
    )

    Player = namedtuple('Player', fields)

    player_list = list()
    while players:
        player_list.append(Player(*players[:num_fields]))
        players = players[num_fields:]

    little_player_list = list()
    for x in player_list:
        little_player_list.append(x[0])
        if cmdline.debug and len(little_player_list) > 1:
            break

    # Print out pretty server name/players
    if cmdline.debug:
        print 'server name: ' + serverinfo[1]
        print 'players : ' + serverinfo[2]
        print 'maxplayers : ' + serverinfo[3]
        print 'mode : ' + serverinfo[4]
        print 'map : ' + serverinfo[5]

    player_count = serverinfo[2] + '/' + serverinfo[3]
    current_map = map_names[serverinfo[5]]
    current_mode = game_modes[serverinfo[4]]
    return little_player_list, player_count, current_map, current_mode

def bf4db_query(player_list):
    player_dict = SortedDict()
    for x in sorted(player_list, key=lambda s: s.lower()):
        time.sleep(0.5)
        try:
            bf4db_json = json_query(bf4db_url + x)
            player_dict[x] = bf4db_json['data']
        except ValueError:
            player_dict[x] = None
        if cmdline.debug:
            print x + ' ' + str(player_dict[x]['cheatscore'])
    return player_dict

class CommandLine:
    def __init__(self):
        self.debug = False
        self.address = ''
        self.server_port = None
        self.file_dir = ''

    def cmdline(self):
        parser = argparse.ArgumentParser(description='Status web page for your BF4 server.')
        parser.add_argument('-d', help='show debug info in terminal',
                            action="store_true")
        parser.add_argument('address', help='Server hostname or IP address')
        parser.add_argument('-p', '--port', type=int, help='Server port number')
        parser.add_argument('file_dir', help='Path to generated HTML file(s)')
        args = parser.parse_args()
        self.address = args.address
        self.file_dir = args.file_dir
        if args.d:
            self.debug = True
        else:
            self.debug = False
        if args.port:
            self.server_port = args.port
        else:
            self.server_port = None

cmdline = CommandLine()
cmdline.cmdline()
process_lock = ProcessLock()
process_lock.get_lock('bf4_server_status.py')
refresh = 60
bf4db_url = 'http://api.bf4db.com/api-player.php?name='
players = server_status(cmdline.address, cmdline.server_port)
player_data = bf4db_query(players[0])
write_template(players[1], players[2], players[3], player_data)
<commit_msg>pep8 cleanup<commit_after>#!/usr/bin/env python
import socket
import sys
from collections import namedtuple
from lib.frostbite_wire.packet import Packet
import argparse
import urllib
import json
import os
import socket
import sys
import time
from django.template import Template, Context
from django.conf import settings
from django.utils.datastructures import SortedDict
# We have to do this to use django templates standalone - see
# http://stackoverflow.com/questions/98135/how-do-i-use-django-templates-without-the-rest-of-django
settings.configure()

# Our template. Could just as easily be stored in a separate file
template = """

<meta http-equiv="refresh" content="{{refresh}}" >
{{player_count}} player(s) on {{current_map}} {{current_mode}}.

        Player
        Cheat Score

    {% for key, value in player_data.items %}

        {{key}}
        {% if value.cheatscore < 10 or value.cheatscore == None %}
            {{value.cheatscore}}
        {% else %}
            {{value.cheatscore}}
        {% endif %}

    {% endfor %}

Last updated at {{update_time}} UTC.
"""

# Mapping engine map names to human-readable names
map_names = {'MP_Abandoned': 'Zavod 311',
             'MP_Damage': 'Lancang Dam',
             'MP_Flooded': 'Flood Zone',
             'MP_Journey': 'Golmud Railway',
             'MP_Naval': 'Paracel Storm',
             'MP_Prison': 'Operation Locker',
             'MP_Resort': 'Hainan Resort',
             'MP_Siege': 'Siege of Shanghai',
             'MP_TheDish': 'Rogue Transmission',
             'MP_Tremors': 'Dawnbreaker',
             'XP1_001': 'Silk Road',
             'XP1_002': 'Altai Range',
             'XP1_003': 'Guilin Peaks',
             'XP1_004': 'Dragon Pass',
             'XP0_Caspian': 'Caspian Border',
             'XP0_Firestorm': 'Operation Firestorm',
             'XP0_Metro': 'Operation Metro',
             'XP0_Oman': 'Gulf of Oman',
             'XP2_001': 'Lost Islands',
             'XP2_002': 'Nansha strike',
             'XP2_003': 'WaveBreaker',
             'XP2_004': 'Operation Mortar',
             'XP3_MarketPl': 'Pearl Market',
             'XP3_Prpganda': 'Propaganda',
             'XP3_UrbanGdn': 'Lumphini Garden',
             'XP3_WtrFront': 'Sunken Dragon'}

# Mapping engine map modes to human-readable names
game_modes = {'AirSuperiority0': 'Air Superiority',
              'CaptureTheFlag0': 'Capture the Flag',
              'CarrierAssaultSmall0': 'Carrier Assault',
              'CarrierAssaultLarge0': 'Carrier Assault Large',
              'Chainlink0': 'Chain Link',
              'ConquestSmall0': 'Conquest Small',
              'ConquestLarge0': 'Conquest Large',
              'Elimination0': 'Defuse',
              'Domination0': 'Domination',
              'Obliteration': 'Obliteration',
              'RushLarge0': 'Rush',
              'SquadDeathMatch0': 'Squad DM',
              'TeamDeathMatch0': 'Team DM'}

# Generic way to write our files
def write_file(filename, text):
    with open(filename, 'w') as f:
        f.write(text)

# http://stackoverflow.com/questions/788411/check-to-see-if-python-script-is-running
class ProcessLock:
    def __init__(self):
        self.lock_socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)

    def get_lock(self, process_name):
        try:
            self.lock_socket.bind('\0' + process_name)
        except socket.error:
            print 'already running.  exiting.'
            sys.exit()

def json_query(json_url):
    retry_limit = range(1, 6)
    for x in retry_limit:
        try:
            result_json = json.load(urllib.urlopen(json_url))
            return result_json
        except:
            print 'query failed - URL was ' + json_url
            print 'attempting retry ' + str(x) + ' of ' + str(retry_limit[-1])
            time.sleep(1)
            if x >= retry_limit[-1]:
                print 'giving up.  exiting.'
                sys.exit(1)
            else:
                continue

def write_template(player_count, current_map, current_mode, player_data):
    write_file(os.path.join(cmdline.file_dir + '/player_count.html'), player_count)
    update_time = time.strftime('%H:%M:%S %m/%d/%Y')
    t = Template(template)
    c = Context({"player_count": player_count,
                 "current_map": current_map,
                 "current_mode": current_mode,
                 "refresh": refresh,
                 "update_time": update_time,
                 "player_data": player_data})
    write_file(os.path.join(cmdline.file_dir + '/index.html'), t.render(c))

def server_status(address, server_port=None):
    def recv(sock):
        # Pull enough to get the int headers and instantiate a Packet
        out = sock.recv(12)
        p = Packet.from_buffer(out)
        packet_size = len(p)
        # Pull one character at a time until we've recv'd
        # up to the reported size
        while len(out) < packet_size:
            out += sock.recv(1)
        return out

    try:
        port = int(server_port)
    except TypeError:
        port = 47200

    server = address, port

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect(server)

    serverinfo = Packet(1, False, True, 'serverinfo')
    sock.sendall(serverinfo.to_buffer())

    response = Packet.from_buffer(recv(sock))
    serverinfo = response.words

    listplayers = Packet(2, False, True, 'listPlayers all')
    sock.sendall(listplayers.to_buffer())
    response = Packet.from_buffer(recv(sock))
    listplayers = response.words

    sock.close()

    # Need both of these to go on
    assert serverinfo[0] == 'OK'
    assert listplayers[0] == 'OK'

    # Chomp on the listplayers output and loop out some namedtuples
    num_fields, the_rest = int(listplayers[1]), listplayers[2:]
    fields, num_players, players = (
        the_rest[:num_fields],
        the_rest[num_fields],
        the_rest[num_fields + 1:]
    )

    Player = namedtuple('Player', fields)

    player_list = list()
    while players:
        player_list.append(Player(*players[:num_fields]))
        players = players[num_fields:]

    little_player_list = list()
    for x in player_list:
        little_player_list.append(x[0])
        if cmdline.debug and len(little_player_list) > 1:
            break

    # Print out pretty server name/players
    if cmdline.debug:
        print 'server name: ' + serverinfo[1]
        print 'players : ' + serverinfo[2]
        print 'maxplayers : ' + serverinfo[3]
        print 'mode : ' + serverinfo[4]
        print 'map : ' + serverinfo[5]

    player_count = serverinfo[2] + '/' + serverinfo[3]
    current_map = map_names[serverinfo[5]]
    current_mode = game_modes[serverinfo[4]]
    return little_player_list, player_count, current_map, current_mode

def bf4db_query(player_list):
    player_dict = SortedDict()
    for x in sorted(player_list, key=lambda s: s.lower()):
        time.sleep(0.5)
        try:
            bf4db_json = json_query(bf4db_url + x)
            player_dict[x] = bf4db_json['data']
        except ValueError:
            player_dict[x] = None
        if cmdline.debug:
            print x + ' ' + str(player_dict[x]['cheatscore'])
    return player_dict

class CommandLine:
    def __init__(self):
        self.debug = False
        self.address = ''
        self.server_port = None
        self.file_dir = ''

    def cmdline(self):
        parser = argparse.ArgumentParser(description='Status web page for your BF4 server.')
        parser.add_argument('-d', help='show debug info in terminal',
                            action="store_true")
        parser.add_argument('address', help='Server hostname or IP address')
        parser.add_argument('-p', '--port', type=int, help='Server port number')
        parser.add_argument('file_dir', help='Path to generated HTML file(s)')
        args = parser.parse_args()
        self.address = args.address
        self.file_dir = args.file_dir
        if args.d:
            self.debug = True
        else:
            self.debug = False
        if args.port:
            self.server_port = args.port
        else:
            self.server_port = None

cmdline = CommandLine()
cmdline.cmdline()
process_lock = ProcessLock()
process_lock.get_lock('bf4_server_status.py')
refresh = 60
bf4db_url = 'http://api.bf4db.com/api-player.php?name='
players = server_status(cmdline.address, cmdline.server_port)
player_data = bf4db_query(players[0])
write_template(players[1], players[2], players[3], player_data)
<|endoftext|>


<commit_before># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import absolute_import, unicode_literals

from ._null_logger import NullLogger

MODULE_NAME = "dataproperty"

try:
    from loguru import logger

    logger.disable(MODULE_NAME)
    LOGURU_INSTALLED = True
except ImportError:
    logger = NullLogger()
    LOGURU_INSTALLED = False

def set_logger(is_enable):
    if is_enable:
        logger.enable(MODULE_NAME)
    else:
        logger.disable(MODULE_NAME)

def set_log_level(log_level):
    """
    Set logging level of this module. The module using
    `logbook <https://logbook.readthedocs.io/en/stable/>`__ module for logging.

    :param int log_level:
        One of the log level of the
        `logbook <https://logbook.readthedocs.io/en/stable/api/base.html>`__.
        Disabled logging if the ``log_level`` is ``logbook.NOTSET``.
    :raises LookupError: If ``log_level`` is an invalid value.
    """

    # deprecated
    return
<commit_msg>Remove an unsed docstring<commit_after># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import absolute_import, unicode_literals

from ._null_logger import NullLogger

MODULE_NAME = "dataproperty"

try:
    from loguru import logger

    logger.disable(MODULE_NAME)
    LOGURU_INSTALLED = True
except ImportError:
    logger = NullLogger()
    LOGURU_INSTALLED = False

def set_logger(is_enable):
    if is_enable:
        logger.enable(MODULE_NAME)
    else:
        logger.disable(MODULE_NAME)

def set_log_level(log_level):
    # deprecated
    return
<|endoftext|>


<commit_before>import tensorflow as tf

"""
Designed to work with either Dataset api or feed-dict "standalone".

tf.reset_default_graph()

epochs = 1
batch_size = 32
in_placeholder = tf.placeholder(tf.float32, in_data.shape)
labels_placeholder = tf.placeholder(tf.float32, oh_labels.shape)
dataset = (tf.data.Dataset
           .from_tensor_slices((in_placeholder, labels_placeholder))
           .repeat()
           .shuffle(100)
           .batch(batch_size))

iterator = dataset.make_initializable_iterator()
inputs, targets = iterator.get_next()
iterator_init_op = iterator.initializer

test.build(inputs, targets)

with tf.Session() as sess:
    sess.run(iterator.initializer,
             feed_dict = {
            in_placeholder: in_data,
            labels_placeholder: oh_labels
        })
    sess.run(tf.global_variables_initializer())

    for i in range(epochs):
        _, loss_val, out = sess.run([test.optimizer, test.loss, test.out])
"""

# fixed architecture net
class fMRIConvNet(object):
    def __init__(self, params, n_classes, input_size, l2=True, learning_rate=0.001, keep_prob=0.5):
        self.n_classes = n_classes
        # pad up to make sure everything will be contained
        self.input_size = input_size
        self.l2 = l2
        self.learning_rate = learning_rate
        self.keep_prob = keep_prob
        self.params = params.copy()

        for p in self.params:
            # if p["kernel_size"] is None:
            #     p["kernel_size"] = calculate_fc_kernel(self.input_size, params)
            if "logit" in p["name"] and p["filters"] is None:
                p["filters"] = self.n_classes

    def _parse_param(self, p, inputs):
        # someday this will be awesome
        op = p["name"].split("_")[0]
        if op == "conv":
            return self._add_conv_relu_layer(p, inputs)
        elif op == "maxpool":
            return self._add_maxpool(p, inputs)
        elif op == "logit":
            return self._add_conv_layer(p, inputs)

    def _standalone_placeholders(self, input_size, n_classes):
        input_shape = tuple([None] + list(input_size))
        self.inputs = tf.placeholder(tf.float32, input_shape, name="inputs")
        self.targets = tf.placeholder(tf.float32, [None, n_classes], name="targets")

    def _add_conv_relu_layer(self, layer_params, inputs):
        if layer_params["kernel_size"] is None:
            layer_params["kernel_size"] = inputs.get_shape().as_list()[1:4]
        return tf.layers.conv3d(
            inputs=inputs,
            filters=layer_params["filters"],
            kernel_size=layer_params["kernel_size"],
            padding=layer_params["padding"],
            name=layer_params["name"],
            activation=tf.nn.relu
        )

    def _add_conv_layer(self, layer_params, inputs):
        return tf.layers.conv3d(
            inputs=inputs,
            filters=layer_params["filters"],
            kernel_size=layer_params["kernel_size"],
            padding=layer_params["padding"],
            name=layer_params["name"],
            activation=None
        )

    def _add_maxpool(self, layer_params, inputs):
        return tf.layers.max_pooling3d(
            inputs=inputs,
            pool_size=layer_params["pool_size"],
            name=layer_params["name"],
            strides=layer_params["strides"],
            padding=layer_params["padding"]
        )

    def _build_loss(self, logits, targets):
        loss = tf.reduce_mean(
            tf.nn.softmax_cross_entropy_with_logits_v2(
                logits=logits,
                labels=targets
            )
        )
        return loss

    def _build_optimizer(self, loss, learning_rate):
        train_op = tf.train.AdamOptimizer(learning_rate)
        optimizer = train_op.minimize(loss)
        return optimizer

    def _build(self, inputs, targets):
        if inputs is None and targets is None:
            self._standalone_placeholders(self.input_size, self.n_classes)
        else:
            self.inputs, self.targets = inputs, targets

        # if train_mode
        # for p in self.params:
        #     self._parse_param(p)
        a = [tf.expand_dims(self.inputs, -1)]
        for i, layer_params in enumerate(self.params):
            a.append(self._parse_param(layer_params, a[i]))

        logits = tf.squeeze(a[-1])
        self.out = tf.nn.softmax(logits, name="predictions")

        self.loss = self._build_loss(logits, self.targets)
        if self.l2:
            self.loss += tf.losses.get_regularization_loss()
        self.optimizer = self._build_optimizer(self.loss, self.learning_rate)

    def build(self, inputs=None, targets=None, train_mode=True):
        self._build(inputs, targets)

    def train(self, sess, x, y):
        pass

    def eval(self, sess, x, y):
        pass

    def predict(self):
        pass

<commit_msg>update saving<commit_after>import tensorflow as tf
import os
import logging
import json
"""
Designed to work with either Dataset api or feed-dict "standalone".

tf.reset_default_graph()

epochs = 1
batch_size = 32
in_placeholder = tf.placeholder(tf.float32, in_data.shape)
labels_placeholder = tf.placeholder(tf.float32, oh_labels.shape)
dataset = (tf.data.Dataset
           .from_tensor_slices((in_placeholder, labels_placeholder))
           .repeat()
           .shuffle(100)
           .batch(batch_size))

iterator = dataset.make_initializable_iterator()
inputs, targets = iterator.get_next()
iterator_init_op = iterator.initializer

test.build(inputs, targets)

with tf.Session() as sess:
    sess.run(iterator.initializer,
             feed_dict = {
            in_placeholder: in_data,
            labels_placeholder: oh_labels
        })
    sess.run(tf.global_variables_initializer())

    for i in range(epochs):
        _, loss_val, out = sess.run([test.optimizer, test.loss, test.out])
"""

# fixed architecture net
class fMRIConvNet(object):
    def __init__(self, params, n_classes, input_size, l2=True, name="cnn-model", save_path="./model", learning_rate=0.001):
        self.name = name
        self.save_path = save_path
        self.n_classes = n_classes
        # pad up to make sure everything will be contained
        self.input_size = input_size
        self.l2 = l2
        self.learning_rate = learning_rate
        self.params = params.copy()
        self.is_built = False
        for p in self.params:
            # if p["kernel_size"] is None:
            #     p["kernel_size"] = calculate_fc_kernel(self.input_size, params)
            if "logit" in p["name"] and p["filters"] is None:
                p["filters"] = self.n_classes

    def _parse_param(self, p, inputs):
        # someday this will be awesome
        op = p["name"].split("_")[0]
        if op == "conv":
            return self._add_conv_relu_layer(p, inputs)
        elif op == "maxpool":
            return self._add_maxpool(p, inputs)
        elif op == "logit":
            return self._add_conv_layer(p, inputs)
        elif op == "drop":
            return self._add_dropout(p, inputs)

    def _add_dropout(self, layer_params, inputs):
        return tf.layers.dropout(
            inputs=inputs,
            rate=layer_params["rate"],
            name=layer_params["name"]
        )

    def _standalone_placeholders(self, input_size, n_classes):
        input_shape = tuple([None] + list(input_size))
        self.inputs = tf.placeholder(tf.float32, input_shape, name="inputs")
        self.targets = tf.placeholder(tf.float32, [None, n_classes], name="targets")

    def _add_conv_relu_layer(self, layer_params, inputs):
        if layer_params["kernel_size"] is None:
            layer_params["kernel_size"] = inputs.get_shape().as_list()[1:4]
        return tf.layers.conv3d(
            inputs=inputs,
            filters=layer_params["filters"],
            kernel_size=layer_params["kernel_size"],
            padding=layer_params["padding"],
            name=layer_params["name"],
            activation=tf.nn.relu
        )

    def _add_conv_layer(self, layer_params, inputs):
        return tf.layers.conv3d(
            inputs=inputs,
            filters=layer_params["filters"],
            kernel_size=layer_params["kernel_size"],
            padding=layer_params["padding"],
            name=layer_params["name"],
            activation=None
        )

    def _add_maxpool(self, layer_params, inputs):
        return tf.layers.max_pooling3d(
            inputs=inputs,
            pool_size=layer_params["pool_size"],
            name=layer_params["name"],
            strides=layer_params["strides"],
            padding=layer_params["padding"]
        )

    def _build_loss(self, logits, targets):
        loss = tf.reduce_mean(
            tf.nn.softmax_cross_entropy_with_logits_v2(
                logits=logits,
                labels=targets
            )
        )
        return loss

    def _compute_accuracy(self, pred, label):
        correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(label, 1))
        return tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    def _build_optimizer(self, loss, learning_rate):
        train_op = tf.train.AdamOptimizer(learning_rate)
        optimizer = train_op.minimize(loss)
        return optimizer

    def _build(self, inputs, targets):
        if self.is_built is False:
            if inputs is None and targets is None:
                self._standalone_placeholders(self.input_size, self.n_classes)
            else:
                self.inputs, self.targets = inputs, targets

            # if train_mode
            # for p in self.params:
            #     self._parse_param(p)
            a = [tf.expand_dims(self.inputs, -1)]
            for i, layer_params in enumerate(self.params):
                a.append(self._parse_param(layer_params, a[i]))

            logits = tf.squeeze(a[-1])
            self.out = tf.nn.softmax(logits, name="predictions")
            self.loss = self._build_loss(logits, self.targets)
            self.accuracy = self._compute_accuracy(self.out, self.targets)
            if self.l2:
                self.loss += tf.losses.get_regularization_loss()
            self.optimizer = self._build_optimizer(self.loss, self.learning_rate)
            self.is_built = True
        else:
            pass

    def build(self, inputs=None, targets=None, rebuild=False):
        if rebuild:
            self.is_built = False
        self._build(inputs, targets)

    def save(self, sess, saver, **params):
        saver.save(sess, os.path.join(self.save_path, self.name), **params)
        with open(os.path.join(self.save_path, "{}.json".format(self.name)), "w") as out:
            json.dump(self.params, out)

    def load(self, sess, saver):
        restore_from = tf.train.latest_checkpoint(self.save_path)
        if restore_from is not None:
            saver.restore(sess, restore_from)
        else:
            pass

    def train(self, sess, x, y):
        pass

    def eval(self, sess, x, y):
        return sess.run(self.accuracy)

    def predict(self, sess, x, y):
        return sess.run(self.out)

<|endoftext|>


<commit_before>from django.conf import settings
from django.contrib.sites.models import Site
from django.template.loader import render_to_string
from django.core.mail import EmailMultiAlternatives, EmailMessage

from django.contrib.auth.models import User

def get_manager_emails():
    """
    Get a list of the managers email addresses.
    """
    addresses = User.objects.filter(is_staff=True).exclude(email='').distinct().values_list('email')
    manager_emails = [m[0] for m in addresses]
    if not manager_emails:
        manager_emails = [m[1] for m in settings.MANAGERS]

    return manager_emails

def send(sender=None, to=(), cc=(), bcc=(), subject='mail',
         attachments=(), template_name=(), text_template_name=(),
         extra_context=None, **kwargs):
    """
    Render and send a (mail) template.
    if text_template_name is not None then a multipart email will be sent using
    template for the html part and text_template_name for the plain part.
    The context will include the current site (and any extra_context specified).
    If no sender is specified then the DEFAULT_FROM_EMAIL or SERVER_EMAIL setting will be used.
    Any extra items passed in with kwargs will be added to the email headers.
    """
    current_site = Site.objects.get_current()

    if sender is None:
        sender = hasattr(settings, 'DEFAULT_FROM_EMAIL') and settings.DEFAULT_FROM_EMAIL or settings.SERVER_EMAIL

    subject = unicode(subject)

    context = {'site': current_site}

    if extra_context is not None:
        context.update(extra_context)

    attachment_list = [[a.name, a.read(), a.content_type] for a in attachments if attachments]

    email_kwargs = {
        'from_email': sender,
        'to': to,
        'cc': cc,
        'bcc': bcc,
        'subject': subject,
        'attachments': attachment_list,
        'headers': kwargs,
    }
    if not text_template_name:
        email_kwargs['body'] = render_to_string(template_name, context)
        msg = EmailMessage(**email_kwargs)
    else:
        email_kwargs['body'] = render_to_string(text_template_name, context)
        msg = EmailMultiAlternatives(**email_kwargs)

        html_content = render_to_string(template_name, context)
        msg.attach_alternative(html_content, 'text/html')

    msg.send()
<commit_msg>Convert to, cc, bcc to lists if not already<commit_after>from django.conf import settings
from django.contrib.sites.models import Site
from django.template.loader import render_to_string
from django.core.mail import EmailMultiAlternatives, EmailMessage

from django.contrib.auth.models import User

def get_manager_emails():
    """
    Get a list of the managers email addresses.
    """
    addresses = User.objects.filter(is_staff=True).exclude(email='').distinct().values_list('email')
    manager_emails = [m[0] for m in addresses]
    if not manager_emails:
        manager_emails = [m[1] for m in settings.MANAGERS]

    return manager_emails

def send(sender=None, to=(), cc=(), bcc=(), subject='mail',
         attachments=(), template_name=(), text_template_name=(),
         extra_context=None, **kwargs):
    """
    Render and send a (mail) template.
    if text_template_name is not None then a multipart email will be sent using
    template for the html part and text_template_name for the plain part.
    The context will include the current site (and any extra_context specified).
    If no sender is specified then the DEFAULT_FROM_EMAIL or SERVER_EMAIL setting will be used.
    Any extra items passed in with kwargs will be added to the email headers.
    """
    current_site = Site.objects.get_current()

    if isinstance(to, basestring):
        to = [to]

    if isinstance(cc, basestring):
        cc = [cc]

    if isinstance(bcc, basestring):
        bcc = [bcc]

    if sender is None:
        sender = hasattr(settings, 'DEFAULT_FROM_EMAIL') and settings.DEFAULT_FROM_EMAIL or settings.SERVER_EMAIL

    subject = unicode(subject)

    context = {'site': current_site}

    if extra_context is not None:
        context.update(extra_context)

    attachment_list = [[a.name, a.read(), a.content_type] for a in attachments if attachments]

    email_kwargs = {
        'from_email': sender,
        'to': to,
        'cc': cc,
        'bcc': bcc,
        'subject': subject,
        'attachments': attachment_list,
        'headers': kwargs,
    }
    if not text_template_name:
        email_kwargs['body'] = render_to_string(template_name, context)
        msg = EmailMessage(**email_kwargs)
    else:
        email_kwargs['body'] = render_to_string(text_template_name, context)
        msg = EmailMultiAlternatives(**email_kwargs)

        html_content = render_to_string(template_name, context)
        msg.attach_alternative(html_content, 'text/html')

    msg.send()
<|endoftext|>


<commit_before># coding=utf-8
import re
from unidecode import unidecode

from sii import __SII_VERSION__
from sii.models import invoices_record

SIGN = {'B': -1, 'A': -1, 'N': 1, 'R': 1}

def get_iva_values(invoice, in_invoice):
    vals = {
        'sujeta_a_iva': False,
        'detalle_iva': [],
        'no_sujeta_a_iva': False,
        'iva_exento': False,
        'iva_no_exento': False,
        'detalle_iva_exento': {'BaseImponible': 0}
    }

    invoice_total = invoice.amount_total

    for inv_tax in invoice.tax_line:
        if 'iva' in inv_tax.name.lower():
            vals['sujeta_a_iva'] = True

            invoice_total -= (abs(inv_tax.tax_amount) + abs(inv_tax.base))

            if inv_tax.tax_id.amount == 0 and inv_tax.tax_id.type == 'percent':
                vals['iva_exento'] = True
                vals['detalle_iva_exento']['BaseImponible'] += inv_tax.base
            else:
                sign = SIGN[invoice.rectificative_type]
                iva = {
                    'BaseImponible': sign * abs(inv_tax.base),
                    'TipoImpositivo': inv_tax.tax_id.amount * 100
                }
                if in_invoice:
                    iva['CuotaSoportada'] = sign * abs(inv_tax.tax_amount)
                else:
                    iva['CuotaRepercutida'] = sign * abs(inv_tax.tax_amount)
                vals['iva_no_exento'] = True
                vals['detalle_iva'].append(iva)

    invoice_total = round(invoice_total, 2)
    if invoice_total > 0:
        vals['no_sujeta_a_iva'] = True
        vals['importe_no_sujeto'] = invoice_total

    return vals

def get_contraparte(partner, in_invoice):
    vat_type = partner.sii_get_vat_type()
    contraparte = {'NombreRazon': partner.name}

    if not partner.aeat_registered and not in_invoice:
        contraparte['IDOtro'] = {
            'CodigoPais': partner.country.code,
            'IDType': '07',
            'ID': partner.vat
        }
    else:
        if vat_type == '02':
            contraparte['NIF'] = partner.vat
        else:
            contraparte['IDOtro'] = {
                'CodigoPais': partner.country.code,
                'IDType': vat_type,
                'ID': partner.vat
            }

    return contraparte

def get_factura_emitida(invoice):
    iva_values = get_iva_values(invoice, in_invoice=False)
    desglose_factura = {}

    if iva_values['sujeta_a_iva']:
        desglose_factura['Sujeta'] = {}
        if iva_values['iva_exento']:
            desglose_factura['Sujeta']['Exenta'] = \
                iva_values['detalle_iva_exento']
        if iva_values['iva_no_exento']:
            desglose_factura['Sujeta']['NoExenta'] = {
                'TipoNoExenta': 'S1',
                'DesgloseIVA': {
                    'DetalleIVA': iva_values['detalle_iva']
                }
            }
    if iva_values['no_sujeta_a_iva']:
        importe_no_sujeto = iva_values['importe_no_sujeto']

        fp = invoice.fiscal_position
        if fp and 'islas canarias' in unidecode(fp.name.lower()):
            desglose_factura['NoSujeta'] = {
                'ImporteTAIReglasLocalizacion': importe_no_sujeto
            }
        else:
            desglose_factura['NoSujeta'] = {
                'ImportePorArticulos7_14_Otros': importe_no_sujeto
            }

    factura_expedida = {
        'TipoFactura': 'R4' if invoice.rectificative_type == 'R' else 'F1',
        'ClaveRegimenEspecialOTrascendencia':
            invoice.sii_out_clave_regimen_especial,
        'ImporteTotal': SIGN[invoice.rectificative_type] * invoice.amount_total,
        'DescripcionOperacion': invoice.journal_id.name,
        'Contraparte': get_contraparte(invoice.partner_id, in_invoice=False),
        'TipoDesglose': {
            'DesgloseFactura': desglose_factura
        }
    }

    return factura_expedida

def get_factura_recibida(invoice):
    iva_values = get_iva_values(invoice, in_invoice=True)
    cuota_deducible = 0

    if iva_values['sujeta_a_iva'] and iva_values['iva_no_exento']:
        desglose_factura = {  # TODO to change
            # 'InversionSujetoPasivo': {
            #     'DetalleIVA': iva_values['detalle_iva']
            # },
            'DesgloseIVA': {
                'DetalleIVA': iva_values['detalle_iva']
            }
        }

        for detalle_iva in iva_values['detalle_iva']:
            cuota_deducible += detalle_iva['CuotaSoportada']
    else:
        desglose_factura = {
            'DesgloseIVA': {
                'DetalleIVA': [{
                    'BaseImponible': 0  # TODO deixem de moment 0 perquè no tindrem inversio sujeto pasivo
                }]
            }
        }

    factura_recibida = {
        'TipoFactura': 'R4' if invoice.rectificative_type == 'R' else 'F1',
        'ClaveRegimenEspecialOTrascendencia':
            invoice.sii_in_clave_regimen_especial,
        'ImporteTotal': SIGN[invoice.rectificative_type] * invoice.amount_total,
        'DescripcionOperacion': invoice.journal_id.name,
        'Contraparte': get_contraparte(invoice.partner_id, in_invoice=True),
        'DesgloseFactura': desglose_factura,
        'CuotaDeducible': cuota_deducible,
        'FechaRegContable': invoice.date_invoice
    }

    return factura_recibida

def get_header(invoice):
    cabecera = {
        'IDVersionSii': __SII_VERSION__,
        'Titular': {
            'NombreRazon': invoice.company_id.partner_id.name,
            'NIF': invoice.company_id.partner_id.vat
        },
        'TipoComunicacion': 'A0' if not invoice.sii_registered else 'A1'
    }

    return cabecera

def get_factura_rectificativa_fields():
    rectificativa_fields = {
        'TipoRectificativa': 'S',  # Por sustitución
        'ImporteRectificacion': {
            'BaseRectificada': 0,
            'CuotaRectificada': 0
        }
    }

    return rectificativa_fields

def get_factura_emitida_dict(invoice, rectificativa=False):
    obj = {
        'SuministroLRFacturasEmitidas': {
            'Cabecera': get_header(invoice),
            'RegistroLRFacturasEmitidas': {
                'PeriodoImpositivo': {
                    'Ejercicio': invoice.period_id.name[3:7],
                    'Periodo': invoice.period_id.name[0:2]
                },
                'IDFactura': {
                    'IDEmisorFactura': {
                        'NIF': invoice.company_id.partner_id.vat
                    },
                    'NumSerieFacturaEmisor': invoice.number,
                    'FechaExpedicionFacturaEmisor': invoice.date_invoice
                },
                'FacturaExpedida': get_factura_emitida(invoice)
            }
        }
    }

    if rectificativa:
        vals = get_factura_rectificativa_fields()

        (
            obj['SuministroLRFacturasEmitidas']['RegistroLRFacturasEmitidas']
            ['FacturaExpedida']
        ).update(vals)

    return obj

def get_factura_recibida_dict(invoice, rectificativa=False):
    obj = {
        'SuministroLRFacturasRecibidas': {
            'Cabecera': get_header(invoice),
            'RegistroLRFacturasRecibidas': {
                'PeriodoImpositivo': {
                    'Ejercicio': invoice.period_id.name[3:7],
                    'Periodo': invoice.period_id.name[0:2]
                },
                'IDFactura': {
                    'IDEmisorFactura': {
                        'NIF': invoice.partner_id.vat
                    },
                    'NumSerieFacturaEmisor': invoice.origin,
                    'FechaExpedicionFacturaEmisor': invoice.origin_date_invoice
                },
                'FacturaRecibida': get_factura_recibida(invoice)
            }
        }
    }

    if rectificativa:
        vals = get_factura_rectificativa_fields()

        (
            obj['SuministroLRFacturasRecibidas']['RegistroLRFacturasRecibidas']
            ['FacturaRecibida']
        ).update(vals)

    return obj

def refactor_nifs(invoice):
    for partner in (invoice.partner_id, invoice.company_id.partner_id):
        partner.vat = re.sub('^ES', '', partner.vat.upper())

class SII(object):
    def __init__(self, invoice):
        self.invoice = invoice
        refactor_nifs(self.invoice)
        rectificativa = invoice.rectificative_type == 'R'
        if invoice.type.startswith('in'):
            self.invoice_model = invoices_record.SuministroFacturasRecibidas()
            self.invoice_dict = get_factura_recibida_dict(
                self.invoice, rectificativa=rectificativa
            )
        elif invoice.type.startswith('out'):
            self.invoice_model = invoices_record.SuministroFacturasEmitidas()
            self.invoice_dict = get_factura_emitida_dict(
                self.invoice, rectificativa=rectificativa
            )
        else:
            raise AttributeError(
                'Valor desconocido en el tipo de factura: {}'.format(
                    invoice.type
                )
            )

    def get_validation_errors_list(self, errors):
        error_messages = []

        for val in errors.values():
            if isinstance(val, dict):
                error_messages += self.get_validation_errors_list(val)
            else:
                error_messages += val

        return error_messages

    def validate_invoice(self):

        res = {}

        errors = self.invoice_model.validate(self.invoice_dict)

        res['successful'] = False if errors else True
        if errors:
            errors_list = self.get_validation_errors_list(errors)
            res['errors'] = errors_list

        return res

    def generate_object(self):

        validation_values = self.validate_invoice()
        if not validation_values['successful']:
            raise Exception(
                'Errors were found while trying to validate the data:',
                validation_values['errors']
            )

        res = self.invoice_model.dump(self.invoice_dict)
        if res.errors:
            raise Exception(
                'Errors were found while trying to generate the dump:',
                res.errors
            )

        return res.data
<commit_msg>Return key in errors list if field type doesn't validate<commit_after># coding=utf-8
import re
from unidecode import unidecode

from sii import __SII_VERSION__
from sii.models import invoices_record

SIGN = {'B': -1, 'A': -1, 'N': 1, 'R': 1}

def get_iva_values(invoice, in_invoice):
    vals = {
        'sujeta_a_iva': False,
        'detalle_iva': [],
        'no_sujeta_a_iva': False,
        'iva_exento': False,
        'iva_no_exento': False,
        'detalle_iva_exento': {'BaseImponible': 0}
    }

    invoice_total = invoice.amount_total

    for inv_tax in invoice.tax_line:
        if 'iva' in inv_tax.name.lower():
            vals['sujeta_a_iva'] = True

            invoice_total -= (abs(inv_tax.tax_amount) + abs(inv_tax.base))

            if inv_tax.tax_id.amount == 0 and inv_tax.tax_id.type == 'percent':
                vals['iva_exento'] = True
                vals['detalle_iva_exento']['BaseImponible'] += inv_tax.base
            else:
                sign = SIGN[invoice.rectificative_type]
                iva = {
                    'BaseImponible': sign * abs(inv_tax.base),
                    'TipoImpositivo': inv_tax.tax_id.amount * 100
                }
                if in_invoice:
                    iva['CuotaSoportada'] = sign * abs(inv_tax.tax_amount)
                else:
                    iva['CuotaRepercutida'] = sign * abs(inv_tax.tax_amount)
                vals['iva_no_exento'] = True
                vals['detalle_iva'].append(iva)

    invoice_total = round(invoice_total, 2)
    if invoice_total > 0:
        vals['no_sujeta_a_iva'] = True
        vals['importe_no_sujeto'] = invoice_total

    return vals

def get_contraparte(partner, in_invoice):
    vat_type = partner.sii_get_vat_type()
    contraparte = {'NombreRazon': partner.name}

    if not partner.aeat_registered and not in_invoice:
        contraparte['IDOtro'] = {
            'CodigoPais': partner.country.code,
            'IDType': '07',
            'ID': partner.vat
        }
    else:
        if vat_type == '02':
            contraparte['NIF'] = partner.vat
        else:
            contraparte['IDOtro'] = {
                'CodigoPais': partner.country.code,
                'IDType': vat_type,
                'ID': partner.vat
            }

    return contraparte

def get_factura_emitida(invoice):
    iva_values = get_iva_values(invoice, in_invoice=False)
    desglose_factura = {}

    if iva_values['sujeta_a_iva']:
        desglose_factura['Sujeta'] = {}
        if iva_values['iva_exento']:
            desglose_factura['Sujeta']['Exenta'] = \
                iva_values['detalle_iva_exento']
        if iva_values['iva_no_exento']:
            desglose_factura['Sujeta']['NoExenta'] = {
                'TipoNoExenta': 'S1',
                'DesgloseIVA': {
                    'DetalleIVA': iva_values['detalle_iva']
                }
            }
    if iva_values['no_sujeta_a_iva']:
        importe_no_sujeto = iva_values['importe_no_sujeto']

        fp = invoice.fiscal_position
        if fp and 'islas canarias' in unidecode(fp.name.lower()):
            desglose_factura['NoSujeta'] = {
                'ImporteTAIReglasLocalizacion': importe_no_sujeto
            }
        else:
            desglose_factura['NoSujeta'] = {
                'ImportePorArticulos7_14_Otros': importe_no_sujeto
            }

    factura_expedida = {
        'TipoFactura': 'R4' if invoice.rectificative_type == 'R' else 'F1',
        'ClaveRegimenEspecialOTrascendencia':
            invoice.sii_out_clave_regimen_especial,
        'ImporteTotal': SIGN[invoice.rectificative_type] * invoice.amount_total,
        'DescripcionOperacion': invoice.journal_id.name,
        'Contraparte': get_contraparte(invoice.partner_id, in_invoice=False),
        'TipoDesglose': {
            'DesgloseFactura': desglose_factura
        }
    }

    return factura_expedida

def get_factura_recibida(invoice):
    iva_values = get_iva_values(invoice, in_invoice=True)
    cuota_deducible = 0

    if iva_values['sujeta_a_iva'] and iva_values['iva_no_exento']:
        desglose_factura = {  # TODO to change
            # 'InversionSujetoPasivo': {
            #     'DetalleIVA': iva_values['detalle_iva']
            # },
            'DesgloseIVA': {
                'DetalleIVA': iva_values['detalle_iva']
            }
        }

        for detalle_iva in iva_values['detalle_iva']:
            cuota_deducible += detalle_iva['CuotaSoportada']
    else:
        desglose_factura = {
            'DesgloseIVA': {
                'DetalleIVA': [{
                    'BaseImponible': 0  # TODO deixem de moment 0 perquè no tindrem inversio sujeto pasivo
                }]
            }
        }

    factura_recibida = {
        'TipoFactura': 'R4' if invoice.rectificative_type == 'R' else 'F1',
        'ClaveRegimenEspecialOTrascendencia':
            invoice.sii_in_clave_regimen_especial,
        'ImporteTotal': SIGN[invoice.rectificative_type] * invoice.amount_total,
        'DescripcionOperacion': invoice.journal_id.name,
        'Contraparte': get_contraparte(invoice.partner_id, in_invoice=True),
        'DesgloseFactura': desglose_factura,
        'CuotaDeducible': cuota_deducible,
        'FechaRegContable': invoice.date_invoice
    }

    return factura_recibida

def get_header(invoice):
    cabecera = {
        'IDVersionSii': __SII_VERSION__,
        'Titular': {
            'NombreRazon': invoice.company_id.partner_id.name,
            'NIF': invoice.company_id.partner_id.vat
        },
        'TipoComunicacion': 'A0' if not invoice.sii_registered else 'A1'
    }

    return cabecera

def get_factura_rectificativa_fields():
    rectificativa_fields = {
        'TipoRectificativa': 'S',  # Por sustitución
        'ImporteRectificacion': {
            'BaseRectificada': 0,
            'CuotaRectificada': 0
        }
    }

    return rectificativa_fields

def get_factura_emitida_dict(invoice, rectificativa=False):
    obj = {
        'SuministroLRFacturasEmitidas': {
            'Cabecera': get_header(invoice),
            'RegistroLRFacturasEmitidas': {
                'PeriodoImpositivo': {
                    'Ejercicio': invoice.period_id.name[3:7],
                    'Periodo': invoice.period_id.name[0:2]
                },
                'IDFactura': {
                    'IDEmisorFactura': {
                        'NIF': invoice.company_id.partner_id.vat
                    },
                    'NumSerieFacturaEmisor': invoice.number,
                    'FechaExpedicionFacturaEmisor': invoice.date_invoice
                },
                'FacturaExpedida': get_factura_emitida(invoice)
            }
        }
    }

    if rectificativa:
        vals = get_factura_rectificativa_fields()

        (
            obj['SuministroLRFacturasEmitidas']['RegistroLRFacturasEmitidas']
            ['FacturaExpedida']
        ).update(vals)

    return obj

def get_factura_recibida_dict(invoice, rectificativa=False):
    obj = {
        'SuministroLRFacturasRecibidas': {
            'Cabecera': get_header(invoice),
            'RegistroLRFacturasRecibidas': {
                'PeriodoImpositivo': {
                    'Ejercicio': invoice.period_id.name[3:7],
                    'Periodo': invoice.period_id.name[0:2]
                },
                'IDFactura': {
                    'IDEmisorFactura': {
                        'NIF': invoice.partner_id.vat
                    },
                    'NumSerieFacturaEmisor': invoice.origin,
                    'FechaExpedicionFacturaEmisor': invoice.origin_date_invoice
                },
                'FacturaRecibida': get_factura_recibida(invoice)
            }
        }
    }

    if rectificativa:
        vals = get_factura_rectificativa_fields()

        (
            obj['SuministroLRFacturasRecibidas']['RegistroLRFacturasRecibidas']
            ['FacturaRecibida']
        ).update(vals)

    return obj

def refactor_nifs(invoice):
    for partner in (invoice.partner_id, invoice.company_id.partner_id):
        partner.vat = re.sub('^ES', '', partner.vat.upper())

class SII(object):
    def __init__(self, invoice):
        self.invoice = invoice
        refactor_nifs(self.invoice)
        rectificativa = invoice.rectificative_type == 'R'
        if invoice.type.startswith('in'):
            self.invoice_model = invoices_record.SuministroFacturasRecibidas()
            self.invoice_dict = get_factura_recibida_dict(
                self.invoice, rectificativa=rectificativa
            )
        elif invoice.type.startswith('out'):
            self.invoice_model = invoices_record.SuministroFacturasEmitidas()
            self.invoice_dict = get_factura_emitida_dict(
                self.invoice, rectificativa=rectificativa
            )
        else:
            raise AttributeError(
                'Valor desconocido en el tipo de factura: {}'.format(
                    invoice.type
                )
            )

    def get_validation_errors_list(self, errors):
        error_messages = []

        for key, values in errors.items():
            if isinstance(values, dict):
                error_messages += self.get_validation_errors_list(values)
            else:
                error_messages += ['{}: {}'.format(key, val) for val in values]

        return error_messages

    def validate_invoice(self):

        res = {}

        errors = self.invoice_model.validate(self.invoice_dict)

        res['successful'] = False if errors else True
        if errors:
            errors_list = self.get_validation_errors_list(errors)
            res['errors'] = errors_list

        return res

    def generate_object(self):

        validation_values = self.validate_invoice()
        if not validation_values['successful']:
            raise Exception(
                'Errors were found while trying to validate the data:',
                validation_values['errors']
            )

        res = self.invoice_model.dump(self.invoice_dict)
        if res.errors:
            raise Exception(
                'Errors were found while trying to generate the dump:',
                res.errors
            )

        return res.data
<|endoftext|>


<commit_before>import copy
import core.image_collection
import core.image_entity
import core.sequence_type
import util.database_helpers as db_help

def import_dataset_from_image_source(db_client, image_source):
    """
    Read an image source, and save it in hte database as an image collection.
    :param db_client: The connection to the database
    :param image_source: The image source to save
    :return:
    """
    image_ids = []
    if image_source.is_stored_in_database():
        # Stop early, this image source is already in the database.
        return
    image_source.begin()
    while not image_source.is_complete():
        image = image_source.get_next_image()
        image_entity = core.image_entity.image_to_entity(image)
        s_image = image_entity.serialize()
        query = db_help.query_to_dot_notation(copy.deepcopy(s_image))
        existing = db_client.image_collection.find_one(query, {'_id': True})
        if existing is None:
            image_entity.save_image_data(db_client)
            image_ids.append(db_client.image_collection.insert(s_image))
        else:
            image_ids.append(existing['_id'])
    s_collection = core.image_collection.ImageCollection.create_serialized(
        image_ids=image_ids,
        sequence_type=image_source.sequence_type
    )
    query = db_help.query_to_dot_notation(copy.deepcopy(s_collection))
    if db_client.image_source_collection.find_one(query, {'_id': True}) is None:
        db_client.image_source_collection.insert(s_collection)
<commit_msg>Change image source saving code to also create sample datasets. Allows filtering of the images in the new collection<commit_after>import copy
import core.image_collection
import core.image_entity
import core.sequence_type
import util.database_helpers as db_help

def import_dataset_from_image_source(db_client, image_source, filter_function=None):
    """
    Read an image source, and save it in the database as an image collection.
    This is used to both save datasets from simulation,
    and to sample existing datasets into new collections.

    :param db_client: The connection to the database
    :param image_source: The image source to save
    :param filter_function: A function used to filter the images that will be part of the new collection.
    :return:
    """
    image_ids = []
    image_source.begin()
    while not image_source.is_complete():
        image = image_source.get_next_image()
        if not callable(filter_function) or filter_function(image):
            if hasattr(image, 'identifier') and image.identifier is not None:
                # Image is already in the database, just store it's id
                image_ids.append(image.identifier)
            else:
                # Image is not in the database, convert it to an entity and store it.
                image_entity = core.image_entity.image_to_entity(image)
                s_image = image_entity.serialize()
                query = db_help.query_to_dot_notation(copy.deepcopy(s_image))
                existing = db_client.image_collection.find_one(query, {'_id': True})
                if existing is None:
                    image_entity.save_image_data(db_client)
                    image_ids.append(db_client.image_collection.insert(s_image))
                else:
                    # An identical image already exists, use that.
                    image_ids.append(existing['_id'])
    if len(image_ids) > 0:
        s_collection = core.image_collection.ImageCollection.create_serialized(
            image_ids=image_ids,
            sequence_type=image_source.sequence_type
        )
        query = db_help.query_to_dot_notation(copy.deepcopy(s_collection))
        if db_client.image_source_collection.find_one(query, {'_id': True}) is None:
            db_client.image_source_collection.insert(s_collection)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright (c) The Spyder Development Team
#
# Licensed under the terms of the MIT License
# (See LICENSE.txt for details)
# -----------------------------------------------------------------------------
"""Tests changelog output."""

# Standard library imports
import os

# Third party imports
import pytest

# Local imports
from loghub.main import create_changelog

REPO = 'spyder-ide/loghub'
TEST_TOKEN = os.environ.get('TEST_TOKEN', '').replace('x', '')
TEST_USER = os.environ.get('TEST_USER', '').replace('x', '')
TEST_PASS = os.environ.get('TEST_CODE', '').replace('x', '')
TEST_MILESTONE = 'test-milestone'
TEST_TAG = 'v0.1.2'
NOT_ON_CI = os.environ.get('CIRCLECI') != 'true'

# --- Tests
# -----------------------------------------------------------------------------
@pytest.mark.skipif(NOT_ON_CI, reason='test on ci server only')
def test_changelog():
    log = create_changelog(
        repo=REPO,
        token=TEST_TOKEN,
        milestone=TEST_MILESTONE,
        branch='master',
        output_format='changelog')
    expected = '''## Version <RELEASE_VERSION> (2016-12-05)

### Issues Closed

* [Issue 26](https://github.com/spyder-ide/loghub/issues/26) - Test number 2
* [Issue 24](https://github.com/spyder-ide/loghub/issues/24) - Issue test

In this release 2 issues were closed.

### Pull Requests Merged

* [PR 25](https://github.com/spyder-ide/loghub/pull/25) - PR: Add tests folder

In this release 1 pull request was closed.
'''
    print([log])
    print([expected])
    assert log == expected

@pytest.mark.skipif(NOT_ON_CI, reason='test on ci server only')
def test_changelog_release():
    log = create_changelog(
        repo=REPO,
        token=TEST_TOKEN,
        milestone=TEST_MILESTONE,
        branch='master',
        output_format='release')
    expected = '''## Version <RELEASE_VERSION> (2016-12-05)

### Issues Closed

* Issue #26 - Test number 2
* Issue #24 - Issue test

In this release 2 issues were closed.

### Pull Requests Merged

* PR #25 - PR: Add tests folder

In this release 1 pull request was closed.
'''
    print([log])
    print([expected])
    assert log == expected
<commit_msg>Add branch test<commit_after># -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright (c) The Spyder Development Team
#
# Licensed under the terms of the MIT License
# (See LICENSE.txt for details)
# -----------------------------------------------------------------------------
"""Tests changelog output."""

# Standard library imports
import os

# Third party imports
import pytest

# Local imports
from loghub.main import create_changelog

REPO = 'spyder-ide/loghub'
TEST_TOKEN = os.environ.get('TEST_TOKEN', '').replace('x', '')
TEST_USER = os.environ.get('TEST_USER', '').replace('x', '')
TEST_PASS = os.environ.get('TEST_CODE', '').replace('x', '')
TEST_MILESTONE = 'test-milestone'
TEST_TAG = 'v0.1.2'
NOT_ON_CI = os.environ.get('CIRCLECI') != 'true'

# --- Tests
# -----------------------------------------------------------------------------
@pytest.mark.skipif(NOT_ON_CI, reason='test on ci server only')
def test_changelog():
    log = create_changelog(
        repo=REPO,
        token=TEST_TOKEN,
        milestone=TEST_MILESTONE,
        branch='master',
        output_format='changelog')
    expected = '''## Version <RELEASE_VERSION> (2016-12-05)

### Issues Closed

* [Issue 26](https://github.com/spyder-ide/loghub/issues/26) - Test number 2
* [Issue 24](https://github.com/spyder-ide/loghub/issues/24) - Issue test

In this release 2 issues were closed.

### Pull Requests Merged

* [PR 25](https://github.com/spyder-ide/loghub/pull/25) - PR: Add tests folder

In this release 1 pull request was closed.
'''
    print([log])
    print([expected])
    assert log == expected

@pytest.mark.skipif(NOT_ON_CI, reason='test on ci server only')
def test_changelog_release():
    log = create_changelog(
        repo=REPO,
        token=TEST_TOKEN,
        milestone=TEST_MILESTONE,
        branch='master',
        output_format='release')
    expected = '''## Version <RELEASE_VERSION> (2016-12-05)

### Issues Closed

* Issue #26 - Test number 2
* Issue #24 - Issue test

In this release 2 issues were closed.

### Pull Requests Merged

* PR #25 - PR: Add tests folder

In this release 1 pull request was closed.
'''
    print([log])
    print([expected])
    assert log == expected

@pytest.mark.skipif(NOT_ON_CI, reason='test on ci server only')
def test_changelog_release_branch():
    log = create_changelog(
        repo=REPO,
        token=TEST_TOKEN,
        milestone=TEST_MILESTONE,
        branch='test-branch',
        output_format='release')
    expected = '''## Version <RELEASE_VERSION> (2016-12-05)

### Issues Closed

* Issue #26 - Test number 2
* Issue #24 - Issue test

In this release 2 issues were closed.

### Pull Requests Merged

* PR #33 - PR: Test change

In this release 1 pull request was closed.
'''
    print([log])
    print([expected])
    assert log == expected
<|endoftext|>


<commit_before>#!/usr/bin/python2.7
import sys
import os
import time
from multiprocessing import Process, Lock, Queue, cpu_count
import struct
import numpy
from xml.dom.minidom import parse, parseString
import gdal
import osr
import gdalnumeric
from gdalconst import *

exitFlag = 0
queueLock = None
writeLock = None
workQueue = None
tasks = None

class Tasks(object):
	def __init__(self):
		self.tasks = []

	def add(self,task):
		self.tasks.append(task)

	def get(self,task):
		return self.tasks[task]

class RenderSubprocess(object):
    def __init__(self, processID, processName, queue, tasks):
        self.processID = processID
        self.processName = processName
        self.queue = queue
        self.tasks = tasks
        #Variable#
        self.strip = None
        self.task = None
        self.tries = 3

    def run(self):
    	while not exitFlag:
    		if self.strip is None:
    			queueLock.acquire()
        		if not workQueue.empty():
            			b, inBand, outBand, y0, y, r, t = self.task = self.tasks.get(self.queue.get())
            			queueLock.release()
            			#==#
            			if t==1:
			            	#print self.processName+" reading rows "+str(y*r)+" to "+str((y*r)+r-1)+" in band "+str(b)+"."
            				try:
            					self.strip = inBand.ReadAsArray(0,y*r,inBand.XSize,r,inBand.XSize,r)
            					self.tries = 10
            				except:
            					self.strip = None
            					print "read failed.  will try again later."
            					print "y:"+str(y)
            					print "r:"+str(r)

            			elif t==2:
			            	#print self.processName+" reading row "+str((y0*r)+y)+" in band "+str(b)+"."
            				try:
            					self.strip = inBand.ReadAsArray(0,(y0*r)+y,inBand.XSize,1,inBand.XSize,1)
            					self.tries = 10
            				except:
            					self.strip = None
            					print "read failed.  will try again later."
            					print "y0:"+str(y0)
            					print "y:"+str(y)
            					print "r:"+str(r)
            					print "(y0*r)+y:"+str((y0*r)+y)

        		else:
            			queueLock.release()
        	else:
        		if self.tries > 0:
        			writeLock.acquire()
        			b, inBand, outBand, y0, y, r, t = self.task
        			if t==1:
			            	#print self.processName+" writing rows "+str(y*r)+" to "+str((y*r)+r-1)+" in band "+str(b)+"."
            				try:
	            				outBand.WriteArray(self.strip,0,y*r)
            					self.strip = None
            				except:
	            				self.tries = self.tries - 1
            					print "write failed.  "+str(self.tries)+" more tries."
            			elif t==2:
			            	#print self.processName+" writing row "+str((y0*r)+y)+" in band "+str(b)+"."
            				try:
            					outBand.WriteArray(self.strip,0,(y0*r)+y)
            					self.strip = None
            				except:
            					self.tries = self.tries -1
            					print "write failed.  "+str(self.tries)+" more tries."
            			writeLock.release()
            		else:
            			print "abandoning write"
            			self.strip = None
        	time.sleep(1)

def execute(subprocess):
	subprocess.run()

class BreakPoint:

	def __init__(self, i = None, o = None):
		self.i = i #Should be double; even though it is most frequently an int
		self.o = o

class Line:

	def __init__(self, b0, b1):
		self.x0 = b0.i
		self.y0 = b0.o
		self.x1 = b1.i
		self.y1 = b1.o
		self.n = float(self.y1-self.y0)
		self.d = float(self.x1-self.x0)
		if self.d != 0:
			self.m = self.n/self.d
			self.b = self.y0 - self.m*self.x0
		else:
			self.m = None
			self.b = None

	def min(self):
		return self.x0

	def max(self):
		return self.x1

	def contains(self, x):
		return x>=self.x0 and x <=self.x1

	def calc(self, x):
		if self.d != 0:
			return int(round(self.m*x+self.b))
		elif self.d == 0 and self.y1 == self.y0:
			return self.y1
		else:
			return x

class Lines:

	def __init__(self, bps):
		self.lines = []
		for i in range(len(bps)-1):
			if (not (bps[i] is None)) and (not (bps[i+1] is None)):
				self.lines.append(Line(bps[i],bps[i+1]))

	def size(self):
		return len(self.lines)

	def min(self):
		return self.lines[0].min()

	def max(self):
		return self.lines[len(self.lines)-1].max()

	def calc(self, x):
		y = x
		for line in self.lines:
			if line.contains(x):
				y = line.calc(x)
				break
		return y

class LookUpTable:

	def __init__(self,bps):
		self.lines = Lines(bps)
		self.table = []
		self.min = int(self.lines.min())
		self.max = int(self.lines.max())

		for i in range(0,256):
			self.table.append(self.lines.calc(i))

class LookUpTable2:

	def __init__(self,bps_red,bps_green,bps_blue):
		self.lines_red = Lines(bps_red)
		self.lines_green = Lines(bps_green)
		self.lines_blue = Lines(bps_blue)
		self.table = []
		for i in range(0,256):
			self.table.append(int((self.lines_red.calc(i)+self.lines_green.calc(i)+self.lines_blue.calc(i))/3.0))

class LookUpTables:

	def __init__ (self, filename):
		self.bps_red = None
		self.bps_green = None
		self.bps_blue = None
		self.tables = None
		self.table = None

		self.file = None
		self.valid = True

		self.init_breakpoints(filename)
		self.init_tables()

	def init_breakpoints(self,filename):
                self.bps_red = []
                self.bps_green = []
                self.bps_blue = []

		if filename.endswith(".cbp") or filename.endswith(".txt"):
			lines = None
			file = open(filename,"r")
			with open(filename) as f:
				lines = f.readlines()

			band = -1
			for line in lines:
				line = line.strip()
				if line=="":
					continue
				elif line=="rgbversion8.3":
					continue
				elif line.startswith("#"):
					continue
				elif line=="RGB":
					band = band+1
				elif line.startswith("Band"):
					band = int(line.split()[1].strip(' \t\n\r'))-1
				elif line.startswith("Break") or line.startswith("-"):
					if band==0:
						self.bps_red.append(None)
					elif band==1:
						self.bps_green.append(None)
					elif band==2:
						self.bps_blue.append(None)
				else:
					terms = line.split()
					bp = BreakPoint(float(terms[0].strip(' \t\n\r')),float(terms[1].strip(' \t\n\r')))
					if band==0:
						self.bps_red.append(bp)
					elif band==1:
						self.bps_green.append(bp)
					elif band==2:
						self.bps_blue.append(bp)

		elif filename.endswith(".qml"):
			file = open(filename,"r")
			dom = parse(file)
			minRed = -1
			maxRed = -1
			minGreen = -1
			maxGreen = -1
			minBlue = -1
			maxBlue = -1
			for nRasterRenderer in dom.getElementsByTagName("rasterrenderer"):
				for node in nRasterRenderer.childNodes:
					#print node.toxml()
					if node.nodeType != node.TEXT_NODE:
						if node.tagName=="redContrastEnhancement":
							minRed = float(node.getElementsByTagName("minValue")[0].firstChild.nodeValue)
							maxRed = float(node.getElementsByTagName("maxValue")[0].firstChild.nodeValue)
						elif node.tagName=="greenContrastEnhancement":
							minGreen = float(node.getElementsByTagName("minValue")[0].firstChild.nodeValue)
							maxGreen = float(node.getElementsByTagName("maxValue")[0].firstChild.nodeValue)
						elif node.tagName=="blueContrastEnhancement":
							minBlue = float(node.getElementsByTagName("minValue")[0].firstChild.nodeValue)
							maxBlue = float(node.getElementsByTagName("maxValue")[0].firstChild.nodeValue)
			if minRed!=-1 and maxRed!=-1:
				self.bps_red.extend(self.buildBreakPoints(minRed,maxRed))
			if minGreen!=-1 and maxGreen!=-1:
				self.bps_green.extend(self.buildBreakPoints(minGreen,maxGreen))
                        if minBlue!=-1 and maxBlue!=-1:
				self.bps_blue.extend(self.buildBreakPoints(minBlue,maxBlue))

	def buildBreakPoints(self,minValue,maxValue):
		bps = []
		bps.append(BreakPoint(0.0,0.0))
		bps.append(BreakPoint(minValue,0.0))
		bps.append(BreakPoint(maxValue,255.0))
		bps.append(BreakPoint(255.0,255.0))
		return bps

	def init_tables(self):
		self.tables = []
		self.tables.append(LookUpTable(self.bps_red))
		self.tables.append(LookUpTable(self.bps_green))
		self.tables.append(LookUpTable(self.bps_blue))
		self.tables.append(LookUpTable2(self.bps_red,self.bps_green,self.bps_blue))

	def isValid(self):
		return self.valid;

def main():
	if(len(sys.argv)==6):
		inputFile = sys.argv[1]
		breakPointsFile = sys.argv[2]
		outputFile = sys.argv[3]
		rows = int(sys.argv[4])
		numberOfThreads = int(sys.argv[5])
		if numberOfThreads > 0:
			if(os.path.exists(inputFile) and os.path.exists(breakPointsFile)):
				if(not os.path.exists(outputFile)):
					inputDataset = gdal.Open(inputFile,GA_ReadOnly)
					lookUpTables = LookUpTables(breakPointsFile)
					if ((not inputDataset is None) and (lookUpTables.isValid())):
						outputFormat = "HFA"
						numberOfBands = 3
						w = inputDataset.RasterXSize
						h = inputDataset.RasterYSize
						outputDataset = initDataset(outputFile,outputFormat,w,h,numberOfBands)
						outputDataset.SetGeoTransform(list(inputDataset.GetGeoTransform()))
						outputDataset.SetProjection(inputDataset.GetProjection())

						if numberOfThreads == 1:
							for b in range(numberOfBands):
								print "Stretching Band "+str(b+1)
								lut = numpy.array(lookUpTables.tables[b].table)
								inBand = inputDataset.GetRasterBand(b+1)
								outBand = outputDataset.GetRasterBand(b+1)

								r = rows
								for y in range(int(inBand.YSize/r)):
									outBand.WriteArray(lut[inBand.ReadAsArray(0,y*r,inBand.XSize,r,inBand.XSize,r)],0,y*r)

								y0 = inBand.YSize/rows
								for y in range(inBand.YSize%r):
									outBand.WriteArray(lut[inBand.ReadAsArray(0,y0+y,inBand.XSize,1,inBand.XSize,1)],0,y0+y)
						elif numberOfThreads > 1:
							print "not implemented yet"

						inputDataset = None
						outputDataset = None
					else:
						print "Error Opening File"
				else:
					print "Output file already exists"
			else:
				print "Input file does not exist."
		else:
			print "You need at least 1 thread."
	else:
		print "Usage: cybergis-script-stretch.py <input_file> <breakpoints_file> <output_file> <rows> <threads>"

def initDataset(outputFile,f,w,h,b):
    driver = gdal.GetDriverByName(f)
    metadata = driver.GetMetadata()
    return driver.Create(outputFile,w,h,b,gdal.GDT_Byte,['ALPHA=YES'])

def initProcesses(count)
    print str(cpu_count())+" CPUs are available."
    processes = []
    for processID in range(count):
        subprocess = RenderSubprocess(processID, ("Thread "+str(processID)), workQueue, tasks)
        process = Process(target=execute,args=(subprocess,))
        process.start()
        processes.append(process)
        processID += 1
    print "Initialized "+str(count)+" threads."
    return processes

main()
<commit_msg>Update cybergis-script-stretch.py<commit_after>#!/usr/bin/python2.7
import sys
import os
import time
from multiprocessing import Process, Lock, Queue, cpu_count
import struct
import numpy
from xml.dom.minidom import parse, parseString
import gdal
import osr
import gdalnumeric
from gdalconst import *

exitFlag = 0
queueLock = None
writeLock = None
workQueue = None
tasks = None

class Tasks(object):
	def __init__(self):
		self.tasks = []

	def add(self,task):
		self.tasks.append(task)

	def get(self,task):
		return self.tasks[task]

class RenderSubprocess(object):
    def __init__(self, processID, processName, queue, tasks):
        self.processID = processID
        self.processName = processName
        self.queue = queue
        self.tasks = tasks
        #Variable#
        self.strip = None
        self.strip_stretched = None
        self.task = None
        self.tries = 3

    def run(self):
    	while not exitFlag:
    		if self.strip is None:
    			queueLock.acquire()
        		if not workQueue.empty():
            			b, inBand, outBand, y0, y, r, t = self.task = self.tasks.get(self.queue.get())
            			queueLock.release()
            			#==#
            			if t==1:
			            	#print self.processName+" reading rows "+str(y*r)+" to "+str((y*r)+r-1)+" in band "+str(b)+"."
            				try:
            					self.strip = inBand.ReadAsArray(0,y*r,inBand.XSize,r,inBand.XSize,r)
            					self.tries = 10
            				except:
            					self.strip = None
            					self.strip_stretched = None
            					print "read failed.  will try again later."
            					print "y:"+str(y)
            					print "r:"+str(r)

            			elif t==2:
			            	#print self.processName+" reading row "+str((y0*r)+y)+" in band "+str(b)+"."
            				try:
            					self.strip = inBand.ReadAsArray(0,(y0*r)+y,inBand.XSize,1,inBand.XSize,1)
            					self.tries = 10
            				except:
            					self.strip = None
            					self.strip_stretched = None
            					print "read failed.  will try again later."
            					print "y0:"+str(y0)
            					print "y:"+str(y)
            					print "r:"+str(r)
            					print "(y0*r)+y:"+str((y0*r)+y)

        		else:
            			queueLock.release()
            	elif self.strip_stretched is None:
            		self.strip_stretched = lut[self.strip]
        	elif (not self.strip is None) and (not self.strip_stretched is None):
        		if self.tries > 0:
        			writeLock.acquire()
        			b, inBand, outBand, y0, y, r, t = self.task
        			if t==1:
			            	#print self.processName+" writing rows "+str(y*r)+" to "+str((y*r)+r-1)+" in band "+str(b)+"."
            				try:
	            				outBand.WriteArray(self.strip_stretched,0,y*r)
            					self.strip = None
            					self.strip_stretched = None
            				except:
	            				self.tries = self.tries - 1
            					print "write failed.  "+str(self.tries)+" more tries."
            			elif t==2:
			            	#print self.processName+" writing row "+str((y0*r)+y)+" in band "+str(b)+"."
            				try:
            					outBand.WriteArray(self.strip_stretched,0,(y0*r)+y)
            					self.strip = None
            					self.strip_stretched = None
            				except:
            					self.tries = self.tries -1
            					print "write failed.  "+str(self.tries)+" more tries."
            			writeLock.release()
            		else:
            			print "abandoning write"
            			self.strip = None
            			self.strip_stretched = None
            	else:
            	    print "abandoning task.  This should never happen."
            	    self.strip = None
            	    self.strip_stretched = None

        	time.sleep(1)

def execute(subprocess):
	subprocess.run()

class BreakPoint:

	def __init__(self, i = None, o = None):
		self.i = i #Should be double; even though it is most frequently an int
		self.o = o

class Line:

	def __init__(self, b0, b1):
		self.x0 = b0.i
		self.y0 = b0.o
		self.x1 = b1.i
		self.y1 = b1.o
		self.n = float(self.y1-self.y0)
		self.d = float(self.x1-self.x0)
		if self.d != 0:
			self.m = self.n/self.d
			self.b = self.y0 - self.m*self.x0
		else:
			self.m = None
			self.b = None

	def min(self):
		return self.x0

	def max(self):
		return self.x1

	def contains(self, x):
		return x>=self.x0 and x <=self.x1

	def calc(self, x):
		if self.d != 0:
			return int(round(self.m*x+self.b))
		elif self.d == 0 and self.y1 == self.y0:
			return self.y1
		else:
			return x

class Lines:

	def __init__(self, bps):
		self.lines = []
		for i in range(len(bps)-1):
			if (not (bps[i] is None)) and (not (bps[i+1] is None)):
				self.lines.append(Line(bps[i],bps[i+1]))

	def size(self):
		return len(self.lines)

	def min(self):
		return self.lines[0].min()

	def max(self):
		return self.lines[len(self.lines)-1].max()

	def calc(self, x):
		y = x
		for line in self.lines:
			if line.contains(x):
				y = line.calc(x)
				break
		return y

class LookUpTable:

	def __init__(self,bps):
		self.lines = Lines(bps)
		self.table = []
		self.min = int(self.lines.min())
		self.max = int(self.lines.max())

		for i in range(0,256):
			self.table.append(self.lines.calc(i))

class LookUpTable2:

	def __init__(self,bps_red,bps_green,bps_blue):
		self.lines_red = Lines(bps_red)
		self.lines_green = Lines(bps_green)
		self.lines_blue = Lines(bps_blue)
		self.table = []
		for i in range(0,256):
			self.table.append(int((self.lines_red.calc(i)+self.lines_green.calc(i)+self.lines_blue.calc(i))/3.0))

class LookUpTables:

	def __init__ (self, filename):
		self.bps_red = None
		self.bps_green = None
		self.bps_blue = None
		self.tables = None
		self.table = None

		self.file = None
		self.valid = True

		self.init_breakpoints(filename)
		self.init_tables()

	def init_breakpoints(self,filename):
                self.bps_red = []
                self.bps_green = []
                self.bps_blue = []

		if filename.endswith(".cbp") or filename.endswith(".txt"):
			lines = None
			file = open(filename,"r")
			with open(filename) as f:
				lines = f.readlines()

			band = -1
			for line in lines:
				line = line.strip()
				if line=="":
					continue
				elif line=="rgbversion8.3":
					continue
				elif line.startswith("#"):
					continue
				elif line=="RGB":
					band = band+1
				elif line.startswith("Band"):
					band = int(line.split()[1].strip(' \t\n\r'))-1
				elif line.startswith("Break") or line.startswith("-"):
					if band==0:
						self.bps_red.append(None)
					elif band==1:
						self.bps_green.append(None)
					elif band==2:
						self.bps_blue.append(None)
				else:
					terms = line.split()
					bp = BreakPoint(float(terms[0].strip(' \t\n\r')),float(terms[1].strip(' \t\n\r')))
					if band==0:
						self.bps_red.append(bp)
					elif band==1:
						self.bps_green.append(bp)
					elif band==2:
						self.bps_blue.append(bp)

		elif filename.endswith(".qml"):
			file = open(filename,"r")
			dom = parse(file)
			minRed = -1
			maxRed = -1
			minGreen = -1
			maxGreen = -1
			minBlue = -1
			maxBlue = -1
			for nRasterRenderer in dom.getElementsByTagName("rasterrenderer"):
				for node in nRasterRenderer.childNodes:
					#print node.toxml()
					if node.nodeType != node.TEXT_NODE:
						if node.tagName=="redContrastEnhancement":
							minRed = float(node.getElementsByTagName("minValue")[0].firstChild.nodeValue)
							maxRed = float(node.getElementsByTagName("maxValue")[0].firstChild.nodeValue)
						elif node.tagName=="greenContrastEnhancement":
							minGreen = float(node.getElementsByTagName("minValue")[0].firstChild.nodeValue)
							maxGreen = float(node.getElementsByTagName("maxValue")[0].firstChild.nodeValue)
						elif node.tagName=="blueContrastEnhancement":
							minBlue = float(node.getElementsByTagName("minValue")[0].firstChild.nodeValue)
							maxBlue = float(node.getElementsByTagName("maxValue")[0].firstChild.nodeValue)
			if minRed!=-1 and maxRed!=-1:
				self.bps_red.extend(self.buildBreakPoints(minRed,maxRed))
			if minGreen!=-1 and maxGreen!=-1:
				self.bps_green.extend(self.buildBreakPoints(minGreen,maxGreen))
                        if minBlue!=-1 and maxBlue!=-1:
				self.bps_blue.extend(self.buildBreakPoints(minBlue,maxBlue))

	def buildBreakPoints(self,minValue,maxValue):
		bps = []
		bps.append(BreakPoint(0.0,0.0))
		bps.append(BreakPoint(minValue,0.0))
		bps.append(BreakPoint(maxValue,255.0))
		bps.append(BreakPoint(255.0,255.0))
		return bps

	def init_tables(self):
		self.tables = []
		self.tables.append(LookUpTable(self.bps_red))
		self.tables.append(LookUpTable(self.bps_green))
		self.tables.append(LookUpTable(self.bps_blue))
		self.tables.append(LookUpTable2(self.bps_red,self.bps_green,self.bps_blue))

	def isValid(self):
		return self.valid;

def main():
	if(len(sys.argv)==6):
		inputFile = sys.argv[1]
		breakPointsFile = sys.argv[2]
		outputFile = sys.argv[3]
		rows = int(sys.argv[4])
		numberOfThreads = int(sys.argv[5])
		if numberOfThreads > 0:
			if(os.path.exists(inputFile) and os.path.exists(breakPointsFile)):
				if(not os.path.exists(outputFile)):
					inputDataset = gdal.Open(inputFile,GA_ReadOnly)
					lookUpTables = LookUpTables(breakPointsFile)
					if ((not inputDataset is None) and (lookUpTables.isValid())):
						outputFormat = "HFA"
						numberOfBands = 3
						w = inputDataset.RasterXSize
						h = inputDataset.RasterYSize
						outputDataset = initDataset(outputFile,outputFormat,w,h,numberOfBands)
						outputDataset.SetGeoTransform(list(inputDataset.GetGeoTransform()))
						outputDataset.SetProjection(inputDataset.GetProjection())

						if numberOfThreads == 1:
							for b in range(numberOfBands):
								print "Stretching Band "+str(b+1)
								lut = numpy.array(lookUpTables.tables[b].table)
								inBand = inputDataset.GetRasterBand(b+1)
								outBand = outputDataset.GetRasterBand(b+1)

								r = rows
								for y in range(int(inBand.YSize/r)):
									outBand.WriteArray(lut[inBand.ReadAsArray(0,y*r,inBand.XSize,r,inBand.XSize,r)],0,y*r)

								y0 = inBand.YSize/rows
								for y in range(inBand.YSize%r):
									outBand.WriteArray(lut[inBand.ReadAsArray(0,y0+y,inBand.XSize,1,inBand.XSize,1)],0,y0+y)
						elif numberOfThreads > 1:
							print "not fully implemented yet"
                                                        global exitFlag
                                                        global queueLock
                                                        global writeLock
                                                        global workQueue
                                                        global tasks
							#
                                                        exitFlag = 0
                                                        queueLock = Lock()
                                                        writeLock = Lock()
                                                        workQueue = Queue(0)
                                                        tasks = Tasks()
							#
							for b in range(inputBands):
								print "Adding tasks for band "+str(b+1)
								lut = numpy.array(lookUpTables.tables[b].table)
								inBand = inputDataset.GetRasterBand(b+1)
								outBand = outputDataset.GetRasterBand(b+1)
								y0 = int(inBand.YSize/r)
								for y in range(int(inBand.YSize/r)):
									task = b+1, inBand, outBand, lut, y0, y, r, 1
									tasks.add(task)
								for y in range(inBand.YSize%r):
									task = b+1, inBand, outBand, lut, y0, y, r, 2
									tasks.add(task)

						inputDataset = None
						outputDataset = None
					else:
						print "Error Opening File"
				else:
					print "Output file already exists"
			else:
				print "Input file does not exist."
		else:
			print "You need at least 1 thread."
	else:
		print "Usage: cybergis-script-stretch.py <input_file> <breakpoints_file> <output_file> <rows> <threads>"

def initDataset(outputFile,f,w,h,b):
    driver = gdal.GetDriverByName(f)
    metadata = driver.GetMetadata()
    return driver.Create(outputFile,w,h,b,gdal.GDT_Byte,['ALPHA=YES'])

def initProcesses(count)
    print str(cpu_count())+" CPUs are available."
    processes = []
    processID = 1
    for processID in range(count):
        subprocess = RenderSubprocess(processID, ("Thread "+str(processID)), workQueue, tasks)
        process = Process(target=execute,args=(subprocess,))
        process.start()
        processes.append(process)
        processID += 1
    print "Initialized "+str(count)+" threads."
    return processes

main()
<|endoftext|>


<commit_before># Copyright 2014 hm authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

__version__ = '0.1.1'
<commit_msg>bump version<commit_after># Copyright 2014 hm authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

__version__ = '0.1.2'
<|endoftext|>


<commit_before># Copyright 2016 hm authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

__version__ = '0.6.8'
<commit_msg>bump to 0.6.9<commit_after># Copyright 2016 hm authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

__version__ = '0.6.9'
<|endoftext|>