<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific ladnguage governing permissions and
# limitations under the License.

import datetime
import os

import batch_predict

PROJECT_ID = os.environ["AUTOML_PROJECT_ID"]
BUCKET_ID = "{}-lcm".format(PROJECT_ID)
MODEL_ID = "TEN0000000000000000000"
PREFIX = "TEST_EXPORT_OUTPUT_" + datetime.datetime.now().strftime(
    "%Y%m%d%H%M%S"
)

def test_batch_predict(capsys):
    # As batch prediction can take a long time. Try to batch predict on a model
    # and confirm that the model was not found, but other elements of the
    # request were valid.
    try:
        input_uri = "gs://{}/entity-extraction/input.jsonl".format(BUCKET_ID)
        output_uri = "gs://{}/{}/".format(BUCKET_ID, PREFIX)
        batch_predict.batch_predict(
            PROJECT_ID, MODEL_ID, input_uri, output_uri
        )
        out, _ = capsys.readouterr()
        assert (
            "The model is either not found or not supported for prediction yet"
            in out
        )
    except Exception as e:
        assert (
            "The model is either not found or not supported for prediction yet"
            in e.message
        )
<commit_msg>automl: fix batch test due to error message change (#3057)<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific ladnguage governing permissions and
# limitations under the License.

import datetime
import os

import batch_predict

PROJECT_ID = os.environ["AUTOML_PROJECT_ID"]
BUCKET_ID = "{}-lcm".format(PROJECT_ID)
MODEL_ID = "TEN0000000000000000000"
PREFIX = "TEST_EXPORT_OUTPUT_" + datetime.datetime.now().strftime(
    "%Y%m%d%H%M%S"
)

def test_batch_predict(capsys):
    # As batch prediction can take a long time. Try to batch predict on a model
    # and confirm that the model was not found, but other elements of the
    # request were valid.
    try:
        input_uri = "gs://{}/entity-extraction/input.jsonl".format(BUCKET_ID)
        output_uri = "gs://{}/{}/".format(BUCKET_ID, PREFIX)
        batch_predict.batch_predict(
            PROJECT_ID, MODEL_ID, input_uri, output_uri
        )
        out, _ = capsys.readouterr()
        assert (
            "does not exist"
            in out
        )
    except Exception as e:
        assert (
            "does not exist"
            in e.message
        )
<|endoftext|>


<commit_before>import commands
import os
import Queue
import settings
import time
import threading
import serial
import sys
import traceback

from thirtybirds_2_0.Network.manager import init as network_init
from thirtybirds_2_0.Updates.manager import init as updates_init

class Network(object):
    def __init__(self, hostname, network_message_handler, network_status_handler):
        self.hostname = hostname
        self.thirtybirds = network_init(
            hostname=hostname,
            role="client",
            discovery_multicastGroup=settings.discovery_multicastGroup,
            discovery_multicastPort=settings.discovery_multicastPort,
            discovery_responsePort=settings.discovery_responsePort,
            pubsub_pubPort=settings.pubsub_pubPort,
            message_callback=network_message_handler,
            status_callback=network_status_handler
        )

########################
## UTILS
########################

class Utils(object):
    def __init__(self, hostname):
        self.hostname = hostname
    def reboot(self):
        os.system("sudo reboot now")

    def remote_update_git(self, oratio, thirtybirds, update, upgrade):
        if oratio:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/oratio')
        if thirtybirds:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/thirtybirds_2_0')
        return

    def remote_update_scripts(self):
        updates_init("/home/pi/oratio", False, True)
        return

    def get_update_script_version(self):
        (updates, ghStatus, bsStatus) = updates_init("/home/pi/oratio", False, False)
        return updates.read_version_pickle()

    def get_git_timestamp(self):
        return commands.getstatusoutput("cd /home/pi/oratio/; git log -1 --format=%cd")[1]

    def get_temp(self):
        return commands.getstatusoutput("/opt/vc/bin/vcgencmd measure_temp")[1]

    def get_cpu(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[12]

    def get_uptime(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[4]

    def get_disk(self):
        # stub for now
        return "0"

    def get_client_status(self):
        return (self.hostname, self.get_update_script_version(), self.get_git_timestamp(), self.get_temp(), self.get_cpu(), self.get_uptime(), self.get_disk())

class Poller(threading.Thread):
    def __init__(self, _main_, poll_delay_time):
        threading.Thread.__init__(self)
        self._main_ = _main_
        self.poll_delay_time = poll_delay_time
    def run(self):
        while True:
            print "Poller Thread"
            self._main_.network.thirtybirds.send("mandala_device_request", True)
            time.sleep(self.poll_delay_time)

# Main handles network send/recv and can see all other classes directly
class Main(threading.Thread):
    def __init__(self, hostname):
        threading.Thread.__init__(self)
        print os. system("stty -F /dev/ttyACM0 -hupcl")

        self.network = Network(hostname, self.network_message_handler, self.network_status_handler)
        self.queue = Queue.Queue()
        self.arduino_connection = open("/dev/ttyACM0",'w')
        self.utils = Utils(hostname)
        self.network.thirtybirds.subscribe_to_topic("mandala_device_status")
        self.UNSET = 0
        self.FAIL = 2000
        self.PASS = 4000
        self.mandala_device_status = None
        self.mandala_tlc_ids = {
            "avl-controller":39,
            "avl-formant-1":15,
            "avl-formant-1-amplifier":4,
            "avl-formant-2":16,
            "avl-formant-2-amplifier":5,
            "avl-formant-3":17,
            "avl-formant-3-amplifier":6,
            "avl-layer-1":40,
            "avl-layer-2":11,
            "avl-layer-3":12,
            "avl-medulla":35,
            "avl-pitch-keys":18,
            "avl-pitch-keys-sensor-1":7,
            "avl-pitch-keys-sensor-2":8,
            "avl-pitch-keys-sensor-3":9,
            "avl-pitch-keys-sensor-4":10,
            "avl-settings":34,
            "avl-settings-adcs":24,
            "avl-transport":13,
            "avl-transport-encoder":0,
            "avl-voice-1":36,
            "avl-voice-1-crystal-frequency-counter":25,
            "avl-voice-1-harmonic-generators":26,
            "avl-voice-1-harmonic-volume":27,
            "avl-voice-2":37,
            "avl-voice-2-crystal-frequency-counter":28,
            "avl-voice-2-harmonic-generators":29,
            "avl-voice-2-harmonic-volume":30,
            "avl-voice-3":38,
            "avl-voice-3-crystal-frequency-counter":31,
            "avl-voice-3-harmonic-generators":32,
            "avl-voice-3-harmonic-volume":33,
            "avl-voice-keys":14,
            "avl-voice-keys-encoder-1":1,
            "avl-voice-keys-encoder-2":2,
            "avl-voice-keys-encoder-3":3
        }
        self.mandala_status = {
            "avl-controller":"pass", # because if this is sending data, it's online.
            "avl-formant-1":"unset",
            "avl-formant-1-amplifier":"unset",
            "avl-formant-2":"unset",
            "avl-formant-2-amplifier":"unset",
            "avl-formant-3":"unset",
            "avl-formant-3-amplifier":"unset",
            "avl-layer-1":"unset",
            "avl-layer-2":"unset",
            "avl-layer-3":"unset",
            "avl-medulla":"pass",# because if this is sending data, it's online.
            "avl-pitch-keys":"unset",
            "avl-pitch-keys-sensor-1":"unset",
            "avl-pitch-keys-sensor-2":"unset",
            "avl-pitch-keys-sensor-3":"unset",
            "avl-pitch-keys-sensor-4":"unset",
            "avl-settings":"unset",
            "avl-settings-adcs":"unset",
            "avl-transport":"unset",
            "avl-transport-encoder":"unset",
            "avl-voice-1":"unset",
            "avl-voice-1-crystal-frequency-counter":"unset",
            "avl-voice-1-harmonic-generators":"unset",
            "avl-voice-1-harmonic-volume":"unset",
            "avl-voice-2":"unset",
            "avl-voice-2-crystal-frequency-counter":"unset",
            "avl-voice-2-harmonic-generators":"unset",
            "avl-voice-2-harmonic-volume":"unset",
            "avl-voice-3":"unset",
            "avl-voice-3-crystal-frequency-counter":"unset",
            "avl-voice-3-harmonic-generators":"unset",
            "avl-voice-3-harmonic-volume":"unset",
            "avl-voice-keys":"unset",
            "avl-voice-keys-encoder-1":"unset",
            "avl-voice-keys-encoder-2":"unset",
            "avl-voice-keys-encoder-3":"unset"
        }
        self.arduino_delay_time = 0.05

        self.poller = Poller(self, 20)
        self.poller.start()

    def network_message_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        topic, msg =  topic_msg # separating just to eval msg.  best to do it early.  it should be done in TB.
        #print "network_message_handler", topic, msg
        #if len(msg) > 0:
        #    msg = eval(msg)
        self.add_to_queue(topic, msg)

    def network_status_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Mains
        print "Main.network_status_handler", topic_msg

    def add_to_queue(self, topic, msg):
        self.queue.put((topic, msg))

    def update_mandala_status(self, devicename, status):
        print "update_mandala_status", devicename, self.mandala_status[devicename], status, self.mandala_status[devicename] == status

        if str(self.mandala_status[devicename]) != str(status):
            self.mandala_status[devicename] = status
            tlc_id_int = self.mandala_tlc_ids[devicename] + 5000
            tlc_id_str = "{}\n".format(tlc_id_int)
            if self.mandala_status[devicename] == "unset":
                tlc_level_int = 0
            if self.mandala_status[devicename] == "fail":
                tlc_level_int = 100
            if self.mandala_status[devicename] == "pass":
                tlc_level_int = 4000
            tlc_level_str = "{}\n".format(tlc_level_int)

            self.write_to_arduino(tlc_id_str,tlc_level_str)

    def write_to_arduino(self, id, level):
        print "write_to_arduino", repr(id), repr(level)
        time.sleep(self.arduino_delay_time)
        #self.arduino_connection.write(id)
        time.sleep(self.arduino_delay_time)
        #self.arduino_connection.write(level)

    def run(self):

        devicenames = self.mandala_tlc_ids.keys()
        devicenames.sort()
        for devicename in devicenames:
            tlc_id_int = self.mandala_tlc_ids[devicename] + 5000
            tlc_id_str = "{}\n".format(tlc_id_int)
            tlc_level_str = "0/n"
            self.write_to_arduino(tlc_id_str,tlc_level_str)
        while True:
            #if not all(status == "pass" for status in self.mandala_status.values()):
            #    self.network.thirtybirds.send("mandala_device_request", True)
            try:
                topic, msg_str = self.queue.get(True)
                if topic == "mandala_device_status":
                    msg = eval(msg_str)
                    #print topic, msg
                    devicename, status = msg
                    self.update_mandala_status(devicename, status)
                time.sleep(0.01)
            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback))

def init(hostname):
    main = Main(hostname)
    main.daemon = True
    main.start()
    return main

<commit_msg>fixing mandala event chain<commit_after>import commands
import os
import Queue
import settings
import time
import threading
import serial
import sys
import traceback

from thirtybirds_2_0.Network.manager import init as network_init
from thirtybirds_2_0.Updates.manager import init as updates_init

class Network(object):
    def __init__(self, hostname, network_message_handler, network_status_handler):
        self.hostname = hostname
        self.thirtybirds = network_init(
            hostname=hostname,
            role="client",
            discovery_multicastGroup=settings.discovery_multicastGroup,
            discovery_multicastPort=settings.discovery_multicastPort,
            discovery_responsePort=settings.discovery_responsePort,
            pubsub_pubPort=settings.pubsub_pubPort,
            message_callback=network_message_handler,
            status_callback=network_status_handler
        )

########################
## UTILS
########################

class Utils(object):
    def __init__(self, hostname):
        self.hostname = hostname
    def reboot(self):
        os.system("sudo reboot now")

    def remote_update_git(self, oratio, thirtybirds, update, upgrade):
        if oratio:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/oratio')
        if thirtybirds:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/thirtybirds_2_0')
        return

    def remote_update_scripts(self):
        updates_init("/home/pi/oratio", False, True)
        return

    def get_update_script_version(self):
        (updates, ghStatus, bsStatus) = updates_init("/home/pi/oratio", False, False)
        return updates.read_version_pickle()

    def get_git_timestamp(self):
        return commands.getstatusoutput("cd /home/pi/oratio/; git log -1 --format=%cd")[1]

    def get_temp(self):
        return commands.getstatusoutput("/opt/vc/bin/vcgencmd measure_temp")[1]

    def get_cpu(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[12]

    def get_uptime(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[4]

    def get_disk(self):
        # stub for now
        return "0"

    def get_client_status(self):
        return (self.hostname, self.get_update_script_version(), self.get_git_timestamp(), self.get_temp(), self.get_cpu(), self.get_uptime(), self.get_disk())

class Poller(threading.Thread):
    def __init__(self, _main_, poll_delay_time):
        threading.Thread.__init__(self)
        self._main_ = _main_
        self.poll_delay_time = poll_delay_time
    def run(self):
        while True:
            print "Poller Thread"
            self._main_.network.thirtybirds.send("mandala_device_request", True)
            time.sleep(self.poll_delay_time)

# Main handles network send/recv and can see all other classes directly
class Main(threading.Thread):
    def __init__(self, hostname):
        threading.Thread.__init__(self)
        print os. system("stty -F /dev/ttyACM0 -hupcl")

        self.network = Network(hostname, self.network_message_handler, self.network_status_handler)
        self.queue = Queue.Queue()
        self.arduino_connection = open("/dev/ttyACM0",'w')
        self.utils = Utils(hostname)
        self.network.thirtybirds.subscribe_to_topic("mandala_device_status")
        self.UNSET = 0
        self.FAIL = 2000
        self.PASS = 4000
        self.mandala_device_status = None
        self.mandala_tlc_ids = {
            "avl-controller":39,
            "avl-formant-1":15,
            "avl-formant-1-amplifier":4,
            "avl-formant-2":16,
            "avl-formant-2-amplifier":5,
            "avl-formant-3":17,
            "avl-formant-3-amplifier":6,
            "avl-layer-1":40,
            "avl-layer-2":11,
            "avl-layer-3":12,
            "avl-medulla":35,
            "avl-pitch-keys":18,
            "avl-pitch-keys-sensor-1":7,
            "avl-pitch-keys-sensor-2":8,
            "avl-pitch-keys-sensor-3":9,
            "avl-pitch-keys-sensor-4":10,
            "avl-settings":34,
            "avl-settings-adcs":24,
            "avl-transport":13,
            "avl-transport-encoder":0,
            "avl-voice-1":36,
            "avl-voice-1-crystal-frequency-counter":25,
            "avl-voice-1-harmonic-generators":26,
            "avl-voice-1-harmonic-volume":27,
            "avl-voice-2":37,
            "avl-voice-2-crystal-frequency-counter":28,
            "avl-voice-2-harmonic-generators":29,
            "avl-voice-2-harmonic-volume":30,
            "avl-voice-3":38,
            "avl-voice-3-crystal-frequency-counter":31,
            "avl-voice-3-harmonic-generators":32,
            "avl-voice-3-harmonic-volume":33,
            "avl-voice-keys":14,
            "avl-voice-keys-encoder-1":1,
            "avl-voice-keys-encoder-2":2,
            "avl-voice-keys-encoder-3":3
        }
        self.mandala_status = {
            "avl-controller":"pass", # because if this is sending data, it's online.
            "avl-formant-1":"unset",
            "avl-formant-1-amplifier":"unset",
            "avl-formant-2":"unset",
            "avl-formant-2-amplifier":"unset",
            "avl-formant-3":"unset",
            "avl-formant-3-amplifier":"unset",
            "avl-layer-1":"unset",
            "avl-layer-2":"unset",
            "avl-layer-3":"unset",
            "avl-medulla":"pass",# because if this is sending data, it's online.
            "avl-pitch-keys":"unset",
            "avl-pitch-keys-sensor-1":"unset",
            "avl-pitch-keys-sensor-2":"unset",
            "avl-pitch-keys-sensor-3":"unset",
            "avl-pitch-keys-sensor-4":"unset",
            "avl-settings":"unset",
            "avl-settings-adcs":"unset",
            "avl-transport":"unset",
            "avl-transport-encoder":"unset",
            "avl-voice-1":"unset",
            "avl-voice-1-crystal-frequency-counter":"unset",
            "avl-voice-1-harmonic-generators":"unset",
            "avl-voice-1-harmonic-volume":"unset",
            "avl-voice-2":"unset",
            "avl-voice-2-crystal-frequency-counter":"unset",
            "avl-voice-2-harmonic-generators":"unset",
            "avl-voice-2-harmonic-volume":"unset",
            "avl-voice-3":"unset",
            "avl-voice-3-crystal-frequency-counter":"unset",
            "avl-voice-3-harmonic-generators":"unset",
            "avl-voice-3-harmonic-volume":"unset",
            "avl-voice-keys":"unset",
            "avl-voice-keys-encoder-1":"unset",
            "avl-voice-keys-encoder-2":"unset",
            "avl-voice-keys-encoder-3":"unset"
        }
        self.arduino_delay_time = 0.05

        self.poller = Poller(self, 20)
        self.poller.start()

    def network_message_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        topic, msg =  topic_msg # separating just to eval msg.  best to do it early.  it should be done in TB.
        #print "network_message_handler", topic, msg
        #if len(msg) > 0:
        #    msg = eval(msg)
        self.add_to_queue(topic, msg)

    def network_status_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Mains
        print "Main.network_status_handler", topic_msg

    def add_to_queue(self, topic, msg):
        self.queue.put((topic, msg))

    def update_mandala_status(self, devicename, status):
        print "update_mandala_status", devicename, self.mandala_status[devicename], status, self.mandala_status[devicename] == status

        #if str(self.mandala_status[devicename]) != str(status):
        self.mandala_status[devicename] = status
        tlc_id_int = self.mandala_tlc_ids[devicename] + 5000
        tlc_id_str = "{}\n".format(tlc_id_int)
        if self.mandala_status[devicename] == "unset":
            tlc_level_int = 0
        if self.mandala_status[devicename] == "fail":
            tlc_level_int = 100
        if self.mandala_status[devicename] == "pass":
            tlc_level_int = 4000
        tlc_level_str = "{}\n".format(tlc_level_int)

        self.write_to_arduino(tlc_id_str,tlc_level_str)

    def write_to_arduino(self, id, level):
        print "write_to_arduino", repr(id), repr(level)
        time.sleep(self.arduino_delay_time)
        self.arduino_connection.write(id)
        time.sleep(self.arduino_delay_time)
        self.arduino_connection.write(level)

    def run(self):

        devicenames = self.mandala_tlc_ids.keys()
        devicenames.sort()
        for devicename in devicenames:
            tlc_id_int = self.mandala_tlc_ids[devicename] + 5000
            tlc_id_str = "{}\n".format(tlc_id_int)
            tlc_level_str = "0/n"
            self.write_to_arduino(tlc_id_str,tlc_level_str)
        while True:
            #if not all(status == "pass" for status in self.mandala_status.values()):
            #    self.network.thirtybirds.send("mandala_device_request", True)
            try:
                topic, msg_str = self.queue.get(True)
                if topic == "mandala_device_status":
                    msg = eval(msg_str)
                    #print topic, msg
                    devicename, status = msg
                    self.update_mandala_status(devicename, status)
                time.sleep(0.01)
            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback))

def init(hostname):
    main = Main(hostname)
    main.daemon = True
    main.start()
    return main

<|endoftext|>


<commit_before>"""View for block details specialized for a GAS"""

from django.conf import settings
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext as _, ugettext_lazy as _lazy

from flexi_auth.models import PrincipalParamRoleRelation, ObjectWithContext

from gasistafelice.consts import EDIT, GAS_MEMBER

from gasistafelice.rest.views.blocks.base import ResourceBlockAction
from gasistafelice.rest.views.blocks import details

from gasistafelice.lib.shortcuts import render_to_context_response

from gasistafelice.gas.forms.base import EditGASForm

import logging
log = logging.getLogger(__name__)

class Block(details.Block):

    BLOCK_NAME = "gas_details"
    BLOCK_VALID_RESOURCE_TYPES = ["gas"]

    def get_description(self):
        return _("Details about %(name)s") % {
            'name' : self.resource.gas.name,
        }

    def _get_edit_form_class(self):
        log.debug ("Loading my edit form class...")
        return EditGASForm

    def _get_user_actions(self, request):
        """Who can edit GAS informations, has also the ability to configure it."""

        user_actions = super(Block, self)._get_user_actions(request)

        if request.user.has_perm(EDIT, obj=ObjectWithContext(request.resource)):

            act_configure = ResourceBlockAction(
                    block_name = self.BLOCK_NAME,
                    resource = request.resource,
                    name="configure", verbose_name=_("Configure"),
                    popup_form=True,
                    url = reverse('admin:gas_gasconfig_change', args=(request.resource.config.pk,))
            )

            for i,act in enumerate(user_actions):
                # Change URL for action EDIT, insert "configure" action
                if act.name == EDIT:
#                   act.url = reverse('admin:gas_gas_change', args=(request.resource.pk,))
                   user_actions.insert(i+1, act_configure)
                   break

        return user_actions

    def manage_roles(self, request):
        """This method is needed to filter out GAS_MEMBER roles"""

        formset_class = self._get_roles_formset_class()

        if request.method == 'POST':
            return super(Block, self).manage_roles(request)

        else:

            data = {}
            roles = self.resource.roles

            # HACK HERE
            # Exclude GAS_MEMBER role which is managed by "Add gasmember" action
            roles = roles.exclude(role__name=GAS_MEMBER)

            # Roles already assigned to resource
            pprrs = PrincipalParamRoleRelation.objects.filter(role__in=roles)

            i = 0
            for i,pprr in enumerate(pprrs):

                key_prefix = 'form-%d' % i
                data.update({
                   '%s-id' % key_prefix : pprr.pk,
                   '%s-person' % key_prefix : pprr.user.person.pk,
                   '%s-role' % key_prefix : pprr.role.pk,
                })

            data['form-TOTAL_FORMS'] = i + formset_class.extra
            data['form-INITIAL_FORMS'] = i
            data['form-MAX_NUM_FORMS'] = 0

            formset = formset_class(request, data)

        context = {

            "formset": formset,
            'opts' : PrincipalParamRoleRelation._meta,
            'is_popup': False,
            'save_as' : False,
            'save_on_top': False,
            #'errors': helpers.AdminErrorList(form, []),
            #'media': mark_safe(adminForm.media),
            'form_url' : request.build_absolute_uri(),
            'add'  : False,
            'change' : True,
            'has_add_permission': False,
            'has_delete_permission': True,
            'has_change_permission': True,
            'show_delete' : True,

        }

        return render_to_context_response(request, "html/formsets.html", context)

<commit_msg>Moved gas_details data from the formset_class to the context.<commit_after>"""View for block details specialized for a GAS"""

from django.conf import settings
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext as _, ugettext_lazy as _lazy

from flexi_auth.models import PrincipalParamRoleRelation, ObjectWithContext

from gasistafelice.consts import EDIT, GAS_MEMBER

from gasistafelice.rest.views.blocks.base import ResourceBlockAction
from gasistafelice.rest.views.blocks import details

from gasistafelice.lib.shortcuts import render_to_context_response

from gasistafelice.gas.forms.base import EditGASForm

import logging
log = logging.getLogger(__name__)

class Block(details.Block):

    BLOCK_NAME = "gas_details"
    BLOCK_VALID_RESOURCE_TYPES = ["gas"]

    def get_description(self):
        return _("Details about %(name)s") % {
            'name' : self.resource.gas.name,
        }

    def _get_edit_form_class(self):
        log.debug ("Loading my edit form class...")
        return EditGASForm

    def _get_user_actions(self, request):
        """Who can edit GAS informations, has also the ability to configure it."""

        user_actions = super(Block, self)._get_user_actions(request)

        if request.user.has_perm(EDIT, obj=ObjectWithContext(request.resource)):

            act_configure = ResourceBlockAction(
                    block_name = self.BLOCK_NAME,
                    resource = request.resource,
                    name="configure", verbose_name=_("Configure"),
                    popup_form=True,
                    url = reverse('admin:gas_gasconfig_change', args=(request.resource.config.pk,))
            )

            for i,act in enumerate(user_actions):
                # Change URL for action EDIT, insert "configure" action
                if act.name == EDIT:
#                   act.url = reverse('admin:gas_gas_change', args=(request.resource.pk,))
                   user_actions.insert(i+1, act_configure)
                   break

        return user_actions

    def manage_roles(self, request):
        """This method is needed to filter out GAS_MEMBER roles"""

        formset_class = self._get_roles_formset_class()

        if request.method == 'POST':
            return super(Block, self).manage_roles(request)

        else:

            data = {}
            roles = self.resource.roles

            # HACK HERE
            # Exclude GAS_MEMBER role which is managed by "Add gasmember" action
            roles = roles.exclude(role__name=GAS_MEMBER)

            # Roles already assigned to resource
            pprrs = PrincipalParamRoleRelation.objects.filter(role__in=roles)

            i = 0
            for i,pprr in enumerate(pprrs):

                key_prefix = 'form-%d' % i
                data.update({
                   '%s-id' % key_prefix : pprr.pk,
                   '%s-person' % key_prefix : pprr.user.person.pk,
                   '%s-role' % key_prefix : pprr.role.pk,
                })

            data['form-TOTAL_FORMS'] = i + formset_class.extra
            data['form-INITIAL_FORMS'] = i
            data['form-MAX_NUM_FORMS'] = 0

            formset = formset_class(request, {})

        context = {
            #data is delivered directly to the template
            "data" : data,
            "formset": formset,
            'opts' : PrincipalParamRoleRelation._meta,
            'is_popup': False,
            'save_as' : False,
            'save_on_top': False,
            #'errors': helpers.AdminErrorList(form, []),
            #'media': mark_safe(adminForm.media),
            'form_url' : request.build_absolute_uri(),
            'add'  : False,
            'change' : True,
            'has_add_permission': False,
            'has_delete_permission': True,
            'has_change_permission': True,
            'show_delete' : True,

        }

        return render_to_context_response(request, "html/formsets.html", context)

<|endoftext|>


<commit_before>from __future__ import unicode_literals

from collections import defaultdict

from django.db.backends.postgresql_psycopg2 import schema
from django.conf import settings

import sqlparse
from sqlparse.tokens import DDL, DML, Keyword

from ...schema import deactivate_schema, is_shared_table
from ...signals import schema_aware_operation

def get_constraints(cursor, table_name, schema_name='__template__'):
    """Return all constraints for a given table

    (in either the given schema, or the public schema: the assumption is made
    that constraint names will not be the same in both).

    """
    cursor.execute("""
WITH constraints AS (

          SELECT tc.constraint_type,
                 tc.constraint_name,
                 COALESCE(ccu.column_name, kcu.column_name) AS column_name
            FROM information_schema.table_constraints AS tc
 LEFT OUTER JOIN information_schema.constraint_column_usage AS ccu
           USING (table_schema, table_name, constraint_name)
 LEFT OUTER JOIN information_schema.key_column_usage AS kcu
           USING (table_schema, table_name, constraint_name)
           WHERE tc.table_schema IN (%s, %s)
             AND tc.table_name = %s

             UNION

          SELECT 'INDEX' AS constraint_type,
                 id.indexname AS constraint_name,
                 attr.attname AS column_name
            FROM pg_catalog.pg_indexes AS id
      INNER JOIN pg_catalog.pg_index  AS idx
              ON (id.schemaname || '.' || id.indexname)::regclass = idx.indexrelid
 LEFT OUTER JOIN pg_catalog.pg_attribute AS attr
              ON idx.indrelid = attr.attrelid AND attr.attnum = ANY(idx.indkey)
           WHERE id.schemaname IN (%s, %s)
             AND id.tablename = %s
),
by_type AS (
  SELECT constraint_type,
         constraint_name,
         array_agg(column_name ORDER BY column_name) AS columns
    FROM constraints
   WHERE column_name IS NOT NULL
GROUP BY constraint_name, constraint_type
),
by_name AS (
  SELECT array_agg(constraint_type) AS constraints,
         constraint_name,
         columns
    FROM by_type
GROUP BY constraint_name, columns
)

SELECT constraint_name,
       columns,
       'PRIMARY KEY' = ANY(constraints) AS "primary_key",
       'UNIQUE' = ANY(constraints) OR 'PRIMARY KEY' = ANY(constraints) AS "unique",
       CASE WHEN 'FOREIGN KEY' = ANY(constraints) THEN
           (SELECT ARRAY[table_name::text, column_name::text]
                    FROM information_schema.constraint_column_usage ccu
                   WHERE by_name.constraint_name = ccu.constraint_name
                   LIMIT 1)
       END AS "foreign_key",
       'CHECK' = ANY(constraints) AS "check",
       'INDEX' = ANY(constraints) AS "index"
FROM by_name""", [settings.PUBLIC_SCHEMA, schema_name, table_name] * 2)
    columns = [x.name for x in cursor.description]
    return {row[0]: dict(zip(columns, row)) for row in cursor}

def get_index_data(cursor, index_name):

    cursor.execute('''SELECT c.relname AS table_name,
                             n.nspname AS schema_name
                        FROM pg_catalog.pg_class c, pg_catalog.pg_class c2,
                             pg_catalog.pg_index idx, pg_catalog.pg_namespace n
                       WHERE c.oid = idx.indrelid
                         AND idx.indexrelid = c2.oid
                         AND n.oid = c.relnamespace
                         AND n.nspname IN (%s, %s)
                         AND c2.relname = %s
    ''', [settings.PUBLIC_SCHEMA, '__template__', index_name])

    return [table_name for (table_name, schema_name) in cursor.fetchall()]

def group_tokens(parsed):
    grouped = defaultdict(list)
    identifiers = []

    for token in parsed.tokens:
        if token.ttype:
            grouped[token.ttype].append(token.value)
        elif token.get_name():
            identifiers.append(token)

    return grouped, identifiers

def get_table_and_schema(sql, cursor):
    try:
        parsed = sqlparse.parse(sql)[0]
    except IndexError:
        # In the case of a CREATE * FUNCTION that is a plpgsql function, we
        # know we won't be able to parse it. Functions should probably be in
        # the public schema anyway.
        sql_upper = sql.upper()
        if (
            'CREATE FUNCTION' in sql_upper or
            'CREATE OR REPLACE FUNCTION' in sql_upper
        ) and 'LANGUAGE PLPGSQL' in sql_upper:
            return None, None
        raise

    grouped, identifiers = group_tokens(parsed)

    if grouped[DDL] and grouped[DDL][0] in ['CREATE', 'DROP', 'ALTER', 'CREATE OR REPLACE']:
        # We may care about this.
        keywords = grouped[Keyword]
        if 'FUNCTION' in keywords:
            # At this point, I'm not convinced that functions belong anywhere
            # other than in the public schema. Perhaps they should, as that
            # could be a nice way to get different behaviour per-tenant.
            return None, None
        elif 'INDEX' in keywords and grouped[DDL][0] == 'DROP':
            # DROP INDEX does not have a table associated with it.
            # We will have to hit the database to see what tables have
            # an index with that name: we can just use the template/public
            # schemata though.
            return get_index_data(cursor, identifiers[0].get_name())[0], None
        elif ('VIEW' in keywords or 'TABLE' in keywords) and identifiers:
            # We care about identifier 0, which will be the name of the view
            # or table.
            return identifiers[0].get_name(), identifiers[0].get_parent_name()
        elif ('TRIGGER' in keywords or 'INDEX' in keywords) and len(identifiers) > 1:
            # We care about identifier 1, as identifier 0 is the name of the
            # function or index: identifier 1 is the table it refers to.
            return identifiers[1].get_name(), identifiers[1].get_parent_name()

    # We also care about other non-DDL statements, as the implication is that
    # they should apply to every known schema, if we are updating as part of a
    # migration.
    if grouped[DML] and \
       grouped[DML][0] in ['INSERT INTO', 'UPDATE', 'DELETE FROM'] and\
       identifiers:
        return identifiers[0].get_name(), identifiers[0].get_parent_name()

    return None, None

class DatabaseSchemaEditor(schema.DatabaseSchemaEditor):

    def __exit__(self, exc_type, exc_value, traceback):
        # It seems that actions that add stuff to the deferred sql
        # will fire per-schema, so we can end up with multiples.
        # We'll reduce that to a unique list.
        # Can't just do a set, as that may change ordering.
        deferred_sql = []
        for sql in self.deferred_sql:
            if sql not in deferred_sql:
                deferred_sql.append(sql)
        self.deferred_sql = deferred_sql
        return super(DatabaseSchemaEditor, self).__exit__(exc_type, exc_value, traceback)
        # If we manage to rewrite the SQL so it injects schema clauses, then we can remove this override.

    def execute(self, sql, params=None):
        execute = super(DatabaseSchemaEditor, self).execute

        table_name, schema_name = get_table_and_schema(sql, self.connection.cursor())

        # TODO: try to get the apps from current project_state, not global apps.
        if table_name and not schema_name and not is_shared_table(table_name):
            schema_aware_operation.send(
                self.__class__,
                db_table=table_name,
                function=execute,
                args=(sql, params)
            )
            deactivate_schema()
        else:
            execute(sql, params)

    def _constraint_names(self, model, column_names=None, unique=None,
                          primary_key=None, index=None, foreign_key=None,
                          check=None):
        """
        Returns all constraint names matching the columns and conditions
        """
        column_names = list(column_names) if column_names else None
        with self.connection.cursor() as cursor:
            constraints = get_constraints(cursor, model._meta.db_table)
        result = []
        for name, infodict in constraints.items():
            if column_names is None or column_names == infodict['columns']:
                if unique is not None and infodict['unique'] != unique:
                    continue
                if primary_key is not None and infodict['primary_key'] != primary_key:
                    continue
                if index is not None and infodict['index'] != index:
                    continue
                if check is not None and infodict['check'] != check:
                    continue
                if foreign_key is not None and not infodict['foreign_key']:
                    continue
                result.append(name)

        return result
<commit_msg>Update documentation.<commit_after>from __future__ import unicode_literals

from collections import defaultdict

from django.db.backends.postgresql_psycopg2 import schema
from django.conf import settings

import sqlparse
from sqlparse.tokens import DDL, DML, Keyword

from ...schema import deactivate_schema, is_shared_table
from ...signals import schema_aware_operation

def get_constraints(cursor, table_name, schema_name='__template__'):
    """Return all constraints for a given table

    (in either the given schema, or the public schema: the assumption is made
    that constraint names will not be the same in both).

    """
    cursor.execute(''
"""WITH constraints AS (

          SELECT tc.constraint_type,
                 tc.constraint_name,
                 COALESCE(ccu.column_name, kcu.column_name) AS column_name
            FROM information_schema.table_constraints AS tc
 LEFT OUTER JOIN information_schema.constraint_column_usage AS ccu
           USING (table_schema, table_name, constraint_name)
 LEFT OUTER JOIN information_schema.key_column_usage AS kcu
           USING (table_schema, table_name, constraint_name)
           WHERE tc.table_schema IN (%s, %s)
             AND tc.table_name = %s

             UNION

          SELECT 'INDEX' AS constraint_type,
                 id.indexname AS constraint_name,
                 attr.attname AS column_name
            FROM pg_catalog.pg_indexes AS id
      INNER JOIN pg_catalog.pg_index  AS idx
              ON (id.schemaname || '.' || id.indexname)::regclass = idx.indexrelid
 LEFT OUTER JOIN pg_catalog.pg_attribute AS attr
              ON idx.indrelid = attr.attrelid AND attr.attnum = ANY(idx.indkey)
           WHERE id.schemaname IN (%s, %s)
             AND id.tablename = %s
),
by_type AS (
  SELECT constraint_type,
         constraint_name,
         array_agg(column_name ORDER BY column_name) AS columns
    FROM constraints
   WHERE column_name IS NOT NULL
GROUP BY constraint_name, constraint_type
),
by_name AS (
  SELECT array_agg(constraint_type) AS constraints,
         constraint_name,
         columns
    FROM by_type
GROUP BY constraint_name, columns
)

SELECT constraint_name,
       columns,
       'PRIMARY KEY' = ANY(constraints) AS "primary_key",
       'UNIQUE' = ANY(constraints) OR 'PRIMARY KEY' = ANY(constraints) AS "unique",
       CASE WHEN 'FOREIGN KEY' = ANY(constraints) THEN
           (SELECT ARRAY[table_name::text, column_name::text]
                    FROM information_schema.constraint_column_usage ccu
                   WHERE by_name.constraint_name = ccu.constraint_name
                   LIMIT 1)
       END AS "foreign_key",
       'CHECK' = ANY(constraints) AS "check",
       'INDEX' = ANY(constraints) AS "index"
FROM by_name""", [settings.PUBLIC_SCHEMA, schema_name, table_name] * 2)
    columns = [x.name for x in cursor.description]
    return {row[0]: dict(zip(columns, row)) for row in cursor}

def get_index_data(cursor, index_name):

    cursor.execute('''SELECT c.relname AS table_name,
                             n.nspname AS schema_name
                        FROM pg_catalog.pg_class c, pg_catalog.pg_class c2,
                             pg_catalog.pg_index idx, pg_catalog.pg_namespace n
                       WHERE c.oid = idx.indrelid
                         AND idx.indexrelid = c2.oid
                         AND n.oid = c.relnamespace
                         AND n.nspname IN (%s, %s)
                         AND c2.relname = %s
    ''', [settings.PUBLIC_SCHEMA, '__template__', index_name])

    return [table_name for (table_name, schema_name) in cursor.fetchall()]

def group_tokens(parsed):
    grouped = defaultdict(list)
    identifiers = []

    for token in parsed.tokens:
        if token.ttype:
            grouped[token.ttype].append(token.value)
        elif token.get_name():
            identifiers.append(token)

    return grouped, identifiers

def get_table_and_schema(sql, cursor):
    """
    Given an SQL statement, determine what the database object that is being
    operated upon is.

    This logic is quite complex. If you find a case that does not work, please
    submit a bug report (or even better, pull request!)
    """
    try:
        parsed = sqlparse.parse(sql)[0]
    except IndexError:
        # In the case of a CREATE * FUNCTION that is a plpgsql function, we
        # know we won't be able to parse it. Functions should probably be in
        # the public schema anyway. If you had different functions required
        # per tenant, then you'd need to come up with a way to do that.
        sql_upper = sql.upper()
        if (
            'CREATE FUNCTION' in sql_upper or
            'CREATE OR REPLACE FUNCTION' in sql_upper
        ) and 'LANGUAGE PLPGSQL' in sql_upper:
            return None, None
        raise

    grouped, identifiers = group_tokens(parsed)

    if grouped[DDL] and grouped[DDL][0] in ['CREATE', 'DROP', 'ALTER', 'CREATE OR REPLACE']:
        # We may care about this.
        keywords = grouped[Keyword]
        if 'FUNCTION' in keywords:
            # At this point, I'm not convinced that functions belong anywhere
            # other than in the public schema. Perhaps they should, as that
            # could be a nice way to get different behaviour per-tenant.
            return None, None
        elif 'INDEX' in keywords and grouped[DDL][0] == 'DROP':
            # DROP INDEX does not have a table associated with it.
            # We will have to hit the database to see what tables have
            # an index with that name: we can just use the template/public
            # schemata though.
            return get_index_data(cursor, identifiers[0].get_name())[0], None
        elif ('VIEW' in keywords or 'TABLE' in keywords) and identifiers:
            # We care about identifier 0, which will be the name of the view
            # or table.
            return identifiers[0].get_name(), identifiers[0].get_parent_name()
        elif ('TRIGGER' in keywords or 'INDEX' in keywords) and len(identifiers) > 1:
            # We care about identifier 1, as identifier 0 is the name of the
            # function or index: identifier 1 is the table it refers to.
            return identifiers[1].get_name(), identifiers[1].get_parent_name()

    # We also care about other non-DDL statements, as the implication is that
    # they should apply to every known schema, if we are updating as part of a
    # migration.
    if grouped[DML] and \
       grouped[DML][0] in ['INSERT INTO', 'UPDATE', 'DELETE FROM'] and\
       identifiers:
        return identifiers[0].get_name(), identifiers[0].get_parent_name()

    return None, None

class DatabaseSchemaEditor(schema.DatabaseSchemaEditor):
    """
    This Schema Editor alters behaviour in three ways.

    1. Remove duplicates of deferred sql statements. These are
       executed using `self.execute()` anyway, so they will get
       applied to all schemata as appropriate.
    2. Fire a signal during `self.execute()` so that listeners may choose
       to apply this statement to all schemata. This signal only fires for
       objects that are private objects.
    3. Change the mechanism for grabbing constraint names to also look in
       the template schema (instead of just `public`, as is hard-coded in
       the original method).
    """

    def __exit__(self, exc_type, exc_value, traceback):
        # It seems that actions that add stuff to the deferred sql
        # will fire per-schema, so we can end up with multiples.
        # We'll reduce that to a unique list.
        # Can't just do a set, as that may change ordering.
        deferred_sql = []
        for sql in self.deferred_sql:
            if sql not in deferred_sql:
                deferred_sql.append(sql)
        self.deferred_sql = deferred_sql
        return super(DatabaseSchemaEditor, self).__exit__(exc_type, exc_value, traceback)
        # If we manage to rewrite the SQL so it injects schema clauses, then we can remove this override.

    def execute(self, sql, params=None):
        execute = super(DatabaseSchemaEditor, self).execute

        table_name, schema_name = get_table_and_schema(sql, self.connection.cursor())

        # TODO: try to get the apps from current project_state, not global apps.
        if table_name and not schema_name and not is_shared_table(table_name):
            schema_aware_operation.send(
                self.__class__,
                db_table=table_name,
                function=execute,
                args=(sql, params)
            )
            deactivate_schema()
        else:
            execute(sql, params)

    def _constraint_names(self, model, column_names=None, unique=None,
                          primary_key=None, index=None, foreign_key=None,
                          check=None):
        """
        Returns all constraint names matching the columns and conditions
        """
        column_names = list(column_names) if column_names else None
        with self.connection.cursor() as cursor:
            constraints = get_constraints(cursor, model._meta.db_table)
        result = []
        for name, infodict in constraints.items():
            if column_names is None or column_names == infodict['columns']:
                if unique is not None and infodict['unique'] != unique:
                    continue
                if primary_key is not None and infodict['primary_key'] != primary_key:
                    continue
                if index is not None and infodict['index'] != index:
                    continue
                if check is not None and infodict['check'] != check:
                    continue
                if foreign_key is not None and not infodict['foreign_key']:
                    continue
                result.append(name)

        return result
<|endoftext|>


<commit_before>from keras.applications.imagenet_utils import _obtain_input_shape
from keras import backend as K
from keras.layers import Input, Convolution2D, MaxPooling2D, Activation, concatenate, Dropout, warnings
from keras.layers import GlobalAveragePooling2D, GlobalMaxPooling2D
from keras.models import Model
from keras.engine.topology import get_source_inputs
from keras.utils import get_file
from keras.utils import layer_utils

sq1x1 = "squeeze1x1"
exp1x1 = "expand1x1"
exp3x3 = "expand3x3"
relu = "relu_"

WEIGHTS_PATH = "https://github.com/rcmalli/keras-squeezenet/releases/download/v1.0/squeezenet_weights_tf_dim_ordering_tf_kernels.h5"
WEIGHTS_PATH_NO_TOP = "https://github.com/rcmalli/keras-squeezenet/releases/download/v1.0/squeezenet_weights_tf_dim_ordering_tf_kernels_notop.h5"

# Modular function for Fire Node

def fire_module(x, fire_id, squeeze=16, expand=64):
    s_id = 'fire' + str(fire_id) + '/'

    if K.image_data_format() == 'channels_first':
        channel_axis = 1
    else:
        channel_axis = 3

    x = Convolution2D(squeeze, (1, 1), padding='valid', name=s_id + sq1x1)(x)
    x = Activation('relu', name=s_id + relu + sq1x1)(x)

    left = Convolution2D(expand, (1, 1), padding='valid', name=s_id + exp1x1)(x)
    left = Activation('relu', name=s_id + relu + exp1x1)(left)

    right = Convolution2D(expand, (3, 3), padding='same', name=s_id + exp3x3)(x)
    right = Activation('relu', name=s_id + relu + exp3x3)(right)

    x = concatenate([left, right], axis=channel_axis, name=s_id + 'concat')
    return x

# Original SqueezeNet from paper.

def SqueezeNet(include_top=True, weights='imagenet',
               input_tensor=None, input_shape=None,
               pooling=None,
               classes=1000):
    """Instantiates the SqueezeNet architecture.
    """

    if weights not in {'imagenet', None}:
        raise ValueError('The `weights` argument should be either '
                         '`None` (random initialization) or `imagenet` '
                         '(pre-training on ImageNet).')

    if weights == 'imagenet' and classes != 1000:
        raise ValueError('If using `weights` as imagenet with `include_top`'
                         ' as true, `classes` should be 1000')

    input_shape = _obtain_input_shape(input_shape,
                                      default_size=227,
                                      min_size=48,
                                      data_format=K.image_data_format(),
                                      require_flatten=include_top)

    if input_tensor is None:
        img_input = Input(shape=input_shape)
    else:
        if not K.is_keras_tensor(input_tensor):
            img_input = Input(tensor=input_tensor, shape=input_shape)
        else:
            img_input = input_tensor

    x = Convolution2D(64, (3, 3), strides=(2, 2), padding='valid', name='conv1')(img_input)
    x = Activation('relu', name='relu_conv1')(x)
    x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), name='pool1')(x)

    x = fire_module(x, fire_id=2, squeeze=16, expand=64)
    x = fire_module(x, fire_id=3, squeeze=16, expand=64)
    x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), name='pool3')(x)

    x = fire_module(x, fire_id=4, squeeze=32, expand=128)
    x = fire_module(x, fire_id=5, squeeze=32, expand=128)
    x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), name='pool5')(x)

    x = fire_module(x, fire_id=6, squeeze=48, expand=192)
    x = fire_module(x, fire_id=7, squeeze=48, expand=192)
    x = fire_module(x, fire_id=8, squeeze=64, expand=256)
    x = fire_module(x, fire_id=9, squeeze=64, expand=256)

    if include_top:
        # It's not obvious where to cut the network...
        # Could do the 8th or 9th layer... some work recommends cutting earlier layers.

        x = Dropout(0.5, name='drop9')(x)

        x = Convolution2D(classes, (1, 1), padding='valid', name='conv10')(x)
        x = Activation('relu', name='relu_conv10')(x)
        x = GlobalAveragePooling2D()(x)
        x = Activation('softmax', name='loss')(x)
    else:
        if pooling == 'avg':
            x = GlobalAveragePooling2D()(x)
        elif pooling=='max':
            x = GlobalMaxPooling2D()(x)
        elif pooling==None:
            pass
        else:
            raise ValueError("Unknown argument for 'pooling'=" + pooling)

    # Ensure that the model takes into account
    # any potential predecessors of `input_tensor`.
    if input_tensor is not None:
        inputs = get_source_inputs(input_tensor)
    else:
        inputs = img_input

    model = Model(inputs, x, name='squeezenet')

    # load weights
    if weights == 'imagenet':
        if include_top:
            weights_path = get_file('squeezenet_weights_tf_dim_ordering_tf_kernels.h5',
                                    WEIGHTS_PATH,
                                    cache_subdir='models')
        else:
            weights_path = get_file('squeezenet_weights_tf_dim_ordering_tf_kernels_notop.h5',
                                    WEIGHTS_PATH_NO_TOP,
                                    cache_subdir='models')

        model.load_weights(weights_path)
        if K.backend() == 'theano':
            layer_utils.convert_all_kernels_in_model(model)

        if K.image_data_format() == 'channels_first':

            if K.backend() == 'tensorflow':
                warnings.warn('You are using the TensorFlow backend, yet you '
                              'are using the Theano '
                              'image data format convention '
                              '(`image_data_format="channels_first"`). '
                              'For best performance, set '
                              '`image_data_format="channels_last"` in '
                              'your Keras config '
                              'at ~/.keras/keras.json.')
    return model

<commit_msg>Delete squeezenet.py<commit_after><|endoftext|>


<commit_before>#
# Copyright 2012 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pytz
import numbers

from hashlib import md5
from datetime import datetime
from zipline.protocol import (
    DATASOURCE_TYPE,
    Event
)

def hash_args(*args, **kwargs):
    """Define a unique string for any set of representable args."""
    arg_string = '_'.join([str(arg) for arg in args])
    kwarg_string = '_'.join([str(key) + '=' + str(value)
                             for key, value in kwargs.iteritems()])
    combined = ':'.join([arg_string, kwarg_string])

    hasher = md5()
    hasher.update(combined)
    return hasher.hexdigest()

def create_trade(sid, price, amount, datetime, source_id="test_factory"):

    trade = Event()

    trade.source_id = source_id
    trade.type = DATASOURCE_TYPE.TRADE
    trade.sid = sid
    trade.dt = datetime
    trade.price = price
    trade.close = price
    trade.open = price
    trade.low = price * .95
    trade.high = price * 1.05
    trade.volume = amount
    trade.TRANSACTION = None

    return trade

def assert_datasource_protocol(event):
    """Assert that an event meets the protocol for datasource outputs."""

    assert isinstance(event.source_id, basestring)
    assert event.type in DATASOURCE_TYPE

    # Done packets have no dt.
    if not event.type == DATASOURCE_TYPE.DONE:
        assert isinstance(event.dt, datetime)
        assert event.dt.tzinfo == pytz.utc

def assert_trade_protocol(event):
    """Assert that an event meets the protocol for datasource TRADE outputs."""
    assert_datasource_protocol(event)

    assert event.type == DATASOURCE_TYPE.TRADE
    assert isinstance(event.sid, int)
    assert isinstance(event.price, numbers.Real)
    assert isinstance(event.volume, numbers.Integral)
    assert isinstance(event.dt, datetime)

def assert_datasource_unframe_protocol(event):
    """Assert that an event is valid output of zp.DATASOURCE_UNFRAME."""
    assert isinstance(event.source_id, basestring)
    assert event.type in DATASOURCE_TYPE

def assert_sort_protocol(event):
    """Assert that an event is valid input to zp.FEED_FRAME."""
    assert isinstance(event.source_id, basestring)
    assert event.type in DATASOURCE_TYPE

def assert_sort_unframe_protocol(event):
    """Same as above."""
    assert isinstance(event.source_id, basestring)
    assert event.type in DATASOURCE_TYPE

def assert_merge_protocol(tnfm_ids, message):
    """Merge should output an ndict with a field for each id
    in its transform set."""
    assert set(tnfm_ids) == set(message.keys())
<commit_msg>MAINT: Updates copyright year on gens.utils module.<commit_after>#
# Copyright 2013 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pytz
import numbers

from hashlib import md5
from datetime import datetime
from zipline.protocol import (
    DATASOURCE_TYPE,
    Event
)

def hash_args(*args, **kwargs):
    """Define a unique string for any set of representable args."""
    arg_string = '_'.join([str(arg) for arg in args])
    kwarg_string = '_'.join([str(key) + '=' + str(value)
                             for key, value in kwargs.iteritems()])
    combined = ':'.join([arg_string, kwarg_string])

    hasher = md5()
    hasher.update(combined)
    return hasher.hexdigest()

def create_trade(sid, price, amount, datetime, source_id="test_factory"):

    trade = Event()

    trade.source_id = source_id
    trade.type = DATASOURCE_TYPE.TRADE
    trade.sid = sid
    trade.dt = datetime
    trade.price = price
    trade.close = price
    trade.open = price
    trade.low = price * .95
    trade.high = price * 1.05
    trade.volume = amount
    trade.TRANSACTION = None

    return trade

def assert_datasource_protocol(event):
    """Assert that an event meets the protocol for datasource outputs."""

    assert isinstance(event.source_id, basestring)
    assert event.type in DATASOURCE_TYPE

    # Done packets have no dt.
    if not event.type == DATASOURCE_TYPE.DONE:
        assert isinstance(event.dt, datetime)
        assert event.dt.tzinfo == pytz.utc

def assert_trade_protocol(event):
    """Assert that an event meets the protocol for datasource TRADE outputs."""
    assert_datasource_protocol(event)

    assert event.type == DATASOURCE_TYPE.TRADE
    assert isinstance(event.sid, int)
    assert isinstance(event.price, numbers.Real)
    assert isinstance(event.volume, numbers.Integral)
    assert isinstance(event.dt, datetime)

def assert_datasource_unframe_protocol(event):
    """Assert that an event is valid output of zp.DATASOURCE_UNFRAME."""
    assert isinstance(event.source_id, basestring)
    assert event.type in DATASOURCE_TYPE

def assert_sort_protocol(event):
    """Assert that an event is valid input to zp.FEED_FRAME."""
    assert isinstance(event.source_id, basestring)
    assert event.type in DATASOURCE_TYPE

def assert_sort_unframe_protocol(event):
    """Same as above."""
    assert isinstance(event.source_id, basestring)
    assert event.type in DATASOURCE_TYPE

def assert_merge_protocol(tnfm_ids, message):
    """Merge should output an ndict with a field for each id
    in its transform set."""
    assert set(tnfm_ids) == set(message.keys())
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-

import time
import operator
import argparse
import sys
import os
import pickle
import json

import dbconf
from utils import *
import ipaddr

import getpass
import smtplib
from email.mime.text import MIMEText

import subprocess

import time
import syslog

class MySysLogger():
	def __init__(self, seconds, annotation, quiet = False):
		self.seconds = seconds
		self.annotation = annotation
		self.total = 0
		self.partial = 0
		self.time = self.start = time.time()
		self.quiet = quiet
		if not self.quiet:
			print "Starting syslogger"

	def run(self):
		now = time.time()
		elapsed = now - self.time
		if (elapsed < self.seconds):
			return

		self.time = time.time()
		if self.seconds > 0:
			line = "partial %0.1f conn/sec %d (%.2f seconds), total %0.1f conn/sec %ld" % (self.partial/elapsed, self.partial, elapsed, self.total/(now-self.start), self.total)
			if not self.quiet:
				print line
			syslog.syslog(syslog.LOG_INFO, line)

		if self.annotation > 0:
			try:
				history = pickle.load(open("/tmp/access_history.p", "rb" ))
			except:
				 history = []
			history.append([int(now), int(self.partial/elapsed)])
			history = history[-self.annotation:]

			pickle.dump(history, open("/tmp/access_history.p", "wb" ))

			current = json.dumps(history)
			store_annotation('access_history', current)

		self.partial = 0

	def increment(self):
		self.partial += 1
		self.total += 1

def openfile(filename):
	logfile = open(filename,"rU")
	logfile.seek(0,2)
	return logfile

def analyze(logfile):
	global configuration, syslogger

	loglines = follow_log(logfile, configuration.showbad)
	total = counter = empties = 0

	""" Number of previous periods to store """
	low_history = 4

	ip_scripts = {}
	ip_users = {}
	ip_counter = {}
	ip_warned = set()
	ip_banned = set()
	ip_periods = []

	for log in loglines:
		if log:
			empties = 0
			counter += 1
			total += 1

			if syslogger:
				syslogger.increment()

			if log['ip'] in ip_counter:
				ip_counter[log['ip']] += 1
			else:
				ip_counter[log['ip']] = 1
				ip_users[log['ip']] = set()
			ip_users[log['ip']].add(log['user'])

			if log['_blocked']:
				ip_banned.add(log['ip'])

		else:
			ip_exceeded = set()
			ip_low_exceeded = set()
			ip_high_exceeded = set()
			to_ban = set()
			rate = counter/configuration.period
			ip_periods_seen = {}

			if total > 0:
				print rate, "c/sec"
			if empties > 2:
				return total
			if counter == 0:
				empties += 1
			else:
				#sorted_ips = sorted(ip_counter.iteritems(), key=operator.itemgetter(1), reverse=True)
				sorted_ips = sorted(ip_counter.items(), key=lambda x:x[1], reverse=True)
				i = 0
				max_len = len(sorted_ips)
				max_count = configuration.rate * configuration.period
				low_count = max_count * 0.6
				high_count = max_count * 2
				""" Give one aditional connection per second to different users """
				while i < max_len and sorted_ips[i][1] >= low_count + (len(ip_users[sorted_ips[i][0]])-1)*configuration.period:
					ip = sorted_ips[i][0]

					""" Never block private IPs """
					ip_addr = ipaddr.IPNetwork(ip)
					if ip_addr.is_private or ip_addr.is_loopback or ip_addr.is_link_local:
						i += 1
						continue

					count = sorted_ips[i][1]
					additional = (len(ip_users[sorted_ips[i][0]])-1)*configuration.period
					if ip not in ip_banned:
						ip_low_exceeded.add(ip)
						if count > max_count + additional:
							ip_exceeded.add(ip)
						if count > high_count + additional:
							ip_high_exceeded.add(ip)

					i += 1

				if not configuration.q:
					print "Top IPs"
					print "    %5s %4s %s" % ("Conn.", "c/s", "IP")
					r = min(len(sorted_ips), 10)
					for i in range(r):
						ip, conns = sorted_ips[i]

						if ip in ip_high_exceeded: print "H",
						elif ip in ip_exceeded: print "+",
						elif ip in ip_low_exceeded: print "-",
						else: print " ",

						if ip in ip_warned: print "*",
						else: print " ",

						print "%5d %4d %s" % (conns, conns/configuration.period, ip),
						print ','.join([x for x in sorted(ip_users[ip], key=str.lower)])
					print

				if configuration.ban:
					to_ban = ip_high_exceeded
					for ip in ip_high_exceeded:
						ip_periods_seen[ip] = 1

					for ip in [x for x in ip_exceeded if x in ip_warned and x not in ip_banned]:
						to_ban.add(ip)
						ip_periods_seen[ip] = 2

					low_intersection = set.intersection(*ip_periods)
					for ip in [x for x in ip_low_exceeded if x in low_intersection and x not in ip_banned]:
						to_ban.add(ip)
						ip_periods_seen[ip] = low_history + 1

					for ip in to_ban.copy():
						rate = ip_counter[ip]/configuration.period

						if len(ip_users[ip]) > 1 or "-" not in ip_users[ip]:
							seconds = 3600 * 2
						else:
							seconds = 86400 * 2
						""" Increase de seconds according to how much it exceeded """
						seconds = int(seconds * ip_counter[ip]/float(max_count))

						users = ','.join([x for x in sorted(ip_users[ip], key=str.lower)])
						reason = "Automatic (%s) %d conn/sec during %d seconds, blocked for %02d:%02d:%02d hs" % \
								(users, rate, ip_periods_seen[ip]*configuration.period, seconds/3600, (seconds%3600)/60, seconds%60)
						# reason = "Automatic (" + ','.join([x for x in sorted(ip_users[ip], key=str.lower)]) + ") " + rate + " conn/sec during " + str(ip_periods_seen[ip]*configuration.period) + " seconds, blocked for " + str(seconds/3600) + ":" + str((seconds%3600)/60) + str(seconds%60)
						if not ban_ip(ip, reason, seconds):
							to_ban.remove(ip) # Don't consider it as banned

			ip_warned = ip_exceeded;
			ip_banned = to_ban;

			ip_periods.append(ip_low_exceeded)
			if len(ip_periods) > low_history:
				del(ip_periods[0])

			ip_scripts = {}
			ip_users = {}
			ip_counter = {}
			counter = 0
			if syslogger:
				syslogger.run()
			time.sleep(configuration.period)
	return total

def ban_ip(ip, reason, time):
	global configuration

	if not configuration.dry:
		try:
			c = DBM.cursor('update')
			c.execute("REPLACE INTO bans (ban_type, ban_text, ban_comment, ban_expire) VALUES (%s, %s, %s, date_add(now(), interval %s second))", ("noaccess", ip, reason, time))
			c.close()
			DBM.commit()
			DBM.close('update')
		except Exception as e:
			DBM.close('update')
			syslog.syslog(syslog.LOG_INFO, "Error in DB blocking IP: " + ip + " " + unicode(e))
			return False

	print "BAN:", ip, reason
	syslog.syslog(syslog.LOG_INFO, "Block IP: " + ip + " " + reason)

	if configuration.mail:
		""" Generate a report """
		try:
			summary = os.path.dirname(os.path.abspath(__file__)) + "/" + "summary_access.py"
			p = subprocess.Popen([summary, ip, "-M", "1"], stdout=subprocess.PIPE)
			(report, err) = p.communicate()
		except Exception as e:
			report = unicode(e)

		try:
			msg = "BANNED IP: " + ip +"\nReason: " + reason + "\n\nSUMMARY REPORT LAST MINUTE:\n" + unicode(report)
			msg = MIMEText(msg.encode('utf-8'), 'plain', 'utf-8')
			msg['Subject'] = "Automatic DoS ban"
			msg['From'] = getpass.getuser()
			msg['To'] = configuration.mail
			s = smtplib.SMTP('localhost')
			s.sendmail(getpass.getuser(), configuration.mail, msg.as_string())
			s.quit()
		except Exception as e:
			syslog.syslog(syslog.LOG_INFO, "Error sending email: " + reason + " (" + unicode(e) + ")")

	return True

if __name__ == '__main__':
	parser = argparse.ArgumentParser()
	parser.add_argument("--period", "-p", type=int, default=10, help="Seconds between analysis, default 10")
	parser.add_argument("--ban", "-b", action="store_true", help="Ban IPs")
	parser.add_argument("-q", action="store_true", help="Quiet mode")
	parser.add_argument("--syslog", "-s", type=int, help="Write a summary every x seconds")
	parser.add_argument("--annotation", "-A", type=int, help="If syslog enabled, store in meneame.annotation the last X values")
	parser.add_argument("--dry", "-d", action="store_true", help="Do not store the ban in the DB")
	parser.add_argument("--rate", "-r", type=int, default=15, help="Set the max number of connections per second, default 15")
	parser.add_argument("--logfile", "-l", default="/var/log/meneame_access.log", help="Logfile pathname, default /var/log/meneame_access.log")
	parser.add_argument("--mail", "-m", help="Send email to this address when an IP is banned")
	parser.add_argument("--showbad", action="store_true", help="Report bad format lines")
	configuration = parser.parse_args()

	if configuration.q:
		f = open(os.devnull, 'w')
		sys.stdout = f

	syslog.openlog("meneame", syslog.LOG_NDELAY, syslog.LOG_USER)
	counter = 0
	if configuration.syslog > 0:
		syslogger = MySysLogger(configuration.syslog, configuration.annotation, configuration.q)
	else:
		syslogger = None

	while True:
		try:
			try:
				logfile = openfile(configuration.logfile)
			except (IOError), e:
				print >> sys.stderr, e
				syslog.syslog(syslog.LOG_INFO, "check_access IOError: " + e)
				exit(1)

			counter += 1
			lines = analyze(logfile)
			mess = "check_access, end: %d, %d, restarting in 1 second" % (counter, lines)
			if not configuration.q:
				print mess
			else:
				syslog.syslog(syslog.LOG_INFO, mess)
			time.sleep(1)
		except (KeyboardInterrupt), e:
			print
			exit(0)

<commit_msg>Check is a ban exists, if so, don't send the email again.<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-

import time
import operator
import argparse
import sys
import os
import pickle
import json

import dbconf
from utils import *
import ipaddr

import getpass
import smtplib
from email.mime.text import MIMEText

import subprocess

import time
import syslog

class MySysLogger():
	def __init__(self, seconds, annotation, quiet = False):
		self.seconds = seconds
		self.annotation = annotation
		self.total = 0
		self.partial = 0
		self.time = self.start = time.time()
		self.quiet = quiet
		if not self.quiet:
			print "Starting syslogger"

	def run(self):
		now = time.time()
		elapsed = now - self.time
		if (elapsed < self.seconds):
			return

		self.time = time.time()
		if self.seconds > 0:
			line = "partial %0.1f conn/sec %d (%.2f seconds), total %0.1f conn/sec %ld" % (self.partial/elapsed, self.partial, elapsed, self.total/(now-self.start), self.total)
			if not self.quiet:
				print line
			syslog.syslog(syslog.LOG_INFO, line)

		if self.annotation > 0:
			try:
				history = pickle.load(open("/tmp/access_history.p", "rb" ))
			except:
				 history = []
			history.append([int(now), int(self.partial/elapsed)])
			history = history[-self.annotation:]

			pickle.dump(history, open("/tmp/access_history.p", "wb" ))

			current = json.dumps(history)
			store_annotation('access_history', current)

		self.partial = 0

	def increment(self):
		self.partial += 1
		self.total += 1

def openfile(filename):
	logfile = open(filename,"rU")
	logfile.seek(0,2)
	return logfile

def analyze(logfile):
	global configuration, syslogger

	loglines = follow_log(logfile, configuration.showbad)
	total = counter = empties = 0

	""" Number of previous periods to store """
	low_history = 4

	ip_scripts = {}
	ip_users = {}
	ip_counter = {}
	ip_warned = set()
	ip_banned = set()
	ip_periods = []

	for log in loglines:
		if log:
			empties = 0
			counter += 1
			total += 1

			if syslogger:
				syslogger.increment()

			if log['ip'] in ip_counter:
				ip_counter[log['ip']] += 1
			else:
				ip_counter[log['ip']] = 1
				ip_users[log['ip']] = set()
			ip_users[log['ip']].add(log['user'])

			if log['_blocked']:
				ip_banned.add(log['ip'])

		else:
			ip_exceeded = set()
			ip_low_exceeded = set()
			ip_high_exceeded = set()
			to_ban = set()
			rate = counter/configuration.period
			ip_periods_seen = {}

			if total > 0:
				print rate, "c/sec"
			if empties > 2:
				return total
			if counter == 0:
				empties += 1
			else:
				#sorted_ips = sorted(ip_counter.iteritems(), key=operator.itemgetter(1), reverse=True)
				sorted_ips = sorted(ip_counter.items(), key=lambda x:x[1], reverse=True)
				i = 0
				max_len = len(sorted_ips)
				max_count = configuration.rate * configuration.period
				low_count = max_count * 0.6
				high_count = max_count * 2
				""" Give one aditional connection per second to different users """
				while i < max_len and sorted_ips[i][1] >= low_count + (len(ip_users[sorted_ips[i][0]])-1)*configuration.period:
					ip = sorted_ips[i][0]

					""" Never block private IPs """
					ip_addr = ipaddr.IPNetwork(ip)
					if ip_addr.is_private or ip_addr.is_loopback or ip_addr.is_link_local:
						i += 1
						continue

					count = sorted_ips[i][1]
					additional = (len(ip_users[sorted_ips[i][0]])-1)*configuration.period
					if ip not in ip_banned:
						ip_low_exceeded.add(ip)
						if count > max_count + additional:
							ip_exceeded.add(ip)
						if count > high_count + additional:
							ip_high_exceeded.add(ip)

					i += 1

				if not configuration.q:
					print "Top IPs"
					print "    %5s %4s %s" % ("Conn.", "c/s", "IP")
					r = min(len(sorted_ips), 10)
					for i in range(r):
						ip, conns = sorted_ips[i]

						if ip in ip_high_exceeded: print "H",
						elif ip in ip_exceeded: print "+",
						elif ip in ip_low_exceeded: print "-",
						else: print " ",

						if ip in ip_warned: print "*",
						else: print " ",

						print "%5d %4d %s" % (conns, conns/configuration.period, ip),
						print ','.join([x for x in sorted(ip_users[ip], key=str.lower)])
					print

				if configuration.ban:
					to_ban = ip_high_exceeded
					for ip in ip_high_exceeded:
						ip_periods_seen[ip] = 1

					for ip in [x for x in ip_exceeded if x in ip_warned and x not in ip_banned]:
						to_ban.add(ip)
						ip_periods_seen[ip] = 2

					low_intersection = set.intersection(*ip_periods)
					for ip in [x for x in ip_low_exceeded if x in low_intersection and x not in ip_banned]:
						to_ban.add(ip)
						ip_periods_seen[ip] = low_history + 1

					for ip in to_ban.copy():
						rate = ip_counter[ip]/configuration.period

						if len(ip_users[ip]) > 1 or "-" not in ip_users[ip]:
							seconds = 3600 * 2
						else:
							seconds = 86400 * 2
						""" Increase de seconds according to how much it exceeded """
						seconds = int(seconds * ip_counter[ip]/float(max_count))

						users = ','.join([x for x in sorted(ip_users[ip], key=str.lower)])
						reason = "Automatic (%s) %d conn/sec during %d seconds, blocked for %02d:%02d:%02d hs" % \
								(users, rate, ip_periods_seen[ip]*configuration.period, seconds/3600, (seconds%3600)/60, seconds%60)
						# reason = "Automatic (" + ','.join([x for x in sorted(ip_users[ip], key=str.lower)]) + ") " + rate + " conn/sec during " + str(ip_periods_seen[ip]*configuration.period) + " seconds, blocked for " + str(seconds/3600) + ":" + str((seconds%3600)/60) + str(seconds%60)
						if not ban_ip(ip, reason, seconds):
							to_ban.remove(ip) # Don't consider it as banned

			ip_warned = ip_exceeded;
			ip_banned = to_ban;

			ip_periods.append(ip_low_exceeded)
			if len(ip_periods) > low_history:
				del(ip_periods[0])

			ip_scripts = {}
			ip_users = {}
			ip_counter = {}
			counter = 0
			if syslogger:
				syslogger.run()
			time.sleep(configuration.period)
	return total

def ban_ip(ip, reason, time):
	global configuration

	if not configuration.dry:
		try:
			c = DBM.cursor('update')
			c.execute("select ban_text from bans where ban_type = %s AND ban_text = %s AND (ban_expire IS null OR ban_expire > now())", ('noaccess', ip))
			exists = c.rowcount

			if not exists:
				c.execute("REPLACE INTO bans (ban_type, ban_text, ban_comment, ban_expire) VALUES (%s, %s, %s, date_add(now(), interval %s second))", ("noaccess", ip, reason, time))

			c.close()
			DBM.commit()
			DBM.close('update')

			if exists:
				return True

		except Exception as e:
			DBM.close('update')
			syslog.syslog(syslog.LOG_INFO, "Error in DB blocking IP: " + ip + " " + unicode(e))
			return False

	print "BAN:", ip, reason
	syslog.syslog(syslog.LOG_INFO, "Block IP: " + ip + " " + reason)

	if configuration.mail:
		""" Generate a report """
		try:
			summary = os.path.dirname(os.path.abspath(__file__)) + "/" + "summary_access.py"
			p = subprocess.Popen([summary, ip, "-M", "1"], stdout=subprocess.PIPE)
			(report, err) = p.communicate()
		except Exception as e:
			report = unicode(e)

		try:
			msg = "BANNED IP: " + ip +"\nReason: " + reason + "\n\nSUMMARY REPORT LAST MINUTE:\n" + unicode(report)
			msg = MIMEText(msg.encode('utf-8'), 'plain', 'utf-8')
			msg['Subject'] = "Automatic DoS ban"
			msg['From'] = getpass.getuser()
			msg['To'] = configuration.mail
			s = smtplib.SMTP('localhost')
			s.sendmail(getpass.getuser(), configuration.mail, msg.as_string())
			s.quit()
		except Exception as e:
			syslog.syslog(syslog.LOG_INFO, "Error sending email: " + reason + " (" + unicode(e) + ")")

	return True

if __name__ == '__main__':
	parser = argparse.ArgumentParser()
	parser.add_argument("--period", "-p", type=int, default=10, help="Seconds between analysis, default 10")
	parser.add_argument("--ban", "-b", action="store_true", help="Ban IPs")
	parser.add_argument("-q", action="store_true", help="Quiet mode")
	parser.add_argument("--syslog", "-s", type=int, help="Write a summary every x seconds")
	parser.add_argument("--annotation", "-A", type=int, help="If syslog enabled, store in meneame.annotation the last X values")
	parser.add_argument("--dry", "-d", action="store_true", help="Do not store the ban in the DB")
	parser.add_argument("--rate", "-r", type=int, default=15, help="Set the max number of connections per second, default 15")
	parser.add_argument("--logfile", "-l", default="/var/log/meneame_access.log", help="Logfile pathname, default /var/log/meneame_access.log")
	parser.add_argument("--mail", "-m", help="Send email to this address when an IP is banned")
	parser.add_argument("--showbad", action="store_true", help="Report bad format lines")
	configuration = parser.parse_args()

	if configuration.q:
		f = open(os.devnull, 'w')
		sys.stdout = f

	syslog.openlog("meneame", syslog.LOG_NDELAY, syslog.LOG_USER)
	counter = 0
	if configuration.syslog > 0:
		syslogger = MySysLogger(configuration.syslog, configuration.annotation, configuration.q)
	else:
		syslogger = None

	while True:
		try:
			try:
				logfile = openfile(configuration.logfile)
			except (IOError), e:
				print >> sys.stderr, e
				syslog.syslog(syslog.LOG_INFO, "check_access IOError: " + e)
				exit(1)

			counter += 1
			lines = analyze(logfile)
			mess = "check_access, end: %d, %d, restarting in 1 second" % (counter, lines)
			if not configuration.q:
				print mess
			else:
				syslog.syslog(syslog.LOG_INFO, mess)
			time.sleep(1)
		except (KeyboardInterrupt), e:
			print
			exit(0)

<|endoftext|>


<commit_before>
import logging

import sys
from ..templates import prepare_service_stage
from ..utils import calculate_hash
from ..utils import (
    get_tender_data,
    get_latest_bid_for_bidder,
    patch_tender_data
)
from ..systemd_msgs_ids import(
    AUCTION_WORKER_API_HANDLE,
    AUCTION_WORKER_SERVICE,
    AUCTION_WORKER_API
)
from barbecue import calculate_coeficient

MULTILINGUAL_FIELDS = ["title", "description"]
ADDITIONAL_LANGUAGES = ["ru", "en"]
ROUNDS = 3
logger = logging.getLogger('Auction Worker')

def get_auction_info(self, prepare=False):
    if not self.debug:
        if prepare:
            self._auction_data = get_tender_data(
                self.tender_url,
                request_id=self.request_id,
                session=self.session
            )
        else:
            self._auction_data = {'data': {}}
        auction_data = get_tender_data(
            self.tender_url + '/auction',
            user=self.worker_defaults["TENDERS_API_TOKEN"],
            request_id=self.request_id,
            session=self.session
        )
        if auction_data:
            self._auction_data['data'].update(auction_data['data'])
            self.startDate = self.convert_datetime(self._auction_data['data']['auctionPeriod']['startDate'])
            del auction_data
        else:
            self.get_auction_document()
            if self.auction_document:
                self.auction_document["current_stage"] = -100
                self.save_auction_document()
                logger.warning("Cancel auction: {}".format(
                    self.auction_doc_id
                ), extra={"JOURNAL_REQUEST_ID": self.request_id,
                          "MESSAGE_ID": AUCTION_WORKER_API_HANDLE})
            else:
                logger.error("Auction {} not exists".format(
                    self.auction_doc_id
                ), extra={"JOURNAL_REQUEST_ID": self.request_id,
                          "MESSAGE_ID": AUCTION_WORKER_API_HANDLE})
            sys.exit(1)
    self.bidders_count = len(self._auction_data["data"]["bids"])
    logger.info("Bidders count: {}".format(self.bidders_count),
                extra={"JOURNAL_REQUEST_ID": self.request_id,
                       "MESSAGE_ID": AUCTION_WORKER_SERVICE})
    self.rounds_stages = []
    for stage in range((self.bidders_count + 1) * ROUNDS + 1):
        if (stage + self.bidders_count) % (self.bidders_count + 1) == 0:
            self.rounds_stages.append(stage)
    self.bidders = [bid["id"] for bid in self._auction_data["data"]["bids"]]
    self.mapping = {}
    self.startDate = self.convert_datetime(
        self._auction_data['data']['auctionPeriod']['startDate']
    )
    if "features" in self._auction_data["data"]:
        self.features = self._auction_data["data"]["features"]

    if not prepare:
        self.bidders_data = []
        if self.features:
            # TODO: remove bidders_features, use bidders_coeficient instead
            self.bidders_features = {}
            self.bidders_coeficient = {}
            self.features = self._auction_data["data"]["features"]
            for bid in self._auction_data["data"]["bids"]:
                self.bidders_features[bid["id"]] = bid["parameters"]
                self.bidders_coeficient[bid["id"]] = calculate_coeficient(self.features, bid["parameters"])
        else:
            self.bidders_features = None
            self.features = None

        for bid in self._auction_data['data']['bids']:
            self.bidders_data.append({
                'id': bid['id'],
                'date': bid['date'],
                'value': bid['value']
            })
            self.bidders_features[bid["id"]] = bid["parameters"]
            self.bidders_coeficient[bid["id"]] = calculate_coeficient(self.features, bid["parameters"])
        self.bidders_count = len(self.bidders_data)

        for index, uid in enumerate(self.bidders_data):
            self.mapping[self.bidders_data[index]['id']] = str(index + 1)

def prepare_auction_document(self):
    self.auction_document.update(
        {"_id": self.auction_doc_id,
         "stages": [],
         "tenderID": self._auction_data["data"].get("tenderID", ""),
         "TENDERS_API_VERSION": self.worker_defaults["TENDERS_API_VERSION"],
         "initial_bids": [],
         "current_stage": -1,
         "results": [],
         "minimalStep": self._auction_data["data"].get("minimalStep", {}),
         "procuringEntity": self._auction_data["data"].get("procuringEntity", {}),
         "items": self._auction_data["data"].get("items", []),
         "value": self._auction_data["data"].get("value", {})}
    )
    if self.features:
        self.auction_document["auction_type"] = "meat"
    else:
        self.auction_document["auction_type"] = "default"

    for key in MULTILINGUAL_FIELDS:
        for lang in ADDITIONAL_LANGUAGES:
            lang_key = "{}_{}".format(key, lang)
            if lang_key in self._auction_data["data"]:
                self.auction_document[lang_key] = self._auction_data["data"][lang_key]
        self.auction_document[key] = self._auction_data["data"].get(key, "")

    self.auction_document['stages'].append(
        prepare_service_stage(
            start=self.startDate.isoformat(),
            type="pause"
        )
    )

    return self.auction_document

def prepare_auction_and_participation_urls(self):
    auction_url = self.worker_defaults["AUCTIONS_URL"].format(
        auction_id=self.tender_id
    )
    patch_data = {"data": {"auctionUrl": auction_url, "bids": []}}
    for bid in self._auction_data["data"]["bids"]:
        participation_url = self.worker_defaults["AUCTIONS_URL"].format(
            auction_id=self.auction_doc_id
        )
        participation_url += '/login?bidder_id={}&hash={}'.format(
            bid["id"],
            calculate_hash(bid["id"], self.worker_defaults["HASH_SECRET"])
        )
        patch_data['data']['bids'].append(
            {"participationUrl": participation_url,
             "id": bid["id"]}
        )
    return patch_data

def post_results_data(self):
    all_bids = self.auction_document["results"]
    logger.info(
        "Approved data: {}".format(all_bids),
        extra={"JOURNAL_REQUEST_ID": self.request_id,
               "MESSAGE_ID": AUCTION_WORKER_API}
    )

    for index, bid_info in enumerate(self._auction_data["data"]["bids"]):
        auction_bid_info = get_latest_bid_for_bidder(all_bids, bid_info["id"])
        self._auction_data["data"]["bids"][index]["value"]["amount"] = auction_bid_info["amount"]
        self._auction_data["data"]["bids"][index]["date"] = auction_bid_info["time"]

    data = {'data': {'bids': self._auction_data["data"]['bids']}}
    return patch_tender_data(
        self.tender_url + '/auction', data=data,
        user=self.worker_defaults["TENDERS_API_TOKEN"],
        method='post',
        request_id=self.request_id, session=self.session
    )

def announce_results_data(self, results):
    bids_information = dict([(bid["id"], bid["tenderers"])
                             for bid in results["data"]["bids"]])
    for section in ['initial_bids', 'stages', 'results']:
        for index, stage in enumerate(self.auction_document[section]):
            if 'bidder_id' in stage and stage['bidder_id'] in bids_information:
                self.auction_document[section][index]["label"]["uk"] = bids_information[stage['bidder_id']][0]["name"]
                self.auction_document[section][index]["label"]["ru"] = bids_information[stage['bidder_id']][0]["name"]
                self.auction_document[section][index]["label"]["en"] = bids_information[stage['bidder_id']][0]["name"]
    return bids_information
<commit_msg>Fix simple tender auction<commit_after>
import logging

import sys
from ..templates import prepare_service_stage
from ..utils import calculate_hash
from ..utils import (
    get_tender_data,
    get_latest_bid_for_bidder,
    patch_tender_data
)
from ..systemd_msgs_ids import(
    AUCTION_WORKER_API_HANDLE,
    AUCTION_WORKER_SERVICE,
    AUCTION_WORKER_API
)
from barbecue import calculate_coeficient

MULTILINGUAL_FIELDS = ["title", "description"]
ADDITIONAL_LANGUAGES = ["ru", "en"]
ROUNDS = 3
logger = logging.getLogger('Auction Worker')

def get_auction_info(self, prepare=False):
    if not self.debug:
        if prepare:
            self._auction_data = get_tender_data(
                self.tender_url,
                request_id=self.request_id,
                session=self.session
            )
        else:
            self._auction_data = {'data': {}}
        auction_data = get_tender_data(
            self.tender_url + '/auction',
            user=self.worker_defaults["TENDERS_API_TOKEN"],
            request_id=self.request_id,
            session=self.session
        )
        if auction_data:
            self._auction_data['data'].update(auction_data['data'])
            self.startDate = self.convert_datetime(self._auction_data['data']['auctionPeriod']['startDate'])
            del auction_data
        else:
            self.get_auction_document()
            if self.auction_document:
                self.auction_document["current_stage"] = -100
                self.save_auction_document()
                logger.warning("Cancel auction: {}".format(
                    self.auction_doc_id
                ), extra={"JOURNAL_REQUEST_ID": self.request_id,
                          "MESSAGE_ID": AUCTION_WORKER_API_HANDLE})
            else:
                logger.error("Auction {} not exists".format(
                    self.auction_doc_id
                ), extra={"JOURNAL_REQUEST_ID": self.request_id,
                          "MESSAGE_ID": AUCTION_WORKER_API_HANDLE})
            sys.exit(1)
    self.bidders_count = len(self._auction_data["data"]["bids"])
    logger.info("Bidders count: {}".format(self.bidders_count),
                extra={"JOURNAL_REQUEST_ID": self.request_id,
                       "MESSAGE_ID": AUCTION_WORKER_SERVICE})
    self.rounds_stages = []
    for stage in range((self.bidders_count + 1) * ROUNDS + 1):
        if (stage + self.bidders_count) % (self.bidders_count + 1) == 0:
            self.rounds_stages.append(stage)
    self.bidders = [bid["id"] for bid in self._auction_data["data"]["bids"]]
    self.mapping = {}
    self.startDate = self.convert_datetime(
        self._auction_data['data']['auctionPeriod']['startDate']
    )
    if "features" in self._auction_data["data"]:
        self.features = self._auction_data["data"]["features"]

    if not prepare:
        self.bidders_data = []
        if self.features:
            # TODO: remove bidders_features, use bidders_coeficient instead
            self.bidders_features = {}
            self.bidders_coeficient = {}
            self.features = self._auction_data["data"]["features"]
            for bid in self._auction_data["data"]["bids"]:
                self.bidders_features[bid["id"]] = bid["parameters"]
                self.bidders_coeficient[bid["id"]] = calculate_coeficient(self.features, bid["parameters"])
        else:
            self.bidders_features = None
            self.features = None

        for bid in self._auction_data['data']['bids']:
            self.bidders_data.append({
                'id': bid['id'],
                'date': bid['date'],
                'value': bid['value']
            })
            if self.features:
                self.bidders_features[bid["id"]] = bid["parameters"]
                self.bidders_coeficient[bid["id"]] = calculate_coeficient(self.features, bid["parameters"])
        self.bidders_count = len(self.bidders_data)

        for index, uid in enumerate(self.bidders_data):
            self.mapping[self.bidders_data[index]['id']] = str(index + 1)

def prepare_auction_document(self):
    self.auction_document.update(
        {"_id": self.auction_doc_id,
         "stages": [],
         "tenderID": self._auction_data["data"].get("tenderID", ""),
         "TENDERS_API_VERSION": self.worker_defaults["TENDERS_API_VERSION"],
         "initial_bids": [],
         "current_stage": -1,
         "results": [],
         "minimalStep": self._auction_data["data"].get("minimalStep", {}),
         "procuringEntity": self._auction_data["data"].get("procuringEntity", {}),
         "items": self._auction_data["data"].get("items", []),
         "value": self._auction_data["data"].get("value", {})}
    )
    if self.features:
        self.auction_document["auction_type"] = "meat"
    else:
        self.auction_document["auction_type"] = "default"

    for key in MULTILINGUAL_FIELDS:
        for lang in ADDITIONAL_LANGUAGES:
            lang_key = "{}_{}".format(key, lang)
            if lang_key in self._auction_data["data"]:
                self.auction_document[lang_key] = self._auction_data["data"][lang_key]
        self.auction_document[key] = self._auction_data["data"].get(key, "")

    self.auction_document['stages'].append(
        prepare_service_stage(
            start=self.startDate.isoformat(),
            type="pause"
        )
    )

    return self.auction_document

def prepare_auction_and_participation_urls(self):
    auction_url = self.worker_defaults["AUCTIONS_URL"].format(
        auction_id=self.tender_id
    )
    patch_data = {"data": {"auctionUrl": auction_url, "bids": []}}
    for bid in self._auction_data["data"]["bids"]:
        participation_url = self.worker_defaults["AUCTIONS_URL"].format(
            auction_id=self.auction_doc_id
        )
        participation_url += '/login?bidder_id={}&hash={}'.format(
            bid["id"],
            calculate_hash(bid["id"], self.worker_defaults["HASH_SECRET"])
        )
        patch_data['data']['bids'].append(
            {"participationUrl": participation_url,
             "id": bid["id"]}
        )
    return patch_data

def post_results_data(self):
    all_bids = self.auction_document["results"]
    logger.info(
        "Approved data: {}".format(all_bids),
        extra={"JOURNAL_REQUEST_ID": self.request_id,
               "MESSAGE_ID": AUCTION_WORKER_API}
    )

    for index, bid_info in enumerate(self._auction_data["data"]["bids"]):
        auction_bid_info = get_latest_bid_for_bidder(all_bids, bid_info["id"])
        self._auction_data["data"]["bids"][index]["value"]["amount"] = auction_bid_info["amount"]
        self._auction_data["data"]["bids"][index]["date"] = auction_bid_info["time"]

    data = {'data': {'bids': self._auction_data["data"]['bids']}}
    return patch_tender_data(
        self.tender_url + '/auction', data=data,
        user=self.worker_defaults["TENDERS_API_TOKEN"],
        method='post',
        request_id=self.request_id, session=self.session
    )

def announce_results_data(self, results):
    bids_information = dict([(bid["id"], bid["tenderers"])
                             for bid in results["data"]["bids"]])
    for section in ['initial_bids', 'stages', 'results']:
        for index, stage in enumerate(self.auction_document[section]):
            if 'bidder_id' in stage and stage['bidder_id'] in bids_information:
                self.auction_document[section][index]["label"]["uk"] = bids_information[stage['bidder_id']][0]["name"]
                self.auction_document[section][index]["label"]["ru"] = bids_information[stage['bidder_id']][0]["name"]
                self.auction_document[section][index]["label"]["en"] = bids_information[stage['bidder_id']][0]["name"]
    return bids_information
<|endoftext|>