<commit_before>import logging
import sys

import cli_ui

from gitlabform import EXIT_PROCESSING_ERROR
from gitlabform.gitlab import GitLab
from gitlabform.gitlab.core import NotFoundException

class BranchProtector(object):
    def __init__(self, gitlab: GitLab, strict: bool):
        self.gitlab = gitlab
        self.strict = strict

    def protect_branch(self, project_and_group, configuration, branch):
        try:
            if (
                "protected" in configuration["branches"][branch]
                and configuration["branches"][branch]["protected"]
            ):
                if ("developers_can_push" and "developers_can_merge") in configuration[
                    "branches"
                ][branch]:
                    logging.debug("Setting branch '%s' as *protected*", branch)
                    # unprotect first to reset 'allowed to merge' and 'allowed to push' fields
                    self.gitlab.unprotect_branch_new_api(project_and_group, branch)
                    self.gitlab.protect_branch(
                        project_and_group,
                        branch,
                        configuration["branches"][branch]["developers_can_push"],
                        configuration["branches"][branch]["developers_can_merge"],
                    )
                elif (
                    "push_access_level"
                    and "merge_access_level"
                    and "unprotect_access_level"
                ) in configuration["branches"][branch]:
                    branch_access_levels = self.gitlab.get_branch_access_levels(project_and_group, branch)
                    levels = ["push_access_levels", "merge_access_levels", "unprotect_access_levels"]
                    if all(configuration["branches"][branch][level[0:-1]] == branch_access_levels[level][0]["access_level"] for level in levels):
                        logging.debug("Skipping set branch '%s' access levels because they're already set")
                        return
                    logging.debug("Setting branch '%s' access level", branch)
                    # unprotect first to reset 'allowed to merge' and 'allowed to push' fields
                    self.gitlab.unprotect_branch_new_api(project_and_group, branch)
                    self.gitlab.branch_access_level(
                        project_and_group,
                        branch,
                        configuration["branches"][branch]["push_access_level"],
                        configuration["branches"][branch]["merge_access_level"],
                        configuration["branches"][branch]["unprotect_access_level"],
                    )
                if "code_owner_approval_required" in configuration["branches"][branch]:
                    logging.debug(
                        "Setting branch '%s' \"code owner approval required\" option",
                        branch,
                    )
                    self.gitlab.branch_code_owner_approval_required(
                        project_and_group,
                        branch,
                        configuration["branches"][branch][
                            "code_owner_approval_required"
                        ],
                    )
            else:
                logging.debug("Setting branch '%s' as unprotected", branch)
                self.gitlab.unprotect_branch_new_api(project_and_group, branch)
        except NotFoundException:
            message = f"Branch '{branch}' not found when trying to set it as protected/unprotected!"
            if self.strict:
                cli_ui.error(message)
                sys.exit(EXIT_PROCESSING_ERROR)
            else:
                cli_ui.warning(message)
<commit_msg>fix: fixes all but one test in the v2 branch<commit_after>import logging
import sys

import cli_ui

from gitlabform import EXIT_PROCESSING_ERROR
from gitlabform.gitlab import GitLab
from gitlabform.gitlab.core import NotFoundException

class BranchProtector(object):
    def __init__(self, gitlab: GitLab, strict: bool):
        self.gitlab = gitlab
        self.strict = strict

    def protect_branch(self, project_and_group, configuration, branch):
        try:
            if (
                "protected" in configuration["branches"][branch]
                and configuration["branches"][branch]["protected"]
            ):
                if ("developers_can_push" and "developers_can_merge") in configuration[
                    "branches"
                ][branch]:
                    logging.debug("Setting branch '%s' as *protected*", branch)
                    # unprotect first to reset 'allowed to merge' and 'allowed to push' fields
                    self.gitlab.unprotect_branch_new_api(project_and_group, branch)
                    self.gitlab.protect_branch(
                        project_and_group,
                        branch,
                        configuration["branches"][branch]["developers_can_push"],
                        configuration["branches"][branch]["developers_can_merge"],
                    )
                elif (
                    "push_access_level"
                    and "merge_access_level"
                    and "unprotect_access_level"
                ) in configuration["branches"][branch]:
                    try:
                        branch_access_levels = self.gitlab.get_branch_access_levels(project_and_group, branch)
                        levels = ["push_access_levels", "merge_access_levels", "unprotect_access_levels"]
                        # Check each access type has the correct access level, if they do, not change is needed
                        # Gitlabform uses access_levels with plural form, and gitlab uses singular form.
                        # [0:-1] removes the plural
                        if all(configuration["branches"][branch][level[0:-1]] == branch_access_levels[level][0]["access_level"] for level in levels):
                            logging.debug("Skipping set branch '%s' access levels because they're already set")
                            return

                    except NotFoundException:
                        logging.debug("No access levels for this branch exist yet")

                    logging.debug("Setting branch '%s' access level", branch)
                    # unprotect first to reset 'allowed to merge' and 'allowed to push' fields
                    self.gitlab.unprotect_branch_new_api(project_and_group, branch)
                    self.gitlab.branch_access_level(
                        project_and_group,
                        branch,
                        configuration["branches"][branch]["push_access_level"],
                        configuration["branches"][branch]["merge_access_level"],
                        configuration["branches"][branch]["unprotect_access_level"],
                    )
                if "code_owner_approval_required" in configuration["branches"][branch]:
                    logging.debug(
                        "Setting branch '%s' \"code owner approval required\" option",
                        branch,
                    )
                    self.gitlab.branch_code_owner_approval_required(
                        project_and_group,
                        branch,
                        configuration["branches"][branch][
                            "code_owner_approval_required"
                        ],
                    )
            else:
                logging.debug("Setting branch '%s' as unprotected", branch)
                self.gitlab.unprotect_branch_new_api(project_and_group, branch)
        except NotFoundException:
            message = f"Branch '{branch}' not found when trying to set it as protected/unprotected!"
            if self.strict:
                cli_ui.error(message)
                sys.exit(EXIT_PROCESSING_ERROR)
            else:
                cli_ui.warning(message)
<|endoftext|>


<commit_before># Copyright 2013 Daniel Narvaez
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from StringIO import StringIO

from gi.repository import Gio
from gi.repository import GLib
from gi.repository import GObject

from gwebsockets import protocol

class MessageBuffer():
    def __init__(self):
        self._data = b""
        self._cursor = 0
        self.available = 0

    def append(self, data):
        self._data = self._data + data
        self._update_available()

    def read(self, size):
        result = self._data[self._cursor:self._cursor + size]

        self._cursor = self._cursor + size
        self._update_available()

        return result

    def _update_available(self):
        self.available = len(self._data) - self._cursor

class Message():
    TYPE_TEXT = 0
    TYPE_BINARY = 1

    def __init__(self, message_type, data):
        self.message_type = message_type
        self.data = data

class Session(GObject.GObject):
    message_received = GObject.Signal("message-received", arg_types=(object,))

    def __init__(self, connection):
        GObject.GObject.__init__(self)

        self._connection = connection
        self._request = StringIO()
        self._message = MessageBuffer()
        self._parse_g = None
        self._ready = False

    def _response_write_cb(self, stream, result, user_data):
        stream.write_bytes_finish(result)
        self._ready = True

    def _do_handshake(self):
        self._request.seek(0)
        response = protocol.make_handshake(self._request)

        stream = self._connection.get_output_stream()
        stream.write_bytes_async(GLib.Bytes.new(response.encode("utf-8")),
                                 GLib.PRIORITY_DEFAULT,
                                 None, self._response_write_cb, None)

    def read_data(self):
        stream = self._connection.get_input_stream()
        stream.read_bytes_async(8192, GLib.PRIORITY_DEFAULT, None,
                                self._read_data_cb, None)

    def _read_data_cb(self, stream, result, user_data):
        data = stream.read_bytes_finish(result).get_data()

        if self._ready:
            if self._message is None:
                self._message = MessageBuffer()

            self._message.append(data)

            if self._parse_g is None:
                self._parse_g = protocol.parse_message(self._message)

            parsed_message = self._parse_g.next()
            if parsed_message:
                self._parse_g = None
                self._message = None

                received = None
                if parsed_message.tp == protocol.OPCODE_TEXT:
                    received = Message(Message.TYPE_TEXT,
                                       parsed_message.data)
                elif parsed_message.tp == protocol.OPCODE_BINARY:
                    received = Message(Message.TYPE_BINARY,
                                       parsed_message.data)

                if received:
                    self.message_received.emit(received)
        else:
            self._request.write(data)
            if data.endswith("\r\n\r\n"):
                self._do_handshake()

        self.read_data()

    def _message_write_cb(self, stream, result, callback):
        written = stream.write_bytes_finish(result)
        if callback:
            callback(written)

    def send_message(self, message, callback=None, binary=False):
        protocol_message = protocol.make_message(message)

        stream = self._connection.get_output_stream()
        stream.write_bytes_async(GLib.Bytes.new(protocol_message),
                                 GLib.PRIORITY_DEFAULT,
                                 None, self._message_write_cb, callback)

class Server(GObject.GObject):
    session_started = GObject.Signal("session-started", arg_types=(object,))

    def _incoming_connection_cb(self, service, connection, user_data):
        session = Session(connection)
        self.session_started.emit(session)
        session.read_data()

    def start(self):
        service = Gio.SocketService()
        service.connect("incoming", self._incoming_connection_cb)
        return service.add_any_inet_port(None)

if __name__ == "__main__":
    def message_received_cb(session, message):
        session.send_message(message.data)

    def session_started_cb(server, session):
        session.connect("message-received", message_received_cb)

    server = Server()
    server.connect("session-started", session_started_cb)
    port = server.start()

    print "Listening on port %d" % port

    main_loop = GLib.MainLoop()
    main_loop.run()
<commit_msg>Parse multiple messages in the same data chunk<commit_after># Copyright 2013 Daniel Narvaez
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from StringIO import StringIO

from gi.repository import Gio
from gi.repository import GLib
from gi.repository import GObject

from gwebsockets import protocol

class MessageBuffer():
    def __init__(self):
        self._data = b""
        self.available = 0

    def append(self, data):
        self._data = self._data + data
        self.available = len(self._data)

    def read(self, size):
        result = self._data[:size]
        self._data = self._data[size:]
        self.available = len(self._data)
        return result

class Message():
    TYPE_TEXT = 0
    TYPE_BINARY = 1

    def __init__(self, message_type, data):
        self.message_type = message_type
        self.data = data

class Session(GObject.GObject):
    message_received = GObject.Signal("message-received", arg_types=(object,))

    def __init__(self, connection):
        GObject.GObject.__init__(self)

        self._connection = connection
        self._request = StringIO()
        self._message = MessageBuffer()
        self._parse_g = None
        self._ready = False

    def _response_write_cb(self, stream, result, user_data):
        stream.write_bytes_finish(result)
        self._ready = True

    def _do_handshake(self):
        self._request.seek(0)
        response = protocol.make_handshake(self._request)

        stream = self._connection.get_output_stream()
        stream.write_bytes_async(GLib.Bytes.new(response.encode("utf-8")),
                                 GLib.PRIORITY_DEFAULT,
                                 None, self._response_write_cb, None)

    def read_data(self):
        stream = self._connection.get_input_stream()
        stream.read_bytes_async(8192, GLib.PRIORITY_DEFAULT, None,
                                self._read_data_cb, None)

    def _read_data_cb(self, stream, result, user_data):
        data = stream.read_bytes_finish(result).get_data()

        if self._ready:
            self._message.append(data)

            while self._message.available > 0:
                if self._parse_g is None:
                    self._parse_g = protocol.parse_message(self._message)

                parsed_message = self._parse_g.next()
                if parsed_message:
                    self._parse_g = None

                    received = None
                    if parsed_message.tp == protocol.OPCODE_TEXT:
                        received = Message(Message.TYPE_TEXT,
                                           parsed_message.data)
                    elif parsed_message.tp == protocol.OPCODE_BINARY:
                        received = Message(Message.TYPE_BINARY,
                                           parsed_message.data)

                    if received:
                        self.message_received.emit(received)
        else:
            self._request.write(data)
            if data.endswith("\r\n\r\n"):
                self._do_handshake()

        self.read_data()

    def _message_write_cb(self, stream, result, callback):
        written = stream.write_bytes_finish(result)
        if callback:
            callback(written)

    def send_message(self, message, callback=None, binary=False):
        protocol_message = protocol.make_message(message)

        stream = self._connection.get_output_stream()
        stream.write_bytes_async(GLib.Bytes.new(protocol_message),
                                 GLib.PRIORITY_DEFAULT,
                                 None, self._message_write_cb, callback)

class Server(GObject.GObject):
    session_started = GObject.Signal("session-started", arg_types=(object,))

    def _incoming_connection_cb(self, service, connection, user_data):
        session = Session(connection)
        self.session_started.emit(session)
        session.read_data()

    def start(self):
        service = Gio.SocketService()
        service.connect("incoming", self._incoming_connection_cb)
        return service.add_any_inet_port(None)

if __name__ == "__main__":
    def message_received_cb(session, message):
        session.send_message(message.data)

    def session_started_cb(server, session):
        session.connect("message-received", message_received_cb)

    server = Server()
    server.connect("session-started", session_started_cb)
    port = server.start()

    print "Listening on port %d" % port

    main_loop = GLib.MainLoop()
    main_loop.run()
<|endoftext|>


<commit_before><commit_msg>printing sst_MAM was writing "SST_NDJ"<commit_after><|endoftext|>


<commit_before>from django.db import models

# Create your models here.
from django.db import models
import datetime

class Record(models.Model):
    price      = models.DecimalField('the price of ltc',max_digits=6,decimal_places=2)
    timestamp  = models.IntegerField('date of record')

    class Meta:
        get_latest_by = 'timestamp'

    def __str__(self):
        offset = datetime.timedelta(hours=8)
        tz  = datetime.timezone(offset,'Asia/Shanghai')
        date = datetime.datetime.fromtimestamp(self.timestamp,tz)
        date = date.replace(tzinfo=datetime.timezone.utc).astimezone(tz=tz)
        return str(date) + '\t' + str(self.price)

    @classmethod
    def create(cls, price, timestamp):
        record = cls(price = price,timestamp = timestamp)
        record.save()
        return record

    def dumpJSON(self):
        json = {}
        json['price'] = self.price
        json['timestamp']  = self.timestamp
        return json
<commit_msg>fix offset<commit_after>from django.db import models

# Create your models here.
from django.db import models
import datetime

class Record(models.Model):
    price      = models.DecimalField('the price of ltc',max_digits=6,decimal_places=2)
    timestamp  = models.IntegerField('date of record')

    class Meta:
        get_latest_by = 'timestamp'

    def __str__(self):
        offset = datetime.timedelta(hours=0)
        tz  = datetime.timezone(offset,'Asia/Shanghai')
        date = datetime.datetime.fromtimestamp(self.timestamp,tz)
        date = date.replace(tzinfo=datetime.timezone.utc).astimezone(tz=tz)
        return str(date) + '\t' + str(self.price)

    @classmethod
    def create(cls, price, timestamp):
        record = cls(price = price,timestamp = timestamp)
        record.save()
        return record

    def dumpJSON(self):
        json = {}
        json['price'] = self.price
        json['timestamp']  = self.timestamp
        return json
<|endoftext|>


<commit_before># Copyright (C) 2009 Robert Collins <robertc@robertcollins.net>
# Licenced under the txaws licence available at /LICENSE in the txaws source.
"""A GTK client for working with aws."""

from __future__ import absolute_import

import gnomekeyring
import gobject
import gtk

# DO NOT IMPORT twisted.internet, or things that import
# twisted.internet.
# Doing so loads the default Reactor, which is bad. thanks.

from txaws.credentials import AWSCredentials

__all__ = ["main"]

class AWSStatusIcon(gtk.StatusIcon):
    """A status icon shown when instances are running."""

    def __init__(self, reactor):
        from txaws.service import AWSServiceRegion
        gtk.StatusIcon.__init__(self)
        self.set_from_stock(gtk.STOCK_NETWORK)
        self.set_visible(True)
        self.reactor = reactor
        self.connect("activate", self.on_activate)
        self.probing = False
        # Nested import because otherwise we get "reactor already installed".
        self.password_dialog = None
        try:
            creds = AWSCredentials()
        except ValueError:
            creds = self.from_gnomekeyring()
        self.region = AWSServiceRegion(creds)
        self.create_client(creds)
        menu = """
            <ui>
             <menubar name="Menubar">
              <menu action="Menu">
               <menuitem action="Stop instances"/>
              </menu>
             </menubar>
            </ui>
        """
        actions = [
            ("Menu",  None, "Menu"),
            ("Stop instances", gtk.STOCK_STOP, "_Stop instances...", None,
                "Stop instances", self.on_stop_instances),
            ]
        ag = gtk.ActionGroup("Actions")
        ag.add_actions(actions)
        self.manager = gtk.UIManager()
        self.manager.insert_action_group(ag, 0)
        self.manager.add_ui_from_string(menu)
        self.menu = self.manager.get_widget(
            "/Menubar/Menu/Stop instances").props.parent
        self.connect("popup-menu", self.on_popup_menu)

    def create_client(self, creds):
        if creds is not None:
            self.client = self.region.get_ec2_client()
            self.on_activate(None)
        else:
            # waiting on user entered credentials.
            self.client = None

    def from_gnomekeyring(self):
        # Try for gtk gui specific credentials.
        creds = None
        try:
            items = gnomekeyring.find_items_sync(
                gnomekeyring.ITEM_GENERIC_SECRET,
                {
                    "aws-host": "aws.amazon.com",
                })
        except (gnomekeyring.NoMatchError,
            gnomekeyring.DeniedError):
            self.show_a_password_dialog()
            return None
        else:
            key_id, secret_key = items[0].secret.split(":")
            return AWSCredentials(access_key=key_id, secret_key=secret_key)

    def show_a_password_dialog(self):
        self.password_dialog = gtk.Dialog(
            "Enter your AWS credentals", None, gtk.DIALOG_MODAL,
            (gtk.STOCK_OK, gtk.RESPONSE_ACCEPT,
            gtk.STOCK_CANCEL,
            gtk.RESPONSE_REJECT))
        content = self.password_dialog.get_content_area()
        def add_entry(name):
            box = gtk.HBox()
            box.show()
            content.add(box)
            label = gtk.Label(name)
            label.show()
            box.add(label)
            entry = gtk.Entry()
            entry.show()
            box.add(entry)
            label.set_use_underline(True)
            label.set_mnemonic_widget(entry)
        add_entry("AWS _Access Key ID")
        add_entry("AWS _Secret Key")

        self.password_dialog.show()
        self.password_dialog.connect("response", self.save_key)
        self.password_dialog.run()

    def on_activate(self, data):
        if self.probing or not self.client:
            # don't ask multiple times, and don't ask until we have
            # credentials.
            return
        self.probing = True
        deferred = self.client.describe_instances()
        deferred.addCallbacks(self.showhide, self.describe_error)

    def on_popup_menu(self, status, button, time):
        self.menu.popup(None, None, None, button, time)

    def on_stop_instances(self, data):
        # It would be nice to popup a window to select instances.. TODO.
        deferred = self.client.describe_instances()
        deferred.addCallbacks(self.shutdown_instances, self.show_error)

    def save_key(self, response_id, data):
        # handle the dialog
        dialog = self.password_dialog
        try:
            if data != gtk.RESPONSE_ACCEPT:
                # User cancelled. They can ask for the password again somehow.
                return
            content = self.password_dialog.get_content_area()
            key_id = content.get_children()[0].get_children()[1].get_text()
            secret_key = content.get_children()[1].get_children()[1].get_text()
            creds = AWSCredentials(access_key=key_id, secret_key=secret_key)
            self.create_client(creds)
            gnomekeyring.item_create_sync(
                None,
                gnomekeyring.ITEM_GENERIC_SECRET,
                "AWS access credentials",
                    {"aws-host": "aws.amazon.com"},
                    "%s:%s" % (key_id, secret_key), True)
        finally:
            self.password_dialog.hide()
            # XXX? Does this leak?
            self.password_dialog = None

    def showhide(self, reservation):
        active = 0
        for instance in reservation:
            if instance.instanceState == "running":
                active += 1
        self.set_tooltip("AWS Status - %d instances" % active)
        self.set_visible(active != 0)
        self.queue_check()

    def shutdown_instances(self, reservation):
        d = self.client.terminate_instances(
            *[instance.instance_id for instance in reservation])
        d.addCallbacks(self.on_activate, self.show_error)

    def queue_check(self):
        self.probing = False
        self.reactor.callLater(60, self.on_activate, None)

    def show_error(self, error):
        # debugging output for now.
        print error.value
        try:
            print error.value.response
        except:
            pass

    def describe_error(self, error):
        from twisted.internet.defer import TimeoutError
        if isinstance(error.value, TimeoutError):
            # timeout errors can be ignored - transient network issue or some
            # such.
            pass
        else:
            # debugging output for now.
            self.show_error(error)
        self.queue_check()

def main(argv, reactor=None):
    """Run the client GUI.

    Typical use:
    >>> sys.exit(main(sys.argv))

    @param argv: The arguments to run it with, e.g. sys.argv.
    @param reactor: The reactor to use. Must be compatible with gtk as this
        module uses gtk API"s.
    @return exitcode: The exit code it returned, as per sys.exit.
    """
    if reactor is None:
        from twisted.internet import gtk2reactor
        gtk2reactor.install()
        from twisted.internet import reactor
    status = AWSStatusIcon(reactor)
    gobject.set_application_name("aws-status")
    reactor.run()
<commit_msg>instanceState -> instance_state in the client.<commit_after># Copyright (C) 2009 Robert Collins <robertc@robertcollins.net>
# Licenced under the txaws licence available at /LICENSE in the txaws source.
"""A GTK client for working with aws."""

from __future__ import absolute_import

import gnomekeyring
import gobject
import gtk

# DO NOT IMPORT twisted.internet, or things that import
# twisted.internet.
# Doing so loads the default Reactor, which is bad. thanks.

from txaws.credentials import AWSCredentials

__all__ = ["main"]

class AWSStatusIcon(gtk.StatusIcon):
    """A status icon shown when instances are running."""

    def __init__(self, reactor):
        from txaws.service import AWSServiceRegion
        gtk.StatusIcon.__init__(self)
        self.set_from_stock(gtk.STOCK_NETWORK)
        self.set_visible(True)
        self.reactor = reactor
        self.connect("activate", self.on_activate)
        self.probing = False
        # Nested import because otherwise we get "reactor already installed".
        self.password_dialog = None
        try:
            creds = AWSCredentials()
        except ValueError:
            creds = self.from_gnomekeyring()
        self.region = AWSServiceRegion(creds)
        self.create_client(creds)
        menu = """
            <ui>
             <menubar name="Menubar">
              <menu action="Menu">
               <menuitem action="Stop instances"/>
              </menu>
             </menubar>
            </ui>
        """
        actions = [
            ("Menu",  None, "Menu"),
            ("Stop instances", gtk.STOCK_STOP, "_Stop instances...", None,
                "Stop instances", self.on_stop_instances),
            ]
        ag = gtk.ActionGroup("Actions")
        ag.add_actions(actions)
        self.manager = gtk.UIManager()
        self.manager.insert_action_group(ag, 0)
        self.manager.add_ui_from_string(menu)
        self.menu = self.manager.get_widget(
            "/Menubar/Menu/Stop instances").props.parent
        self.connect("popup-menu", self.on_popup_menu)

    def create_client(self, creds):
        if creds is not None:
            self.client = self.region.get_ec2_client()
            self.on_activate(None)
        else:
            # waiting on user entered credentials.
            self.client = None

    def from_gnomekeyring(self):
        # Try for gtk gui specific credentials.
        creds = None
        try:
            items = gnomekeyring.find_items_sync(
                gnomekeyring.ITEM_GENERIC_SECRET,
                {
                    "aws-host": "aws.amazon.com",
                })
        except (gnomekeyring.NoMatchError,
            gnomekeyring.DeniedError):
            self.show_a_password_dialog()
            return None
        else:
            key_id, secret_key = items[0].secret.split(":")
            return AWSCredentials(access_key=key_id, secret_key=secret_key)

    def show_a_password_dialog(self):
        self.password_dialog = gtk.Dialog(
            "Enter your AWS credentals", None, gtk.DIALOG_MODAL,
            (gtk.STOCK_OK, gtk.RESPONSE_ACCEPT,
            gtk.STOCK_CANCEL,
            gtk.RESPONSE_REJECT))
        content = self.password_dialog.get_content_area()
        def add_entry(name):
            box = gtk.HBox()
            box.show()
            content.add(box)
            label = gtk.Label(name)
            label.show()
            box.add(label)
            entry = gtk.Entry()
            entry.show()
            box.add(entry)
            label.set_use_underline(True)
            label.set_mnemonic_widget(entry)
        add_entry("AWS _Access Key ID")
        add_entry("AWS _Secret Key")

        self.password_dialog.show()
        self.password_dialog.connect("response", self.save_key)
        self.password_dialog.run()

    def on_activate(self, data):
        if self.probing or not self.client:
            # don't ask multiple times, and don't ask until we have
            # credentials.
            return
        self.probing = True
        deferred = self.client.describe_instances()
        deferred.addCallbacks(self.showhide, self.describe_error)

    def on_popup_menu(self, status, button, time):
        self.menu.popup(None, None, None, button, time)

    def on_stop_instances(self, data):
        # It would be nice to popup a window to select instances.. TODO.
        deferred = self.client.describe_instances()
        deferred.addCallbacks(self.shutdown_instances, self.show_error)

    def save_key(self, response_id, data):
        # handle the dialog
        dialog = self.password_dialog
        try:
            if data != gtk.RESPONSE_ACCEPT:
                # User cancelled. They can ask for the password again somehow.
                return
            content = self.password_dialog.get_content_area()
            key_id = content.get_children()[0].get_children()[1].get_text()
            secret_key = content.get_children()[1].get_children()[1].get_text()
            creds = AWSCredentials(access_key=key_id, secret_key=secret_key)
            self.create_client(creds)
            gnomekeyring.item_create_sync(
                None,
                gnomekeyring.ITEM_GENERIC_SECRET,
                "AWS access credentials",
                    {"aws-host": "aws.amazon.com"},
                    "%s:%s" % (key_id, secret_key), True)
        finally:
            self.password_dialog.hide()
            # XXX? Does this leak?
            self.password_dialog = None

    def showhide(self, reservation):
        active = 0
        for instance in reservation:
            if instance.instance_state == "running":
                active += 1
        self.set_tooltip("AWS Status - %d instances" % active)
        self.set_visible(active != 0)
        self.queue_check()

    def shutdown_instances(self, reservation):
        d = self.client.terminate_instances(
            *[instance.instance_id for instance in reservation])
        d.addCallbacks(self.on_activate, self.show_error)

    def queue_check(self):
        self.probing = False
        self.reactor.callLater(60, self.on_activate, None)

    def show_error(self, error):
        # debugging output for now.
        print error.value
        try:
            print error.value.response
        except:
            pass

    def describe_error(self, error):
        from twisted.internet.defer import TimeoutError
        if isinstance(error.value, TimeoutError):
            # timeout errors can be ignored - transient network issue or some
            # such.
            pass
        else:
            # debugging output for now.
            self.show_error(error)
        self.queue_check()

def main(argv, reactor=None):
    """Run the client GUI.

    Typical use:
    >>> sys.exit(main(sys.argv))

    @param argv: The arguments to run it with, e.g. sys.argv.
    @param reactor: The reactor to use. Must be compatible with gtk as this
        module uses gtk API"s.
    @return exitcode: The exit code it returned, as per sys.exit.
    """
    if reactor is None:
        from twisted.internet import gtk2reactor
        gtk2reactor.install()
        from twisted.internet import reactor
    status = AWSStatusIcon(reactor)
    gobject.set_application_name("aws-status")
    reactor.run()
<|endoftext|>


<commit_before>''' imports from installed packages '''
from django.core.management.base import BaseCommand, CommandError

from django_mongokit import get_database

try:
    from bson import ObjectId
except ImportError:  # old pymongo
    from pymongo.objectid import ObjectId

''' imports from application folders/files '''
from gnowsys_ndf.ndf.models import GSystemType
from gnowsys_ndf.ndf.models import GSystem

from gnowsys_ndf.settings import GAPPS

####################################################################################################################

class Command(BaseCommand):
    help = "Based on GAPPS list, inserts few documents (consisting of dummy values) into the following collections:\n\n" \
           "\tGSystemType, GSystemType"

    def handle(self, *args, **options):
        db = get_database()

        db.drop_collection(GSystem.collection_name)
        db.drop_collection(GSystemType.collection_name)

        gst_collection = db[GSystemType.collection_name]
        gst_node = []

        for i in range(0, len(GAPPS)):
            gst_node.append(gst_collection.GSystemType())

            gst_node[i].name = unicode(GAPPS[i])
            gst_node[i].save()

        # --- End of handle() ---
<commit_msg>minor changes in filldb.py<commit_after>''' imports from installed packages '''
from django.core.management.base import BaseCommand, CommandError

from django_mongokit import get_database

try:
    from bson import ObjectId
except ImportError:  # old pymongo
    from pymongo.objectid import ObjectId

''' imports from application folders/files '''
from gnowsys_ndf.ndf.models import GSystemType
from gnowsys_ndf.ndf.models import GSystem

from gnowsys_ndf.settings import GAPPS

####################################################################################################################

class Command(BaseCommand):
    help = "Based on GAPPS list, inserts few documents (consisting of dummy values) into the following collections:\n\n" \
           "\tGSystemType, GSystemType"

    def handle(self, *args, **options):
        db = get_database()

        db.drop_collection(GSystem.collection_name)
        db.drop_collection(GSystemType.collection_name)

        gst_collection = db[GSystemType.collection_name]
        gst_node = []
        user_id = 1
        for i in range(0, len(GAPPS)):
            gst_node.append(gst_collection.GSystemType())

            gst_node[i].name = unicode(GAPPS[i])
            gst_node[i].created_by = user_id
            gst_node[i].modified_by.append(user_id)
            gst_node[i].save()

        # --- End of handle() ---
<|endoftext|>


<commit_before>#
# Copyright (c) 2010 Doug Hellmann.  All rights reserved.
#
"""Spelling checker extension for Sphinx.
"""

# TODO - Words with multiple uppercase letters treated as classes and ignored

import builtins
import importlib
import subprocess
import xmlrpc.client as xmlrpc_client

from enchant.tokenize import Filter, get_tokenizer, tokenize, unit_tokenize

from sphinx.util import logging

logger = logging.getLogger(__name__)

class AcronymFilter(Filter):
    """If a word looks like an acronym (all upper case letters),
    ignore it.
    """
    def _skip(self, word):
        return (
            word.isupper() or  # all caps
            # pluralized acronym ("URLs")
            (word[-1].lower() == 's' and word[:-1].isupper())
        )

class list_tokenize(tokenize):

    def __init__(self, words):
        super().__init__('')
        self._words = words

    def next(self):
        if not self._words:
            raise StopIteration()
        word = self._words.pop(0)
        return (word, 0)

class ContractionFilter(Filter):
    """Strip common contractions from words.
    """
    splits = {
        "aren't": ["are", "not"],
        "can't": ["can", "not"],
        "could've": ["could", "have"],
        "couldn't": ["could", "not"],
        "didn't": ["did", "not"],
        "doesn't": ["does", "not"],
        "don't": ["do", "not"],
        "hadn't": ["had", "not"],
        "hasn't": ["has", "not"],
        "haven't": ["have", "not"],
        "he'd": ["he", "would"],
        "he'll": ["he", "will"],
        "he's": ["he", "is"],
        "how'd": ["how", "would"],
        "how'll": ["how", "will"],
        "how's": ["how", "is"],
        "i'd": ["I", "would"],
        "i'll": ["I", "will"],
        "i'm": ["I", "am"],
        "i've": ["I", "have"],
        "isn't": ["is", "not"],
        "it'd": ["it", "would"],
        "it'll": ["it", "will"],
        "it's": ["it", "is"],
        "ma'am": ["madam"],
        "might've": ["might", "have"],
        "mightn't": ["might", "not"],
        "must've": ["must", "have"],
        "mustn't": ["must", "not"],
        "o'": ["of"],
        "o'clock": ["of", "the", "clock"],
        "she'd": ["she", "would"],
        "she'll": ["she", "will"],
        "she's": ["she", "is"],
        "should've": ["should", "have"],
        "shouldn't": ["should", "not"],
        "that'd": ["that", "would"],
        "that'll": ["that", "will"],
        "that's": ["that", "is"],
        "they'd": ["they", "would"],
        "they'll": ["they", "will"],
        "they're": ["they", "are"],
        "they've": ["they", "have"],
        "wasn't": ["was", "not"],
        "we'd": ["we", "would"],
        "we'll": ["we", "will"],
        "we're": ["we", "are"],
        "we've": ["we", "have"],
        "weren't": ["were", "not"],
        "what'd": ["what", "would"],
        "what'll": ["what", "will"],
        "what're": ["what", "are"],
        "what's": ["what", "is"],
        "when'd": ["when", "would"],
        "when'll": ["when", "will"],
        "when's": ["when", "is"],
        "where'd": ["where", "would"],
        "where'll": ["where", "will"],
        "where's": ["where", "is"],
        "who'd": ["who", "would"],
        "who'll": ["who", "will"],
        "who's": ["who", "is"],
        "why'd": ["why", "would"],
        "why'll": ["why", "will"],
        "why's": ["why", "is"],
        "won't": ["will", "not"],
        "would've": ["would", "have"],
        "wouldn't": ["would", "not"],
        "you'd": ["you", "would"],
        "you'll": ["you", "will"],
        "you're": ["you", "are"],
        "you've": ["you", "have"],
    }

    def _split(self, word):
        # Fixed responses
        if word.lower() in self.splits:
            return list_tokenize(self.splits[word.lower()])

        # Possessive
        if word.lower().endswith("'s"):
            return unit_tokenize(word[:-2])

        # * not
        if word.lower().endswith("n't"):
            return unit_tokenize(word[:-3])

        return unit_tokenize(word)

class IgnoreWordsFilter(Filter):
    """Given a set of words, ignore them all.
    """

    def __init__(self, tokenizer, word_set):
        self.word_set = set(word_set)
        super().__init__(tokenizer)

    def _skip(self, word):
        return word in self.word_set

class IgnoreWordsFilterFactory:

    def __init__(self, words):
        self.words = words

    def __call__(self, tokenizer):
        return IgnoreWordsFilter(tokenizer, self.words)

class PyPIFilterFactory(IgnoreWordsFilterFactory):
    """Build an IgnoreWordsFilter for all of the names of packages on PyPI.
    """
    def __init__(self):
        client = xmlrpc_client.ServerProxy('https://pypi.python.org/pypi')
        super().__init__(client.list_packages())

class PythonBuiltinsFilter(Filter):
    """Ignore names of built-in Python symbols.
    """
    def _skip(self, word):
        try:
            return hasattr(builtins, word)
        except UnicodeEncodeError:
            return False

class ImportableModuleFilter(Filter):
    """Ignore names of modules that we could import.
    """
    def __init__(self, tokenizer):
        super().__init__(tokenizer)
        self.found_modules = set()
        self.sought_modules = set()

    def _skip(self, word):
        if word not in self.sought_modules:
            self.sought_modules.add(word)
            try:
                spec = importlib.util.find_spec(word)
            except (AttributeError, ImportError):
                # Raises ModuleNotFoundError (subclass of ImportError) instead
                # of AttributeError in Python 3.7+.
                return False
            if spec is None:
                return False
            self.found_modules.add(word)
            return True
        return word in self.found_modules

class ContributorFilter(IgnoreWordsFilter):
    """Accept information about contributors as spelled correctly.

    Look in the git history for authors and committers and accept
    tokens that are in the set.
    """

    _pretty_format = (
        '%(trailers:key=Co-Authored-By,separator=%x0A)%x0A%an%x0A%cn'
    )

    def __init__(self, tokenizer):
        contributors = self._get_contributors()
        super().__init__(tokenizer, contributors)

    def _get_contributors(self):
        logger.info('Scanning contributors')
        cmd = [
            'git', 'log', '--quiet', '--no-color',
            '--pretty=format:' + self._pretty_format,
        ]
        try:
            p = subprocess.run(cmd, check=True, stdout=subprocess.PIPE)
        except subprocess.CalledProcessError as err:
            logger.warning('Called: {}'.format(' '.join(cmd)))
            logger.warning(f'Failed to scan contributors: {err}')
            return set()
        output = p.stdout.decode('utf-8')
        tokenizer = get_tokenizer('en_US', filters=[])
        return {word for word, pos in tokenizer(output)}
<commit_msg>Remove unnecessary UnicodeEncodeError (due to Python 3)<commit_after>#
# Copyright (c) 2010 Doug Hellmann.  All rights reserved.
#
"""Spelling checker extension for Sphinx.
"""

# TODO - Words with multiple uppercase letters treated as classes and ignored

import builtins
import importlib
import subprocess
import xmlrpc.client as xmlrpc_client

from enchant.tokenize import Filter, get_tokenizer, tokenize, unit_tokenize

from sphinx.util import logging

logger = logging.getLogger(__name__)

class AcronymFilter(Filter):
    """If a word looks like an acronym (all upper case letters),
    ignore it.
    """
    def _skip(self, word):
        return (
            word.isupper() or  # all caps
            # pluralized acronym ("URLs")
            (word[-1].lower() == 's' and word[:-1].isupper())
        )

class list_tokenize(tokenize):

    def __init__(self, words):
        super().__init__('')
        self._words = words

    def next(self):
        if not self._words:
            raise StopIteration()
        word = self._words.pop(0)
        return (word, 0)

class ContractionFilter(Filter):
    """Strip common contractions from words.
    """
    splits = {
        "aren't": ["are", "not"],
        "can't": ["can", "not"],
        "could've": ["could", "have"],
        "couldn't": ["could", "not"],
        "didn't": ["did", "not"],
        "doesn't": ["does", "not"],
        "don't": ["do", "not"],
        "hadn't": ["had", "not"],
        "hasn't": ["has", "not"],
        "haven't": ["have", "not"],
        "he'd": ["he", "would"],
        "he'll": ["he", "will"],
        "he's": ["he", "is"],
        "how'd": ["how", "would"],
        "how'll": ["how", "will"],
        "how's": ["how", "is"],
        "i'd": ["I", "would"],
        "i'll": ["I", "will"],
        "i'm": ["I", "am"],
        "i've": ["I", "have"],
        "isn't": ["is", "not"],
        "it'd": ["it", "would"],
        "it'll": ["it", "will"],
        "it's": ["it", "is"],
        "ma'am": ["madam"],
        "might've": ["might", "have"],
        "mightn't": ["might", "not"],
        "must've": ["must", "have"],
        "mustn't": ["must", "not"],
        "o'": ["of"],
        "o'clock": ["of", "the", "clock"],
        "she'd": ["she", "would"],
        "she'll": ["she", "will"],
        "she's": ["she", "is"],
        "should've": ["should", "have"],
        "shouldn't": ["should", "not"],
        "that'd": ["that", "would"],
        "that'll": ["that", "will"],
        "that's": ["that", "is"],
        "they'd": ["they", "would"],
        "they'll": ["they", "will"],
        "they're": ["they", "are"],
        "they've": ["they", "have"],
        "wasn't": ["was", "not"],
        "we'd": ["we", "would"],
        "we'll": ["we", "will"],
        "we're": ["we", "are"],
        "we've": ["we", "have"],
        "weren't": ["were", "not"],
        "what'd": ["what", "would"],
        "what'll": ["what", "will"],
        "what're": ["what", "are"],
        "what's": ["what", "is"],
        "when'd": ["when", "would"],
        "when'll": ["when", "will"],
        "when's": ["when", "is"],
        "where'd": ["where", "would"],
        "where'll": ["where", "will"],
        "where's": ["where", "is"],
        "who'd": ["who", "would"],
        "who'll": ["who", "will"],
        "who's": ["who", "is"],
        "why'd": ["why", "would"],
        "why'll": ["why", "will"],
        "why's": ["why", "is"],
        "won't": ["will", "not"],
        "would've": ["would", "have"],
        "wouldn't": ["would", "not"],
        "you'd": ["you", "would"],
        "you'll": ["you", "will"],
        "you're": ["you", "are"],
        "you've": ["you", "have"],
    }

    def _split(self, word):
        # Fixed responses
        if word.lower() in self.splits:
            return list_tokenize(self.splits[word.lower()])

        # Possessive
        if word.lower().endswith("'s"):
            return unit_tokenize(word[:-2])

        # * not
        if word.lower().endswith("n't"):
            return unit_tokenize(word[:-3])

        return unit_tokenize(word)

class IgnoreWordsFilter(Filter):
    """Given a set of words, ignore them all.
    """

    def __init__(self, tokenizer, word_set):
        self.word_set = set(word_set)
        super().__init__(tokenizer)

    def _skip(self, word):
        return word in self.word_set

class IgnoreWordsFilterFactory:

    def __init__(self, words):
        self.words = words

    def __call__(self, tokenizer):
        return IgnoreWordsFilter(tokenizer, self.words)

class PyPIFilterFactory(IgnoreWordsFilterFactory):
    """Build an IgnoreWordsFilter for all of the names of packages on PyPI.
    """
    def __init__(self):
        client = xmlrpc_client.ServerProxy('https://pypi.python.org/pypi')
        super().__init__(client.list_packages())

class PythonBuiltinsFilter(Filter):
    """Ignore names of built-in Python symbols.
    """
    def _skip(self, word):
        return hasattr(builtins, word)

class ImportableModuleFilter(Filter):
    """Ignore names of modules that we could import.
    """
    def __init__(self, tokenizer):
        super().__init__(tokenizer)
        self.found_modules = set()
        self.sought_modules = set()

    def _skip(self, word):
        if word not in self.sought_modules:
            self.sought_modules.add(word)
            try:
                spec = importlib.util.find_spec(word)
            except (AttributeError, ImportError):
                # Raises ModuleNotFoundError (subclass of ImportError) instead
                # of AttributeError in Python 3.7+.
                return False
            if spec is None:
                return False
            self.found_modules.add(word)
            return True
        return word in self.found_modules

class ContributorFilter(IgnoreWordsFilter):
    """Accept information about contributors as spelled correctly.

    Look in the git history for authors and committers and accept
    tokens that are in the set.
    """

    _pretty_format = (
        '%(trailers:key=Co-Authored-By,separator=%x0A)%x0A%an%x0A%cn'
    )

    def __init__(self, tokenizer):
        contributors = self._get_contributors()
        super().__init__(tokenizer, contributors)

    def _get_contributors(self):
        logger.info('Scanning contributors')
        cmd = [
            'git', 'log', '--quiet', '--no-color',
            '--pretty=format:' + self._pretty_format,
        ]
        try:
            p = subprocess.run(cmd, check=True, stdout=subprocess.PIPE)
        except subprocess.CalledProcessError as err:
            logger.warning('Called: {}'.format(' '.join(cmd)))
            logger.warning(f'Failed to scan contributors: {err}')
            return set()
        output = p.stdout.decode('utf-8')
        tokenizer = get_tokenizer('en_US', filters=[])
        return {word for word, pos in tokenizer(output)}
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
reads .DMCdata files and displays them
 Michael Hirsch
NOTE: Observe the dtype=np.int64, this is for Windows Python, that wants to
   default to int32 instead of int64 like everyone else!
 """
from pathlib import Path
import logging
import re
from dateutil.parser import parse
from numpy import int64,uint16,zeros,arange,fromfile
from re import search
from astropy.io import fits
from shutil import disk_usage
#
from . import req2frame,dir2fn,getRawInd,meta2rawInd
from .h5imgwriter import setupimgh5,imgwriteincr
from .timedmc import frame2ut1,ut12frame
#
try:
    import tifffile
except ImportError:
    tifffile=None
#
bpp = 16

def goRead(fn:Path,
           xyPix,xyBin,
           FrameIndReq=None, ut1Req=None,kineticraw=None,
           startUTC=None,cmosinit=None,verbose=0,outfn=None,nHeadBytes:int=4):

    fn = Path(fn).expanduser()
    ext = fn.suffix
#%% optional output file setup
    outfn = dir2fn(outfn,fn,'.h5')
#%% setup data parameters
    if ext in ('.DMCdata','.dat'):
        # preallocate *** LABVIEW USES ROW-MAJOR ORDERING C ORDER
        finf = getDMCparam(fn,xyPix,xyBin,FrameIndReq,ut1Req,kineticraw,startUTC,nHeadBytes,verbose)
        outsize = finf['bytesperframe']*finf['nframeextract']
        if outfn:
            freeout =  disk_usage(outfn.parent).free
            if freeout < 10*outsize:
                raise RuntimeError(f'out of disk space on {outfn.parent}.  {freeout/1e9} GB free, wanting to write {outsize/1e9} GB.')

        rawFrameInd = zeros(finf['nframeextract'], dtype=int64)
#%% output (variable or file)
        if outfn:
            setupimgh5(outfn,finf['nframeextract'],finf['supery'],finf['superx'])
            data = None
        else:
            data = zeros((finf['nframeextract'],finf['supery'],finf['superx']),
                    dtype=uint16, order='C')
#%% read
        with fn.open('rb') as fid: #NOTE: not pathlib due to Python 2.7, Numpy 1.11 incompat. Py3.5 OK
            for j,i in enumerate(finf['frameindrel']): #j and i are NOT the same in general when not starting from beginning of file!
                D, rawFrameInd[j] = getDMCframe(fid,i,finf,verbose)
                if outfn:
                    imgwriteincr(outfn,D,j)
                else:
                    data[j,...] = D
#%% absolute time estimate, software timing (at your peril)
        finf['ut1'] = frame2ut1(startUTC,kineticraw,rawFrameInd)

    elif ext[:4] == '.tif':
        finf,data = getNeoParam(fn,FrameIndReq,ut1Req,kineticraw,startUTC,cmosinit,verbose)
        rawFrameInd = finf['frameind'] #FIXME this is for individual file, not start of night.
    else:
        raise ValueError(f'not sure to do with file {fn}')

    return data, rawFrameInd,finf#,ut1_unix
#%% workers
def getserialnum(flist):
    """
    This function assumes the serial number of the camera is in a particular place in the filename.
    Yes, this is a little lame, but it's how the original 2011 image-writing program worked, and I've
    carried over the scheme rather than appending bits to dozens of TB of files.
    """
    sn = []
    for f in flist:
        tmp = search(r'(?<=CamSer)\d{3,6}',f)
        if tmp:
            ser = int(tmp.group())
        else:
            ser = None
        sn.append(ser)
    return sn

def getDMCparam(fn:Path,xyPix,xyBin,
                FrameIndReq=None,ut1req=None,kineticsec=None,startUTC=None,nHeadBytes=4,verbose=0):
    """
    nHeadBytes=4 for 2013-2016 data
    nHeadBytes=0 for 2011 data
    """
    Nmetadata = nHeadBytes//2 #FIXME for DMCdata version 1 only

    if not fn.is_file(): #leave this here, getsize() doesn't fail on directory
        raise ValueError(f'{fn} is not a file!')

    print(f'reading {fn}')

    #np.int64() in case we are fed a float or int
    SuperX = int64(xyPix[0]) // int64(xyBin[0]) # "//" keeps as integer
    SuperY = int64(xyPix[1]) // int64(xyBin[1])

    PixelsPerImage,BytesPerImage,BytesPerFrame = howbig(SuperX,SuperY,nHeadBytes)

    (firstRawInd,lastRawInd) = getRawInd(fn,BytesPerImage,nHeadBytes,Nmetadata)

    FrameIndRel = whichframes(fn,FrameIndReq,kineticsec,ut1req,startUTC,firstRawInd,lastRawInd,
                              BytesPerImage,BytesPerFrame,verbose)

    return {'superx':SuperX, 'supery':SuperY, 'nmetadata':Nmetadata,
            'bytesperframe':BytesPerFrame, 'pixelsperimage':PixelsPerImage,
            'nframeextract':FrameIndRel.size,
            'frameindrel':FrameIndRel}

def getNeoParam(fn,FrameIndReq=None,ut1req=None,kineticsec=None,startUTC=None,cmosinit={},verbose=False):
    """ assumption is that this is a Neo sCMOS FITS/TIFF file, where Solis chooses to break up the recordings
    into smaller files. Verify if this timing estimate makes sense for your application!
    I did not want to do regexp on the filename or USERTXT1 as I felt this too prone to error.

    inputs:
    -------
    cmosinit = {'firstrawind','lastrawind'}
    """
    fn = Path(fn).expanduser()

    nHeadBytes=0

    if fn.suffix.lower() in '.tiff':
        if tifffile is None: raise ImportError('pip install tifffile')
        #FIXME didn't the 2011 TIFFs have headers? maybe not.
        with tifffile.TiffFile(str(fn)) as f:
            data = f.asarray()
            Y,X = data.shape[-2:]
    elif fn.suffix.lower() in '.fits':
        with fits.open(str(fn),mode='readonly',memmap=False) as f:
            data = None #f[0].data  #NOTE You can read the data if you want, I didn't need it here.

            kineticsec = f[0].header['KCT']
            startseries = parse(f[0].header['DATE'] + 'Z') #TODO start of night's recording (with some Solis versionss)

            #TODO wish there was a better way
            try:
                frametxt = f[0].header['USERTXT1']
                m = re.search('(?<=Images\:)\d+-\d+(?=\.)',frametxt)
                inds = m.group(0).split('-')
            except KeyError: # just a single file?
                inds = [1,f[0].shape[0]] #yes start with 1, end without adding 1 for Andor Solis

            cmosinit={'firstrawind':int(inds[0]),
                      'lastrawind':int(inds[1])}

            #start = parse(f[0].header['FRAME']+'Z') No, incorrect by several hours with some 2015 Solis versions!

            Y,X = f[0].shape[-2:]

        startUTC = startseries.timestamp()

#%% FrameInd relative to this file
    PixelsPerImage,BytesPerImage,BytesPerFrame = howbig(X,Y,nHeadBytes)

    FrameIndRel = whichframes(fn,FrameIndReq,kineticsec,ut1req,startUTC,
                              cmosinit['firstrawind'],cmosinit['lastrawind'],
                              BytesPerImage,BytesPerFrame,verbose)

    assert isinstance(FrameIndReq,int) or FrameIndReq is None, 'TODO: add multi-frame request case'
    rawFrameInd = arange(cmosinit['firstrawind'],cmosinit['lastrawind']+1,FrameIndReq,dtype=int64)

    finf = {'superx':X,
            'supery':Y,
            'nframeextract':FrameIndRel.size,
            'nframe':rawFrameInd.size,
            'frameindrel':FrameIndRel,
            'frameind':rawFrameInd,
            'kineticsec':kineticsec}
#%% absolute frame timing (software, yikes)
    finf['ut1'] = frame2ut1(startUTC,kineticsec,rawFrameInd)

    return finf, data

def howbig(SuperX,SuperY,nHeadBytes):
    PixelsPerImage= SuperX * SuperY
    BytesPerImage = PixelsPerImage*bpp//8
    BytesPerFrame = BytesPerImage + nHeadBytes
    return PixelsPerImage,BytesPerImage,BytesPerFrame

def whichframes(fn,FrameIndReq,kineticsec,ut1req,startUTC,firstRawInd,lastRawInd,
                BytesPerImage,BytesPerFrame,verbose):
    ext = Path(fn).suffix
#%% get file size
    fileSizeBytes = fn.stat().st_size

    if fileSizeBytes < BytesPerImage:
        raise ValueError('File size {} is smaller than a single image frame!'.format(fileSizeBytes))

    if ext=='.DMCdata' and fileSizeBytes % BytesPerFrame:
        logging.error(f"Looks like I am not reading this file correctly, with BPF: {BytesPerFrame:d}")

    if ext=='.DMCdata':
        nFrame = fileSizeBytes // BytesPerFrame
        logging.info(f'{nFrame} frames, Bytes: {fileSizeBytes} in file {fn}')

        nFrameRaw = (lastRawInd-firstRawInd+1)
        if nFrameRaw != nFrame:
             logging.warning(f'there may be missed frames: nFrameRaw {nFrameRow}   nFrame {nFrame}')
    else:
        nFrame = lastRawInd-firstRawInd+1

    allrawframe = arange(firstRawInd,lastRawInd+1,1,dtype=int64)
    logging.info(f"first / last raw frame #'s: {firstRawInd}  / {lastRawInd} ")
#%% absolute time estimate
    ut1_unix_all = frame2ut1(startUTC,kineticsec,allrawframe)
#%% setup frame indices
    """
    if no requested frames were specified, read all frames. Otherwise, just
    return the requested frames
    note these assignments have to be "int64", not just python "int", because on windows python 2.7 64-bit on files >2.1GB, the bytes will wrap
    """
    FrameIndRel = ut12frame(ut1req,arange(0,nFrame,1,dtype=int64),ut1_unix_all)

    if FrameIndRel is None or len(FrameIndRel)==0: #NOTE: no ut1req or problems with ut1req, canNOT use else, need to test len() in case index is [0] validly
        FrameIndRel = req2frame(FrameIndReq, nFrame)

    badReqInd = (FrameIndRel>nFrame) | (FrameIndRel<0)
# check if we requested frames beyond what the BigFN contains
    if badReqInd.any():
        raise ValueError('You have requested frames outside the times covered in {}'.format(fn)) #don't include frames in case of None

    nFrameExtract = FrameIndRel.size #to preallocate properly
    nBytesExtract = nFrameExtract * BytesPerFrame
    if verbose > 0:
        print('Extracted {} frames from {} totaling {} bytes.'.format(nFrameExtract,fn,nBytesExtract))
    if nBytesExtract > 4e9:
        logging.info('This will require {:.2f} Gigabytes of RAM.'.format(nBytesExtract/1e9))

    return FrameIndRel

def getDMCframe(f,iFrm,finf,verbose=0):
    # on windows, "int" is int32 and overflows at 2.1GB!  We need np.int64
    currByte = iFrm * finf['bytesperframe']
#%% advance to start of frame in bytes
    if verbose>0:
        print(f'seeking to byte {currByte}')

    assert isinstance(currByte,int64),'int32 will fail on files > 2GB'
    try:
        f.seek(currByte,0) #no return value
    except IOError as e:
        raise IOError('I couldnt seek to byte {:d}. try using a 64-bit integer for iFrm \n'
              'is {} a DMCdata file?  {}'.format(currByte,f.name,e))
#%% read data ***LABVIEW USES ROW-MAJOR C ORDERING!!
    try:
        currFrame = fromfile(f, uint16,
                            finf['pixelsperimage']).reshape((finf['supery'],finf['superx']),
                            order='C')
    except ValueError as e:
        raise ValueError('we may have read past end of file?  {}'.format(e))

    rawFrameInd = meta2rawInd(f,finf['nmetadata'])

    if rawFrameInd is None: #2011 no metadata file
        rawFrameInd = iFrm+1 #fallback

    return currFrame,rawFrameInd
<commit_msg>default to print size out of output file<commit_after>#!/usr/bin/env python
"""
reads .DMCdata files and displays them
 Michael Hirsch
NOTE: Observe the dtype=np.int64, this is for Windows Python, that wants to
   default to int32 instead of int64 like everyone else!
 """
from pathlib import Path
import logging
import re
from dateutil.parser import parse
from numpy import int64,uint16,zeros,arange,fromfile
from re import search
from astropy.io import fits
from shutil import disk_usage
#
from . import req2frame,dir2fn,getRawInd,meta2rawInd
from .h5imgwriter import setupimgh5,imgwriteincr
from .timedmc import frame2ut1,ut12frame
#
try:
    import tifffile
except ImportError:
    tifffile=None
#
bpp = 16

def goRead(fn:Path,
           xyPix,xyBin,
           FrameIndReq=None, ut1Req=None,kineticraw=None,
           startUTC=None,cmosinit=None,verbose=0,outfn=None,nHeadBytes:int=4):

    fn = Path(fn).expanduser()
    ext = fn.suffix
#%% optional output file setup
    outfn = dir2fn(outfn,fn,'.h5')
#%% setup data parameters
    if ext in ('.DMCdata','.dat'):
        # preallocate *** LABVIEW USES ROW-MAJOR ORDERING C ORDER
        finf = getDMCparam(fn,xyPix,xyBin,FrameIndReq,ut1Req,kineticraw,startUTC,nHeadBytes,verbose)
        outsize = finf['bytesperframe']*finf['nframeextract']
        if outfn:
            freeout =  disk_usage(outfn.parent).free
            if freeout < 10*outsize:
                raise RuntimeError(f'out of disk space on {outfn.parent}.  {freeout/1e9} GB free, wanting to write {outsize/1e9} GB.')

        rawFrameInd = zeros(finf['nframeextract'], dtype=int64)
#%% output (variable or file)
        if outfn:
            setupimgh5(outfn,finf['nframeextract'],finf['supery'],finf['superx'])
            data = None
        else:
            data = zeros((finf['nframeextract'],finf['supery'],finf['superx']),
                    dtype=uint16, order='C')
#%% read
        with fn.open('rb') as fid: #NOTE: not pathlib due to Python 2.7, Numpy 1.11 incompat. Py3.5 OK
            for j,i in enumerate(finf['frameindrel']): #j and i are NOT the same in general when not starting from beginning of file!
                D, rawFrameInd[j] = getDMCframe(fid,i,finf,verbose)
                if outfn:
                    imgwriteincr(outfn,D,j)
                else:
                    data[j,...] = D
#%% absolute time estimate, software timing (at your peril)
        finf['ut1'] = frame2ut1(startUTC,kineticraw,rawFrameInd)

    elif ext[:4] == '.tif':
        finf,data = getNeoParam(fn,FrameIndReq,ut1Req,kineticraw,startUTC,cmosinit,verbose)
        rawFrameInd = finf['frameind'] #FIXME this is for individual file, not start of night.
    else:
        raise ValueError(f'not sure to do with file {fn}')

    return data, rawFrameInd,finf#,ut1_unix
#%% workers
def getserialnum(flist):
    """
    This function assumes the serial number of the camera is in a particular place in the filename.
    Yes, this is a little lame, but it's how the original 2011 image-writing program worked, and I've
    carried over the scheme rather than appending bits to dozens of TB of files.
    """
    sn = []
    for f in flist:
        tmp = search(r'(?<=CamSer)\d{3,6}',f)
        if tmp:
            ser = int(tmp.group())
        else:
            ser = None
        sn.append(ser)
    return sn

def getDMCparam(fn:Path,xyPix,xyBin,
                FrameIndReq=None,ut1req=None,kineticsec=None,startUTC=None,nHeadBytes=4,verbose=0):
    """
    nHeadBytes=4 for 2013-2016 data
    nHeadBytes=0 for 2011 data
    """
    Nmetadata = nHeadBytes//2 #FIXME for DMCdata version 1 only

    if not fn.is_file(): #leave this here, getsize() doesn't fail on directory
        raise ValueError(f'{fn} is not a file!')

    print(f'reading {fn}')

    #np.int64() in case we are fed a float or int
    SuperX = int64(xyPix[0]) // int64(xyBin[0]) # "//" keeps as integer
    SuperY = int64(xyPix[1]) // int64(xyBin[1])

    PixelsPerImage,BytesPerImage,BytesPerFrame = howbig(SuperX,SuperY,nHeadBytes)

    (firstRawInd,lastRawInd) = getRawInd(fn,BytesPerImage,nHeadBytes,Nmetadata)

    FrameIndRel = whichframes(fn,FrameIndReq,kineticsec,ut1req,startUTC,firstRawInd,lastRawInd,
                              BytesPerImage,BytesPerFrame,verbose)

    return {'superx':SuperX, 'supery':SuperY, 'nmetadata':Nmetadata,
            'bytesperframe':BytesPerFrame, 'pixelsperimage':PixelsPerImage,
            'nframeextract':FrameIndRel.size,
            'frameindrel':FrameIndRel}

def getNeoParam(fn,FrameIndReq=None,ut1req=None,kineticsec=None,startUTC=None,cmosinit={},verbose=False):
    """ assumption is that this is a Neo sCMOS FITS/TIFF file, where Solis chooses to break up the recordings
    into smaller files. Verify if this timing estimate makes sense for your application!
    I did not want to do regexp on the filename or USERTXT1 as I felt this too prone to error.

    inputs:
    -------
    cmosinit = {'firstrawind','lastrawind'}
    """
    fn = Path(fn).expanduser()

    nHeadBytes=0

    if fn.suffix.lower() in '.tiff':
        if tifffile is None: raise ImportError('pip install tifffile')
        #FIXME didn't the 2011 TIFFs have headers? maybe not.
        with tifffile.TiffFile(str(fn)) as f:
            data = f.asarray()
            Y,X = data.shape[-2:]
    elif fn.suffix.lower() in '.fits':
        with fits.open(str(fn),mode='readonly',memmap=False) as f:
            data = None #f[0].data  #NOTE You can read the data if you want, I didn't need it here.

            kineticsec = f[0].header['KCT']
            startseries = parse(f[0].header['DATE'] + 'Z') #TODO start of night's recording (with some Solis versionss)

            #TODO wish there was a better way
            try:
                frametxt = f[0].header['USERTXT1']
                m = re.search('(?<=Images\:)\d+-\d+(?=\.)',frametxt)
                inds = m.group(0).split('-')
            except KeyError: # just a single file?
                inds = [1,f[0].shape[0]] #yes start with 1, end without adding 1 for Andor Solis

            cmosinit={'firstrawind':int(inds[0]),
                      'lastrawind':int(inds[1])}

            #start = parse(f[0].header['FRAME']+'Z') No, incorrect by several hours with some 2015 Solis versions!

            Y,X = f[0].shape[-2:]

        startUTC = startseries.timestamp()

#%% FrameInd relative to this file
    PixelsPerImage,BytesPerImage,BytesPerFrame = howbig(X,Y,nHeadBytes)

    FrameIndRel = whichframes(fn,FrameIndReq,kineticsec,ut1req,startUTC,
                              cmosinit['firstrawind'],cmosinit['lastrawind'],
                              BytesPerImage,BytesPerFrame,verbose)

    assert isinstance(FrameIndReq,int) or FrameIndReq is None, 'TODO: add multi-frame request case'
    rawFrameInd = arange(cmosinit['firstrawind'],cmosinit['lastrawind']+1,FrameIndReq,dtype=int64)

    finf = {'superx':X,
            'supery':Y,
            'nframeextract':FrameIndRel.size,
            'nframe':rawFrameInd.size,
            'frameindrel':FrameIndRel,
            'frameind':rawFrameInd,
            'kineticsec':kineticsec}
#%% absolute frame timing (software, yikes)
    finf['ut1'] = frame2ut1(startUTC,kineticsec,rawFrameInd)

    return finf, data

def howbig(SuperX,SuperY,nHeadBytes):
    PixelsPerImage= SuperX * SuperY
    BytesPerImage = PixelsPerImage*bpp//8
    BytesPerFrame = BytesPerImage + nHeadBytes
    return PixelsPerImage,BytesPerImage,BytesPerFrame

def whichframes(fn,FrameIndReq,kineticsec,ut1req,startUTC,firstRawInd,lastRawInd,
                BytesPerImage,BytesPerFrame,verbose):
    ext = Path(fn).suffix
#%% get file size
    fileSizeBytes = fn.stat().st_size

    if fileSizeBytes < BytesPerImage:
        raise ValueError('File size {} is smaller than a single image frame!'.format(fileSizeBytes))

    if ext=='.DMCdata' and fileSizeBytes % BytesPerFrame:
        logging.error(f"Looks like I am not reading this file correctly, with BPF: {BytesPerFrame:d}")

    if ext=='.DMCdata':
        nFrame = fileSizeBytes // BytesPerFrame
        logging.info(f'{nFrame} frames, Bytes: {fileSizeBytes} in file {fn}')

        nFrameRaw = (lastRawInd-firstRawInd+1)
        if nFrameRaw != nFrame:
             logging.warning(f'there may be missed frames: nFrameRaw {nFrameRow}   nFrame {nFrame}')
    else:
        nFrame = lastRawInd-firstRawInd+1

    allrawframe = arange(firstRawInd,lastRawInd+1,1,dtype=int64)
    logging.info(f"first / last raw frame #'s: {firstRawInd}  / {lastRawInd} ")
#%% absolute time estimate
    ut1_unix_all = frame2ut1(startUTC,kineticsec,allrawframe)
#%% setup frame indices
    """
    if no requested frames were specified, read all frames. Otherwise, just
    return the requested frames
    note these assignments have to be "int64", not just python "int", because on windows python 2.7 64-bit on files >2.1GB, the bytes will wrap
    """
    FrameIndRel = ut12frame(ut1req,arange(0,nFrame,1,dtype=int64),ut1_unix_all)

    if FrameIndRel is None or len(FrameIndRel)==0: #NOTE: no ut1req or problems with ut1req, canNOT use else, need to test len() in case index is [0] validly
        FrameIndRel = req2frame(FrameIndReq, nFrame)

    badReqInd = (FrameIndRel>nFrame) | (FrameIndRel<0)
# check if we requested frames beyond what the BigFN contains
    if badReqInd.any():
        raise ValueError('You have requested frames outside the times covered in {}'.format(fn)) #don't include frames in case of None

    nFrameExtract = FrameIndRel.size #to preallocate properly
    nBytesExtract = nFrameExtract * BytesPerFrame
    print(f'Extracted {nFrameExtract} frames from {fn} totaling {nBytesExtract/1e9:.2f} GB.')

    if nBytesExtract > 4e9:
        print('This will require {nBytesExtract/1e9:.2f} GB of RAM.')

    return FrameIndRel

def getDMCframe(f,iFrm,finf,verbose=0):
    # on windows, "int" is int32 and overflows at 2.1GB!  We need np.int64
    currByte = iFrm * finf['bytesperframe']
#%% advance to start of frame in bytes
    if verbose>0:
        print(f'seeking to byte {currByte}')

    assert isinstance(currByte,int64),'int32 will fail on files > 2GB'
    try:
        f.seek(currByte,0) #no return value
    except IOError as e:
        raise IOError('I couldnt seek to byte {:d}. try using a 64-bit integer for iFrm \n'
              'is {} a DMCdata file?  {}'.format(currByte,f.name,e))
#%% read data ***LABVIEW USES ROW-MAJOR C ORDERING!!
    try:
        currFrame = fromfile(f, uint16,
                            finf['pixelsperimage']).reshape((finf['supery'],finf['superx']),
                            order='C')
    except ValueError as e:
        raise ValueError('we may have read past end of file?  {}'.format(e))

    rawFrameInd = meta2rawInd(f,finf['nmetadata'])

    if rawFrameInd is None: #2011 no metadata file
        rawFrameInd = iFrm+1 #fallback

    return currFrame,rawFrameInd
<|endoftext|>


<commit_before>from django.utils.translation import ugettext as _, ugettext_lazy as _lazy
from django.core import urlresolvers

from gasistafelice.rest.views.blocks.base import BlockSSDataTables, ResourceBlockAction, CREATE_PDF
from gasistafelice.consts import CREATE, EDIT, EDIT_MULTIPLE, VIEW

from gasistafelice.lib.shortcuts import render_to_xml_response, render_to_context_response

from gasistafelice.supplier.models import Supplier
from gasistafelice.supplier.forms import SupplierForm, BaseFormSetWithRequest, formset_factory

from django.http import HttpResponse
from django.template.loader import get_template
#from django.template.loader import render_to_string
from django.template import Context
#from django.template import RequestContext
import xhtml2pdf.pisa as pisa
import cStringIO as StringIO
import cgi, os
from django.conf import settings
from gasistafelice.des.models import Siteattr

from django.utils.encoding import smart_unicode
from flexi_auth.models import ObjectWithContext

import logging
log = logging.getLogger(__name__)

#------------------------------------------------------------------------------#
#                                                                              #
#------------------------------------------------------------------------------#

class Block(BlockSSDataTables):

    BLOCK_NAME = "suppliers_report"
    BLOCK_DESCRIPTION = _("Suppliers")
    BLOCK_VALID_RESOURCE_TYPES = ["site", "gas"]

    COLUMN_INDEX_NAME_MAP = {
        0: 'pk',
        1: 'name',
        2: 'frontman',
        3: 'city',
        4: 'mail',
        5: 'phone',
        6: 'tot_stocks',
        7: 'tot_pacts',
        8: 'tot_eco',
        9: 'certifications_list'
    }

#        10: 'enabled'

    def _get_user_actions(self, request):

        user_actions = []
        des = Siteattr.get_site()

        if request.user.has_perm(CREATE, \
            obj=ObjectWithContext(Supplier, context={'site':des})):

            user_actions.append(
                ResourceBlockAction(
                    block_name = self.BLOCK_NAME,
                    resource = request.resource,
                    name=CREATE, verbose_name=_("Add supplier"),
                    url=urlresolvers.reverse('admin:supplier_supplier_add')
                )
            )

        user_actions += [
            ResourceBlockAction(
                block_name = self.BLOCK_NAME,
                resource = request.resource,
                name=CREATE_PDF, verbose_name=_("Create PDF"),
                popup_form=False,
            ),
        ]

        if request.user.has_perm(EDIT, obj=ObjectWithContext(request.resource)):
            user_actions += [
                ResourceBlockAction(
                    block_name = self.BLOCK_NAME,
                    resource = request.resource,
                    name=VIEW, verbose_name=_("Show"),
                    popup_form=False,
                    method="get",
                ),
                ResourceBlockAction(
                    block_name = self.BLOCK_NAME,
                    resource = request.resource,
                    name=EDIT_MULTIPLE, verbose_name=_("Edit"),
                    popup_form=False,
                    method="get",
                ),
            ]
        return user_actions

    def _get_resource_list(self, request):
        # Suppliers objects filtered without PRIVATE
        qry = request.resource.suppliers.order_by('name')
        #FIXME: change queryset into list
        #qry = filter(lambda t: not t.is_private, qry)
        return qry

    def _get_edit_multiple_form_class(self):
        qs = self._get_resource_list(self.request)
        return formset_factory(
                    form=SupplierForm,
                    formset=BaseFormSetWithRequest,
                    extra=qs.count()
        )

    def _get_records(self, request, querySet):
        """Return records of rendered table fields."""

        data = {}
        i = 0
        c = querySet.count()
        map_info = { }
        av = True

        for i,el in enumerate(querySet):

            key_prefix = 'form-%d' % i
            data.update({
               '%s-id' % key_prefix : el.pk,
               '%s-enabled' % key_prefix : bool(av),
            })

            map_info[el.pk] = {'formset_index' : i}

        data['form-TOTAL_FORMS'] = c #i
        data['form-INITIAL_FORMS'] = c #0
        data['form-MAX_NUM_FORMS'] = 0

        formset = self._get_edit_multiple_form_class()(request, data)

        records = []
        for i, el in enumerate(querySet):

            form = formset[map_info[el.pk]['formset_index']]

            records.append({
               'id' : el.pk,
               'product' : el.product,
               'price' : el.order_price,
               'price_changed' : el.has_changed,
               'tot_gasmembers' : el.tot_gasmembers,
               'unconfirmed' : el.unconfirmed_orders,
               'ordered_amount' : el.tot_amount,
               'ordered_total' : el.tot_price,
               'field_enabled' : "%s %s" % (form['id'], form['enabled']),
            })

        return formset, records, {}

    def _get_pdfrecords_pacts(self, querySet):
        """Return records of rendered table fields."""

        records = []
        #memorize pact, economic and number of products
        nSup = querySet.count()
        pact_count = 0
        nProducts = 0

        for el in querySet:
            if el.pk in (5, 12, 73):
                continue
            pact_count += el.tot_pacts
            nProducts += el.tot_stocks

            records.append({
               'id' : el.pk,
               'name' : el.name,
               'frontman' : el.frontman.report_name,
               'address' : el.address,
               'email' : el.preferred_email_address,
               'phone' : el.preferred_phone_address,
               'fax' : el.preferred_fax_address,
               'tot_stocks' : el.tot_stocks,
               'tot_pacts' : el.tot_pacts,
               'tot_eco' : el.tot_eco,
               'certs' : el.certifications_list,
            })

#            {{row.tot_stocks|floatformat:"-2"}}
#            {{row.tot_pacts|floatformat:"-2"}}
#             € {{row.tot_eco|floatformat:"2"}}
#            {{row.certs|escapejs}}

        return records, nSup -3 , nProducts, pact_count

    def get_response(self, request, resource_type, resource_id, args):

        self.request = request
        self.resource = resource = request.resource

        if args == CREATE_PDF:
            return self._create_pdf()
        else:
            return super(Block, self).get_response(request, resource_type, resource_id, args)

    def _create_pdf(self):

        # Dati di esempio
        #order = self.resource.order
        xres = self.resource
        #querySet = self._get_resource_list(self.request).distinct()
        querySet, nSup, nProducts, nPacts = self._get_pdfrecords_pacts(self._get_resource_list(self.request).order_by('name'))
        context_dict = {
            'order' : xres,
            'recSup' : querySet,
            'Suppliers_count' : nSup,
            'pacts_count' : nPacts,
            'products_count' : nProducts,
            'user' : self.request.user,
        }

        REPORT_TEMPLATE = "blocks/%s/report.html" % self.BLOCK_NAME

        template = get_template(REPORT_TEMPLATE)
        context = Context(context_dict)
        html = template.render(context)
        result = StringIO.StringIO()
        #pdf = pisa.pisaDocument(StringIO.StringIO(html.encode("ISO-8859-1", "ignore")), result)
        pdf = pisa.pisaDocument(StringIO.StringIO(html.encode("UTF-8")), result ) #, link_callback = fetch_resources )
        if not pdf.err:
            response = HttpResponse(result.getvalue(), mimetype='application/pdf')
            response['Content-Disposition'] = "attachment; filename=Suppliers.pdf"
            return response
        return self.response_error(_('We had some errors
%s
') % cgi.escape(html))

    def fetch_resources(uri, rel):
        path = os.path.join(settings.MEDIA_ROOT, uri.replace(settings.MEDIA_URL, ""))
        log.debug("Order report Pisa image path (%s)" % path)
        path = os.path.join(settings.MEDIA_ROOT, '/img/icon_beta3.jpg')
        log.debug("Order report Pisa image path (%s)" % path)
        return path

<commit_msg>Suppliers grid for private<commit_after>from django.utils.translation import ugettext as _, ugettext_lazy as _lazy
from django.core import urlresolvers

from gasistafelice.rest.views.blocks.base import BlockSSDataTables, ResourceBlockAction, CREATE_PDF
from gasistafelice.consts import CREATE, EDIT, EDIT_MULTIPLE, VIEW

from gasistafelice.lib.shortcuts import render_to_xml_response, render_to_context_response

from gasistafelice.supplier.models import Supplier
from gasistafelice.supplier.forms import SupplierForm, BaseFormSetWithRequest, formset_factory

from django.http import HttpResponse
from django.template.loader import get_template
#from django.template.loader import render_to_string
from django.template import Context
#from django.template import RequestContext
import xhtml2pdf.pisa as pisa
import cStringIO as StringIO
import cgi, os
from django.conf import settings
from gasistafelice.des.models import Siteattr

from django.utils.encoding import smart_unicode
from flexi_auth.models import ObjectWithContext

import logging
log = logging.getLogger(__name__)

#------------------------------------------------------------------------------#
#                                                                              #
#------------------------------------------------------------------------------#

class Block(BlockSSDataTables):

    BLOCK_NAME = "suppliers_report"
    BLOCK_DESCRIPTION = _("Suppliers")
    BLOCK_VALID_RESOURCE_TYPES = ["site", "gas"]

    COLUMN_INDEX_NAME_MAP = {
        0: 'pk',
        1: 'name',
        2: 'frontman',
        3: 'city',
        4: 'mail',
        5: 'phone',
        6: 'tot_stocks',
        7: 'tot_pacts',
        8: 'tot_eco',
        9: 'certifications_list'
    }

#        10: 'enabled'

    def _get_user_actions(self, request):

        user_actions = []
        des = Siteattr.get_site()

        if request.user.has_perm(CREATE, \
            obj=ObjectWithContext(Supplier, context={'site':des})):

            user_actions.append(
                ResourceBlockAction(
                    block_name = self.BLOCK_NAME,
                    resource = request.resource,
                    name=CREATE, verbose_name=_("Add supplier"),
                    url=urlresolvers.reverse('admin:supplier_supplier_add')
                )
            )

        user_actions += [
            ResourceBlockAction(
                block_name = self.BLOCK_NAME,
                resource = request.resource,
                name=CREATE_PDF, verbose_name=_("Create PDF"),
                popup_form=False,
            ),
        ]

        if request.user.has_perm(EDIT, obj=ObjectWithContext(request.resource)):
            user_actions += [
                ResourceBlockAction(
                    block_name = self.BLOCK_NAME,
                    resource = request.resource,
                    name=VIEW, verbose_name=_("Show"),
                    popup_form=False,
                    method="get",
                ),
                ResourceBlockAction(
                    block_name = self.BLOCK_NAME,
                    resource = request.resource,
                    name=EDIT_MULTIPLE, verbose_name=_("Edit"),
                    popup_form=False,
                    method="get",
                ),
            ]
        return user_actions

    def _get_resource_list(self, request):
        # Suppliers objects filtered without PRIVATE
        qry = request.resource.suppliers.order_by('name')
        #FIXME: change queryset into list
        #qry = filter(lambda t: not t.is_private, qry)
        return qry

    def _get_edit_multiple_form_class(self):
        qs = self._get_resource_list(self.request)
        return formset_factory(
                    form=SupplierForm,
                    formset=BaseFormSetWithRequest,
                    extra=qs.count()
        )

    def _get_records(self, request, querySet):
        """Return records of rendered table fields."""

        data = {}
        i = 0
        c = querySet.count()
        map_info = { }
        av = True

        for i,el in enumerate(querySet):

            key_prefix = 'form-%d' % i
            data.update({
               '%s-id' % key_prefix : el.pk,
               '%s-enabled' % key_prefix : bool(av),
            })

            map_info[el.pk] = {'formset_index' : i}

        data['form-TOTAL_FORMS'] = c #i
        data['form-INITIAL_FORMS'] = c #0
        data['form-MAX_NUM_FORMS'] = 0

        formset = self._get_edit_multiple_form_class()(request, data)

        records = []
        for i, el in enumerate(querySet):

            form = formset[map_info[el.pk]['formset_index']]

            records.append({
               'id' : el.pk,
               'product' : el.product,
               'price' : el.order_price,
               'price_changed' : el.has_changed,
               'tot_gasmembers' : el.tot_gasmembers,
               'unconfirmed' : el.unconfirmed_orders,
               'ordered_amount' : el.tot_amount,
               'ordered_total' : el.tot_price,
               'field_enabled' : "%s %s" % (form['id'], form['enabled']),
            })

        return formset, records, {}

    def _get_pdfrecords_pacts(self, querySet):
        """Return records of rendered table fields."""

        records = []
        #memorize pact, economic and number of products
        nSup = querySet.count()
        pact_count = 0
        nProducts = 0

        for el in querySet:
            pact_count += el.tot_pacts
            nProducts += el.tot_stocks

            records.append({
               'id' : el.pk,
               'name' : el.name,
               'frontman' : el.frontman.report_name,
               'address' : el.address,
               'email' : el.preferred_email_address,
               'phone' : el.preferred_phone_address,
               'fax' : el.preferred_fax_address,
               'tot_stocks' : el.tot_stocks,
               'tot_pacts' : el.tot_pacts,
               'tot_eco' : el.tot_eco,
               'certs' : el.certifications_list,
            })

#            {{row.tot_stocks|floatformat:"-2"}}
#            {{row.tot_pacts|floatformat:"-2"}}
#             € {{row.tot_eco|floatformat:"2"}}
#            {{row.certs|escapejs}}

        return records, nSup -3 , nProducts, pact_count

    def get_response(self, request, resource_type, resource_id, args):

        self.request = request
        self.resource = resource = request.resource

        if args == CREATE_PDF:
            return self._create_pdf()
        else:
            return super(Block, self).get_response(request, resource_type, resource_id, args)

    def _create_pdf(self):

        # Dati di esempio
        #order = self.resource.order
        xres = self.resource
        #querySet = self._get_resource_list(self.request).distinct()
        querySet, nSup, nProducts, nPacts = self._get_pdfrecords_pacts(self._get_resource_list(self.request).order_by('name'))
        context_dict = {
            'order' : xres,
            'recSup' : querySet,
            'Suppliers_count' : nSup,
            'pacts_count' : nPacts,
            'products_count' : nProducts,
            'user' : self.request.user,
        }

        REPORT_TEMPLATE = "blocks/%s/report.html" % self.BLOCK_NAME

        template = get_template(REPORT_TEMPLATE)
        context = Context(context_dict)
        html = template.render(context)
        result = StringIO.StringIO()
        #pdf = pisa.pisaDocument(StringIO.StringIO(html.encode("ISO-8859-1", "ignore")), result)
        pdf = pisa.pisaDocument(StringIO.StringIO(html.encode("UTF-8")), result ) #, link_callback = fetch_resources )
        if not pdf.err:
            response = HttpResponse(result.getvalue(), mimetype='application/pdf')
            response['Content-Disposition'] = "attachment; filename=Suppliers.pdf"
            return response
        return self.response_error(_('We had some errors
%s
') % cgi.escape(html))

    def fetch_resources(uri, rel):
        path = os.path.join(settings.MEDIA_ROOT, uri.replace(settings.MEDIA_URL, ""))
        log.debug("Order report Pisa image path (%s)" % path)
        path = os.path.join(settings.MEDIA_ROOT, '/img/icon_beta3.jpg')
        log.debug("Order report Pisa image path (%s)" % path)
        return path

<|endoftext|>


<commit_before>from django.test import TestCase
from mock import patch

from smart_manager.base import BaseSmartManager
from smart_manager.tests.smart_managers import UpsertSmartManager
from smart_manager.tests.models import UpsertModel

class BaseSmartManagerTest(TestCase):
    """
    Tests functionality in the base model template.
    """
    def test_template_deepcopy(self):
        """
        Tests that the template can be modified after and it is still intact in the template
        class.
        """
        template = {'hello': 'world'}
        smart_manager = BaseSmartManager(template)

        # Modify the input template
        template['hello'] = 'world2'

        # The template in the model template should remain the same
        self.assertEquals(smart_manager._template, {'hello': 'world'})

    def test_build_object(self):
        """
        Tests that the build_object function upserts the object and adds it to the built objects list.
        """
        smart_manager = BaseSmartManager({})
        smart_manager.build_obj(UpsertModel, char_field='hi', updates={
            'int_field': 1,
        })
        smart_manager.build_obj(UpsertModel, char_field='hi', updates={
            'int_field': 2,
        })

        # There should be one UpsertModel with an int_field of 2
        upsert_model = UpsertModel.objects.get()
        self.assertEquals(upsert_model.char_field, 'hi')
        self.assertEquals(upsert_model.int_field, 2)
        self.assertEquals(UpsertModel.objects.count(), 1)
        self.assertEquals(smart_manager.built_objs, set([upsert_model]))

    def test_build_using(self):
        """
        Tests building using another model template.
        """
        smart_manager = BaseSmartManager({})
        built_objs = smart_manager.build_using(UpsertSmartManager, {'int_field': 1, 'char_field': '2'})
        self.assertTrue(type(built_objs) in (list, tuple,))

        upsert_model = UpsertModel.objects.get()
        self.assertEquals(upsert_model.int_field, 1)
        self.assertEquals(upsert_model.char_field, '2')
        self.assertEquals(UpsertModel.objects.count(), 1)
        self.assertEquals(smart_manager.built_objs, set([upsert_model]))

    @patch('smart_manager.base.BaseSmartManager.build', spec_set=True)
    def test_multi_build_using(self, mock_build):
        """
        Hits the branch for .build returning a list or tuple
        :type mock_build: Mock
        """
        mock_build.return_value = ['one', 'two']

        smart_manager = BaseSmartManager({})
        built_objs = smart_manager.build_using(UpsertSmartManager, {'int_field': 1, 'char_field': '2'})
        self.assertTrue(type(built_objs) in (list, tuple,))
<commit_msg>* patch correct method<commit_after>from django.test import TestCase
from mock import patch

from smart_manager.base import BaseSmartManager
from smart_manager.tests.smart_managers import UpsertSmartManager
from smart_manager.tests.models import UpsertModel

class BaseSmartManagerTest(TestCase):
    """
    Tests functionality in the base model template.
    """
    def test_template_deepcopy(self):
        """
        Tests that the template can be modified after and it is still intact in the template
        class.
        """
        template = {'hello': 'world'}
        smart_manager = BaseSmartManager(template)

        # Modify the input template
        template['hello'] = 'world2'

        # The template in the model template should remain the same
        self.assertEquals(smart_manager._template, {'hello': 'world'})

    def test_build_object(self):
        """
        Tests that the build_object function upserts the object and adds it to the built objects list.
        """
        smart_manager = BaseSmartManager({})
        smart_manager.build_obj(UpsertModel, char_field='hi', updates={
            'int_field': 1,
        })
        smart_manager.build_obj(UpsertModel, char_field='hi', updates={
            'int_field': 2,
        })

        # There should be one UpsertModel with an int_field of 2
        upsert_model = UpsertModel.objects.get()
        self.assertEquals(upsert_model.char_field, 'hi')
        self.assertEquals(upsert_model.int_field, 2)
        self.assertEquals(UpsertModel.objects.count(), 1)
        self.assertEquals(smart_manager.built_objs, set([upsert_model]))

    def test_build_using(self):
        """
        Tests building using another model template.
        """
        smart_manager = BaseSmartManager({})
        built_objs = smart_manager.build_using(UpsertSmartManager, {'int_field': 1, 'char_field': '2'})
        self.assertTrue(type(built_objs) in (list, tuple,))

        upsert_model = UpsertModel.objects.get()
        self.assertEquals(upsert_model.int_field, 1)
        self.assertEquals(upsert_model.char_field, '2')
        self.assertEquals(UpsertModel.objects.count(), 1)
        self.assertEquals(smart_manager.built_objs, set([upsert_model]))

    @patch('smart_manager.tests.smart_managers.UpsertSmartManager.build', spec_set=True)
    def test_multi_build_using(self, mock_build):
        """
        Hits the branch for .build returning a list or tuple
        :type mock_build: Mock
        """
        mock_build.return_value = ['one', 'two']

        smart_manager = BaseSmartManager({})
        built_objs = smart_manager.build_using(UpsertSmartManager, {'int_field': 1, 'char_field': '2'})
        self.assertTrue(type(built_objs) in (list, tuple,))
<|endoftext|>


<commit_before>from django.shortcuts import render_to_response
from search.models import *
import query

def index(request):
    q = request.GET.get('q') # the query terms
    s = request.GET.get('s') # sort type: a - authority, h - hubbiness
    b = request.GET.get('b') # beta: weigting between word frequency and authority/hubbiness

    try:
        b = float(b)
        if b > 1.0 or b < 0.0: b = 0.7
    except Exception:
        b = .7

    print b
    print float(b)

    if q:
        results = query.get_results(q, "authority" if s == "a" else "hubbiness", b)
        return render_to_response('search/results.html', { 'q': q, 's': s, 'b': b, 'results': results })
    else:
        return render_to_response('search/index.html', { })
<commit_msg>removed unnecessary print statements in views.py<commit_after>from django.shortcuts import render_to_response
from search.models import *
import query

def index(request):
    q = request.GET.get('q') # the query terms
    s = request.GET.get('s') # sort type: a - authority, h - hubbiness
    b = request.GET.get('b') # beta: weigting between word frequency and authority/hubbiness

    try:
        b = float(b)
        if b > 1.0 or b < 0.0: b = 0.7
    except Exception:
        b = .7

    if q:
        results = query.get_results(q, "authority" if s == "a" else "hubbiness", b)
        return render_to_response('search/results.html', { 'q': q, 's': s, 'b': b, 'results': results })
    else:
        return render_to_response('search/index.html', { })
<|endoftext|>