<commit_before>#from itertools import chain
import sqlalchemy as sa
#from sqlalchemy_utils.functions import primary_keys
from .operation import Operation
from .utils import versioned_column_properties, parent_class

def first_level(paths):
    for path in paths:
        yield path.split('.')[0]

def subpaths(paths, name):
    for path in paths:
        parts = path.split('.')
        if len(parts) > 1 and parts[0] == name:
            yield '.'.join(parts[1:])

class ReverterException(Exception):
    pass

class Reverter(object):
    def __init__(self, obj, visited_objects=[], relations=[]):
        self.visited_objects = visited_objects
        self.obj = obj
        self.version_parent = self.obj.version_parent
        self.parent_class = parent_class(self.obj.__class__)
        self.parent_mapper = sa.inspect(self.parent_class)

        self.relations = list(relations)
        for path in relations:
            subpath = path.split('.')[0]
            if subpath not in self.parent_mapper.relationships:
                raise ReverterException(
                    "Could not initialize Reverter. Class '%s' does not have "
                    "relationship '%s'." % (
                        parent_class(self.obj.__class__).__name__,
                        subpath
                    )
                )

    def revert_properties(self):
        for prop in versioned_column_properties(self.parent_class):
            setattr(
                self.version_parent,
                prop.key,
                getattr(self.obj, prop.key)
            )

    def revert_relationships(self):
        for prop in self.parent_mapper.iterate_properties:
            if isinstance(prop, sa.orm.RelationshipProperty):
                if prop.key in ['versions', 'transaction']:
                    continue

                if prop.key not in first_level(self.relations):
                    continue

                if prop.secondary is not None:
                    setattr(self.version_parent, prop.key, [])
                    for value in getattr(self.obj, prop.key):
                        value = Reverter(
                            value,
                            visited_objects=self.visited_objects,
                            relations=subpaths(self.relations, prop.key)
                        )()
                        if value:
                            getattr(self.version_parent, prop.key).append(
                                value
                            )
                else:
                    for value in getattr(self.obj, prop.key):
                        Reverter(
                            value,
                            visited_objects=self.visited_objects,
                            relations=subpaths(self.relations, prop.key)
                        )()

    def __call__(self):
        if self.obj in self.visited_objects:
            return

        session = sa.orm.object_session(self.obj)

        if self.obj.operation_type == Operation.DELETE:
            session.delete(self.version_parent)
            return

        self.visited_objects.append(self.obj)

        # Check if parent object has been deleted
        if self.version_parent is None:
            self.version_parent = parent_class(self.obj.__class__)()
            session.add(self.version_parent)

        # Before reifying relations we need to reify object properties. This
        # is needed because reifying relations might need to flush the session
        # which leads to errors when sqlalchemy tries to insert null values
        # into parent object (if parent object has not null constraints).
        self.revert_properties()
        self.revert_relationships()

        return self.version_parent
<commit_msg>Refactor reverter<commit_after>#from itertools import chain
import sqlalchemy as sa
#from sqlalchemy_utils.functions import primary_keys
from .operation import Operation
from .utils import versioned_column_properties, parent_class

def first_level(paths):
    for path in paths:
        yield path.split('.')[0]

def subpaths(paths, name):
    for path in paths:
        parts = path.split('.')
        if len(parts) > 1 and parts[0] == name:
            yield '.'.join(parts[1:])

class ReverterException(Exception):
    pass

class Reverter(object):
    def __init__(self, obj, visited_objects=[], relations=[]):
        self.visited_objects = visited_objects
        self.obj = obj
        self.version_parent = self.obj.version_parent
        self.parent_class = parent_class(self.obj.__class__)
        self.parent_mapper = sa.inspect(self.parent_class)

        self.relations = list(relations)
        for path in relations:
            subpath = path.split('.')[0]
            if subpath not in self.parent_mapper.relationships:
                raise ReverterException(
                    "Could not initialize Reverter. Class '%s' does not have "
                    "relationship '%s'." % (
                        parent_class(self.obj.__class__).__name__,
                        subpath
                    )
                )

    def revert_properties(self):
        for prop in versioned_column_properties(self.parent_class):
            setattr(
                self.version_parent,
                prop.key,
                getattr(self.obj, prop.key)
            )

    def revert_relationship(self, prop):
        if prop.secondary is not None:
            setattr(self.version_parent, prop.key, [])
            for value in getattr(self.obj, prop.key):
                value = Reverter(
                    value,
                    visited_objects=self.visited_objects,
                    relations=subpaths(self.relations, prop.key)
                )()
                if value:
                    getattr(self.version_parent, prop.key).append(
                        value
                    )
        else:
            for value in getattr(self.obj, prop.key):
                Reverter(
                    value,
                    visited_objects=self.visited_objects,
                    relations=subpaths(self.relations, prop.key)
                )()

    def revert_relationships(self):
        for prop in self.parent_mapper.iterate_properties:
            if isinstance(prop, sa.orm.RelationshipProperty):
                if prop.key in ['versions', 'transaction']:
                    continue

                if prop.key not in first_level(self.relations):
                    continue

                self.revert_relationship(prop)

    def __call__(self):
        if self.obj in self.visited_objects:
            return

        session = sa.orm.object_session(self.obj)

        if self.obj.operation_type == Operation.DELETE:
            session.delete(self.version_parent)
            return

        self.visited_objects.append(self.obj)

        # Check if parent object has been deleted
        if self.version_parent is None:
            self.version_parent = parent_class(self.obj.__class__)()
            session.add(self.version_parent)

        # Before reifying relations we need to reify object properties. This
        # is needed because reifying relations might need to flush the session
        # which leads to errors when sqlalchemy tries to insert null values
        # into parent object (if parent object has not null constraints).
        self.revert_properties()
        self.revert_relationships()

        return self.version_parent
<|endoftext|>


<commit_before>from datetime import datetime

from sqlobject import col, events, SQLObject, AND

class Version(SQLObject):
    def restore(self):
        values = self.sqlmeta.asDict()
        del values['id']
        del values['masterID']
        del values['dateArchived']
        for col in self.extraCols:
            del values[col]
        self.masterClass.get(self.masterID).set(**values)

    def nextVersion(self):
        version = self.select(
            AND(self.q.masterID == self.masterID, self.q.id > self.id),
            orderBy=self.q.id)
        if version.count():
            return version[0]
        else:
            return self.master

    def getChangedFields(self):
        next = self.nextVersion()
        columns = self.masterClass.sqlmeta.columns
        fields = []
        for column in columns:
            if column not in ["dateArchived", "id", "masterID"]:
                if getattr(self, column) != getattr(next, column):
                    fields.append(column.title())

        return fields

    @classmethod
    def select(cls, clause=None, *args, **kw):
        if not getattr(cls, '_connection', None):
            cls._connection = cls.masterClass._connection
        return super(Version, cls).select(clause, *args, **kw)

    def __getattr__(self, attr):
        if attr in self.__dict__:
            return self.__dict__[attr]
        else:
            return getattr(self.master, attr)

def getColumns(columns, cls):
    for column, defi in cls.sqlmeta.columnDefinitions.items():
        if column.endswith("ID") and isinstance(defi, col.ForeignKey):
            column = column[:-2]

        # remove incompatible constraints
        kwds = dict(defi._kw)
        for kw in ["alternateID", "unique"]:
            if kw in kwds:
                del kwds[kw]
        columns[column] = defi.__class__(**kwds)

    # ascend heirarchy
    if cls.sqlmeta.parentClass:
        getColumns(columns, cls.sqlmeta.parentClass)

class Versioning(object):
    def __init__(self, extraCols=None):
        if extraCols:
            self.extraCols = extraCols
        else:
            self.extraCols = {}
        pass

    def __addtoclass__(self, soClass, name):
        self.name = name
        self.soClass = soClass

        attrs = {'dateArchived': col.DateTimeCol(default=datetime.now),
                 'master': col.ForeignKey(self.soClass.__name__),
                 'masterClass': self.soClass,
                 'extraCols': self.extraCols
                 }

        getColumns(attrs, self.soClass)

        attrs.update(self.extraCols)

        self.versionClass = type(self.soClass.__name__ + 'Versions',
                                 (Version,),
                                 attrs)

        if '_connection' in self.soClass.__dict__:
            self.versionClass._connection = \
                self.soClass.__dict__['_connection']

        events.listen(self.createTable,
                      soClass, events.CreateTableSignal)
        events.listen(self.rowUpdate, soClass,
                      events.RowUpdateSignal)

    def createVersionTable(self, cls, conn):
        self.versionClass.createTable(ifNotExists=True, connection=conn)

    def createTable(self, soClass, connection, extra_sql, post_funcs):
        assert soClass is self.soClass
        post_funcs.append(self.createVersionTable)

    def rowUpdate(self, instance, kwargs):
        if instance.childName and instance.childName != self.soClass.__name__:
            return  # if you want your child class versioned, version it

        values = instance.sqlmeta.asDict()
        del values['id']
        values['masterID'] = instance.id
        self.versionClass(connection=instance._connection, **values)

    def __get__(self, obj, type=None):
        if obj is None:
            return self
        return self.versionClass.select(
            self.versionClass.q.masterID == obj.id, connection=obj._connection)
<commit_msg>Fix #33 flake8 F402 import 'name' shadowed by loop variable<commit_after>from datetime import datetime
from sqlobject import col, events, SQLObject, AND

class Version(SQLObject):
    def restore(self):
        values = self.sqlmeta.asDict()
        del values['id']
        del values['masterID']
        del values['dateArchived']
        for _col in self.extraCols:
            del values[_col]
        self.masterClass.get(self.masterID).set(**values)

    def nextVersion(self):
        version = self.select(
            AND(self.q.masterID == self.masterID, self.q.id > self.id),
            orderBy=self.q.id)
        if version.count():
            return version[0]
        else:
            return self.master

    def getChangedFields(self):
        next = self.nextVersion()
        columns = self.masterClass.sqlmeta.columns
        fields = []
        for column in columns:
            if column not in ["dateArchived", "id", "masterID"]:
                if getattr(self, column) != getattr(next, column):
                    fields.append(column.title())

        return fields

    @classmethod
    def select(cls, clause=None, *args, **kw):
        if not getattr(cls, '_connection', None):
            cls._connection = cls.masterClass._connection
        return super(Version, cls).select(clause, *args, **kw)

    def __getattr__(self, attr):
        if attr in self.__dict__:
            return self.__dict__[attr]
        else:
            return getattr(self.master, attr)

def getColumns(columns, cls):
    for column, defi in cls.sqlmeta.columnDefinitions.items():
        if column.endswith("ID") and isinstance(defi, col.ForeignKey):
            column = column[:-2]

        # remove incompatible constraints
        kwds = dict(defi._kw)
        for kw in ["alternateID", "unique"]:
            if kw in kwds:
                del kwds[kw]
        columns[column] = defi.__class__(**kwds)

    # ascend heirarchy
    if cls.sqlmeta.parentClass:
        getColumns(columns, cls.sqlmeta.parentClass)

class Versioning(object):
    def __init__(self, extraCols=None):
        if extraCols:
            self.extraCols = extraCols
        else:
            self.extraCols = {}
        pass

    def __addtoclass__(self, soClass, name):
        self.name = name
        self.soClass = soClass

        attrs = {'dateArchived': col.DateTimeCol(default=datetime.now),
                 'master': col.ForeignKey(self.soClass.__name__),
                 'masterClass': self.soClass,
                 'extraCols': self.extraCols
                 }

        getColumns(attrs, self.soClass)

        attrs.update(self.extraCols)

        self.versionClass = type(self.soClass.__name__ + 'Versions',
                                 (Version,),
                                 attrs)

        if '_connection' in self.soClass.__dict__:
            self.versionClass._connection = \
                self.soClass.__dict__['_connection']

        events.listen(self.createTable,
                      soClass, events.CreateTableSignal)
        events.listen(self.rowUpdate, soClass,
                      events.RowUpdateSignal)

    def createVersionTable(self, cls, conn):
        self.versionClass.createTable(ifNotExists=True, connection=conn)

    def createTable(self, soClass, connection, extra_sql, post_funcs):
        assert soClass is self.soClass
        post_funcs.append(self.createVersionTable)

    def rowUpdate(self, instance, kwargs):
        if instance.childName and instance.childName != self.soClass.__name__:
            return  # if you want your child class versioned, version it

        values = instance.sqlmeta.asDict()
        del values['id']
        values['masterID'] = instance.id
        self.versionClass(connection=instance._connection, **values)

    def __get__(self, obj, type=None):
        if obj is None:
            return self
        return self.versionClass.select(
            self.versionClass.q.masterID == obj.id, connection=obj._connection)
<|endoftext|>


<commit_before>from django.forms import CharField, ModelForm, DateInput
from django.utils.translation import gettext_lazy as _

from modeltranslation import forms as mt_forms, translator

from submission import models as submission_models

class JanewayTranslationModelForm(mt_forms.TranslationModelForm):
    def __init__(self, *args, **kwargs):
        super(JanewayTranslationModelForm, self).__init__(*args, **kwargs)
        opts = translator.translator.get_options_for_model(self._meta.model)
        self.translated_field_names = opts.get_field_names()

class FakeModelForm(ModelForm):
    """ A form that can't be saved

    Usefull for rendering a sample form
    """

    def __init__(self, *args, disable_fields=False, **kwargs):
        super().__init__(*args, **kwargs)
        self.disable_fields = disable_fields
        if disable_fields is True:
            for field in self.fields:
                self.fields[field].widget.attrs["readonly"] = True

    def save(self, *args, **kwargs):
        raise NotImplementedError("FakeModelForm can't be saved")

    def clean(self, *args, **kwargs):
        if self.disable_fields is True:
            raise NotImplementedError(
                "FakeModelForm can't be cleaned: disable_fields is True",
            )
        return super().clean(*args, **kwargs)

class KeywordModelForm(ModelForm):
    """ A ModelForm for models implementing a Keyword M2M relationship """
    keywords = CharField(
            required=False, help_text=_("Hit Enter to add a new keyword."))

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        current_keywords = self.instance.keywords.values_list(
            "word", flat=True)
        field = self.fields["keywords"]
        field.initial = ",".join(current_keywords)

    def save(self, commit=True, *args, **kwargs):
        instance = super().save(commit=commit, *args, **kwargs)

        posted_keywords = self.cleaned_data.get('keywords', '').split(',')
        instance.keywords.clear()
        for i, keyword in enumerate(posted_keywords):
                obj, _ = submission_models.Keyword.objects.get_or_create(
                        word=keyword)
                instance.keywords.add(obj)
        if commit:
            instance.save()
        return instance

class HTMLDateInput(DateInput):
    input_type = 'date'

    def __init__(self, **kwargs):
        kwargs["format"] = "%Y-%m-%d"
        super().__init__(**kwargs)
<commit_msg>Avoid posting empty keywords via article edit form<commit_after>from django.forms import CharField, ModelForm, DateInput
from django.utils.translation import gettext_lazy as _

from modeltranslation import forms as mt_forms, translator

from submission import models as submission_models

class JanewayTranslationModelForm(mt_forms.TranslationModelForm):
    def __init__(self, *args, **kwargs):
        super(JanewayTranslationModelForm, self).__init__(*args, **kwargs)
        opts = translator.translator.get_options_for_model(self._meta.model)
        self.translated_field_names = opts.get_field_names()

class FakeModelForm(ModelForm):
    """ A form that can't be saved

    Usefull for rendering a sample form
    """

    def __init__(self, *args, disable_fields=False, **kwargs):
        super().__init__(*args, **kwargs)
        self.disable_fields = disable_fields
        if disable_fields is True:
            for field in self.fields:
                self.fields[field].widget.attrs["readonly"] = True

    def save(self, *args, **kwargs):
        raise NotImplementedError("FakeModelForm can't be saved")

    def clean(self, *args, **kwargs):
        if self.disable_fields is True:
            raise NotImplementedError(
                "FakeModelForm can't be cleaned: disable_fields is True",
            )
        return super().clean(*args, **kwargs)

class KeywordModelForm(ModelForm):
    """ A ModelForm for models implementing a Keyword M2M relationship """
    keywords = CharField(
            required=False, help_text=_("Hit Enter to add a new keyword."))

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        current_keywords = self.instance.keywords.values_list(
            "word", flat=True)
        field = self.fields["keywords"]
        field.initial = ",".join(current_keywords)

    def save(self, commit=True, *args, **kwargs):
        instance = super().save(commit=commit, *args, **kwargs)
        instance.keywords.clear()
        posted_keywords = self.cleaned_data.get(
            'keywords', '').split(',')
        if posted_keywords:
            for i, keyword in enumerate(posted_keywords):
                obj, _ = submission_models.Keyword.objects.get_or_create(
                    word=keyword)
                instance.keywords.add(obj)
        if commit:
            instance.save()
        return instance

class HTMLDateInput(DateInput):
    input_type = 'date'

    def __init__(self, **kwargs):
        kwargs["format"] = "%Y-%m-%d"
        super().__init__(**kwargs)
<|endoftext|>


<commit_before>import matplotlib.pyplot as plt
from keras.preprocessing import image as keras_image_preprocessor
from keras.applications.vgg16 import preprocess_input
import glob
import numpy as np
from utils.boxes import decode_boxes
from utils.boxes import filter_boxes

def preprocess_images(image_array):
    return preprocess_input(image_array)

# remove this function
def list_files_in_directory(path_name='*'):
    return glob.glob(path_name)

def predict_boxes(model, image_array, prior_boxes,
                    class_threshold=.1,
                    box_scale_factors=[.1, .1, .2, .2],
                    num_classes=21, background_id=0):
    image_array = np.expand_dims(image_array, axis=0)
    image_array = preprocess_images(image_array)
    predictions = model.predict(image_array)
    predictions = np.squeeze(predictions)
    predictions = decode_boxes(predictions, prior_boxes,
                                    box_scale_factors)
    predictions = filter_boxes(predictions, num_classes,
                                background_id, class_threshold)

def load_image(image_path, grayscale=False ,target_size=None):
    image = keras_image_preprocessor.load_img(image_path,
                                                grayscale ,
                                    target_size=target_size)
    return keras_image_preprocessor.img_to_array(image)

def scheduler(epoch, decay=0.9, base_learning_rate=3e-4):
    return base_learning_rate * decay**(epoch)

def split_data(ground_truths, training_ratio=.8):
    ground_truth_keys = sorted(ground_truths.keys())
    num_train = int(round(training_ratio * len(ground_truth_keys)))
    train_keys = ground_truth_keys[:num_train]
    validation_keys = ground_truth_keys[num_train:]
    return train_keys, validation_keys

def plot_images(original_image, transformed_image):
    plt.figure(1)
    plt.subplot(121)
    plt.title('Original image')
    plt.imshow(original_image.astype('uint8'))
    plt.subplot(122)
    plt.title('Transformed image')
    plt.imshow(transformed_image.astype('uint8'))
    plt.show()

# move this function to dataset module
def get_class_names(dataset_name='VOC2007'):
    if dataset_name == 'VOC2007':
        class_names = ['background','aeroplane', 'bicycle', 'bird', 'boat',
                       'bottle', 'bus', 'car', 'cat', 'chair', 'cow',
                       'diningtable', 'dog', 'horse', 'motorbike', 'person',
                       'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor']
    elif dataset_name == 'COCO':
        class_names = ['background', 'person', 'bicycle', 'car', 'motorcycle',
                        'airplane', 'bus', 'train', 'truck', 'boat',
                        'traffic light', 'fire hydrant', 'stop sign',
                        'parking meter', 'bench', 'bird', 'cat', 'dog',
                        'horse', 'sheep', 'cow', 'elephant', 'bear',
                        'zebra', 'giraffe', 'backpack', 'umbrella',
                        'handbag', 'tie', 'suitcase', 'frisbee', 'skis',
                        'snowboard', 'sports ball', 'kite', 'baseball bat',
                        'baseball glove', 'skateboard', 'surfboard',
                        'tennis racket', 'bottle', 'wine glass',
                        'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana',
                        'apple', 'sandwich', 'orange', 'broccoli', 'carrot',
                        'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch',
                        'potted plant', 'bed', 'dining table', 'toilet',
                        'tv', 'laptop', 'mouse', 'remote', 'keyboard',
                        'cell phone', 'microwave', 'oven', 'toaster',
                        'sink', 'refrigerator', 'book', 'clock', 'vase',
                        'scissors', 'teddy bear', 'hair drier', 'toothbrush']
    else:
        raise Exception('Invalid dataset', dataset_name)
    return class_names

def get_arg_to_class(class_names):
    return dict(zip(list(range(len(class_names))), class_names))
<commit_msg>Distributed all function to the other utils.<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# Name: seccheck
# Purpose: Check an IP against Nessus Security Center for Known Exploitable Vulnerabilities
# By:       Jerry Gamblin
# Date:     13.05.15
# Modified  13.05.15
# Rev Level 0.1
# Run 'pip install pysecuritycenter' for library
# -----------------------------------------------

import os
import sys
import json
import urllib
import urllib2
import hashlib
import argparse
import re
import socket
import ssl
import socket
import getopt
from urllib2 import urlopen
from securitycenter import SecurityCenter

# Provide the login info & security center address here:
username = ''
password = ''
host = ''

def color(text, color_code):
    if sys.platform == "win32" and os.getenv("TERM") != "xterm":
        return text

    return '\x1b[%dm%s\x1b[0m' % (color_code, text)

def red(text):
    return color(text, 31)

def blink(text):
    return color(text, 5)

def green(text):
    return color(text, 32)

def blue(text):
    return color(text, 34)

#Get IP To SCAN
if len(sys.argv) != 2:
    print("Error: specify an IP to connect to!")
    exit(0)

ip = sys.argv[1]

checkip = sys.argv[1]

#Welcome:
print 'Quickly Check Security For Exploitable Vulnerabilities For An IP'
print 'A quick and dirty script by @jgamblin'
print('\n')

#Ignore TLS Cert Error
if hasattr(ssl, '_create_unverified_context'):
	ssl._create_default_https_context = ssl._create_unverified_context

# Instantiate a Security Center instance and login with the credentials provided
sc = SecurityCenter(host, username, password)

#
vulns = sc.query('vulndetails', exploitAvailable='true', pluginType='active', severity='3,4', ip=checkip)

# Set IP Address:
ips ={}
if not vulns:
    print 'You Probably Cant Hack %. Congrats!  : )'  %ip
else:
    for vuln in vulns:
        if vuln['ip'] not in ips:
            ips[vuln['ip']] = []
        ips[vuln['ip']].append(vuln)

# Now to print the output to the screen.  This could easily be rewritten to
# output to a file as well, or even parse it into a CSV file if needed.
for ip in ips:
    print 'Yep, you can hack %. Here is how:' %ip
    print '\n'
    for vuln in ips[ip]:
	print '%s' % vuln['pluginName']
	print 'Base CVS Score: %s' % vuln['baseScore']
	print 'Severity: %s' % vuln['severity']
	print ((red('Synopsis:')) +'%s') % vuln['synopsis']
	print ((red('CVE:')) +'%s') % vuln['cve']
	print '\n'
<commit_msg>Update canyouhack.py<commit_after>#!/usr/bin/env python
# Name: seccheck
# Purpose: Check an IP against Nessus Security Center for Known Exploitable Vulnerabilities
# By:       Jerry Gamblin
# Date:     13.05.15
# Modified  13.05.15
# Rev Level 0.1
# Run 'pip install pysecuritycenter' for library
# -----------------------------------------------

import os
import sys
import json
import urllib
import urllib2
import hashlib
import argparse
import re
import socket
import ssl
import socket
import getopt
from urllib2 import urlopen
from securitycenter import SecurityCenter

# Provide the login info & security center address here:
username = ''
password = ''
host = ''

def color(text, color_code):
    if sys.platform == "win32" and os.getenv("TERM") != "xterm":
        return text

    return '\x1b[%dm%s\x1b[0m' % (color_code, text)

def red(text):
    return color(text, 31)

def blink(text):
    return color(text, 5)

def green(text):
    return color(text, 32)

def blue(text):
    return color(text, 34)

#Get IP To SCAN
if len(sys.argv) != 2:
    print("Error: specify an IP to connect to!")
    exit(0)

ip = sys.argv[1]

checkip = sys.argv[1]

#Ignore TLS Cert Error
if hasattr(ssl, '_create_unverified_context'):
	ssl._create_default_https_context = ssl._create_unverified_context

# Instantiate a Security Center instance and login with the credentials provided
sc = SecurityCenter(host, username, password)

#
vulns = sc.query('vulndetails', exploitAvailable='true', pluginType='active', severity='3,4', ip=checkip)

# Set IP Address:
ips ={}
if not vulns:
    print '\nYou Probably Cant Hack %s. Congrats!  : ) \n' % ip
else:
    for vuln in vulns:
        if vuln['ip'] not in ips:
            ips[vuln['ip']] = []
        ips[vuln['ip']].append(vuln)

# Now to print the output to the screen.  This could easily be rewritten to
# output to a file as well, or even parse it into a CSV file if needed.
for ip in ips:
    print 'Yep, you can hack %s. \nHere is how:' %ip
    print '\n'
    for vuln in ips[ip]:
	print '%s' % vuln['pluginName']
	print 'Base CVS Score: %s' % vuln['baseScore']
	print 'Severity: %s' % vuln['severity']
	print ((red('Synopsis:')) +'%s') % vuln['synopsis']
	print ((red('CVE:')) +'%s') % vuln['cve']
	print '\n'
<|endoftext|>


<commit_before>import threading
import SocketServer
import serial
import cv2
import numpy as np
import math
# distance data measured by ultrasonic sensor
sensor_data = " "

class DistanceToCamera(object):

    def __init__(self):
        # camera params
        self.alpha = 8.0 * math.pi / 180
        self.v0 = 119.865631204
        self.ay = 332.262498472

    def calculate(self, v, h, x_shift, image):
        # compute and return the distance from the target point to the camera
        d = h / math.tan(self.alpha + math.atan((v - self.v0) / self.ay))
        if d > 0:
            cv2.putText(image, "%.1fcm" % d,
                (image.shape[1] - x_shift, image.shape[0] - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2)
        return d

class SensorDataHandler(SocketServer.BaseRequestHandler):

    data = " "

    def handle(self):
        global sensor_data
        try:
            while self.data:
                self.data = self.request.recv(1024)
                sensor_data = round(float(self.data), 1)
                #print "{} sent:".format(self.client_address[0])
                print sensor_data
        finally:
            print "Connection closed on thread 2"

class VideoStreamHandler(SocketServer.StreamRequestHandler):

    # h1: stop sign
    h1 = 15.5 - 10  # cm
    # h2: traffic light
    h2 = 15.5 - 10
    d_to_camera = DistanceToCamera()

    def handle(self):
        global sensor_data
        stream_bytes = ' '
        stop_flag = False
        stop_sign_active = True
        # stream video frames one by one
        try:
            while True:
                stream_bytes += self.rfile.read(1024)
                first = stream_bytes.find('\xff\xd8')
                last = stream_bytes.find('\xff\xd9')
                if first != -1 and last != -1:
                    jpg = stream_bytes[first:last+2]
                    stream_bytes = stream_bytes[last+2:]
                    gray = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8), cv2.CV_LOAD_IMAGE_GRAYSCALE)
                    image = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8), cv2.CV_LOAD_IMAGE_UNCHANGED)

                    # lower half of the image
                    half_gray = gray[120:240, :]

                cv2.imshow('image', image)
                # cv2.imshow('mlp_image', half_gray)
                if cv2.waitKey(1) & 0xFF == ord('q'):
                    self.rc_car.stop()
                    break
            cv2.destroyAllWindows()
        finally:
            print "Connection closed on thread 1"
class ThreadServer(object):

    def server_thread(host, port):
        server = SocketServer.TCPServer((host, port), VideoStreamHandler)
        server.serve_forever()

    def server_thread2(host, port):
        server = SocketServer.TCPServer((host, port), SensorDataHandler)
        server.serve_forever()

    distance_thread = threading.Thread(target=server_thread2, args=('127.0.0.1', 8004))
    distance_thread.start()

    video_thread = threading.Thread(target=server_thread('127.0.0.1', 8002))
    video_thread.start()
if __name__ == '__main__':
    ThreadServer()<commit_msg>client server<commit_after>import threading
import SocketServer
import serial
import cv2
import numpy as np
import math
# distance data measured by ultrasonic sensor
sensor_data = " "

class DistanceToCamera(object):

    def __init__(self):
        # camera params
        self.alpha = 8.0 * math.pi / 180
        self.v0 = 119.865631204
        self.ay = 332.262498472

    def calculate(self, v, h, x_shift, image):
        # compute and return the distance from the target point to the camera
        d = h / math.tan(self.alpha + math.atan((v - self.v0) / self.ay))
        if d > 0:
            cv2.putText(image, "%.1fcm" % d,
                (image.shape[1] - x_shift, image.shape[0] - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2)
        return d

class SensorDataHandler(SocketServer.BaseRequestHandler):

    data = " "

    def handle(self):
        global sensor_data
        try:
            while self.data:
                self.data = self.request.recv(1024)
                #3sensor_data = round(float(self.data), 1)
                print "{} sent:".format(self.client_address[0])
                print sensor_data
        finally:
            print "Connection closed on thread 2"

class VideoStreamHandler(SocketServer.StreamRequestHandler):

    # h1: stop sign
    h1 = 15.5 - 10  # cm
    # h2: traffic light
    h2 = 15.5 - 10
    d_to_camera = DistanceToCamera()

    def handle(self):
        global sensor_data
        stream_bytes = ' '
        stop_flag = False
        stop_sign_active = True
        # stream video frames one by one
        try:
            while True:
                stream_bytes += self.rfile.read(1024)
                first = stream_bytes.find('\xff\xd8')
                last = stream_bytes.find('\xff\xd9')
                if first != -1 and last != -1:
                    jpg = stream_bytes[first:last+2]
                    stream_bytes = stream_bytes[last+2:]
                    gray = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8), cv2.CV_LOAD_IMAGE_GRAYSCALE)
                    image = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8), cv2.CV_LOAD_IMAGE_UNCHANGED)

                    # lower half of the image
                    half_gray = gray[120:240, :]

                    cv2.imshow('image', image)
                # cv2.imshow('mlp_image', half_gray)
                if cv2.waitKey(1) & 0xFF == ord('q'):
                    self.rc_car.stop()
                    break
            cv2.destroyAllWindows()
        finally:
            print "Connection closed on thread 1"
class ThreadServer(object):

    def server_thread(host, port):
        server = SocketServer.TCPServer((host, port), VideoStreamHandler)
        server.serve_forever()

    def server_thread2(host, port):
        server = SocketServer.TCPServer((host, port), SensorDataHandler)
        server.serve_forever()

    distance_thread = threading.Thread(target=server_thread2, args=('127.0.0.1', 8004))
    distance_thread.start()

    video_thread = threading.Thread(target=server_thread('127.0.0.1', 8002))
    video_thread.start()
if __name__ == '__main__':
    ThreadServer()<|endoftext|>


<commit_before>from Whisperer import Card, Mana

class Azar(Card):
    def __init__(self):
        name = "Azar"
        text = "The first Whisperer card!"
        mana = Mana(red=1, black=1, blue=1)
        super().__init__(name, text, mana)
<commit_msg>Removed excess mana cost from Azar<commit_after>from Whisperer import Card, Mana

class Azar(Card):
    def __init__(self):
        name = "Azar"
        text = "The first Whisperer card!"
        mana = Mana(red=1)
        super().__init__(name, text, mana)
<|endoftext|>


<commit_before>import numpy as np

def get_virtual_treenodes(treenodes_response, z_depth):
    """
    Get the locations of all virtual nodes in the given skeleton response,
    with the IDs of the material nodes proximal and distal to the virtual slab.

    Parameters
    ----------
    treenodes_response: sequence of sequences of any
        "nodes" element of compact-skeleton or compact-arbor API response:
        row values are [treenode_id, parent_id, _, location_x, location_y, location_z, ...]
    z_depth: float
        Z resolution of the stack

    Returns
    -------
    iterator of (
        (proximal_treenode_id, distal_treenode_id),
        [
            (proximal_vnode_x, proximal_vnode_y, proximal_vnode_z),
            ...
            (distal_vnode_x, distal_vnode_y, distal_vnode_z)
        ]
    )
    """
    data = dict()
    for row in treenodes_response:
        parent = row[1]
        data[int(row[0])] = {
            "parent": None if parent is None else int(parent),
            "location": np.array(row[3:6], dtype=float)
        }

    for tnid, d in data.items():
        parent_id = d["parent"]
        if parent_id is None:
            continue
        parent_loc = data[parent_id]["location"]
        virtual_treenodes = list(
            interpolate_treenodes(parent_loc, d["location"], z_depth)
        )
        if virtual_treenodes:
            yield (d["parent"], tnid), virtual_treenodes

def interpolate_treenodes(parent_xyz, child_xyz, z_depth):
    """
    Yield the locations of virtual treenodes between two material treenodes.

    The parent and child's z coordinates will be quantized to the nearest multiple of z_depth.
    Virtual nodes will be projected onto any multiples of z_depth between the parent and child,
    and their (x, y, z) coordinates will be yielded in parent -> child order.

    Parameters
    ----------
    parent_xyz : sequence of (float, float, float)
    child_xyz : sequence of (float, float, float)
    z_depth : float
        e.g. z-resolution

    Returns
    -------
    Iterator of (x,y,z) tuples
    """
    parent_child_xyz = np.array((parent_xyz, child_xyz), dtype=float)
    z_slices = np.round(parent_child_xyz[:, 2] / z_depth)

    n_vnodes = int(abs(np.diff(z_slices)[0])) - 1

    if n_vnodes < 1:
        return

    parent_child_xyz[:, 2] = z_slices * z_depth

    it = zip(
        *[np.linspace(*parent_child_xyz[:, idx], num=n_vnodes + 2) for idx in range(3)]
    )
    next(it)  # discard first (parent location)
    for _ in range(n_vnodes):
        yield next(it)
    # discard last (child location)
<commit_msg>fix py2 zip<commit_after>import numpy as np

from six.moves import zip

def get_virtual_treenodes(treenodes_response, z_depth):
    """
    Get the locations of all virtual nodes in the given skeleton response,
    with the IDs of the material nodes proximal and distal to the virtual slab.

    Parameters
    ----------
    treenodes_response: sequence of sequences of any
        "nodes" element of compact-skeleton or compact-arbor API response:
        row values are [treenode_id, parent_id, _, location_x, location_y, location_z, ...]
    z_depth: float
        Z resolution of the stack

    Returns
    -------
    iterator of (
        (proximal_treenode_id, distal_treenode_id),
        [
            (proximal_vnode_x, proximal_vnode_y, proximal_vnode_z),
            ...
            (distal_vnode_x, distal_vnode_y, distal_vnode_z)
        ]
    )
    """
    data = dict()
    for row in treenodes_response:
        parent = row[1]
        data[int(row[0])] = {
            "parent": None if parent is None else int(parent),
            "location": np.array(row[3:6], dtype=float)
        }

    for tnid, d in data.items():
        parent_id = d["parent"]
        if parent_id is None:
            continue
        parent_loc = data[parent_id]["location"]
        virtual_treenodes = list(
            interpolate_treenodes(parent_loc, d["location"], z_depth)
        )
        if virtual_treenodes:
            yield (d["parent"], tnid), virtual_treenodes

def interpolate_treenodes(parent_xyz, child_xyz, z_depth):
    """
    Yield the locations of virtual treenodes between two material treenodes.

    The parent and child's z coordinates will be quantized to the nearest multiple of z_depth.
    Virtual nodes will be projected onto any multiples of z_depth between the parent and child,
    and their (x, y, z) coordinates will be yielded in parent -> child order.

    Parameters
    ----------
    parent_xyz : sequence of (float, float, float)
    child_xyz : sequence of (float, float, float)
    z_depth : float
        e.g. z-resolution

    Returns
    -------
    Iterator of (x,y,z) tuples
    """
    parent_child_xyz = np.array((parent_xyz, child_xyz), dtype=float)
    z_slices = np.round(parent_child_xyz[:, 2] / z_depth)

    n_vnodes = int(abs(np.diff(z_slices)[0])) - 1

    if n_vnodes < 1:
        return

    parent_child_xyz[:, 2] = z_slices * z_depth

    it = zip(
        *[np.linspace(*parent_child_xyz[:, idx], num=n_vnodes + 2) for idx in range(3)]
    )
    next(it)  # discard first (parent location)
    for _ in range(n_vnodes):
        yield next(it)
    # discard last (child location)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#coding=utf-8

import sys
major, minor = sys.version_info[:2]
if major < 3:
    reload(sys)
    sys.setdefaultencoding('utf-8')

import os
from catsup.options import g, enable_pretty_logging

enable_pretty_logging()

g.catsup_path = os.path.abspath(os.path.dirname(__file__))
g.public_templates_path = os.path.join(g.catsup_path, 'templates')

try:
    import catsup
except ImportError:
    import site
    site.addsitedir(os.path.dirname(g.catsup_path))

doc = """catsup v%s

Usage:
    catsup init [<path>]
    catsup build [-s <file>|--settings=<file>]
    catsup server [-s <file>|--settings=<file>] [-p <port>|--port=<port>]
    catsup webhook [-s <file>|--settings=<file>] [-p <port>|--port=<port>]
    catsup themes
    catsup install <theme> [-g|--global]
    catsup -h | --help
    catsup --version

Options:
    -h --help               show this screen.
    -s --settings=<file>    specify a config file.
    -p --port=<port>        specify the server port.
    -g --global             install theme to global theme folder.
""" % catsup.__version__

import catsup.config
import catsup.server
import catsup.themes
import catsup.build

from parguments import Parguments

parguments = Parguments(doc, version=catsup.__version__)

@parguments.command
def init(args):
    """
Usage:
    catsup init [<path>] [-s <file>|--settings=<file>]

Options:
    -h --help               show this screen.
    -s --settings=<file>    specify a setting file.
    """
    catsup.config.init(args.get('<path>'))

@parguments.command
def build(args):
    """
Usage:
    catsup build [-s <file>|--settings=<file>]

Options:
    -h --help               show this screen.
    -s --settings=<file>    specify a setting file.
    """
    path = args.get('--settings') or 'config.json'
    catsup.config.load(path)
    catsup.build.build()

@parguments.command
def server(args):
    """
Usage:
    catsup server [-s <file>|--settings=<file>] [-p <port>|--port=<port>]

Options:
    -h --help               show this screen.
    -s --settings=<file>    specify a setting file.
    -p --port=<port>        specify the server port.
    """
    path = args.get('--settings') or 'config.json'
    catsup.config.load(path)
    port = args.get('--port') or '8888'
    catsup.server.preview(port=port)

@parguments.command
def webhook(args):
    """
Usage:
    catsup webhook [-s <file>|--settings=<file>] [-p <port>|--port=<port>]

Options:
    -h --help               show this screen.
    -s --settings=<file>    specify a setting file.
    -p --port=<port>        specify the server port.
    """
    path = args.get('--settings') or 'config.json'
    catsup.config.load(path)
    port = args.get('--port') or '8888'
    catsup.server.webhook(port=port)

@parguments.command
def themes(args):
    """
Usage:
    catsup themes

Options:
    -h --help               show this screen.
    """
    catsup.themes.list()

@parguments.command
def install(args):
    """
Usage:
    catsup install <theme> [-g|--global]

Options:
    -g --global             install theme to global theme folder.
    -h --help               show this screen.
    """
    g = args.get('--global')
    catsup.themes.install(g=g)

def main():
    parguments.run()

if __name__ == '__main__':
    main()
<commit_msg>display default in Options<commit_after>#!/usr/bin/env python
#coding=utf-8

import sys
major, minor = sys.version_info[:2]
if major < 3:
    reload(sys)
    sys.setdefaultencoding('utf-8')

import os
from catsup.options import g, enable_pretty_logging

enable_pretty_logging()

g.catsup_path = os.path.abspath(os.path.dirname(__file__))
g.public_templates_path = os.path.join(g.catsup_path, 'templates')

try:
    import catsup
except ImportError:
    import site
    site.addsitedir(os.path.dirname(g.catsup_path))

doc = """catsup v%s

Usage:
    catsup init [<path>]
    catsup build [-s <file>|--settings=<file>]
    catsup server [-s <file>|--settings=<file>] [-p <port>|--port=<port>]
    catsup webhook [-s <file>|--settings=<file>] [-p <port>|--port=<port>]
    catsup themes
    catsup install <theme> [-g|--global]
    catsup -h | --help
    catsup --version

Options:
    -h --help               show this screen.
    -s --settings=<file>    specify a config file. [default: config.json]
    -p --port=<port>        specify the server port. [default: 8888]
    -g --global             install theme to global theme folder.
""" % catsup.__version__

import catsup.config
import catsup.server
import catsup.themes
import catsup.build

from parguments import Parguments

parguments = Parguments(doc, version=catsup.__version__)

@parguments.command
def init(args):
    """
Usage:
    catsup init [<path>] [-s <file>|--settings=<file>]

Options:
    -h --help               show this screen.
    -s --settings=<file>    specify a setting file. [default: config.json]
    """
    catsup.config.init(args.get('<path>'))

@parguments.command
def build(args):
    """
Usage:
    catsup build [-s <file>|--settings=<file>]

Options:
    -h --help               show this screen.
    -s --settings=<file>    specify a setting file. [default: config.json]
    """
    path = args.get('--settings')
    catsup.config.load(path)
    catsup.build.build()

@parguments.command
def server(args):
    """
Usage:
    catsup server [-s <file>|--settings=<file>] [-p <port>|--port=<port>]

Options:
    -h --help               show this screen.
    -s --settings=<file>    specify a setting file. [default: config.json]
    -p --port=<port>        specify the server port. [default: 8888]
    """
    path = args.get('--settings')
    catsup.config.load(path)
    port = args.get('--port')
    catsup.server.preview(port=port)

@parguments.command
def webhook(args):
    """
Usage:
    catsup webhook [-s <file>|--settings=<file>] [-p <port>|--port=<port>]

Options:
    -h --help               show this screen.
    -s --settings=<file>    specify a setting file. [default: config.json]
    -p --port=<port>        specify the server port. [default: 8888]
    """
    path = args.get('--settings')
    catsup.config.load(path)
    port = args.get('--port')
    catsup.server.webhook(port=port)

@parguments.command
def themes(args):
    """
Usage:
    catsup themes

Options:
    -h --help               show this screen.
    """
    catsup.themes.list()

@parguments.command
def install(args):
    """
Usage:
    catsup install <theme> [-g|--global]

Options:
    -g --global             install theme to global theme folder.
    -h --help               show this screen.
    """
    g = args.get('--global')
    catsup.themes.install(g=g)

def main():
    parguments.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import sys
import time
import datetime
import numpy as np
import code, traceback, signal

#=============================================================================
# Tiling utilities
#=============================================================================
def amin(a, b):
    return np.vstack([a, b]).min(axis=0)

def amax(a, b):
    return np.vstack([a, b]).max(axis=0)

class Tile(object):
    def __init__(self, left, right=None, mins=None, maxs=None):
        if right is None:
            right = left
            left = np.zeros(len(left), dtype='int')

        if not hasattr(left, '__iter__'):
            left = np.array([left]*3, dtype='int')

        if not hasattr(right, '__iter__'):
            right = np.array([right]*3, dtype='int')

        if mins is not None:
            if not hasattr(mins, '__iter__'):
                mins = np.array([mins]*3)
            left = amax(left, mins)

        if maxs is not None:
            if not hasattr(maxs, '__iter__'):
                maxs = np.array([maxs]*3)
            right = amin(right, maxs)

        l, r = left, right
        self.l = np.array(l)
        self.r = np.array(r)
        self.bounds = (l, r)
        self.shape = self.r - self.l
        self.slicer = np.s_[l[0]:r[0], l[1]:r[1], l[2]:r[2]]

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return str(self.__class__.__name__)+" {} -> {} ({})".format(
                list(self.l), list(self.r), list(self.shape))

def cdd(d, k):
    """ Conditionally delete key (or list of keys) 'k' from dict 'd' """
    if not isinstance(k, list):
        k = [k]
    for i in k:
        if d.has_key(i):
            d.pop(i)

#=============================================================================
# Progress bar
#=============================================================================
class ProgressBar(object):
    def __init__(self, num, label='Progress', value=0, screen=79,
            time_remaining=True, bar=True, bar_symbol='=', bar_caps='[]',
            bar_decimals=2):
        """
        ProgressBar class which creates a dynamic ASCII progress bar of two
        different varieties:
            1) A bar chart that looks like the following:
                Progress [================      ]  63.00%

            2) A simple number completed look:
                Progress :   17 / 289

        Parameters:
        -----------
        num : integer
            The number of tasks that need to be completed

        label : string [default: 'Progress']
            The label for this particular progress indicator,

        value : integer [default: 0]
            Starting value

        screen : integer [default: 79]
            Size the screen to use for the progress bar

        time_remaining : boolean [default: True]
            Display estimated time remaining

        bar : boolean [default: True]
            Whether or not to display the bar chart

        bar_symbol : char [default: '=']
            The character to use to fill in the bar chart

        bar_caps : string [default: '[]']
            Characters to use as the end caps of the.  The string will be split in
            half and each half put on a side of the chart

        bar_decimals: integer [default: 2]
            Number of decimal places to include in the _percentage
        """
        # TODO -- add estimated time remaining
        self.num = num
        self.value = value
        self._percent = 0
        self.time_remaining = time_remaining
        self._deltas = []

        self.label = label
        self.bar = bar
        self._bar_symbol = bar_symbol
        self._bar_caps = bar_caps
        self._decimals = bar_decimals
        self.screen = screen

        if len(self._bar_caps) % 2 != 0:
            raise AttributeError("End caps must be even number of symbols")

        if self.bar:
            # 3 digit _percent + decimal places + '.'
            self._numsize = 3 + self._decimals + 1

            # end caps size calculation
            self._cap_len = len(self._bar_caps)/2
            self._capl = self._bar_caps[:self._cap_len]
            self._capr = self._bar_caps[self._cap_len:]

            # time remaining calculation for space
            self._time_space = 11 if self.time_remaining else 0

            # the space available for the progress bar is
            # 79 (screen) - (label) - (number) - 2 ([]) - 2 (space) - 1 (%)
            self._barsize = (
                    self.screen - len(self.label) - self._numsize -
                    len(self._bar_caps) - 2 - 1 - self._time_space
                )

            self._formatstr = '\r{label} {_capl}{_bars:<{_barsize}}{_capr} {_percent:>{_numsize}.{_decimals}f}%'

            self._percent = 0
            self._dt = '--:--:--'
            self._bars = ''

            if self.time_remaining:
                self._formatstr += " ({_dt})"
        else:
            self._digits = str(int(np.ceil(np.log10(self.num))))
            self._formatstr = '\r{label} : {value:>{_digits}} / {num:>{_digits}}'

            self._dt = '--:--:--'
            if self.time_remaining:
                self._formatstr += " ({_dt})"

        self.update()

    def _estimate_time(self):
        if len(self._deltas) < 3:
            self._dt = '--:--:--'
        else:
            dt = np.diff(self._deltas[-25:]).mean() * (self.num - self.value)
            self._dt = time.strftime('%H:%M:%S', time.gmtime(dt))

    def _draw(self):
        """ Interal draw method, simply prints to screen """
        print self._formatstr.format(**self.__dict__),
        sys.stdout.flush()

    def increment(self):
        self.update(self.value + 1)

    def update(self, value=0):
        """
        Update the value of the progress and update progress bar.

        Parameters:
        -----------
        value : integer
            The current iteration of the progress
        """
        self._deltas.append(time.time())

        self.value = value
        self._percent = 100.0 * self.value / self.num

        if self.bar:
            self._bars = self._bar_symbol*int(np.round(self._percent / 100. * self._barsize))

        if (len(self._deltas) < 2) or (self._deltas[-1] - self._deltas[-2]) > 1e-1:
            self._estimate_time()
            self._draw()

        if self.value == self.num:
            self.end()

    def end(self):
        print '\r{lett:>{screen}}'.format(**{'lett':'', 'screen': self.screen})

#=============================================================================
# debugging / python interpreter / logging
#=============================================================================
def debug(sig, frame):
    """Interrupt running process, and provide a python prompt for
    interactive debugging."""
    d={'_frame':frame}         # Allow access to frame object.
    d.update(frame.f_globals)  # Unless shadowed by global
    d.update(frame.f_locals)

    i = code.InteractiveConsole(d)
    message  = "Signal received : entering python shell.\nTraceback:\n"
    message += ''.join(traceback.format_stack(frame))
    i.interact(message)

def listen():
    # register handler
    signal.signal(signal.SIGUSR1, debug)
<commit_msg>adding display parameter so that code doesn't diverge everywhere<commit_after>import sys
import time
import datetime
import numpy as np
import code, traceback, signal

#=============================================================================
# Tiling utilities
#=============================================================================
def amin(a, b):
    return np.vstack([a, b]).min(axis=0)

def amax(a, b):
    return np.vstack([a, b]).max(axis=0)

class Tile(object):
    def __init__(self, left, right=None, mins=None, maxs=None):
        if right is None:
            right = left
            left = np.zeros(len(left), dtype='int')

        if not hasattr(left, '__iter__'):
            left = np.array([left]*3, dtype='int')

        if not hasattr(right, '__iter__'):
            right = np.array([right]*3, dtype='int')

        if mins is not None:
            if not hasattr(mins, '__iter__'):
                mins = np.array([mins]*3)
            left = amax(left, mins)

        if maxs is not None:
            if not hasattr(maxs, '__iter__'):
                maxs = np.array([maxs]*3)
            right = amin(right, maxs)

        l, r = left, right
        self.l = np.array(l)
        self.r = np.array(r)
        self.bounds = (l, r)
        self.shape = self.r - self.l
        self.slicer = np.s_[l[0]:r[0], l[1]:r[1], l[2]:r[2]]

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return str(self.__class__.__name__)+" {} -> {} ({})".format(
                list(self.l), list(self.r), list(self.shape))

def cdd(d, k):
    """ Conditionally delete key (or list of keys) 'k' from dict 'd' """
    if not isinstance(k, list):
        k = [k]
    for i in k:
        if d.has_key(i):
            d.pop(i)

#=============================================================================
# Progress bar
#=============================================================================
class ProgressBar(object):
    def __init__(self, num, label='Progress', value=0, screen=79,
            time_remaining=True, bar=True, bar_symbol='=', bar_caps='[]',
            bar_decimals=2, display=True):
        """
        ProgressBar class which creates a dynamic ASCII progress bar of two
        different varieties:
            1) A bar chart that looks like the following:
                Progress [================      ]  63.00%

            2) A simple number completed look:
                Progress :   17 / 289

        Parameters:
        -----------
        num : integer
            The number of tasks that need to be completed

        label : string [default: 'Progress']
            The label for this particular progress indicator,

        value : integer [default: 0]
            Starting value

        screen : integer [default: 79]
            Size the screen to use for the progress bar

        time_remaining : boolean [default: True]
            Display estimated time remaining

        bar : boolean [default: True]
            Whether or not to display the bar chart

        bar_symbol : char [default: '=']
            The character to use to fill in the bar chart

        bar_caps : string [default: '[]']
            Characters to use as the end caps of the.  The string will be split in
            half and each half put on a side of the chart

        bar_decimals: integer [default: 2]
            Number of decimal places to include in the _percentage

        display : boolean [default: True]
            a crutch so that we don't have a lot of `if`s later.  display
            or don't display the progress bar
        """
        # TODO -- add estimated time remaining
        self.num = num
        self.value = value
        self._percent = 0
        self.time_remaining = time_remaining
        self._deltas = []
        self.display = display

        self.label = label
        self.bar = bar
        self._bar_symbol = bar_symbol
        self._bar_caps = bar_caps
        self._decimals = bar_decimals
        self.screen = screen

        if len(self._bar_caps) % 2 != 0:
            raise AttributeError("End caps must be even number of symbols")

        if self.bar:
            # 3 digit _percent + decimal places + '.'
            self._numsize = 3 + self._decimals + 1

            # end caps size calculation
            self._cap_len = len(self._bar_caps)/2
            self._capl = self._bar_caps[:self._cap_len]
            self._capr = self._bar_caps[self._cap_len:]

            # time remaining calculation for space
            self._time_space = 11 if self.time_remaining else 0

            # the space available for the progress bar is
            # 79 (screen) - (label) - (number) - 2 ([]) - 2 (space) - 1 (%)
            self._barsize = (
                    self.screen - len(self.label) - self._numsize -
                    len(self._bar_caps) - 2 - 1 - self._time_space
                )

            self._formatstr = '\r{label} {_capl}{_bars:<{_barsize}}{_capr} {_percent:>{_numsize}.{_decimals}f}%'

            self._percent = 0
            self._dt = '--:--:--'
            self._bars = ''

            if self.time_remaining:
                self._formatstr += " ({_dt})"
        else:
            self._digits = str(int(np.ceil(np.log10(self.num))))
            self._formatstr = '\r{label} : {value:>{_digits}} / {num:>{_digits}}'

            self._dt = '--:--:--'
            if self.time_remaining:
                self._formatstr += " ({_dt})"

        self.update()

    def _estimate_time(self):
        if len(self._deltas) < 3:
            self._dt = '--:--:--'
        else:
            dt = np.diff(self._deltas[-25:]).mean() * (self.num - self.value)
            self._dt = time.strftime('%H:%M:%S', time.gmtime(dt))

    def _draw(self):
        """ Interal draw method, simply prints to screen """
        if self.display:
            print self._formatstr.format(**self.__dict__),
            sys.stdout.flush()

    def increment(self):
        self.update(self.value + 1)

    def update(self, value=0):
        """
        Update the value of the progress and update progress bar.

        Parameters:
        -----------
        value : integer
            The current iteration of the progress
        """
        self._deltas.append(time.time())

        self.value = value
        self._percent = 100.0 * self.value / self.num

        if self.bar:
            self._bars = self._bar_symbol*int(np.round(self._percent / 100. * self._barsize))

        if (len(self._deltas) < 2) or (self._deltas[-1] - self._deltas[-2]) > 1e-1:
            self._estimate_time()
            self._draw()

        if self.value == self.num:
            self.end()

    def end(self):
        if self.display:
            print '\r{lett:>{screen}}'.format(**{'lett':'', 'screen': self.screen})

#=============================================================================
# debugging / python interpreter / logging
#=============================================================================
def debug(sig, frame):
    """Interrupt running process, and provide a python prompt for
    interactive debugging."""
    d={'_frame':frame}         # Allow access to frame object.
    d.update(frame.f_globals)  # Unless shadowed by global
    d.update(frame.f_locals)

    i = code.InteractiveConsole(d)
    message  = "Signal received : entering python shell.\nTraceback:\n"
    message += ''.join(traceback.format_stack(frame))
    i.interact(message)

def listen():
    # register handler
    signal.signal(signal.SIGUSR1, debug)
<|endoftext|>


<commit_before>from django.db import models

class ProjectManager(models.Manager):
    def get_by_natural_key(self, name):
        return self.get(name=name)

class Project(models.Model):
    """ Represents a project in a python project hierarchy """

    name = models.CharField(max_length=200, unique=True)

    objects = ProjectManager()

    def __unicode__(self):
        return self.name

    def natural_key(self):
        return (self.name,)

    @models.permalink
    def get_absolute_url(self):
        return ('project-detail', (), {
            'package': self.name,
        })

class ProjectVersionManager(models.Manager):

    def get_by_natural_key(self, name, version_number):
        return self.get(
            project=Project.objects.get_by_natural_key(name=name),
            version_number=version_number
            )

    def get_latest(self, name):
        return self.order_by('-version_number',)[0]

class ProjectVersion(models.Model):
    """ Represents a particular version of a project in a python project hierarchy """

    project = models.ForeignKey(Project)
    version_number = models.CharField(max_length=200)

    objects = ProjectVersionManager()

    class Meta:
        unique_together = ('project', 'version_number')
        ordering = ('-version_number',)

    def __unicode__(self):
        return self.project.name + " " + self.version_number

    def natural_key(self):
        return self.project.natural_key() + (self.version_number,)
    natural_key.dependencies = ['cbv.Project']

    @models.permalink
    def get_absolute_url(self):
        return ('version-detail', (), {
            'package': self.project.name,
            'version': self.version_number,
        })

    @property
    def docs_version_number(self):
        return '.'.join(self.version_number.split('.')[:2])

class ModuleManager(models.Manager):
    def get_by_natural_key(self, module_name, project_name, version_number):
        return self.get(
            name=module_name,
            project_version=ProjectVersion.objects.get_by_natural_key(
                name=project_name,
                version_number=version_number)
            )

class Module(models.Model):
    """ Represents a module of a python project """

    project_version = models.ForeignKey(ProjectVersion)
    name = models.CharField(max_length=200)
    docstring = models.TextField(blank=True, default='')
    filename = models.CharField(max_length=511, default='')

    objects = ModuleManager()

    class Meta:
        unique_together = ('project_version', 'name')

    def __unicode__(self):
        return self.name

    def short_name(self):
        return self.name.split('.')[-1]

    def natural_key(self):
        return (self.name,) + self.project_version.natural_key()
    natural_key.dependencies = ['cbv.ProjectVersion']

    @models.permalink
    def get_absolute_url(self):
        return ('module-detail', (), {
            'package': self.project_version.project.name,
            'version': self.project_version.version_number,
            'module': self.name,
        })

class KlassManager(models.Manager):
    def get_by_natural_key(self, klass_name, module_name, project_name, version_number):
        return self.get(
            name=klass_name,
            module=Module.objects.get_by_natural_key(
                module_name=module_name,
                project_name=project_name,
                version_number=version_number)
            )

    def get_latest_for_name(self, klass_name, project_name):
        qs = self.filter(
            name=klass_name,
            module__project_version__project__name=project_name,
        ) or self.filter(
            name__iexact=klass_name,
            module__project_version__project__name__iexact=project_name,
        )
        return qs.get(
            module__project_version__version_number=qs.aggregate(
                models.Max('module__project_version__version_number')
            )['module__project_version__version_number__max']
        )

# TODO: quite a few of the methods on here should probably be denormed.
class Klass(models.Model):
    """ Represents a class in a module of a python project hierarchy """

    module = models.ForeignKey(Module)
    name = models.CharField(max_length=200)
    docstring = models.TextField(blank=True, default='')
    line_number = models.IntegerField()
    import_path = models.CharField(max_length=255)

    objects = KlassManager()

    class Meta:
        unique_together = ('module', 'name')
        ordering = ('module__name',)

    def __unicode__(self):
        return self.name

    def natural_key(self):
        return (self.name,) + self.module.natural_key()
    natural_key.dependencies = ['cbv.Module']

    def is_secondary(self):
        return (self.name.startswith('Base') or
                self.name.endswith('Mixin') or
                self.name.endswith('Error') or
                self.name == 'ProcessFormView')

    @models.permalink
    def get_absolute_url(self):
        return ('klass-detail', (), {
            'package': self.module.project_version.project.name,
            'version': self.module.project_version.version_number,
            'module': self.module.name,
            'klass': self.name
        })

    def get_djangodocs_url(self):
        url = 'https://docs.djangoproject.com/en/{version}/ref/class-based-views/#{module}.{klass}'
        return url.format(
            version=self.module.project_version.docs_version_number,
            module=self.module.name,
            klass=self.name,
        )

    def get_source_url(self):
        url = 'https://github.com/django/django/blob/'
        return url + '{version}{path}#L{line}'.format(
            version=self.module.project_version.version_number,
            path=self.module.filename,
            line=self.line_number,
        )

    def get_ancestors(self):
        if not hasattr(self, '_ancestors'):
            self._ancestors = Klass.objects.filter(inheritance__child=self).order_by('inheritance__order')
        return self._ancestors

    def get_children(self):
        if not hasattr(self, '_descendants'):
            self._descendants = Klass.objects.filter(ancestor_relationships__parent=self).order_by('name')
        return self._descendants

    #TODO: This is all mucho inefficient. Perhaps we should use mptt for
    #       get_all_ancestors, get_all_children, get_methods, & get_attributes?
    def get_all_ancestors(self):
        if not hasattr(self, '_all_ancestors'):
            ancestors = self.get_ancestors().select_related('module__project_version__project')
            tree = []
            for ancestor in ancestors:
                tree += [ancestor]
                tree += ancestor.get_all_ancestors()
            self._all_ancestors = tree
        return self._all_ancestors

    def get_all_children(self):
        if not hasattr(self, '_all_descendants'):
            children = self.get_children().select_related('module__project_version__project')
            for child in children:
                children = children | child.get_all_children()
            self._all_descendants = children
        return self._all_descendants

    def get_methods(self):
        if not hasattr(self, '_methods'):
            methods = self.method_set.all().select_related('klass')
            for ancestor in self.get_all_ancestors():
                methods = methods | ancestor.get_methods()
            self._methods = methods
        return self._methods

    def get_attributes(self):
        if not hasattr(self, '_attributes'):
            attrs = self.attribute_set.all()
            for ancestor in self.get_all_ancestors():
                attrs = attrs | ancestor.get_attributes()
            self._attributes = attrs
        return self._attributes

    def get_prepared_attributes(self):
        attributes = self.get_attributes()
        # Make a dictionary of attributes based on name
        attribute_names = {}
        for attr in attributes:
            try:
                attribute_names[attr.name] += [attr]
            except KeyError:
                attribute_names[attr.name] = [attr]

        ancestors = self.get_all_ancestors()

        # Find overridden attributes
        for name, attrs in attribute_names.iteritems():
            # Skip if we have only one attribute.
            if len(attrs) == 1:
                continue

            # Sort the attributes by ancestors.
            def _key(a):
                try:
                    # If ancestor, return the index (>= 0)
                    return ancestors.index(a.klass)
                except:
                    # else a.klass == self, so return -1
                    return -1
            sorted_attrs = sorted(attrs, key=_key)

            # Mark overriden KlassAttributes
            for a in sorted_attrs[1:]:
                a.overridden = True
        return attributes

class Inheritance(models.Model):
    """ Represents the inheritance relationships for a Klass """

    parent = models.ForeignKey(Klass)
    child = models.ForeignKey(Klass, related_name='ancestor_relationships')
    order = models.IntegerField()

    class Meta:
        ordering = ('order',)
        unique_together = ('child', 'order')

    def __unicode__(self):
        return u'%s <- %s (%d)' % (self.parent, self.child, self.order)

class KlassAttribute(models.Model):
    """ Represents an attribute on a Klass """

    klass = models.ForeignKey(Klass, related_name='attribute_set')
    name = models.CharField(max_length=200)
    value = models.CharField(max_length=200)
    line_number = models.IntegerField()

    class Meta:
        ordering = ('name',)
        unique_together = ('klass', 'name')

    def __unicode__(self):
        return u'%s = %s' % (self.name, self.value)

class ModuleAttribute(models.Model):
    """ Represents an attribute on a Module """

    module = models.ForeignKey(Module, related_name='attribute_set')
    name = models.CharField(max_length=200)
    value = models.CharField(max_length=200)
    line_number = models.IntegerField()

    class Meta:
        ordering = ('name',)
        unique_together = ('module', 'name')

    def __unicode__(self):
        return u'%s = %s' % (self.name, self.value)

class Method(models.Model):
    """ Represents a method on a Klass """

    klass = models.ForeignKey(Klass)
    name = models.CharField(max_length=200)
    docstring = models.TextField(blank=True, default='')
    code = models.TextField()
    kwargs = models.CharField(max_length=200)
    line_number = models.IntegerField()

    def __unicode__(self):
        return self.name

    class Meta:
        ordering = ('name',)

class Function(models.Model):
    """ Represents a function on a Module """

    module = models.ForeignKey(Module)
    name = models.CharField(max_length=200)
    docstring = models.TextField(blank=True, default='')
    code = models.TextField()
    kwargs = models.CharField(max_length=200)
    line_number = models.IntegerField()

    def __unicode__(self):
        return self.name

    class Meta:
        ordering = ('name',)
<commit_msg>Fix get_latest_for_name on Klass<commit_after>from django.db import models

class ProjectManager(models.Manager):
    def get_by_natural_key(self, name):
        return self.get(name=name)

class Project(models.Model):
    """ Represents a project in a python project hierarchy """

    name = models.CharField(max_length=200, unique=True)

    objects = ProjectManager()

    def __unicode__(self):
        return self.name

    def natural_key(self):
        return (self.name,)

    @models.permalink
    def get_absolute_url(self):
        return ('project-detail', (), {
            'package': self.name,
        })

class ProjectVersionManager(models.Manager):

    def get_by_natural_key(self, name, version_number):
        return self.get(
            project=Project.objects.get_by_natural_key(name=name),
            version_number=version_number
            )

    def get_latest(self, name):
        return self.order_by('-version_number',)[0]

class ProjectVersion(models.Model):
    """ Represents a particular version of a project in a python project hierarchy """

    project = models.ForeignKey(Project)
    version_number = models.CharField(max_length=200)

    objects = ProjectVersionManager()

    class Meta:
        unique_together = ('project', 'version_number')
        ordering = ('-version_number',)

    def __unicode__(self):
        return self.project.name + " " + self.version_number

    def natural_key(self):
        return self.project.natural_key() + (self.version_number,)
    natural_key.dependencies = ['cbv.Project']

    @models.permalink
    def get_absolute_url(self):
        return ('version-detail', (), {
            'package': self.project.name,
            'version': self.version_number,
        })

    @property
    def docs_version_number(self):
        return '.'.join(self.version_number.split('.')[:2])

class ModuleManager(models.Manager):
    def get_by_natural_key(self, module_name, project_name, version_number):
        return self.get(
            name=module_name,
            project_version=ProjectVersion.objects.get_by_natural_key(
                name=project_name,
                version_number=version_number)
            )

class Module(models.Model):
    """ Represents a module of a python project """

    project_version = models.ForeignKey(ProjectVersion)
    name = models.CharField(max_length=200)
    docstring = models.TextField(blank=True, default='')
    filename = models.CharField(max_length=511, default='')

    objects = ModuleManager()

    class Meta:
        unique_together = ('project_version', 'name')

    def __unicode__(self):
        return self.name

    def short_name(self):
        return self.name.split('.')[-1]

    def natural_key(self):
        return (self.name,) + self.project_version.natural_key()
    natural_key.dependencies = ['cbv.ProjectVersion']

    @models.permalink
    def get_absolute_url(self):
        return ('module-detail', (), {
            'package': self.project_version.project.name,
            'version': self.project_version.version_number,
            'module': self.name,
        })

class KlassManager(models.Manager):
    def get_by_natural_key(self, klass_name, module_name, project_name, version_number):
        return self.get(
            name=klass_name,
            module=Module.objects.get_by_natural_key(
                module_name=module_name,
                project_name=project_name,
                version_number=version_number)
            )

    def get_latest_for_name(self, klass_name, project_name):
        qs = self.filter(
            name=klass_name,
            module__project_version__project__name=project_name,
        ) or self.filter(
            name__iexact=klass_name,
            module__project_version__project__name__iexact=project_name,
        )
        try:
            obj = qs.order_by('-module__project_version__version_number',)[0]
        except IndexError:
            raise self.model.DoesNotExist()
        else:
            return obj

# TODO: quite a few of the methods on here should probably be denormed.
class Klass(models.Model):
    """ Represents a class in a module of a python project hierarchy """

    module = models.ForeignKey(Module)
    name = models.CharField(max_length=200)
    docstring = models.TextField(blank=True, default='')
    line_number = models.IntegerField()
    import_path = models.CharField(max_length=255)

    objects = KlassManager()

    class Meta:
        unique_together = ('module', 'name')
        ordering = ('module__name',)

    def __unicode__(self):
        return self.name

    def natural_key(self):
        return (self.name,) + self.module.natural_key()
    natural_key.dependencies = ['cbv.Module']

    def is_secondary(self):
        return (self.name.startswith('Base') or
                self.name.endswith('Mixin') or
                self.name.endswith('Error') or
                self.name == 'ProcessFormView')

    @models.permalink
    def get_absolute_url(self):
        return ('klass-detail', (), {
            'package': self.module.project_version.project.name,
            'version': self.module.project_version.version_number,
            'module': self.module.name,
            'klass': self.name
        })

    def get_djangodocs_url(self):
        url = 'https://docs.djangoproject.com/en/{version}/ref/class-based-views/#{module}.{klass}'
        return url.format(
            version=self.module.project_version.docs_version_number,
            module=self.module.name,
            klass=self.name,
        )

    def get_source_url(self):
        url = 'https://github.com/django/django/blob/'
        return url + '{version}{path}#L{line}'.format(
            version=self.module.project_version.version_number,
            path=self.module.filename,
            line=self.line_number,
        )

    def get_ancestors(self):
        if not hasattr(self, '_ancestors'):
            self._ancestors = Klass.objects.filter(inheritance__child=self).order_by('inheritance__order')
        return self._ancestors

    def get_children(self):
        if not hasattr(self, '_descendants'):
            self._descendants = Klass.objects.filter(ancestor_relationships__parent=self).order_by('name')
        return self._descendants

    #TODO: This is all mucho inefficient. Perhaps we should use mptt for
    #       get_all_ancestors, get_all_children, get_methods, & get_attributes?
    def get_all_ancestors(self):
        if not hasattr(self, '_all_ancestors'):
            ancestors = self.get_ancestors().select_related('module__project_version__project')
            tree = []
            for ancestor in ancestors:
                tree += [ancestor]
                tree += ancestor.get_all_ancestors()
            self._all_ancestors = tree
        return self._all_ancestors

    def get_all_children(self):
        if not hasattr(self, '_all_descendants'):
            children = self.get_children().select_related('module__project_version__project')
            for child in children:
                children = children | child.get_all_children()
            self._all_descendants = children
        return self._all_descendants

    def get_methods(self):
        if not hasattr(self, '_methods'):
            methods = self.method_set.all().select_related('klass')
            for ancestor in self.get_all_ancestors():
                methods = methods | ancestor.get_methods()
            self._methods = methods
        return self._methods

    def get_attributes(self):
        if not hasattr(self, '_attributes'):
            attrs = self.attribute_set.all()
            for ancestor in self.get_all_ancestors():
                attrs = attrs | ancestor.get_attributes()
            self._attributes = attrs
        return self._attributes

    def get_prepared_attributes(self):
        attributes = self.get_attributes()
        # Make a dictionary of attributes based on name
        attribute_names = {}
        for attr in attributes:
            try:
                attribute_names[attr.name] += [attr]
            except KeyError:
                attribute_names[attr.name] = [attr]

        ancestors = self.get_all_ancestors()

        # Find overridden attributes
        for name, attrs in attribute_names.iteritems():
            # Skip if we have only one attribute.
            if len(attrs) == 1:
                continue

            # Sort the attributes by ancestors.
            def _key(a):
                try:
                    # If ancestor, return the index (>= 0)
                    return ancestors.index(a.klass)
                except:
                    # else a.klass == self, so return -1
                    return -1
            sorted_attrs = sorted(attrs, key=_key)

            # Mark overriden KlassAttributes
            for a in sorted_attrs[1:]:
                a.overridden = True
        return attributes

class Inheritance(models.Model):
    """ Represents the inheritance relationships for a Klass """

    parent = models.ForeignKey(Klass)
    child = models.ForeignKey(Klass, related_name='ancestor_relationships')
    order = models.IntegerField()

    class Meta:
        ordering = ('order',)
        unique_together = ('child', 'order')

    def __unicode__(self):
        return u'%s <- %s (%d)' % (self.parent, self.child, self.order)

class KlassAttribute(models.Model):
    """ Represents an attribute on a Klass """

    klass = models.ForeignKey(Klass, related_name='attribute_set')
    name = models.CharField(max_length=200)
    value = models.CharField(max_length=200)
    line_number = models.IntegerField()

    class Meta:
        ordering = ('name',)
        unique_together = ('klass', 'name')

    def __unicode__(self):
        return u'%s = %s' % (self.name, self.value)

class ModuleAttribute(models.Model):
    """ Represents an attribute on a Module """

    module = models.ForeignKey(Module, related_name='attribute_set')
    name = models.CharField(max_length=200)
    value = models.CharField(max_length=200)
    line_number = models.IntegerField()

    class Meta:
        ordering = ('name',)
        unique_together = ('module', 'name')

    def __unicode__(self):
        return u'%s = %s' % (self.name, self.value)

class Method(models.Model):
    """ Represents a method on a Klass """

    klass = models.ForeignKey(Klass)
    name = models.CharField(max_length=200)
    docstring = models.TextField(blank=True, default='')
    code = models.TextField()
    kwargs = models.CharField(max_length=200)
    line_number = models.IntegerField()

    def __unicode__(self):
        return self.name

    class Meta:
        ordering = ('name',)

class Function(models.Model):
    """ Represents a function on a Module """

    module = models.ForeignKey(Module)
    name = models.CharField(max_length=200)
    docstring = models.TextField(blank=True, default='')
    code = models.TextField()
    kwargs = models.CharField(max_length=200)
    line_number = models.IntegerField()

    def __unicode__(self):
        return self.name

    class Meta:
        ordering = ('name',)
<|endoftext|>


<commit_before><commit_msg>manual copy-paste of master demand<commit_after>"""
===========================
Analytical energy demand model algorithm
===========================
File history and credits:
J. Fonseca  script development          24.08.15
D. Thomas   formatting, refactoring, debugging and cleaning
D. Thomas   integration in toolbox
J. Fonseca  refactoring to new properties file   22.03.16
"""
from __future__ import division
import pandas as pd
import numpy as np
import functions as f
import maker as m
import globalvar
from geopandas import GeoDataFrame as gpdf
import inputlocator

reload(f)
reload(globalvar)

def demand_calculation(locator, gv):
    """
    Algorithm to calculate the hourly demand of energy services in buildings
    using the integrated model of Fonseca et al. 2015. Applied energy.
    Parameters
    ----------
    :param locator: An InputLocator to locate input files
    :type locator: inputlocator.InputLocator
    path_radiation:
        path to solar radiation file in vertical surfaces
        RadiationYearFinal.csv
    path_schedules: string
        path to folder containing occupancy profile schedules
    path_weather : string
        path to weather data file weather_design_hour.csv
    path_results : string
        path to demand results folder demand
    path_properties: string
        path to folder with .shp files created with the properties script there should be a total of 6
    Returns
    -------
    single_demand: .csv
        csv file for every building with hourly demand data
    total_demand: .csv
        csv file of yearly demand data per buidling.
    """
    # local variables
    gv.log("reading input files")

    weather_data = pd.read_csv(locator.get_weather_hourly(), usecols=['te', 'RH'])
    solar = pd.read_csv(locator.get_radiation()).set_index('Name')
    surfaces = pd.read_csv(locator.get_surfaces())
    prop_geometry = gpdf.from_file(locator.get_building_geometry())
    prop_geometry['footprint'] = prop_geometry.area
    prop_geometry['perimeter'] = prop_geometry.length
    prop_geometry = prop_geometry.drop('geometry', axis=1).set_index('Name')
    prop_HVAC = gpdf.from_file(locator.get_building_hvac()).drop('geometry', axis=1)
    prop_thermal = gpdf.from_file(locator.get_building_thermal()).drop('geometry', axis=1).set_index('Name')
    prop_occupancy_df = gpdf.from_file(locator.get_building_occupancy()).drop('geometry', axis=1).set_index('Name')
    prop_occupancy = prop_occupancy_df.loc[:, (prop_occupancy_df != 0).any(axis=0)] # trick to erase occupancies that are not being used (it speeds up the code)
    prop_architecture = gpdf.from_file(locator.get_building_architecture()).drop('geometry', axis=1).set_index('Name')
    prop_age = gpdf.from_file(locator.get_building_age()).drop('geometry', axis=1).set_index('Name')
    prop_comfort = gpdf.from_file(locator.get_building_comfort()).drop('geometry', axis=1).set_index('Name')
    prop_internal_loads = gpdf.from_file(locator.get_building_internal()).drop('geometry', axis=1).set_index('Name')
    # get temperatures of operation
    prop_HVAC_result= get_temperatures(locator, prop_HVAC).set_index('Name')
    # weather conditions
    T_ext = np.array(weather_data.te)
    RH_ext = np.array(weather_data.RH)
    #get list of uses
    list_uses = list(prop_occupancy.drop('PFloor', axis=1).columns)
    #get date
    date = pd.date_range(gv.date_start, periods=8760, freq='H')
    gv.log('done')

    gv.log("reading occupancy schedules")
    # get schedules
    schedules = m.schedule_maker(date, locator, list_uses)
    gv.log("done")

    gv.log("calculating thermal properties")
    # get thermal properties for RC model
    prop_RC_model = f.get_prop_RC_model(prop_occupancy, prop_architecture, prop_thermal, prop_geometry,
                                        prop_HVAC_result, surfaces, gv)
    gv.log("done")

    # get timeseries of demand
    num_buildings = len(prop_RC_model.index)
    counter = 0
    for building in prop_RC_model.index:
        gv.models['calc-thermal-loads'](building, prop_occupancy.ix[building], prop_architecture.ix[building],
                           prop_geometry.ix[building], prop_HVAC_result.ix[building], prop_RC_model.ix[building],
                           prop_comfort.ix[building],prop_internal_loads.ix[building],
                           prop_age.ix[building], solar.ix[building], locator.get_demand_results_folder(),
                           schedules, T_ext, RH_ext, locator.get_temporary_folder(), gv, date, list_uses)
        gv.log('Building No. %(bno)i completed out of %(btot)i', bno=counter + 1, btot=num_buildings)
        counter += 1

    # get total file

    counter = 0
    for name in prop_RC_model.index:
        temporary_file = locator.get_temporary_file('%(name)sT.csv' % locals())
        # TODO: check this logic
        if counter == 0:
            df = pd.read_csv(temporary_file)
            counter += 1
        else:
            df2 = pd.read_csv(temporary_file)
            df = df.append(df2, ignore_index=True)
    df.to_csv(locator.get_total_demand(), index=False, float_format='%.2f')

    gv.log('finished')

def get_temperatures(locator, prop_HVAC):
    prop_emission_heating = pd.read_excel(locator.get_technical_emission_systems(), 'heating')
    prop_emission_cooling = pd.read_excel(locator.get_technical_emission_systems(), 'cooling')
    prop_emission_dhw = pd.read_excel(locator.get_technical_emission_systems(), 'dhw')

    df = prop_HVAC.merge(prop_emission_heating, left_on='type_hs', right_on='code')
    df2 = prop_HVAC.merge(prop_emission_cooling, left_on='type_cs', right_on='code')
    df3 = prop_HVAC.merge(prop_emission_dhw, left_on='type_dhw', right_on='code')

    fields = ['Name', 'type_hs', 'type_cs', 'type_dhw',  'type_ctrl', 'Tshs0_C', 'dThs0_C', 'Qhsmax_Wm2']
    fields2 = ['Name', 'Tscs0_C', 'dTcs0_C', 'Qcsmax_Wm2']
    fields3 = ['Name', 'Tsww0_C', 'dTww0_C', 'Qwwmax_Wm2']

    result = df[fields].merge(df2[fields2], on='Name').merge(df3[fields3], on='Name')
    return result

def test_demand():
    locator = inputlocator.InputLocator(scenario_path=r'C:\reference-case\baseline')
    gv = globalvar.GlobalVariables()
    demand_calculation(locator=locator, gv=gv)

if __name__ == '__main__':
    test_demand()<|endoftext|>