<commit_before># Licensed under the MIT license
# http://opensource.org/licenses/mit-license.php

# Copyright (C) 2006 Fluendo, S.A. (www.fluendo.com).
# Copyright 2006, Frank Scholz <coherence@beebits.net>

from coherence.extern.et import parse_xml as et_parse_xml

from twisted.web import server, http, static
from twisted.web import client, error
from twisted.internet import reactor
from twisted.python import failure

import socket
import fcntl
import struct
import string

def parse_xml(data, encoding="utf-8"):
    return et_parse_xml(data,encoding)

def parse_http_response(data):

    header, payload = data.split('\r\n\r\n')

    lines = header.split('\r\n')
    cmd = lines[0].split(' ')
    lines = map(lambda x: x.replace(': ', ':', 1), lines[1:])
    lines = filter(lambda x: len(x) > 0, lines)

    headers = [x.split(':', 1) for x in lines]
    headers = dict(map(lambda x: (x[0].lower(), x[1]), headers))

    return cmd, headers

def get_ip_address(ifname):
    """
    determine the IP address by interface name

    http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/439094
    (c) Paul Cannon
    Uses the Linux SIOCGIFADDR ioctl to find the IP address associated
    with a network interface, given the name of that interface, e.g. "eth0".
    The address is returned as a string containing a dotted quad.
    """

    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    return socket.inet_ntoa(fcntl.ioctl(
        s.fileno(),
        0x8915,  # SIOCGIFADDR
        struct.pack('256s', ifname[:15])
    )[20:24])

def get_host_address():
    """ try to get determine the interface used for
        the default route, as this is most likely
        the interface we should bind to (on a single homed host!)
    """
    try:
        route_file = '/proc/net/route'
        route = open(route_file)
        if (route):
            tmp = route.readline() #skip first line
            while (tmp != ''):
                tmp = route.readline()
                l = tmp.split('\t')
                if (len(l) > 2):
                    if l[2] != '00000000': #default gateway...
                        route.close()
                        return get_ip_address(l[0])
    except IOerror:
        """ fallback to parsing the output of netstat """
        import os, posix
        (osname,_, _, _,_) = os.uname()
        osname = osname.lower()
        f = posix.popen('netstat -rn')
        lines = f.readlines()
        f.close()
        for l in lines:
            parts = [x.strip() for x in l.split(' ') if len(x) > 0]
            if parts[0] in ('0.0.0.0','default'):
                if osname[:6] == 'darwin':
                    return get_ip_address(parts[5])
                else:
                    return get_ip_address(parts[-1])

    """ return localhost if we havn't found anything """
    return '127.0.0.1'

class Site(server.Site):

    noisy = False

    def startFactory(self):
        http._logDateTimeStart()

class myHTTPPageGetter(client.HTTPPageGetter):

    def handleResponse(self, response):
        if self.quietLoss:
            return
        if self.failed:
            self.factory.noPage(
                failure.Failure(
                    error.Error(
                        self.status, self.message, response)))
        elif self.factory.method != 'HEAD' and self.length != None and self.length != 0:
            self.factory.noPage(failure.Failure(
                client.PartialDownloadError(self.status, self.message, response)))
        else:
            self.factory.page(response)
        # server might be stupid and not close connection. admittedly
        # the fact we do only one request per connection is also
        # stupid...
        self.transport.loseConnection()

class HeaderAwareHTTPClientFactory(client.HTTPClientFactory):

    protocol = myHTTPPageGetter
    noisy = False

    def page(self, page):
        if self.waiting:
            self.waiting = 0
            self.deferred.callback((page, self.response_headers))

class HeaderAwareHTTPDownloader(client.HTTPDownloader):

    def gotHeaders(self, headers):
        self.value = headers
        if self.requestedPartial:
            contentRange = headers.get("content-range", None)
            if not contentRange:
                # server doesn't support partial requests, oh well
                self.requestedPartial = 0
                return
            start, end, realLength = http.parseContentRange(contentRange[0])
            if start != self.requestedPartial:
                # server is acting wierdly
                self.requestedPartial = 0

def getPage(url, contextFactory=None, *args, **kwargs):
    """Download a web page as a string.

    Download a page. Return a deferred, which will callback with a
    page (as a string) or errback with a description of the error.

    See HTTPClientFactory to see what extra args can be passed.
    """
    scheme, host, port, path = client._parse(url)
    factory = HeaderAwareHTTPClientFactory(url, *args, **kwargs)
    if scheme == 'https':
        from twisted.internet import ssl
        if contextFactory is None:
            contextFactory = ssl.ClientContextFactory()
        reactor.connectSSL(host, port, factory, contextFactory)
    else:
        reactor.connectTCP(host, port, factory)
    return factory.deferred

def downloadPage(url, file, contextFactory=None, *args, **kwargs):
    """Download a web page to a file.

    @param file: path to file on filesystem, or file-like object.

    See HTTPDownloader to see what extra args can be passed.
    """
    scheme, host, port, path = client._parse(url)
    factory = HeaderAwareHTTPDownloader(url, file, *args, **kwargs)
    factory.noisy = False
    if scheme == 'https':
        from twisted.internet import ssl
        if contextFactory is None:
            contextFactory = ssl.ClientContextFactory()
        reactor.connectSSL(host, port, factory, contextFactory)
    else:
        reactor.connectTCP(host, port, factory)
    return factory.deferred

class StaticFile(static.File):
    """ taken from twisted.web.static and modified
        accordingly to the patch by John-Mark Gurney
        http://resnet.uoregon.edu/~gurney_j/jmpc/dist/twisted.web.static.patch
    """

    def render(self, request):
        """You know what you doing."""
        self.restat()

        if self.type is None:
            self.type, self.encoding = static.getTypeAndEncoding(self.basename(),
                                                          self.contentTypes,
                                                          self.contentEncodings,
                                                          self.defaultType)

        if not self.exists():
            return self.childNotFound.render(request)

        if self.isdir():
            return self.redirect(request)

        #for content-length
        fsize = size = self.getFileSize()

        request.setHeader('accept-ranges','bytes')

        if self.type:
            request.setHeader('content-type', self.type)
        if self.encoding:
            request.setHeader('content-encoding', self.encoding)

        try:
            f = self.openForReading()
        except IOError, e:
            import errno
            if e[0] == errno.EACCES:
                return error.ForbiddenResource().render(request)
            else:
                raise

        if request.setLastModified(self.getmtime()) is http.CACHED:
            return ''

        trans = True

        range = request.getHeader('range')

        tsize = size
        if range is not None:
            # This is a request for partial data...
            bytesrange = string.split(range, '=')
            assert bytesrange[0] == 'bytes',\
                   "Syntactically invalid http range header!"
            start, end = string.split(bytesrange[1],'-', 1)
            if start:
                f.seek(int(start))
                if end:
                    end = int(end)
                else:
                    end = size - 1
            else:
                lastbytes = int(end)
                if size < lastbytes:
                    lastbytes = size
                start = size - lastbytes
                f.seek(start)
                fsize = lastbytes
                end = size - 1
            size = end + 1
            fsize = end - int(start) + 1
            # start is the byte offset to begin, and end is the byte offset
            # to end..  fsize is size to send, tsize is the real size of
            # the file, and size is the byte position to stop sending.

            if fsize <= 0:
                request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE)
                fsize = tsize
                trans = False
            else:
                request.setResponseCode(http.PARTIAL_CONTENT)
                request.setHeader('content-range',"bytes %s-%s/%s " % (
                    str(start), str(end), str(tsize)))

        request.setHeader('content-length', str(fsize))
        if request.method == 'HEAD' or trans == False:
            # pretend we're a HEAD request, so content-length
            # won't be overwritten.
            request.method = 'HEAD'
            return ''

        # return data
        # size is the byte position to stop sending, not how many bytes to send
        static.FileTransfer(f, size, request)
        # and make sure the connection doesn't get closed
        return server.NOT_DONE_YET

from datetime import datetime, tzinfo, timedelta
import random

class CET(tzinfo):

    def __init__(self):
        self.__offset = timedelta(minutes=60)
        self.__name = 'CET'

    def utcoffset(self, dt):
        return self.__offset

    def tzname(self, dt):
        return self.__name

    def dst(self,dt):
        return timedelta(0)

class CEST(tzinfo):

    def __init__(self):
        self.__offset = timedelta(minutes=120)
        self.__name = 'CEST'

    def utcoffset(self, dt):
        return self.__offset

    def tzname(self, dt):
        return self.__name

    def dst(self,dt):
        return timedelta(0)

bdates = [ datetime(1997,2,28,17,20,tzinfo=CET()),   # Sebastian Oliver
           datetime(1999,9,19,4,12,tzinfo=CEST()),   # Patrick Niklas
           datetime(2000,9,23,4,8,tzinfo=CEST()),    # Saskia Alexa
           datetime(2003,7,23,1,18,tzinfo=CEST()),   # Mara Sophie
                                                     # you are the best!
         ]

def datefaker():
    return random.choice(bdates)
<commit_msg>support for IP detection on BSD systems. Thx for the patch, kraft!<commit_after># Licensed under the MIT license
# http://opensource.org/licenses/mit-license.php

# Copyright (C) 2006 Fluendo, S.A. (www.fluendo.com).
# Copyright 2006, Frank Scholz <coherence@beebits.net>

from coherence.extern.et import parse_xml as et_parse_xml

from twisted.web import server, http, static
from twisted.web import client, error
from twisted.internet import reactor
from twisted.python import failure

import socket
import fcntl
import struct
import string
import os

def parse_xml(data, encoding="utf-8"):
    return et_parse_xml(data,encoding)

def parse_http_response(data):

    header, payload = data.split('\r\n\r\n')

    lines = header.split('\r\n')
    cmd = lines[0].split(' ')
    lines = map(lambda x: x.replace(': ', ':', 1), lines[1:])
    lines = filter(lambda x: len(x) > 0, lines)

    headers = [x.split(':', 1) for x in lines]
    headers = dict(map(lambda x: (x[0].lower(), x[1]), headers))

    return cmd, headers

def get_ip_address(ifname):
    """
    determine the IP address by interface name

    http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/439094
    (c) Paul Cannon
    Uses the Linux SIOCGIFADDR ioctl to find the IP address associated
    with a network interface, given the name of that interface, e.g. "eth0".
    The address is returned as a string containing a dotted quad.

    Updated to work on BSD. OpenBSD and OSX share the same value for
    SIOCGIFADDR, and its likely that other BSDs do too.
    """

    system_type = os.uname()[0]
    if system_type == "Linux":
        SIOCGIFADDR = 0x8915
    else:
        SIOCGIFADDR = 0xc0206921

    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    return socket.inet_ntoa(fcntl.ioctl(
        s.fileno(),
        SIOCGIFADDR,
        struct.pack('256s', ifname[:15])
    )[20:24])

def get_host_address():
    """ try to get determine the interface used for
        the default route, as this is most likely
        the interface we should bind to (on a single homed host!)
    """
    try:
        route_file = '/proc/net/route'
        route = open(route_file)
        if (route):
            tmp = route.readline() #skip first line
            while (tmp != ''):
                tmp = route.readline()
                l = tmp.split('\t')
                if (len(l) > 2):
                    if l[2] != '00000000': #default gateway...
                        route.close()
                        return get_ip_address(l[0])
    except IOerror:
        """ fallback to parsing the output of netstat """
        import os, posix
        (osname,_, _, _,_) = os.uname()
        osname = osname.lower()
        f = posix.popen('netstat -rn')
        lines = f.readlines()
        f.close()
        for l in lines:
            parts = [x.strip() for x in l.split(' ') if len(x) > 0]
            if parts[0] in ('0.0.0.0','default'):
                if osname[:6] == 'darwin':
                    return get_ip_address(parts[5])
                else:
                    return get_ip_address(parts[-1])

    """ return localhost if we havn't found anything """
    return '127.0.0.1'

class Site(server.Site):

    noisy = False

    def startFactory(self):
        http._logDateTimeStart()

class myHTTPPageGetter(client.HTTPPageGetter):

    def handleResponse(self, response):
        if self.quietLoss:
            return
        if self.failed:
            self.factory.noPage(
                failure.Failure(
                    error.Error(
                        self.status, self.message, response)))
        elif self.factory.method != 'HEAD' and self.length != None and self.length != 0:
            self.factory.noPage(failure.Failure(
                client.PartialDownloadError(self.status, self.message, response)))
        else:
            self.factory.page(response)
        # server might be stupid and not close connection. admittedly
        # the fact we do only one request per connection is also
        # stupid...
        self.transport.loseConnection()

class HeaderAwareHTTPClientFactory(client.HTTPClientFactory):

    protocol = myHTTPPageGetter
    noisy = False

    def page(self, page):
        if self.waiting:
            self.waiting = 0
            self.deferred.callback((page, self.response_headers))

class HeaderAwareHTTPDownloader(client.HTTPDownloader):

    def gotHeaders(self, headers):
        self.value = headers
        if self.requestedPartial:
            contentRange = headers.get("content-range", None)
            if not contentRange:
                # server doesn't support partial requests, oh well
                self.requestedPartial = 0
                return
            start, end, realLength = http.parseContentRange(contentRange[0])
            if start != self.requestedPartial:
                # server is acting wierdly
                self.requestedPartial = 0

def getPage(url, contextFactory=None, *args, **kwargs):
    """Download a web page as a string.

    Download a page. Return a deferred, which will callback with a
    page (as a string) or errback with a description of the error.

    See HTTPClientFactory to see what extra args can be passed.
    """
    scheme, host, port, path = client._parse(url)
    factory = HeaderAwareHTTPClientFactory(url, *args, **kwargs)
    if scheme == 'https':
        from twisted.internet import ssl
        if contextFactory is None:
            contextFactory = ssl.ClientContextFactory()
        reactor.connectSSL(host, port, factory, contextFactory)
    else:
        reactor.connectTCP(host, port, factory)
    return factory.deferred

def downloadPage(url, file, contextFactory=None, *args, **kwargs):
    """Download a web page to a file.

    @param file: path to file on filesystem, or file-like object.

    See HTTPDownloader to see what extra args can be passed.
    """
    scheme, host, port, path = client._parse(url)
    factory = HeaderAwareHTTPDownloader(url, file, *args, **kwargs)
    factory.noisy = False
    if scheme == 'https':
        from twisted.internet import ssl
        if contextFactory is None:
            contextFactory = ssl.ClientContextFactory()
        reactor.connectSSL(host, port, factory, contextFactory)
    else:
        reactor.connectTCP(host, port, factory)
    return factory.deferred

class StaticFile(static.File):
    """ taken from twisted.web.static and modified
        accordingly to the patch by John-Mark Gurney
        http://resnet.uoregon.edu/~gurney_j/jmpc/dist/twisted.web.static.patch
    """

    def render(self, request):
        """You know what you doing."""
        self.restat()

        if self.type is None:
            self.type, self.encoding = static.getTypeAndEncoding(self.basename(),
                                                          self.contentTypes,
                                                          self.contentEncodings,
                                                          self.defaultType)

        if not self.exists():
            return self.childNotFound.render(request)

        if self.isdir():
            return self.redirect(request)

        #for content-length
        fsize = size = self.getFileSize()

        request.setHeader('accept-ranges','bytes')

        if self.type:
            request.setHeader('content-type', self.type)
        if self.encoding:
            request.setHeader('content-encoding', self.encoding)

        try:
            f = self.openForReading()
        except IOError, e:
            import errno
            if e[0] == errno.EACCES:
                return error.ForbiddenResource().render(request)
            else:
                raise

        if request.setLastModified(self.getmtime()) is http.CACHED:
            return ''

        trans = True

        range = request.getHeader('range')

        tsize = size
        if range is not None:
            # This is a request for partial data...
            bytesrange = string.split(range, '=')
            assert bytesrange[0] == 'bytes',\
                   "Syntactically invalid http range header!"
            start, end = string.split(bytesrange[1],'-', 1)
            if start:
                f.seek(int(start))
                if end:
                    end = int(end)
                else:
                    end = size - 1
            else:
                lastbytes = int(end)
                if size < lastbytes:
                    lastbytes = size
                start = size - lastbytes
                f.seek(start)
                fsize = lastbytes
                end = size - 1
            size = end + 1
            fsize = end - int(start) + 1
            # start is the byte offset to begin, and end is the byte offset
            # to end..  fsize is size to send, tsize is the real size of
            # the file, and size is the byte position to stop sending.

            if fsize <= 0:
                request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE)
                fsize = tsize
                trans = False
            else:
                request.setResponseCode(http.PARTIAL_CONTENT)
                request.setHeader('content-range',"bytes %s-%s/%s " % (
                    str(start), str(end), str(tsize)))

        request.setHeader('content-length', str(fsize))
        if request.method == 'HEAD' or trans == False:
            # pretend we're a HEAD request, so content-length
            # won't be overwritten.
            request.method = 'HEAD'
            return ''

        # return data
        # size is the byte position to stop sending, not how many bytes to send
        static.FileTransfer(f, size, request)
        # and make sure the connection doesn't get closed
        return server.NOT_DONE_YET

from datetime import datetime, tzinfo, timedelta
import random

class CET(tzinfo):

    def __init__(self):
        self.__offset = timedelta(minutes=60)
        self.__name = 'CET'

    def utcoffset(self, dt):
        return self.__offset

    def tzname(self, dt):
        return self.__name

    def dst(self,dt):
        return timedelta(0)

class CEST(tzinfo):

    def __init__(self):
        self.__offset = timedelta(minutes=120)
        self.__name = 'CEST'

    def utcoffset(self, dt):
        return self.__offset

    def tzname(self, dt):
        return self.__name

    def dst(self,dt):
        return timedelta(0)

bdates = [ datetime(1997,2,28,17,20,tzinfo=CET()),   # Sebastian Oliver
           datetime(1999,9,19,4,12,tzinfo=CEST()),   # Patrick Niklas
           datetime(2000,9,23,4,8,tzinfo=CEST()),    # Saskia Alexa
           datetime(2003,7,23,1,18,tzinfo=CEST()),   # Mara Sophie
                                                     # you are the best!
         ]

def datefaker():
    return random.choice(bdates)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import json
import random
import re
import string
import types

from datetime import datetime, timedelta

data = {
    'NUMBER' : tuple("0123456789"),
    'LETTER_UPPER' : tuple(string.uppercase),
    'LETTER_LOWER' : tuple(string.lowercase),
    'MALE_FIRST_NAME' : ("James", "John", "Robert", "Michael", "William", "David",
        "Richard", "Charles", "Joseph", "Thomas", "Christopher", "Daniel",
        "Paul", "Mark", "Donald", "George", "Kenneth", "Steven", "Edward",
        "Brian", "Ronald", "Anthony", "Kevin", "Jason", "Matthew", "Gary",
        "Timothy", "Jose", "Larry", "Jeffrey", "Frank", "Scott", "Eric"),
    'FEMALE_FIRST_NAME' : ("Mary", "Patricia", "Linda", "Barbara", "Elizabeth",
        "Jennifer", "Maria", "Susan", "Margaret", "Dorothy", "Lisa", "Nancy",
        "Karen", "Betty", "Helen", "Sandra", "Donna", "Carol", "Ruth", "Sharon",
        "Michelle", "Laura", "Sarah", "Kimberly", "Deborah", "Jessica",
        "Shirley", "Cynthia", "Angela", "Melissa", "Brenda", "Amy", "Anna"),
    'LAST_NAME' : ("Smith", "Johnson", "Williams", "Brown", "Jones", "Miller",
        "Davis", "Garcia", "Rodriguez", "Wilson", "Martinez", "Anderson",
        "Taylor", "Thomas", "Hernandez", "Moore", "Martin", "Jackson",
        "Thompson", "White", "Lopez", "Lee", "Gonzalez", "Harris", "Clark",
        "Lewis", "Robinson", "Walker", "Perez", "Hall", "Young", "Allen")
}

def _random_data(key) :
    key = key.lstrip('@')

    params = re.findall(r"\(([^\)]+)\)", key)
    params = params if params else []

    if not data.has_key(key):
        return key #FIXME log or exception

    d = data[key]
    k_type = type(d)
    if k_type is list or k_type is tuple:
        return d[random.randrange(len(d))]
    elif k_type is types.FunctionType:
        return d
    raise Exception('invalid key type')

def _random_email():
    l = _random_data('@LETTER_LOWER')
    n1 = _random_data('@LAST_NAME').lower()
    n2 = _random_data('@LAST_NAME').lower()
    return l + '.' + n1 + '@' + n2 + '.com'

def _lorem():
    words = tuple("""lorem ipsum dolor sit amet consectetur adipisicing elit sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Ut enim ad minim veniam quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur Excepteur sint occaecat cupidatat non proident sunt in culpa qui officia deserunt mollit anim id est laborum""".split())
    return words[random.randrange(len(words))]

def _lorem_ipsum():
    words = tuple("""Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum""".split())
    length = random.randrange(len(words)/2)

    # TODO min length here, apparently I don't understand python ternarys
    #length = length if length < 3 else 3
    result = ''
    for i in range(length):
        result += ' ' + words[random.randrange(len(words))]
    return result.strip()

def _random_date():
    return datetime.today() - timedelta(days = random.randrange(6571, 27375))

def _random_year():
    return str(_random_date().year)

def _random_month():
    return str(_random_date().month).zfill(2)

def _random_day():
    return str(_random_date().day).zfill(2)

def _random_hour():
    return str(_random_date().hour).zfill(2)

def _random_minutes():
    return str(_random_date().minute).zfill(2)

def _random_seconds():
    return str(_random_date().second).zfill(2)

# additional data definitions that are dependent on functions
data['EMAIL'] = _random_email
data['LOREM'] = _lorem
data['LOREM_IPSUM'] = _lorem_ipsum
data['DATE_YYYY'] = _random_year
data['DATE_MM'] = _random_month
data['DATE_DD'] = _random_day
data['TIME_HH'] = _random_hour
data['TIME_MM'] = _random_minutes
data['TIME_SS'] = _random_seconds

def generate_json_object(template, name=None):
    #print template, name

    length = 0
    if name:
        matches = re.search(r"\w+\|(\d+)-(\d+)", name)
        if matches:
            groups = matches.groups()
            length_min = int(groups[0])
            length_max = int(groups[1])
            length = random.randint(length_min, length_max)

    t_type = type(template)

    if t_type is dict:
        generated = {}
        for key, value in template.iteritems():
            stripped_key = re.sub(r"\|(\d+-\d+|\+\d+)",'', key)
            generated[stripped_key] = generate_json_object(value, key)

            # handle increments
            inc_matches = re.search(r"\w+\|\+(\d+)", key)
            if inc_matches and type(template[key]) is int:
                increment = int(inc_matches.groups()[0])
                template[key] += increment
    elif t_type is list:
        generated = []
        for i in range(length):
            generated.append(generate_json_object(template[0]))
    elif t_type is int:
        generated = length if matches else template
    elif t_type is bool:
        # apparently getrandbits(1) is faster...
        generated = random.choice([True, False]) if matches else template
    # is this always just going to be unicode here?
    elif t_type is str or t_type is unicode:
        if template:
            generated = ''
            length = length if length else 1
            for i in range(length):
                generated += template
            matches = re.findall(r"(@[A-Z_0-9\(\),]+)", generated)
            if matches:
                for key in matches:
                    generated = generated.replace(key, _random_data(key))
        else:
            generated = ''.join(random.choice(string.letters) for i in xrange(length))
    else:
        generated = template
    return generated

def generate_json(template, name=None):
    return json.dumps(generate_json_object(json_data), sort_keys=False)

if __name__ == '__main__':
    with open('test.template.json') as f:
        json_data = json.load(f)
    print(generate_json(json_data))
<commit_msg>take file from system args<commit_after>#!/usr/bin/env python

import json
import random
import re
import string
import sys
import types

from datetime import datetime, timedelta

data = {
    'NUMBER' : tuple("0123456789"),
    'LETTER_UPPER' : tuple(string.uppercase),
    'LETTER_LOWER' : tuple(string.lowercase),
    'MALE_FIRST_NAME' : ("James", "John", "Robert", "Michael", "William", "David",
        "Richard", "Charles", "Joseph", "Thomas", "Christopher", "Daniel",
        "Paul", "Mark", "Donald", "George", "Kenneth", "Steven", "Edward",
        "Brian", "Ronald", "Anthony", "Kevin", "Jason", "Matthew", "Gary",
        "Timothy", "Jose", "Larry", "Jeffrey", "Frank", "Scott", "Eric"),
    'FEMALE_FIRST_NAME' : ("Mary", "Patricia", "Linda", "Barbara", "Elizabeth",
        "Jennifer", "Maria", "Susan", "Margaret", "Dorothy", "Lisa", "Nancy",
        "Karen", "Betty", "Helen", "Sandra", "Donna", "Carol", "Ruth", "Sharon",
        "Michelle", "Laura", "Sarah", "Kimberly", "Deborah", "Jessica",
        "Shirley", "Cynthia", "Angela", "Melissa", "Brenda", "Amy", "Anna"),
    'LAST_NAME' : ("Smith", "Johnson", "Williams", "Brown", "Jones", "Miller",
        "Davis", "Garcia", "Rodriguez", "Wilson", "Martinez", "Anderson",
        "Taylor", "Thomas", "Hernandez", "Moore", "Martin", "Jackson",
        "Thompson", "White", "Lopez", "Lee", "Gonzalez", "Harris", "Clark",
        "Lewis", "Robinson", "Walker", "Perez", "Hall", "Young", "Allen")
}

def _random_data(key) :
    key = key.lstrip('@')

    params = re.findall(r"\(([^\)]+)\)", key)
    params = params if params else []

    if not data.has_key(key):
        return key #FIXME log or exception

    d = data[key]
    k_type = type(d)
    if k_type is list or k_type is tuple:
        return d[random.randrange(len(d))]
    elif k_type is types.FunctionType:
        return d
    raise Exception('invalid key type')

def _random_email():
    l = _random_data('@LETTER_LOWER')
    n1 = _random_data('@LAST_NAME').lower()
    n2 = _random_data('@LAST_NAME').lower()
    return l + '.' + n1 + '@' + n2 + '.com'

def _lorem():
    words = tuple("""lorem ipsum dolor sit amet consectetur adipisicing elit sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Ut enim ad minim veniam quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur Excepteur sint occaecat cupidatat non proident sunt in culpa qui officia deserunt mollit anim id est laborum""".split())
    return words[random.randrange(len(words))]

def _lorem_ipsum():
    words = tuple("""Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum""".split())
    length = random.randrange(len(words)/2)

    # TODO min length here, apparently I don't understand python ternarys
    #length = length if length < 3 else 3
    result = ''
    for i in range(length):
        result += ' ' + words[random.randrange(len(words))]
    return result.strip()

def _random_date():
    return datetime.today() - timedelta(days = random.randrange(6571, 27375))

def _random_year():
    return str(_random_date().year)

def _random_month():
    return str(_random_date().month).zfill(2)

def _random_day():
    return str(_random_date().day).zfill(2)

def _random_hour():
    return str(_random_date().hour).zfill(2)

def _random_minutes():
    return str(_random_date().minute).zfill(2)

def _random_seconds():
    return str(_random_date().second).zfill(2)

# additional data definitions that are dependent on functions
data['EMAIL'] = _random_email
data['LOREM'] = _lorem
data['LOREM_IPSUM'] = _lorem_ipsum
data['DATE_YYYY'] = _random_year
data['DATE_MM'] = _random_month
data['DATE_DD'] = _random_day
data['TIME_HH'] = _random_hour
data['TIME_MM'] = _random_minutes
data['TIME_SS'] = _random_seconds

def generate_json_object(template, name=None):
    #print template, name

    length = 0
    if name:
        matches = re.search(r"\w+\|(\d+)-(\d+)", name)
        if matches:
            groups = matches.groups()
            length_min = int(groups[0])
            length_max = int(groups[1])
            length = random.randint(length_min, length_max)

    t_type = type(template)

    if t_type is dict:
        generated = {}
        for key, value in template.iteritems():
            stripped_key = re.sub(r"\|(\d+-\d+|\+\d+)",'', key)
            generated[stripped_key] = generate_json_object(value, key)

            # handle increments
            inc_matches = re.search(r"\w+\|\+(\d+)", key)
            if inc_matches and type(template[key]) is int:
                increment = int(inc_matches.groups()[0])
                template[key] += increment
    elif t_type is list:
        generated = []
        for i in range(length):
            generated.append(generate_json_object(template[0]))
    elif t_type is int:
        generated = length if matches else template
    elif t_type is bool:
        # apparently getrandbits(1) is faster...
        generated = random.choice([True, False]) if matches else template
    # is this always just going to be unicode here?
    elif t_type is str or t_type is unicode:
        if template:
            generated = ''
            length = length if length else 1
            for i in range(length):
                generated += template
            matches = re.findall(r"(@[A-Z_0-9\(\),]+)", generated)
            if matches:
                for key in matches:
                    generated = generated.replace(key, _random_data(key))
        else:
            generated = ''.join(random.choice(string.letters) for i in xrange(length))
    else:
        generated = template
    return generated

def generate_json(template, name=None):
    return json.dumps(generate_json_object(json_data), sort_keys=False)

if __name__ == '__main__':
    arg = sys.argv[1:]
    with open(arg) as f:
        json_data = json.load(f)
    print(generate_json(json_data))
<|endoftext|>


<commit_before>#########
# Copyright (c) 2018 Cloudify Platform Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  * See the License for the specific language governing permissions and
#  * limitations under the License.

from datetime import datetime
from flask_restful.reqparse import Argument

from cloudify._compat import text_type
from cloudify import constants as common_constants
from cloudify.workflows import events as common_events

from manager_rest import manager_exceptions
from manager_rest.rest.rest_utils import (
    get_args_and_verify_arguments,
    get_json_and_verify_params,
)
from manager_rest.rest.rest_decorators import (
    marshal_with,
    paginate
)
from manager_rest.storage import (
    get_storage_manager,
    models,
    db
)
from manager_rest.security.authorization import authorize
from manager_rest.resource_manager import get_resource_manager
from manager_rest.security import SecuredResource
from manager_rest.security.authorization import is_user_action_allowed
from manager_rest.execution_token import current_execution

class Operations(SecuredResource):
    @authorize('operations')
    @marshal_with(models.Operation)
    @paginate
    def get(self, _include=None, pagination=None, **kwargs):
        args = get_args_and_verify_arguments([
            Argument('graph_id', type=text_type, required=True)
        ])
        sm = get_storage_manager()
        graph_id = args.get('graph_id')
        tasks_graph = sm.list(models.TasksGraph, filters={'id': graph_id})[0]
        return sm.list(
            models.Operation,
            filters={'tasks_graph': tasks_graph},
            pagination=pagination,
            include=_include
        )

class OperationsId(SecuredResource):
    @authorize('operations')
    @marshal_with(models.Operation)
    def get(self, operation_id, **kwargs):
        return get_storage_manager().get(models.Operation, operation_id)

    @authorize('operations')
    @marshal_with(models.Operation)
    def put(self, operation_id, **kwargs):
        params = get_json_and_verify_params({
            'name': {'type': text_type, 'required': True},
            'graph_id': {'type': text_type, 'required': True},
            'dependencies': {'type': list, 'required': True},
            'parameters': {'type': dict},
            'type': {'type': text_type}
        })
        operation = get_resource_manager().create_operation(
            operation_id,
            name=params['name'],
            graph_id=params['graph_id'],
            dependencies=params['dependencies'],
            type=params['type'],
            parameters=params['parameters']
        )
        return operation, 201

    @authorize('operations', allow_if_execution=True)
    @marshal_with(models.Operation)
    def patch(self, operation_id, **kwargs):
        request_dict = get_json_and_verify_params({
            'state': {'type': text_type},
            'result': {'optional': True},
            'exception': {'optional': True},
            'exception_causes': {'optional': True},
        })
        sm = get_storage_manager()
        with sm.transaction():
            instance = sm.get(models.Operation, operation_id, locking=True)
            old_state = instance.state
            instance.state = request_dict.get('state', instance.state)
            if instance.state == common_constants.TASK_SUCCEEDED:
                self._on_task_success(sm, instance)
            self._insert_event(
                instance,
                request_dict.get('result'),
                request_dict.get('exception'),
                request_dict.get('exception_causes')
            )
            if not instance.is_nop and \
                    old_state not in common_constants.TERMINATED_STATES and \
                    instance.state in common_constants.TERMINATED_STATES:
                self._modify_execution_operations_counts(instance, 1)
            instance = sm.update(instance, modified_attrs=('state',))
        return instance

    def _on_task_success(self, sm, operation):
        handler = getattr(self, f'_on_success_{operation.type}', None)
        if handler:
            handler(sm, operation)

    def _on_success_SetNodeInstanceStateTask(self, sm, operation):
        required_permission = 'node_instance_update'
        tenant_name = current_execution.tenant.name
        if not is_user_action_allowed(required_permission,
                                      tenant_name=tenant_name):
            raise manager_exceptions.ForbiddenError(
                f'Execution is not permitted to perform the action '
                f'{required_permission} in the tenant {tenant_name}'
            )
        try:
            kwargs = operation.parameters['task_kwargs']
            node_instance_id = kwargs['node_instance_id']
            state = kwargs['state']
        except KeyError:
            return
        node_instance = sm.get(models.NodeInstance, node_instance_id)
        node_instance.state = state
        sm.update(node_instance)

    def _insert_event(self, operation, result=None, exception=None,
                      exception_causes=None):
        if operation.type != 'RemoteWorkflowTask':
            return
        if not current_execution:
            return
        try:
            context = operation.parameters['task_kwargs']['kwargs'][
                '__cloudify_context']
        except (KeyError, TypeError):
            return
        if exception is not None:
            operation.parameters.setdefault('error', str(exception))
        current_retries = context.get('current_retries') or 0
        total_retries = context.get('total_retries') or 0

        try:
            message = common_events.format_event_message(
                operation.name,
                operation.state,
                result,
                exception,
                current_retries,
                total_retries,
            )
            event_type = common_events.get_event_type(operation.state)
        except RuntimeError:
            return

        db.session.execute(models.Event.__table__.insert().values(
            timestamp=datetime.utcnow(),
            reported_timestamp=datetime.utcnow(),
            event_type=event_type,
            message=message,
            message_code=None,
            operation=operation.name,
            node_id=context.get('node_id'),
            source_id=context.get('source_id'),
            target_id=context.get('target_id'),
            error_causes=exception_causes,
            _execution_fk=current_execution._storage_id,
            _tenant_id=current_execution._tenant_id,
            _creator_id=current_execution._creator_id,
            visibility=current_execution.visibility,
        ))

    def _modify_execution_operations_counts(self, operation, finished_delta,
                                            total_delta=0):
        """Increase finished_operations for this operation's execution

        This is a separate sql-level update query, rather than ORM-level
        calls, for performance: the operation state-update call is on
        the critical path for all operations in a workflow; this saves
        about 3ms over the ORM approach (which requires fetching the
        execution; more if the DB is not local).
        """
        exc_table = models.Execution.__table__
        tg_table = models.TasksGraph.__table__
        values = {}
        if finished_delta:
            values['finished_operations'] =\
                exc_table.c.finished_operations + finished_delta
        if total_delta:
            values['total_operations'] =\
                exc_table.c.total_operations + total_delta
        db.session.execute(
            exc_table.update()
            .where(db.and_(
                tg_table.c._execution_fk == exc_table.c._storage_id,
                tg_table.c._storage_id == operation._tasks_graph_fk,
            ))
            .values(**values)
        )

    @authorize('operations')
    @marshal_with(models.Operation)
    def delete(self, operation_id, **kwargs):
        sm = get_storage_manager()
        with sm.transaction():
            instance = sm.get(models.Operation, operation_id, locking=True)
            if not instance.is_nop:
                finished_delta = (
                    -1
                    if instance.state in common_constants.TERMINATED_STATES
                    else 0
                )
                self._modify_execution_operations_counts(
                    instance, finished_delta, total_delta=-1)
            sm.delete(instance)
        return instance, 200

class TasksGraphs(SecuredResource):
    @authorize('operations')
    @marshal_with(models.TasksGraph)
    @paginate
    def get(self, _include=None, pagination=None, **kwargs):
        args = get_args_and_verify_arguments([
            Argument('execution_id', type=text_type, required=True),
            Argument('name', type=text_type, required=True)
        ])
        sm = get_storage_manager()
        execution_id = args.get('execution_id')
        name = args.get('name')
        execution = sm.list(models.Execution, filters={'id': execution_id})[0]
        return sm.list(
            models.TasksGraph,
            filters={'execution': execution, 'name': name},
            pagination=pagination
        )

class TasksGraphsId(SecuredResource):
    @authorize('operations')
    @marshal_with(models.TasksGraph)
    def post(self, **kwargs):
        params = get_json_and_verify_params({
            'name': {'type': text_type, 'required': True},
            'execution_id': {'type': text_type, 'required': True},
            'operations': {'required': False}
        })
        sm = get_storage_manager()
        with sm.transaction():
            tasks_graph = get_resource_manager().create_tasks_graph(
                name=params['name'],
                execution_id=params['execution_id'],
                operations=params.get('operations', [])
            )
        return tasks_graph, 201

    @authorize('operations')
    @marshal_with(models.TasksGraph)
    def patch(self, tasks_graph_id, **kwargs):
        request_dict = get_json_and_verify_params(
            {'state': {'type': text_type}}
        )
        sm = get_storage_manager()
        instance = sm.get(models.TasksGraph, tasks_graph_id, locking=True)
        instance.state = request_dict.get('state', instance.state)
        return sm.update(instance)
<commit_msg>RD-2650 Automatically insert node-events on task success (#3128)<commit_after>#########
# Copyright (c) 2018 Cloudify Platform Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  * See the License for the specific language governing permissions and
#  * limitations under the License.

from datetime import datetime
from flask_restful.reqparse import Argument

from cloudify._compat import text_type
from cloudify import constants as common_constants
from cloudify.workflows import events as common_events

from manager_rest import manager_exceptions
from manager_rest.rest.rest_utils import (
    get_args_and_verify_arguments,
    get_json_and_verify_params,
)
from manager_rest.rest.rest_decorators import (
    marshal_with,
    paginate
)
from manager_rest.storage import (
    get_storage_manager,
    models,
    db
)
from manager_rest.security.authorization import authorize
from manager_rest.resource_manager import get_resource_manager
from manager_rest.security import SecuredResource
from manager_rest.security.authorization import is_user_action_allowed
from manager_rest.execution_token import current_execution

class Operations(SecuredResource):
    @authorize('operations')
    @marshal_with(models.Operation)
    @paginate
    def get(self, _include=None, pagination=None, **kwargs):
        args = get_args_and_verify_arguments([
            Argument('graph_id', type=text_type, required=True)
        ])
        sm = get_storage_manager()
        graph_id = args.get('graph_id')
        tasks_graph = sm.list(models.TasksGraph, filters={'id': graph_id})[0]
        return sm.list(
            models.Operation,
            filters={'tasks_graph': tasks_graph},
            pagination=pagination,
            include=_include
        )

class OperationsId(SecuredResource):
    @authorize('operations')
    @marshal_with(models.Operation)
    def get(self, operation_id, **kwargs):
        return get_storage_manager().get(models.Operation, operation_id)

    @authorize('operations')
    @marshal_with(models.Operation)
    def put(self, operation_id, **kwargs):
        params = get_json_and_verify_params({
            'name': {'type': text_type, 'required': True},
            'graph_id': {'type': text_type, 'required': True},
            'dependencies': {'type': list, 'required': True},
            'parameters': {'type': dict},
            'type': {'type': text_type}
        })
        operation = get_resource_manager().create_operation(
            operation_id,
            name=params['name'],
            graph_id=params['graph_id'],
            dependencies=params['dependencies'],
            type=params['type'],
            parameters=params['parameters']
        )
        return operation, 201

    @authorize('operations', allow_if_execution=True)
    @marshal_with(models.Operation)
    def patch(self, operation_id, **kwargs):
        request_dict = get_json_and_verify_params({
            'state': {'type': text_type},
            'result': {'optional': True},
            'exception': {'optional': True},
            'exception_causes': {'optional': True},
        })
        sm = get_storage_manager()
        with sm.transaction():
            instance = sm.get(models.Operation, operation_id, locking=True)
            old_state = instance.state
            instance.state = request_dict.get('state', instance.state)
            if instance.state == common_constants.TASK_SUCCEEDED:
                self._on_task_success(sm, instance)
            self._insert_event(
                instance,
                request_dict.get('result'),
                request_dict.get('exception'),
                request_dict.get('exception_causes')
            )
            if not instance.is_nop and \
                    old_state not in common_constants.TERMINATED_STATES and \
                    instance.state in common_constants.TERMINATED_STATES:
                self._modify_execution_operations_counts(instance, 1)
            instance = sm.update(instance, modified_attrs=('state',))
        return instance

    def _on_task_success(self, sm, operation):
        handler = getattr(self, f'_on_success_{operation.type}', None)
        if handler:
            handler(sm, operation)

    def _on_success_SetNodeInstanceStateTask(self, sm, operation):
        required_permission = 'node_instance_update'
        tenant_name = current_execution.tenant.name
        if not is_user_action_allowed(required_permission,
                                      tenant_name=tenant_name):
            raise manager_exceptions.ForbiddenError(
                f'Execution is not permitted to perform the action '
                f'{required_permission} in the tenant {tenant_name}'
            )
        try:
            kwargs = operation.parameters['task_kwargs']
            node_instance_id = kwargs['node_instance_id']
            state = kwargs['state']
        except KeyError:
            return
        node_instance = sm.get(models.NodeInstance, node_instance_id)
        node_instance.state = state
        sm.update(node_instance)

    def _on_success_SendNodeEventTask(self, sm, operation):
        try:
            kwargs = operation.parameters['task_kwargs']
        except KeyError:
            pass
        db.session.execute(models.Event.__table__.insert().values(
            timestamp=datetime.utcnow(),
            reported_timestamp=datetime.utcnow(),
            event_type='workflow_node_event',
            message=kwargs['event'],
            message_code=None,
            operation=None,
            node_id=kwargs['node_instance_id'],
            _execution_fk=current_execution._storage_id,
            _tenant_id=current_execution._tenant_id,
            _creator_id=current_execution._creator_id,
            visibility=current_execution.visibility,
        ))

    def _insert_event(self, operation, result=None, exception=None,
                      exception_causes=None):
        if operation.type != 'RemoteWorkflowTask':
            return
        if not current_execution:
            return
        try:
            context = operation.parameters['task_kwargs']['kwargs'][
                '__cloudify_context']
        except (KeyError, TypeError):
            return
        if exception is not None:
            operation.parameters.setdefault('error', str(exception))
        current_retries = context.get('current_retries') or 0
        total_retries = context.get('total_retries') or 0

        try:
            message = common_events.format_event_message(
                operation.name,
                operation.state,
                result,
                exception,
                current_retries,
                total_retries,
            )
            event_type = common_events.get_event_type(operation.state)
        except RuntimeError:
            return

        db.session.execute(models.Event.__table__.insert().values(
            timestamp=datetime.utcnow(),
            reported_timestamp=datetime.utcnow(),
            event_type=event_type,
            message=message,
            message_code=None,
            operation=operation.name,
            node_id=context.get('node_id'),
            source_id=context.get('source_id'),
            target_id=context.get('target_id'),
            error_causes=exception_causes,
            _execution_fk=current_execution._storage_id,
            _tenant_id=current_execution._tenant_id,
            _creator_id=current_execution._creator_id,
            visibility=current_execution.visibility,
        ))

    def _modify_execution_operations_counts(self, operation, finished_delta,
                                            total_delta=0):
        """Increase finished_operations for this operation's execution

        This is a separate sql-level update query, rather than ORM-level
        calls, for performance: the operation state-update call is on
        the critical path for all operations in a workflow; this saves
        about 3ms over the ORM approach (which requires fetching the
        execution; more if the DB is not local).
        """
        exc_table = models.Execution.__table__
        tg_table = models.TasksGraph.__table__
        values = {}
        if finished_delta:
            values['finished_operations'] =\
                exc_table.c.finished_operations + finished_delta
        if total_delta:
            values['total_operations'] =\
                exc_table.c.total_operations + total_delta
        db.session.execute(
            exc_table.update()
            .where(db.and_(
                tg_table.c._execution_fk == exc_table.c._storage_id,
                tg_table.c._storage_id == operation._tasks_graph_fk,
            ))
            .values(**values)
        )

    @authorize('operations')
    @marshal_with(models.Operation)
    def delete(self, operation_id, **kwargs):
        sm = get_storage_manager()
        with sm.transaction():
            instance = sm.get(models.Operation, operation_id, locking=True)
            if not instance.is_nop:
                finished_delta = (
                    -1
                    if instance.state in common_constants.TERMINATED_STATES
                    else 0
                )
                self._modify_execution_operations_counts(
                    instance, finished_delta, total_delta=-1)
            sm.delete(instance)
        return instance, 200

class TasksGraphs(SecuredResource):
    @authorize('operations')
    @marshal_with(models.TasksGraph)
    @paginate
    def get(self, _include=None, pagination=None, **kwargs):
        args = get_args_and_verify_arguments([
            Argument('execution_id', type=text_type, required=True),
            Argument('name', type=text_type, required=True)
        ])
        sm = get_storage_manager()
        execution_id = args.get('execution_id')
        name = args.get('name')
        execution = sm.list(models.Execution, filters={'id': execution_id})[0]
        return sm.list(
            models.TasksGraph,
            filters={'execution': execution, 'name': name},
            pagination=pagination
        )

class TasksGraphsId(SecuredResource):
    @authorize('operations')
    @marshal_with(models.TasksGraph)
    def post(self, **kwargs):
        params = get_json_and_verify_params({
            'name': {'type': text_type, 'required': True},
            'execution_id': {'type': text_type, 'required': True},
            'operations': {'required': False}
        })
        sm = get_storage_manager()
        with sm.transaction():
            tasks_graph = get_resource_manager().create_tasks_graph(
                name=params['name'],
                execution_id=params['execution_id'],
                operations=params.get('operations', [])
            )
        return tasks_graph, 201

    @authorize('operations')
    @marshal_with(models.TasksGraph)
    def patch(self, tasks_graph_id, **kwargs):
        request_dict = get_json_and_verify_params(
            {'state': {'type': text_type}}
        )
        sm = get_storage_manager()
        instance = sm.get(models.TasksGraph, tasks_graph_id, locking=True)
        instance.state = request_dict.get('state', instance.state)
        return sm.update(instance)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import datetime
from dateutil.relativedelta import relativedelta
import calendar
from itertools import islice
import math
from decimal import Decimal
from collections import defaultdict

class Payment(object):
    def __init__(self, date, payment, interest_payment, principal_payment, current_principal):
        self.date = date
        self.payment = payment
        self.interest_payment = interest_payment
        self.principal_payment = principal_payment
        self.current_principal = current_principal
        self.not_used = False

class RegularPayment(Payment):
    def __init__(self, date, payment, interest_payment, principal_payment, current_principal):
        return super().__init__(date, payment, interest_payment, principal_payment, current_principal)

    def __str__(self):
        if not self.not_used:
            return '%s - %.2f - %.2f - %.2f - %.2f - regular' % (str(self.date), self.payment, self.interest_payment, self.principal_payment, self.current_principal)
        else:
            return '%s - %.2f - not used' % (str(self.date), self.payment)

class NonRegularPayment(Payment):
    def __init__(self, date, payment, interest_payment, principal_payment, current_principal):
        super().__init__(date, payment, interest_payment, principal_payment, current_principal)

    def __str__(self):
        if not self.not_used:
            return '%s - %.2f - %.2f - %.2f - %.2f - non regular' % (str(self.date), self.payment, self.interest_payment, self.principal_payment, self.current_principal)
        else:
            return '%s - %.2f - not used' % (str(self.date), self.payment)

class Period(object):
    def __init__(self, start, end):
        self.start = start
        self.end = end
        self.payments = []
        self.principal_paid = False

    def add_payment(self, payment):
        self.payments.append(payment)

class Result(object):
    def __init__(self):
        self.payments = defaultdict(list)
        self.year_interest_payments = defaultdict(Decimal)

class Mortgage(object):
    def __init__(self, interest, months, start_date):
        self.interest = interest
        self.interest_rate = Decimal(interest / 100)
        self.monthly_interest = Decimal(self.interest_rate / 12)

        # срок кредита
        self.total_months = months

        # дата получения кредита
        self.start_date = start_date

        self.cached_calendar = [self.start_date + relativedelta(months=m) for m in range(0, self.total_months + 1)]

        self.leap_years_cache = [366 if calendar.isleap(self.start_date.year + y) else 365 for y in range(0, int(self.total_months / 12))]

    def calc_monthly_payment(self, principal, months):
        return Decimal(round(principal * (self.monthly_interest / (1 - (1 + self.monthly_interest) ** (- months))), 2))

    def calc_interest_payment(self, pay_date, prev_pay_date, principal):
        # первый месяц в году, т.е. январь
        # в этом случае надо немного по-другому считать
        days_in_year = self.leap_years_cache[self.start_date.year - pay_date.year]
        if pay_date.month == 1 and prev_pay_date.month == 12:
            december31 = datetime.date(prev_pay_date.year, 12, 31)

            days1 = december31 - prev_pay_date
            days1 = days1.days
            p1 = Decimal(days1 / self.leap_years_cache[self.start_date.year - prev_pay_date.year])
            interest_payment1 = Decimal(round(principal * self.interest_rate * p1, 2))

            days2 = pay_date - december31
            days2 = days2.days
            p2 = Decimal(days2 / days_in_year)
            interest_payment2 = Decimal(round(principal * self.interest_rate * p2, 2))

            return interest_payment1 + interest_payment2
        else:
            d = pay_date - prev_pay_date
            days = d.days

            return Decimal(round(principal * self.interest_rate * days / days_in_year, 2))

    def is_pay_in_regular_date(self, pay):
        return pay.date.day == self.start_date.day

    def calc(self, principal, predefined_payments = {}, initial_month = None):
        current_principal = principal
        monthly_payment = self.calc_monthly_payment(current_principal, self.total_months)

        result = Result()
        cur_period = self.cached_calendar[0 if initial_month == None else initial_month]

        for m in range(1 if initial_month == None else initial_month + 1, self.total_months + 1):
            end_period = self.cached_calendar[m]
            principal_paid = False

            non_reg_payments = predefined_payments.get((end_period.year, end_period.month), [])
            for non_reg_pay in non_reg_payments:
                if self.is_pay_in_regular_date(non_reg_pay):

                    if current_principal <= non_reg_pay.payment: # хотим заплатить больше, чем должны. скорректируем
                        non_reg_pay.payment = current_principal
                    current_principal -= non_reg_pay.payment

                    non_reg_pay.current_principal = current_principal
                    non_reg_pay.principal_payment = non_reg_pay.payment
                    non_reg_pay.interest_payment = 0
                    result.payments[non_reg_pay.date].append(non_reg_pay)

                    if current_principal == 0:
                        break

                    # так как долг поменялся, пересчитаем аннуитентный платёж
                    left_months = self.total_months - m + 1 # сколько месяцев осталось платить
                    monthly_payment = self.calc_monthly_payment(current_principal, left_months)
                else:
                    # считаем набежавшие проценты от последнего платежа до даты досрочки
                    p = self.calc_interest_payment(non_reg_pay.date, cur_period, current_principal)
                    result.year_interest_payments[non_reg_pay.date.year] += p

                    if non_reg_pay.payment < p:
                        non_reg_pay.not_used = True
                        result.payments[non_reg_pay.date].append(non_reg_pay)
                        pass # не хватает для выплаты процентов
                    else:
                        # обновим остаток долга с учётом процентов

                        if current_principal <= non_reg_pay.payment - p: # хотим заплатить больше, чем должны. скорректируем
                            non_reg_pay.payment = current_principal + p
                        current_principal -= non_reg_pay.payment - p

                        # обновим инфу о платеже и добавим её в список платежей
                        non_reg_pay.interest_payment = p
                        non_reg_pay.current_principal = current_principal
                        non_reg_pay.principal_payment = non_reg_pay.payment - p

                        principal_paid = True # в этом периоде больше за основной долг не платим
                        result.payments[non_reg_pay.date].append(non_reg_pay)

                        if current_principal == 0:
                            break

                        # так как долг поменялся, пересчитаем аннуитентный платёж
                        left_months = self.total_months - m # сколько месяцев осталось платить
                        monthly_payment = self.calc_monthly_payment(current_principal, left_months)

                        cur_period = non_reg_pay.date

            if current_principal == 0:
                break

            # регулярный платёж
            interest_payment = self.calc_interest_payment(end_period, cur_period, current_principal)
            result.year_interest_payments[end_period.year] += interest_payment

            if m == self.total_months and monthly_payment > current_principal: # last payment
                principal_payment = current_principal
                monthly_payment = interest_payment + principal_payment
            elif not principal_paid:
                principal_payment = monthly_payment - interest_payment
            else:
                principal_payment = 0

            current_principal -= principal_payment

            payment = RegularPayment(end_period, interest_payment + principal_payment, interest_payment, principal_payment, current_principal)
            principal_paid = True
            result.payments[payment.date].append(payment)

            if current_principal == 0:
                break

            cur_period = end_period

        return result

def make_non_reg_payment(year, month, day, sum):
    return NonRegularPayment(datetime.date(year, month, day), Decimal(round(sum, 2)), 0, 0, 0)

def add_non_reg_payment(payments, reg_pay_day, year, month, day, sum):
    if day < reg_pay_day:
        t = (year, month)
    else:
        d = datetime.date(year, month, 1)
        d += relativedelta(months=1)
        t = (d.year, d.month)
    payments[t].append(make_non_reg_payment(year, month, day, sum))

def calc_interest_payments(year, periods):
    s = 0
    for p in periods:
        if p.end.year == year:
            for pay in p.payments:
                s += pay.interest_payment

    return s

def diff_dates(dt1, dt2):
    diff = relativedelta(dt1, dt2)
    return diff.years * 12 + diff.months

# каждый месяц добавляем в досрочку разницу между текущим ежемесячным платежём и первоначальным
def calc():

    start_date = datetime.date(2012, 4, 18)
    m = Mortgage(12.4, 12*20, start_date)
    initial_principal = 4900000
    total_months = 12*20
    initial_monthly_payment = m.calc_monthly_payment(initial_principal, total_months)
    next_pay_date = datetime.date(2016, 6, 18)
    non_reg_payments = defaultdict(list)
    add_non_reg_payment(non_reg_payments, start_date.day, 2016, 1, 15, 800000)
    add_non_reg_payment(non_reg_payments, start_date.day, 2016, 2, 18, 10000)
    add_non_reg_payment(non_reg_payments, start_date.day, 2016, 3, 18, 80000)
    add_non_reg_payment(non_reg_payments, start_date.day, 2016, 4, 18, 16000)

    tax_deduction = None#datetime.date(next_pay_date.year, 5, 18) # учёт налогового вычета
    result = None
    while True:
        result = m.calc(initial_principal, non_reg_payments)
        date, delta, principal = sim(m, initial_monthly_payment, result.payments, next_pay_date)
        if date is None:
            break

        non_reg_payments[(date.year, date.month)].append(NonRegularPayment(next_pay_date, delta, 0, 0, 0))

        if tax_deduction and next_pay_date == tax_deduction:
            v = round(result.year_interest_payments[next_pay_date.year - 1] * Decimal("0.13"), 2)
            v = math.ceil(v / 1000) * 1000
            non_reg_payments[(date.year, date.month)].append(NonRegularPayment(next_pay_date, v, 0, 0, 0))

        next_pay_date = date

    return result

def sim(mortgage, initial_monthly_payment, payments, date):

    pays = payments.get(date, [])
    for pay in pays:
        if isinstance(pay, RegularPayment):
            delta = initial_monthly_payment - pay.payment
            delta = math.ceil(delta / 1000) * 1000 # округлим до тысячи в большую сторону
            if delta > pay.current_principal: # осталось заплатить меньше, чем получившаяся разница
                delta = pay.current_principal
            next_pay_date = pay.date + relativedelta(months=1)
            return next_pay_date, delta, pay.current_principal
    return None, None, None

def print_payments(payments):
    s1 = 0 # сколько уплатим банку по процентам
    for k, v in sorted(payments.items()):
        for p in v:
            s1 += p.interest_payment
            print(p)
    print("Total interest payment: %s - %% = %s" % (s1, round(s1*Decimal("0.13"), 2)))

if __name__ == '__main__':

    # 2024-09-18 - best result
    # 4587670.65 - total interests payment
    r = calc()
    print_payments(r.payments)
    print("Interest payments by years:")
    for k, v in sorted(r.year_interest_payments.items()):
        print("%s - %s - %% = %s" % (k, v, round(v * Decimal("0.13"), 2)))<commit_msg>:lipstick:<commit_after># -*- coding: utf-8 -*-

import datetime
from dateutil.relativedelta import relativedelta
import calendar
from itertools import islice
import math
from decimal import Decimal
from collections import defaultdict

class Payment(object):
    def __init__(self, date, payment, interest_payment, principal_payment, current_principal):
        self.date = date
        self.payment = payment
        self.interest_payment = interest_payment
        self.principal_payment = principal_payment
        self.current_principal = current_principal
        self.not_used = False

class RegularPayment(Payment):
    def __init__(self, date, payment, interest_payment, principal_payment, current_principal):
        return super().__init__(date, payment, interest_payment, principal_payment, current_principal)

    def __str__(self):
        if not self.not_used:
            return '%s - %.2f - %.2f - %.2f - %.2f - regular' % (str(self.date), self.payment, self.interest_payment, self.principal_payment, self.current_principal)
        else:
            return '%s - %.2f - not used' % (str(self.date), self.payment)

class NonRegularPayment(Payment):
    def __init__(self, date, payment, interest_payment, principal_payment, current_principal):
        super().__init__(date, payment, interest_payment, principal_payment, current_principal)

    def __str__(self):
        if not self.not_used:
            return '%s - %.2f - %.2f - %.2f - %.2f - non regular' % (str(self.date), self.payment, self.interest_payment, self.principal_payment, self.current_principal)
        else:
            return '%s - %.2f - not used' % (str(self.date), self.payment)

class Period(object):
    def __init__(self, start, end):
        self.start = start
        self.end = end
        self.payments = []
        self.principal_paid = False

    def add_payment(self, payment):
        self.payments.append(payment)

class Result(object):
    def __init__(self):
        self.payments = defaultdict(list)
        self.year_interest_payments = defaultdict(Decimal)

class Mortgage(object):
    def __init__(self, interest, months, start_date):
        self.interest = interest
        self.interest_rate = Decimal(interest / 100)
        self.monthly_interest = Decimal(self.interest_rate / 12)

        # срок кредита
        self.total_months = months

        # дата получения кредита
        self.start_date = start_date

        self.cached_calendar = [self.start_date + relativedelta(months=m) for m in range(0, self.total_months + 1)]

        self.leap_years_cache = [366 if calendar.isleap(self.start_date.year + y) else 365 for y in range(0, int(self.total_months / 12))]

    def calc_monthly_payment(self, principal, months):
        return Decimal(round(principal * (self.monthly_interest / (1 - (1 + self.monthly_interest) ** (- months))), 2))

    def calc_interest_payment(self, pay_date, prev_pay_date, principal):
        # первый месяц в году, т.е. январь
        # в этом случае надо немного по-другому считать
        days_in_year = self.leap_years_cache[self.start_date.year - pay_date.year]
        if pay_date.month == 1 and prev_pay_date.month == 12:
            december31 = datetime.date(prev_pay_date.year, 12, 31)

            days1 = december31 - prev_pay_date
            days1 = days1.days
            p1 = Decimal(days1 / self.leap_years_cache[self.start_date.year - prev_pay_date.year])
            interest_payment1 = Decimal(round(principal * self.interest_rate * p1, 2))

            days2 = pay_date - december31
            days2 = days2.days
            p2 = Decimal(days2 / days_in_year)
            interest_payment2 = Decimal(round(principal * self.interest_rate * p2, 2))

            return interest_payment1 + interest_payment2
        else:
            d = pay_date - prev_pay_date
            days = d.days

            return Decimal(round(principal * self.interest_rate * days / days_in_year, 2))

    def is_pay_in_regular_date(self, pay):
        return pay.date.day == self.start_date.day

    def calc(self, principal, predefined_payments = {}, initial_month = None):
        current_principal = principal
        monthly_payment = self.calc_monthly_payment(current_principal, self.total_months)

        result = Result()
        cur_period = self.cached_calendar[0 if initial_month == None else initial_month]

        for m in range(1 if initial_month == None else initial_month + 1, self.total_months + 1):
            end_period = self.cached_calendar[m]
            principal_paid = False

            non_reg_payments = predefined_payments.get((end_period.year, end_period.month), [])
            for non_reg_pay in non_reg_payments:
                if self.is_pay_in_regular_date(non_reg_pay):

                    if current_principal <= non_reg_pay.payment: # хотим заплатить больше, чем должны. скорректируем
                        non_reg_pay.payment = current_principal
                    current_principal -= non_reg_pay.payment

                    non_reg_pay.current_principal = current_principal
                    non_reg_pay.principal_payment = non_reg_pay.payment
                    non_reg_pay.interest_payment = 0
                    result.payments[non_reg_pay.date].append(non_reg_pay)

                    if current_principal == 0:
                        break

                    # так как долг поменялся, пересчитаем аннуитентный платёж
                    left_months = self.total_months - m + 1 # сколько месяцев осталось платить
                    monthly_payment = self.calc_monthly_payment(current_principal, left_months)
                else:
                    # считаем набежавшие проценты от последнего платежа до даты досрочки
                    p = self.calc_interest_payment(non_reg_pay.date, cur_period, current_principal)
                    result.year_interest_payments[non_reg_pay.date.year] += p

                    if non_reg_pay.payment < p:
                        non_reg_pay.not_used = True
                        result.payments[non_reg_pay.date].append(non_reg_pay)
                        pass # не хватает для выплаты процентов
                    else:
                        # обновим остаток долга с учётом процентов

                        if current_principal <= non_reg_pay.payment - p: # хотим заплатить больше, чем должны. скорректируем
                            non_reg_pay.payment = current_principal + p
                        current_principal -= non_reg_pay.payment - p

                        # обновим инфу о платеже и добавим её в список платежей
                        non_reg_pay.interest_payment = p
                        non_reg_pay.current_principal = current_principal
                        non_reg_pay.principal_payment = non_reg_pay.payment - p

                        principal_paid = True # в этом периоде больше за основной долг не платим
                        result.payments[non_reg_pay.date].append(non_reg_pay)

                        if current_principal == 0:
                            break

                        # так как долг поменялся, пересчитаем аннуитентный платёж
                        left_months = self.total_months - m # сколько месяцев осталось платить
                        monthly_payment = self.calc_monthly_payment(current_principal, left_months)

                        cur_period = non_reg_pay.date

            if current_principal == 0:
                break

            # регулярный платёж
            interest_payment = self.calc_interest_payment(end_period, cur_period, current_principal)
            result.year_interest_payments[end_period.year] += interest_payment

            if m == self.total_months and monthly_payment > current_principal: # last payment
                principal_payment = current_principal
                monthly_payment = interest_payment + principal_payment
            elif not principal_paid:
                principal_payment = monthly_payment - interest_payment
            else:
                principal_payment = 0

            current_principal -= principal_payment

            payment = RegularPayment(end_period, interest_payment + principal_payment, interest_payment, principal_payment, current_principal)
            principal_paid = True
            result.payments[payment.date].append(payment)

            if current_principal == 0:
                break

            cur_period = end_period

        return result

def make_non_reg_payment(year, month, day, sum):
    return NonRegularPayment(datetime.date(year, month, day), Decimal(round(sum, 2)), 0, 0, 0)

def add_non_reg_payment(payments, reg_pay_day, year, month, day, sum):
    if day < reg_pay_day:
        t = (year, month)
    else:
        d = datetime.date(year, month, 1)
        d += relativedelta(months=1)
        t = (d.year, d.month)
    payments[t].append(make_non_reg_payment(year, month, day, sum))

def calc_interest_payments(year, periods):
    s = 0
    for p in periods:
        if p.end.year == year:
            for pay in p.payments:
                s += pay.interest_payment

    return s

def diff_dates(dt1, dt2):
    diff = relativedelta(dt1, dt2)
    return diff.years * 12 + diff.months

# каждый месяц добавляем в досрочку разницу между текущим ежемесячным платежём и первоначальным
def calc_auto_4dp_v1(non_reg_payments, next_pay_date, tax_deduction):

    start_date = datetime.date(2012, 4, 18) # дата получения кредита
    total_months = 12*20 # сколько платить
    initial_principal = 4900000 # долг

    m = Mortgage(12.4, 12*20, start_date)
    initial_monthly_payment = m.calc_monthly_payment(initial_principal, total_months)

    result = None
    while True:
        result = m.calc(initial_principal, non_reg_payments)
        date, delta = sim_auto_4dp(initial_monthly_payment, result.payments, next_pay_date)
        if date is None:
            break

        non_reg_payments[(date.year, date.month)].append(NonRegularPayment(next_pay_date, delta, 0, 0, 0))

        if tax_deduction and next_pay_date == tax_deduction:
            v = round(result.year_interest_payments[next_pay_date.year - 1] * Decimal("0.13"), 2)
            v = math.ceil(v / 1000) * 1000
            non_reg_payments[(date.year, date.month)].append(NonRegularPayment(next_pay_date, v, 0, 0, 0))

        next_pay_date = date

    return result

def sim_auto_4dp(initial_monthly_payment, payments, start_date):

    pays = payments.get(start_date, [])
    for pay in pays:
        if isinstance(pay, RegularPayment):
            delta = initial_monthly_payment - pay.payment # разница между изначальным платёжом и сколько теперь платим
            delta = math.ceil(delta / 1000) * 1000 # округлим до тысячи в большую сторону
            if delta > pay.current_principal: # осталось заплатить меньше, чем получившаяся разница
                delta = pay.current_principal
            next_pay_date = pay.date + relativedelta(months=1) # следующий платёж + 1 месяц
            return next_pay_date, delta
    return None, None

def calc_auto_4dp_v2(non_reg_payments, next_pay_date, tax_deduction):
    pass

def print_payments(payments):
    s1 = 0 # сколько уплатим банку по процентам
    for k, v in sorted(payments.items()):
        for p in v:
            s1 += p.interest_payment
            print(p)
    print("Total interest payment: %s - %% = %s" % (s1, round(s1*Decimal("0.13"), 2)))

if __name__ == '__main__':

    start_date = datetime.date(2016, 10, 18) # дата, с которой начинаем авто ЧДП
    regular_date_pay = 18 # день платежа по графику

    # уже совершённые ЧДП
    non_reg_payments = defaultdict(list)
    add_non_reg_payment(non_reg_payments, regular_date_pay, 2016, 1, 15, 800000)
    add_non_reg_payment(non_reg_payments, regular_date_pay, 2016, 2, 18, 10000)
    add_non_reg_payment(non_reg_payments, regular_date_pay, 2016, 3, 18, 80000)
    add_non_reg_payment(non_reg_payments, regular_date_pay, 2016, 4, 18, 16000)
    add_non_reg_payment(non_reg_payments, regular_date_pay, 2016, 6, 9,  130000)
    add_non_reg_payment(non_reg_payments, regular_date_pay, 2016, 6, 14, 51182.53)
    add_non_reg_payment(non_reg_payments, regular_date_pay, 2016, 8, 18, 15000)
    add_non_reg_payment(non_reg_payments, regular_date_pay, 2016, 9, 16, 55798.25)

    #add_non_reg_payment(non_reg_payments, regular_date_pay, 2016, 9, 26, 20588.90)
    #add_non_reg_payment(non_reg_payments, regular_date_pay, 2016, 10, 11, 28968.55 + 107.57)

    r = calc_auto_4dp_v1(non_reg_payments, next_pay_date=start_date, tax_deduction=None)
    print_payments(r.payments)
    print("Interest payments by years:")
    for k, v in sorted(r.year_interest_payments.items()):
        print("%s - %s - %% = %s" % (k, v, round(v * Decimal("0.13"), 2)))<|endoftext|>


<commit_before>#!/usr/bin/python

'''
The main mrMeshPy qapplication window built with Qt5

Andre' Gouws 2017
'''

import sys
import vtk

# some local modules
from mrMeshPyQtTCPServer import mrMeshPyQtTCPServer
from mp_MainWindow import Ui_MrMeshMainWindow
from mp_MenuCallbacks import Ui_setupMenuBarCallbacks
from mp_setupVTKWindow import mrMeshVTKWindow

# for the gui
from PyQt5 import QtCore, QtGui, QtNetwork, QtWidgets

#for debugging/testing
debug = False
test = False

class mrMeshPyMainWindow(QtWidgets.QMainWindow):

    def __init__(self, parent = None):

        # set up the initial user interface window on Qt
        QtWidgets.QMainWindow.__init__(self, parent)
        self.ui = Ui_MrMeshMainWindow()
        self.ui.setupUi(self)

        # set up a tcp server instance (Qt TCP server)
        self.server = mrMeshPyQtTCPServer(self)
        self.ui.TCP_server = self.server ## this puts the service instance into the scope of the user interface (ui)

        # TODO - hard code TCP port 9999 for now #next available may be better
        if not self.server.listen(QtNetwork.QHostAddress('127.0.0.1'), 9999):
            print('Error starting TCP instance on requested port - quitting.')
            return
        print('Running TCP instance on port %d' % self.server.serverPort())

        # set up the menu bar in the UI window with its callbacks
        Ui_setupMenuBarCallbacks(self.ui)

        # lets keep track of how many vtk instances are open in a list that is in the scope of the ui
        self.ui.vtkInstances = [] #empty for now
        # and we are going to use a dictionary to keep track of unique instances
        # the unique string generated by matlab will point at an index to the vtkInstances list #TODO is this sensible?
        self.ui.vtkDict = {}

        self.ui.statusbar.showMessage(' ... Waiting for matlab to send a mesh ...')

        # ---Ready to go unless testing/debugging ---------------------------------

        # some debug options
        if debug:
            # ---------------
            # add a vtk window!
            mrMeshVTKWindow(self.ui, 'debug')
            self.ui.statusbar.showMessage(' ... in debug mode; loaded test VTK window ...')

            def testMenuPrint(self):
                print('test menu print message')

            print(dir(self.ui.actionDraw))
            self.ui.actionDraw.triggered.connect(testMenuPrint)

        # some other test options
        if test:
            from testRender import mp_launchVTKWindow
            mp_launchVTKWindow(self.ui)

        # -------------------------------------------------------------------------

if __name__ == '__main__':

    app = QtWidgets.QApplication(sys.argv)
    window = mrMeshPyMainWindow()
    window.show()
    sys.exit(app.exec_())

<commit_msg>Test dev branch<commit_after>#!/usr/bin/python

'''
The main mrMeshPy qapplication window built with Qt5

Andre' Gouws 2017
'''

import sys
import vtk

# some local modules
from mrMeshPyQtTCPServer import mrMeshPyQtTCPServer
from mp_MainWindow import Ui_MrMeshMainWindow
from mp_MenuCallbacks import Ui_setupMenuBarCallbacks
from mp_setupVTKWindow import mrMeshVTKWindow

# for the gui
from PyQt5 import QtCore, QtGui, QtNetwork, QtWidgets

#for debugging/testing
debug = True
test = False

if debug: print 'hello'

class mrMeshPyMainWindow(QtWidgets.QMainWindow):

    def __init__(self, parent = None):

        # set up the initial user interface window on Qt
        QtWidgets.QMainWindow.__init__(self, parent)
        self.ui = Ui_MrMeshMainWindow()
        self.ui.setupUi(self)

        # set up a tcp server instance (Qt TCP server)
        self.server = mrMeshPyQtTCPServer(self)
        self.ui.TCP_server = self.server ## this puts the service instance into the scope of the user interface (ui)

        # TODO - hard code TCP port 9999 for now #next available may be better
        if not self.server.listen(QtNetwork.QHostAddress('127.0.0.1'), 9999):
            print('Error starting TCP instance on requested port - quitting.')
            return
        print('Running TCP instance on port %d' % self.server.serverPort())

        # set up the menu bar in the UI window with its callbacks
        Ui_setupMenuBarCallbacks(self.ui)

        # lets keep track of how many vtk instances are open in a list that is in the scope of the ui
        self.ui.vtkInstances = [] #empty for now
        # and we are going to use a dictionary to keep track of unique instances
        # the unique string generated by matlab will point at an index to the vtkInstances list #TODO is this sensible?
        self.ui.vtkDict = {}

        self.ui.statusbar.showMessage(' ... Waiting for matlab to send a mesh ...')

        # ---Ready to go unless testing/debugging ---------------------------------

        # some debug options
        if debug:
            # ---------------
            # add a vtk window!
            mrMeshVTKWindow(self.ui, 'debug')
            self.ui.statusbar.showMessage(' ... in debug mode; loaded test VTK window ...')

            def testMenuPrint(self):
                print('test menu print message')

            print(dir(self.ui.actionDraw))
            self.ui.actionDraw.triggered.connect(testMenuPrint)

        # some other test options
        if test:
            from testRender import mp_launchVTKWindow
            mp_launchVTKWindow(self.ui)

        # -------------------------------------------------------------------------

if __name__ == '__main__':

    app = QtWidgets.QApplication(sys.argv)
    window = mrMeshPyMainWindow()
    window.show()
    sys.exit(app.exec_())

<|endoftext|>


<commit_before>"""Implements muli-dimensional threshold discovery via binary search."""
from collections import namedtuple, deque
from itertools import chain, combinations
from pathlib import Path

import numpy as np
from numpy import array
import svgwrite
import funcy as fn

Rec = namedtuple("Rec", "bot top")

def binsearch(r: Rec, is_member, eps=0.001) -> (array, array, array):
    """Binary search over the diagonal of the rectangle.

    Returns the lower and upper approximation on the diagonal.
    """
    lo, hi = 0, 1
    diag = r.top - r.bot
    f = lambda t: r.bot + t * diag
    while hi - lo > eps:
        mid = lo + (hi - lo) / 2
        if not is_member(f(mid)):
            lo, hi = mid, hi
        else:
            lo, hi = lo, mid
    return f(lo), f(mid), f(hi)

def forward_cone(p: array, r: Rec) -> Rec:
    """Computes the forward cone from point p."""
    return Rec(p, r.top)

def backward_cone(p: array, r: Rec) -> Rec:
    """Computes the backward cone from point p."""
    return Rec(r.bot, p)

def basis_vec(i, dim):
    """Basis vector i"""
    a = np.zeros(dim)
    a[i] = 1.0
    return a

@fn.memoize
def basis_vecs(dim):
    """Standard orthonormal basis."""
    return [basis_vec(i, dim) for i in range(dim)]

def generate_incomparables(mid, r):
    """Generate all incomparable hyper-boxes."""
    forward, backward = forward_cone(mid, r), backward_cone(mid, r)
    bases = (backward.bot, forward.bot)
    diags = (backward.top - backward.bot, forward.top - forward.bot)
    dim = len(bases[0])
    basis = basis_vecs(dim)
    for i in range(1, dim):
        for es in combinations(basis, i):
            vs = tuple(sum((diag @e)*e for e in es) for diag in diags)
            yield Rec(*[base + v for base, v in zip(bases, vs)])

def subdivide(low, mid, high, r: Rec) -> [Rec]:
    """Computes the set of incomparable cones of point p."""
    forward = forward_cone(high, r)
    backward = backward_cone(low, r)
    incomparables = list(generate_incomparables(mid, r))
    return backward, forward, incomparables

def multidim_search(rec: Rec, is_member) -> [({Rec}, {Rec})]:
    """Generator for iteratively approximating the oracle's threshold."""
    queue = deque([rec])
    good_approx, bad_approx = [], []
    while True:
        rec = queue.pop()
        low, mid, high = binsearch(rec, is_member)
        backward, forward, incomparables = subdivide(low, mid, high, rec)

        bad_approx.append(backward)
        good_approx.append(forward)
        queue.extendleft(incomparables)

        yield bad_approx, good_approx

def draw_rec(dwg, r: Rec, is_member: bool):
    """TODO: handle different orientations."""
    bot = tuple(map(float, r.bot))
    dim = tuple(map(float, r.top - r.bot))
    color = "red" if is_member else "green"
    return dwg.rect(bot, dim, fill=color)

def draw_domain(r: Rec, good: {Rec}, bad: {Rec}, scale):
    width, height = tuple(map(float, scale(r.top - r.bot)))
    dwg = svgwrite.Drawing(width=width, height=height)
    scale_rec = lambda x: Rec(scale(x.bot), scale(x.top))
    good_recs = (draw_rec(dwg, scale_rec(r), True) for r in good)
    bad_recs = (draw_rec(dwg, scale_rec(r), False) for r in bad)

    for svg_rec in chain(good_recs, bad_recs):
        dwg.add(svg_rec)
    return dwg

def multidim_search_and_draw(rec, is_member, n, save_path):
    bad, good = fn.nth(n, multidim_search(rec, is_member))
    # TODO automate detecting a good scale
    # currently assumes -1, 1 to 0, 100 transformation
    scale = lambda x: 100 * (x + 1)
    dwg = draw_domain(rec, good=good, bad=bad, scale=scale)
    with Path(save_path).open('w') as f:
        dwg.write(f)

def main():
    R = Rec(-np.ones(2), np.ones(2))
    n = np.array([1, 1]) / np.sqrt(2)
    f = lambda x: x @n > 0
    multidim_search_and_draw(R, f, 100, "foo.svg")

    f = lambda x: x[0] > 0
    multidim_search_and_draw(R, f, 100, "foo2.svg")

    f = lambda x: x[1] > 0
    multidim_search_and_draw(R, f, 100, "foo3.svg")

    f = lambda x: (x @n > 0 and x[0] > 0) or (x @n > -0.2 and x[0] < 0)
    multidim_search_and_draw(R, f, 100, "foo4.svg")

if __name__ == "__main__":
    main()
<commit_msg>update examples<commit_after>"""Implements muli-dimensional threshold discovery via binary search."""
from collections import namedtuple, deque
from itertools import chain, combinations
from pathlib import Path

import numpy as np
from numpy import array
import svgwrite
import funcy as fn

Rec = namedtuple("Rec", "bot top")

def binsearch(r: Rec, is_member, eps=0.001) -> (array, array, array):
    """Binary search over the diagonal of the rectangle.

    Returns the lower and upper approximation on the diagonal.
    """
    lo, hi = 0, 1
    diag = r.top - r.bot
    f = lambda t: r.bot + t * diag
    while hi - lo > eps:
        mid = lo + (hi - lo) / 2
        if not is_member(f(mid)):
            lo, hi = mid, hi
        else:
            lo, hi = lo, mid
    return f(lo), f(mid), f(hi)

def forward_cone(p: array, r: Rec) -> Rec:
    """Computes the forward cone from point p."""
    return Rec(p, r.top)

def backward_cone(p: array, r: Rec) -> Rec:
    """Computes the backward cone from point p."""
    return Rec(r.bot, p)

def basis_vec(i, dim):
    """Basis vector i"""
    a = np.zeros(dim)
    a[i] = 1.0
    return a

@fn.memoize
def basis_vecs(dim):
    """Standard orthonormal basis."""
    return [basis_vec(i, dim) for i in range(dim)]

def generate_incomparables(mid, r):
    """Generate all incomparable hyper-boxes."""
    forward, backward = forward_cone(mid, r), backward_cone(mid, r)
    bases = (backward.bot, forward.bot)
    diags = (backward.top - backward.bot, forward.top - forward.bot)
    dim = len(bases[0])
    basis = basis_vecs(dim)
    for i in range(1, dim):
        for es in combinations(basis, i):
            vs = tuple(sum((diag @e)*e for e in es) for diag in diags)
            yield Rec(*[base + v for base, v in zip(bases, vs)])

def subdivide(low, mid, high, r: Rec) -> [Rec]:
    """Computes the set of incomparable cones of point p."""
    forward = forward_cone(high, r)
    backward = backward_cone(low, r)
    incomparables = list(generate_incomparables(mid, r))
    return backward, forward, incomparables

def multidim_search(rec: Rec, is_member) -> [({Rec}, {Rec})]:
    """Generator for iteratively approximating the oracle's threshold."""
    queue = deque([rec])
    good_approx, bad_approx = [], []
    while True:
        rec = queue.pop()
        low, mid, high = binsearch(rec, is_member)
        backward, forward, incomparables = subdivide(low, mid, high, rec)

        bad_approx.append(backward)
        good_approx.append(forward)
        queue.extendleft(incomparables)

        yield bad_approx, good_approx

def draw_rec(dwg, r: Rec, is_member: bool):
    """TODO: handle different orientations."""
    bot = tuple(map(float, r.bot))
    dim = tuple(map(float, r.top - r.bot))
    color = "red" if is_member else "green"
    return dwg.rect(bot, dim, fill=color)

def draw_domain(r: Rec, good: {Rec}, bad: {Rec}, scale):
    width, height = tuple(map(float, scale(r.top - r.bot)))
    dwg = svgwrite.Drawing(width=width, height=height)
    scale_rec = lambda x: Rec(scale(x.bot), scale(x.top))
    good_recs = (draw_rec(dwg, scale_rec(r), True) for r in good)
    bad_recs = (draw_rec(dwg, scale_rec(r), False) for r in bad)

    for svg_rec in chain(good_recs, bad_recs):
        dwg.add(svg_rec)
    return dwg

def multidim_search_and_draw(rec, is_member, n, save_path):
    bad, good = fn.nth(n, multidim_search(rec, is_member))
    # TODO automate detecting a good scale
    # currently assumes -1, 1 to 0, 100 transformation
    scale = lambda x: 100 * (x + 1)
    dwg = draw_domain(rec, good=good, bad=bad, scale=scale)
    with Path(save_path).open('w') as f:
        dwg.write(f)

def main():
    R = Rec(-np.ones(2), np.ones(2))
    n = np.array([1, 1]) / np.sqrt(2)
    f = lambda x: x @n > 0
    multidim_search_and_draw(R, f, 1000, "foo.svg")

    f = lambda x: x[0] > 0
    multidim_search_and_draw(R, f, 1000, "foo2.svg")

    f = lambda x: x[1] > 0
    multidim_search_and_draw(R, f, 1000, "foo3.svg")

    f = lambda x: (x @n > 0 and x[0] > 0) or (x @n > 0.2 and x[0] < 0)
    multidim_search_and_draw(R, f, 1000, "foo4.svg")

    f = lambda x: np.abs(x[1]) > x[0]**2 if x[0] < 0 else x@n > 0
    multidim_search_and_draw(R, f, 1000, "foo5.svg")

if __name__ == "__main__":
    main()
<|endoftext|>