<commit_before>import json

from flask import g, render_template, request, redirect, url_for

from charat2.model.connections import use_db

@use_db
def home():
    posts = json.loads(feed())
    return render_template(
        "blog/home.html",
        logged_in=g.user is not None,
        posts=posts
    )

@use_db
def post(id):
    posts = json.loads(feed())
    #post = posts[int(id)]
    #return render_template(
    #    "blog/home.html",
    #    logged_in=g.user is not None,
    #    post=post
    #)
    return id

@use_db
def feed():
    data = {
    1 : {
        "id" : 1,
        "title" : "Test Blog",
        "author" : "Sho Tran",
        "content" : "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."
    },
    2 : {
        "id" : 2,
        "title" : "Test Bloggy Stuff",
        "author" : "Sho Tran",
        "content" : "lorem ipsum testing everything yyyyeee"
    },
    3 : {
        "id" : 3,
        "title" : "Test Bloggy Stuff",
        "author" : "Sho Tran",
        "content" : "lorem ipsum testing everything yyyyeee"
    },
    4 : {
        "id" : 4,
        "title" : "Test Bloggy Stuff",
        "author" : "Sho Tran",
        "content" : "lorem ipsum testing everything yyyyeee"
    }}
    return json.dumps(data, sort_keys=True)
<commit_msg>blog post enhance<commit_after>import json

from flask import g, render_template, request, redirect, url_for

from charat2.model.connections import use_db

@use_db
def home():
    posts = json.loads(feed())
    return render_template(
        "blog/home.html",
        logged_in=g.user is not None,
        posts=posts
    )

@use_db
def post(id):
    posts = json.loads(feed())
    post = posts[id]
    return render_template(
        "blog/home.html",
        logged_in=g.user is not None,
        post=post
    )

@use_db
def feed():
    data = {
    1 : {
        "id" : 1,
        "title" : "Test Blog",
        "author" : "Sho Tran",
        "content" : "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."
    },
    2 : {
        "id" : 2,
        "title" : "Test Bloggy Stuff",
        "author" : "Sho Tran",
        "content" : "lorem ipsum testing everything yyyyeee"
    },
    3 : {
        "id" : 3,
        "title" : "Test Bloggy Stuff",
        "author" : "Sho Tran",
        "content" : "lorem ipsum testing everything yyyyeee"
    },
    4 : {
        "id" : 4,
        "title" : "Test Bloggy Stuff",
        "author" : "Sho Tran",
        "content" : "lorem ipsum testing everything yyyyeee"
    }}
    return json.dumps(data, sort_keys=True)
<|endoftext|>


<commit_before>from cherrypy.test import test
test.prefer_parent_path()

import gzip, StringIO
import cherrypy
europoundUnicode = u'\x80\xa3'
europoundUtf8 = u'\x80\xa3'.encode('utf-8')
sing = u"\u6bdb\u6cfd\u4e1c: Sing, Little Birdie?"
sing8 = sing.encode('utf-8')
sing16 = sing.encode('utf-16')

def setup_server():
    class Root:
        def index(self, param):
            assert param == europoundUnicode
            yield europoundUnicode
        index.exposed = True

        def mao_zedong(self):
            return sing
        mao_zedong.exposed = True

        def utf8(self):
            return sing8
        utf8.exposed = True
        utf8._cp_config = {'tools.encode.encoding': 'utf-8'}

    class GZIP:
        def index(self):
            yield "Hello, world"
        index.exposed = True

        def noshow(self):
            # Test for ticket #147, where yield showed no exceptions (content-
            # encoding was still gzip even though traceback wasn't zipped).
            raise IndexError()
            yield "Here be dragons"
        noshow.exposed = True

        def noshow_stream(self):
            # Test for ticket #147, where yield showed no exceptions (content-
            # encoding was still gzip even though traceback wasn't zipped).
            raise IndexError()
            yield "Here be dragons"
        noshow_stream.exposed = True
        noshow_stream._cp_config = {'response.stream': True}

    cherrypy.config.update({
            'environment': 'test_suite',
            'tools.encode.on': True,
            'tools.decode.on': True,
    })

    root = Root()
    root.gzip = GZIP()
    cherrypy.tree.mount(root, config={'/gzip': {'tools.gzip.on': True}})

from cherrypy.test import helper

class EncodingTests(helper.CPWebCase):

    def testDecoding(self):
        europoundUtf8 = europoundUnicode.encode('utf-8')
        self.getPage('/?param=%s' % europoundUtf8)
        self.assertBody(europoundUtf8)

    def testEncoding(self):
        # Default encoding should be utf-8
        self.getPage('/mao_zedong')
        self.assertBody(sing8)

        # Ask for utf-16.
        self.getPage('/mao_zedong', [('Accept-Charset', 'utf-16')])
        self.assertHeader('Content-Type', 'text/html;charset=utf-16')
        self.assertBody(sing16)

        # Ask for multiple encodings. ISO-8859-1 should fail, and utf-16
        # should be produced.
        self.getPage('/mao_zedong', [('Accept-Charset',
                                      'iso-8859-1;q=1, utf-16;q=0.5')])
        self.assertBody(sing16)

        # The "*" value should default to our default_encoding, utf-8
        self.getPage('/mao_zedong', [('Accept-Charset', '*;q=1, utf-7;q=.2')])
        self.assertBody(sing8)

        # Only allow iso-8859-1, which should fail and raise 406.
        self.getPage('/mao_zedong', [('Accept-Charset', 'iso-8859-1, *;q=0')])
        self.assertStatus("406 Not Acceptable")
        self.assertInBody("Your client sent this Accept-Charset header: "
                          "iso-8859-1, *;q=0. We tried these charsets: "
                          "iso-8859-1.")

        # Ask for x-mac-ce, which should be unknown. See ticket #569.
        self.getPage('/mao_zedong', [('Accept-Charset',
                                      'us-ascii, ISO-8859-1, x-mac-ce')])
        self.assertStatus("406 Not Acceptable")
        self.assertInBody("Your client sent this Accept-Charset header: "
                          "us-ascii, ISO-8859-1, x-mac-ce. We tried these "
                          "charsets: x-mac-ce, us-ascii, ISO-8859-1.")

        # Test the 'encoding' arg to encode.
        self.getPage('/utf8')
        self.assertBody(sing8)
        self.getPage('/utf8', [('Accept-Charset', 'us-ascii, ISO-8859-1')])
        self.assertStatus("406 Not Acceptable")

    def testGzip(self):
        zbuf = StringIO.StringIO()
        zfile = gzip.GzipFile(mode='wb', fileobj=zbuf, compresslevel=9)
        zfile.write("Hello, world")
        zfile.close()

        self.getPage('/gzip/', headers=[("Accept-Encoding", "gzip")])
        self.assertInBody(zbuf.getvalue()[:3])
        self.assertHeader("Vary", "Accept-Encoding")
        self.assertHeader("Content-Encoding", "gzip")

        # Test when gzip is denied.
        self.getPage('/gzip/', headers=[("Accept-Encoding", "identity")])
        self.assertNoHeader("Vary")
        self.assertBody("Hello, world")

        self.getPage('/gzip/', headers=[("Accept-Encoding", "gzip;q=0")])
        self.assertNoHeader("Vary")
        self.assertBody("Hello, world")

        self.getPage('/gzip/', headers=[("Accept-Encoding", "*;q=0")])
        self.assertStatus(406)
        self.assertNoHeader("Vary")
        self.assertErrorPage(406, "identity, gzip")

        # Test for ticket #147
        self.getPage('/gzip/noshow', headers=[("Accept-Encoding", "gzip")])
        self.assertNoHeader('Content-Encoding')
        self.assertStatus(500)
        self.assertErrorPage(500, pattern="IndexError\n")

        # In this case, there's nothing we can do to deliver a
        # readable page, since 1) the gzip header is already set,
        # and 2) we may have already written some of the body.
        # The fix is to never stream yields when using gzip.
        if cherrypy.server.protocol_version == "HTTP/1.0":
            self.getPage('/gzip/noshow_stream',
                         headers=[("Accept-Encoding", "gzip")])
            self.assertHeader('Content-Encoding', 'gzip')
            self.assertInBody('\x1f\x8b\x08\x00')
        else:
            # The wsgiserver will simply stop sending data, and the HTTP client
            # will error due to an incomplete chunk-encoded stream.
            self.assertRaises(ValueError, self.getPage, '/gzip/noshow_stream',
                              headers=[("Accept-Encoding", "gzip")])

if __name__ == "__main__":
    setup_server()
    helper.testmain()
<commit_msg>utf8 decode test.<commit_after>from cherrypy.test import test
test.prefer_parent_path()

import gzip, StringIO
import cherrypy
europoundUnicode = u'\x80\xa3'
europoundUtf8 = u'\x80\xa3'.encode('utf-8')
sing = u"\u6bdb\u6cfd\u4e1c: Sing, Little Birdie?"
sing8 = sing.encode('utf-8')
sing16 = sing.encode('utf-16')

def setup_server():
    class Root:
        def index(self, param):
            assert param == europoundUnicode
            yield europoundUnicode
        index.exposed = True

        def mao_zedong(self):
            return sing
        mao_zedong.exposed = True

        def utf8(self):
            return sing8
        utf8.exposed = True
        utf8._cp_config = {'tools.encode.encoding': 'utf-8'}

        def reqparams(self, *args, **kwargs):
            return repr(cherrypy.request.params)
        reqparams.exposed = True

    class GZIP:
        def index(self):
            yield "Hello, world"
        index.exposed = True

        def noshow(self):
            # Test for ticket #147, where yield showed no exceptions (content-
            # encoding was still gzip even though traceback wasn't zipped).
            raise IndexError()
            yield "Here be dragons"
        noshow.exposed = True

        def noshow_stream(self):
            # Test for ticket #147, where yield showed no exceptions (content-
            # encoding was still gzip even though traceback wasn't zipped).
            raise IndexError()
            yield "Here be dragons"
        noshow_stream.exposed = True
        noshow_stream._cp_config = {'response.stream': True}

    cherrypy.config.update({
            'environment': 'test_suite',
            'tools.encode.on': True,
            'tools.decode.on': True,
    })

    root = Root()
    root.gzip = GZIP()
    cherrypy.tree.mount(root, config={'/gzip': {'tools.gzip.on': True}})

from cherrypy.test import helper

class EncodingTests(helper.CPWebCase):

    def testDecoding(self):
        europoundUtf8 = europoundUnicode.encode('utf-8')
        self.getPage('/?param=%s' % europoundUtf8)
        self.assertBody(europoundUtf8)

        # Make sure that encoded utf8 gets parsed correctly
        self.getPage("/reqparams?q=%C2%A3")
        self.assertBody(r"{'q': '\xc2\xa3'}")

    def testEncoding(self):
        # Default encoding should be utf-8
        self.getPage('/mao_zedong')
        self.assertBody(sing8)

        # Ask for utf-16.
        self.getPage('/mao_zedong', [('Accept-Charset', 'utf-16')])
        self.assertHeader('Content-Type', 'text/html;charset=utf-16')
        self.assertBody(sing16)

        # Ask for multiple encodings. ISO-8859-1 should fail, and utf-16
        # should be produced.
        self.getPage('/mao_zedong', [('Accept-Charset',
                                      'iso-8859-1;q=1, utf-16;q=0.5')])
        self.assertBody(sing16)

        # The "*" value should default to our default_encoding, utf-8
        self.getPage('/mao_zedong', [('Accept-Charset', '*;q=1, utf-7;q=.2')])
        self.assertBody(sing8)

        # Only allow iso-8859-1, which should fail and raise 406.
        self.getPage('/mao_zedong', [('Accept-Charset', 'iso-8859-1, *;q=0')])
        self.assertStatus("406 Not Acceptable")
        self.assertInBody("Your client sent this Accept-Charset header: "
                          "iso-8859-1, *;q=0. We tried these charsets: "
                          "iso-8859-1.")

        # Ask for x-mac-ce, which should be unknown. See ticket #569.
        self.getPage('/mao_zedong', [('Accept-Charset',
                                      'us-ascii, ISO-8859-1, x-mac-ce')])
        self.assertStatus("406 Not Acceptable")
        self.assertInBody("Your client sent this Accept-Charset header: "
                          "us-ascii, ISO-8859-1, x-mac-ce. We tried these "
                          "charsets: x-mac-ce, us-ascii, ISO-8859-1.")

        # Test the 'encoding' arg to encode.
        self.getPage('/utf8')
        self.assertBody(sing8)
        self.getPage('/utf8', [('Accept-Charset', 'us-ascii, ISO-8859-1')])
        self.assertStatus("406 Not Acceptable")

    def testGzip(self):
        zbuf = StringIO.StringIO()
        zfile = gzip.GzipFile(mode='wb', fileobj=zbuf, compresslevel=9)
        zfile.write("Hello, world")
        zfile.close()

        self.getPage('/gzip/', headers=[("Accept-Encoding", "gzip")])
        self.assertInBody(zbuf.getvalue()[:3])
        self.assertHeader("Vary", "Accept-Encoding")
        self.assertHeader("Content-Encoding", "gzip")

        # Test when gzip is denied.
        self.getPage('/gzip/', headers=[("Accept-Encoding", "identity")])
        self.assertNoHeader("Vary")
        self.assertBody("Hello, world")

        self.getPage('/gzip/', headers=[("Accept-Encoding", "gzip;q=0")])
        self.assertNoHeader("Vary")
        self.assertBody("Hello, world")

        self.getPage('/gzip/', headers=[("Accept-Encoding", "*;q=0")])
        self.assertStatus(406)
        self.assertNoHeader("Vary")
        self.assertErrorPage(406, "identity, gzip")

        # Test for ticket #147
        self.getPage('/gzip/noshow', headers=[("Accept-Encoding", "gzip")])
        self.assertNoHeader('Content-Encoding')
        self.assertStatus(500)
        self.assertErrorPage(500, pattern="IndexError\n")

        # In this case, there's nothing we can do to deliver a
        # readable page, since 1) the gzip header is already set,
        # and 2) we may have already written some of the body.
        # The fix is to never stream yields when using gzip.
        if cherrypy.server.protocol_version == "HTTP/1.0":
            self.getPage('/gzip/noshow_stream',
                         headers=[("Accept-Encoding", "gzip")])
            self.assertHeader('Content-Encoding', 'gzip')
            self.assertInBody('\x1f\x8b\x08\x00')
        else:
            # The wsgiserver will simply stop sending data, and the HTTP client
            # will error due to an incomplete chunk-encoded stream.
            self.assertRaises(ValueError, self.getPage, '/gzip/noshow_stream',
                              headers=[("Accept-Encoding", "gzip")])

if __name__ == "__main__":
    setup_server()
    helper.testmain()
<|endoftext|>


<commit_before>"""
HandBrake queue parser

Uses HandBrake to compress movies ripped by makeMKV

This script can be run with a simple cron, every hour should be fine

An optional script used to rename and compress movies to an acceptable standard
which still delivers quallity audio and video but reduces the file size
dramatly.
Using a nice value of 15 by default, it runs HandBrake as a background task
that allows other critical tasks to complete first.

Using this script with the auto ripper allows for multiple movies to be ripped
and encoded with very little effot.

Released under the MIT license
Copyright (c) 2012, Jason Millward

@category   misc
@version    $Id: 1.5, 2013-10-20 20:40:30 CST $;
@author     Jason Millward <jason@jcode.me>
@license    http://opensource.org/licenses/MIT
"""

import os
import ConfigParser
from handbrake import HandBrake
from timer import Timer

DIR = os.path.dirname(os.path.realpath(__file__))
CONFIG_FILE = "%s/../settings.cfg" % DIR

def read_value(key):
    """
    read_value temp docstring
    """
    config = ConfigParser.RawConfigParser()
    config.read(CONFIG_FILE)
    to_return = config.get('HANDBRAKE', key)
    config = None
    return to_return

def compress():
    """
    compress temp docstring
    """
    hb_nice = read_value('nice')
    hb_cli = read_value('com')
    hb_out = read_value('temp_output')

    hb_api = HandBrake()

    if not hb_api.findProcess():
        if hb_api.loadMovie():
            print "Encoding and compressing %s" % hb_api.getMovieTitle()
            stopwatch = Timer()

            if hb_api.convert(args=hb_cli, nice=hb_nice, output=hb_out):
                print "Movie was compressed and encoded successfully"

                stopwatch.stop()
                print ("It took %s minutes to compress %s"
                    %
                    (stopwatch.getTime(), hb_api.getMovieTitle()))
            else:
                stopwatch.stop()
                print "HandBrake did not complete successfully"
        else:
            print "Queue does not exist or is empty"
    else:
        print "Process already running skipper"

if __name__ == '__main__':
    compress()
<commit_msg>Fixed typecasting<commit_after>"""
HandBrake queue parser

Uses HandBrake to compress movies ripped by makeMKV

This script can be run with a simple cron, every hour should be fine

An optional script used to rename and compress movies to an acceptable standard
which still delivers quallity audio and video but reduces the file size
dramatly.
Using a nice value of 15 by default, it runs HandBrake as a background task
that allows other critical tasks to complete first.

Using this script with the auto ripper allows for multiple movies to be ripped
and encoded with very little effot.

Released under the MIT license
Copyright (c) 2012, Jason Millward

@category   misc
@version    $Id: 1.5, 2013-10-20 20:40:30 CST $;
@author     Jason Millward <jason@jcode.me>
@license    http://opensource.org/licenses/MIT
"""

import os
import ConfigParser
from handbrake import HandBrake
from timer import Timer

DIR = os.path.dirname(os.path.realpath(__file__))
CONFIG_FILE = "%s/../settings.cfg" % DIR

def read_value(key):
    """
    read_value temp docstring
    """
    config = ConfigParser.RawConfigParser()
    config.read(CONFIG_FILE)
    to_return = config.get('HANDBRAKE', key)
    config = None
    return to_return

def compress():
    """
    compress temp docstring
    """
    hb_nice = int(read_value('nice'))
    hb_cli = read_value('com')
    hb_out = read_value('temp_output')

    hb_api = HandBrake()

    if not hb_api.findProcess():
        if hb_api.loadMovie():
            print "Encoding and compressing %s" % hb_api.getMovieTitle()
            stopwatch = Timer()

            if hb_api.convert(args=hb_cli, nice=hb_nice, output=hb_out):
                print "Movie was compressed and encoded successfully"

                stopwatch.stop()
                print ("It took %s minutes to compress %s"
                    %
                    (stopwatch.getTime(), hb_api.getMovieTitle()))
            else:
                stopwatch.stop()
                print "HandBrake did not complete successfully"
        else:
            print "Queue does not exist or is empty"
    else:
        print "Process already running skipper"

if __name__ == '__main__':
    compress()
<|endoftext|>


<commit_before># Module:   sockets
# Date:     26th June 2006
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""Sockets Test Suite

Test all functionality of the sockets module.
"""

import unittest

from circuits import Component
from circuits.lib.sockets import TCPServer, TCPClient, UDPServer, UDPClient

def wait():
    for x in xrange(100000):
        pass

class Client(Component):

    def __init__(self, client):
        super(Client, self).__init__()

        self.client = client
        self.manager += self.client

        self.connected = False
        self.disconnected = False
        self.data = ""

    def connected(self, host, port):
        self.connected = True

    def disconnected(self):
        self.disconnected = True

    def read(self, data):
        self.data = data

class Server(Component):

    def __init__(self, server):
        super(Server, self).__init__()

        self.server = server
        self.manager += self.server

        self.data = ""

    def connected(self, sock, host, port):
        self.server.write(sock, "Ready")

    def read(self, sock, data):
        self.data = data

class SocketsTestCase(unittest.TestCase):

    def testTCPClientServer(self):
        """Test sockets.TCPClient and sockets.TCPServer

        Test that communication between a TCPClient and
        TCPServer work correctly.
        """

        server = Server(TCPServer(9999))
        client = Client(TCPClient("localhost", 9999))

        server.start()
        client.start()

        try:
            client.client.connect()
            wait()

            self.assertTrue(client.connected)
            self.assertTrue(client.data == "Ready")

            client.client.write("foo")
            wait()

            self.assertTrue(server.data == "foo")

            client.client.close()
            wait()

            self.assertTrue(client.disconnected)
        finally:
            server.stop()
            client.stop()

    def testUDPClientServer(self):
        """Test sockets.UDPClient and sockets.UDPServer

        Test that communication between a UDPClient and
        UDPServer work correctly.
        """

        server = Server(UDPServer(9999))
        client = Client(UDPClient(10000))

        server.start()
        client.start()

        try:
            client.client.write(("localhost", 9999), "foo")
            wait()

            self.assertTrue(server.data == "foo")
        finally:
            server.stop()
            client.stop()

def suite():
    return unittest.makeSuite(SocketsTestCase, "test")

if __name__ == "__main__":
    unittest.main()
<commit_msg>tests.test_sockets: Fixed TCPServer and TCPCLient test and half fixed UDPServer/UDPClient test (1 failure still :/)<commit_after># Module:   sockets
# Date:     26th June 2006
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""Sockets Test Suite

Test all functionality of the sockets module.
"""

import unittest
from time import sleep

from circuits.tools import inspect
from circuits import Component, Debugger
from circuits.lib.sockets import *

def wait():
    sleep(0.1)

class Client(Component):

    channel = "client"

    def __init__(self):
        super(Client, self).__init__()

        self.connected = False
        self.disconnected = False
        self.data = ""

    def connected(self, host, port):
        self.connected = True

    def disconnected(self):
        self.disconnected = True

    def read(self, data):
        self.data = data

class Server(Component):

    channel = "server"

    def __init__(self):
        super(Server, self).__init__()

        self.connected = False
        self.disconnected = False
        self.data = ""

    def connect(self, sock, host, port):
        self.connected = True
        self.push(Write(sock, "Ready"), "write")

    def disconnect(self, sock):
        self.disconnected = True

    def read(self, sock, data):
        self.data = data

class SocketsTestCase(unittest.TestCase):

    def testTCPClientServer(self):
        """Test sockets.TCPClient and sockets.TCPServer

        Test that communication between a TCPClient and
        TCPServer work correctly.
        """

        server = Server() + TCPServer(9999)
        client = Client() + TCPClient()

        server.start()
        client.start()

        try:
            client.push(Connect("127.0.0.1", 9999), "connect")
            wait()

            self.assertTrue(client.connected)
            self.assertTrue(server.connected)
            self.assertTrue(client.data == "Ready")

            client.push(Write("foo"), "write")
            wait()

            self.assertTrue(server.data == "foo")

            client.push(Close(), "close")
            wait()

            self.assertTrue(client.disconnected)
            self.assertTrue(server.disconnected)
        finally:
            server.stop()
            client.stop()

    def testUDPClientServer(self):
        """Test sockets.UDPClient and sockets.UDPServer

        Test that communication between a UDPClient and
        UDPServer work correctly.
        """

        server = Server() + UDPServer(9999)# + Debugger()
        client = Client() + UDPClient(10000)# + Debugger()

        server.start()
        client.start()

        try:
            client.push(Write(("127.0.0.1", 9999), "foo"), "write")
            wait()
            client.push(Write(("127.0.0.1", 9999), "foo"), "write")
            sleep(1)
            print repr(server.data)
            self.assertTrue(server.data == "foo")
        finally:
            server.stop()
            client.stop()

def suite():
    return unittest.makeSuite(SocketsTestCase, "test")

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>"""Class Namespaces (internal module).

All of the guts of the class namespace implementation.

"""

# See https://blog.ionelmc.ro/2015/02/09/understanding-python-metaclasses/

import collections.abc
import functools
import itertools
import weakref

from . import ops
from .descriptor_inspector import _DescriptorInspector
from .flags import ENABLE_SET_NAME
from .proxy import _Proxy
from .scope_proxy import _ScopeProxy

_PROXY_INFOS = weakref.WeakKeyDictionary()

def _mro_to_chained(mro, dct):
    """Return a chained map of lookups for the given namespace and mro."""
    return collections.ChainMap(*[
        Namespace.get_namespace(cls, dct.path) for cls in
        itertools.takewhile(
            functools.partial(Namespace.no_blocker, dct),
            (cls for cls in mro if isinstance(cls, _Namespaceable))) if
        Namespace.namespace_exists(dct.path, cls)])

def _instance_map(ns_proxy):
    """Return a map, possibly chained, of lookups for the given instance."""
    dct, instance, _ = _PROXY_INFOS[ns_proxy]
    if instance is not None:
        if isinstance(instance, _Namespaceable):
            return _mro_to_chained(instance.__mro__, dct)
        else:
            return Namespace.get_namespace(instance, dct.path)
    else:
        return {}

def _mro_map(ns_proxy):
    """Return a chained map of lookups for the given owner class."""
    dct, _, owner = _PROXY_INFOS[ns_proxy]
    mro = owner.__mro__
    mro = mro[mro.index(dct.parent_object):]
    return _mro_to_chained(mro, dct)

def _retarget(ns_proxy):
    """Convert a class lookup to an instance lookup, if needed."""
    dct, instance, owner = _PROXY_INFOS[ns_proxy]
    if instance is None and isinstance(type(owner), _Namespaceable):
        instance, owner = owner, type(owner)
        dct = Namespace.get_namespace(owner, dct.path)
        ns_proxy = _NamespaceProxy(dct, instance, owner)
    return ns_proxy

class _NamespaceProxy(_Proxy):

    """Proxy object for manipulating and querying namespaces."""

    __slots__ = '__weakref__',

    def __init__(self, dct, instance, owner):
        _PROXY_INFOS[self] = dct, instance, owner

    def __dir__(self):
        return collections.ChainMap(_instance_map(self), _mro_map(self))

    def __getattribute__(self, name):
        self = _retarget(self)
        _, instance, owner = _PROXY_INFOS[self]
        instance_map = _instance_map(self)
        mro_map = _mro_map(self)
        instance_value = ops.get(instance_map, name)
        mro_value = ops.get(mro_map, name)
        if ops.is_data(mro_value):
            return mro_value.get(instance, owner)
        elif issubclass(owner, type) and ops.has_get(instance_value):
            return instance_value.get(None, instance)
        elif instance_value is not None:
            return instance_value.object
        elif ops.has_get(mro_value):
            return mro_value.get(instance, owner)
        elif mro_value is not None:
            return mro_value.object
        else:
            raise AttributeError(name)

    def __setattr__(self, name, value):
        self = _retarget(self)
        dct, instance, owner = _PROXY_INFOS[self]
        if instance is None:
            real_map = Namespace.get_namespace(owner, dct.path)
            real_map[name] = value
            return
        mro_map = _mro_map(self)
        target_value = ops.get_data(mro_map, name)
        if target_value is not None:
            target_value.set(instance, value)
            return
        instance_map = Namespace.get_namespace(instance, dct.path)
        instance_map[name] = value

    def __delattr__(self, name):
        self = _retarget(self)
        dct, instance, owner = _PROXY_INFOS[self]
        real_map = Namespace.get_namespace(owner, dct.path)
        if instance is None:
            ops.delete(real_map, name)
            return
        value = ops.get_data(real_map, name)
        if value is not None:
            value.delete(instance)
            return
        instance_map = Namespace.get_namespace(instance, dct.path)
        ops.delete(instance_map, name)

class Namespace(dict):

    """Namespace."""

    __slots__ = 'name', 'scope', 'parent', 'active', 'parent_object'

    __namespaces = {}

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        bad_values = tuple(
            value for value in self.values() if
            isinstance(value, (Namespace, _Proxy)))
        if bad_values:
            raise ValueError('Bad values: {}'.format(bad_values))
        self.name = None
        self.scope = None
        self.parent = None
        self.active = False
        self.parent_object = None

    @classmethod
    def premake(cls, name, parent):
        """Return an empty namespace with the given name and parent."""
        self = cls()
        self.name = name
        self.parent = parent
        return self

    def __setitem__(self, key, value):
        if (
                self.parent_object is not None and
                isinstance(value, Namespace) and value.name != key):
            value.push(key, self.scope)
            value.add(self.parent_object)
        super().__setitem__(key, value)

    def __enter__(self):
        self.activate()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if self.name is None:
            raise RuntimeError('Namespace must be named.')
        self.deactivate()

    @property
    def path(self):
        """Return the full path of the namespace."""
        if self.name is None or self.parent is None:
            raise ValueError
        if isinstance(self.parent, Namespace):
            parent_path = self.parent.path
        else:
            parent_path = ()
        return parent_path + (self.name,)

    @classmethod
    def __get_helper(cls, target, path):
        """Return the namespace for `target` at `path`, create if needed."""
        if isinstance(target, _Namespaceable):
            path_ = target, path
            namespaces = cls.__namespaces
        else:
            path_ = path
            try:
                namespaces = target.__namespaces
            except AttributeError:
                namespaces = {}
                target.__namespaces = namespaces
        return path_, namespaces

    @classmethod
    def namespace_exists(cls, path, target):
        """Return whether the given namespace exists."""
        path_, namespaces = cls.__get_helper(target, path)
        return path_ in namespaces

    @classmethod
    def get_namespace(cls, target, path):
        """Return a namespace with given target and path, create if needed."""
        path_, namespaces = cls.__get_helper(target, path)
        try:
            return namespaces[path_]
        except KeyError:
            if len(path) == 1:
                parent = {}
            else:
                parent = cls.get_namespace(target, path[:-1])
            return namespaces.setdefault(path_, cls.premake(path[-1], parent))

    @classmethod
    def no_blocker(cls, dct, cls_):
        """Return False if there's a non-Namespace object in the path."""
        try:
            namespace = vars(cls_)
            for name in dct.path:
                namespace = namespace[name]
                if not isinstance(namespace, cls):
                    return False
        except KeyError:
            pass
        return True

    def add(self, target):
        """Add self as a namespace under target."""
        path, namespaces = self.__get_helper(target, self.path)
        res = namespaces.setdefault(path, self)
        if res is self:
            self.parent_object = target

    def set_if_none(self, name, value):
        """Set the attribute `name` to `value`, if it's initially None."""
        if getattr(self, name) is None:
            setattr(self, name, value)

    def push(self, name, scope):
        """Bind self to the given name and scope, and activate."""
        self.set_if_none('name', name)
        self.set_if_none('scope', scope)
        self.set_if_none('parent', scope.dicts[0])
        if name != self.name:
            raise ValueError('Cannot rename namespace')
        if scope is not self.scope:
            raise ValueError('Cannot reuse namespace')
        if scope.dicts[0] is not self.parent:
            raise ValueError('Cannot reparent namespace')
        self.scope.namespaces.append(self)
        self.activate()

    def activate(self):
        """Take over as the scope for the target."""
        if self.scope is not None and not self.active:
            if self.scope.dicts[0] is not self.parent:
                raise ValueError('Cannot reparent namespace')
            self.active = True
            self.scope.dicts.insert(0, self)

    def deactivate(self):
        """Stop being the scope for the target."""
        if self.scope is not None and self.active:
            self.active = False
            self.scope.dicts.pop(0)

    def __get__(self, instance, owner):
        return _NamespaceProxy(self, instance, owner)

class _NamespaceScope(collections.abc.MutableMapping):

    """The class creation namespace for _Namespaceables."""

    __slots__ = 'dicts', 'namespaces', 'proxies', 'scope_proxy'

    def __init__(self, dct):
        self.dicts = [dct]
        self.namespaces = []
        self.proxies = proxies = weakref.WeakKeyDictionary()

        class ScopeProxy(_ScopeProxy):

            """Local version of ScopeProxy for this scope."""

            __slots__ = ()

            def __init__(self, dct):
                super().__init__(dct, proxies)

        self.scope_proxy = ScopeProxy

    def __getitem__(self, key):
        value = collections.ChainMap(*self.dicts)[key]
        if isinstance(value, Namespace):
            value = self.scope_proxy(value)
        return value

    def __setitem__(self, key, value):
        dct = self.dicts[0]
        if isinstance(value, self.scope_proxy):
            value = self.proxies[value]
        if isinstance(value, Namespace) and value.name != key:
            value.push(key, self)
        dct[key] = value

    def __delitem__(self, key):
        del self.dicts[0][key]

    def __iter__(self):
        return iter(collections.ChainMap(*self.dicts))

    def __len__(self):
        return len(collections.ChainMap(*self.dicts))

_NAMESPACE_SCOPES = weakref.WeakKeyDictionary()

class _NamespaceBase:

    """Common base class for Namespaceable and its metaclass."""

    __slots__ = ()

    def __setattr__(self, name, value):
        super(_NamespaceBase, type(self)).__setattr__(self, name, value)

    def __delattr__(self, name):
        super(_NamespaceBase, type(self)).__delattr__(self, name)

class _Namespaceable(_NamespaceBase, type):

    """Metaclass for classes that can contain namespaces.

    Using the metaclass directly is a bad idea. Use a base class instead.
    """

    @classmethod
    def __prepare__(mcs, name, bases, **kwargs):
        return _NamespaceScope(super().__prepare__(name, bases, **kwargs))

    def __new__(mcs, name, bases, dct, **kwargs):
        cls = super().__new__(mcs, name, bases, dct.dicts[0], **kwargs)
        if _DEFINED and not issubclass(cls, Namespaceable):
            raise ValueError(
                'Cannot create a _Namespaceable that does not inherit from '
                'Namespaceable')
        _NAMESPACE_SCOPES[cls] = dct
        for namespace in dct.namespaces:
            namespace.add(cls)
            if ENABLE_SET_NAME:
                for name, value in namespace.items():
                    wrapped = _DescriptorInspector(value)
                    if wrapped.has_set_name:
                        wrapped.set_name(cls, name)
        return cls

    def __setattr__(cls, name, value):
        if isinstance(value, Namespace) and value.name != name:
            value.push(name, _NAMESPACE_SCOPES[cls])
            value.add(cls)
        super(_Namespaceable, type(cls)).__setattr__(cls, name, value)

_DEFINED = False

class Namespaceable(_NamespaceBase, metaclass=_Namespaceable):

    """Base class for classes that can contain namespaces."""

_DEFINED = True
<commit_msg>All attribute forwarding works.<commit_after>"""Class Namespaces (internal module).

All of the guts of the class namespace implementation.

"""

# See https://blog.ionelmc.ro/2015/02/09/understanding-python-metaclasses/

import collections.abc
import functools
import itertools
import weakref

from . import ops
from .descriptor_inspector import _DescriptorInspector
from .flags import ENABLE_SET_NAME
from .proxy import _Proxy
from .scope_proxy import _ScopeProxy

_PROXY_INFOS = weakref.WeakKeyDictionary()

def _mro_to_chained(mro, dct):
    """Return a chained map of lookups for the given namespace and mro."""
    return collections.ChainMap(*[
        Namespace.get_namespace(cls, dct.path) for cls in
        itertools.takewhile(
            functools.partial(Namespace.no_blocker, dct),
            (cls for cls in mro if isinstance(cls, _Namespaceable))) if
        Namespace.namespace_exists(dct.path, cls)])

def _instance_map(ns_proxy):
    """Return a map, possibly chained, of lookups for the given instance."""
    dct, instance, _ = _PROXY_INFOS[ns_proxy]
    if instance is not None:
        if isinstance(instance, _Namespaceable):
            return _mro_to_chained(instance.__mro__, dct)
        else:
            return Namespace.get_namespace(instance, dct.path)
    else:
        return {}

def _mro_map(ns_proxy):
    """Return a chained map of lookups for the given owner class."""
    dct, _, owner = _PROXY_INFOS[ns_proxy]
    mro = owner.__mro__
    mro = mro[mro.index(dct.parent_object):]
    return _mro_to_chained(mro, dct)

def _retarget(ns_proxy):
    """Convert a class lookup to an instance lookup, if needed."""
    dct, instance, owner = _PROXY_INFOS[ns_proxy]
    if instance is None and isinstance(type(owner), _Namespaceable):
        instance, owner = owner, type(owner)
        dct = Namespace.get_namespace(owner, dct.path)
        ns_proxy = _NamespaceProxy(dct, instance, owner)
    return ns_proxy

class _NamespaceProxy(_Proxy):

    """Proxy object for manipulating and querying namespaces."""

    __slots__ = '__weakref__',

    def __init__(self, dct, instance, owner):
        _PROXY_INFOS[self] = dct, instance, owner

    def __dir__(self):
        return collections.ChainMap(_instance_map(self), _mro_map(self))

    def __getattribute__(self, name):
        self = _retarget(self)
        _, instance, owner = _PROXY_INFOS[self]
        instance_map = _instance_map(self)
        mro_map = _mro_map(self)
        instance_value = ops.get(instance_map, name)
        mro_value = ops.get(mro_map, name)
        if ops.is_data(mro_value):
            return mro_value.get(instance, owner)
        elif issubclass(owner, type) and ops.has_get(instance_value):
            return instance_value.get(None, instance)
        elif instance_value is not None:
            return instance_value.object
        elif ops.has_get(mro_value):
            return mro_value.get(instance, owner)
        elif mro_value is not None:
            return mro_value.object
        else:
            raise AttributeError(name)

    def __setattr__(self, name, value):
        self = _retarget(self)
        dct, instance, owner = _PROXY_INFOS[self]
        if instance is None:
            real_map = Namespace.get_namespace(owner, dct.path)
            real_map[name] = value
            return
        mro_map = _mro_map(self)
        target_value = ops.get_data(mro_map, name)
        if target_value is not None:
            target_value.set(instance, value)
            return
        instance_map = Namespace.get_namespace(instance, dct.path)
        instance_map[name] = value

    def __delattr__(self, name):
        self = _retarget(self)
        dct, instance, owner = _PROXY_INFOS[self]
        real_map = Namespace.get_namespace(owner, dct.path)
        if instance is None:
            ops.delete(real_map, name)
            return
        value = ops.get_data(real_map, name)
        if value is not None:
            value.delete(instance)
            return
        instance_map = Namespace.get_namespace(instance, dct.path)
        ops.delete(instance_map, name)

class Namespace(dict):

    """Namespace."""

    __slots__ = 'name', 'scope', 'parent', 'active', 'parent_object'

    __namespaces = {}

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        bad_values = tuple(
            value for value in self.values() if
            isinstance(value, (Namespace, _Proxy)))
        if bad_values:
            raise ValueError('Bad values: {}'.format(bad_values))
        self.name = None
        self.scope = None
        self.parent = None
        self.active = False
        self.parent_object = None

    @classmethod
    def premake(cls, name, parent):
        """Return an empty namespace with the given name and parent."""
        self = cls()
        self.name = name
        self.parent = parent
        return self

    def __setitem__(self, key, value):
        if (
                self.parent_object is not None and
                isinstance(value, Namespace) and value.name != key):
            value.push(key, self.scope)
            value.add(self.parent_object)
        super().__setitem__(key, value)

    def __enter__(self):
        self.activate()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if self.name is None:
            raise RuntimeError('Namespace must be named.')
        self.deactivate()

    @property
    def path(self):
        """Return the full path of the namespace."""
        if self.name is None or self.parent is None:
            raise ValueError
        if isinstance(self.parent, Namespace):
            parent_path = self.parent.path
        else:
            parent_path = ()
        return parent_path + (self.name,)

    @classmethod
    def __get_helper(cls, target, path):
        """Return the namespace for `target` at `path`, create if needed."""
        if isinstance(target, _Namespaceable):
            path_ = target, path
            namespaces = cls.__namespaces
        else:
            path_ = path
            try:
                namespaces = target.__namespaces
            except AttributeError:
                namespaces = {}
                target.__namespaces = namespaces
        return path_, namespaces

    @classmethod
    def namespace_exists(cls, path, target):
        """Return whether the given namespace exists."""
        path_, namespaces = cls.__get_helper(target, path)
        return path_ in namespaces

    @classmethod
    def get_namespace(cls, target, path):
        """Return a namespace with given target and path, create if needed."""
        path_, namespaces = cls.__get_helper(target, path)
        try:
            return namespaces[path_]
        except KeyError:
            if len(path) == 1:
                parent = {}
            else:
                parent = cls.get_namespace(target, path[:-1])
            return namespaces.setdefault(path_, cls.premake(path[-1], parent))

    @classmethod
    def no_blocker(cls, dct, cls_):
        """Return False if there's a non-Namespace object in the path."""
        try:
            namespace = vars(cls_)
            for name in dct.path:
                namespace = namespace[name]
                if not isinstance(namespace, cls):
                    return False
        except KeyError:
            pass
        return True

    def add(self, target):
        """Add self as a namespace under target."""
        path, namespaces = self.__get_helper(target, self.path)
        res = namespaces.setdefault(path, self)
        if res is self:
            self.parent_object = target

    def set_if_none(self, name, value):
        """Set the attribute `name` to `value`, if it's initially None."""
        if getattr(self, name) is None:
            setattr(self, name, value)

    def push(self, name, scope):
        """Bind self to the given name and scope, and activate."""
        self.set_if_none('name', name)
        self.set_if_none('scope', scope)
        self.set_if_none('parent', scope.dicts[0])
        if name != self.name:
            raise ValueError('Cannot rename namespace')
        if scope is not self.scope:
            raise ValueError('Cannot reuse namespace')
        if scope.dicts[0] is not self.parent:
            raise ValueError('Cannot reparent namespace')
        self.scope.namespaces.append(self)
        self.activate()

    def activate(self):
        """Take over as the scope for the target."""
        if self.scope is not None and not self.active:
            if self.scope.dicts[0] is not self.parent:
                raise ValueError('Cannot reparent namespace')
            self.active = True
            self.scope.dicts.insert(0, self)

    def deactivate(self):
        """Stop being the scope for the target."""
        if self.scope is not None and self.active:
            self.active = False
            self.scope.dicts.pop(0)

    def __get__(self, instance, owner):
        return _NamespaceProxy(self, instance, owner)

class _NamespaceScope(collections.abc.MutableMapping):

    """The class creation namespace for _Namespaceables."""

    __slots__ = 'dicts', 'namespaces', 'proxies', 'scope_proxy'

    def __init__(self, dct):
        self.dicts = [dct]
        self.namespaces = []
        self.proxies = proxies = weakref.WeakKeyDictionary()

        class ScopeProxy(_ScopeProxy):

            """Local version of ScopeProxy for this scope."""

            __slots__ = ()

            def __init__(self, dct):
                super().__init__(dct, proxies)

        self.scope_proxy = ScopeProxy

    def __getitem__(self, key):
        value = collections.ChainMap(*self.dicts)[key]
        if isinstance(value, Namespace):
            value = self.scope_proxy(value)
        return value

    def __setitem__(self, key, value):
        dct = self.dicts[0]
        if isinstance(value, self.scope_proxy):
            value = self.proxies[value]
        if isinstance(value, Namespace) and value.name != key:
            value.push(key, self)
        dct[key] = value

    def __delitem__(self, key):
        del self.dicts[0][key]

    def __iter__(self):
        return iter(collections.ChainMap(*self.dicts))

    def __len__(self):
        return len(collections.ChainMap(*self.dicts))

_NAMESPACE_SCOPES = weakref.WeakKeyDictionary()

class _NamespaceBase:

    """Common base class for Namespaceable and its metaclass."""

    __slots__ = ()

    def __getattribute__(self, name):
        return super(_NamespaceBase, type(self)).__getattribute__(self, name)

    def __setattr__(self, name, value):
        super(_NamespaceBase, type(self)).__setattr__(self, name, value)

    def __delattr__(self, name):
        super(_NamespaceBase, type(self)).__delattr__(self, name)

class _Namespaceable(_NamespaceBase, type):

    """Metaclass for classes that can contain namespaces.

    Using the metaclass directly is a bad idea. Use a base class instead.
    """

    @classmethod
    def __prepare__(mcs, name, bases, **kwargs):
        return _NamespaceScope(super().__prepare__(name, bases, **kwargs))

    def __new__(mcs, name, bases, dct, **kwargs):
        cls = super().__new__(mcs, name, bases, dct.dicts[0], **kwargs)
        if _DEFINED and not issubclass(cls, Namespaceable):
            raise ValueError(
                'Cannot create a _Namespaceable that does not inherit from '
                'Namespaceable')
        _NAMESPACE_SCOPES[cls] = dct
        for namespace in dct.namespaces:
            namespace.add(cls)
            if ENABLE_SET_NAME:
                for name, value in namespace.items():
                    wrapped = _DescriptorInspector(value)
                    if wrapped.has_set_name:
                        wrapped.set_name(cls, name)
        return cls

    def __setattr__(cls, name, value):
        if isinstance(value, Namespace) and value.name != name:
            value.push(name, _NAMESPACE_SCOPES[cls])
            value.add(cls)
        super(_Namespaceable, type(cls)).__setattr__(cls, name, value)

_DEFINED = False

class Namespaceable(_NamespaceBase, metaclass=_Namespaceable):

    """Base class for classes that can contain namespaces."""

_DEFINED = True
<|endoftext|>


<commit_before>#!/usr/bin/env python

import click
import copy
import os
import popper.utils as pu
import signal
import subprocess
import time
import sys
import re
import requests

from popper.cli import pass_context
from subprocess import check_output

@click.command('run',
               short_help='Run one or more pipelines and report on their '
                          'status.')
@click.argument('pipeline', required=False)
@click.option(
    '--timeout',
    help='Timeout limit for pipeline. Use s for seconds, m for minutes and h '
         'for hours. A single integer can also be used to specify timeout '
         'in seconds. Use double quotes if you wish to use more than one unit.'
         'For example: --timeout "2m 20s" will mean 140 seconds.',
    required=False,
    show_default=True,
    default="10800s"
)
@click.option(
    '--output',
    help='Directory where logs are stored, relative to pipeline folder.',
    required=False,
    show_default=True,
    default="popper/"
)
@click.option(
    '--skip',
    help="Comma-separated list of stages to skip. Only works when a single"
         "pipeline is being executed.",
    required=False,
)
@click.option(
    '--ignore-errors',
    help='Execute all pipelines even if there is a failure.',
    is_flag=True,
    required=False,
)
@click.option(
    '--requirement-level',
    type=click.Choice(['warn', 'ignore', 'fail']), default='fail',
    help='Determines how to handle missing requirements. warn simply'
         'emits a wanring and continues normally; ignore runs the'
         'specified pipelines despite missing unfulfilled'
         'requirements; fail exits with an error on unfilfilled'
         'requirements.'
)
@click.option(
    '--no-badge-update',
    help='Do not update the status of the pipeline against the badge server.',
    is_flag=True,
    required=False,
)
@pass_context
def cli(ctx, pipeline, timeout, skip, ignore_errors, output,
        no_badge_update, requirement_level):
    """Executes one or more pipelines and reports on their status. When
    PIPELINE is given, it executes only the pipeline with that name. If the
    argument is omitted, all pipelines are executed in lexicographical order.
    """
    project_pipelines = pu.read_config()['pipelines']

    if len(project_pipelines) == 0:
        pu.info("No pipelines defined in .popper.yml. "
                "Run popper init --help for more info.", fg='yellow')
        sys.exit(0)

    project_root = pu.get_project_root()

    cwd = os.getcwd()

    pipelines = get_pipelines_to_execute(cwd, pipeline, project_pipelines)

    if os.environ.get('CI', False):
        pipes_from_log = pipelines_from_commit_message(project_pipelines)
        if len(pipes_from_log) != 0:
            pu.info("Found 'CI', ignoring PIPELINE argument.")
            pipelines = pipes_from_log

    pipelines = {pipe_n: pipe_c for pipe_n, pipe_c in pipelines.items()
                 if check_requirements(pipe_n, pipe_c, requirement_level)}

    for pipe_n, pipe_d in pipelines.items():
        for env in pipe_d.get('envs', ['host']):
            status = run_pipeline(project_root, pipe_n, pipe_d, env, timeout,
                                  skip, ignore_errors, output)
            if status == 'FAIL' and not ignore_errors:
                break
    if not len(pipelines):
        pu.info("No pipelines to execute")
        sys.exit(0)

    os.chdir(cwd)

    if os.environ.get('CI', False) and not no_badge_update:
        update_badge(status)

    if status == 'FAIL':
        pu.fail("Failed to execute pipeline")

def get_pipelines_to_execute(cwd, pipe_n, project_pipelines):
    pipelines = {}
    if pipe_n:
        if pipe_n not in project_pipelines:
            pu.fail("Pipeline '{}' not in .popper.yml".format(pipe_n))
        return {pipe_n: project_pipelines[pipe_n]}
    else:
        cwd_pipe_n = os.path.basename(cwd)
        if cwd_pipe_n in project_pipelines:
            pipelines = {cwd_pipe_n: project_pipelines[cwd_pipe_n]}
        else:
            pipelines = project_pipelines

    return pipelines

def update_badge(status):
    if pu.is_repo_empty():
        pu.warn('No commit log found. Skipping badge server update')

    remote_url = pu.get_remote_url()

    if remote_url:
        baseurl = pu.read_config().get(
            'badge-server-url', 'http://badges.falsifiable.us'
        )
        org, repo = remote_url.split('/')[-2:]
        badge_server_url = '{}/{}/{}'.format(baseurl, org, repo)
        data = {
            'timestamp': int(time.time()),
            'commit_id': pu.get_head_commit(),
            'status': status
        }
        try:
            r = requests.post(badge_server_url, data=data)
            if r.status_code != 201:
                pu.warn("Could not create a record on the badge server")
        except requests.exceptions.RequestException:
            pu.warn("Could not communicate with the badge server")

def pipelines_from_commit_message(project_pipelines):

    # check if repo is empty
    if pu.is_repo_empty():
        return {}

    args = ['git', 'log', '-1', '--pretty=%B']

    msg = check_output(args)

    # check for pull requests
    if 'Merge' in msg:
        pu.info("Merge detected. Reading message from merged commit.")
        commit_id = re.search('Merge (.+?) into', msg).group(1)

        args = ['git', 'show', '-s', '--format=%B', commit_id]

        msg = str(check_output(args))

    if 'popper:skip[' in msg:
        pu.info("Found 'popper:skip' keyword.")
        re_expr = r'popper:skip\[(.+?)\]'
    elif 'popper:whitelist[' in msg:
        pu.info("Found 'popper:whitelist' keyword.")
        re_expr = r'popper:whitelist\[(.+?)\]'
    else:
        return project_pipelines

    try:
        pipe_list = re.search(re_expr, msg).group(1).split(',')
    except AttributeError:
        pu.fail("Error parsing commit message keyword.")

    if 'skip' in re_expr:
        pipelines = copy.deepcopy(project_pipelines)
        for p in pipe_list:
            pipelines.pop(p)
    else:
        pipelines = {}
        for p in pipe_list:
            pipelines.update({p: project_pipelines[p]})

    print('pipes: {}'.format(pipelines))
    return pipelines

def check_requirements(pipe_n, pipeline, requirement_level):
    if 'requirements' not in pipeline:
        return True

    reqs = pipeline['requirements'].get('vars', [])
    missing_reqs = [envvar for envvar in reqs if envvar not in os.environ]

    if len(missing_reqs):
        msg = ('Required environment variables for pipeline {} unset: {}'
               .format(pipe_n, ','.join(missing_reqs)))

        if requirement_level == 'fail':
            pu.fail(msg)

        pu.warn(msg)

        if requirement_level == 'warn':
            pu.info('Skipping pipeline {}'.format(pipe_n))

        return requirement_level == 'ignore'
    return True

def run_in_docker(project_root, pipe_n, pipe_d, env, timeout, skip,
                  ignore_errors, output_dir):

    abs_path = '{}/{}'.format(project_root, pipe_d['path'])
    docker_cmd = 'docker run --rm -v {0}:{0}'.format(project_root)
    docker_cmd += ' --workdir={}'.format(abs_path)

    if '/' in env:
        img = env
    else:
        img = 'falsifiable/popper:{}'.format(env)

    docker_cmd += ' {} run '.format(img)

    popper_flags = ' --timeout={}'.format(timeout)
    popper_flags += ' --skip {}'.format(','.join(skip)) if skip else ''
    popper_flags += ' --output {}'.format(output_dir)
    popper_flags += ' --ignore-errors' if ignore_errors else ''

    cmd = '{} {}'.format(docker_cmd, popper_flags)
    pu.info('Running in Docker with: {}'.format(cmd))
    check_output(cmd, shell=True)

    with open(os.path.join(abs_path, output_dir, 'popper_status'), 'r') as f:
        status = f.read()

    return status

def run_on_host(project_root, pipe_n, pipe_d, skip_list, timeout, output_dir):

    os.chdir(os.path.join(project_root, pipe_d['path']))

    check_output('rm -rf {}'.format(output_dir), shell=True)
    check_output('mkdir -p {}'.format(output_dir), shell=True)

    status = "SUCCESS"

    with click.progressbar(pipe_d['stages'], show_eta=False,
                           item_show_func=(lambda s: status if s is None
                                           else str(s)),
                           bar_template='[%(bar)s] %(info)s',
                           show_percent=False) as stages:

        for stage in stages:

            stage_file = pu.get_filename('.', stage)

            if not stage_file or stage in skip_list:
                continue

            ecode = execute(stage_file, timeout, output_dir, stages)

            if ecode != 0:
                pu.info("\n\nStage '{}' failed.".format(stage))
                status = "FAIL"
                for t in ['.err', '.out']:
                    logfile = os.path.join(output_dir, stage_file + t)
                    with open(logfile, 'r') as f:
                        pu.info("\n" + t + ":\n", bold=True, fg='red')
                        pu.info(f.read())

                # Execute teardown when some stage fails and then break
                teardown_file = pu.get_filename('.', 'teardown')
                if (teardown_file and
                        stage != 'teardown' and
                        'teardown' in pipe_d['stages'] and
                        'teardown' in skip_list):
                    execute(teardown_file, timeout, output_dir)

                break

            if 'valid' in stage:
                status = "GOLD"
                fpath = os.path.join(output_dir, 'validate.sh.out')
                with open(fpath, 'r') as f:
                    validate_output = f.readlines()
                    if len(validate_output) == 0:
                        status = "SUCCESS"
                    for line in validate_output:
                        if '[true]' not in line:
                            status = "SUCCESS"

    with open(os.path.join(output_dir, 'popper_status'), 'w') as f:
        f.write(status + '\n')

    if status == "SUCCESS":
        fg = 'green'
    elif status == "GOLD":
        fg = 'yellow'
    else:
        fg = 'red'
    pu.info('\nstatus: {}\n'.format(status), fg=fg, bold=True)

    os.chdir(project_root)

    return status

def execute(stage, timeout, output_dir, bar=None):
    time_limit = time.time() + timeout
    sleep_time = 1
    out_fname = os.path.join(output_dir, stage + '.out')
    err_fname = os.path.join(output_dir, stage + '.err')

    with open(out_fname, "w") as outf, open(err_fname, "w") as errf:
        p = subprocess.Popen(os.path.join('.', stage), shell=True, stdout=outf,
                             stderr=errf, preexec_fn=os.setsid)

        if os.environ.get('CI', False):
            # print info when in CI environment
            print('')
            sys.stdout.write('Running: {}'.format(stage))

        while p.poll() is None:

            if time.time() > time_limit:
                os.killpg(os.getpgid(p.pid), signal.SIGTERM)
                sys.stdout.write(' time out!')
                break

            if sleep_time < 300:
                sleep_time *= 2

            if os.environ.get('CI', False):
                # print dot every 10 secs when in CI environment
                for i in range(sleep_time):
                    if i % 10 == 0:
                        sys.stdout.write('.')
                time.sleep(sleep_time)
                continue

            if bar:
                for i in range(sleep_time):
                    bar.label = bar.label + '\b_'
                    bar.render_progress()
                    time.sleep(0.5)
                    bar.label = bar.label + '\b '
                    bar.render_progress()
                    time.sleep(0.5)

    return p.poll()

def run_pipeline(project_root, pipe_n, pipe_d, env, timeout,
                 skip, ignore_errors, output_dir):
    timeout_parsed = pu.parse_timeout(timeout)

    skip_list = skip.split(',') if skip else []

    click.echo('Executing pipeline: {}'.format(pipe_n))

    if os.path.isfile('/.dockerenv'):
        return run_on_host(project_root, pipe_n, pipe_d, skip_list,
                           timeout_parsed, output_dir)

    if env != 'host':
        return run_in_docker(project_root, pipe_n, pipe_d, env, timeout, skip,
                             ignore_errors, '{}/{}'.format(
                                 output_dir, env.replace('/', '_')))

    return run_on_host(project_root, pipe_n, pipe_d, skip_list,
                       timeout_parsed, os.path.join(output_dir, 'host'))

class Unbuffered(object):
    def __init__(self, stream):
        self.stream = stream

    def write(self, data):
        self.stream.write(data)
        self.stream.flush()

    def __getattr__(self, attr):
        return getattr(self.stream, attr)
<commit_msg>Print only pipeline names<commit_after>#!/usr/bin/env python

import click
import copy
import os
import popper.utils as pu
import signal
import subprocess
import time
import sys
import re
import requests

from popper.cli import pass_context
from subprocess import check_output

@click.command('run',
               short_help='Run one or more pipelines and report on their '
                          'status.')
@click.argument('pipeline', required=False)
@click.option(
    '--timeout',
    help='Timeout limit for pipeline. Use s for seconds, m for minutes and h '
         'for hours. A single integer can also be used to specify timeout '
         'in seconds. Use double quotes if you wish to use more than one unit.'
         'For example: --timeout "2m 20s" will mean 140 seconds.',
    required=False,
    show_default=True,
    default="10800s"
)
@click.option(
    '--output',
    help='Directory where logs are stored, relative to pipeline folder.',
    required=False,
    show_default=True,
    default="popper/"
)
@click.option(
    '--skip',
    help="Comma-separated list of stages to skip. Only works when a single"
         "pipeline is being executed.",
    required=False,
)
@click.option(
    '--ignore-errors',
    help='Execute all pipelines even if there is a failure.',
    is_flag=True,
    required=False,
)
@click.option(
    '--requirement-level',
    type=click.Choice(['warn', 'ignore', 'fail']), default='fail',
    help='Determines how to handle missing requirements. warn simply'
         'emits a wanring and continues normally; ignore runs the'
         'specified pipelines despite missing unfulfilled'
         'requirements; fail exits with an error on unfilfilled'
         'requirements.'
)
@click.option(
    '--no-badge-update',
    help='Do not update the status of the pipeline against the badge server.',
    is_flag=True,
    required=False,
)
@pass_context
def cli(ctx, pipeline, timeout, skip, ignore_errors, output,
        no_badge_update, requirement_level):
    """Executes one or more pipelines and reports on their status. When
    PIPELINE is given, it executes only the pipeline with that name. If the
    argument is omitted, all pipelines are executed in lexicographical order.
    """
    project_pipelines = pu.read_config()['pipelines']

    if len(project_pipelines) == 0:
        pu.info("No pipelines defined in .popper.yml. "
                "Run popper init --help for more info.", fg='yellow')
        sys.exit(0)

    project_root = pu.get_project_root()

    cwd = os.getcwd()

    pipelines = get_pipelines_to_execute(cwd, pipeline, project_pipelines)

    if os.environ.get('CI', False):
        pipes_from_log = pipelines_from_commit_message(project_pipelines)
        if len(pipes_from_log) != 0:
            pu.info("Found 'CI', ignoring PIPELINE argument.")
            pipelines = pipes_from_log

    pipelines = {pipe_n: pipe_c for pipe_n, pipe_c in pipelines.items()
                 if check_requirements(pipe_n, pipe_c, requirement_level)}

    for pipe_n, pipe_d in pipelines.items():
        for env in pipe_d.get('envs', ['host']):
            status = run_pipeline(project_root, pipe_n, pipe_d, env, timeout,
                                  skip, ignore_errors, output)
            if status == 'FAIL' and not ignore_errors:
                break
    if not len(pipelines):
        pu.info("No pipelines to execute")
        sys.exit(0)

    os.chdir(cwd)

    if os.environ.get('CI', False) and not no_badge_update:
        update_badge(status)

    if status == 'FAIL':
        pu.fail("Failed to execute pipeline")

def get_pipelines_to_execute(cwd, pipe_n, project_pipelines):
    pipelines = {}
    if pipe_n:
        if pipe_n not in project_pipelines:
            pu.fail("Pipeline '{}' not in .popper.yml".format(pipe_n))
        return {pipe_n: project_pipelines[pipe_n]}
    else:
        cwd_pipe_n = os.path.basename(cwd)
        if cwd_pipe_n in project_pipelines:
            pipelines = {cwd_pipe_n: project_pipelines[cwd_pipe_n]}
        else:
            pipelines = project_pipelines

    return pipelines

def update_badge(status):
    if pu.is_repo_empty():
        pu.warn('No commit log found. Skipping badge server update')

    remote_url = pu.get_remote_url()

    if remote_url:
        baseurl = pu.read_config().get(
            'badge-server-url', 'http://badges.falsifiable.us'
        )
        org, repo = remote_url.split('/')[-2:]
        badge_server_url = '{}/{}/{}'.format(baseurl, org, repo)
        data = {
            'timestamp': int(time.time()),
            'commit_id': pu.get_head_commit(),
            'status': status
        }
        try:
            r = requests.post(badge_server_url, data=data)
            if r.status_code != 201:
                pu.warn("Could not create a record on the badge server")
        except requests.exceptions.RequestException:
            pu.warn("Could not communicate with the badge server")

def pipelines_from_commit_message(project_pipelines):

    # check if repo is empty
    if pu.is_repo_empty():
        return {}

    args = ['git', 'log', '-1', '--pretty=%B']

    msg = str(check_output(args))

    # check for pull requests
    if 'Merge' in msg:
        pu.info("Merge detected. Reading message from merged commit.")
        commit_id = re.search('Merge (.+?) into', msg).group(1)

        args = ['git', 'show', '-s', '--format=%B', commit_id]

        msg = str(check_output(args))

    if 'popper:skip[' in msg:
        pu.info("Found 'popper:skip' keyword.")
        re_expr = r'popper:skip\[(.+?)\]'
    elif 'popper:whitelist[' in msg:
        pu.info("Found 'popper:whitelist' keyword.")
        re_expr = r'popper:whitelist\[(.+?)\]'
    else:
        return project_pipelines

    try:
        pipe_list = re.search(re_expr, msg).group(1).split(',')
    except AttributeError:
        pu.fail("Error parsing commit message keyword.")

    if 'skip' in re_expr:
        pipelines = copy.deepcopy(project_pipelines)
        for p in pipe_list:
            pipelines.pop(p)
    else:
        pipelines = {}
        for p in pipe_list:
            pipelines.update({p: project_pipelines[p]})

    print('Only running pipes: {}'.format(', '.join(pipelines.keys())))
    return pipelines

def check_requirements(pipe_n, pipeline, requirement_level):
    if 'requirements' not in pipeline:
        return True

    reqs = pipeline['requirements'].get('vars', [])
    missing_reqs = [envvar for envvar in reqs if envvar not in os.environ]

    if len(missing_reqs):
        msg = ('Required environment variables for pipeline {} unset: {}'
               .format(pipe_n, ','.join(missing_reqs)))

        if requirement_level == 'fail':
            pu.fail(msg)

        pu.warn(msg)

        if requirement_level == 'warn':
            pu.info('Skipping pipeline {}'.format(pipe_n))

        return requirement_level == 'ignore'
    return True

def run_in_docker(project_root, pipe_n, pipe_d, env, timeout, skip,
                  ignore_errors, output_dir):

    abs_path = '{}/{}'.format(project_root, pipe_d['path'])
    docker_cmd = 'docker run --rm -v {0}:{0}'.format(project_root)
    docker_cmd += ' --workdir={}'.format(abs_path)

    if '/' in env:
        img = env
    else:
        img = 'falsifiable/popper:{}'.format(env)

    docker_cmd += ' {} run '.format(img)

    popper_flags = ' --timeout={}'.format(timeout)
    popper_flags += ' --skip {}'.format(','.join(skip)) if skip else ''
    popper_flags += ' --output {}'.format(output_dir)
    popper_flags += ' --ignore-errors' if ignore_errors else ''

    cmd = '{} {}'.format(docker_cmd, popper_flags)
    pu.info('Running in Docker with: {}'.format(cmd))
    check_output(cmd, shell=True)

    with open(os.path.join(abs_path, output_dir, 'popper_status'), 'r') as f:
        status = f.read()

    return status

def run_on_host(project_root, pipe_n, pipe_d, skip_list, timeout, output_dir):

    os.chdir(os.path.join(project_root, pipe_d['path']))

    check_output('rm -rf {}'.format(output_dir), shell=True)
    check_output('mkdir -p {}'.format(output_dir), shell=True)

    status = "SUCCESS"

    with click.progressbar(pipe_d['stages'], show_eta=False,
                           item_show_func=(lambda s: status if s is None
                                           else str(s)),
                           bar_template='[%(bar)s] %(info)s',
                           show_percent=False) as stages:

        for stage in stages:

            stage_file = pu.get_filename('.', stage)

            if not stage_file or stage in skip_list:
                continue

            ecode = execute(stage_file, timeout, output_dir, stages)

            if ecode != 0:
                pu.info("\n\nStage '{}' failed.".format(stage))
                status = "FAIL"
                for t in ['.err', '.out']:
                    logfile = os.path.join(output_dir, stage_file + t)
                    with open(logfile, 'r') as f:
                        pu.info("\n" + t + ":\n", bold=True, fg='red')
                        pu.info(f.read())

                # Execute teardown when some stage fails and then break
                teardown_file = pu.get_filename('.', 'teardown')
                if (teardown_file and
                        stage != 'teardown' and
                        'teardown' in pipe_d['stages'] and
                        'teardown' in skip_list):
                    execute(teardown_file, timeout, output_dir)

                break

            if 'valid' in stage:
                status = "GOLD"
                fpath = os.path.join(output_dir, 'validate.sh.out')
                with open(fpath, 'r') as f:
                    validate_output = f.readlines()
                    if len(validate_output) == 0:
                        status = "SUCCESS"
                    for line in validate_output:
                        if '[true]' not in line:
                            status = "SUCCESS"

    with open(os.path.join(output_dir, 'popper_status'), 'w') as f:
        f.write(status + '\n')

    if status == "SUCCESS":
        fg = 'green'
    elif status == "GOLD":
        fg = 'yellow'
    else:
        fg = 'red'
    pu.info('\nstatus: {}\n'.format(status), fg=fg, bold=True)

    os.chdir(project_root)

    return status

def execute(stage, timeout, output_dir, bar=None):
    time_limit = time.time() + timeout
    sleep_time = 1
    out_fname = os.path.join(output_dir, stage + '.out')
    err_fname = os.path.join(output_dir, stage + '.err')

    with open(out_fname, "w") as outf, open(err_fname, "w") as errf:
        p = subprocess.Popen(os.path.join('.', stage), shell=True, stdout=outf,
                             stderr=errf, preexec_fn=os.setsid)

        if os.environ.get('CI', False):
            # print info when in CI environment
            print('')
            sys.stdout.write('Running: {}'.format(stage))

        while p.poll() is None:

            if time.time() > time_limit:
                os.killpg(os.getpgid(p.pid), signal.SIGTERM)
                sys.stdout.write(' time out!')
                break

            if sleep_time < 300:
                sleep_time *= 2

            if os.environ.get('CI', False):
                # print dot every 10 secs when in CI environment
                for i in range(sleep_time):
                    if i % 10 == 0:
                        sys.stdout.write('.')
                time.sleep(sleep_time)
                continue

            if bar:
                for i in range(sleep_time):
                    bar.label = bar.label + '\b_'
                    bar.render_progress()
                    time.sleep(0.5)
                    bar.label = bar.label + '\b '
                    bar.render_progress()
                    time.sleep(0.5)

    return p.poll()

def run_pipeline(project_root, pipe_n, pipe_d, env, timeout,
                 skip, ignore_errors, output_dir):
    timeout_parsed = pu.parse_timeout(timeout)

    skip_list = skip.split(',') if skip else []

    click.echo('Executing pipeline: {}'.format(pipe_n))

    if os.path.isfile('/.dockerenv'):
        return run_on_host(project_root, pipe_n, pipe_d, skip_list,
                           timeout_parsed, output_dir)

    if env != 'host':
        return run_in_docker(project_root, pipe_n, pipe_d, env, timeout, skip,
                             ignore_errors, '{}/{}'.format(
                                 output_dir, env.replace('/', '_')))

    return run_on_host(project_root, pipe_n, pipe_d, skip_list,
                       timeout_parsed, os.path.join(output_dir, 'host'))

class Unbuffered(object):
    def __init__(self, stream):
        self.stream = stream

    def write(self, data):
        self.stream.write(data)
        self.stream.flush()

    def __getattr__(self, attr):
        return getattr(self.stream, attr)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import shutil
import os

import numpy as np

from openfisca_core import periods
from openfisca_core.periods import ETERNITY
from openfisca_core.indexed_enums import EnumArray

class InMemoryStorage(object):
    """
    Low-level class responsible for storing and retrieving calculated vectors in memory
    """

    def __init__(self, is_eternal = False):
        self._arrays = {}
        self.is_eternal = is_eternal

    def get(self, period):
        if self.is_eternal:
            period = periods.period(ETERNITY)
        period = periods.period(period)

        values = self._arrays.get(period)
        if values is None:
            return None
        if isinstance(values, dict):
            return next(iter(values.values()))
        return values

    def put(self, value, period):
        if self.is_eternal:
            period = periods.period(ETERNITY)
        period = periods.period(period)

        self._arrays[period] = value

    def delete(self, period = None):
        if period is None:
            self._arrays = {}
            return

        if self.is_eternal:
            period = periods.period(ETERNITY)
        period = periods.period(period)

        self._arrays = {
            period_item: value
            for period_item, value in self._arrays.items()
            if not period.contains(period_item)
            }

    def get_known_periods(self):
        return self._arrays.keys()

    def get_memory_usage(self):
        if not self._arrays:
            return dict(
                nb_arrays = 0,
                total_nb_bytes = 0,
                cell_size = np.nan,
                )

        nb_arrays = sum([
            len(array_or_dict) if isinstance(array_or_dict, dict) else 1
            for array_or_dict in self._arrays.values()
            ])

        array = next(iter(self._arrays.values()))
        if isinstance(array, dict):
            array = list(array.values())[0]
        return dict(
            nb_arrays = nb_arrays,
            total_nb_bytes = array.nbytes * nb_arrays,
            cell_size = array.itemsize,
            )

class OnDiskStorage(object):
    """
    Low-level class responsible for storing and retrieving calculated vectors on disk
    """

    def __init__(self, storage_dir, is_eternal = False, preserve_storage_dir = False):
        self._files = {}
        self._enums = {}
        self.is_eternal = is_eternal
        self.preserve_storage_dir = preserve_storage_dir
        self.storage_dir = storage_dir

    def _decode_file(self, file):
        enum = self._enums.get(file)
        if enum is not None:
            return EnumArray(np.load(file), enum)
        else:
            return np.load(file)

    def get(self, period):
        if self.is_eternal:
            period = periods.period(ETERNITY)
        period = periods.period(period)

        values = self._files.get(period)
        if values is None:
            return None
        if isinstance(values, dict):
            return self._decode_file(next(iter(values.values())))
        return self._decode_file(values)

    def put(self, value, period):
        if self.is_eternal:
            period = periods.period(ETERNITY)
        period = periods.period(period)

        filename = str(period)
        path = os.path.join(self.storage_dir, filename) + '.npy'
        if isinstance(value, EnumArray):
            self._enums[path] = value.possible_values
            value = value.view(np.ndarray)
        np.save(path, value)
        self._files[period] = path

    def delete(self, period = None):
        if period is None:
            self._files = {}
            return

        if self.is_eternal:
            period = periods.period(ETERNITY)
        period = periods.period(period)

        if period is not None:
            self._files = {
                period_item: value
                for period_item, value in self._files.items()
                if not period.contains(period_item)
                }

    def get_known_periods(self):
        return self._files.keys()

    def restore(self):
        self._files = files = {}
        # Restore self._files from content of storage_dir.
        for filename in os.listdir(self.storage_dir):
            if not filename.endswith('.npy'):
                continue
            path = os.path.join(self.storage_dir, filename)
            filename_core = filename.rsplit('.', 1)[0]
            period = periods.period(filename_core)
            files[period] = path

    def __del__(self):
        if self.preserve_storage_dir:
            return
        shutil.rmtree(self.storage_dir)  # Remove the holder temporary files
        # If the simulation temporary directory is empty, remove it
        parent_dir = os.path.abspath(os.path.join(self.storage_dir, os.pardir))
        if not os.listdir(parent_dir):
            shutil.rmtree(parent_dir)
<commit_msg>Remove extra param handling in storage<commit_after># -*- coding: utf-8 -*-

import shutil
import os

import numpy as np

from openfisca_core import periods
from openfisca_core.periods import ETERNITY
from openfisca_core.indexed_enums import EnumArray

class InMemoryStorage(object):
    """
    Low-level class responsible for storing and retrieving calculated vectors in memory
    """

    def __init__(self, is_eternal = False):
        self._arrays = {}
        self.is_eternal = is_eternal

    def get(self, period):
        if self.is_eternal:
            period = periods.period(ETERNITY)
        period = periods.period(period)

        values = self._arrays.get(period)
        if values is None:
            return None
        return values

    def put(self, value, period):
        if self.is_eternal:
            period = periods.period(ETERNITY)
        period = periods.period(period)

        self._arrays[period] = value

    def delete(self, period = None):
        if period is None:
            self._arrays = {}
            return

        if self.is_eternal:
            period = periods.period(ETERNITY)
        period = periods.period(period)

        self._arrays = {
            period_item: value
            for period_item, value in self._arrays.items()
            if not period.contains(period_item)
            }

    def get_known_periods(self):
        return self._arrays.keys()

    def get_memory_usage(self):
        if not self._arrays:
            return dict(
                nb_arrays = 0,
                total_nb_bytes = 0,
                cell_size = np.nan,
                )

        nb_arrays = len(self._arrays)
        array = next(iter(self._arrays.values()))
        return dict(
            nb_arrays = nb_arrays,
            total_nb_bytes = array.nbytes * nb_arrays,
            cell_size = array.itemsize,
            )

class OnDiskStorage(object):
    """
    Low-level class responsible for storing and retrieving calculated vectors on disk
    """

    def __init__(self, storage_dir, is_eternal = False, preserve_storage_dir = False):
        self._files = {}
        self._enums = {}
        self.is_eternal = is_eternal
        self.preserve_storage_dir = preserve_storage_dir
        self.storage_dir = storage_dir

    def _decode_file(self, file):
        enum = self._enums.get(file)
        if enum is not None:
            return EnumArray(np.load(file), enum)
        else:
            return np.load(file)

    def get(self, period):
        if self.is_eternal:
            period = periods.period(ETERNITY)
        period = periods.period(period)

        values = self._files.get(period)
        if values is None:
            return None
        return self._decode_file(values)

    def put(self, value, period):
        if self.is_eternal:
            period = periods.period(ETERNITY)
        period = periods.period(period)

        filename = str(period)
        path = os.path.join(self.storage_dir, filename) + '.npy'
        if isinstance(value, EnumArray):
            self._enums[path] = value.possible_values
            value = value.view(np.ndarray)
        np.save(path, value)
        self._files[period] = path

    def delete(self, period = None):
        if period is None:
            self._files = {}
            return

        if self.is_eternal:
            period = periods.period(ETERNITY)
        period = periods.period(period)

        if period is not None:
            self._files = {
                period_item: value
                for period_item, value in self._files.items()
                if not period.contains(period_item)
                }

    def get_known_periods(self):
        return self._files.keys()

    def restore(self):
        self._files = files = {}
        # Restore self._files from content of storage_dir.
        for filename in os.listdir(self.storage_dir):
            if not filename.endswith('.npy'):
                continue
            path = os.path.join(self.storage_dir, filename)
            filename_core = filename.rsplit('.', 1)[0]
            period = periods.period(filename_core)
            files[period] = path

    def __del__(self):
        if self.preserve_storage_dir:
            return
        shutil.rmtree(self.storage_dir)  # Remove the holder temporary files
        # If the simulation temporary directory is empty, remove it
        parent_dir = os.path.abspath(os.path.join(self.storage_dir, os.pardir))
        if not os.listdir(parent_dir):
            shutil.rmtree(parent_dir)
<|endoftext|>


<commit_before>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (c) 2015, GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import numpy
from openquake.commonlib import InvalidFile

def parse_header(header):
    """
    Convert a list of the form `['fieldname:fieldtype:fieldsize',...]`
    into a numpy composite dtype. Here is an example:

    >>> parse_header(['PGA:float64:3', 'PGV:float64:2', 'avg:float:1'])
    dtype([('PGA', '<f8', (3,)), ('PGV', '<f8', (2,)), ('avg', '<f8', (1,))])

    :params header: a list of type descriptions
    :returns: the corresponding composite dtype
    """
    triples = []
    for col_str in header:
        col = col_str.split(':')
        field = col[0]
        numpytype = col[1]
        shape = () if not col[2].strip() else (int(col[2]),)
        triples.append((field, numpytype, shape))
    return numpy.dtype(triples)

def get_subtypes(dtype):
    """
    :param: a composite numpy dtype
    :returns: the subtypes contained in the dtype
    """
    sd = []
    for f in dtype.fields:
        dt = dtype.fields[f][0]
        sd.append((dt.subdtype[0].type if dt.subdtype else dt.type, dt.shape))
    return sd

def _cast_str(col, type, shape, lineno, fname):
    # an utility to convert strings into tuples or numbers
    try:
        if shape:
            return tuple(map(type, col.split()))
        else:
            return type(col)
    except:
        raise InvalidFile('Could not parse %r in file %s, line %d' % (
            col, fname, lineno))

# NB: this only works with flat composite arrays
def read_composite_array(fname, sep=','):
    r"""
    Convert a CSV file with header into a numpy array of records.

    >>> from openquake.baselib.general import writetmp
    >>> fname = writetmp('PGA:float64:3,PGV:float64:2,avg:float64:1\n'
    ...                  '.1 .2 .3,.4 .5,.6\n')
    >>> print read_composite_array(fname)  # array of shape (1,)
    [([0.1, 0.2, 0.3], [0.4, 0.5], [0.6])]
    """
    with open(fname) as f:
        header = next(f)
        dtype = parse_header(header.split(sep))
        subtypes = get_subtypes(dtype)
        col_ids = range(1, len(subtypes) + 1)
        num_columns = len(col_ids)
        records = []
        col, col_id = '', 0
        for i, line in enumerate(f, 2):
            row = line.split(sep)
            if len(row) != num_columns:
                raise InvalidFile(
                    'expected %d columns, found %d in file %s, line %d' %
                    (num_columns, len(row), fname, i))
            try:
                record = []
                for (type, shape), col, col_id in zip(subtypes, row, col_ids):
                    record.append(_cast_str(col, type, shape, i, fname))
                records.append(tuple(record))
            except Exception as e:
                raise InvalidFile(
                    'Could not cast %r in file %s, line %d, column %d '
                    'using %s: %s' % (col, fname, i, col_id,
                                      (type.__name__,) + shape), e)
        return numpy.array(records, dtype)

def read_array(fname, sep=','):
    r"""
    Convert a CSV file without header into a numpy array of floats.

    >>> from openquake.baselib.general import writetmp
    >>> print read_array(writetmp('.1 .2, .3 .4, .5 .6\n'))
    [[[ 0.1  0.2]
      [ 0.3  0.4]
      [ 0.5  0.6]]]
    """
    with open(fname) as f:
        records = []
        for line in f:
            row = line.split(sep)
            record = [map(float, col.split()) for col in row]
            records.append(record)
        return numpy.array(records)
<commit_msg>Small cleanup<commit_after>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (c) 2015, GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import numpy
from openquake.commonlib import InvalidFile

def parse_header(header):
    """
    Convert a list of the form `['fieldname:fieldtype:fieldsize',...]`
    into a numpy composite dtype. Here is an example:

    >>> parse_header(['PGA:float64:3', 'PGV:float64:2', 'avg:float:1'])
    dtype([('PGA', '<f8', (3,)), ('PGV', '<f8', (2,)), ('avg', '<f8', (1,))])

    :params header: a list of type descriptions
    :returns: the corresponding composite dtype
    """
    triples = []
    for col_str in header:
        col = col_str.split(':')
        field = col[0]
        numpytype = col[1]
        shape = () if not col[2].strip() else (int(col[2]),)
        triples.append((field, numpytype, shape))
    return numpy.dtype(triples)

def get_subtypes(dtype):
    """
    :param: a composite numpy dtype
    :returns: the subtypes contained in the dtype
    """
    sd = []
    for f in dtype.fields:
        dt = dtype.fields[f][0]
        sd.append((dt.subdtype[0].type if dt.subdtype else dt.type, dt.shape))
    return sd

def _cast_str(col, type, shape, lineno, fname):
    # convert strings into tuples or numbers, used inside read_composite_array
    if shape:
        return tuple(map(type, col.split()))
    else:
        return type(col)

# NB: this only works with flat composite arrays
def read_composite_array(fname, sep=','):
    r"""
    Convert a CSV file with header into a numpy array of records.

    >>> from openquake.baselib.general import writetmp
    >>> fname = writetmp('PGA:float64:3,PGV:float64:2,avg:float64:1\n'
    ...                  '.1 .2 .3,.4 .5,.6\n')
    >>> print read_composite_array(fname)  # array of shape (1,)
    [([0.1, 0.2, 0.3], [0.4, 0.5], [0.6])]
    """
    with open(fname) as f:
        header = next(f)
        dtype = parse_header(header.split(sep))
        subtypes = get_subtypes(dtype)
        col_ids = range(1, len(subtypes) + 1)
        num_columns = len(col_ids)
        records = []
        col, col_id = '', 0
        for i, line in enumerate(f, 2):
            row = line.split(sep)
            if len(row) != num_columns:
                raise InvalidFile(
                    'expected %d columns, found %d in file %s, line %d' %
                    (num_columns, len(row), fname, i))
            try:
                record = []
                for (type, shape), col, col_id in zip(subtypes, row, col_ids):
                    record.append(_cast_str(col, type, shape, i, fname))
                records.append(tuple(record))
            except Exception as e:
                raise InvalidFile(
                    'Could not cast %r in file %s, line %d, column %d '
                    'using %s: %s' % (col, fname, i, col_id,
                                      (type.__name__,) + shape), e)
        return numpy.array(records, dtype)

def read_array(fname, sep=','):
    r"""
    Convert a CSV file without header into a numpy array of floats.

    >>> from openquake.baselib.general import writetmp
    >>> print read_array(writetmp('.1 .2, .3 .4, .5 .6\n'))
    [[[ 0.1  0.2]
      [ 0.3  0.4]
      [ 0.5  0.6]]]
    """
    with open(fname) as f:
        records = []
        for line in f:
            row = line.split(sep)
            record = [map(float, col.split()) for col in row]
            records.append(record)
        return numpy.array(records)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# (C) 2016-2018 Ed Bueler

from __future__ import print_function
import numpy as np
import argparse

parser = argparse.ArgumentParser(description='Generate a .geo file for a Koch snowflake.')
parser.add_argument('-l', type=int, metavar='LEVEL', default=2,
                    help='recursion level for snowflake; default=2')
parser.add_argument('-o', default='koch.geo', metavar='FILE',
                    help='name for .geo output file; default=koch.geo')
args = parser.parse_args()
lev = args.l

N = 3 * 4**lev   # number of boundary nodes
print('creating level %d Koch snowflake polygon with %d segments ...' % (lev,N))

# generate string code for snowflake; see https://commons.wikimedia.org/wiki/Koch_snowflake
koch_flake = "FRFRF"
for i in range(lev):
    koch_flake = koch_flake.replace("F","FLFRFLF")

# draw by saving array of "turtle" positions
nodes = np.zeros((N,2))
pos = np.array([0.0,0.0])  # initial location; arbitrary because of rescale (below)
vec = np.array([1.0,0.0])  # initial direction
dist = 1.0  # edge length; really arbitrary because of rescale
count = 0
for move in koch_flake:
    if move == "F":
        nodes[count] = pos
        count += 1
        pos += dist * vec
    elif move == "L":
        cl = np.cos(np.pi/3.0)
        sl = np.sin(np.pi/3.0)
        vec = np.array([cl * vec[0] - sl * vec[1], sl * vec[0] + cl * vec[1]])
    elif move == "R":
        cl = np.cos(-2.0*np.pi/3.0)
        sl = np.sin(-2.0*np.pi/3.0)
        vec = np.array([cl * vec[0] - sl * vec[1], sl * vec[0] + cl * vec[1]])

# rescale so it is centered at (0,0) and fits into unit circle
c = np.average(nodes,axis=0)
for j in range(N):
    nodes[j] -= c
r = np.sqrt(nodes[:,0]**2+nodes[:,1]**2).max()
for j in range(N):
    nodes[j] /= r

# get length of a first segment
h0 = np.sqrt(sum((nodes[1,:] - nodes[0,:])**2))

# generate Gmsh-readable .geo file
print('writing Gmsh-readable file  %s  ...' % args.o)
f = open(args.o, 'w')
f.write('// level %d Koch domain with %d boundary segments\n\n' % (lev,N))
f.write('cl = %.10e; // characteristic length\n' % h0)
for j in range(N):
    f.write('Point(%d) = {%.10f,%.10f,0,cl};\n' % (j+1,nodes[j,0],nodes[j,1]))
for j in range(N-1):
    f.write('Line(%d) = {%d,%d};\n' % (N+1+j,j+1,j+2))
f.write('Line(%d) = {%d,1};\n' % (2*N,N))
f.write('Line Loop(%d) = {' % (2*N+1))
for j in range(N-1):
    f.write('%d,' % (N+1+j))
f.write('%d};\n' % (2*N))
f.write('Plane Surface(%d) = {%d};\n' % (2*N+2,2*N+1))
f.write('Physical Line("dirichlet") = {')
for j in range(N-1):
    f.write('%d,' % (N+1+j))
f.write('%d};\n' % (2*N))
f.write('Physical Line("neumann") = {};\n')
f.write('Physical Surface("interior") = {%d};\n' % (2*N+2))
f.close()

<commit_msg>add option to mark entire boundary as Neumann; useful for plotting<commit_after>#!/usr/bin/env python
#
# (C) 2016-2018 Ed Bueler

from __future__ import print_function
import numpy as np
import argparse

parser = argparse.ArgumentParser(description='Generate a .geo file for a Koch snowflake.')
parser.add_argument('--neumann', action='store_true', default=False,
                    help='mark entire boundary as Neumann (useful for plotting polygon only; see petsc2tikz.py)')
parser.add_argument('-l', type=int, metavar='LEVEL', default=2,
                    help='recursion level for snowflake; default=2')
parser.add_argument('-o', default='koch.geo', metavar='FILE',
                    help='name for .geo output file; default=koch.geo')
args = parser.parse_args()
lev = args.l

N = 3 * 4**lev   # number of boundary nodes
print('creating level %d Koch snowflake polygon with %d segments ...' % (lev,N))

# generate string code for snowflake; see https://commons.wikimedia.org/wiki/Koch_snowflake
koch_flake = "FRFRF"
for i in range(lev):
    koch_flake = koch_flake.replace("F","FLFRFLF")

# draw by saving array of "turtle" positions
nodes = np.zeros((N,2))
pos = np.array([0.0,0.0])  # initial location; arbitrary because of rescale (below)
vec = np.array([1.0,0.0])  # initial direction
dist = 1.0  # edge length; really arbitrary because of rescale
count = 0
for move in koch_flake:
    if move == "F":
        nodes[count] = pos
        count += 1
        pos += dist * vec
    elif move == "L":
        cl = np.cos(np.pi/3.0)
        sl = np.sin(np.pi/3.0)
        vec = np.array([cl * vec[0] - sl * vec[1], sl * vec[0] + cl * vec[1]])
    elif move == "R":
        cl = np.cos(-2.0*np.pi/3.0)
        sl = np.sin(-2.0*np.pi/3.0)
        vec = np.array([cl * vec[0] - sl * vec[1], sl * vec[0] + cl * vec[1]])

# rescale so it is centered at (0,0) and fits into unit circle
c = np.average(nodes,axis=0)
for j in range(N):
    nodes[j] -= c
r = np.sqrt(nodes[:,0]**2+nodes[:,1]**2).max()
for j in range(N):
    nodes[j] /= r

# get length of a first segment
h0 = np.sqrt(sum((nodes[1,:] - nodes[0,:])**2))

# generate Gmsh-readable .geo file
print('writing Gmsh-readable file  %s  ...' % args.o)
f = open(args.o, 'w')
f.write('// level %d Koch domain with %d boundary segments\n\n' % (lev,N))
f.write('cl = %.10e; // characteristic length\n' % h0)
for j in range(N):
    f.write('Point(%d) = {%.10f,%.10f,0,cl};\n' % (j+1,nodes[j,0],nodes[j,1]))
for j in range(N-1):
    f.write('Line(%d) = {%d,%d};\n' % (N+1+j,j+1,j+2))
f.write('Line(%d) = {%d,1};\n' % (2*N,N))
f.write('Line Loop(%d) = {' % (2*N+1))
for j in range(N-1):
    f.write('%d,' % (N+1+j))
f.write('%d};\n' % (2*N))
f.write('Plane Surface(%d) = {%d};\n' % (2*N+2,2*N+1))
if args.neumann:
    f.write('Physical Line("neumann") = {')
else:
    f.write('Physical Line("dirichlet") = {')
for j in range(N-1):
    f.write('%d,' % (N+1+j))
f.write('%d};\n' % (2*N))
if args.neumann:
    f.write('Physical Line("dirichlet") = {};\n')
else:
    f.write('Physical Line("neumann") = {};\n')
f.write('Physical Surface("interior") = {%d};\n' % (2*N+2))
f.close()

<|endoftext|>