<commit_before>"""Coordinates the twitter api with the markov chain models"""
import json
from pprint import pprint

from tweepy import Stream, OAuthHandler, API
from tweepy.streaming import StreamListener

from holy_markov import OldTestaPassagesMarkov
from twitter_secrets import api_tokens as at

class HolyListener(StreamListener):

    old_testa = OldTestaPassagesMarkov()
    bot_name = 'HolyStupidArt'

    def send_passage(self, screen_name):

        if screen_name != self.bot_name:
            print('Passage sent to @' + screen_name)
            passage = self.old_testa.twitter_message(line_length=(140 - len(screen_name) - 2))
            tweet = ''.join(['@', screen_name, ' ', passage])

            self.api.update_status(tweet)

    def on_connect(self):
        print("Connection established!!")

    def on_disconnect(self, notice):
        print("Connection lost!! : ", notice)

    def on_direct_message(self, status):
        try:
            dm = status._json['direct_message']
            self.send_passage(screen_name=dm['sender_screen_name'])
        except BaseException as e:
            print("Failed on_direct_message()", str(e))
            pprint(status._json)

        return True

    def on_event(self, status):
        print('Entered on_event()')
        print(status)

    def on_status(self, status):
        print('Entered on_status()')
        try:
            self.send_passage(screen_name=status._json['user']['screen_name'])
        except BaseException as e:
            print("Failed on_status()", str(e))
            pprint(status._json)
        return True

    def on_error(self, status):
        print('Entered on_error()')
        print(status)

def main():
    """The main event loop for the holy twitter bot

    It watches for twitter events, and posts randomly generated holy text to twitter.
    """
    auth = OAuthHandler(at['CONSUMER_KEY'], at['CONSUMER_SECRET'])
    auth.secure = True
    auth.set_access_token(at['ACCESS_KEY'], at['ACCESS_SECRET'])
    api = API(auth)

    # If the authentication was successful, you should
    # see the name of the account print out
    print(api.me().name)
    stream = Stream(auth, HolyListener(api=api))
    stream.userstream()

if __name__ == '__main__':
    main()
<commit_msg>Setup on_event to send passage upon a follow event<commit_after>"""Coordinates the twitter api with the markov chain models"""
import json
from pprint import pprint

from tweepy import Stream, OAuthHandler, API
from tweepy.streaming import StreamListener

from holy_markov import OldTestaPassagesMarkov
from twitter_secrets import api_tokens as at

class HolyListener(StreamListener):

    old_testa = OldTestaPassagesMarkov()
    bot_name = 'HolyStupidArt'

    def send_passage(self, screen_name):

        if screen_name != self.bot_name:
            print('Passage sent to @' + screen_name)
            passage = self.old_testa.twitter_message(line_length=(140 - len(screen_name) - 2))
            tweet = ''.join(['@', screen_name, ' ', passage])

            self.api.update_status(tweet)

    def on_connect(self):
        print("Connection established!!")

    def on_disconnect(self, notice):
        print("Connection lost!! : ", notice)

    def on_direct_message(self, status):
        try:
            dm = status._json['direct_message']
            self.send_passage(screen_name=dm['sender_screen_name'])
        except BaseException as e:
            print("Failed on_direct_message()", str(e))
            pprint(status._json)

        return True

    def on_event(self, status):
        try:
            event = status._json['event']
        except BaseException as e:
            print("Failed on_event()", str(e))
            pprint(status._json)
        else:
            if event == 'follow':
                self.send_passage(screen_name=status._json['source']['screen_name'])

    def on_status(self, status):
        print('Entered on_status()')
        try:
            self.send_passage(screen_name=status._json['user']['screen_name'])
        except BaseException as e:
            print("Failed on_status()", str(e))
            pprint(status._json)
        return True

    def on_error(self, status):
        print('Entered on_error()')
        print(status)

def main():
    """The main event loop for the holy twitter bot

    It watches for twitter events, and posts randomly generated holy text to twitter.
    """
    auth = OAuthHandler(at['CONSUMER_KEY'], at['CONSUMER_SECRET'])
    auth.secure = True
    auth.set_access_token(at['ACCESS_KEY'], at['ACCESS_SECRET'])
    api = API(auth)

    # If the authentication was successful, you should
    # see the name of the account print out
    print(api.me().name)
    stream = Stream(auth, HolyListener(api=api))
    stream.userstream()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Sun Apr 17 22:13:36 2011

@author: josef
"""

import numpy as np
from numpy.testing import assert_

from statsmodels.sandbox.distributions.extras import (skewnorm,
                skewnorm2, ACSkewT_gen)

def test_skewnorm():
    #library("sn")
    #dsn(c(-2,-1,0,1,2), shape=10)
    #psn(c(-2,-1,0,1,2), shape=10)
    #noquote(sprintf("%.15e,", snp))
    pdf_r = np.array([2.973416551551523e-90, 3.687562713971017e-24,
                      3.989422804014327e-01, 4.839414490382867e-01,
                      1.079819330263761e-01])
    pdf_sn = skewnorm.pdf([-2,-1,0,1,2], 10)

    #res = (snp-snp_r)/snp
    assert_(np.allclose(pdf_sn, pdf_r,rtol=1e-13, atol=0))

    pdf_sn2 = skewnorm2.pdf([-2,-1,0,1,2], 10)
    assert_(np.allclose(pdf_sn2, pdf_r, rtol=1e-13, atol=0))

    cdf_r = np.array([0.000000000000000e+00, 0.000000000000000e+00,
                      3.172551743055357e-02, 6.826894921370859e-01,
                      9.544997361036416e-01])
    cdf_sn = skewnorm.cdf([-2,-1,0,1,2], 10)
    assert_(np.allclose(cdf_sn, cdf_r, rtol=1e-13, atol=1e-25))

    cdf_sn2 = skewnorm2.cdf([-2,-1,0,1,2], 10)
    assert_(np.allclose(cdf_sn2, cdf_r, rtol=1e-13, atol=1e-25))

def test_skewt():
    skewt = ACSkewT_gen()
    x = [-2, -1, -0.5, 0, 1, 2]
    #noquote(sprintf("%.15e,", dst(c(-2,-1, -0.5,0,1,2), shape=10)))
    #default in R:sn is df=inf
    pdf_r = np.array([2.973416551551523e-90, 3.687562713971017e-24,
                      2.018401586422970e-07, 3.989422804014327e-01,
                      4.839414490382867e-01, 1.079819330263761e-01])
    pdf_st = skewt.pdf(x, 1000000, 10)
    pass
    np.allclose(pdf_st, pdf_r, rtol=0, atol=1e-6)
    np.allclose(pdf_st, pdf_r, rtol=1e-1, atol=0)

    #noquote(sprintf("%.15e,", pst(c(-2,-1, -0.5,0,1,2), shape=10)))
    cdf_r = np.array([0.000000000000000e+00, 0.000000000000000e+00,
                      3.729478836866917e-09, 3.172551743055357e-02,
                      6.826894921370859e-01, 9.544997361036416e-01])
    cdf_st = skewt.cdf(x, 1000000, 10)
    np.allclose(cdf_st, cdf_r, rtol=0, atol=1e-6)
    np.allclose(cdf_st, cdf_r, rtol=1e-1, atol=0)
    #assert_(np.allclose(cdf_st, cdf_r, rtol=1e-13, atol=1e-15))

    #noquote(sprintf("%.15e,", dst(c(-2,-1, -0.5,0,1,2), shape=10, df=5)))
    pdf_r = np.array([2.185448836190663e-07, 1.272381597868587e-05,
                      5.746937644959992e-04, 3.796066898224945e-01,
                      4.393468708859825e-01, 1.301804021075493e-01])
    pdf_st = skewt.pdf(x, 5, 10)  #args = (df, alpha)
    assert_(np.allclose(pdf_st, pdf_r, rtol=1e-13, atol=1e-25))

    #noquote(sprintf("%.15e,", pst(c(-2,-1, -0.5,0,1,2), shape=10, df=5)))
    cdf_r = np.array([8.822783669199699e-08, 2.638467463775795e-06,
                      6.573106017198583e-05, 3.172551743055352e-02,
                      6.367851708183412e-01, 8.980606093979784e-01])
    cdf_st = skewt.cdf(x, 5, 10)  #args = (df, alpha)
    assert_(np.allclose(cdf_st, cdf_r, rtol=1e-10, atol=0))

    #noquote(sprintf("%.15e,", dst(c(-2,-1, -0.5,0,1,2), shape=10, df=1)))
    pdf_r = np.array([3.941955996757291e-04, 1.568067236862745e-03,
                      6.136996029432048e-03, 3.183098861837907e-01,
                      3.167418189469279e-01, 1.269297588738406e-01])
    pdf_st = skewt.pdf(x, 1, 10)  #args = (df, alpha) = (1, 10))
    assert_(np.allclose(pdf_st, pdf_r, rtol=1e-13, atol=1e-25))

    #noquote(sprintf("%.15e,", pst(c(-2,-1, -0.5,0,1,2), shape=10, df=1)))
    cdf_r = np.array([7.893671370544414e-04, 1.575817262600422e-03,
                      3.128720749105560e-03, 3.172551743055351e-02,
                      5.015758172626005e-01, 7.056221318361879e-01])
    cdf_st = skewt.cdf(x, 1, 10)  #args = (df, alpha) = (1, 10)
    assert_(np.allclose(cdf_st, cdf_r, rtol=1e-13, atol=1e-25))

if __name__ == '__main__':
    import nose
    nose.runmodule(argv=['__main__','-vvs','-x','--pdb', '--pdb-failure'],
                   exit=False)

    print ('Done')

'''
>>> skewt.pdf([-2,-1,0,1,2], 10000000, 10)
array([  2.98557345e-90,   3.68850289e-24,   3.98942271e-01,
         4.83941426e-01,   1.07981952e-01])
>>> skewt.pdf([-2,-1,0,1,2], np.inf, 10)
array([ nan,  nan,  nan,  nan,  nan])
'''
<commit_msg>TST: skewnorm: lower precision, change to assert_almost_equal<commit_after># -*- coding: utf-8 -*-
"""
Created on Sun Apr 17 22:13:36 2011

@author: josef
"""

import numpy as np
from numpy.testing import assert_, assert_almost_equal

from statsmodels.sandbox.distributions.extras import (skewnorm,
                skewnorm2, ACSkewT_gen)

def test_skewnorm():
    #library("sn")
    #dsn(c(-2,-1,0,1,2), shape=10)
    #psn(c(-2,-1,0,1,2), shape=10)
    #noquote(sprintf("%.15e,", snp))
    pdf_r = np.array([2.973416551551523e-90, 3.687562713971017e-24,
                      3.989422804014327e-01, 4.839414490382867e-01,
                      1.079819330263761e-01])
    pdf_sn = skewnorm.pdf([-2,-1,0,1,2], 10)

    #res = (snp-snp_r)/snp
    assert_(np.allclose(pdf_sn, pdf_r,rtol=1e-13, atol=0))

    pdf_sn2 = skewnorm2.pdf([-2,-1,0,1,2], 10)
    assert_(np.allclose(pdf_sn2, pdf_r, rtol=1e-13, atol=0))

    cdf_r = np.array([0.000000000000000e+00, 0.000000000000000e+00,
                      3.172551743055357e-02, 6.826894921370859e-01,
                      9.544997361036416e-01])
    cdf_sn = skewnorm.cdf([-2,-1,0,1,2], 10)
    maxabs = np.max(np.abs(cdf_sn - cdf_r))
    maxrel = np.max(np.abs(cdf_sn - cdf_r)/(cdf_r+1e-50))
    msg = "maxabs=%15.13g, maxrel=%15.13g\n%r\n%r" % (maxabs, maxrel, cdf_sn,
                                                       cdf_r)
    #assert_(np.allclose(cdf_sn, cdf_r, rtol=1e-13, atol=1e-25), msg=msg)
    assert_almost_equal(cdf_sn, cdf_r, decimal=10)

    cdf_sn2 = skewnorm2.cdf([-2,-1,0,1,2], 10)
    maxabs = np.max(np.abs(cdf_sn2 - cdf_r))
    maxrel = np.max(np.abs(cdf_sn2 - cdf_r)/(cdf_r+1e-50))
    msg = "maxabs=%15.13g, maxrel=%15.13g" % (maxabs, maxrel)
    #assert_(np.allclose(cdf_sn2, cdf_r, rtol=1e-13, atol=1e-25), msg=msg)
    assert_almost_equal(cdf_sn2, cdf_r, decimal=10, err_msg=msg)

def test_skewt():
    skewt = ACSkewT_gen()
    x = [-2, -1, -0.5, 0, 1, 2]
    #noquote(sprintf("%.15e,", dst(c(-2,-1, -0.5,0,1,2), shape=10)))
    #default in R:sn is df=inf
    pdf_r = np.array([2.973416551551523e-90, 3.687562713971017e-24,
                      2.018401586422970e-07, 3.989422804014327e-01,
                      4.839414490382867e-01, 1.079819330263761e-01])
    pdf_st = skewt.pdf(x, 1000000, 10)
    pass
    np.allclose(pdf_st, pdf_r, rtol=0, atol=1e-6)
    np.allclose(pdf_st, pdf_r, rtol=1e-1, atol=0)

    #noquote(sprintf("%.15e,", pst(c(-2,-1, -0.5,0,1,2), shape=10)))
    cdf_r = np.array([0.000000000000000e+00, 0.000000000000000e+00,
                      3.729478836866917e-09, 3.172551743055357e-02,
                      6.826894921370859e-01, 9.544997361036416e-01])
    cdf_st = skewt.cdf(x, 1000000, 10)
    np.allclose(cdf_st, cdf_r, rtol=0, atol=1e-6)
    np.allclose(cdf_st, cdf_r, rtol=1e-1, atol=0)
    #assert_(np.allclose(cdf_st, cdf_r, rtol=1e-13, atol=1e-15))

    #noquote(sprintf("%.15e,", dst(c(-2,-1, -0.5,0,1,2), shape=10, df=5)))
    pdf_r = np.array([2.185448836190663e-07, 1.272381597868587e-05,
                      5.746937644959992e-04, 3.796066898224945e-01,
                      4.393468708859825e-01, 1.301804021075493e-01])
    pdf_st = skewt.pdf(x, 5, 10)  #args = (df, alpha)
    assert_(np.allclose(pdf_st, pdf_r, rtol=1e-13, atol=1e-25))

    #noquote(sprintf("%.15e,", pst(c(-2,-1, -0.5,0,1,2), shape=10, df=5)))
    cdf_r = np.array([8.822783669199699e-08, 2.638467463775795e-06,
                      6.573106017198583e-05, 3.172551743055352e-02,
                      6.367851708183412e-01, 8.980606093979784e-01])
    cdf_st = skewt.cdf(x, 5, 10)  #args = (df, alpha)
    assert_(np.allclose(cdf_st, cdf_r, rtol=1e-10, atol=0))

    #noquote(sprintf("%.15e,", dst(c(-2,-1, -0.5,0,1,2), shape=10, df=1)))
    pdf_r = np.array([3.941955996757291e-04, 1.568067236862745e-03,
                      6.136996029432048e-03, 3.183098861837907e-01,
                      3.167418189469279e-01, 1.269297588738406e-01])
    pdf_st = skewt.pdf(x, 1, 10)  #args = (df, alpha) = (1, 10))
    assert_(np.allclose(pdf_st, pdf_r, rtol=1e-13, atol=1e-25))

    #noquote(sprintf("%.15e,", pst(c(-2,-1, -0.5,0,1,2), shape=10, df=1)))
    cdf_r = np.array([7.893671370544414e-04, 1.575817262600422e-03,
                      3.128720749105560e-03, 3.172551743055351e-02,
                      5.015758172626005e-01, 7.056221318361879e-01])
    cdf_st = skewt.cdf(x, 1, 10)  #args = (df, alpha) = (1, 10)
    assert_(np.allclose(cdf_st, cdf_r, rtol=1e-13, atol=1e-25))

if __name__ == '__main__':
    import nose
    nose.runmodule(argv=['__main__','-vvs','-x','--pdb', '--pdb-failure'],
                   exit=False)

    print ('Done')

'''
>>> skewt.pdf([-2,-1,0,1,2], 10000000, 10)
array([  2.98557345e-90,   3.68850289e-24,   3.98942271e-01,
         4.83941426e-01,   1.07981952e-01])
>>> skewt.pdf([-2,-1,0,1,2], np.inf, 10)
array([ nan,  nan,  nan,  nan,  nan])
'''
<|endoftext|>


<commit_before>import dateutil.parser
import json
import re
import requests
import sys
from requests.exceptions import HTTPError, ConnectTimeout

knownWindowsLayers = {
    "sha256:3889bb8d808bbae6fa5a33e07093e65c31371bcf9e4c38c21be6b9af52ad1548": "microsoft/windowsservercore:10.0.14393.576 update",
    "sha256:04ee5d718c7adc0144556d740900f778129e41be806c95191710d1d92051a7b3": "microsoft/windowsservercore:10.0.14393.576 base",
    "sha256:d33fff6043a134da85e10360f9932543f1dfc0c3a22e1edd062aa9b088a86c5b": "microsoft/windowsservercore:10.0.14393.447 update",
    "sha256:de5064718b3f2749727c8b5ffddf2da7698189277afe0df6fc0a57ad573bca0f": "microsoft/windowsservercore:10.0.14393.321 update",
    "sha256:9c7f9c7d9bc2915388ecc5d08e89a7583658285469d7325281f95d8ee279cc60": "microsoft/windowsservercore:10.0.14393.206 base",
    "sha256:1239394e5a8ab79fbd3b751dc5d98decf5886f14339958fdf5c1f96c89da58a7": "microsoft/windowsservercore:10.0.14300.1030",

    "sha256:10bf725c5388a1909f7184467b5ec75dbad3ece68508aa5fa4074baa0b20cc6f": "microsoft/nanoserver:10.0.14393.576 update",
    "sha256:bce2fbc256ea437a87dadac2f69aabd25bed4f56255549090056c1131fad0277": "microsoft/nanoserver:10.0.14393.576 base",
    "sha256:482ab31872a23b32cbdeca13edb7a0b97290714c0b5edcce96fbb3e34221ea91": "microsoft/nanoserver:10.0.14393.447 update",
    "sha256:94b4ce7ac4c7c7d4ed75ac2bd9359a87204ad2d5a909759d8e77953874d8e7c2": "microsoft/nanoserver:10.0.14393.321 update",
    "sha256:5496abde368a3dd39999745bf998c877ddc6a390a943bc3fd99ffaabf728ed88": "microsoft/nanoserver:10.0.14393.206 base",
    "sha256:cf62dbf6d334601f3e026a976218e4d73641ab8c3e66a842a4e4dbdc72768b18": "microsoft/nanoserver:10.0.14300.1030",
}

class DockerImageInspector(object):

    def __init__(self, registry_hostname, repository_name, tag="latest"):
        self.base_url = "https://%s" % registry_hostname
        self.repository_name = repository_name
        self.token = None
        self.tag = tag
        self.create_date = None
        self.create_os = None
        self.create_docker_version = None
        self.create_os_version = ""
        self.layers = []
        self.tags = []
        self.manifest = None
        self.get_tags()
        self.get_manifest()

    def get_tags(self):
        url = "{base_url}/v2/{name}/tags/list".format(
                base_url=self.base_url,
                name=self.repository_name)
        headers = {}
        if self.token is not None:
            headers["Authorization"] = "Bearer %s" % self.token
        r = requests.get(url, headers=headers, timeout=(3.05,10))
        r.raise_for_status()
        self.tags = r.json()["tags"]

    def get_manifest(self):
        url = "{base_url}/v2/{name}/manifests/{reference}".format(
                base_url=self.base_url,
                name=self.repository_name,
                reference=self.tag)
        headers = {}
        if self.token is not None:
            headers["Authorization"] = "Bearer %s" % self.token
        r = requests.get(url, headers=headers, timeout=(3.05,10))
        r.raise_for_status()
        self.manifest = r.json()
        if "history" in self.manifest:
            if "v1Compatibility" in self.manifest["history"][0]:
                hist = json.loads(self.manifest["history"][0]["v1Compatibility"])
                self.create_date = dateutil.parser.parse(hist["created"])
                self.create_os = hist["os"]
                if "os.version" in hist:
                    self.create_os_version = hist["os.version"]
                self.create_docker_version = hist["docker_version"]
        self.manifest_content_type = r.headers["content-type"]
        headers["Accept"] = "application/vnd.docker.distribution.manifest.v2+json"
        r = requests.get(url, headers=headers, timeout=(3.05,10))
        r.raise_for_status()
        manifest = r.json()
        self.manifest["layers"] = manifest["layers"]
        for l in self.manifest["layers"]:
            self.layers.append(l)

class DockerHubImageInspector(DockerImageInspector):

    def __init__(self, repository_name, tag="latest"):
        self.base_url = "https://registry.hub.docker.com"
        self.repository_name = repository_name
        self.__get_token()
        self.tag = tag
        self.layers = []
        self.tags = []
        self.manifest = None
        self.create_os_version = ""
        self.get_tags()
        self.get_manifest()

    def __get_token(self):
        url = "https://auth.docker.io/token"
        params = {
            "service": "registry.docker.io",
            "scope": "repository:{name}:pull".format(
                name=self.repository_name)
        }
        r = requests.get(url, params=params)
        r.raise_for_status()
        self.token = r.json()["token"]

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description='Inspect an image from DockerHub')
    parser.add_argument('image', metavar='IMAGE',
                        help='The image to look for, e.g. "library/redis"')

    args = parser.parse_args()
    if ":" in args.image:
        (image, tag) = args.image.split(":")
    else:
        image = args.image
        tag = "latest"

    image_parts = image.split("/")
    registry = None
    namespace = None
    repository = None
    if len(image_parts) == 3:
        registry = image_parts[-3]
    else:
        registry = "index.docker.io"
    if len(image_parts) >= 2:
        namespace = image_parts[-2]
    else:
        namespace = "library"
    repository = image_parts[-1]

    image = "/".join([registry, namespace, repository])

    try:
        if registry == "index.docker.io":
            dii = DockerHubImageInspector(namespace + "/" + repository, tag)
        else:
            dii = DockerImageInspector(registry, namespace + "/" + repository, tag)
    except HTTPError, e:
        sys.stderr.write(str(e) + "\n")
        sys.exit(1)
    except ConnectTimeout, e:
        sys.stderr.write(str(e) + "\n")
        sys.exit(2)
    except Exception, e:
        sys.stderr.write("Unknown error: " + str(e) + "\n")
        sys.exit(3)

    print("Image name: %s" % dii.repository_name)
    print("Tag: %s" % dii.tag)
    print("Number of layers: %d" % len(dii.layers))
    print("Schema version: %s" % dii.manifest["schemaVersion"])
    print("Architecture: %s" % dii.manifest["architecture"])
    print("Number of history entries: %d" % len(dii.manifest["history"]))
    print("Created: %s with Docker %s on %s %s" % (dii.create_date.strftime("%Y-%m-%d %H:%M:%S"), dii.create_docker_version, dii.create_os, dii.create_os_version))
    totalSize=0
    appSize=0
    print("Sizes of layers:")
    for l in dii.layers:
        print("  %s - %s byte" % (l["digest"], l["size"]))
        totalSize += l["size"]
        if l["mediaType"] != "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip":
            appSize += l["size"]
    print("Total size (including Windows base layers): %s byte" % totalSize)
    print("Application size (w/o Windows base layers): %s byte" % appSize)
    print("Windows base image used:")
    if dii.create_os != "windows":
        print("  It does not seem to be a Windows image")
    for l in dii.layers:
        if l["mediaType"] == "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip":
            if l["digest"] in knownWindowsLayers:
                print("  %s" % knownWindowsLayers[l["digest"]])
            else:
                print("  %s - unknown Windows layer" % l["digest"])
    print("History:")
    for h in reversed(dii.manifest["history"]):
        hist = json.loads(h["v1Compatibility"])["container_config"]
        if "(nop)" in json.dumps(hist["Cmd"]):
            cmd = hist["Cmd"][-1]
            cmd = re.sub(r".*\(nop\)\s*", "", cmd)
            if "(nop)" in hist["Cmd"][-1]:
                print("  %s" % cmd)
            else:
                print("  %s" % cmd)
        else:
            print("  RUN %s" % json.dumps(hist["Cmd"]))
<commit_msg>Fix for Windows base images<commit_after>import dateutil.parser
import json
import re
import requests
import sys
from requests.exceptions import HTTPError, ConnectTimeout

knownWindowsLayers = {
    "sha256:3889bb8d808bbae6fa5a33e07093e65c31371bcf9e4c38c21be6b9af52ad1548": "microsoft/windowsservercore:10.0.14393.576 update",
    "sha256:04ee5d718c7adc0144556d740900f778129e41be806c95191710d1d92051a7b3": "microsoft/windowsservercore:10.0.14393.576 base",
    "sha256:d33fff6043a134da85e10360f9932543f1dfc0c3a22e1edd062aa9b088a86c5b": "microsoft/windowsservercore:10.0.14393.447 update",
    "sha256:de5064718b3f2749727c8b5ffddf2da7698189277afe0df6fc0a57ad573bca0f": "microsoft/windowsservercore:10.0.14393.321 update",
    "sha256:9c7f9c7d9bc2915388ecc5d08e89a7583658285469d7325281f95d8ee279cc60": "microsoft/windowsservercore:10.0.14393.206 base",
    "sha256:1239394e5a8ab79fbd3b751dc5d98decf5886f14339958fdf5c1f96c89da58a7": "microsoft/windowsservercore:10.0.14300.1030",

    "sha256:10bf725c5388a1909f7184467b5ec75dbad3ece68508aa5fa4074baa0b20cc6f": "microsoft/nanoserver:10.0.14393.576 update",
    "sha256:bce2fbc256ea437a87dadac2f69aabd25bed4f56255549090056c1131fad0277": "microsoft/nanoserver:10.0.14393.576 base",
    "sha256:482ab31872a23b32cbdeca13edb7a0b97290714c0b5edcce96fbb3e34221ea91": "microsoft/nanoserver:10.0.14393.447 update",
    "sha256:94b4ce7ac4c7c7d4ed75ac2bd9359a87204ad2d5a909759d8e77953874d8e7c2": "microsoft/nanoserver:10.0.14393.321 update",
    "sha256:5496abde368a3dd39999745bf998c877ddc6a390a943bc3fd99ffaabf728ed88": "microsoft/nanoserver:10.0.14393.206 base",
    "sha256:cf62dbf6d334601f3e026a976218e4d73641ab8c3e66a842a4e4dbdc72768b18": "microsoft/nanoserver:10.0.14300.1030",
}

class DockerImageInspector(object):

    def __init__(self, registry_hostname, repository_name, tag="latest"):
        self.base_url = "https://%s" % registry_hostname
        self.repository_name = repository_name
        self.token = None
        self.tag = tag
        self.create_date = None
        self.create_os = None
        self.create_docker_version = None
        self.create_os_version = ""
        self.layers = []
        self.tags = []
        self.manifest = None
        self.get_tags()
        self.get_manifest()

    def get_tags(self):
        url = "{base_url}/v2/{name}/tags/list".format(
                base_url=self.base_url,
                name=self.repository_name)
        headers = {}
        if self.token is not None:
            headers["Authorization"] = "Bearer %s" % self.token
        r = requests.get(url, headers=headers, timeout=(3.05,10))
        r.raise_for_status()
        self.tags = r.json()["tags"]

    def get_manifest(self):
        url = "{base_url}/v2/{name}/manifests/{reference}".format(
                base_url=self.base_url,
                name=self.repository_name,
                reference=self.tag)
        headers = {}
        if self.token is not None:
            headers["Authorization"] = "Bearer %s" % self.token
        try:
            r = requests.get(url, headers=headers, timeout=(3.05,10))
            r.raise_for_status()
            self.manifest = r.json()
            if "history" in self.manifest:
                if "v1Compatibility" in self.manifest["history"][0]:
                    hist = json.loads(self.manifest["history"][0]["v1Compatibility"])
                    self.create_date = dateutil.parser.parse(hist["created"])
                    self.create_os = hist["os"]
                    if "os.version" in hist:
                        self.create_os_version = hist["os.version"]
                    self.create_docker_version = hist["docker_version"]
            self.manifest_content_type = r.headers["content-type"]
        except:
            # seems we directly check a Windows base image
            self.manifest = {}
            self.create_os = "windows"
        headers["Accept"] = "application/vnd.docker.distribution.manifest.v2+json"
        r = requests.get(url, headers=headers, timeout=(3.05,10))
        r.raise_for_status()
        manifest = r.json()
        self.manifest["layers"] = manifest["layers"]
        for l in self.manifest["layers"]:
            self.layers.append(l)

class DockerHubImageInspector(DockerImageInspector):

    def __init__(self, repository_name, tag="latest"):
        self.base_url = "https://registry.hub.docker.com"
        self.repository_name = repository_name
        self.__get_token()
        self.tag = tag
        self.layers = []
        self.tags = []
        self.manifest = None
        self.create_os_version = ""
        self.get_tags()
        self.get_manifest()

    def __get_token(self):
        url = "https://auth.docker.io/token"
        params = {
            "service": "registry.docker.io",
            "scope": "repository:{name}:pull".format(
                name=self.repository_name)
        }
        r = requests.get(url, params=params)
        r.raise_for_status()
        self.token = r.json()["token"]

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description='Inspect an image from DockerHub')
    parser.add_argument('image', metavar='IMAGE',
                        help='The image to look for, e.g. "library/redis"')

    args = parser.parse_args()
    if ":" in args.image:
        (image, tag) = args.image.split(":")
    else:
        image = args.image
        tag = "latest"

    image_parts = image.split("/")
    registry = None
    namespace = None
    repository = None
    if len(image_parts) == 3:
        registry = image_parts[-3]
    else:
        registry = "index.docker.io"
    if len(image_parts) >= 2:
        namespace = image_parts[-2]
    else:
        namespace = "library"
    repository = image_parts[-1]

    image = "/".join([registry, namespace, repository])

    try:
        if registry == "index.docker.io":
            dii = DockerHubImageInspector(namespace + "/" + repository, tag)
        else:
            dii = DockerImageInspector(registry, namespace + "/" + repository, tag)
    except HTTPError, e:
        sys.stderr.write(str(e) + "\n")
        sys.exit(1)
    except ConnectTimeout, e:
        sys.stderr.write(str(e) + "\n")
        sys.exit(2)
    except Exception, e:
        sys.stderr.write("Unknown error: " + str(e) + "\n")
        sys.exit(3)

    print("Image name: %s" % dii.repository_name)
    print("Tag: %s" % dii.tag)
    print("Number of layers: %d" % len(dii.layers))
    if "schemaVersion" in dii.manifest:
        print("Schema version: %s" % dii.manifest["schemaVersion"])
        print("Architecture: %s" % dii.manifest["architecture"])
        print("Created: %s with Docker %s on %s %s" % (dii.create_date.strftime("%Y-%m-%d %H:%M:%S"), dii.create_docker_version, dii.create_os, dii.create_os_version))
    totalSize=0
    appSize=0
    print("Sizes of layers:")
    for l in dii.layers:
        print("  %s - %s byte" % (l["digest"], l["size"]))
        totalSize += l["size"]
        if l["mediaType"] != "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip":
            appSize += l["size"]
    print("Total size (including Windows base layers): %s byte" % totalSize)
    print("Application size (w/o Windows base layers): %s byte" % appSize)
    print("Windows base image used:")
    if dii.create_os != "windows":
        print("  It does not seem to be a Windows image")
    for l in dii.layers:
        if l["mediaType"] == "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip":
            if l["digest"] in knownWindowsLayers:
                print("  %s" % knownWindowsLayers[l["digest"]])
            else:
                print("  %s - unknown Windows layer" % l["digest"])
    if "history" in dii.manifest:
        print("History:")
        for h in reversed(dii.manifest["history"]):
            hist = json.loads(h["v1Compatibility"])["container_config"]
            if "(nop)" in json.dumps(hist["Cmd"]):
                cmd = hist["Cmd"][-1]
                cmd = re.sub(r".*\(nop\)\s*", "", cmd)
                if "(nop)" in hist["Cmd"][-1]:
                    print("  %s" % cmd)
                else:
                    print("  %s" % cmd)
            else:
                print("  RUN %s" % json.dumps(hist["Cmd"]))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import sys
import json
import time
import requests
import datetime

REPO_URI = 'http://{es_node}:9200/_snapshot/{repo}'

def create_s3_repo(node, repo, bucket):
    _sys_print('Creating S3 repository...')

    repository_config = {
        "type": "s3",
        "settings": {
            "bucket": bucket,
            "base_path": "elasticsearch_snapshots"
        }
    }

    r = requests.put(REPO_URI.format(es_node=node, repo=repo),
                     data=json.dumps(repository_config))

    result = r.json()
    assert r.status_code == 200, result
    if not result['acknowledged']:
        raise Exception(
            'Was not able to create s3 repository!\nReason: {}'.format(result))
        exit(1)
    _sys_print(' [OK]\n')

def poll_snapshot_status(snapshot_uri):
    status_uri = "{snapshot_uri}/_status".format(snapshot_uri=snapshot_uri)
    failure_count = 0
    sleep_time = 1
    print "Files:"
    counter = 0
    spinner = ['|', '/', '-', '\\']
    while True:
        r = requests.get(status_uri)

        if r.status_code == 200:
            snapshot_status = r.json()
            status = snapshot_status['snapshots'][0]['state']
            if status == 'STARTED':
                stats = snapshot_status['snapshots'][0]['stats']
                _sys_print(
                    '\r{}/{} '.format(
                        stats['processed_files'], stats['number_of_files']))
            elif status == 'SUCCESS':
                return print_success_message(snapshot_status)
            else:
                raise Exception('Bad snapshot status: {}'.format(r.json()))
        else:
            failure_count += 1
            sleep_time = sleep_time * 2

        if failure_count > 10:
            raise Exception(
                'Bad status code {} {}'.format(r.status_code, r.content))
        counter += 1
        _sys_print(spinner[counter % len(spinner)])
        time.sleep(sleep_time)

def create_snapshot(node, repo):
    date_stamp = datetime.datetime.now().date().strftime('%Y%m%d')
    _sys_print('Creating snapshot for {} ...'.format(date_stamp))

    snapshot_uri = '{repo}/snapshot_{date}'.format(
        repo=REPO_URI.format(es_node=node, repo=repo), date=date_stamp)

    r = requests.put(snapshot_uri)

    result = r.json()
    assert r.status_code == 200, result

    if not result['accepted']:
        raise Exception('Snapshot Failed\nReason: {}'.format(result))

    _sys_print(' [OK]\n')

    poll_snapshot_status(snapshot_uri)

def print_success_message(snapshot_status):
    stats = snapshot_status['snapshots'][0]['stats']
    shard_stats = snapshot_status['snapshots'][0]['shards_stats']
    print '\n'
    print '-'*80
    print 'Stats: Files: {}/{}; Shards - {}/{}; Duration - {}s'.format(
        stats['processed_files'], stats['number_of_files'],
        shard_stats['done'], shard_stats['total'],
        stats['time_in_millis']/100)
    print '-'*80

def _sys_print(m):
    sys.stdout.write(m)
    sys.stdout.flush()
<commit_msg>add seconds to timestamp in snapshot name<commit_after># -*- coding: utf-8 -*-
import sys
import json
import time
import requests
import datetime

REPO_URI = 'http://{es_node}:9200/_snapshot/{repo}'

def create_s3_repo(node, repo, bucket):
    _sys_print('Creating S3 repository...')

    repository_config = {
        "type": "s3",
        "settings": {
            "bucket": bucket,
            "base_path": "elasticsearch_snapshots"
        }
    }

    r = requests.put(REPO_URI.format(es_node=node, repo=repo),
                     data=json.dumps(repository_config))

    result = r.json()
    assert r.status_code == 200, result
    if not result['acknowledged']:
        raise Exception(
            'Was not able to create s3 repository!\nReason: {}'.format(result))
        exit(1)
    _sys_print(' [OK]\n')

def poll_snapshot_status(snapshot_uri):
    status_uri = "{snapshot_uri}/_status".format(snapshot_uri=snapshot_uri)
    failure_count = 0
    sleep_time = 1
    print "Files:"
    counter = 0
    spinner = ['|', '/', '-', '\\']
    while True:
        r = requests.get(status_uri)

        if r.status_code == 200:
            snapshot_status = r.json()
            status = snapshot_status['snapshots'][0]['state']
            if status == 'STARTED':
                stats = snapshot_status['snapshots'][0]['stats']
                _sys_print(
                    '\r{}/{} '.format(
                        stats['processed_files'], stats['number_of_files']))
            elif status == 'SUCCESS':
                return print_success_message(snapshot_status)
            else:
                raise Exception('Bad snapshot status: {}'.format(r.json()))
        else:
            failure_count += 1
            sleep_time = sleep_time * 2

        if failure_count > 10:
            raise Exception(
                'Bad status code {} {}'.format(r.status_code, r.content))
        counter += 1
        _sys_print(spinner[counter % len(spinner)])
        time.sleep(sleep_time)

def create_snapshot(node, repo):
    date_stamp = datetime.datetime.now().strftime('%Y%m%d_%s')
    _sys_print('Creating snapshot for {} ...'.format(date_stamp))

    snapshot_uri = '{repo}/snapshot_{date}'.format(
        repo=REPO_URI.format(es_node=node, repo=repo), date=date_stamp)

    r = requests.put(snapshot_uri)

    result = r.json()
    assert r.status_code == 200, result

    if not result['accepted']:
        raise Exception('Snapshot Failed\nReason: {}'.format(result))

    _sys_print(' [OK]\n')

    poll_snapshot_status(snapshot_uri)

def print_success_message(snapshot_status):
    stats = snapshot_status['snapshots'][0]['stats']
    shard_stats = snapshot_status['snapshots'][0]['shards_stats']
    print '\n'
    print '-'*80
    print 'Stats: Files: {}/{}; Shards - {}/{}; Duration - {}s'.format(
        stats['processed_files'], stats['number_of_files'],
        shard_stats['done'], shard_stats['total'],
        stats['time_in_millis']/100)
    print '-'*80

def _sys_print(m):
    sys.stdout.write(m)
    sys.stdout.flush()
<|endoftext|>


<commit_before>#!/usr/bin/python
__version__ = "0.0.1"
__author__ = "david"
__email__ = "db@d1b.org"

import argparse
import csv
import json
from multiprocessing import Process
import os
import socket
import time

import netaddr
from netaddr import EUI
from scapy.all import sniff, ARP

def get_hostname(ip):
	try:
		ret = socket.gethostbyaddr(ip)
		if ret:
			return ret[0]
	except socket.error:
		return ""

def persistent_pkt_info(filename, ** kwargs):
	with open(filename, 'a+') as csvfile:
		csv_w = csv.writer(csvfile)
		csv_w.writerow([kwargs['hwsrc'], json.dumps(kwargs)])

def handle_new_pkt(pkt, ** kwargs):
	hwsrc = kwargs['hwsrc']
	src_ip = pkt[ARP].psrc
	hostname = get_hostname(src_ip)
	try:
		org = EUI(hwsrc).oui.registration().org
	except netaddr.core.NotRegisteredError:
		org = 'NA'
	print src_ip, hwsrc, hostname, org
	filename = kwargs.get('filename', None)
	if filename:
		cur_time = time.time()
		persistent_pkt_info(filename, hwsrc=hwsrc, src_ip=src_ip,
			time=cur_time, org=org, hostname=hostname)

class SimpleNetworkMonitor(object):
	def __init__(self, filename, ** kwargs):
		self.filename = filename
		self.csv_file = self._get_csv_file()
		self.load_known_from_persist()

	def _get_csv_file(self):
		return open(self.filename, 'a+')

	def __get_csv_writer(self):
		return csv.writer(self.csv_file)

	def __get_csv_reader(self):
		""" re-open the file so it can be read from the start """
		return csv.reader(self._get_csv_file())

	def load_known_from_persist(self):
		self.known_mac = set([row[0] for row in self.__get_csv_reader()
			if row])

	def arp_monitor_callback(self, pkt):
		if ARP not in pkt:
			return
		hwsrc = pkt[ARP].hwsrc
		if self.is_mac_known(hwsrc):
			return
		self.known_mac.add(hwsrc)
		p = Process(target=handle_new_pkt, args=(pkt),
			kwargs={'hwsrc' : hwsrc, 'filename' : self.filename})
		p.start()

	def is_mac_known(self, hwsrc):
		if hwsrc in self.known_mac:
			return True
		return False

	def show_known(self):
		for row in self.__get_csv_reader():
			if not row:
				continue
			print row[0], json.loads(row[1])

def parse_args():
	parser = argparse.ArgumentParser(description="SimpleNetworkMonitor")
	parser.add_argument("-f", "--file", dest="filename", default=
		os.path.expanduser("~/.simple_arp_watcher"),
		help="Filename to load and store arp and ip entries into." +
		"\nDefaults to ~/.simple_arp_watcher")
	parser.add_argument("-s", "--show-known-entries", dest="show_known",
		help="Show known mac address network information", action=
		"store_true")
	parser.add_argument("-i", "--ip-monitor", dest="ip_monitor",
		action="store_true", help="Alert on mac & ip address change." +
		"\nCurrently This option does nothing." )
	parser.add_argument("-e", "--email", dest="email",
		help="Email address to notify on network changes." +
		"\n This is not required and it currently does nothing.")
	args = parser.parse_args()
	return args

def main():
	args = parse_args()
	simpNetMon = SimpleNetworkMonitor(args.filename)
	if args.show_known:
		simpNetMon.show_known()
		return
	sniff(prn=simpNetMon.arp_monitor_callback, filter='arp',
		store=0, iface="eth0")

if __name__ =="__main__":
	main()
<commit_msg>Perform a few more style tweaks. Signed-off-by: David <db@d1b.org><commit_after>#!/usr/bin/python
__version__ = "0.0.1"
__author__ = "david"
__email__ = "db@d1b.org"

import argparse
import csv
import json
from multiprocessing import Process
import os
import socket
import time

import netaddr
from netaddr import EUI
from scapy.all import sniff, ARP

def get_hostname(ip):
	try:
		ret = socket.gethostbyaddr(ip)
		if ret:
			return ret[0]
	except socket.error:
		return ""

def persistent_pkt_info(filename, ** kwargs):
	with open(filename, 'a+') as csvfile:
		csv_w = csv.writer(csvfile)
		csv_w.writerow([kwargs['hwsrc'], json.dumps(kwargs)])

def handle_new_pkt(pkt, ** kwargs):
	hwsrc = kwargs['hwsrc']
	src_ip = pkt[ARP].psrc
	hostname = get_hostname(src_ip)
	try:
		org = EUI(hwsrc).oui.registration().org
	except netaddr.core.NotRegisteredError:
		org = 'NA'
	print src_ip, hwsrc, hostname, org
	filename = kwargs.get('filename', None)
	if filename:
		cur_time = time.time()
		persistent_pkt_info(filename, hwsrc=hwsrc, src_ip=src_ip,
			time=cur_time, org=org, hostname=hostname)

class SimpleNetworkMonitor(object):
	def __init__(self, filename, ** kwargs):
		self.filename = filename
		self.csv_file = self._get_csv_file()
		self.load_known_from_persist()

	def _get_csv_file(self):
		return open(self.filename, 'a+')

	def __get_csv_writer(self):
		return csv.writer(self.csv_file)

	def __get_csv_reader(self):
		""" re-open the file so it can be read from the start """
		return csv.reader(self._get_csv_file())

	def load_known_from_persist(self):
		self.known_mac = set([row[0] for row in self.__get_csv_reader()
			if row])

	def arp_monitor_callback(self, pkt):
		if ARP not in pkt:
			return
		hwsrc = pkt[ARP].hwsrc
		if self.is_mac_known(hwsrc):
			return
		self.known_mac.add(hwsrc)
		p = Process(target=handle_new_pkt, args=(pkt),
			kwargs={'hwsrc' : hwsrc, 'filename' : self.filename})
		p.start()

	def is_mac_known(self, hwsrc):
		if hwsrc in self.known_mac:
			return True
		return False

	def show_known(self):
		for row in self.__get_csv_reader():
			if not row:
				continue
			print row[0], json.loads(row[1])

def parse_args():
	parser = argparse.ArgumentParser(description="SimpleNetworkMonitor")
	parser.add_argument("-f", "--file", dest="filename", default=
		os.path.expanduser("~/.simple_arp_watcher"),
		help="Filename to load and store arp and ip entries into."
		"\nDefaults to ~/.simple_arp_watcher")
	parser.add_argument("-s", "--show-known-entries", dest="show_known",
		help="Show known mac address network information.",
		action="store_true")
	parser.add_argument("-i", "--ip-monitor", dest="ip_monitor",
		action="store_true", help="Alert on mac & ip address change."
		"\nCurrently This option does nothing." )
	parser.add_argument("-e", "--email", dest="email",
		help="Email address to notify on network changes."
		"\n This is not required and it currently does nothing.")
	args = parser.parse_args()
	return args

def main():
	args = parse_args()
	simpNetMon = SimpleNetworkMonitor(args.filename)
	if args.show_known:
		simpNetMon.show_known()
		return
	sniff(prn=simpNetMon.arp_monitor_callback, filter='arp',
		store=0, iface='eth0')

if __name__ =="__main__":
	main()
<|endoftext|>


<commit_before>__author__ = 'Calvin Lobo'
__version__ = '1.78'

from .property import Property
from .dictproperty import DictProperty
from .listproperty import ListProperty
from .unitproperty import UnitProperty
from .setproperty import SetProperty
from .limitproperty import LimitProperty
from .weakrefproperty import WeakRefProperty
from .stringproperty import StringProperty, _
from .scheduledevent import ScheduledEvent

import contextlib
from collections import defaultdict

class BindError(Exception):
    pass

class EventDispatcher(object):
    def __init__(self, *args, **kwargs):
        self.event_dispatcher_event_callbacks = {}
        self.event_dispatcher_properties = {}
        bindings = {}
        # Walk backwards through the MRO looking for Property attributes in the classes. Then register and bind them to
        # 'on_<prop_name>' if it exists. Walking backwards allows you to override the default value for a superclass.
        for cls in reversed(self.__class__.__mro__):
            for prop_name, prop in cls.__dict__.iteritems():
                if isinstance(prop, Property):
                    prop.name = prop_name
                    prop.register(self, prop_name, prop.default_value)
                    if hasattr(self, 'on_{}'.format(prop_name)):
                        func = getattr(self, 'on_{}'.format(prop_name))
                        bindings.update({prop_name: func})

        self.bind(**bindings)

    def force_dispatch(self, prop_name, value):
        """
        Assigns the value to the property and then dispatches the event, regardless of whether that value is the same
        as the previous value.
        :param prop_name: property name
        :param value: value to assign to the property
        """
        previous_value = getattr(self, prop_name)
        if previous_value == value:
            self.dispatch(prop_name, self, previous_value)
        else:
            setattr(self, prop_name, value)

    def dispatch(self, key, *args, **kwargs):
        for callback in self.event_dispatcher_properties[key]['callbacks']:
            if callback(*args, **kwargs):
                break

    def dispatch_event(self, event, *args, **kwargs):
        for callback in self.event_dispatcher_event_callbacks[event]:
            if callback(*args, **kwargs):
                break

    def register_event(self, name):
        if hasattr(self, 'on_{}'.format(name)):
            self.event_dispatcher_event_callbacks[name] = [getattr(self, 'on_{}'.format(name))]
        else:
            self.event_dispatcher_event_callbacks[name] = []

    def unbind(self, **kwargs):
        all_properties = self.event_dispatcher_properties
        for prop_name, callback in kwargs.iteritems():
            if prop_name in all_properties:
                try:
                    all_properties[prop_name]['callbacks'].remove(callback)
                except ValueError:
                    raise BindError("No binding for {} in property '{}'".format(callback.__name__, prop_name))
            elif prop_name in self.event_dispatcher_event_callbacks:
                try:
                    self.event_dispatcher_event_callbacks[prop_name].remove(callback)
                except ValueError:
                    raise BindError("No binding for {} in event '{}'".format(callback.__name__, prop_name))
            else:
                raise BindError('No property or event by the name of %s' % prop_name)

    def unbind_all(self, *args):
        all_properties = self.event_dispatcher_properties
        for prop_name in args:
            if prop_name in all_properties:
                del all_properties[prop_name]['callbacks'][:]
            elif prop_name in self.event_dispatcher_event_callbacks:
                del self.event_dispatcher_event_callbacks[prop_name][:]
            else:
                raise BindError("No such property or event '%s'" % prop_name)

    def bind(self, **kwargs):
        for prop_name, callback in kwargs.iteritems():
            if prop_name in self.event_dispatcher_properties:
                # Queue the callback into the property
                self.event_dispatcher_properties[prop_name]['callbacks'].append(callback)
            elif prop_name in self.event_dispatcher_event_callbacks:
                # If a property was not found, search in events
                self.event_dispatcher_event_callbacks[prop_name].append(callback)
            else:
                raise BindError("No property or event by the name of '%s'" % prop_name)

    def setter(self, prop_name):
        return lambda inst, value: setattr(self, prop_name, value)

    def get_dispatcher_property(self, prop_name):
        return self.event_dispatcher_properties[prop_name]['property']

    @contextlib.contextmanager
    def temp_unbind(self, **bindings):
        # Enter / With
        all_properties = self.event_dispatcher_properties
        callbacks = {}
        for prop_name, binding in bindings.iteritems():
            if prop_name in all_properties:
                # Make a copy of the callback sequence so we can revert back
                callbacks[prop_name] = all_properties[prop_name]['callbacks'][:]
                # Remove the specified bindings
                if binding in all_properties[prop_name]['callbacks']:
                    all_properties[prop_name]['callbacks'].remove(binding)
            elif prop_name in self.event_dispatcher_event_callbacks:
                callbacks[prop_name] = self.event_dispatcher_event_callbacks[prop_name][:]
                self.event_dispatcher_event_callbacks[prop_name].remove(binding)
        # Inside of with statement
        yield None
        # Finally / Exit
        for prop_name, cb in callbacks.iteritems():
            if prop_name in all_properties:
                all_properties[prop_name]['callbacks'] = cb
            elif prop_name in self.event_dispatcher_event_callbacks:
                self.event_dispatcher_event_callbacks[prop_name] = callbacks[prop_name]<commit_msg>bump to 1.79<commit_after>__author__ = 'Calvin Lobo'
__version__ = '1.79'

from .property import Property
from .dictproperty import DictProperty
from .listproperty import ListProperty
from .unitproperty import UnitProperty
from .setproperty import SetProperty
from .limitproperty import LimitProperty
from .weakrefproperty import WeakRefProperty
from .stringproperty import StringProperty, _
from .scheduledevent import ScheduledEvent

import contextlib
from collections import defaultdict

class BindError(Exception):
    pass

class EventDispatcher(object):
    def __init__(self, *args, **kwargs):
        self.event_dispatcher_event_callbacks = {}
        self.event_dispatcher_properties = {}
        bindings = {}
        # Walk backwards through the MRO looking for Property attributes in the classes. Then register and bind them to
        # 'on_<prop_name>' if it exists. Walking backwards allows you to override the default value for a superclass.
        for cls in reversed(self.__class__.__mro__):
            for prop_name, prop in cls.__dict__.iteritems():
                if isinstance(prop, Property):
                    prop.name = prop_name
                    prop.register(self, prop_name, prop.default_value)
                    if hasattr(self, 'on_{}'.format(prop_name)):
                        func = getattr(self, 'on_{}'.format(prop_name))
                        bindings.update({prop_name: func})

        self.bind(**bindings)

    def force_dispatch(self, prop_name, value):
        """
        Assigns the value to the property and then dispatches the event, regardless of whether that value is the same
        as the previous value.
        :param prop_name: property name
        :param value: value to assign to the property
        """
        previous_value = getattr(self, prop_name)
        if previous_value == value:
            self.dispatch(prop_name, self, previous_value)
        else:
            setattr(self, prop_name, value)

    def dispatch(self, key, *args, **kwargs):
        for callback in self.event_dispatcher_properties[key]['callbacks']:
            if callback(*args, **kwargs):
                break

    def dispatch_event(self, event, *args, **kwargs):
        for callback in self.event_dispatcher_event_callbacks[event]:
            if callback(*args, **kwargs):
                break

    def register_event(self, name):
        if hasattr(self, 'on_{}'.format(name)):
            self.event_dispatcher_event_callbacks[name] = [getattr(self, 'on_{}'.format(name))]
        else:
            self.event_dispatcher_event_callbacks[name] = []

    def unbind(self, **kwargs):
        all_properties = self.event_dispatcher_properties
        for prop_name, callback in kwargs.iteritems():
            if prop_name in all_properties:
                try:
                    all_properties[prop_name]['callbacks'].remove(callback)
                except ValueError:
                    raise BindError("No binding for {} in property '{}'".format(callback.__name__, prop_name))
            elif prop_name in self.event_dispatcher_event_callbacks:
                try:
                    self.event_dispatcher_event_callbacks[prop_name].remove(callback)
                except ValueError:
                    raise BindError("No binding for {} in event '{}'".format(callback.__name__, prop_name))
            else:
                raise BindError('No property or event by the name of %s' % prop_name)

    def unbind_all(self, *args):
        all_properties = self.event_dispatcher_properties
        for prop_name in args:
            if prop_name in all_properties:
                del all_properties[prop_name]['callbacks'][:]
            elif prop_name in self.event_dispatcher_event_callbacks:
                del self.event_dispatcher_event_callbacks[prop_name][:]
            else:
                raise BindError("No such property or event '%s'" % prop_name)

    def bind(self, **kwargs):
        for prop_name, callback in kwargs.iteritems():
            if prop_name in self.event_dispatcher_properties:
                # Queue the callback into the property
                self.event_dispatcher_properties[prop_name]['callbacks'].append(callback)
            elif prop_name in self.event_dispatcher_event_callbacks:
                # If a property was not found, search in events
                self.event_dispatcher_event_callbacks[prop_name].append(callback)
            else:
                raise BindError("No property or event by the name of '%s'" % prop_name)

    def setter(self, prop_name):
        return lambda inst, value: setattr(self, prop_name, value)

    def get_dispatcher_property(self, prop_name):
        return self.event_dispatcher_properties[prop_name]['property']

    @contextlib.contextmanager
    def temp_unbind(self, **bindings):
        # Enter / With
        all_properties = self.event_dispatcher_properties
        callbacks = {}
        for prop_name, binding in bindings.iteritems():
            if prop_name in all_properties:
                # Make a copy of the callback sequence so we can revert back
                callbacks[prop_name] = all_properties[prop_name]['callbacks'][:]
                # Remove the specified bindings
                if binding in all_properties[prop_name]['callbacks']:
                    all_properties[prop_name]['callbacks'].remove(binding)
            elif prop_name in self.event_dispatcher_event_callbacks:
                callbacks[prop_name] = self.event_dispatcher_event_callbacks[prop_name][:]
                self.event_dispatcher_event_callbacks[prop_name].remove(binding)
        # Inside of with statement
        yield None
        # Finally / Exit
        for prop_name, cb in callbacks.iteritems():
            if prop_name in all_properties:
                all_properties[prop_name]['callbacks'] = cb
            elif prop_name in self.event_dispatcher_event_callbacks:
                self.event_dispatcher_event_callbacks[prop_name] = callbacks[prop_name]
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
This is a quick demo hack of a wavenet trainer, with some help
of infrastructure from ibab.

Carl Quillen
'''

from __future__ import print_function

import optparse
import sys
import time
from operator import mul

import tensorflow as tf
import tensorflow.contrib.layers as layers
from audio_reader import AudioReader
from ops import mu_law_encode, mu_law_decode
from wavenet import wavenet

parser = optparse.OptionParser()
parser.add_option('-p', '--param_file', dest='param_file',
                  default=None, help='File to set parameters')
parser.add_option('-l', '--logdir', dest='logdir',
                  default=None, help='Tensorflow event logdir')
parser.add_option('-i', '--input_file', dest='input_file',
                  default=None, help='Input checkpoint file')
parser.add_option('-o', '--output_file', dest='output_file',
                  default='ckpt', help='Output checkpoint file')
parser.add_option('-d', '--data', dest='data_dir',
                  default='VCTK-Corpus', help='VCTK data directory')
parser.add_option('-c', '--checkpoint_rate', dest='checkpoint_rate',
                  type=int, default=1000, help='Rate to checkpoint.')
parser.add_option('-s', '--summary_rate', dest='summary_rate',
                  type=int, default=20, help='Rate to output summaries.')
parser.add_option('-S', '--silence_threshold', dest='silence_threshold',
                  type=float, default=0.2,
                  help='Silence classifier energy threshold')
parser.add_option('-Z', '--audio_chunk_size', dest='audio_chunk_size',
                  type=int, default=100000, help='Audio chunk size per batch.')
parser.add_option('-L', '--base_learning_rate', dest='base_learning_rate',
                  type=float, default=1e-03,
                  help='The initial learning rate. ' +
                  'lr = base_learning_rate/(lr_offet + timestep/const)')
parser.add_option('-O', '--lr_offset', dest='lr_offset',
                  type=float, default=1.0,
                  help="lr = base_learning_rate/(lr_offset + timestep/const)")
parser.add_option('-H', '--histogram_summaries', dest='histogram_summaries',
                  action='store_true', default=False,
                  help='Do histogram summaries')
parser.add_option('-b', '--batch_norm', dest='batch_norm',
                  action='store_true', default=False,
                  help='Do batch normalization')
parser.add_option('-w', '--which_future', dest='which_future',
                  type=int, default=1,
                  help='Which sample in the future to predict')

opts, cmdline_args = parser.parse_args()

# Set these further options in a parameter file.
opts.CANONICAL_EPOCH_SIZE = 20000.0
opts.batch_size = 1         # How many utterances to train at once.
opts.input_kernel_size = 32  # The size of the input layer kernel.
opts.num_outputs = 64       # The number of convolutional channels.
opts.skip_dimension = 512   # The dimension for skip connections.
opts.kernel_size = 2
opts.dilations = [[2, 4, 8, 16, 32, 128, 256],
                  [2, 4, 8, 16, 32, 128, 256]]
opts.epsilon = 1e-4      # Adams optimizer epsilon.
opts.max_steps = 200000
opts.sample_rate = 16000
opts.quantization_channels = 256
opts.one_hot_input = False

assert opts.which_future > 0 and opts.which_future < 20

# Set opts.* parameters from a parameter file if you want:
if opts.param_file is not None:
    with open(opts.param_file) as f:
        exec(f)

sess = tf.Session()

coord = tf.train.Coordinator()  # Is this used for anything?
data = AudioReader(opts.data_dir, coord, sample_rate=opts.sample_rate,
                   sample_size=opts.audio_chunk_size,
                   silence_threshold=opts.silence_threshold, queue_size=16)

data.start_threads(sess)         # start data reader threads.

# Define the computational graph.
with tf.name_scope("input_massaging"):
    batch = data.dequeue(num_elements=opts.batch_size)

    # We will try to predict the encoded_batch, which is a quantized version
    # of the input.
    encoded_batch = mu_law_encode(tf.reshape(batch, [opts.batch_size, -1]),
                                  opts.quantization_channels)
    if opts.one_hot_input:
        batch = tf.one_hot(encoded_batch, depth=opts.quantization_channels)

    # shift left to predict two samples into the future.
    encoded_batch = encoded_batch[:, opts.which_future:]

wavenet_out = wavenet(batch, opts)
for i in xrange(1, opts.which_future):
    if opts.one_hot_input:
        wavenet_out = wavenet(wavenet_out, opts, reuse=True)
    else:
        wavenet_out = tf.reshape(tf.arg_max(wavenet_out, dimension=2),
                                 shape=(-1, -1, 1))
        wavenet_out = mu_law_decode(wavenet_out, opts.quantization_channels)
        wavenet_out = wavenet(wavenet_out, opts, reuse=True)

# That should have created all training variables.  Now we can make a saver.
saver = tf.train.Saver(tf.trainable_variables())

if opts.histogram_summaries:
    tf.summary.histogram(name="wavenet", values=wavenet_out)
    layers.summaries.summarize_variables()

loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(
        wavenet_out[:, 0:-opts.which_future], encoded_batch))

tf.summary.scalar(name="loss", tensor=loss)

learning_rate = tf.placeholder(tf.float32, shape=())
# adams_epsilon probably should be reduced near the end of training.
adams_epsilon = tf.placeholder(tf.float32, shape=())
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
                                   epsilon=adams_epsilon)
minimize = optimizer.minimize(loss, var_list=tf.trainable_variables())

summaries = tf.summary.merge_all()

init = tf.global_variables_initializer()

# Finalize the graph, so that any new ops cannot be created.
# this is good for avoiding memory leaks.
tf.get_default_graph().finalize()

print("Model variables:")
total_params = 0
for var in tf.trainable_variables():
    vshape = var.get_shape().as_list()
    total_params += reduce(mul, vshape)
    print("  ", var.name, vshape)
print("Total model parameters:", total_params)
sys.stdout.flush()

if opts.logdir is not None:
    summary_writer = tf.summary.FileWriter(logdir=opts.logdir,
                                           graph=tf.get_default_graph())

# Initialize everything.
sess.run(init)

if opts.input_file is not None:
    print("Restoring from", opts.input_file)
    saver.restore(sess, opts.input_file)

# Main training loop:
last_time = time.time()

for global_step in xrange(opts.max_steps):
    cur_lr = opts.base_learning_rate/(
        global_step/opts.CANONICAL_EPOCH_SIZE + opts.lr_offset)

    if (global_step + 1) % opts.summary_rate == 0 and opts.logdir is not None:
        cur_loss, summary_pb = sess.run([loss, summaries, minimize],
                                        feed_dict={learning_rate: cur_lr,
                                        adams_epsilon: opts.epsilon})[0:2]
        summary_writer.add_summary(summary_pb, global_step)
    else:
        cur_loss = sess.run([loss, minimize],
                            feed_dict={learning_rate: cur_lr,
                            adams_epsilon: opts.epsilon})[0]
    new_time = time.time()
    print("loss[{}]: {:.3f} dt {:.3f}".format(global_step, cur_loss,
                                              new_time - last_time))
    last_time = new_time

    if (global_step + 1) % opts.checkpoint_rate == 0 and \
            opts.output_file is not None:
        saver.save(sess, opts.output_file, global_step)
    sys.stdout.flush()

print("Training done.")
saver.save(sess, opts.output_file)
sess.close()
<commit_msg>Add 1 rate!<commit_after>#!/usr/bin/env python
'''
This is a quick demo hack of a wavenet trainer, with some help
of infrastructure from ibab.

Carl Quillen
'''

from __future__ import print_function

import optparse
import sys
import time
from operator import mul

import tensorflow as tf
import tensorflow.contrib.layers as layers
from audio_reader import AudioReader
from ops import mu_law_encode, mu_law_decode
from wavenet import wavenet

# Options from the command line:
parser = optparse.OptionParser()
parser.add_option('-p', '--param_file', dest='param_file',
                  default=None, help='File to set parameters')
parser.add_option('-l', '--logdir', dest='logdir',
                  default=None, help='Tensorflow event logdir')
parser.add_option('-i', '--input_file', dest='input_file',
                  default=None, help='Input checkpoint file')
parser.add_option('-o', '--output_file', dest='output_file',
                  default='ckpt', help='Output checkpoint file')
parser.add_option('-d', '--data', dest='data_dir',
                  default='VCTK-Corpus', help='VCTK data directory')
parser.add_option('-c', '--checkpoint_rate', dest='checkpoint_rate',
                  type=int, default=1000, help='Rate to checkpoint.')
parser.add_option('-s', '--summary_rate', dest='summary_rate',
                  type=int, default=20, help='Rate to output summaries.')
parser.add_option('-S', '--silence_threshold', dest='silence_threshold',
                  type=float, default=0.2,
                  help='Silence classifier energy threshold')
parser.add_option('-Z', '--audio_chunk_size', dest='audio_chunk_size',
                  type=int, default=100000, help='Audio chunk size per batch.')
parser.add_option('-L', '--base_learning_rate', dest='base_learning_rate',
                  type=float, default=1e-03,
                  help='The initial learning rate. ' +
                  'lr = base_learning_rate/(lr_offet + timestep/const)')
parser.add_option('-O', '--lr_offset', dest='lr_offset',
                  type=float, default=1.0,
                  help="lr = base_learning_rate/(lr_offset + timestep/const)")
parser.add_option('-H', '--histogram_summaries', dest='histogram_summaries',
                  action='store_true', default=False,
                  help='Do histogram summaries')
parser.add_option('-b', '--batch_norm', dest='batch_norm',
                  action='store_true', default=False,
                  help='Do batch normalization')
parser.add_option('-w', '--which_future', dest='which_future',
                  type=int, default=1,
                  help='Which sample in the future to predict')

opts, cmdline_args = parser.parse_args()

# Options that can be set in a parameter file:
opts.canonical_epoch_size = 20000.0
opts.batch_size = 1         # How many utterances to train at once.
opts.input_kernel_size = 32  # The size of the input layer kernel.
opts.kernel_size = 2        # The size of other kernels.
opts.num_outputs = 64       # The number of convolutional channels.
opts.skip_dimension = 512   # The dimension for skip connections.
opts.dilations = [[1, 2, 4, 8, 16, 32, 128, 256],
                  [1, 2, 4, 8, 16, 32, 128, 256]]
opts.epsilon = 1e-4      # Adams optimizer epsilon.
opts.max_steps = 200000
opts.sample_rate = 16000
opts.quantization_channels = 256
opts.one_hot_input = False

assert opts.which_future > 0 and opts.which_future < 20

# Set opts.* parameters from a parameter file if you want:
if opts.param_file is not None:
    with open(opts.param_file) as f:
        exec(f)

sess = tf.Session()

coord = tf.train.Coordinator()  # Is this used for anything?
data = AudioReader(opts.data_dir, coord, sample_rate=opts.sample_rate,
                   sample_size=opts.audio_chunk_size,
                   silence_threshold=opts.silence_threshold, queue_size=16)

data.start_threads(sess)         # start data reader threads.

# Define the computational graph.
with tf.name_scope("input_massaging"):
    batch = data.dequeue(num_elements=opts.batch_size)

    # We will try to predict the encoded_batch, which is a quantized version
    # of the input.
    encoded_batch = mu_law_encode(tf.reshape(batch, [opts.batch_size, -1]),
                                  opts.quantization_channels)
    if opts.one_hot_input:
        batch = tf.one_hot(encoded_batch, depth=opts.quantization_channels)

    # shift left to predict which_future samples into the future.
    encoded_batch = encoded_batch[:, opts.which_future:]

wavenet_out = wavenet(batch, opts)
for i in xrange(1, opts.which_future):
    if opts.one_hot_input:
        wavenet_out = wavenet(wavenet_out, opts, reuse=True)
    else:
        wavenet_out = tf.reshape(tf.arg_max(wavenet_out, dimension=2),
                                 shape=(-1, -1, 1))
        wavenet_out = mu_law_decode(wavenet_out, opts.quantization_channels)
        wavenet_out = wavenet(wavenet_out, opts, reuse=True)

# That should have created all training variables.  Now we can make a saver.
saver = tf.train.Saver(tf.trainable_variables())

if opts.histogram_summaries:
    tf.summary.histogram(name="wavenet", values=wavenet_out)
    layers.summaries.summarize_variables()

loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(
        wavenet_out[:, 0:-opts.which_future], encoded_batch))

tf.summary.scalar(name="loss", tensor=loss)

learning_rate = tf.placeholder(tf.float32, shape=())
# adams_epsilon probably should be reduced near the end of training.
adams_epsilon = tf.placeholder(tf.float32, shape=())
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
                                   epsilon=adams_epsilon)
minimize = optimizer.minimize(loss, var_list=tf.trainable_variables())

summaries = tf.summary.merge_all()

init = tf.global_variables_initializer()

# Finalize the graph, so that any new ops cannot be created.
# this is good for avoiding memory leaks.
tf.get_default_graph().finalize()

print("Model variables:")
total_params = 0
for var in tf.trainable_variables():
    vshape = var.get_shape().as_list()
    total_params += reduce(mul, vshape)
    print("  ", var.name, vshape)
print("Total model parameters:", total_params)
sys.stdout.flush()

if opts.logdir is not None:
    summary_writer = tf.summary.FileWriter(logdir=opts.logdir,
                                           graph=tf.get_default_graph())

# Initialize everything.
sess.run(init)

if opts.input_file is not None:
    print("Restoring from", opts.input_file)
    saver.restore(sess, opts.input_file)

# Main training loop:
last_time = time.time()

for global_step in xrange(opts.max_steps):
    cur_lr = opts.base_learning_rate/(
        global_step/opts.canonical_epoch_size + opts.lr_offset)

    if (global_step + 1) % opts.summary_rate == 0 and opts.logdir is not None:
        cur_loss, summary_pb = sess.run([loss, summaries, minimize],
                                        feed_dict={learning_rate: cur_lr,
                                        adams_epsilon: opts.epsilon})[0:2]
        summary_writer.add_summary(summary_pb, global_step)
    else:
        cur_loss = sess.run([loss, minimize],
                            feed_dict={learning_rate: cur_lr,
                            adams_epsilon: opts.epsilon})[0]
    new_time = time.time()
    print("loss[{}]: {:.3f} dt {:.3f}".format(global_step, cur_loss,
                                              new_time - last_time))
    last_time = new_time

    if (global_step + 1) % opts.checkpoint_rate == 0 and \
            opts.output_file is not None:
        saver.save(sess, opts.output_file, global_step)
    sys.stdout.flush()

print("Training done.")
saver.save(sess, opts.output_file)
sess.close()
<|endoftext|>