<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import with_statement

import os

from _ast import *
try:
    from ast import parse
except ImportError:
    parse = lambda source, filename: compile(source, filename, "exec", PyCF_ONLY_AST)

import sys
import shutil
import urllib2
import hashlib
import tempfile
from datetime import datetime
from os.path import abspath, join, dirname, isfile, isdir, basename

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = "1.0"

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named "sphinx.ext.*") or your custom ones.
extensions = [
    "sphinx.ext.autodoc",
    "sphinx.ext.doctest",
    "sphinx.ext.coverage",
    "sphinx.ext.ifconfig",
    "sphinx.ext.viewcode",
    "sphinx.ext.intersphinx"
]

pymajor, pyminor = sys.version_info[0:2]
intersphinx_mapping = {
    "python": ("http://docs.python.org/%s.%s" % (pymajor, pyminor), None),
    "sqlalchemy": ("http://www.sqlalchemy.org/docs/", None),
    "flask": ("http://flask.readthedocs.org/en/latest/", None),
    "numpy": ("http://docs.scipy.org/doc/numpy", None)
}

templates_path = ["_templates"]
source_suffix = ".rst"
master_doc = "index"

project = u"PyFarm"
root = abspath(join(dirname(__file__), "..", ".."))
sys.path.insert(0, root)
docroot = join(root, "docs", "source")
initpy = join(root, project.lower(), "__init__.py")
tmpdir = tempfile.mkdtemp(suffix="-pyfarm-docs")
assert isfile(initpy), "%s does not exist" % initpy

print "generating dynamic content"

# Parse the __init__.py file instead of importing it.  So even if we
# have code that can't be imported we can at least read the proper version
# information
print "..parsing version/author(s)"
with open(initpy, "r") as stream:
    module = parse(stream.read(), stream.name)

author = None
parsed_version = None
for obj in module.body:
    if isinstance(obj, Assign) and obj.targets[0].id == "__version__":
        parsed_version = map(lambda num: num.n, obj.value.elts)
    elif isinstance(obj, Assign) and obj.targets[0].id == "__author__":
        author = obj.value.s

assert isinstance(parsed_version, list), "did not find __version__"
assert isinstance(author, basestring), "did not find __author__"

# General information about the project.
now = datetime.now()
copyright = "%s, %s" % (now.year, author)
release = ".".join(map(str, parsed_version))
version = ".".join(map(str, parsed_version[0:2]))

# create a requirements file to
print "..python requirements"
import setup as _setup
python_requirements = join(docroot, "include", "python_requirements.rst")

with open(python_requirements, "w") as destination:
    supported_versions = ((2, 5), (2, 6), (2, 7))

    for major, minor in supported_versions:
        header = "Python %s.%s" % (major, minor)
        print >> destination, header
        print >> destination, "+" * len(header)

        for requirement in sorted(_setup.requirements(major, minor)):
            print >> destination, "* %s" % requirement

        print >> destination

print "..TODO: jobtypes"
print "..download links"

download_release = (
    (
        "6/10/2009",
        "Stable - 0.3.216",
        "http://pyfarm.net/downloads/pyfarm_stable_v0.3.216.tar.gz",
        "f3b48856c4a4f82a1b2607b6c1b807f9",
        "Legacy Release"
    ),
)

download_other = (
    (
        "11/16/2011",
        "Client Testing - Milestone 1",
        "http://pyfarm.net/downloads/client_milestone1.tar.gz",
        "5a817b22b69cc7e5d8c8a66c723e0ea6",
        "Testing xmlrc/process protocols"
    ),
    (
        "11/25/2011",
        "Client Testing - Milestone 1.5",
        "http://pyfarm.net/downloads/client_milestone1_5.tar.gz",
        "641817a7f95d8f4fa01f8bf0ef4db3d5",
        "adding client preferences"
    ),
    (
        "12/04/2011",
        "Client Testing - Milestone 2",
        "http://pyfarm.net/downloads/client_milestone2.tar.gz",
        "54fe83c91019ac1905a1263a05a1ce82",
        "logging and remote log retrieval"
    ),
    (
        "12/11/2011",
        "Client Testing - Milestone 3",
        "http://pyfarm.net/downloads/client_milestone3.tar.gz",
        "c9bcdb17309735403142d1fc61f5e685",
        "basic job object and environment control"
    ),
    (
        "12/17/2011",
        "Client Testing - Milestone 4",
        "http://pyfarm.net/downloads/client_milestone4.tar.gz",
        "e7d5acdfbd9587f9956dc31d0ddfcb8f",
        "running job statistical information"

    ),
    (
        "1/25/2012",
        "Client Testing - Milestone 5",
        "http://pyfarm.net/downloads/client_milestone5.tar.gz",
        "a420ce6c3c1433053e3207a5644618f9",
        "updated preference objects, multicast discovery"
    )
)

download_directory = join(
    tempfile.gettempdir(),"pyfarm", "docbuild",  "downloads"
)

USE_SERVER_URL = True

if not USE_SERVER_URL and not isdir(join(docroot, "downloads")):
    os.makedirs(join(docroot, "downloads"))

if not isdir(download_directory):
    os.makedirs(download_directory)

def downloadfile(url, md5):
    destination = join(download_directory, url.split("/")[-1])

    # download the file if it does not exist
    if not isfile(destination):
        print "...downloading %s -> %s" % (url, destination)
        wwwresource = urllib2.urlopen(url)
        with open(destination, "w") as outstream:
            outstream.write(wwwresource.read())
        wwwresource.close()

    # check the hash
    filehash = hashlib.md5()
    with open(destination, "r") as sourcestream:
        while True:
            data = sourcestream.read(128)
            if not data:
                break

            filehash.update(data)

    assert filehash.hexdigest() == md5, "hash check failed for %s" % destination

    if USE_SERVER_URL:
        return destination
    else:
        local = join(basename(dirname(destination)), basename(destination))

        if not isfile(local):
            shutil.copy(destination, local)

        return local

def generatetable(stream, source_data):
    for row in source_data:
        date, name, source, md5, desc = row
        localpath = downloadfile(source, md5)
        size = os.stat(localpath).st_size / 1024

        if USE_SERVER_URL:
            data = (date, name, source, size, md5, desc)
        else:
            data = (date, name, localpath, size, md5, desc)

        print >> stream, row_template % data

downloadsrst = join(docroot, "downloads.rst")

if USE_SERVER_URL:
    row_template = '    "%s", "`%s <%s>`_", %s, %s, "%s"'
else:
    row_template = '    "%s", ":download:`%s <%s>`", %s, %s, "%s"'

table_header = '    :header: "Date", "Filename", "Size (KB)", "MD5", "Description"'
table_widths = '    :widths: 5, 10, 3, 10, 15'

with open(downloadsrst, "w") as stream:
    print >> stream, ".. this page was dynamically generated by conf.py, DO"
    print >> stream, ".. NOT MODIFY."
    print >> stream, ""
    print >> stream, "Downloads"
    print >> stream, "========="
    print >> stream, "Below are several downloads containing the source code "
    print >> stream, "for PyFarm.  In addition to releases several prototypes"
    print >> stream, "and tests are included for reference."
    print >> stream, ""
    print >> stream, ""
    print >> stream, "Releases"
    print >> stream, "--------"
    print >> stream, ".. csv-table::"
    print >> stream, '    :header: "Date", "Filename", "Size (KB)", "MD5", "Description"'
    print >> stream, table_widths
    print >> stream, ""

    # for name, source, md5, date, desc in sorted(download_release, sort_dates):
    generatetable(stream, download_release)

    print >> stream, ""
    print >> stream, "Testing/Other"
    print >> stream, "-------------"
    print >> stream, ".. csv-table::"
    print >> stream, table_header
    print >> stream, table_widths
    print >> stream, ""
    generatetable(stream, download_other)

print "generation complete"

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ["include/*", "downloads/*"]
pygments_style = "sphinx"
html_theme = "default"
html_static_path = ["_static"]
htmlhelp_basename = "PyFarmdoc"
latex_elements = {}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ("index", "PyFarm.tex", u"PyFarm Documentation",
   u"Oliver Palmer", "manual"),
]

man_pages = [
    ("index", "pyfarm", u"PyFarm Documentation",
     [u"Oliver Palmer"], 1)
]

texinfo_documents = [
  ("index", "PyFarm", u"PyFarm Documentation",
   u"Oliver Palmer", "PyFarm", "A Python based distributed job system",
   "Miscellaneous"),
]<commit_msg>adding /ping resource<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import with_statement

import os

from _ast import *
try:
    from ast import parse
except ImportError:
    parse = lambda source, filename: compile(source, filename, "exec", PyCF_ONLY_AST)

import sys
import shutil
import urllib2
import hashlib
import tempfile
from datetime import datetime
from os.path import abspath, join, dirname, isfile, isdir, basename

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = "1.0"

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named "sphinx.ext.*") or your custom ones.
extensions = [
    "sphinx.ext.autodoc",
    "sphinx.ext.doctest",
    "sphinx.ext.coverage",
    "sphinx.ext.ifconfig",
    "sphinx.ext.viewcode",
    "sphinx.ext.intersphinx"
]

pymajor, pyminor = sys.version_info[0:2]
intersphinx_mapping = {
    "python": ("http://docs.python.org/%s.%s" % (pymajor, pyminor), None),
    "sqlalchemy": ("http://www.sqlalchemy.org/docs/", None),
    "flask": ("http://flask.readthedocs.org/en/latest/", None),
    "numpy": ("http://docs.scipy.org/doc/numpy", None),
    "flask_restful": ("http://flask-restful.readthedocs.org/en/latest/", None)
}
intersphinx_mapping["flask-restful"] = intersphinx_mapping["flask_restful"]

templates_path = ["_templates"]
source_suffix = ".rst"
master_doc = "index"

project = u"PyFarm"
root = abspath(join(dirname(__file__), "..", ".."))
sys.path.insert(0, root)
docroot = join(root, "docs", "source")
initpy = join(root, project.lower(), "__init__.py")
tmpdir = tempfile.mkdtemp(suffix="-pyfarm-docs")
assert isfile(initpy), "%s does not exist" % initpy

print "generating dynamic content"

# Parse the __init__.py file instead of importing it.  So even if we
# have code that can't be imported we can at least read the proper version
# information
print "..parsing version/author(s)"
with open(initpy, "r") as stream:
    module = parse(stream.read(), stream.name)

author = None
parsed_version = None
for obj in module.body:
    if isinstance(obj, Assign) and obj.targets[0].id == "__version__":
        parsed_version = map(lambda num: num.n, obj.value.elts)
    elif isinstance(obj, Assign) and obj.targets[0].id == "__author__":
        author = obj.value.s

assert isinstance(parsed_version, list), "did not find __version__"
assert isinstance(author, basestring), "did not find __author__"

# General information about the project.
now = datetime.now()
copyright = "%s, %s" % (now.year, author)
release = ".".join(map(str, parsed_version))
version = ".".join(map(str, parsed_version[0:2]))

# create a requirements file to
print "..python requirements"
import setup as _setup
python_requirements = join(docroot, "include", "python_requirements.rst")

with open(python_requirements, "w") as destination:
    supported_versions = ((2, 5), (2, 6), (2, 7))

    for major, minor in supported_versions:
        header = "Python %s.%s" % (major, minor)
        print >> destination, header
        print >> destination, "+" * len(header)

        for requirement in sorted(_setup.requirements(major, minor)):
            print >> destination, "* %s" % requirement

        print >> destination

print "..TODO: jobtypes"
print "..download links"

download_release = (
    (
        "6/10/2009",
        "Stable - 0.3.216",
        "http://pyfarm.net/downloads/pyfarm_stable_v0.3.216.tar.gz",
        "f3b48856c4a4f82a1b2607b6c1b807f9",
        "Legacy Release"
    ),
)

download_other = (
    (
        "11/16/2011",
        "Client Testing - Milestone 1",
        "http://pyfarm.net/downloads/client_milestone1.tar.gz",
        "5a817b22b69cc7e5d8c8a66c723e0ea6",
        "Testing xmlrc/process protocols"
    ),
    (
        "11/25/2011",
        "Client Testing - Milestone 1.5",
        "http://pyfarm.net/downloads/client_milestone1_5.tar.gz",
        "641817a7f95d8f4fa01f8bf0ef4db3d5",
        "adding client preferences"
    ),
    (
        "12/04/2011",
        "Client Testing - Milestone 2",
        "http://pyfarm.net/downloads/client_milestone2.tar.gz",
        "54fe83c91019ac1905a1263a05a1ce82",
        "logging and remote log retrieval"
    ),
    (
        "12/11/2011",
        "Client Testing - Milestone 3",
        "http://pyfarm.net/downloads/client_milestone3.tar.gz",
        "c9bcdb17309735403142d1fc61f5e685",
        "basic job object and environment control"
    ),
    (
        "12/17/2011",
        "Client Testing - Milestone 4",
        "http://pyfarm.net/downloads/client_milestone4.tar.gz",
        "e7d5acdfbd9587f9956dc31d0ddfcb8f",
        "running job statistical information"

    ),
    (
        "1/25/2012",
        "Client Testing - Milestone 5",
        "http://pyfarm.net/downloads/client_milestone5.tar.gz",
        "a420ce6c3c1433053e3207a5644618f9",
        "updated preference objects, multicast discovery"
    )
)

download_directory = join(
    tempfile.gettempdir(),"pyfarm", "docbuild",  "downloads"
)

USE_SERVER_URL = True

if not USE_SERVER_URL and not isdir(join(docroot, "downloads")):
    os.makedirs(join(docroot, "downloads"))

if not isdir(download_directory):
    os.makedirs(download_directory)

def downloadfile(url, md5):
    destination = join(download_directory, url.split("/")[-1])

    # download the file if it does not exist
    if not isfile(destination):
        print "...downloading %s -> %s" % (url, destination)
        wwwresource = urllib2.urlopen(url)
        with open(destination, "w") as outstream:
            outstream.write(wwwresource.read())
        wwwresource.close()

    # check the hash
    filehash = hashlib.md5()
    with open(destination, "r") as sourcestream:
        while True:
            data = sourcestream.read(128)
            if not data:
                break

            filehash.update(data)

    assert filehash.hexdigest() == md5, "hash check failed for %s" % destination

    if USE_SERVER_URL:
        return destination
    else:
        local = join(basename(dirname(destination)), basename(destination))

        if not isfile(local):
            shutil.copy(destination, local)

        return local

def generatetable(stream, source_data):
    for row in source_data:
        date, name, source, md5, desc = row
        localpath = downloadfile(source, md5)
        size = os.stat(localpath).st_size / 1024

        if USE_SERVER_URL:
            data = (date, name, source, size, md5, desc)
        else:
            data = (date, name, localpath, size, md5, desc)

        print >> stream, row_template % data

downloadsrst = join(docroot, "downloads.rst")

if USE_SERVER_URL:
    row_template = '    "%s", "`%s <%s>`_", %s, %s, "%s"'
else:
    row_template = '    "%s", ":download:`%s <%s>`", %s, %s, "%s"'

table_header = '    :header: "Date", "Filename", "Size (KB)", "MD5", "Description"'
table_widths = '    :widths: 5, 10, 3, 10, 15'

with open(downloadsrst, "w") as stream:
    print >> stream, ".. this page was dynamically generated by conf.py, DO"
    print >> stream, ".. NOT MODIFY."
    print >> stream, ""
    print >> stream, "Downloads"
    print >> stream, "========="
    print >> stream, "Below are several downloads containing the source code "
    print >> stream, "for PyFarm.  In addition to releases several prototypes"
    print >> stream, "and tests are included for reference."
    print >> stream, ""
    print >> stream, ""
    print >> stream, "Releases"
    print >> stream, "--------"
    print >> stream, ".. csv-table::"
    print >> stream, '    :header: "Date", "Filename", "Size (KB)", "MD5", "Description"'
    print >> stream, table_widths
    print >> stream, ""

    # for name, source, md5, date, desc in sorted(download_release, sort_dates):
    generatetable(stream, download_release)

    print >> stream, ""
    print >> stream, "Testing/Other"
    print >> stream, "-------------"
    print >> stream, ".. csv-table::"
    print >> stream, table_header
    print >> stream, table_widths
    print >> stream, ""
    generatetable(stream, download_other)

print "generation complete"

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ["include/*", "downloads/*"]
pygments_style = "sphinx"
html_theme = "default"
html_static_path = ["_static"]
htmlhelp_basename = "PyFarmdoc"
latex_elements = {}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ("index", "PyFarm.tex", u"PyFarm Documentation",
   u"Oliver Palmer", "manual"),
]

man_pages = [
    ("index", "pyfarm", u"PyFarm Documentation",
     [u"Oliver Palmer"], 1)
]

texinfo_documents = [
  ("index", "PyFarm", u"PyFarm Documentation",
   u"Oliver Palmer", "PyFarm", "A Python based distributed job system",
   "Miscellaneous"),
]<|endoftext|>


<commit_before>#!/usr/bin/python
"""interactive module for vctools."""
from __future__ import print_function
from cmd import Cmd
# pylint: disable=no-name-in-module
from pyVmomi import vim
from vctools.auth import Auth
from vctools.vmconfig import VMConfig
from vctools.query import Query

# pylint: disable=no-self-use
# pylint: disable=too-many-public-methods
# pylint: disable=unused-argument
class Wwwyzzerdd(Cmd):
    """
    It's a Wwwyzzerdd! An interactive wizard for vctools.
    """

    def __init__(self, intro='wwwyzzerdd!', prompt='# '):
        Cmd.__init__(self)
        self.auth = None
        self.intro = intro
        self.prompt = prompt

    def default(self, arg):
        print('%s: Invalid command. Type "help" for a list.' % (arg))

    def do_connect(self, vcenter):
        """ connect to vcenter."""
        self.auth = Auth(vcenter)
        self.auth.login()

    def do_query(self, args):
        """ query sub-category."""
        query_cmds = QueryCMDs()
        query_cmds.cmdloop()

    def do_exit(self, args):
        """ exit interactive mode."""
        if self.auth:
            self.auth.logout()
            return True
        else:
            return True

class QueryCMDs(Cmd):
    """ sub category of cmd options for querying info."""
    def __init__(self, intro='query commands', prompt='(query) # '):
        Cmd.__init__(self)
        self.query = Query()
        self.intro = intro
        self.prompt = prompt

    def do_networks(self, arg):
        """ show networks associated with cluster."""
        cluster_container = self.query.create_container(
            self.auth.session, self.auth.session.content.rootFolder,
            [vim.ComputeResource], True
        )
        cluster = self.query.get_obj(
            cluster_container.view, arg
        )
        networks = self.query.list_obj_attrs(
            cluster.network, 'name', view=False
        )
        networks.sort()
        for net in networks:
            print(net)

    def do_datastores(self, arg):
        """ show datastores associated with cluster."""
        cluster_container = self.query.create_container(
            self.auth.session, self.auth.session.content.rootFolder,
            [vim.ComputeResource], True
        )
        self.query.list_datastore_info(
            cluster_container.view, arg
        )

    def do_folders(self, arg):
        """ show available folders."""
        folder_container = self.query.create_container(
            self.auth.session, self.auth.session.content.rootFolder,
            [vim.Folder], True
        )
        folders = self.query.list_obj_attrs(folder_container, 'name')
        folders.sort()
        for folder in folders:
            print(folder)

    def do_clusters(self):
        """ show available clusters."""
        cluster_container = self.query.create_container(
            self.auth.session, self.auth.session.content.rootFolder,
            [vim.ComputeResource], True
        )
        clusters = self.query.list_obj_attrs(cluster_container, 'name')
        clusters.sort()
        for cluster in clusters:
            print(cluster)

<commit_msg>more changes to wwwyzzerdd.<commit_after>#!/usr/bin/python
"""interactive module for vctools."""
from __future__ import print_function
from cmd import Cmd
# pylint: disable=no-name-in-module
from pyVmomi import vim
from vctools.auth import Auth
from vctools.vmconfig import VMConfig
from vctools.query import Query

# pylint: disable=no-self-use
# pylint: disable=too-many-public-methods
# pylint: disable=unused-argument
class Wwwyzzerdd(Cmd):
    """
    It's a Wwwyzzerdd! An interactive wizard for vctools.
    """

    def __init__(self, intro='wwwyzzerdd!', prompt='# '):
        Cmd.__init__(self)
        Wwwyzzerdd.auth = None
        self.intro = intro
        self.prompt = prompt

    def default(self, arg):
        print('%s: Invalid command. Type "help" for a list.' % (arg))

    def do_connect(self, vcenter):
        """ connect to vcenter."""
        Wwwyzzerdd.auth = Auth(vcenter)
        Wwwyzzerdd.auth.login()

    def do_query(self, args):
        """ query sub-category."""
        query_cmds = QueryCMDs()
        query_cmds.cmdloop()

    def do_ticket(self, args):
        if self.auth:
            print(self.auth.ticket)
        else:
            print(None)

    def do_exit(self, args):
        """ exit interactive mode."""
        if self.auth:
            self.auth.logout()
            return True
        else:
            return True

class QueryCMDs(Cmd):
    """ sub category of cmd options for querying info."""
    def __init__(self, intro='query commands', prompt='(query) # '):
        Cmd.__init__(self)
        self.query = Query()
        self.intro = intro
        self.prompt = prompt

    def do_back(self, args):
        """ exit interactive mode."""
        return True

    def do_networks(self, args):
        """ show networks associated with cluster."""
        cluster_container = self.query.create_container(
            Wwwyzzerdd.auth.session, Wwwyzzerdd.auth.session.content.rootFolder,
            [vim.ComputeResource], True
        )
        cluster = self.query.get_obj(
            cluster_container.view, args
        )
        networks = self.query.list_obj_attrs(
            cluster.network, 'name', view=False
        )
        networks.sort()
        for net in networks:
            print(net)

    def do_datastores(self, args):
        """ show datastores associated with cluster."""
        cluster_container = self.query.create_container(
            Wwwyzzerdd.auth.session, Wwwyzzerdd.auth.session.content.rootFolder,
            [vim.ComputeResource], True
        )
        self.query.list_datastore_info(
            cluster_container.view, args
        )

    def do_folders(self, args):
        """ show available folders."""
        folder_container = self.query.create_container(
            Wwwyzzerdd.auth.session, Wwwyzzerdd.auth.session.content.rootFolder,
            [vim.Folder], True
        )
        folders = self.query.list_obj_attrs(folder_container, 'name')
        folders.sort()
        for folder in folders:
            print(folder)

    def do_clusters(self, args):
        """ show available clusters."""
        cluster_container = self.query.create_container(
            Wwwyzzerdd.auth.session, Wwwyzzerdd.auth.session.content.rootFolder,
            [vim.ComputeResource], True
        )
        clusters = self.query.list_obj_attrs(cluster_container, 'name')
        clusters.sort()
        for cluster in clusters:
            print(cluster)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# slimta documentation build configuration file, created by
# sphinx-quickstart on Wed Oct 30 10:25:18 2013.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'slimta'
copyright = u'2013, Ian Good'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
from slimta.core import __version__ as slimta_version
version_parts = slimta_version.split('.')
version = '.'.join(version_parts[0:2])
# The full version, including alpha/beta/rc tags.
release = slimta_version

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = []

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# -- Options for HTML output ----------------------------------------------

import cloud_sptheme as csp

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'cloud'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
html_theme_options = {'externalrefs': False}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []
html_theme_path = [csp.get_theme_dir()]

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
html_logo = 'slimta.svg'

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
html_show_sourcelink = False

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
html_show_sphinx = False

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'slimtadoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  ('index', 'slimta.tex', u'slimta Documentation',
   u'Ian Good', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'slimta', u'slimta Documentation',
     [u'Ian Good'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'slimta', u'slimta Documentation',
   u'Ian Good', 'slimta', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'py': ('http://docs.python.org/', None),
                       'gevent': ('http://gevent.org/', None),
                       'redis': ('http://redis-py.readthedocs.org/en/latest/', None),
                       'celery': ('http://docs.celeryproject.org/en/latest/', None),
                       'boto': ('http://boto.readthedocs.org/en/latest/', None)}

autodoc_member_order = 'bysource'
autodoc_docstring_signature = True

<commit_msg>improved signature generation in autodocs<commit_after># -*- coding: utf-8 -*-
#
# slimta documentation build configuration file, created by
# sphinx-quickstart on Wed Oct 30 10:25:18 2013.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'slimta'
copyright = u'2013, Ian Good'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
from slimta.core import __version__ as slimta_version
version_parts = slimta_version.split('.')
version = '.'.join(version_parts[0:2])
# The full version, including alpha/beta/rc tags.
release = slimta_version

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = []

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# -- Options for HTML output ----------------------------------------------

import cloud_sptheme as csp

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'cloud'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
html_theme_options = {'externalrefs': False}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []
html_theme_path = [csp.get_theme_dir()]

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
html_logo = 'slimta.svg'

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
html_show_sourcelink = False

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
html_show_sphinx = False

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'slimtadoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  ('index', 'slimta.tex', u'slimta Documentation',
   u'Ian Good', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'slimta', u'slimta Documentation',
     [u'Ian Good'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'slimta', u'slimta Documentation',
   u'Ian Good', 'slimta', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'py': ('http://docs.python.org/', None),
                       'gevent': ('http://gevent.org/', None),
                       'redis': ('http://redis-py.readthedocs.org/en/latest/', None),
                       'celery': ('http://docs.celeryproject.org/en/latest/', None),
                       'boto': ('http://boto.readthedocs.org/en/latest/', None)}

autodoc_member_order = 'bysource'
autodoc_docstring_signature = True

# {{{ reformat_signatures()
def reformat_signatures(app, what, name, obj, options, signature,
        return_annotation):
    import inspect
    import repr

    if what == 'attribute':
        return None, 'derp'
    if what not in ('class', 'exception', 'method', 'function'):
        return
    if what in ('class', 'exception'):
        obj = getattr(obj, '__init__')
    try:
        args, varargs, keywords, defaults = inspect.getargspec(obj)
    except TypeError:
        return
    if args and args[0] == 'self':
        del args[0]
    doc_repr = repr.Repr()
    doc_repr.maxstring = 40
    with_defaults = []
    without_defaults = []
    if defaults:
        num_without_defaults = len(args) - len(defaults)
        for name in args[:num_without_defaults]:
            without_defaults.append(name)
        for i, value in enumerate(defaults):
            name = args[num_without_defaults+i]
            value_str = doc_repr.repr(value)
            with_defaults.append('{0}={1}'.format(name, value_str))
    else:
        without_defaults = args
    if varargs is not None:
        with_defaults.append('*{0}'.format(varargs))
    if keywords is not None:
        with_defaults.append('**{0}'.format(keywords))
    result = ', '.join(without_defaults)
    if with_defaults:
        if result:
            result += ', '
        result += ', '.join(['['+piece for piece in with_defaults])
        result += ']'*len(with_defaults)
    result = '({0})'.format(result)
    return result, return_annotation
# }}}

def setup(app):
    app.connect('autodoc-process-signature', reformat_signatures)

# vim:sw=4:ts=4:sts=4:fdm=marker:et:
<|endoftext|>


<commit_before># Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Routine for decoding the CIFAR-10 binary file format."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os

import tensorflow.python.platform
from six.moves import xrange  # pylint: disable=redefined-builtin
import tensorflow as tf

import gzip
import os
import re
import sys
import tarfile
from six.moves import urllib

from tensorflow.python.platform import gfile

import params

def read_cifar10(filename_queue):
  """Reads and parses examples from CIFAR10 data files.

  Recommendation: if you want N-way read parallelism, call this function
  N times.  This will give you N independent Readers reading different
  files & positions within those files, which will give better mixing of
  examples.

  Args:
    filename_queue: A queue of strings with the filenames to read from.

  Returns:
    An object representing a single example, with the following fields:
      height: number of rows in the result (32)
      width: number of columns in the result (32)
      depth: number of color channels in the result (3)
      key: a scalar string Tensor describing the filename & record number
        for this example.
      label: an int32 Tensor with the label in the range 0..9.
      uint8image: a [height, width, depth] uint8 Tensor with the image data
  """

  class CIFAR10Record(object):
    pass
  result = CIFAR10Record()

  # Dimensions of the images in the CIFAR-10 dataset.
  # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the
  # input format.
  label_bytes = 1  # 2 for CIFAR-100
  result.height = 32
  result.width = 32
  result.depth = 3
  image_bytes = result.height * result.width * result.depth
  # Every record consists of a label followed by the image, with a
  # fixed number of bytes for each.
  record_bytes = label_bytes + image_bytes

  # Read a record, getting filenames from the filename_queue.  No
  # header or footer in the CIFAR-10 format, so we leave header_bytes
  # and footer_bytes at their default of 0.
  reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
  result.key, value = reader.read(filename_queue)

  # Convert from a string to a vector of uint8 that is record_bytes long.
  record_bytes = tf.decode_raw(value, tf.uint8)

  # The first bytes represent the label, which we convert from uint8->int32.
  result.label = tf.cast(
      tf.slice(record_bytes, [0], [label_bytes]), tf.int32)

  # The remaining bytes after the label represent the image, which we reshape
  # from [depth * height * width] to [depth, height, width].
  depth_major = tf.reshape(tf.slice(record_bytes, [label_bytes], [image_bytes]),
                           [result.depth, result.height, result.width])
  # Convert from [depth, height, width] to [height, width, depth].
  result.uint8image = tf.transpose(depth_major, [1, 2, 0])

  return result

def _generate_image_and_label_batch(image, label, min_queue_examples,
                                    batch_size):
  """Construct a queued batch of images and labels.

  Args:
    image: 3-D Tensor of [height, width, 3] of type.float32.
    label: 1-D Tensor of type.int32
    min_queue_examples: int32, minimum number of samples to retain
      in the queue that provides of batches of examples.
    batch_size: Number of images per batch.

  Returns:
    images: Images. 4D tensor of [batch_size, height, width, 3] size.
    labels: Labels. 1D tensor of [batch_size] size.
  """
  # Create a queue that shuffles the examples, and then
  # read 'batch_size' images + labels from the example queue.
  num_preprocess_threads = 2
  images, label_batch = tf.train.shuffle_batch(
      [image, label],
      batch_size=batch_size,
      num_threads=num_preprocess_threads,
      capacity=min_queue_examples + 3 * batch_size,
      min_after_dequeue=min_queue_examples)

  # Display the training images in the visualizer.
  tf.image_summary('images', images)

  return images, tf.reshape(label_batch, [batch_size])

def distorted_inputs(data_dir, batch_size):
  """Construct distorted input for CIFAR training using the Reader ops.

  Args:
    data_dir: Path to the CIFAR-10 data directory.
    batch_size: Number of images per batch.

  Returns:
    images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
    labels: Labels. 1D tensor of [batch_size] size.
  """
  data_dir = os.path.join(data_dir, "cifar-10-batches-bin")
  filenames = [os.path.join(data_dir, 'data_batch_%d.bin' % i)
               for i in xrange(1, 6)]
  for f in filenames:
    if not gfile.Exists(f):
      raise ValueError('Failed to find file: ' + f)

  # Create a queue that produces the filenames to read.
  filename_queue = tf.train.string_input_producer(filenames)

  # Read examples from files in the filename queue.
  read_input = read_cifar10(filename_queue)
  reshaped_image = tf.cast(read_input.uint8image, tf.float32)

  height = params.image_size
  width = params.image_size

  # Image processing for training the network. Note the many random
  # distortions applied to the image.

  # Randomly crop a [height, width] section of the image.
  distorted_image = tf.image.random_crop(reshaped_image, [height, width])

  # Randomly flip the image horizontally.
  distorted_image = tf.image.random_flip_left_right(distorted_image)

  # Because these operations are not commutative, consider randomizing
  # randomize the order their operation.
  distorted_image = tf.image.random_brightness(distorted_image,
                                               max_delta=63)
  distorted_image = tf.image.random_contrast(distorted_image,
                                             lower=0.2, upper=1.8)

  # Subtract off the mean and divide by the variance of the pixels.
  float_image = tf.image.per_image_whitening(distorted_image)

  # Ensure that the random shuffling has good mixing properties.
  min_fraction_of_examples_in_queue = 0.1
  min_queue_examples = int(params.num_examples_per_epoch_for_train *
                           min_fraction_of_examples_in_queue)
  print ('Filling queue with %d CIFAR images before starting to train. '
         'This will take a few minutes.' % min_queue_examples)

  # Generate a batch of images and labels by building up a queue of examples.
  return _generate_image_and_label_batch(float_image, read_input.label,
                                         min_queue_examples, batch_size)

def inputs(eval_data, data_dir, batch_size):
  """Construct input for CIFAR evaluation using the Reader ops.

  Args:
    eval_data: bool, indicating if one should use the train or eval data set.
    data_dir: Path to the CIFAR-10 data directory.
    batch_size: Number of images per batch.

  Returns:
    images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
    labels: Labels. 1D tensor of [batch_size] size.
  """
  data_dir = os.path.join(data_dir, "cifar-10-batches-bin")
  if not eval_data:
    filenames = [os.path.join(data_dir, 'data_batch_%d.bin' % i)
                 for i in xrange(1, 6)]
    num_examples_per_epoch = params.num_examples_per_epoch_for_train
  else:
    filenames = [os.path.join(data_dir, 'test_batch.bin')]
    num_examples_per_epoch = params.num_examples_per_epoch_for_eval

  for f in filenames:
    if not gfile.Exists(f):
      raise ValueError('Failed to find file: ' + f)

  # Create a queue that produces the filenames to read.
  filename_queue = tf.train.string_input_producer(filenames)

  # Read examples from files in the filename queue.
  read_input = read_cifar10(filename_queue)
  reshaped_image = tf.cast(read_input.uint8image, tf.float32)

  height = params.image_size
  width = params.image_size

  # Image processing for evaluation.
  # Crop the central [height, width] of the image.
  resized_image = tf.image.resize_image_with_crop_or_pad(reshaped_image,
                                                         width, height)

  # Subtract off the mean and divide by the variance of the pixels.
  float_image = tf.image.per_image_whitening(resized_image)

  # Ensure that the random shuffling has good mixing properties.
  min_fraction_of_examples_in_queue = 0.01
  min_queue_examples = int(num_examples_per_epoch *
                           min_fraction_of_examples_in_queue)

  # Generate a batch of images and labels by building up a queue of examples.
  return _generate_image_and_label_batch(float_image, read_input.label,
                                         min_queue_examples, batch_size)

def maybe_download_and_extract(data_dir):
  """Download and extract the tarball from Alex's website."""
  dest_directory = data_dir
  if not os.path.exists(dest_directory):
    os.makedirs(dest_directory)
  filename = params.data_url.split('/')[-1]
  filepath = os.path.join(dest_directory, filename)
  if not os.path.exists(filepath):
    def _progress(count, block_size, total_size):
      sys.stdout.write('\r>> Downloading %s %.1f%%' % (filename,
          float(count * block_size) / float(total_size) * 100.0))
      sys.stdout.flush()
    filepath, _ = urllib.request.urlretrieve(params.data_url, filepath,
                                             reporthook=_progress)
    print()
    statinfo = os.stat(filepath)
    print('Succesfully downloaded', filename, statinfo.st_size, 'bytes.')
    tarfile.open(filepath, 'r:gz').extractall(dest_directory)
<commit_msg>change cifar10 to be compatible with tensorflow 0.71<commit_after># Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Routine for decoding the CIFAR-10 binary file format."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os

import tensorflow.python.platform
from six.moves import xrange  # pylint: disable=redefined-builtin
import tensorflow as tf

import gzip
import os
import re
import sys
import tarfile
from six.moves import urllib

from tensorflow.python.platform import gfile

import params

def read_cifar10(filename_queue):
  """Reads and parses examples from CIFAR10 data files.

  Recommendation: if you want N-way read parallelism, call this function
  N times.  This will give you N independent Readers reading different
  files & positions within those files, which will give better mixing of
  examples.

  Args:
    filename_queue: A queue of strings with the filenames to read from.

  Returns:
    An object representing a single example, with the following fields:
      height: number of rows in the result (32)
      width: number of columns in the result (32)
      depth: number of color channels in the result (3)
      key: a scalar string Tensor describing the filename & record number
        for this example.
      label: an int32 Tensor with the label in the range 0..9.
      uint8image: a [height, width, depth] uint8 Tensor with the image data
  """

  class CIFAR10Record(object):
    pass
  result = CIFAR10Record()

  # Dimensions of the images in the CIFAR-10 dataset.
  # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the
  # input format.
  label_bytes = 1  # 2 for CIFAR-100
  result.height = 32
  result.width = 32
  result.depth = 3
  image_bytes = result.height * result.width * result.depth
  # Every record consists of a label followed by the image, with a
  # fixed number of bytes for each.
  record_bytes = label_bytes + image_bytes

  # Read a record, getting filenames from the filename_queue.  No
  # header or footer in the CIFAR-10 format, so we leave header_bytes
  # and footer_bytes at their default of 0.
  reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
  result.key, value = reader.read(filename_queue)

  # Convert from a string to a vector of uint8 that is record_bytes long.
  record_bytes = tf.decode_raw(value, tf.uint8)

  # The first bytes represent the label, which we convert from uint8->int32.
  result.label = tf.cast(
      tf.slice(record_bytes, [0], [label_bytes]), tf.int32)

  # The remaining bytes after the label represent the image, which we reshape
  # from [depth * height * width] to [depth, height, width].
  depth_major = tf.reshape(tf.slice(record_bytes, [label_bytes], [image_bytes]),
                           [result.depth, result.height, result.width])
  # Convert from [depth, height, width] to [height, width, depth].
  result.uint8image = tf.transpose(depth_major, [1, 2, 0])

  return result

def _generate_image_and_label_batch(image, label, min_queue_examples,
                                    batch_size):
  """Construct a queued batch of images and labels.

  Args:
    image: 3-D Tensor of [height, width, 3] of type.float32.
    label: 1-D Tensor of type.int32
    min_queue_examples: int32, minimum number of samples to retain
      in the queue that provides of batches of examples.
    batch_size: Number of images per batch.

  Returns:
    images: Images. 4D tensor of [batch_size, height, width, 3] size.
    labels: Labels. 1D tensor of [batch_size] size.
  """
  # Create a queue that shuffles the examples, and then
  # read 'batch_size' images + labels from the example queue.
  num_preprocess_threads = 2
  images, label_batch = tf.train.shuffle_batch(
      [image, label],
      batch_size=batch_size,
      num_threads=num_preprocess_threads,
      capacity=min_queue_examples + 3 * batch_size,
      min_after_dequeue=min_queue_examples)

  # Display the training images in the visualizer.
  tf.image_summary('images', images)

  return images, tf.reshape(label_batch, [batch_size])

def distorted_inputs(data_dir, batch_size):
  """Construct distorted input for CIFAR training using the Reader ops.

  Args:
    data_dir: Path to the CIFAR-10 data directory.
    batch_size: Number of images per batch.

  Returns:
    images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
    labels: Labels. 1D tensor of [batch_size] size.
  """
  data_dir = os.path.join(data_dir, "cifar-10-batches-bin")
  filenames = [os.path.join(data_dir, 'data_batch_%d.bin' % i)
               for i in xrange(1, 6)]
  for f in filenames:
    if not gfile.Exists(f):
      raise ValueError('Failed to find file: ' + f)

  # Create a queue that produces the filenames to read.
  filename_queue = tf.train.string_input_producer(filenames)

  # Read examples from files in the filename queue.
  read_input = read_cifar10(filename_queue)
  reshaped_image = tf.cast(read_input.uint8image, tf.float32)

  height = params.image_size
  width = params.image_size

  # Image processing for training the network. Note the many random
  # distortions applied to the image.

  # Randomly crop a [height, width] section of the image.
  distorted_image = tf.random_crop(reshaped_image,
                                   [height, width, params.num_channels])

  # Randomly flip the image horizontally.
  distorted_image = tf.image.random_flip_left_right(distorted_image)

  # Because these operations are not commutative, consider randomizing
  # randomize the order their operation.
  distorted_image = tf.image.random_brightness(distorted_image,
                                               max_delta=63)
  distorted_image = tf.image.random_contrast(distorted_image,
                                             lower=0.2, upper=1.8)

  # Subtract off the mean and divide by the variance of the pixels.
  float_image = tf.image.per_image_whitening(distorted_image)

  # Ensure that the random shuffling has good mixing properties.
  min_fraction_of_examples_in_queue = 0.1
  min_queue_examples = int(params.num_examples_per_epoch_for_train *
                           min_fraction_of_examples_in_queue)
  print ('Filling queue with %d CIFAR images before starting to train. '
         'This will take a few minutes.' % min_queue_examples)

  # Generate a batch of images and labels by building up a queue of examples.
  return _generate_image_and_label_batch(float_image, read_input.label,
                                         min_queue_examples, batch_size)

def inputs(eval_data, data_dir, batch_size):
  """Construct input for CIFAR evaluation using the Reader ops.

  Args:
    eval_data: bool, indicating if one should use the train or eval data set.
    data_dir: Path to the CIFAR-10 data directory.
    batch_size: Number of images per batch.

  Returns:
    images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
    labels: Labels. 1D tensor of [batch_size] size.
  """
  data_dir = os.path.join(data_dir, "cifar-10-batches-bin")
  if not eval_data:
    filenames = [os.path.join(data_dir, 'data_batch_%d.bin' % i)
                 for i in xrange(1, 6)]
    num_examples_per_epoch = params.num_examples_per_epoch_for_train
  else:
    filenames = [os.path.join(data_dir, 'test_batch.bin')]
    num_examples_per_epoch = params.num_examples_per_epoch_for_eval

  for f in filenames:
    if not gfile.Exists(f):
      raise ValueError('Failed to find file: ' + f)

  # Create a queue that produces the filenames to read.
  filename_queue = tf.train.string_input_producer(filenames)

  # Read examples from files in the filename queue.
  read_input = read_cifar10(filename_queue)
  reshaped_image = tf.cast(read_input.uint8image, tf.float32)

  height = params.image_size
  width = params.image_size

  # Image processing for evaluation.
  # Crop the central [height, width] of the image.
  resized_image = tf.image.resize_image_with_crop_or_pad(reshaped_image,
                                                         width, height)

  # Subtract off the mean and divide by the variance of the pixels.
  float_image = tf.image.per_image_whitening(resized_image)

  # Ensure that the random shuffling has good mixing properties.
  min_fraction_of_examples_in_queue = 0.01
  min_queue_examples = int(num_examples_per_epoch *
                           min_fraction_of_examples_in_queue)

  # Generate a batch of images and labels by building up a queue of examples.
  return _generate_image_and_label_batch(float_image, read_input.label,
                                         min_queue_examples, batch_size)

def maybe_download_and_extract(data_dir):
  """Download and extract the tarball from Alex's website."""
  dest_directory = data_dir
  if not os.path.exists(dest_directory):
    os.makedirs(dest_directory)
  filename = params.data_url.split('/')[-1]
  filepath = os.path.join(dest_directory, filename)
  if not os.path.exists(filepath):
    def _progress(count, block_size, total_size):
      sys.stdout.write('\r>> Downloading %s %.1f%%' % (filename,
          float(count * block_size) / float(total_size) * 100.0))
      sys.stdout.flush()
    filepath, _ = urllib.request.urlretrieve(params.data_url, filepath,
                                             reporthook=_progress)
    print()
    statinfo = os.stat(filepath)
    print('Succesfully downloaded', filename, statinfo.st_size, 'bytes.')
    tarfile.open(filepath, 'r:gz').extractall(dest_directory)
<|endoftext|>


<commit_before>
'''Test cases for QLayout handling of child widgets references'''

import unittest
from sys import getrefcount

from PySide.QtGui import QHBoxLayout, QVBoxLayout, QGridLayout
from PySide.QtGui import QStackedLayout, QFormLayout
from PySide.QtGui import QApplication, QPushButton, QLabel

from helper import UsesQApplication

class SaveReference(UsesQApplication):
    '''Test case to check if QLayout-derived classes increment the refcount
    of widgets passed to addWidget()'''

    # Adding here as nose can't see the qapplication attrib we inherit
    qapplication = True

    def setUp(self):
        #Acquire resources
        super(SaveReference, self).setUp()
        self.widget1 = QPushButton('click me')
        self.widget2 = QLabel('aaa')

    def tearDown(self):
        #Release resources
        del self.widget2
        del self.widget1
        super(SaveReference, self).tearDown()

    def checkLayoutReference(self, layout):
        #Checks the reference cound handling of layout.addWidget
        self.assertEqual(getrefcount(self.widget1), 2)
        layout.addWidget(self.widget1)
        self.assertEqual(getrefcount(self.widget1), 3)

        self.assertEqual(getrefcount(self.widget2), 2)
        layout.addWidget(self.widget2)
        self.assertEqual(getrefcount(self.widget2), 3)

        # Check if doesn't mess around with previous widget refcount
        self.assertEqual(getrefcount(self.widget1), 3)

    def testHBoxReference(self):
        #QHBoxLayout.addWidget reference count
        self.checkLayoutReference(QHBoxLayout())

    def testVBoxReference(self):
        #QVBoxLayout.addWidget reference count
        self.checkLayoutReference(QVBoxLayout())

    def testGridReference(self):
        #QGridLayout.addWidget reference count
        self.checkLayoutReference(QGridLayout())

    def testFormReference(self):
        #QFormLayout.addWidget reference count
        self.checkLayoutReference(QFormLayout())

    def testStackedReference(self):
        #QStackedLayout.addWidget reference count
        self.checkLayoutReference(QStackedLayout())

class MultipleAdd(UsesQApplication):
    '''Test case to check if refcount is incremented only once when multiple
    calls to addWidget are made with the same widget'''

    qapplication = True

    def setUp(self):
        #Acquire resources
        super(MultipleAdd, self).setUp()
        self.widget = QPushButton('click me')
        self.layout = QHBoxLayout()

    def tearDown(self):
        #Release resources
        del self.widget
        del self.layout
        super(MultipleAdd, self).tearDown()

    def testRefCount(self):
        #Multiple QLayout.addWidget calls on the same widget
        self.assertEqual(getrefcount(self.widget), 2)
        self.layout.addWidget(self.widget)
        self.assertEqual(getrefcount(self.widget), 3)
        self.layout.addWidget(self.widget)
        self.assertEqual(getrefcount(self.widget), 3)
        self.layout.addWidget(self.widget)
        self.assertEqual(getrefcount(self.widget), 3)

if __name__ == '__main__':
    unittest.main()
<commit_msg>For now on, the c++ ownership is mandatory on QLayout family classes.<commit_after>
'''Test cases for QLayout handling of child widgets references'''

import unittest
from sys import getrefcount

from PySide.QtGui import QHBoxLayout, QVBoxLayout, QGridLayout, QWidget
from PySide.QtGui import QStackedLayout, QFormLayout
from PySide.QtGui import QApplication, QPushButton, QLabel

from helper import UsesQApplication

class SaveReference(UsesQApplication):
    '''Test case to check if QLayout-derived classes increment the refcount
    of widgets passed to addWidget()'''

    # Adding here as nose can't see the qapplication attrib we inherit
    qapplication = True

    def setUp(self):
        #Acquire resources
        super(SaveReference, self).setUp()
        self.widget1 = QPushButton('click me')
        self.widget2 = QLabel('aaa')

    def tearDown(self):
        #Release resources
        del self.widget2
        del self.widget1
        super(SaveReference, self).tearDown()

    def checkLayoutReference(self, layout):
        #Checks the reference cound handling of layout.addWidget
        self.assertEqual(getrefcount(self.widget1), 2)
        layout.addWidget(self.widget1)
        self.assertEqual(getrefcount(self.widget1), 3)

        self.assertEqual(getrefcount(self.widget2), 2)
        layout.addWidget(self.widget2)
        self.assertEqual(getrefcount(self.widget2), 3)

        # Check if doesn't mess around with previous widget refcount
        self.assertEqual(getrefcount(self.widget1), 3)

    def testMoveLayout(self):
        l = QHBoxLayout()
        self.assertEqual(getrefcount(self.widget1), 2)
        l.addWidget(self.widget1)
        self.assertEqual(getrefcount(self.widget1), 2)

        w = QWidget()
        w.setLayout(l)
        self.assertEqual(getrefcount(self.widget1), 3)

    def testHBoxReference(self):
        #QHBoxLayout.addWidget reference count
        w = QWidget()
        self.checkLayoutReference(QHBoxLayout(w))

    def testVBoxReference(self):
        #QVBoxLayout.addWidget reference count
        w = QWidget()
        self.checkLayoutReference(QVBoxLayout(w))

    def testGridReference(self):
        #QGridLayout.addWidget reference count
        w = QWidget()
        self.checkLayoutReference(QGridLayout(w))

    def testFormReference(self):
        #QFormLayout.addWidget reference count
        w = QWidget()
        self.checkLayoutReference(QFormLayout(w))

    def testStackedReference(self):
        #QStackedLayout.addWidget reference count
        w = QWidget()
        self.checkLayoutReference(QStackedLayout(w))

class MultipleAdd(UsesQApplication):
    '''Test case to check if refcount is incremented only once when multiple
    calls to addWidget are made with the same widget'''

    qapplication = True

    def setUp(self):
        #Acquire resources
        super(MultipleAdd, self).setUp()
        self.widget = QPushButton('click me')
        self.win = QWidget()
        self.layout = QHBoxLayout(self.win)

    def tearDown(self):
        #Release resources
        del self.widget
        del self.layout
        del self.win
        super(MultipleAdd, self).tearDown()

    def testRefCount(self):
        #Multiple QLayout.addWidget calls on the same widget
        self.assertEqual(getrefcount(self.widget), 2)
        self.layout.addWidget(self.widget)
        self.assertEqual(getrefcount(self.widget), 3)
        self.layout.addWidget(self.widget)
        self.assertEqual(getrefcount(self.widget), 3)
        self.layout.addWidget(self.widget)
        self.assertEqual(getrefcount(self.widget), 3)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>"""
Process emails containing CSV data.

The CSVEmailParser class requires the implementor to provide it with
email messages.
It does not contain any code for accessing email accounts.
"""

import csv
import email
from io import StringIO
import logging
import quopri
import re

logger = logging.getLogger(__name__)

class CSVEmailParser:

	required_columns = None

	def process_message(self, message):
		"""
		Process CSV in an email.

		Uses `self.get_message_content()` to get the raw mail content.

		Calls `self.process_csv_content()` with the decoded mail content as the
		last step in this method.

		Parameters
		----------

		message : email.message.Message
			https://docs.python.org/3.4/library/email.message.html#email.message.Message
		"""

		raw_content = self.get_message_content(message)

		content_bytes = quopri.decodestring(raw_content, False)

		content = content_bytes.decode('utf-8')

		self.process_csv_content(content)

	def get_message_content(self, message):
		"""
		Get message content from an email.

		Uses `is_multipart()` and `get_payload()` to recursively call this
		function and build the message content from the parts.

		Parameters
		----------

		message : email.message.Message
			https://docs.python.org/3.4/library/email.message.html#email.message.Message

		Returns
		-------

		str
			Body of the message as a string.
		"""

		if not message.is_multipart():
			return message.get_payload()

		parts = [
			self.get_message_content(payload)
			for payload in message.get_payload()
		]

		return ''.join(parts)

	def process_csv_content(self, content):
		"""
		Process CSV from a string.

		Having this intermediate step between processing the entire CSV file
		and then each of the rows allows another action to happen.

		This may be saving the content to a file as a backup.

		Parameters
		----------

		content : str
			Decoded mail content text containing only CSV.
		"""

		self.process_csv_file(StringIO(content))

	def process_csv_file(self, content_file):
		"""
		Process CSV in a file.

		Creates a `csv.DictReader` from the file and loops over rows contained
		within it.

		Uses `self.check_row_is_valid()` on each row, which throws an exception
		if it fails.

		Calls `self.process_csv_row()` if the row is valid.

		Parameters
		----------

		content_file : file object
			This could be `open(file_name)` or `StringIO(text)`.
		"""

		with content_file as f:
			reader = csv.DictReader(f)
			for row in reader:
				# This will throw an error if the columns are missing.
				self.check_row_is_valid(row)

				self.process_csv_row(row)

	def check_row_is_valid(self, row):
		"""
		Check that required columns exist in the row.

		Column names are case-sensitive.

		Parameters
		----------

		row : dict
			Column names and their values.

		Raises
		------

		KeyError
			If a required column is missing.
		"""

		if self.required_columns is not None:
			# Make sure all the required columns are in the row.
			for column_name in self.required_columns:
				if column_name not in row:
					raise KeyError('Missing column "%s"', column_name)

	def process_csv_row(self, row):
		"""
		Process a single row of CSV content.

		This method should be replaced in a subclass to do something useful.

		Parameters
		----------

		row : dict
			Column names and their values.
		"""

		pass

def type_accepts_message(message, csv_type):
	email_from = email.utils.parseaddr(message['From'])[1]

	check = csv_type['check']

	required_from = check['from']

	# Skip this message if it did not come from the correct sender.
	if email_from != required_from:
		msg = 'Email is not from the correct sender (%s != %s).'
		logger.warning(msg, email_from, required_from)
		return None

	# Use the compiled RegEx if it is available.
	if 'subject_regex_compiled' in check:
		subject_regex = check['subject_regex_compiled']

	# Compile and save the RegEx otherwise.
	else:
		subject_regex_list = check['subject_regex']
		subject_regex = re.compile(''.join(subject_regex_list))
		check['subject_regex_compiled'] = subject_regex

	# Check if the message subject matches the RegEx.
	match_data = subject_regex.match(subject)

	# Skip this message if the subject does not match the RegEx.
	if match_data is None:
		logger.warning('Email subject does not match the required format.')
		return None

	# Get a dict of the values matched in the regex.
	return match_data.groupdict()
<commit_msg>Add missing email subject variable<commit_after>"""
Process emails containing CSV data.

The CSVEmailParser class requires the implementor to provide it with
email messages.
It does not contain any code for accessing email accounts.
"""

import csv
import email
from io import StringIO
import logging
import quopri
import re

logger = logging.getLogger(__name__)

class CSVEmailParser:

	required_columns = None

	def process_message(self, message):
		"""
		Process CSV in an email.

		Uses `self.get_message_content()` to get the raw mail content.

		Calls `self.process_csv_content()` with the decoded mail content as the
		last step in this method.

		Parameters
		----------

		message : email.message.Message
			https://docs.python.org/3.4/library/email.message.html#email.message.Message
		"""

		raw_content = self.get_message_content(message)

		content_bytes = quopri.decodestring(raw_content, False)

		content = content_bytes.decode('utf-8')

		self.process_csv_content(content)

	def get_message_content(self, message):
		"""
		Get message content from an email.

		Uses `is_multipart()` and `get_payload()` to recursively call this
		function and build the message content from the parts.

		Parameters
		----------

		message : email.message.Message
			https://docs.python.org/3.4/library/email.message.html#email.message.Message

		Returns
		-------

		str
			Body of the message as a string.
		"""

		if not message.is_multipart():
			return message.get_payload()

		parts = [
			self.get_message_content(payload)
			for payload in message.get_payload()
		]

		return ''.join(parts)

	def process_csv_content(self, content):
		"""
		Process CSV from a string.

		Having this intermediate step between processing the entire CSV file
		and then each of the rows allows another action to happen.

		This may be saving the content to a file as a backup.

		Parameters
		----------

		content : str
			Decoded mail content text containing only CSV.
		"""

		self.process_csv_file(StringIO(content))

	def process_csv_file(self, content_file):
		"""
		Process CSV in a file.

		Creates a `csv.DictReader` from the file and loops over rows contained
		within it.

		Uses `self.check_row_is_valid()` on each row, which throws an exception
		if it fails.

		Calls `self.process_csv_row()` if the row is valid.

		Parameters
		----------

		content_file : file object
			This could be `open(file_name)` or `StringIO(text)`.
		"""

		with content_file as f:
			reader = csv.DictReader(f)
			for row in reader:
				# This will throw an error if the columns are missing.
				self.check_row_is_valid(row)

				self.process_csv_row(row)

	def check_row_is_valid(self, row):
		"""
		Check that required columns exist in the row.

		Column names are case-sensitive.

		Parameters
		----------

		row : dict
			Column names and their values.

		Raises
		------

		KeyError
			If a required column is missing.
		"""

		if self.required_columns is not None:
			# Make sure all the required columns are in the row.
			for column_name in self.required_columns:
				if column_name not in row:
					raise KeyError('Missing column "%s"', column_name)

	def process_csv_row(self, row):
		"""
		Process a single row of CSV content.

		This method should be replaced in a subclass to do something useful.

		Parameters
		----------

		row : dict
			Column names and their values.
		"""

		pass

def type_accepts_message(message, csv_type):
	email_from = email.utils.parseaddr(message['From'])[1]

	subject = message['Subject']

	check = csv_type['check']

	required_from = check['from']

	# Skip this message if it did not come from the correct sender.
	if email_from != required_from:
		msg = 'Email is not from the correct sender (%s != %s).'
		logger.warning(msg, email_from, required_from)
		return None

	# Use the compiled RegEx if it is available.
	if 'subject_regex_compiled' in check:
		subject_regex = check['subject_regex_compiled']

	# Compile and save the RegEx otherwise.
	else:
		subject_regex_list = check['subject_regex']
		subject_regex = re.compile(''.join(subject_regex_list))
		check['subject_regex_compiled'] = subject_regex

	# Check if the message subject matches the RegEx.
	match_data = subject_regex.match(subject)

	# Skip this message if the subject does not match the RegEx.
	if match_data is None:
		logger.warning('Email subject does not match the required format.')
		return None

	# Get a dict of the values matched in the regex.
	return match_data.groupdict()
<|endoftext|>


<commit_before>import nose.tools
from source.referee_highchair import *

def test_referee_is_an_object():
    referee = Referee()
    assert isinstance(referee, object) == True

def test_check_for_draw_can_do_so():
    referee = Referee()
    false_board = [1,10,1,0,10,0,1,0,10]
    true_board = [1,1,10,10,10,1,1,10,1]
    assert referee.check_for_draw(false_board) == False
    assert referee.check_for_draw(true_board) == True

<commit_msg>fix test, we don't need to pass arguments<commit_after>import nose.tools
from source.referee_highchair import *

def test_referee_is_an_object():
    referee = Referee()
    assert isinstance(referee, object) == True

def test_check_for_draw_can_do_so():
    referee = Referee()
    false_board = [1,10,1,0,10,0,1,0,10]
    true_board = [1,1,10,10,10,1,1,10,1]
    assert referee.check_for_draw() == False
    assert referee.check_for_draw() == True

<|endoftext|>


<commit_before># script to grab aspect_1_ids for asp1 products in the database and
# make a table to assist in cron job V&V processing
# Licensed under a 3-clause BSD style license - see LICENSE.rst

import os
import re
import logging
from astropy.io import fits
import Ska.DBI
from glob import glob
import gzip
from pathlib import Path

from mica.common import MICA_ARCHIVE

DEFAULT_CONFIG = dict(data_root=os.path.join(MICA_ARCHIVE, 'asp1'),
                      sql_def='processing_asp_l1_def.sql')

def update(obsids, config=None):
    from mica.archive import asp_l1
    """
    For a list of obsids, update the mica table of aspect 1 processing
    """
    if config is None:
        config = DEFAULT_CONFIG
    logger = logging.getLogger('asp 1 proc table')
    logger.setLevel(logging.INFO)
    if not len(logger.handlers):
        logger.addHandler(logging.StreamHandler())

    apstat_db = dict(dbi='sybase',
                     server='sqlsao',
                     database='axafapstat')
    proc_db_file = os.path.join(MICA_ARCHIVE, 'asp1', 'processing_asp_l1.db3')
    if not os.path.exists(proc_db_file) or os.stat(proc_db_file).st_size == 0:
        if not os.path.exists(config['data_root']):
            os.makedirs(config['data_root'])
        logger.info("creating aspect_1_proc db from {}".format(
            config['sql_def']))
        db_sql = Path(__file__).parent / config['sql_def']
        db_init_cmds = open(db_sql).read()
        proc_db = Ska.DBI.DBI(dbi='sqlite', server=proc_db_file)
        proc_db.execute(db_init_cmds)
    else:
        proc_db = Ska.DBI.DBI(dbi='sqlite', server=proc_db_file)
    archdb = Ska.DBI.DBI(dbi='sqlite',
                         server=os.path.join(config['data_root'],
                                             'archfiles.db3'))
    for obs in obsids:
        logger.info("Adding asp1 processing for obs {}".format(obs))
        asols = asp_l1.get_files(obsid=obs, content='ASPSOL', revision='all')
        for sol in asols:
            logger.info("\tprocessing {}".format(sol))
            procdir = os.path.dirname(sol)
            logfile = glob(os.path.join(procdir, "*log*"))[0]
            aspect_log = gzip.open(logfile).read()
            # read the obspar version with a regex from the log
            obspar_version = int(
                re.search("axaff\d{5}_\d{3}N(\d{3})_obs0a\.par",
                          aspect_log).group(1))
            hdus = fits.open(sol)
            obi = hdus[1].header['OBI_NUM']
            revision = hdus[1].header['REVISION']
            with Ska.DBI.DBI(**apstat_db) as db:
                aspect_1 = db.fetchall("""SELECT * FROM aspect_1
                                             WHERE obsid = {obsid}
                                             AND obi = {obi}
                                             AND revision = {revision}
                                          """.format(obsid=obs,
                                                     obi=obi,
                                                     revision=revision))
            if len(aspect_1) > 1:
                raise ValueError
            indb = proc_db.fetchall("""SELECT * from aspect_1_proc
                                       WHERE aspect_1_id = {}
                                    """.format(aspect_1[0]['aspect_1_id']))
            archrec = archdb.fetchall("""select * from archfiles
                                     where obsid = {obsid}
                                     and revision = {revision}
                                     and content = 'ASPSOL'
                                  """.format(obsid=obs,
                                             revision=revision))
            # if already in the database, reset the V&V
            # it doesn't really need to be redone (could just be relinked)
            # but this should at least make the bookkeeping consistent
            if len(indb):
                logger.info("Resetting vv_complete to 0 for obsid {obsid} rev {revision}".format(
                        obsid=obs, revision=revision))
                proc_db.execute("""UPDATE aspect_1_proc SET vv_complete = 0
                                   WHERE obsid = {obsid}
                                   AND revision = {revision}
                                """.format(obsid=obs,
                                           revision=revision))
            else:
                proc_db.insert(dict(aspect_1_id=aspect_1[0]['aspect_1_id'],
                                    obsid=obs,
                                    obi=obi,
                                    revision=revision,
                                    obspar_version=obspar_version,
                                    ap_date=str(aspect_1[0]['ap_date'])),
                               'aspect_1_proc')
            isdefault = archrec[0]['isdefault']
            if isdefault is not None:
                proc_db.execute("""UPDATE aspect_1_proc SET isdefault = {isdefault}
                                   WHERE obsid = {obsid}
                                   AND revision = {revision}
                                """.format(isdefault=archrec[0]['isdefault'],
                                           obsid=obs,
                                           revision=revision))
            logger.info("\tUpdated table for {}".format(obs))
<commit_msg>Update for py3 and new log file name<commit_after># script to grab aspect_1_ids for asp1 products in the database and
# make a table to assist in cron job V&V processing
# Licensed under a 3-clause BSD style license - see LICENSE.rst

import os
import re
import logging
from astropy.io import fits
import Ska.DBI
from glob import glob
import gzip
from pathlib import Path

from mica.common import MICA_ARCHIVE

DEFAULT_CONFIG = dict(data_root=os.path.join(MICA_ARCHIVE, 'asp1'),
                      sql_def='processing_asp_l1_def.sql')

def update(obsids, config=None):
    from mica.archive import asp_l1
    """
    For a list of obsids, update the mica table of aspect 1 processing
    """
    if config is None:
        config = DEFAULT_CONFIG
    logger = logging.getLogger('asp 1 proc table')
    logger.setLevel(logging.INFO)
    if not len(logger.handlers):
        logger.addHandler(logging.StreamHandler())

    apstat_db = dict(dbi='sybase',
                     server='sqlsao',
                     database='axafapstat')
    proc_db_file = os.path.join(MICA_ARCHIVE, 'asp1', 'processing_asp_l1.db3')
    if not os.path.exists(proc_db_file) or os.stat(proc_db_file).st_size == 0:
        if not os.path.exists(config['data_root']):
            os.makedirs(config['data_root'])
        logger.info("creating aspect_1_proc db from {}".format(
            config['sql_def']))
        db_sql = Path(__file__).parent / config['sql_def']
        db_init_cmds = open(db_sql).read()
        proc_db = Ska.DBI.DBI(dbi='sqlite', server=proc_db_file)
        proc_db.execute(db_init_cmds)
    else:
        proc_db = Ska.DBI.DBI(dbi='sqlite', server=proc_db_file)
    archdb = Ska.DBI.DBI(dbi='sqlite',
                         server=os.path.join(config['data_root'],
                                             'archfiles.db3'))
    for obs in obsids:
        logger.info("Adding asp1 processing for obs {}".format(obs))
        asols = asp_l1.get_files(obsid=obs, content='ASPSOL', revision='all')
        for sol in asols:
            logger.info("\tprocessing {}".format(sol))
            procdir = os.path.dirname(sol)

            # As of DS 10.8.3, there are both "com" logs and per-ai logs.
            # This glob should get the per-ai logs.
            logfile = glob(os.path.join(procdir, "asp_l1_f*log*"))[0]
            aspect_log = gzip.open(logfile, 'rt').read()

            # read the obspar version with a regex from the log
            obspar_version = int(
                re.search("axaff\d{5}_\d{3}N(\d{3})_obs0a\.par",
                          aspect_log).group(1))
            hdus = fits.open(sol)
            obi = hdus[1].header['OBI_NUM']
            revision = hdus[1].header['REVISION']
            with Ska.DBI.DBI(**apstat_db) as db:
                aspect_1 = db.fetchall("""SELECT * FROM aspect_1
                                             WHERE obsid = {obsid}
                                             AND obi = {obi}
                                             AND revision = {revision}
                                          """.format(obsid=obs,
                                                     obi=obi,
                                                     revision=revision))
            if len(aspect_1) > 1:
                raise ValueError
            indb = proc_db.fetchall("""SELECT * from aspect_1_proc
                                       WHERE aspect_1_id = {}
                                    """.format(aspect_1[0]['aspect_1_id']))
            archrec = archdb.fetchall("""select * from archfiles
                                     where obsid = {obsid}
                                     and revision = {revision}
                                     and content = 'ASPSOL'
                                  """.format(obsid=obs,
                                             revision=revision))
            # if already in the database, reset the V&V
            # it doesn't really need to be redone (could just be relinked)
            # but this should at least make the bookkeeping consistent
            if len(indb):
                logger.info("Resetting vv_complete to 0 for obsid {obsid} rev {revision}".format(
                        obsid=obs, revision=revision))
                proc_db.execute("""UPDATE aspect_1_proc SET vv_complete = 0
                                   WHERE obsid = {obsid}
                                   AND revision = {revision}
                                """.format(obsid=obs,
                                           revision=revision))
            else:
                proc_db.insert(dict(aspect_1_id=aspect_1[0]['aspect_1_id'],
                                    obsid=obs,
                                    obi=obi,
                                    revision=revision,
                                    obspar_version=obspar_version,
                                    ap_date=str(aspect_1[0]['ap_date'])),
                               'aspect_1_proc')
            isdefault = archrec[0]['isdefault']
            if isdefault is not None:
                proc_db.execute("""UPDATE aspect_1_proc SET isdefault = {isdefault}
                                   WHERE obsid = {obsid}
                                   AND revision = {revision}
                                """.format(isdefault=archrec[0]['isdefault'],
                                           obsid=obs,
                                           revision=revision))
            logger.info("\tUpdated table for {}".format(obs))
<|endoftext|>


<commit_before>from infi.clickhouse_orm import migrations

operations = [
    migrations.RunSQL("INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-01', 1, 1, 'test') "),
    migrations.RunSQL([
        "INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-02', 2, 2, 'test2') ",
        "INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-03', 3, 3, 'test3') ",
    ])
<commit_msg>Missing brackets<commit_after>from infi.clickhouse_orm import migrations

operations = [
    migrations.RunSQL("INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-01', 1, 1, 'test') "),
    migrations.RunSQL([
        "INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-02', 2, 2, 'test2') ",
        "INSERT INTO `mig` (date, f1, f3, f4) VALUES ('2016-01-03', 3, 3, 'test3') ",
    ])
]
<|endoftext|>


<commit_before>"""

Common Exceptions

"""

UNREGISTERED_FMT = """
Task of kind %s is not registered, please make sure it's imported.
""".strip()

class QueueNotFound(KeyError):
    """Task routed to a queue not in CELERY_QUEUES."""
    pass

class TimeLimitExceeded(Exception):
    """The time limit has been exceeded and the job has been terminated."""
    pass

class SoftTimeLimitExceeded(Exception):
    """The soft time limit has been exceeded. This exception is raised
    to give the task a chance to clean up."""
    pass

class WorkerLostError(Exception):
    """The worker processing a task has exited prematurely."""
    pass

class ImproperlyConfigured(Exception):
    """Celery is somehow improperly configured."""
    pass

class NotRegistered(KeyError):
    """The task is not registered."""

    def __init__(self, message, *args, **kwargs):
        message = UNREGISTERED_FMT % str(message)
        KeyError.__init__(self, message, *args, **kwargs)

class AlreadyRegistered(Exception):
    """The task is already registered."""
    pass

class TimeoutError(Exception):
    """The operation timed out."""
    pass

class MaxRetriesExceededError(Exception):
    """The tasks max restart limit has been exceeded."""
    pass

class TaskRevokedError(Exception):
    """The task has been revoked, so no result available."""
    pass

class NotConfigured(UserWarning):
    """Celery has not been configured, as no config module has been found."""

<commit_msg>Inserting Exception that was accidentally removed<commit_after>"""

Common Exceptions

"""

UNREGISTERED_FMT = """
Task of kind %s is not registered, please make sure it's imported.
""".strip()

class QueueNotFound(KeyError):
    """Task routed to a queue not in CELERY_QUEUES."""
    pass

class TimeLimitExceeded(Exception):
    """The time limit has been exceeded and the job has been terminated."""
    pass

class SoftTimeLimitExceeded(Exception):
    """The soft time limit has been exceeded. This exception is raised
    to give the task a chance to clean up."""
    pass

class WorkerLostError(Exception):
    """The worker processing a task has exited prematurely."""
    pass

class ImproperlyConfigured(Exception):
    """Celery is somehow improperly configured."""
    pass

class NotRegistered(KeyError):
    """The task is not registered."""

    def __init__(self, message, *args, **kwargs):
        message = UNREGISTERED_FMT % str(message)
        KeyError.__init__(self, message, *args, **kwargs)

class AlreadyRegistered(Exception):
    """The task is already registered."""
    pass

class TimeoutError(Exception):
    """The operation timed out."""
    pass

class MaxRetriesExceededError(Exception):
    """The tasks max restart limit has been exceeded."""
    pass

class RetryTaskError(Exception):
    """The task is to be retried later."""

    def __init__(self, message, exc, *args, **kwargs):
        self.exc = exc
        Exception.__init__(self, message, exc, *args,
                           **kwargs)

class TaskRevokedError(Exception):
    """The task has been revoked, so no result available."""
    pass

class NotConfigured(UserWarning):
    """Celery has not been configured, as no config module has been found."""

<|endoftext|>