<commit_before># -*- coding: utf-8 -*-
#
# Hermes documentation build configuration file, created by
# sphinx-quickstart on Sat May 14 12:32:54 2011.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

class Mock(object):
    def __init__(self, *args, **kwargs):
        pass

    def __call__(self, *args, **kwargs):
        return Mock()

    @classmethod
    def __getattr__(cls, name):
        if name in ('__file__', '__path__'):
            return '/dev/null'
        elif name[0] == name[0].upper():
            mockType = type(name, (), {})
            mockType.__module__ = __name__
            return mockType
        else:
            return Mock()

MOCK_MODULES = ['pycurl', 'pyaml', 'pyyaml', 'pyvbox']
for mod_name in MOCK_MODULES:
    sys.modules[mod_name] = Mock()

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc',
              #'rst2pdf.pdfbuilder',
              'sphinx.ext.doctest',
              'sphinx.ext.intersphinx',
              'sphinx.ext.todo',
              'sphinx.ext.coverage',
              'sphinx.ext.pngmath',
              'sphinx.ext.ifconfig',
              'sphinx.ext.viewcode',
              #'sphinxcontrib.httpdomain',    ## see http://packages.python.org/sphinxcontrib-httpdomain/
              #'sphinx.ext.graphviz ',
              #'sphinx.ext.inheritance_diagram',
              #'qubic.sphinx.graphvizinclude',
              #'sphinxcontrib.programoutput'
              ]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['.templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'doc_index'

# General information about the project.
project = u'candelabra'
copyright = u'Alvaro Saurin'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.0'
# The full version, including alpha/beta/rc tags.
release = '1.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# Add any paths that contain custom themes here, relative to this directory.
sys.path.append(os.path.abspath('themes'))
html_theme_path = ['themes']

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
# use 'agogo', 'nature','haiku'
html_theme = 'kr'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'candelabra-doc'

# -- Options for LaTeX output --------------------------------------------------
# http://sphinx.pocoo.org/config.html#options-for-latex-output

# The paper size ('letter' or 'a4').
latex_paper_size = 'a4'

# The font size ('10pt', '11pt' or '12pt').
latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('doc_index',
   'candelabra.tex',
   u'Candelabra Documentation',
   u'Alvaro Saurin',
   'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = "doc_logo.png"

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = False

latex_elements = {
    'fontpkg': '',
    'fncychap': '\\usepackage[Sonny]{fncychap}',
    'printindex': '',                                  ## this removes the index
}

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'candelabra', u'Candelabra Documentation',
     [u'Alvaro Saurin'], 1)
]

# -- Options for Epub output ---------------------------------------------------

# Bibliographic Dublin Core info.
epub_title = u'candelabra'
epub_author = u'Author'
epub_publisher = u'Alvaro Saurin'
epub_copyright = u'Alvaro Saurin'

# The language of the text. It defaults to the language option
# or en if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files shat should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
#epub_exclude_files = []

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

# Allow duplicate toc entries.
#epub_tocdup = True

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'http://docs.python.org/': None}

# -- Options for PDF output --------------------------------------------------

# Grouping the document tree into PDF files. List of tuples
# (source start file, target name, title, author, options).
#
# If there is more than one author, separate them with \\.
# For example: r'Guido van Rossum\\Fred L. Drake, Jr., editor'
#
# The options element is a dictionary that lets you override
# this config per-document.
# For example,
# ('index', u'MyProject', u'My Project', u'Author Name',
#  dict(pdf_compressed = True))
# would mean that specific document would be compressed
# regardless of the global pdf_compressed setting.

pdf_documents = [
  ('index',
   u'candelabra',
   u'Candelabra Documentation',
   u'Alvaro Saurin'),
]

# A comma-separated list of custom stylesheets. Example:
pdf_stylesheets = ['sphinx', 'kerning', 'a4']

# Create a compressed PDF
# Use True/False or 1/0
# Example: compressed=True
pdf_compressed = False

# A colon-separated list of folders to search for fonts. Example:
pdf_font_path = ['/usr/share/fonts', '/usr/share/texmf-dist/fonts/']

# Language to be used for hyphenation support
#pdf_language = "en_US"

# Mode for literal blocks wider than the frame. Can be
# overflow, shrink or truncate
#pdf_fit_mode = "shrink"

# Section level that forces a break page.
# For example: 1 means top-level sections start in a new page
# 0 means disabled
pdf_break_level = 1

# When a section starts in a new page, force it to be 'even', 'odd',
# or just use 'any'
#pdf_breakside = 'any'

# Insert footnotes where they are defined instead of
# at the end.
pdf_inline_footnotes = True

# verbosity level. 0 1 or 2
#pdf_verbosity = 0

# If false, no index is generated.
pdf_use_index = False

# If false, no modindex is generated.
#pdf_use_modindex = True

# If false, no coverpage is generated.
#pdf_use_coverpage = True

# Name of the cover page template to use
#pdf_cover_template = 'sphinxcover.tmpl'

# Documents to append as an appendix to all manuals.
#pdf_appendices = []

# Enable experimental feature to split table cells. Use it
# if you get "DelayedTable too big" errors
#pdf_splittables = False

# Set the default DPI for images
#pdf_default_dpi = 72

# Enable rst2pdf extension modules (default is only vectorpdf)
# you need vectorpdf if you want to use sphinx's graphviz support
#pdf_extensions = ['vectorpdf']

# Page template name for "regular" pages
#pdf_page_template = 'cutePage'

# Show Table Of Contents at the beginning?
#pdf_use_toc = True

# How many levels deep should the table of contents be?
pdf_toc_depth = 9999

# Add section number to section references
pdf_use_numbered_links = False

# Background images fitting mode
pdf_fit_background_mode = 'scale'
<commit_msg>Minor docs updates<commit_after># -*- coding: utf-8 -*-
#
# Hermes documentation build configuration file, created by
# sphinx-quickstart on Sat May 14 12:32:54 2011.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

class Mock(object):
    def __init__(self, *args, **kwargs):
        pass

    def __call__(self, *args, **kwargs):
        return Mock()

    @classmethod
    def __getattr__(cls, name):
        if name in ('__file__', '__path__'):
            return '/dev/null'
        elif name[0] == name[0].upper():
            mockType = type(name, (), {})
            mockType.__module__ = __name__
            return mockType
        else:
            return Mock()

MOCK_MODULES = ['pyaml', 'PyYAML', 'pyvbox']
for mod_name in MOCK_MODULES:
    sys.modules[mod_name] = Mock()

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc',
              #'rst2pdf.pdfbuilder',
              'sphinx.ext.doctest',
              'sphinx.ext.intersphinx',
              'sphinx.ext.todo',
              'sphinx.ext.coverage',
              'sphinx.ext.pngmath',
              'sphinx.ext.ifconfig',
              'sphinx.ext.viewcode',
              #'sphinxcontrib.httpdomain',    ## see http://packages.python.org/sphinxcontrib-httpdomain/
              #'sphinx.ext.graphviz ',
              #'sphinx.ext.inheritance_diagram',
              #'qubic.sphinx.graphvizinclude',
              #'sphinxcontrib.programoutput'
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['.templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'doc_index'

# General information about the project.
project = u'candelabra'
copyright = u'Alvaro Saurin'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.0'
# The full version, including alpha/beta/rc tags.
release = '1.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# Add any paths that contain custom themes here, relative to this directory.
sys.path.append(os.path.abspath('themes'))
html_theme_path = ['themes']

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
# use 'agogo', 'nature','haiku'
html_theme = 'kr'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'candelabra-doc'

# -- Options for LaTeX output --------------------------------------------------
# http://sphinx.pocoo.org/config.html#options-for-latex-output

# The paper size ('letter' or 'a4').
latex_paper_size = 'a4'

# The font size ('10pt', '11pt' or '12pt').
latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
    ('doc_index',
     'candelabra.tex',
     u'Candelabra Documentation',
     u'Alvaro Saurin',
     'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = "doc_logo.png"

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = False

latex_elements = {
    'fontpkg': '',
    'fncychap': '\\usepackage[Sonny]{fncychap}',
    'printindex': '', ## this removes the index
}

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'candelabra', u'Candelabra Documentation',
     [u'Alvaro Saurin'], 1)
]

# -- Options for Epub output ---------------------------------------------------

# Bibliographic Dublin Core info.
epub_title = u'candelabra'
epub_author = u'Author'
epub_publisher = u'Alvaro Saurin'
epub_copyright = u'Alvaro Saurin'

# The language of the text. It defaults to the language option
# or en if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files shat should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
#epub_exclude_files = []

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

# Allow duplicate toc entries.
#epub_tocdup = True

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'http://docs.python.org/': None}

# -- Options for PDF output --------------------------------------------------

# Grouping the document tree into PDF files. List of tuples
# (source start file, target name, title, author, options).
#
# If there is more than one author, separate them with \\.
# For example: r'Guido van Rossum\\Fred L. Drake, Jr., editor'
#
# The options element is a dictionary that lets you override
# this config per-document.
# For example,
# ('index', u'MyProject', u'My Project', u'Author Name',
#  dict(pdf_compressed = True))
# would mean that specific document would be compressed
# regardless of the global pdf_compressed setting.

pdf_documents = [
    ('index',
     u'candelabra',
     u'Candelabra Documentation',
     u'Alvaro Saurin'),
]

# A comma-separated list of custom stylesheets. Example:
pdf_stylesheets = ['sphinx', 'kerning', 'a4']

# Create a compressed PDF
# Use True/False or 1/0
# Example: compressed=True
pdf_compressed = False

# A colon-separated list of folders to search for fonts. Example:
pdf_font_path = ['/usr/share/fonts', '/usr/share/texmf-dist/fonts/']

# Language to be used for hyphenation support
#pdf_language = "en_US"

# Mode for literal blocks wider than the frame. Can be
# overflow, shrink or truncate
#pdf_fit_mode = "shrink"

# Section level that forces a break page.
# For example: 1 means top-level sections start in a new page
# 0 means disabled
pdf_break_level = 1

# When a section starts in a new page, force it to be 'even', 'odd',
# or just use 'any'
#pdf_breakside = 'any'

# Insert footnotes where they are defined instead of
# at the end.
pdf_inline_footnotes = True

# verbosity level. 0 1 or 2
#pdf_verbosity = 0

# If false, no index is generated.
pdf_use_index = False

# If false, no modindex is generated.
#pdf_use_modindex = True

# If false, no coverpage is generated.
#pdf_use_coverpage = True

# Name of the cover page template to use
#pdf_cover_template = 'sphinxcover.tmpl'

# Documents to append as an appendix to all manuals.
#pdf_appendices = []

# Enable experimental feature to split table cells. Use it
# if you get "DelayedTable too big" errors
#pdf_splittables = False

# Set the default DPI for images
#pdf_default_dpi = 72

# Enable rst2pdf extension modules (default is only vectorpdf)
# you need vectorpdf if you want to use sphinx's graphviz support
#pdf_extensions = ['vectorpdf']

# Page template name for "regular" pages
#pdf_page_template = 'cutePage'

# Show Table Of Contents at the beginning?
#pdf_use_toc = True

# How many levels deep should the table of contents be?
pdf_toc_depth = 9999

# Add section number to section references
pdf_use_numbered_links = False

# Background images fitting mode
pdf_fit_background_mode = 'scale'
<|endoftext|>


<commit_before>'''
Created on Oct 3, 2010

@author: Mark V Systems Limited
(c) Copyright 2010 Mark V Systems Limited, All rights reserved.
'''
import tempfile, os, pickle, sys, logging, gettext, json
from arelle import ModelManager
from arelle.Locale import getLanguageCodes
from collections import defaultdict

class Cntlr(object):

    __version__ = "0.0.4"

    def __init__(self, logFileName=None, logFileMode=None, logFileEncoding=None, logFormat=None):
        self.hasWin32gui = False
        self.hasGui = False
        if sys.platform == "darwin":
            self.isMac = True
            self.isMSW = False
            self.userAppDir = os.path.expanduser("~") + "/Library/Application Support/Arelle"
            self.contextMenuClick = "<Button-2>"
            self.hasClipboard = True
            self.updateURL = "http://arelle.org/downloads/8"
        elif sys.platform.startswith("win"):
            self.isMac = False
            self.isMSW = True
            tempDir = tempfile.gettempdir()
            if tempDir.endswith('local\\temp'):
                self.userAppDir = tempDir[:-10] + 'local\\Arelle'
            else:
                self.userAppDir = tempDir + os.sep + 'arelle'
            try:
                import win32clipboard
                self.hasClipboard = True
            except ImportError:
                self.hasClipboard = False
            try:
                import win32gui
                self.hasWin32gui = True # active state for open file dialogs
            except ImportError:
                pass
            self.contextMenuClick = "<Button-3>"
            if "64 bit" in sys.version:
                self.updateURL = "http://arelle.org/downloads/9"
            else: # 32 bit
                self.updateURL = "http://arelle.org/downloads/10"
        else: # Unix/Linux
            self.isMac = False
            self.isMSW = False
            self.userAppDir = os.path.join(
                   os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")),
                   "arelle")
            try:
                import gtk
                self.hasClipboard = True
            except ImportError:
                self.hasClipboard = False
            self.contextMenuClick = "<Button-3>"
        self.moduleDir = os.path.dirname(__file__)
        # for python 3.2 remove __pycache__
        if self.moduleDir.endswith("__pycache__"):
            self.moduleDir = os.path.dirname(self.moduleDir)
        if self.moduleDir.endswith("python32.zip/arelle"):
            '''
            distZipFile = os.path.dirname(self.moduleDir)
            d = os.path.join(self.userAppDir, "arelle")
            self.configDir = os.path.join(d, "config")
            self.imagesDir = os.path.join(d, "images")
            import zipfile
            distZip = zipfile.ZipFile(distZipFile, mode="r")
            distNames = distZip.namelist()
            distZip.extractall(path=self.userAppDir,
                               members=[f for f in distNames if "/config/" in f or "/images/" in f]
                               )
            distZip.close()
            '''
            resources = os.path.dirname(os.path.dirname(os.path.dirname(self.moduleDir)))
            self.configDir = os.path.join(resources, "config")
            self.imagesDir = os.path.join(resources, "images")
            self.localeDir = os.path.join(resources, "locale")
        elif self.moduleDir.endswith("library.zip\\arelle"): # cx_Freexe
            resources = os.path.dirname(os.path.dirname(self.moduleDir))
            self.configDir = os.path.join(resources, "config")
            self.imagesDir = os.path.join(resources, "images")
            self.localeDir = os.path.join(resources, "locale")
        else:
            self.configDir = os.path.join(self.moduleDir, "config")
            self.imagesDir = os.path.join(self.moduleDir, "images")
            self.localeDir = os.path.join(self.moduleDir, "locale")
        try:
            from arelle import webserver
            self.hasWebServer = True
        except ImportError:
            self.hasWebServer = False
        # assert that app dir must exist
        if not os.path.exists(self.userAppDir):
            os.makedirs(self.userAppDir)
        # load config if it exists
        self.configPickleFile = self.userAppDir + os.sep + "config.pickle"
        self.config = None
        if os.path.exists(self.configPickleFile):
            try:
                with open(self.configPickleFile, 'rb') as f:
                    self.config = pickle.load(f)
            except Exception as ex:
                self.config = None # restart with a new config
        if not self.config:
            self.config = {
                'fileHistory': [],
                'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100),
            }

        # start language translation for domain
        try:
            gettext.translation("arelle", self.localeDir, getLanguageCodes()).install()
        except Exception as msg:
            gettext.install("arelle", self.localeDir)

        from arelle.WebCache import WebCache
        self.webCache = WebCache(self, self.config.get("proxySettings"))
        self.modelManager = ModelManager.initialize(self)

        if logFileName: # use default logging
            self.logger = logging.getLogger("arelle")
            if logFileName == "logToPrint":
                self.logHandler = LogToPrintHandler()
            elif logFileName == "logToBuffer":
                self.logHandler = LogToBufferHandler()
            elif logFileName.endswith(".xml"):
                self.logHandler = LogToXmlHandler(filename=logFileName)
                logFormat = "%(message)s"
            else:
                self.logHandler = logging.FileHandler(filename=logFileName,
                                                      mode=logFileMode if logFileMode else "w",
                                                      encoding=logFileEncoding if logFileEncoding else "utf-8")
            self.logHandler.level = logging.DEBUG
            self.logHandler.setFormatter(LogFormatter(logFormat if logFormat else "%(asctime)s [%(messageCode)s] %(message)s - %(file)s\n"))
            self.logger.addHandler(self.logHandler)
        else:
            self.logger = None

    def addToLog(self, message, messageCode="", file=""):
        # if there is a default logger, use it with dummy file name and arguments
        if self.logger is not None:
            self.logger.info(message, extra={"messageCode":messageCode,"refs":[{"href": file}]})
        else:
            print(message) # allows printing on standard out

    def showStatus(self, message, clearAfter=None):
        # dummy status line for batch operation
        pass

    def close(self, saveConfig=False):
        if saveConfig:
            self.saveConfig()
        if self.logger is not None:
            self.logHandler.close()

    def saveConfig(self):
        with open(self.configPickleFile, 'wb') as f:
            pickle.dump(self.config, f, pickle.HIGHEST_PROTOCOL)

    # default non-threaded viewModelObject
    def viewModelObject(self, modelXbrl, objectId):
        modelXbrl.viewModelObject(objectId)

    def reloadViews(self, modelXbrl):
        pass

    def rssWatchUpdateOption(self, **args):
        pass

    # default web authentication password
    def internet_user_password(self, host, realm):
        return ('myusername','mypassword')

    # if no text, then return what is on the clipboard, otherwise place text onto clipboard
    def clipboardData(self, text=None):
        if self.hasClipboard:
            try:
                if sys.platform == "darwin":
                    import subprocess
                    if text is None:
                        p = subprocess.Popen(['pbpaste'], stdout=subprocess.PIPE)
                        retcode = p.wait()
                        text = p.stdout.read()
                        return text
                    else:
                        p = subprocess.Popen(['pbcopy'], stdin=subprocess.PIPE)
                        p.stdin.write(text)
                        p.stdin.close()
                        retcode = p.wait()
                elif sys.platform.startswith("win"):
                    import win32clipboard
                    win32clipboard.OpenClipboard()
                    if text is None:
                        if win32clipboard.IsClipboardFormatAvailable(win32clipboard.CF_TEXT):
                            return win32clipboard.GetClipboardData().decode("utf8")
                    else:
                        win32clipboard.EmptyClipboard()
                        win32clipboard.SetClipboardData(win32clipboard.CF_TEXT, text.encode("utf8"))
                    win32clipboard.CloseClipboard()
                else: # Unix/Linux
                    import gtk
                    clipbd = gtk.Clipboard(display=gtk.gdk.display_get_default(), selection="CLIPBOARD")
                    if text is None:
                        return clipbd.wait_for_text().decode("utf8")
                    else:
                        clipbd.set_text(text.encode("utf8"), len=-1)
            except Exception:
                pass
        return None

class LogFormatter(logging.Formatter):
    def __init__(self, fmt=None, datefmt=None):
        super(LogFormatter, self).__init__(fmt, datefmt)

    def format(self, record):
        # provide a file parameter made up from refs entries
        fileLines = defaultdict(set)
        for ref in record.refs:
            fileLines[ref["href"].partition("#")[0]].add(ref.get("sourceLine"))
        record.file = ", ".join(file + " " + ', '.join(str(line) for line in lines if line)
                                for file, lines in sorted(fileLines.items()))
        formattedMessage = super(LogFormatter, self).format(record)
        del record.file
        return formattedMessage

class LogToPrintHandler(logging.Handler):
    def emit(self, logRecord):
        print(self.format(logRecord))

class LogHandlerWithXml(logging.Handler):
    def __init__(self, filename):
        super(LogHandlerWithXml, self).__init__()

    def recordToXml(self, logRec):
        msg = self.format(logRec)
        if logRec.args:
            args = "".join([' {0}="{1}"'.format(n, v.replace('"','"')) for n, v in logRec.args.items()])
        else:
            args = ""
        refs = "".join('<ref href="{0}"{1}/>'.format(
                        ref["href"],
                        ' sourceLine="{0}"'.format(ref["sourceLine"]) if "sourceLine" in ref else '')
                       for ref in logRec.refs)
        return ('<entry code="{0}" level="{1}" file="{2}" sourceLine="{3}">'
                '<message{4}>{5}</message>{6}'
                '</entry>\n'.format(logRec.messageCode,
                                    logRec.levelname.lower(),
                                    args, msg.replace("&","&").replace("<","<"),
                                    refs))

class LogToXmlHandler(LogHandlerWithXml):
    def __init__(self, filename):
        super(LogToXmlHandler, self).__init__()
        self.filename = filename
        self.logRecordBuffer = []
    def flush(self):
        with open(self.filename, "w", encoding='utf-8') as fh:
            fh.write('<?xml version="1.0" encoding="utf-8"?>\n')
            fh.write('<log>\n')
            for logRec in self.logRecordBuffer:
                fh.write(self.recordToXml(logRec))
            fh.write('</log>\n')
    def emit(self, logRecord):
        self.logRecordBuffer.append(logRecord)

class LogToBufferHandler(LogHandlerWithXml):
    def __init__(self):
        super(LogToBufferHandler, self).__init__()
        self.logRecordBuffer = []

    def flush(self):
        pass # do nothing

    def getXml(self):
        xml = ['<?xml version="1.0" encoding="utf-8"?>\n',
               '<log>']
        for logRec in self.logRecordBuffer:
            xml.append(self.recordToXml(logRec))
        xml.append('</log>')
        self.logRecordBuffer = []
        return '\n'.join(xml)

    def getJson(self):
        entries = []
        for logRec in self.logRecordBuffer:
            message = { "text": self.format(logRec) }
            if logRec.args:
                for n, v in logRec.args.items():
                    message[n] = v
            entry = {"code": logRec.messageCode,
                     "level": logRec.levelname.lower(),
                     "refs": logRec.refs,
                     "message": message}
            entries.append(entry)
        self.logRecordBuffer = []
        return json.dumps( {"log": entries} )

    def getLines(self):
        lines = [self.format(logRec) for logRec in self.logRecordBuffer]
        self.logRecordBuffer = []
        return lines

    def getText(self, separator='\n'):
        return separator.join(self.getLines())

    def emit(self, logRecord):
        self.logRecordBuffer.append(logRecord)

<commit_msg>trying new-style comments add encoding to print output log handler<commit_after># -*- coding: utf-8 -*-
"""
:mod:`arelle.cntlr`
~~~~~~~~~~~~~~~~~~~

.. module:: arelle.cntlr
   :copyright: Copyright 2010-2012 Mark V Systems Limited, All rights reserved.
   :license: Apache-2.
   :synopsis: Common controller class to initialize for platform and setup common logger functions
"""
from arelle import PythonUtil # define 2.x or 3.x string types
import tempfile, os, pickle, sys, logging, gettext, json
from arelle import ModelManager
from arelle.Locale import getLanguageCodes
from collections import defaultdict

class Cntlr:
    """
    .. class:: Cntlr(logFileName=None, logFileMode=None, logFileEncoding=None, logFormat=None)

    Initialization sets up for platform

    - Platform directories for application, configuration, locale, and cache
    - Context menu click event (TKinter)
    - Clipboard presence
    - Update URL
    - Reloads proir config (pickled) user preferences
    - Sets up proxy and web cache
    - Sets up logging
    """
    __version__ = "1.0.0"

    def __init__(self, logFileName=None, logFileMode=None, logFileEncoding=None, logFormat=None):
        self.hasWin32gui = False
        self.hasGui = False
        if sys.platform == "darwin":
            self.isMac = True
            self.isMSW = False
            self.userAppDir = os.path.expanduser("~") + "/Library/Application Support/Arelle"
            self.contextMenuClick = "<Button-2>"
            self.hasClipboard = True
            self.updateURL = "http://arelle.org/downloads/8"
        elif sys.platform.startswith("win"):
            self.isMac = False
            self.isMSW = True
            tempDir = tempfile.gettempdir()
            if tempDir.endswith('local\\temp'):
                self.userAppDir = tempDir[:-10] + 'local\\Arelle'
            else:
                self.userAppDir = tempDir + os.sep + 'arelle'
            try:
                import win32clipboard
                self.hasClipboard = True
            except ImportError:
                self.hasClipboard = False
            try:
                import win32gui
                self.hasWin32gui = True # active state for open file dialogs
            except ImportError:
                pass
            self.contextMenuClick = "<Button-3>"
            if "64 bit" in sys.version:
                self.updateURL = "http://arelle.org/downloads/9"
            else: # 32 bit
                self.updateURL = "http://arelle.org/downloads/10"
        else: # Unix/Linux
            self.isMac = False
            self.isMSW = False
            self.userAppDir = os.path.join(
                   os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")),
                   "arelle")
            try:
                import gtk
                self.hasClipboard = True
            except ImportError:
                self.hasClipboard = False
            self.contextMenuClick = "<Button-3>"
        self.moduleDir = os.path.dirname(__file__)
        # for python 3.2 remove __pycache__
        if self.moduleDir.endswith("__pycache__"):
            self.moduleDir = os.path.dirname(self.moduleDir)
        if self.moduleDir.endswith("python32.zip/arelle"):
            '''
            distZipFile = os.path.dirname(self.moduleDir)
            d = os.path.join(self.userAppDir, "arelle")
            self.configDir = os.path.join(d, "config")
            self.imagesDir = os.path.join(d, "images")
            import zipfile
            distZip = zipfile.ZipFile(distZipFile, mode="r")
            distNames = distZip.namelist()
            distZip.extractall(path=self.userAppDir,
                               members=[f for f in distNames if "/config/" in f or "/images/" in f]
                               )
            distZip.close()
            '''
            resources = os.path.dirname(os.path.dirname(os.path.dirname(self.moduleDir)))
            self.configDir = os.path.join(resources, "config")
            self.imagesDir = os.path.join(resources, "images")
            self.localeDir = os.path.join(resources, "locale")
        elif self.moduleDir.endswith("library.zip\\arelle"): # cx_Freexe
            resources = os.path.dirname(os.path.dirname(self.moduleDir))
            self.configDir = os.path.join(resources, "config")
            self.imagesDir = os.path.join(resources, "images")
            self.localeDir = os.path.join(resources, "locale")
        else:
            self.configDir = os.path.join(self.moduleDir, "config")
            self.imagesDir = os.path.join(self.moduleDir, "images")
            self.localeDir = os.path.join(self.moduleDir, "locale")
        try:
            from arelle import webserver
            self.hasWebServer = True
        except ImportError:
            self.hasWebServer = False
        # assert that app dir must exist
        if not os.path.exists(self.userAppDir):
            os.makedirs(self.userAppDir)
        # load config if it exists
        self.configPickleFile = self.userAppDir + os.sep + "config.pickle"
        self.config = None
        if os.path.exists(self.configPickleFile):
            try:
                with open(self.configPickleFile, 'rb') as f:
                    self.config = pickle.load(f)
            except Exception as ex:
                self.config = None # restart with a new config
        if not self.config:
            self.config = {
                'fileHistory': [],
                'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100),
            }

        # start language translation for domain
        try:
            gettext.translation("arelle", self.localeDir, getLanguageCodes()).install()
        except Exception as msg:
            gettext.install("arelle", self.localeDir)

        from arelle.WebCache import WebCache
        self.webCache = WebCache(self, self.config.get("proxySettings"))
        self.modelManager = ModelManager.initialize(self)

        if logFileName: # use default logging
            self.logger = logging.getLogger("arelle")
            if logFileName == "logToPrint":
                self.logHandler = LogToPrintHandler()
            elif logFileName == "logToBuffer":
                self.logHandler = LogToBufferHandler()
            elif logFileName.endswith(".xml"):
                self.logHandler = LogToXmlHandler(filename=logFileName)
                logFormat = "%(message)s"
            else:
                self.logHandler = logging.FileHandler(filename=logFileName,
                                                      mode=logFileMode if logFileMode else "w",
                                                      encoding=logFileEncoding if logFileEncoding else "utf-8")
            self.logHandler.level = logging.DEBUG
            self.logHandler.setFormatter(LogFormatter(logFormat if logFormat else "%(asctime)s [%(messageCode)s] %(message)s - %(file)s\n"))
            self.logger.addHandler(self.logHandler)
        else:
            self.logger = None

    def addToLog(self, message, messageCode="", file=""):
        """.. method:: addToLog(message, messageCode="", file="")
           Add a simple info message to the default logger"""
        if self.logger is not None:
            self.logger.info(message, extra={"messageCode":messageCode,"refs":[{"href": file}]})
        else:
            print(message) # allows printing on standard out

    def showStatus(self, message, clearAfter=None):
        """.. method:: addToLog(message, clearAfter=None)
           Dummy for subclasses to specialize, provides user feedback on status line of GUI or web page"""
        pass

    def close(self, saveConfig=False):
        """.. method:: close(saveConfig=False)
           Close controller and its logger, optionally saaving the user preferences configuration
           :param saveConfig: save the user preferences configuration"""
        if saveConfig:
            self.saveConfig()
        if self.logger is not None:
            self.logHandler.close()

    def saveConfig(self):
        """.. method:: saveConfig()
           Save user preferences configuration (in a pickle file)."""
        with open(self.configPickleFile, 'wb') as f:
            pickle.dump(self.config, f, pickle.HIGHEST_PROTOCOL)

    # default non-threaded viewModelObject
    def viewModelObject(self, modelXbrl, objectId):
        """.. method:: viewModelObject(modelXbrl, objectId)
           Notification to watching views to show and highlight selected object
           :param modelXbrl: ModelXbrl whose views are to be notified
           :param objectId: Selected object."""
        modelXbrl.viewModelObject(objectId)

    def reloadViews(self, modelXbrl):
        """.. method:: reloadViews(modelXbrl)
           Notification to reload views (probably due to change within modelXbrl).  Dummy
           for subclasses to specialize when they have a GUI or web page.
           :param modelXbrl: ModelXbrl whose views are to be reloaded"""
        pass

    def rssWatchUpdateOption(self, **args):
        """.. method:: rssWatchUpdateOption(**args)
           Notification to change rssWatch options, as passed in, usually from a modal dialog."""
        pass

    # default web authentication password
    def internet_user_password(self, host, realm):
        """.. method:: internet_user_password(self, host, realm)
           Request (for an interactive UI or web page) to obtain user ID and password (usually for a proxy
           or when getting a web page that requires entry of a password).
           :param host: The host that is requesting the password
           :param realm: The domain on the host that is requesting the password
           :rtype string: xzzzzzz"""
        return ('myusername','mypassword')

    # if no text, then return what is on the clipboard, otherwise place text onto clipboard
    def clipboardData(self, text=None):
        """.. method:: clipboardData(self, text=None)
           Places text onto the clipboard (if text is not None), otherwise retrieves and returns text from the clipboard.
           Only supported for those platforms that have clipboard support in the current python implementation (macOS
           or ActiveState Windows Python).
           :param text: Text to place onto clipboard if not None, otherwise retrieval of text from clipboard."""
        if self.hasClipboard:
            try:
                if sys.platform == "darwin":
                    import subprocess
                    if text is None:
                        p = subprocess.Popen(['pbpaste'], stdout=subprocess.PIPE)
                        retcode = p.wait()
                        text = p.stdout.read()
                        return text
                    else:
                        p = subprocess.Popen(['pbcopy'], stdin=subprocess.PIPE)
                        p.stdin.write(text)
                        p.stdin.close()
                        retcode = p.wait()
                elif sys.platform.startswith("win"):
                    import win32clipboard
                    win32clipboard.OpenClipboard()
                    if text is None:
                        if win32clipboard.IsClipboardFormatAvailable(win32clipboard.CF_TEXT):
                            return win32clipboard.GetClipboardData().decode("utf8")
                    else:
                        win32clipboard.EmptyClipboard()
                        win32clipboard.SetClipboardData(win32clipboard.CF_TEXT, text.encode("utf8"))
                    win32clipboard.CloseClipboard()
                else: # Unix/Linux
                    import gtk
                    clipbd = gtk.Clipboard(display=gtk.gdk.display_get_default(), selection="CLIPBOARD")
                    if text is None:
                        return clipbd.wait_for_text().decode("utf8")
                    else:
                        clipbd.set_text(text.encode("utf8"), len=-1)
            except Exception:
                pass
        return None

class LogFormatter(logging.Formatter):
    def __init__(self, fmt=None, datefmt=None):
        super(LogFormatter, self).__init__(fmt, datefmt)

    def format(self, record):
        # provide a file parameter made up from refs entries
        fileLines = defaultdict(set)
        for ref in record.refs:
            fileLines[ref["href"].partition("#")[0]].add(ref.get("sourceLine"))
        record.file = ", ".join(file + " " + ', '.join(str(line) for line in lines if line)
                                for file, lines in sorted(fileLines.items()))
        formattedMessage = super(LogFormatter, self).format(record)
        del record.file
        return formattedMessage

class LogToPrintHandler(logging.Handler):
    """
    .. class:: LogToPrintHandler()

    A log handler that emits log entries to standard out as they are logged.

    CAUTION: Output is utf-8 encoded, which is fine for saving to files, but may not display correctly in terminal windows.
    """
    def emit(self, logRecord):
        print(self.format(logRecord).encode("utf-8"))

class LogHandlerWithXml(logging.Handler):
    def __init__(self):
        super(LogHandlerWithXml, self).__init__()

    def recordToXml(self, logRec):
        msg = self.format(logRec)
        if logRec.args:
            args = "".join([' {0}="{1}"'.format(n, v.replace('"','"')) for n, v in logRec.args.items()])
        else:
            args = ""
        refs = "".join('<ref href="{0}"{1}/>'.format(
                        ref["href"],
                        ' sourceLine="{0}"'.format(ref["sourceLine"]) if "sourceLine" in ref else '')
                       for ref in logRec.refs)
        return ('<entry code="{0}" level="{1}" file="{2}" sourceLine="{3}">'
                '<message{4}>{5}</message>{6}'
                '</entry>\n'.format(logRec.messageCode,
                                    logRec.levelname.lower(),
                                    args, msg.replace("&","&").replace("<","<"),
                                    refs))

class LogToXmlHandler(LogHandlerWithXml):
    """
    .. class:: LogToXmlHandler(filename)

    A log handler that writes log entries to named XML file (utf-8 encoded) upon closing the application.
    """
    def __init__(self, filename):
        super(LogToXmlHandler, self).__init__()
        self.filename = filename
        self.logRecordBuffer = []
    def flush(self):
        with open(self.filename, "w", encoding='utf-8') as fh:
            fh.write('<?xml version="1.0" encoding="utf-8"?>\n')
            fh.write('<log>\n')
            for logRec in self.logRecordBuffer:
                fh.write(self.recordToXml(logRec))
            fh.write('</log>\n')
    def emit(self, logRecord):
        self.logRecordBuffer.append(logRecord)

class LogToBufferHandler(LogHandlerWithXml):
    """
    .. class:: LogToBufferHandler()

    A log handler that writes log entries to a memory buffer for later retrieval (to a string) in XML, JSON, or text lines,
    usually for return to a web service or web page call.
    """
    def __init__(self):
        super(LogToBufferHandler, self).__init__()
        self.logRecordBuffer = []

    def flush(self):
        pass # do nothing

    def getXml(self):
        """.. method:: getXml()
           Returns an XML document (as a string) representing the messages in the log buffer, and clears the buffer."""
        xml = ['<?xml version="1.0" encoding="utf-8"?>\n',
               '<log>']
        for logRec in self.logRecordBuffer:
            xml.append(self.recordToXml(logRec))
        xml.append('</log>')
        self.logRecordBuffer = []
        return '\n'.join(xml)

    def getJson(self):
        """.. method:: getJson()
           Returns an JSON string representing the messages in the log buffer, and clears the buffer."""
        entries = []
        for logRec in self.logRecordBuffer:
            message = { "text": self.format(logRec) }
            if logRec.args:
                for n, v in logRec.args.items():
                    message[n] = v
            entry = {"code": logRec.messageCode,
                     "level": logRec.levelname.lower(),
                     "refs": logRec.refs,
                     "message": message}
            entries.append(entry)
        self.logRecordBuffer = []
        return json.dumps( {"log": entries} )

    def getLines(self):
        """.. method:: getLines()
           Returns a list of the message strings in the log buffer, and clears the buffer."""
        lines = [self.format(logRec) for logRec in self.logRecordBuffer]
        self.logRecordBuffer = []
        return lines

    def getText(self, separator='\n'):
        """.. method:: getText()
           :param separator: Line separator (default is platform's newline)
           Returns a text string of the messages in the log buffer, and clears the buffer."""
        return separator.join(self.getLines())

    def emit(self, logRecord):
        self.logRecordBuffer.append(logRecord)

<|endoftext|>


<commit_before>__version__ = '2.0.0b1'

about_text = ('deen (DEcoderENcoder) v%s\n\nA decoding/encoding application for arbitrary data.\n\n'
              'https://github.com/takeshixx/deen') % __version__

icon_path = '/media/icon.png'

verbose_log_format = '[ %(levelname)s - %(asctime)s - %(name)s - '
verbose_log_format += '%(filename)s:%(lineno)s - %(funcName)s() ] '
verbose_log_format += '%(message)s'

cli_description = 'Apply encodings, compression, hashing and other types of transformations to arbitrary input data.'

cli_epilog = """examples:
  open a file in the deen GUI:
    $ deen /bin/ls

  open file from STDIN in deen GUI:
    $ cat /bin/ls | deen -

  base64 encode a string:
    $ deen -b base64 -d admin:admin
    YWRtaW46YWRtaW4=

  base64 encode a string with subcommand:
    $ deen base64 admin:admin
    YWRtaW46YWRtaW4=

  decode Base64 string:
    $ deen -b base64 -r -d YWRtaW46YWRtaW4=
    admin:admin

  decode Base64 string with subcommand:
    $ deen base64 -r YWRtaW46YWRtaW4=
    admin:admin

  calculate the SHA256 hash of file:
    $ deen sha256 /bin/ls
    df285ab34ad10d8b641e65f39fa11a7d5b44571a37f94314debbfe7233021755
"""
<commit_msg>Updated CLI examples<commit_after>__version__ = '2.0.0b1'

about_text = ('deen (DEcoderENcoder) v%s\n\nA decoding/encoding application for arbitrary data.\n\n'
              'https://github.com/takeshixx/deen') % __version__

icon_path = '/media/icon.png'

verbose_log_format = '[ %(levelname)s - %(asctime)s - %(name)s - '
verbose_log_format += '%(filename)s:%(lineno)s - %(funcName)s() ] '
verbose_log_format += '%(message)s'

cli_description = 'Apply encodings, compression, hashing and other types of transformations to arbitrary input data.'

cli_epilog = """examples:
  open a file in the deen GUI:
    $ deen /bin/ls

  open file from STDIN in deen GUI:
    $ cat /bin/ls | deen -

  base64 encode a string:
    $ deen base64 -d admin:admin
    YWRtaW46YWRtaW4=

  base64 encode a string with subcommand alias:
    $ deen b64 admin:admin
    YWRtaW46YWRtaW4=

  decode Base64 string:
    $ deen base64 -r -d YWRtaW46YWRtaW4=
    admin:admin

  calculate the SHA256 hash of file:
    $ deen sha256 /bin/ls
    df285ab34ad10d8b641e65f39fa11a7d5b44571a37f94314debbfe7233021755
"""
<|endoftext|>


<commit_before><commit_msg>fixed volunteer delete<commit_after><|endoftext|>


<commit_before>import numpy as np
import pandas as pd
from lexos.helpers.error_messages import SEG_NON_POSITIVE_MESSAGE, \
    NOT_ENOUGH_CLASSES_MESSAGE
from lexos.models.topword_model import TopwordModel, TopwordTestOptions
from lexos.receivers.topword_receiver import TopwordFrontEndOption

# ---------------------------- Test for z-test ------------------------------
# noinspection PyProtectedMember
class TestZTest:
    def test_normal_case(self):
        assert round(TopwordModel._z_test(p1=0.1, p2=0.3, n1=10, n2=1000), 2) \
            == -1.38
        assert round(TopwordModel._z_test(p1=0.3, p2=0.1, n1=100, n2=100), 2) \
            == 3.54
        assert TopwordModel._z_test(p1=1, p2=1, n1=100, n2=100) == 0

    def test_special_case(self):
        try:
            _ = TopwordModel._z_test(p1=0.1, p2=0.3, n1=100, n2=0)
            raise AssertionError("Error message did not raise")
        except AssertionError as error:
            assert str(error) == SEG_NON_POSITIVE_MESSAGE

        try:
            _ = TopwordModel._z_test(p1=0.1, p2=0.3, n1=0, n2=100)
            raise AssertionError("Error message did not raise")
        except AssertionError as error:
            assert str(error) == SEG_NON_POSITIVE_MESSAGE
# ---------------------------------------------------------------------------

# ------------------- Test method analyze para to group ---------------------
# Create test suit for normal case.
test_dtm = pd.DataFrame(data=np.array([(1, 1, 0, 0), (0, 0, 1, 10)]),
                        index=np.array([0, 1]),
                        columns=np.array(["A", "B", "C", "D"]))
test_id_temp_label_map = {0: "F1", 1: "F2"}
test_front_end_option = TopwordFrontEndOption(analysis_option="allToPara")
test_option = TopwordTestOptions(doc_term_matrix=test_dtm,
                                 id_temp_label_map=test_id_temp_label_map,
                                 front_end_option=test_front_end_option)
test_topword_model = TopwordModel(test_options=test_option)

# Create test suit for special case.
test_option_empty = TopwordTestOptions(
    doc_term_matrix=pd.DataFrame(data=[], index=[], columns=[]),
    id_temp_label_map={},
    front_end_option=test_front_end_option)
test_topword_model_empty = TopwordModel(test_options=test_option_empty)

# Fake input for class division map.
fake_class_division_map = pd.DataFrame(data=np.array([(True, True)]),
                                       index=["C1"],
                                       columns=["F1", "F2"])

class TestParaToGroup:
    def test_normal_case_result(self):
        pd.testing.assert_series_equal(
            test_topword_model.get_readable_result(
                class_division_map=fake_class_division_map).results[0],
            pd.Series([-2.1483], index=["D"],
                      name='Document "F1" compared to the whole corpus'))
        pd.testing.assert_series_equal(
            test_topword_model.get_readable_result(
                class_division_map=fake_class_division_map).results[1],
            pd.Series([], index=[],
                      name='Document "F2" compared to the whole corpus'))

    def test_normal_case_header(self):
        assert test_topword_model.get_readable_result(
            class_division_map=fake_class_division_map).header \
            == "Compare Each Document to All the Documents As a Whole"

    def test_special_case(self):
        try:
            _ = test_topword_model_empty.get_readable_result(
                class_division_map=fake_class_division_map)
            raise AssertionError("Error message did not raise")
        except AssertionError as error:
            assert str(error) == SEG_NON_POSITIVE_MESSAGE
# ---------------------------------------------------------------------------

"""
# -------------------- Test method analyze class to all ---------------------
# Create test suit for normal case.
test_dtm = pd.DataFrame(data=np.array([(1, 1, 0, 0, 0, 0, 0, 0),
                                       (0, 0, 1, 1, 0, 0, 0, 0),
                                       (0, 0, 0, 0, 1, 1, 0, 0),
                                       (0, 0, 0, 0, 0, 0, 1, 100)]),
                        index=np.array([0, 1, 2, 3]),
                        columns=np.array(["A", "B", "C", "D",
                                          "E", "F", "G", "H"]))
test_id_temp_label_map = {0: "F1", 1: "F2", 2: "F3", 3: "F4"}
test_front_end_option = TopwordFrontEndOption(analysis_option="classToPara")
test_option = TopwordTestOptions(doc_term_matrix=test_dtm,
                                 id_temp_label_map=test_id_temp_label_map,
                                 front_end_option=test_front_end_option)
test_topword_model_one = TopwordModel(test_options=test_option)

# Create test suit for special case.
test_option_empty = TopwordTestOptions(
    doc_term_matrix=pd.DataFrame(data=[], index=[], columns=[]),
    id_temp_label_map={},
    front_end_option=test_front_end_option)
test_topword_model_empty_one = TopwordModel(test_options=test_option_empty)

# Fake class division map.
test_class_division_map = pd.DataFrame(
    data=np.array([(True, True, False, False), (False, False, True, True)]),
    index=np.array(["C1", "C2"]),
    columns=np.array(["F1", "F2", "F3", "F4"]))

# Testing starts here
class TestClassToAll:
    def test_normal_case_result(self):
        pd.testing.assert_series_equal(
            test_topword_model_one.get_readable_result(
                class_division_map=test_class_division_map).results[0],
            pd.Series([7.2108, 7.2108, -6.3857], index=["A", "B", "H"],
                      name='Document "F1" compared to Class "C2"'))
        pd.testing.assert_series_equal(
            test_topword_model_one.get_readable_result(
                class_division_map=test_class_division_map).results[2],
            pd.Series([], index=[],
                      name='Document "F3" compared to Class "C1"'))

    def test_normal_case_header(self):
        assert test_topword_model_one.get_readable_result(
            class_division_map=test_class_division_map).header == \
            "Compare Each Document to Other Class(es)"

    def test_special_case(self):
        try:
            _ = test_topword_model_empty_one.get_readable_result(
                class_division_map=test_class_division_map)
            raise AssertionError("Error message did not raise")
        except AssertionError as error:
            assert str(error) == SEG_NON_POSITIVE_MESSAGE

        try:
            _ = test_topword_model_empty_one.get_readable_result(
                class_division_map=fake_class_division_map)
            raise AssertionError("Error message did not raise")
        except AssertionError as error:
            assert str(error) == NOT_ENOUGH_CLASSES_MESSAGE
# ---------------------------------------------------------------------------
"""

# Create test suit for normal case.
test_dtm = pd.DataFrame(data=np.array([(1, 1, 0, 0, 0, 0, 0, 0),
                                       (0, 0, 1, 1, 0, 0, 0, 0),
                                       (0, 0, 0, 0, 1, 1, 0, 0),
                                       (0, 0, 0, 0, 0, 0, 1, 100)]),
                        index=np.array([0, 1, 2, 3]),
                        columns=np.array(["A", "B", "C", "D",
                                          "E", "F", "G", "H"]))
test_id_temp_label_map = {0: "F1", 1: "F2", 2: "F3", 3: "F4"}
test_front_end_option = TopwordFrontEndOption(analysis_option="classToPara")
test_option = TopwordTestOptions(doc_term_matrix=test_dtm,
                                 id_temp_label_map=test_id_temp_label_map,
                                 front_end_option=test_front_end_option)
test_topword_model_one = TopwordModel(test_options=test_option)

# Create test suit for special case.
test_option_empty = TopwordTestOptions(
    doc_term_matrix=pd.DataFrame(data=[], index=[], columns=[]),
    id_temp_label_map={},
    front_end_option=test_front_end_option)
test_topword_model_empty_one = TopwordModel(test_options=test_option_empty)

# Fake class division map.
test_class_division_map = pd.DataFrame(
    data=np.array([(True, True, False, False), (False, False, True, True)]),
    index=np.array(["C1", "C2"]),
    columns=np.array(["F1", "F2", "F3", "F4"]))

# ------------------- Test method analyze class to class --------------------
# Create test suite for normal case.
test_front_end_option = TopwordFrontEndOption(analysis_option="classToClass")
test_option = TopwordTestOptions(doc_term_matrix=test_dtm,
                                 id_temp_label_map=test_id_temp_label_map,
                                 front_end_option=test_front_end_option)
test_topword_model_two = TopwordModel(test_options=test_option)

# Create test suit for special case.
test_option_empty = TopwordTestOptions(
    doc_term_matrix=pd.DataFrame(data=[], index=[], columns=[]),
    id_temp_label_map={},
    front_end_option=test_front_end_option)
test_topword_model_empty_two = TopwordModel(test_options=test_option_empty)

class TestClassToClass:
    def test_normal_case_result(self):
        pd.testing.assert_series_equal(
            test_topword_model_two.get_readable_result(
                class_division_map=test_class_division_map).results[0],
            pd.Series([-7.70470, 5.09830, 5.09830, 5.09830, 5.09830],
                      index=["H", "A", "B", "C", "D"],
                      name='Class "C1" compared to Class "C2"'))

    def test_normal_case_header(self):
        assert test_topword_model_two.get_readable_result(
            class_division_map=test_class_division_map).header == \
            'Compare a Class to Each Other Class'

    def test_special_case(self):
        try:
            _ = test_topword_model_empty_two.get_readable_result(
                class_division_map=test_class_division_map)
            raise AssertionError("Error message did not raise")
        except AssertionError as error:
            assert str(error) == SEG_NON_POSITIVE_MESSAGE

        try:
            _ = test_topword_model_empty_two.get_readable_result(
                class_division_map=fake_class_division_map)
            raise AssertionError("Error message did not raise")
        except AssertionError as error:
            assert str(error) == NOT_ENOUGH_CLASSES_MESSAGE
# ---------------------------------------------------------------------------

<commit_msg>Corresponding changes in unit test<commit_after>import numpy as np
import pandas as pd
from lexos.helpers.error_messages import SEG_NON_POSITIVE_MESSAGE, \
    NOT_ENOUGH_CLASSES_MESSAGE
from lexos.models.topword_model import TopwordModel, TopwordTestOptions
from lexos.receivers.topword_receiver import TopwordFrontEndOption

# ---------------------------- Test for z-test ------------------------------
# noinspection PyProtectedMember
class TestZTest:
    def test_normal_case(self):
        assert round(TopwordModel._z_test(p1=0.1, p2=0.3, n1=10, n2=1000), 2) \
            == -1.38
        assert round(TopwordModel._z_test(p1=0.3, p2=0.1, n1=100, n2=100), 2) \
            == 3.54
        assert TopwordModel._z_test(p1=1, p2=1, n1=100, n2=100) == 0

    def test_special_case(self):
        try:
            _ = TopwordModel._z_test(p1=0.1, p2=0.3, n1=100, n2=0)
            raise AssertionError("Error message did not raise")
        except AssertionError as error:
            assert str(error) == SEG_NON_POSITIVE_MESSAGE

        try:
            _ = TopwordModel._z_test(p1=0.1, p2=0.3, n1=0, n2=100)
            raise AssertionError("Error message did not raise")
        except AssertionError as error:
            assert str(error) == SEG_NON_POSITIVE_MESSAGE
# ---------------------------------------------------------------------------

# ------------------- Test method analyze para to group ---------------------
# Create test suit for normal case.
test_dtm = pd.DataFrame(data=np.array([(1, 1, 0, 0), (0, 0, 1, 10)]),
                        index=np.array([0, 1]),
                        columns=np.array(["A", "B", "C", "D"]))
test_id_temp_label_map = {0: "F1", 1: "F2"}
test_front_end_option = TopwordFrontEndOption(analysis_option="allToPara")
test_option = TopwordTestOptions(doc_term_matrix=test_dtm,
                                 id_temp_label_map=test_id_temp_label_map,
                                 front_end_option=test_front_end_option)
test_topword_model = TopwordModel(test_options=test_option)

# Create test suit for special case.
test_option_empty = TopwordTestOptions(
    doc_term_matrix=pd.DataFrame(data=[], index=[], columns=[]),
    id_temp_label_map={},
    front_end_option=test_front_end_option)
test_topword_model_empty = TopwordModel(test_options=test_option_empty)

# Fake input for class division map.
fake_class_division_map = pd.DataFrame(data=np.array([(True, True)]),
                                       index=["C1"],
                                       columns=[0, 1])

class TestParaToGroup:
    def test_normal_case_result(self):
        pd.testing.assert_series_equal(
            test_topword_model.get_readable_result(
                class_division_map=fake_class_division_map).results[0],
            pd.Series([-2.1483], index=["D"],
                      name='Document "F1" compared to the whole corpus'))
        pd.testing.assert_series_equal(
            test_topword_model.get_readable_result(
                class_division_map=fake_class_division_map).results[1],
            pd.Series([], index=[],
                      name='Document "F2" compared to the whole corpus'))

    def test_normal_case_header(self):
        assert test_topword_model.get_readable_result(
            class_division_map=fake_class_division_map).header \
            == "Compare Each Document to All the Documents As a Whole"

    def test_special_case(self):
        try:
            _ = test_topword_model_empty.get_readable_result(
                class_division_map=fake_class_division_map)
            raise AssertionError("Error message did not raise")
        except AssertionError as error:
            assert str(error) == SEG_NON_POSITIVE_MESSAGE
# ---------------------------------------------------------------------------

# -------------------- Test method analyze class to all ---------------------
# Create test suit for normal case.
test_dtm = pd.DataFrame(data=np.array([(1, 1, 0, 0, 0, 0, 0, 0),
                                       (0, 0, 1, 1, 0, 0, 0, 0),
                                       (0, 0, 0, 0, 1, 1, 0, 0),
                                       (0, 0, 0, 0, 0, 0, 1, 100)]),
                        index=np.array([0, 1, 2, 3]),
                        columns=np.array(["A", "B", "C", "D",
                                          "E", "F", "G", "H"]))
test_id_temp_label_map = {0: "F1", 1: "F2", 2: "F3", 3: "F4"}
test_front_end_option = TopwordFrontEndOption(analysis_option="classToPara")
test_option = TopwordTestOptions(doc_term_matrix=test_dtm,
                                 id_temp_label_map=test_id_temp_label_map,
                                 front_end_option=test_front_end_option)
test_topword_model_one = TopwordModel(test_options=test_option)

# Create test suit for special case.
test_option_empty = TopwordTestOptions(
    doc_term_matrix=pd.DataFrame(data=[], index=[], columns=[]),
    id_temp_label_map={},
    front_end_option=test_front_end_option)
test_topword_model_empty_one = TopwordModel(test_options=test_option_empty)

# Fake class division map.
test_class_division_map = pd.DataFrame(
    data=np.array([(True, True, False, False), (False, False, True, True)]),
    index=np.array(["C1", "C2"]),
    columns=np.array([0, 1, 2, 3]))

# Testing starts here
class TestClassToAll:
    def test_normal_case_result(self):
        pd.testing.assert_series_equal(
            test_topword_model_one.get_readable_result(
                class_division_map=test_class_division_map).results[0],
            pd.Series([7.2108, 7.2108, -6.3857], index=["A", "B", "H"],
                      name='Document "F1" compared to Class "C2"'))
        pd.testing.assert_series_equal(
            test_topword_model_one.get_readable_result(
                class_division_map=test_class_division_map).results[2],
            pd.Series([], index=[],
                      name='Document "F3" compared to Class "C1"'))

    def test_normal_case_header(self):
        assert test_topword_model_one.get_readable_result(
            class_division_map=test_class_division_map).header == \
            "Compare Each Document to Other Class(es)"

    def test_special_case(self):
        try:
            _ = test_topword_model_empty_one.get_readable_result(
                class_division_map=test_class_division_map)
            raise AssertionError("Error message did not raise")
        except AssertionError as error:
            assert str(error) == SEG_NON_POSITIVE_MESSAGE

        try:
            _ = test_topword_model_empty_one.get_readable_result(
                class_division_map=fake_class_division_map)
            raise AssertionError("Error message did not raise")
        except AssertionError as error:
            assert str(error) == NOT_ENOUGH_CLASSES_MESSAGE
# ---------------------------------------------------------------------------

# ------------------- Test method analyze class to class --------------------
# Create test suite for normal case.
test_front_end_option = TopwordFrontEndOption(analysis_option="classToClass")
test_option = TopwordTestOptions(doc_term_matrix=test_dtm,
                                 id_temp_label_map=test_id_temp_label_map,
                                 front_end_option=test_front_end_option)
test_topword_model_two = TopwordModel(test_options=test_option)

# Create test suit for special case.
test_option_empty = TopwordTestOptions(
    doc_term_matrix=pd.DataFrame(data=[], index=[], columns=[]),
    id_temp_label_map={},
    front_end_option=test_front_end_option)
test_topword_model_empty_two = TopwordModel(test_options=test_option_empty)

class TestClassToClass:
    def test_normal_case_result(self):
        pd.testing.assert_series_equal(
            test_topword_model_two.get_readable_result(
                class_division_map=test_class_division_map).results[0],
            pd.Series([-7.70470, 5.09830, 5.09830, 5.09830, 5.09830],
                      index=["H", "A", "B", "C", "D"],
                      name='Class "C1" compared to Class "C2"'))

    def test_normal_case_header(self):
        assert test_topword_model_two.get_readable_result(
            class_division_map=test_class_division_map).header == \
            'Compare a Class to Each Other Class'

    def test_special_case(self):
        try:
            _ = test_topword_model_empty_two.get_readable_result(
                class_division_map=test_class_division_map)
            raise AssertionError("Error message did not raise")
        except AssertionError as error:
            assert str(error) == SEG_NON_POSITIVE_MESSAGE

        try:
            _ = test_topword_model_empty_two.get_readable_result(
                class_division_map=fake_class_division_map)
            raise AssertionError("Error message did not raise")
        except AssertionError as error:
            assert str(error) == NOT_ENOUGH_CLASSES_MESSAGE
# ---------------------------------------------------------------------------
<|endoftext|>


<commit_before>import json

from django import forms
from django.core.files.storage import FileSystemStorage
from django.core.urlresolvers import reverse
from django.http import JsonResponse, HttpResponseNotFound
from django.views.generic import View

storage = FileSystemStorage()

class UploadForm(forms.Form):
    name = forms.FileField()

def file_info(name):
    return {
        'name': name,
        'size': storage.size(name),
        'created': storage.created_time(name),
        'links': {
            'self': reverse('upload-detail', kwargs={'name': name})
        }
    }

class FileListView(View):
    """Get a list of all available files or create a new file."""

    def get(self, request):
        """List all files."""
        _, files = storage.listdir('')
        info = []
        for name in files:
            info.append(file_info(name))
        result = {
            'files': info,
            'count': len(info),
        }
        return JsonResponse(result)

    def post(self, request):
        """Add a new file."""
        form = UploadForm(request.POST, request.FILES)
        if form.is_valid():
            upload = form.cleaned_data['name']
            storage.save(upload.name, upload)
            result = file_info(upload.name)
            return JsonResponse(result, status=201)
        else:
            return HttpResponse(form.errors.as_json(), status=400, content_type='application/json')

class FileDetailView(View):
    """Get details for a single file."""

    def get(self, request, name):
        """Get details for a file."""
        if storage.exists(name):
            result = file_info(name)
            return JsonResponse(result)
        else:
            return HttpResponseNotFound()
<commit_msg>Delete files via the API. Include previously missing import.<commit_after>import json

from django import forms
from django.core.files.storage import FileSystemStorage
from django.core.urlresolvers import reverse
from django.http import JsonResponse, HttpResponseNotFound, HttpResponse
from django.views.generic import View

storage = FileSystemStorage()

class UploadForm(forms.Form):
    name = forms.FileField()

def file_info(name):
    return {
        'name': name,
        'size': storage.size(name),
        'created': storage.created_time(name),
        'links': {
            'self': reverse('upload-detail', kwargs={'name': name})
        }
    }

class FileListView(View):
    """Get a list of all available files or create a new file."""

    def get(self, request):
        """List all files."""
        _, files = storage.listdir('')
        info = []
        for name in files:
            info.append(file_info(name))
        result = {
            'files': info,
            'count': len(info),
        }
        return JsonResponse(result)

    def post(self, request):
        """Add a new file."""
        form = UploadForm(request.POST, request.FILES)
        if form.is_valid():
            upload = form.cleaned_data['name']
            storage.save(upload.name, upload)
            result = file_info(upload.name)
            return JsonResponse(result, status=201)
        else:
            return HttpResponse(form.errors.as_json(), status=400, content_type='application/json')

class FileDetailView(View):
    """Get details for a single file or delete the file."""

    def get(self, request, name):
        """Get details for a file."""
        if storage.exists(name):
            result = file_info(name)
            return JsonResponse(result)
        else:
            return HttpResponseNotFound()

    def delete(self, request, name):
        """Delete a file."""
        if storage.exists(name):
            storage.delete(name)
            return HttpResponse(status=204)
        else:
            return HttpResponse(status=410)
<|endoftext|>


<commit_before>import argparse
import os
import uuid
import tensorflow as tf
import hypergan as hg
import hyperchamber as hc
import matplotlib.pyplot as plt
from hypergan.loaders import *
from hypergan.util.hc_tf import *

import math

def parse_args():
    parser = argparse.ArgumentParser(description='Train a 2d test!', add_help=True)
    parser.add_argument('--batch_size', '-b', type=int, default=32, help='Examples to include in each batch.  If using batch norm, this needs to be preserved when in server mode')
    parser.add_argument('--device', '-d', type=str, default='/gpu:0', help='In the form "/gpu:0", "/cpu:0", etc.  Always use a GPU (or TPU) to train')
    parser.add_argument('--format', '-f', type=str, default='png', help='jpg or png')
    parser.add_argument('--config', '-c', type=str, default='2d-test', help='config name')
    parser.add_argument('--distribution', '-t', type=str, default='circle', help='what distribution to test, options are circle, modes')
    return parser.parse_args()

def no_regularizer(amt):
    return None

def custom_discriminator_config():
    return {
            'create': custom_discriminator
    }

def custom_generator_config():
    return {
            'create': custom_generator
    }

def custom_discriminator(gan, config, x, g, xs, gs, prefix='d_'):
    net = tf.concat(axis=0, values=[x,g])
    net = linear(net, 128, scope=prefix+'lin1')
    net = tf.nn.relu(net)
    net = linear(net, 128, scope=prefix+'lin2')
    return net

def custom_generator(config, gan, net):
    net = linear(net, 128, scope="g_lin_proj")
    net = batch_norm_1(gan.config.batch_size, name='g_bn_1')(net)
    net = tf.nn.relu(net)
    net = linear(net, 2, scope="g_lin_proj3")
    net = tf.tanh(net)
    return [net]

def batch_accuracy(a, b):
    "Each point of a is measured against the closest point on b.  Distance differences are added together."
    tiled_a = a
    tiled_a = tf.reshape(tiled_a, [int(tiled_a.get_shape()[0]), 1, int(tiled_a.get_shape()[1])])

    tiled_a = tf.tile(tiled_a, [1, int(tiled_a.get_shape()[0]), 1])

    tiled_b = b
    tiled_b = tf.reshape(tiled_b, [1, int(tiled_b.get_shape()[0]), int(tiled_b.get_shape()[1])])
    tiled_b = tf.tile(tiled_b, [int(tiled_b.get_shape()[0]), 1, 1])

    difference = tf.abs(tiled_a-tiled_b)
    difference = tf.reduce_min(difference, axis=1)
    difference = tf.reduce_sum(difference, axis=1)
    return tf.reduce_sum(difference, axis=0)

args = parse_args()

def train():
    selector = hg.config.selector(args)
    config_name="2d-measure-accuracy-"+str(uuid.uuid4())

    config = selector.random_config()
    config_filename = os.path.expanduser('~/.hypergan/configs/'+config_name+'.json')

    trainers = []

    rms_opts = {
        'g_momentum': [0,0.1,0.01,1e-6,1e-5,1e-1,0.9,0.999, 0.5],
        'd_momentum': [0,0.1,0.01,1e-6,1e-5,1e-1,0.9,0.999, 0.5],
        'd_decay': [0.8, 0.9, 0.99,0.999,0.995,0.9999,1],
        'g_decay': [0.8, 0.9, 0.99,0.999,0.995,0.9999,1],
        'clipped_gradients': [False, 1e-2],
        'clipped_d_weights': [False, 1e-2],
        'd_learn_rate': [1e-3,1e-4,5e-4,1e-6,4e-4, 5e-5],
        'g_learn_rate': [1e-3,1e-4,5e-4,1e-6,4e-4, 5e-5]
    }
    trainers.append(hg.trainers.rmsprop_trainer.config(**rms_opts))

    adam_opts = {}

    adam_opts = {
        'd_learn_rate': [1e-3,1e-4,5e-4,1e-2,1e-6],
        'g_learn_rate': [1e-3,1e-4,5e-4,1e-2,1e-6],
        'd_beta1': [0.9, 0.99, 0.999, 0.1, 0.01, 0.2, 1e-8],
        'd_beta2': [0.9, 0.99, 0.999, 0.1, 0.01, 0.2, 1e-8],
        'g_beta1': [0.9, 0.99, 0.999, 0.1, 0.01, 0.2, 1e-8],
        'g_beta2': [0.9, 0.99, 0.999, 0.1, 0.01, 0.2, 1e-8],
        'd_epsilon': [1e-8, 1, 0.1, 0.5],
        'g_epsilon': [1e-8, 1, 0.1, 0.5],
        'd_clipped_weights': [False, 0.01],
        'clipped_gradients': [False, 0.01]
    }

    trainers.append(hg.trainers.adam_trainer.config(**adam_opts))
    encoders = []

    projections = []
    projections.append([hg.encoders.linear_encoder.modal])
    projections.append([hg.encoders.linear_encoder.modal, hg.encoders.linear_encoder.sphere, hg.encoders.linear_encoder.linear])
    projections.append([hg.encoders.linear_encoder.binary, hg.encoders.linear_encoder.sphere])
    projections.append([hg.encoders.linear_encoder.sphere, hg.encoders.linear_encoder.linear])
    encoder_opts = {
            'z': [2,4,8,16,32],
            'modes': [2, 4, 8, 16],
            'projections': projections
            }

    losses = []
    losses.append([hg.losses.wgan_loss.config()])
    encoders.append([hg.encoders.linear_encoder.config(**encoder_opts)])
    custom_config = {
        'model': args.config,
        'batch_size': args.batch_size,
        'trainer': trainers,
        'generator': custom_generator_config(),
        'discriminators': [[custom_discriminator_config()]],
        'losses': losses,
        'encoders': encoders
    }

    custom_config_selector = hc.Selector()
    for key,value in custom_config.items():
        custom_config_selector.set(key, value)
        print("Set ", key, value)

    custom_config_selection = custom_config_selector.random_config()

    for key,value in custom_config_selection.items():
        config[key]=value

    config['dtype']=tf.float32
    config = hg.config.lookup_functions(config)

    def circle(x):
        spherenet = tf.square(x)
        spherenet = tf.reduce_sum(spherenet, 1)
        lam = tf.sqrt(spherenet)
        return x/tf.reshape(lam,[int(lam.get_shape()[0]), 1])

    def modes(x):
        return tf.round(x*2)/2.0

    if args.distribution == 'circle':
        x = tf.random_normal([args.batch_size, 2])
        x = circle(x)
    elif args.distribution == 'modes':
        x = tf.random_uniform([args.batch_size, 2], -1, 1)
        x = modes(x)
    elif args.distribution == 'sin':
        x = tf.random_uniform((1, args.batch_size), -10.5, 10.5 )
        x = tf.transpose(x)
        r_data = tf.random_normal((args.batch_size,1), mean=0, stddev=0.1)
        xy = tf.sin(0.75*x)*7.0+x*0.5+r_data*1.0
        x = tf.concat([xy,x], 1)/16.0
    elif args.distribution == 'arch':
        offset1 = tf.random_uniform((1, args.batch_size), -10, 10 )
        xa = tf.random_uniform((1, 1), 1, 4 )
        xb = tf.random_uniform((1, 1), 1, 4 )
        x1 = tf.random_uniform((1, args.batch_size), -1, 1 )
        xcos = tf.cos(x1*np.pi + offset1)*xa
        xsin = tf.sin(x1*np.pi + offset1)*xb
        x = tf.transpose(tf.concat([xcos,xsin], 0))/16.0

    initial_graph = {
            'x':x,
            'num_labels':1,
            }

    selector.save(config_filename, config)

    with tf.device(args.device):
        gan = hg.GAN(config, initial_graph)

        accuracy_x_to_g=batch_accuracy(gan.graph.x, gan.graph.g[0])
        accuracy_g_to_x=batch_accuracy(gan.graph.g[0], gan.graph.x)
        x_0 = gan.sess.run(gan.graph.x)
        z_0 = gan.sess.run(gan.graph.z[0])

        gan.initialize_graph()

        ax_sum = 0
        ag_sum = 0
        dlog = 0
        last_i = 0

        tf.train.start_queue_runners(sess=gan.sess)
        for i in range(1000):
            d_loss, g_loss = gan.train()

            if(i > 500):
                ax, ag, dl = gan.sess.run([accuracy_x_to_g, accuracy_g_to_x, gan.graph.d_log], {gan.graph.x: x_0, gan.graph.z[0]: z_0})
                ax_sum += ax
                ag_sum += ag
                dlog = dl

        with open("results.csv", "a") as myfile:
            if not any(isinf, [axsum, agsum, ax_sum*ag_sum]):
                myfile.write(config_name+","+str(int(ax_sum))+","+str(int(ag_sum))+","+ str(int(ax_sum+ag_sum))+","+str(int(ax_sum*ag_sum))+","+str(dlog)+","+str(last_i)+"\n")
        tf.reset_default_graph()
        gan.sess.close()

while(True):
    train()
<commit_msg>[fix] isinf check<commit_after>import argparse
import os
import uuid
import tensorflow as tf
import hypergan as hg
import hyperchamber as hc
import matplotlib.pyplot as plt
from hypergan.loaders import *
from hypergan.util.hc_tf import *

import math

def parse_args():
    parser = argparse.ArgumentParser(description='Train a 2d test!', add_help=True)
    parser.add_argument('--batch_size', '-b', type=int, default=32, help='Examples to include in each batch.  If using batch norm, this needs to be preserved when in server mode')
    parser.add_argument('--device', '-d', type=str, default='/gpu:0', help='In the form "/gpu:0", "/cpu:0", etc.  Always use a GPU (or TPU) to train')
    parser.add_argument('--format', '-f', type=str, default='png', help='jpg or png')
    parser.add_argument('--config', '-c', type=str, default='2d-test', help='config name')
    parser.add_argument('--distribution', '-t', type=str, default='circle', help='what distribution to test, options are circle, modes')
    return parser.parse_args()

def no_regularizer(amt):
    return None

def custom_discriminator_config():
    return {
            'create': custom_discriminator
    }

def custom_generator_config():
    return {
            'create': custom_generator
    }

def custom_discriminator(gan, config, x, g, xs, gs, prefix='d_'):
    net = tf.concat(axis=0, values=[x,g])
    net = linear(net, 128, scope=prefix+'lin1')
    net = tf.nn.relu(net)
    net = linear(net, 128, scope=prefix+'lin2')
    return net

def custom_generator(config, gan, net):
    net = linear(net, 128, scope="g_lin_proj")
    net = batch_norm_1(gan.config.batch_size, name='g_bn_1')(net)
    net = tf.nn.relu(net)
    net = linear(net, 2, scope="g_lin_proj3")
    net = tf.tanh(net)
    return [net]

def batch_accuracy(a, b):
    "Each point of a is measured against the closest point on b.  Distance differences are added together."
    tiled_a = a
    tiled_a = tf.reshape(tiled_a, [int(tiled_a.get_shape()[0]), 1, int(tiled_a.get_shape()[1])])

    tiled_a = tf.tile(tiled_a, [1, int(tiled_a.get_shape()[0]), 1])

    tiled_b = b
    tiled_b = tf.reshape(tiled_b, [1, int(tiled_b.get_shape()[0]), int(tiled_b.get_shape()[1])])
    tiled_b = tf.tile(tiled_b, [int(tiled_b.get_shape()[0]), 1, 1])

    difference = tf.abs(tiled_a-tiled_b)
    difference = tf.reduce_min(difference, axis=1)
    difference = tf.reduce_sum(difference, axis=1)
    return tf.reduce_sum(difference, axis=0)

args = parse_args()

def train():
    selector = hg.config.selector(args)
    config_name="2d-measure-accuracy-"+str(uuid.uuid4())

    config = selector.random_config()
    config_filename = os.path.expanduser('~/.hypergan/configs/'+config_name+'.json')

    trainers = []

    rms_opts = {
        'g_momentum': [0,0.1,0.01,1e-6,1e-5,1e-1,0.9,0.999, 0.5],
        'd_momentum': [0,0.1,0.01,1e-6,1e-5,1e-1,0.9,0.999, 0.5],
        'd_decay': [0.8, 0.9, 0.99,0.999,0.995,0.9999,1],
        'g_decay': [0.8, 0.9, 0.99,0.999,0.995,0.9999,1],
        'clipped_gradients': [False, 1e-2],
        'clipped_d_weights': [False, 1e-2],
        'd_learn_rate': [1e-3,1e-4,5e-4,1e-6,4e-4, 5e-5],
        'g_learn_rate': [1e-3,1e-4,5e-4,1e-6,4e-4, 5e-5]
    }
    trainers.append(hg.trainers.rmsprop_trainer.config(**rms_opts))

    adam_opts = {}

    adam_opts = {
        'd_learn_rate': [1e-3,1e-4,5e-4,1e-2,1e-6],
        'g_learn_rate': [1e-3,1e-4,5e-4,1e-2,1e-6],
        'd_beta1': [0.9, 0.99, 0.999, 0.1, 0.01, 0.2, 1e-8],
        'd_beta2': [0.9, 0.99, 0.999, 0.1, 0.01, 0.2, 1e-8],
        'g_beta1': [0.9, 0.99, 0.999, 0.1, 0.01, 0.2, 1e-8],
        'g_beta2': [0.9, 0.99, 0.999, 0.1, 0.01, 0.2, 1e-8],
        'd_epsilon': [1e-8, 1, 0.1, 0.5],
        'g_epsilon': [1e-8, 1, 0.1, 0.5],
        'd_clipped_weights': [False, 0.01],
        'clipped_gradients': [False, 0.01]
    }

    trainers.append(hg.trainers.adam_trainer.config(**adam_opts))
    encoders = []

    projections = []
    projections.append([hg.encoders.linear_encoder.modal])
    projections.append([hg.encoders.linear_encoder.modal, hg.encoders.linear_encoder.sphere, hg.encoders.linear_encoder.linear])
    projections.append([hg.encoders.linear_encoder.binary, hg.encoders.linear_encoder.sphere])
    projections.append([hg.encoders.linear_encoder.sphere, hg.encoders.linear_encoder.linear])
    encoder_opts = {
            'z': [2,4,8,16,32],
            'modes': [2, 4, 8, 16],
            'projections': projections
            }

    losses = []
    losses.append([hg.losses.wgan_loss.config()])
    encoders.append([hg.encoders.linear_encoder.config(**encoder_opts)])
    custom_config = {
        'model': args.config,
        'batch_size': args.batch_size,
        'trainer': trainers,
        'generator': custom_generator_config(),
        'discriminators': [[custom_discriminator_config()]],
        'losses': losses,
        'encoders': encoders
    }

    custom_config_selector = hc.Selector()
    for key,value in custom_config.items():
        custom_config_selector.set(key, value)
        print("Set ", key, value)

    custom_config_selection = custom_config_selector.random_config()

    for key,value in custom_config_selection.items():
        config[key]=value

    config['dtype']=tf.float32
    config = hg.config.lookup_functions(config)

    def circle(x):
        spherenet = tf.square(x)
        spherenet = tf.reduce_sum(spherenet, 1)
        lam = tf.sqrt(spherenet)
        return x/tf.reshape(lam,[int(lam.get_shape()[0]), 1])

    def modes(x):
        return tf.round(x*2)/2.0

    if args.distribution == 'circle':
        x = tf.random_normal([args.batch_size, 2])
        x = circle(x)
    elif args.distribution == 'modes':
        x = tf.random_uniform([args.batch_size, 2], -1, 1)
        x = modes(x)
    elif args.distribution == 'sin':
        x = tf.random_uniform((1, args.batch_size), -10.5, 10.5 )
        x = tf.transpose(x)
        r_data = tf.random_normal((args.batch_size,1), mean=0, stddev=0.1)
        xy = tf.sin(0.75*x)*7.0+x*0.5+r_data*1.0
        x = tf.concat([xy,x], 1)/16.0
    elif args.distribution == 'arch':
        offset1 = tf.random_uniform((1, args.batch_size), -10, 10 )
        xa = tf.random_uniform((1, 1), 1, 4 )
        xb = tf.random_uniform((1, 1), 1, 4 )
        x1 = tf.random_uniform((1, args.batch_size), -1, 1 )
        xcos = tf.cos(x1*np.pi + offset1)*xa
        xsin = tf.sin(x1*np.pi + offset1)*xb
        x = tf.transpose(tf.concat([xcos,xsin], 0))/16.0

    initial_graph = {
            'x':x,
            'num_labels':1,
            }

    selector.save(config_filename, config)

    with tf.device(args.device):
        gan = hg.GAN(config, initial_graph)

        accuracy_x_to_g=batch_accuracy(gan.graph.x, gan.graph.g[0])
        accuracy_g_to_x=batch_accuracy(gan.graph.g[0], gan.graph.x)
        x_0 = gan.sess.run(gan.graph.x)
        z_0 = gan.sess.run(gan.graph.z[0])

        gan.initialize_graph()

        ax_sum = 0
        ag_sum = 0
        dlog = 0
        last_i = 0

        tf.train.start_queue_runners(sess=gan.sess)
        for i in range(1000):
            d_loss, g_loss = gan.train()

            if(i > 500):
                ax, ag, dl = gan.sess.run([accuracy_x_to_g, accuracy_g_to_x, gan.graph.d_log], {gan.graph.x: x_0, gan.graph.z[0]: z_0})
                ax_sum += ax
                ag_sum += ag
                dlog = dl

        with open("results.csv", "a") as myfile:
            if not math.isinf(ax_sum*ag_sum):
                myfile.write(config_name+","+str(int(ax_sum))+","+str(int(ag_sum))+","+ str(int(ax_sum+ag_sum))+","+str(int(ax_sum*ag_sum))+","+str(dlog)+","+str(last_i)+"\n")
        tf.reset_default_graph()
        gan.sess.close()

while(True):
    train()
<|endoftext|>