<commit_before>from threading import Thread, Lock
from queue import Queue
from mpi4py import MPI

import commutils
from enums import CsState, MsgTag

class BaseCommAdapter(object):
    def __init__(self, comm, logging=False):
        self.logging = logging
        self._comm = comm

    def __getattr__(self, name):
        return getattr(self._comm, name)

    def __enter__(self):
        self.open()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()

    def ssend(self, msg, dest, tag=0):
        self.ssend(msg, dest=dest, tag=tag)

    def srecv(self, source, tag=0):
        return self.srecv(source=source, tag=tag)

    def isfaulty(self, procid):
        return self._comm.isfaulty(procid)

    def _log(self, msg, *args, **kwargs):
        if self.logging:
            M = '{:15}: {}'.format(type(self).__name__, msg)
            commutils.log(M, *args, **kwargs)

class MPICommAdapter(BaseCommAdapter):
    def __init__(self, comm, logging=False, m=1):
        super().__init__(comm, logging=logging)
        size = comm.Get_size()
        self._m = m
        self._recv_thread = Thread(target=self._recv_daemon)
        self._cs_state = CsState.OUT
        self._id = comm.Get_rank()
        self._lrd = 0
        self._clock = 0
        self._used_by = [0 for i in size]
        self._perm_delayed = [0 for i in size]
        self._lock = Lock()

    def open(self):
        self._log('Starting receiver thread')
        self._recv_thread.start()

    def close(self):
        self._log('Closing communicator')

    def send(self, msg, dest, tag=MsgTag.REQUEST):
        self.ssend(msg, dest=dest, tag=tag)

    def recv(self, source, tag=MsgTag.REQUEST):
        return self.srecv(source=source, tag=tag)

    def iprobe(self, source, tag=MsgTag.REQUEST):
        self._comm.iprobe(source=source, tag=tag)

    ssend = send

    srecv = recv

    def _recv_daemon(self):
        self._log('Receiver thread started')
        status = MPI.Status()
        while True:
            if self.iprobe(source=MPI.ANY_SOURCE, tag=MsgTag.REQUEST):
                pass
            if self.iprobe(source=MPI.ANY_SOURCE, tag=MsgTag.NOT_USED):
                pass
            result = self._comm.recv(source=MPI.ANY_SOURCE, status=status, tag=MPI.ANY_TAG)

    def acquire_resource(self, k, type='basic'):
        with self._lock:
            self._cs_state = CsState.TRYING
            self._lrd = self._clock + 1
            all_indexes = range(0, self._comm.Get_size())
            indexes_to_check = set(all_indexes) - set(self._id)
            for j in indexes_to_check:
                self.send(
                    {'lrd': self._lrd, 'id': self._id, 'k': k, 'type': type}, j, MsgTag.REQUEST)
                self._used_by[j] += self._m
            self._used_by[self._id] = k
        while True:
            with self._lock:
                if sum(self._used_by) <= self._m:
                    break
        self._cs_state = CsState.IN

    def release_resource(self, k):
        with self._lock:
            self._cs_state = CsState.OUT
            indexes_to_check = set(range(0, self._comm.Get_size()))
            for j in indexes_to_check:
                if self._perm_delayed[j] > 0:
                    self._comm.send(
                        {'lrd': self._lrd, 'id': self._id, 'k': k, 'type': type}, j, MsgTag.NOT_USED)
            self._perm_delayed = [0 for i in self._comm.Get_size()]

COMM_WORLD = MPICommAdapter(MPI.COMM_WORLD, logging=False)
<commit_msg>update<commit_after>from threading import Thread, Lock
from queue import Queue
from mpi4py import MPI

import commutils
from enums import CsState, MsgTag

class BaseCommAdapter(object):
    def __init__(self, comm, logging=False):
        self.logging = logging
        self._comm = comm

    def __getattr__(self, name):
        return getattr(self._comm, name)

    def __enter__(self):
        self.open()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()

    def ssend(self, msg, dest, tag=0):
        self.ssend(msg, dest=dest, tag=tag)

    def srecv(self, source, tag=0):
        return self.srecv(source=source, tag=tag)

    def isfaulty(self, procid):
        return self._comm.isfaulty(procid)

    def _log(self, msg, *args, **kwargs):
        if self.logging:
            M = '{:15}: {}'.format(type(self).__name__, msg)
            commutils.log(M, *args, **kwargs)

class MPICommAdapter(BaseCommAdapter):
    def __init__(self, comm, logging=False, m=1):
        super().__init__(comm, logging=logging)
        size = comm.Get_size()
        self._m = m
        self._recv_thread = Thread(target=self._recv_daemon)
        self._id = comm.Get_rank()
        self._cs_state = CsState.OUT
        self._lrd = 0
        self._clock = 0
        self._used_by = [0 for i in size]
        self._perm_delayed = [0 for i in size]
        self._prio = False
        self._lock = Lock()

    def open(self):
        self._log('Starting receiver thread')
        self._recv_thread.start()

    def close(self):
        self._log('Closing communicator')

    def send(self, msg, dest, tag=MsgTag.REQUEST):
        self.ssend(msg, dest=dest, tag=tag)

    def recv(self, source, tag=MsgTag.REQUEST):
        return self.srecv(source=source, tag=tag)

    def iprobe(self, source, tag=MsgTag.REQUEST):
        self._comm.iprobe(source=source, tag=tag)

    ssend = send

    srecv = recv

    def _recv_daemon(self):
        self._log('Receiver thread started')
        while True:
            if self.iprobe(source=MPI.ANY_SOURCE, tag=MsgTag.REQUEST):
                with self._lock:
                    result = self.recv(source=MPI.ANY_SOURCE, tag=MsgTag.REQUEST)
                    self._clock = max(self._clock, result['k'])
                    less_op = self._lrd < result['lrd'] | \
                                          self._lrd == result['lrd'] & self._id < result['id']
                    self._prio = self._cs_state is not CsState.OUT and less_op
                    if not self._prio or (self._prio and self._perm_delayed[result['id']]):
                        self._log('Sending permission for {0} {1}'.format(self._m,' instances'))
                        self.send(
                            {'lrd': self._lrd, 'id': self._id, 'k': self._m, 'type': type},
                            result['id'], MsgTag.NOT_USED)
                    else:
                        if result['k'] != self._m:
                            self._log(
                                'Sending permission for {0} {1}'.format(
                                    self._m - result['k'], ' instances'))
                            self.send(
                                {'lrd': self._lrd, 'id': self._id, 'k': (self._m,  - result['k']), 'type': type},
                                result['id'], MsgTag.NOT_USED)
                        self._perm_delayed[result['id']] += 1
            if self.iprobe(source=MPI.ANY_SOURCE, tag=MsgTag.NOT_USED):
                with self._lock:
                    result = self.recv(source=MPI.ANY_SOURCE, tag=MsgTag.NOT_USED)
                    self._used_by[result['id']] -= result['k']

    def acquire_resource(self, k, type='basic'):
        with self._lock:
            self._cs_state = CsState.TRYING
            self._lrd = self._clock + 1
            all_indexes = range(0, self._comm.Get_size())
            indexes_to_check = set(all_indexes) - set(self._id)
            for j in indexes_to_check:
                self._log(
                    'Sending request for {0} {1}'.format(k, ' instances'))
                self.send(
                    {'lrd': self._lrd, 'id': self._id, 'k': k, 'type': type}, j, MsgTag.REQUEST)
                self._used_by[j] += self._m
            self._used_by[self._id] = k
        while True:
            with self._lock:
                if sum(self._used_by) <= self._m:
                    break
        self._cs_state = CsState.IN

    def release_resource(self, k):
        with self._lock:
            self._cs_state = CsState.OUT
            indexes_to_check = set(range(0, self._comm.Get_size()))
            for j in indexes_to_check:
                if self._perm_delayed[j] > 0:
                    self._log(
                        'Sending permission for {0} {1}'.format(k, ' instances'))
                    self._comm.send(
                        {'lrd': self._lrd, 'id': self._id, 'k': k, 'type': type}, j, MsgTag.NOT_USED)
            self._perm_delayed = [0 for i in self._comm.Get_size()]

COMM_WORLD = MPICommAdapter(MPI.COMM_WORLD, logging=False)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# conan documentation build configuration file, created by
# sphinx-quickstart on Wed Apr 27 13:10:21 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.

sys.path.insert(0, os.path.abspath('./_themes'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.todo',
    'conan',
]

# The short X.Y version.
version = "1.3"
# The full version, including alpha/beta/rc tags.
release = u'1.3.3'
versions_dict = {"master": "latest"}  #,
               # "release/1.4.0": "1.4"}

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#

html_context = {
    "versions": versions_dict,
    "current_version": version
}

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'conan'
copyright = u'2016-2018, JFrog'
author = u'conan'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------
import conan

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
# html_theme = 'sphinx_rtd_theme'
html_theme = "conan"
html_theme_path = conan.get_html_theme_path()

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
html_theme_options = {
  "base_url": "http://docs.conan.io/en/latest/",
}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = "_static/conan_logo.png"

# The name of an image file (relative to this directory) to use as a favicon of
# the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
#html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'conandoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'conan.tex', u'conan Documentation',
     u'conan', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'conan', u'conan Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'conan', u'conan Documentation',
     author, 'conan', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# -- Options for Epub output ----------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project
epub_author = author
epub_publisher = author
epub_copyright = copyright

# The basename for the epub file. It defaults to the project name.
#epub_basename = project

# The HTML theme for the epub output. Since the default themes are not
# optimized for small screen space, using the same theme for HTML and epub
# output is usually not wise. This defaults to 'epub', a theme designed to save
# visual space.
#epub_theme = 'epub'

# The language of the text. It defaults to the language option
# or 'en' if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
#epub_cover = ()

# A sequence of (type, uri, title) tuples for the guide element of content.opf.
#epub_guide = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files that should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

# Allow duplicate toc entries.
#epub_tocdup = True

# Choose between 'default' and 'includehidden'.
#epub_tocscope = 'default'

# Fix unsupported image types using the Pillow.
#epub_fix_images = False

# Scale large images.
#epub_max_image_width = 0

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#epub_show_urls = 'inline'

# If false, no index is generated.
#epub_use_index = True
<commit_msg>Fixed base url<commit_after># -*- coding: utf-8 -*-
#
# conan documentation build configuration file, created by
# sphinx-quickstart on Wed Apr 27 13:10:21 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.

sys.path.insert(0, os.path.abspath('./_themes'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.todo',
    'conan',
]

# The short X.Y version.
version = "1.3"
# The full version, including alpha/beta/rc tags.
release = u'1.3.3'
versions_dict = {"master": "latest"}  #,
               # "release/1.4.0": "1.4"}

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#

html_context = {
    "versions": versions_dict,
    "current_version": version
}

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'conan'
copyright = u'2016-2018, JFrog'
author = u'conan'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------
import conan

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
# html_theme = 'sphinx_rtd_theme'
html_theme = "conan"
html_theme_path = conan.get_html_theme_path()

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
html_theme_options = {
  "base_url": "https://docs.conan.io/en/latest/",
}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = "_static/conan_logo.png"

# The name of an image file (relative to this directory) to use as a favicon of
# the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
#html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'conandoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'conan.tex', u'conan Documentation',
     u'conan', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'conan', u'conan Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'conan', u'conan Documentation',
     author, 'conan', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# -- Options for Epub output ----------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project
epub_author = author
epub_publisher = author
epub_copyright = copyright

# The basename for the epub file. It defaults to the project name.
#epub_basename = project

# The HTML theme for the epub output. Since the default themes are not
# optimized for small screen space, using the same theme for HTML and epub
# output is usually not wise. This defaults to 'epub', a theme designed to save
# visual space.
#epub_theme = 'epub'

# The language of the text. It defaults to the language option
# or 'en' if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
#epub_cover = ()

# A sequence of (type, uri, title) tuples for the guide element of content.opf.
#epub_guide = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files that should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

# Allow duplicate toc entries.
#epub_tocdup = True

# Choose between 'default' and 'includehidden'.
#epub_tocscope = 'default'

# Fix unsupported image types using the Pillow.
#epub_fix_images = False

# Scale large images.
#epub_max_image_width = 0

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#epub_show_urls = 'inline'

# If false, no index is generated.
#epub_use_index = True
<|endoftext|>


<commit_before>import os
import time
import collections
from datetime import datetime
from abc import ABCMeta, abstractmethod

import numpy as np
import tensorflow as tf
import tensortools as tt

TRAIN_DIR = 'train' # (automatially train_<ModelClassName> ?)
GPU_ALLOW_GROWTH = True # fixed? :)
GPU_MEMORY_FRACTION = 1.0 # fixed? :)

FIXED_NUM_STEPS_PER_DECAY = 10000
NUM_EPOCHS_PER_DECAY = 75.0 # used if FIXED_NUM_STEPS_PER_DECAY is None

INITIAL_LEARNING_RATE = 0.0001
LEARNING_RATE_DECAY_FACTOR = 0.5

class AbstractRuntime(object):
    __metaclass__ = ABCMeta

    def __init__(self):
        self._graph = tf.Graph() # graph really needed?
        self._session = None
        self._datasets = collections.namedtuple("datasets", ("train", "valid", "test"))
        self._model = None
        self._inferences = []

        self._feed_func = None

        self._x = None
        self._y_ = None

        self._is_training = tf.placeholder(tf.bool, name='is_training')
        self._global_step = tf.Variable(0, trainable=False)

        self._summary_writer = None

        self._train_op = None
        self._total_loss = None
        self._loss = None
        self._summary_op = None

    def register_datasets(self, train_ds, valid_ds=None, test_ds=None):
        self._datasets.train = train_ds
        self._datasets.valid = valid_ds
        self._datasets.test = test_ds

    def register_model(self, model):
        self._model = model

    def build(self, is_autoencoder=False):
        # FIXME: placeholders not required, when datasets uses input_queue (change after DS refactoring)
        self._x = tf.placeholder(tf.float32, [None] + self._datasets.train.input_shape, "X")
        if is_autoencoder:
            self._y_ = tf.placeholder(tf.float32, [None] + self._datasets.train.input_shape, "Y_")
            self._feed_func = lambda x_ph, y_ph, inputs, targets : {x_ph: inputs, y_ph: inputs}
        else:
            self._y_ = tf.placeholder(tf.float32, [None] + self._datasets.train.target_shape, "Y_")
            self._feed_func = lambda x_ph, y_ph, inputs, targets : {x_ph: inputs, y_ph: targets}

        train_op, total_loss, loss, summaries = self._build_internal(self._x, self._y_)

        self._train_op = train_op
        self._total_loss = total_loss
        self._loss = loss
        if summaries is None:
            self._summary_op = tf.merge_all_summaries()
        else:
            self._summary_op = tf.merge_summary(summaries)

        # start session and init all variables
        self.session.run(tf.initialize_all_variables())

    def train(self, steps=-1, epochs=-1, display_step=10):
        assert not(steps <= 0 and epochs <= 0), "Either set 'steps' or 'epochs' parameter"
        assert not(steps > 0 and epochs > 0), "Not allowed to set both, 'steps' and 'epochs' parameter"

        if epochs > 0:
            steps = self.datasets.train.batches_per_epoch * epochs

        # Create a saver to store checkpoints of the model
        saver = tf.train.Saver(tf.all_variables())

        # TODO: this is only required for inpute Queue!!!
        # Start input enqueue threads
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=self.session, coord=coord)

        self.datasets.train.reset()

        try:
            this_step = 0
            total_loss_sum = 0
            loss_sum = 0
            while not coord.should_stop():
                this_step += 1
                if (this_step > steps):
                    break

                if this_step % self.datasets.train.batches_per_epoch == 1:
                    epoch = (this_step - 1) // self.datasets.train.batches_per_epoch + 1
                    print("Starting epoch {}...".format(epoch))

                start_time = time.time()

                batch_x, batch_y = self.datasets.train.get_batch()
                feed = self._feed_func(self._x, self._y_, batch_x, batch_y)
                feed.update({self._is_training: True})

                # step counter is increment when train_op is executed
                _, gstep, total_loss, loss = self.session.run([self._train_op,
                                                              self._global_step,
                                                              self._total_loss,
                                                              self._loss],
                                                             feed_dict=feed)
                duration = time.time() - start_time

                assert not np.isnan(loss), 'Warning: Model diverged with loss = NaN'

                total_loss_sum += total_loss
                loss_sum += loss
                if gstep % display_step == 0:
                    # info
                    num_examples_per_step = self.datasets.train.batch_size
                    examples_per_sec = num_examples_per_step / duration
                    sec_per_batch = float(duration)
                    avg_total_loss = total_loss_sum / display_step
                    avg_loss = loss_sum / display_step
                    total_loss_sum = 0
                    loss_sum = 0
                    print("@{:6d}: loss: {:9.3f}, total-loss: {:9.3f} ({:7.1f} examples/sec, {:5.2f} sec/batch)" \
                          .format(gstep, avg_loss, avg_total_loss, examples_per_sec, sec_per_batch))

                if gstep % 100 or this_step == steps:
                    # summary
                    summary_str = self.session.run(self._summary_op, feed_dict=feed)
                    self.summary_writer.add_summary(summary_str, gstep)
                    self.summary_writer.flush()

                if gstep == 100 or this_step == steps or epochs == -1 and gstep % 1000 == 0 or \
                   epochs > 0 and this_step % self.datasets.train.batches_per_epoch == 0:
                    # validate
                    print
                    self._test_internal(self.datasets.valid, "validation", True)
                    print

                if gstep % 1000 == 0 or this_step == steps:
                    # save regular checkpoint
                    checkpoint_path = os.path.join(TRAIN_DIR, "model.ckpt")
                    saver.save(self.session, checkpoint_path, global_step=self._global_step)

                if epochs > 0:
                    if this_step % self.datasets.train.batches_per_epoch == 0:
                        # save epoch checkpoint
                        checkpoint_path = os.path.join(TRAIN_DIR, "ep-{}_model.ckpt".format(epoch))
                        saver.save(self.session, checkpoint_path, global_step=self._global_step)

        except tf.errors.OutOfRangeError:
            print("Done training -- epoch limit reached")
        finally:
            # When done, ask the threads to stop
            coord.request_stop()

        # Wait for threads to finish
        coord.join(threads)

    @abstractmethod
    def _build_internal(self, x, y_):
        pass

    def predict(self, inputs):
        feed = self._feed_func(self._x, self._y_, inputs, None)
        feed.update({self._is_training: False})
        return self.session.run(self._inferences[0], feed_dict=feed)

    def validate(self):
        self._test_internal(self.datasets.valid, "validation")

    def test(self):
        self._test_internal(self.datasets.test, "test")

    def _test_internal(self, dataset, title, summary=False):
        if dataset is None:
            print("No {} dataset registered. Skipping.".format(title))
            return

        num_batches = dataset.batches_per_epoch
        gstep = self.session.run(self._global_step)
        print("@{:6d}: Starting {} (batch-size: {}, dataset-size: {}):" \
              .format(gstep, title, dataset.batch_size, dataset.size))

        dataset.reset()
        loss_sum = 0
        progress = tt.utils.ui.ProgressBar(num_batches * dataset.batch_size)
        for b in xrange(num_batches):
            batch_x, batch_y = self.datasets.train.get_batch()
            feed = self._feed_func(self._x, self._y_, batch_x, batch_y)
            feed.update({self._is_training: False})

            this_loss = self.session.run(self._loss, feed_dict=feed)
            loss_sum += this_loss
            progress.update((b+1) * dataset.batch_size, [('loss', this_loss)])

        if summary:
            avg_loss = loss_sum / num_batches
            loss_summary = tf.scalar_summary("{}_loss".format(title), avg_loss)
            summary_str = self.session.run(loss_summary)
            self.summary_writer.add_summary(summary_str, gstep)
            self.summary_writer.flush()

    def _create_session(self):
        gpu_options = tf.GPUOptions(
            per_process_gpu_memory_fraction=GPU_MEMORY_FRACTION,
            allow_growth=GPU_ALLOW_GROWTH)
        return tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))

    def close(self):
        self.session.close()

    @property
    def graph(self):
        return self._graph

    @property
    def session(self):
        if self._session is None:
            self._session = self._create_session()
        return self._session

    @property
    def datasets(self):
        return self._datasets

    @property
    def summary_writer(self):
        if self._summary_writer is None:
            self._summary_writer = tf.train.SummaryWriter(TRAIN_DIR, self.session.graph)
        return self._summary_writer

class DefaultRuntime(AbstractRuntime):

    def __init__(self):
        super(AbstractRuntime, self).__init__()

class MultiGpuRuntime(AbstractRuntime):

    def __init__(self, num_gpus=2):

        device_list = tt.hardware.get_cuda_devices()
        assert len(device_list) >= num_gpus, "Not enough GPU devices available."
        print("Launing Multi-GPU runtime...")
        print("Selecting devices: {}".format(device_list[0:num_gpus]))
        self._num_gpus = num_gpus

        super(MultiGpuRuntime, self).__init__()

    @tt.utils.attr.override
    def _build_internal(self, x, y_):
        """Train sequence model.
        predictions_for_input:
            Either the same as predictions or None
        """
        # Variables that affect learning rate
        batch_size = self.datasets.train.batch_size
        num_batches_per_epoch = self.datasets.train.size / batch_size
        decay_steps = num_batches_per_epoch * NUM_EPOCHS_PER_DECAY

        if FIXED_NUM_STEPS_PER_DECAY is not None:
            decay_steps = FIXED_NUM_STEPS_PER_DECAY

        # Decay the learning rate exponentially based on the number of steps
        lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE,
                                        self._global_step,
                                        decay_steps,
                                        LEARNING_RATE_DECAY_FACTOR,
                                        staircase=True)
        tf.scalar_summary('learning_rate', lr)

        # Compute gradients
        opt = tf.train.AdamOptimizer(lr) # TODO: make optimizer parameterizable? or spcify in model?
        # Calculate the gradients for each model tower.
        tower_grads = []
        tower_total_losses = []
        tower_losses = []
        batch_size_per_gpu = (batch_size // self.num_gpus)
        for i in xrange(self.num_gpus):
            with tf.device('/gpu:%d' % i, ):
                with tf.name_scope('%s_%d' % ('tower', i)) as scope:
                    this_inputs = x[i*batch_size_per_gpu:(i+1)*batch_size_per_gpu, :, :, :, :]
                    this_targets = y_[i*batch_size_per_gpu:(i+1)*batch_size_per_gpu, :, :, :, :]

                    # Build inference Graph.This function constructs
                    # the entire model but shares the variables across all towers.
                    inference = self._model.inference(this_inputs, this_targets,
                                                      is_training=self._is_training,
                                                      device_scope=scope,
                                                      memory_device='/cpu:0')
                    self._inferences.append(inference)

                    loss = self._model.loss(inference, this_targets)
                    total_loss = self._model.total_loss(loss, inference, this_targets)

                    # Calculate the moving averages of the loss for one tower of the model
                    loss_averages_op = tt.board.loss_summary([total_loss, loss] +
                                                             tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) + # FIXME: redundant!??! Is the list empty???
                                                             tf.get_collection('intermediate_losses', scope),
                                                             decay=0.9)

                    with tf.control_dependencies([loss_averages_op]):
                        this_total_loss = tf.identity(total_loss)
                        this_loss = tf.identity(loss)

                    tower_total_losses.append(this_total_loss)
                    tower_losses.append(this_loss)

                    # Reuse variables for the next tower.
                    tf.get_variable_scope().reuse_variables()

                    # Retain the summaries from the final tower.
                    summaries = tf.get_collection(tf.GraphKeys.SUMMARIES, scope)

                    # Calculate the gradients for the batch of data on this tower.
                    grads = opt.compute_gradients(this_total_loss)

                    # Keep track of the gradients ackeep_probross all towers.
                    tower_grads.append(grads)

        # We must calculate the mean of each gradient.
        # This is also the synchronization point across all towers.
        grads = tt.training.average_gradients(tower_grads)

        total_loss = tf.reduce_mean(tower_total_losses)
        summaries.append(tf.scalar_summary('mean_total_loss', total_loss)) # TODO: really needed?

        loss = tf.reduce_mean(tower_losses)
        summaries.append(tf.scalar_summary('mean_loss', loss)) # TODO: really needed?

        # Add a summary to track the learning rate.
        summaries.append(tf.scalar_summary('learning_rate', lr))

        # Add histograms for gradients
        summaries.extend(tt.board.gradients_histogram_summary(grads))

        # Apply the gradients to adjust the shared variables and increment the step counter
        apply_gradient_op = opt.apply_gradients(grads, global_step=self._global_step)

        summaries.extend(tt.board.variables_histogram_summary())

        # Track the moving averages of all trainable variables
        variable_averages = tf.train.ExponentialMovingAverage(0.9999, self._global_step)
        variables_averages_op = variable_averages.apply(tf.trainable_variables())

        # Group all updates to into a single train op.
        train_op = tf.group(apply_gradient_op, variables_averages_op, name="train_op")

        return train_op, total_loss, loss, summaries

    def start_training(self):
        #with self.graph.as_default(), tf.device('/cpu:0'):
        with tf.device('/cpu:0'):
            return super(MultiGpuRuntime, self).start_training()

    @property
    def num_gpus(self):
        return self._num_gpus
            <commit_msg>Implemented DefaultRuntime for CPU/single-GPU execution<commit_after>import os
import time
import collections
from datetime import datetime
from abc import ABCMeta, abstractmethod

import numpy as np
import tensorflow as tf
import tensortools as tt

TRAIN_DIR = 'train' # (automatially train_<ModelClassName> ?)
GPU_ALLOW_GROWTH = True # fixed? :)
GPU_MEMORY_FRACTION = 1.0 # fixed? :)

FIXED_NUM_STEPS_PER_DECAY = 10000
NUM_EPOCHS_PER_DECAY = 75.0 # used if FIXED_NUM_STEPS_PER_DECAY is None

INITIAL_LEARNING_RATE = 0.0001
LEARNING_RATE_DECAY_FACTOR = 0.5

class AbstractRuntime(object):
    __metaclass__ = ABCMeta

    def __init__(self):
        self._graph = tf.Graph() # graph really needed?
        self._session = None
        self._datasets = collections.namedtuple("datasets", ("train", "valid", "test"))
        self._model = None
        self._inferences = []

        self._feed_func = None

        self._x = None
        self._y_ = None

        self._is_training = tf.placeholder(tf.bool, name='is_training')
        self._global_step = tf.Variable(0, trainable=False)

        self._summary_writer = None

        self._train_op = None
        self._total_loss = None
        self._loss = None
        self._summary_op = None

    def register_datasets(self, train_ds, valid_ds=None, test_ds=None):
        self._datasets.train = train_ds
        self._datasets.valid = valid_ds
        self._datasets.test = test_ds

    def register_model(self, model):
        self._model = model

    def build(self, is_autoencoder=False):
        # FIXME: placeholders not required, when datasets uses input_queue (change after DS refactoring)
        self._x = tf.placeholder(tf.float32, [None] + self._datasets.train.input_shape, "X")
        if is_autoencoder:
            self._y_ = tf.placeholder(tf.float32, [None] + self._datasets.train.input_shape, "Y_")
            self._feed_func = lambda x_ph, y_ph, inputs, targets : {x_ph: inputs, y_ph: inputs}
        else:
            self._y_ = tf.placeholder(tf.float32, [None] + self._datasets.train.target_shape, "Y_")
            self._feed_func = lambda x_ph, y_ph, inputs, targets : {x_ph: inputs, y_ph: targets}

        train_op, total_loss, loss, summaries = self._build_internal(self._x, self._y_)

        self._train_op = train_op
        self._total_loss = total_loss
        self._loss = loss
        if summaries is None:
            self._summary_op = tf.merge_all_summaries()
        else:
            self._summary_op = tf.merge_summary(summaries)

        # start session and init all variables
        self.session.run(tf.initialize_all_variables())

    def train(self, steps=-1, epochs=-1, display_step=10):
        assert not(steps <= 0 and epochs <= 0), "Either set 'steps' or 'epochs' parameter"
        assert not(steps > 0 and epochs > 0), "Not allowed to set both, 'steps' and 'epochs' parameter"

        if epochs > 0:
            steps = self.datasets.train.batches_per_epoch * epochs

        # Create a saver to store checkpoints of the model
        saver = tf.train.Saver(tf.all_variables())

        # TODO: this is only required for inpute Queue!!!
        # Start input enqueue threads
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=self.session, coord=coord)

        self.datasets.train.reset()

        try:
            this_step = 0
            total_loss_sum = 0
            loss_sum = 0
            while not coord.should_stop():
                this_step += 1
                if (this_step > steps):
                    break

                if this_step % self.datasets.train.batches_per_epoch == 1:
                    epoch = (this_step - 1) // self.datasets.train.batches_per_epoch + 1
                    print("Starting epoch {}...".format(epoch))

                start_time = time.time()

                batch_x, batch_y = self.datasets.train.get_batch()
                feed = self._feed_func(self._x, self._y_, batch_x, batch_y)
                feed.update({self._is_training: True})

                # step counter is increment when train_op is executed
                _, gstep, total_loss, loss = self.session.run([self._train_op,
                                                              self._global_step,
                                                              self._total_loss,
                                                              self._loss],
                                                             feed_dict=feed)
                duration = time.time() - start_time

                assert not np.isnan(loss), 'Warning: Model diverged with loss = NaN'

                total_loss_sum += total_loss
                loss_sum += loss
                if gstep % display_step == 0:
                    # info
                    num_examples_per_step = self.datasets.train.batch_size
                    examples_per_sec = num_examples_per_step / duration
                    sec_per_batch = float(duration)
                    avg_total_loss = total_loss_sum / display_step
                    avg_loss = loss_sum / display_step
                    total_loss_sum = 0
                    loss_sum = 0
                    print("@{:6d}: loss: {:9.3f}, total-loss: {:9.3f} ({:7.1f} examples/sec, {:5.2f} sec/batch)" \
                          .format(gstep, avg_loss, avg_total_loss, examples_per_sec, sec_per_batch))

                if gstep % 100 or this_step == steps:
                    # summary
                    summary_str = self.session.run(self._summary_op, feed_dict=feed)
                    self.summary_writer.add_summary(summary_str, gstep)
                    self.summary_writer.flush()

                if gstep == 100 or this_step == steps or epochs == -1 and gstep % 1000 == 0 or \
                   epochs > 0 and this_step % self.datasets.train.batches_per_epoch == 0:
                    # validate
                    print
                    self._test_internal(self.datasets.valid, "validation", True)
                    print

                if gstep % 1000 == 0 or this_step == steps:
                    # save regular checkpoint
                    checkpoint_path = os.path.join(TRAIN_DIR, "model.ckpt")
                    saver.save(self.session, checkpoint_path, global_step=self._global_step)

                if epochs > 0:
                    if this_step % self.datasets.train.batches_per_epoch == 0:
                        # save epoch checkpoint
                        checkpoint_path = os.path.join(TRAIN_DIR, "ep-{}_model.ckpt".format(epoch))
                        saver.save(self.session, checkpoint_path, global_step=self._global_step)

        except tf.errors.OutOfRangeError:
            print("Done training -- epoch limit reached")
        finally:
            # When done, ask the threads to stop
            coord.request_stop()

        # Wait for threads to finish
        coord.join(threads)

    @abstractmethod
    def _build_internal(self, x, y_):
        pass

    def predict(self, inputs):
        feed = self._feed_func(self._x, self._y_, inputs, None)
        feed.update({self._is_training: False})
        return self.session.run(self._inferences[0], feed_dict=feed)

    def validate(self):
        self._test_internal(self.datasets.valid, "validation")

    def test(self):
        self._test_internal(self.datasets.test, "test")

    def _test_internal(self, dataset, title, summary=False):
        if dataset is None:
            print("No {} dataset registered. Skipping.".format(title))
            return

        num_batches = dataset.batches_per_epoch
        gstep = self.session.run(self._global_step)
        print("@{:6d}: Starting {} (batch-size: {}, dataset-size: {}):" \
              .format(gstep, title, dataset.batch_size, dataset.size))

        dataset.reset()
        loss_sum = 0
        progress = tt.utils.ui.ProgressBar(num_batches * dataset.batch_size)
        for b in xrange(num_batches):
            batch_x, batch_y = self.datasets.train.get_batch()
            feed = self._feed_func(self._x, self._y_, batch_x, batch_y)
            feed.update({self._is_training: False})

            this_loss = self.session.run(self._loss, feed_dict=feed)
            loss_sum += this_loss
            progress.update((b+1) * dataset.batch_size, [('loss', this_loss)])

        if summary:
            avg_loss = loss_sum / num_batches
            loss_summary = tf.scalar_summary("{}_loss".format(title), avg_loss)
            summary_str = self.session.run(loss_summary)
            self.summary_writer.add_summary(summary_str, gstep)
            self.summary_writer.flush()

    def _create_session(self):
        gpu_options = tf.GPUOptions(
            per_process_gpu_memory_fraction=GPU_MEMORY_FRACTION,
            allow_growth=GPU_ALLOW_GROWTH)
        return tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))

    def close(self):
        self.session.close()

    @property
    def graph(self):
        return self._graph

    @property
    def session(self):
        if self._session is None:
            self._session = self._create_session()
        return self._session

    @property
    def datasets(self):
        return self._datasets

    @property
    def summary_writer(self):
        if self._summary_writer is None:
            self._summary_writer = tf.train.SummaryWriter(TRAIN_DIR, self.session.graph)
        return self._summary_writer

class DefaultRuntime(AbstractRuntime):

    def __init__(self):
        print("Launing default runtime...")
        super(DefaultRuntime, self).__init__()

    @tt.utils.attr.override
    def _build_internal(self, x, y_):
        # Variables that affect learning rate
        batch_size = self.datasets.train.batch_size
        num_batches_per_epoch = self.datasets.train.size / batch_size
        decay_steps = num_batches_per_epoch * NUM_EPOCHS_PER_DECAY

        if FIXED_NUM_STEPS_PER_DECAY is not None:
            decay_steps = FIXED_NUM_STEPS_PER_DECAY

        # Decay the learning rate exponentially based on the number of steps
        lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE,
                                        self._global_step,
                                        decay_steps,
                                        LEARNING_RATE_DECAY_FACTOR,
                                        staircase=True)
        tf.scalar_summary('learning_rate', lr)

        # Build inference Graph.This function constructs
        # the entire model but shares the variables across all towers.
        inference = self._model.inference(x, y_,
                                          is_training=self._is_training)
        self._inferences.append(inference)

        loss = self._model.loss(inference, y_)
        total_loss = self._model.total_loss(loss, inference, y_)

        # Generate moving averages of all losses and associated summaries
        loss_averages_op = tt.board.loss_summary([total_loss, loss] +
                                                 tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) + # FIXME: redundant!??! Is the list empty???
                                                 tf.get_collection('intermediate_losses'),
                                                 decay=0.9)

        # Compute gradients
        with tf.control_dependencies([loss_averages_op]):
            opt = tf.train.AdamOptimizer(lr)
            grads = opt.compute_gradients(total_loss)

        # Apply gradients
        apply_gradient_op = opt.apply_gradients(grads, global_step=self._global_step)

        tt.board.variables_histogram_summary()

        # Add histograms for gradients
        tt.board.gradients_histogram_summary(grads)

        # Track the moving averages of all trainable variables
        variable_averages = tf.train.ExponentialMovingAverage(0.9999, self._global_step)
        variables_averages_op = variable_averages.apply(tf.trainable_variables())

        train_op = tf.group(apply_gradient_op, variables_averages_op, name="train_op")
        return train_op, total_loss, loss, None # FIXME: collect summaries above? Return get_all_sum?

class MultiGpuRuntime(AbstractRuntime):

    def __init__(self, num_gpus=2):

        device_list = tt.hardware.get_cuda_devices()
        assert len(device_list) >= num_gpus, "Not enough GPU devices available."
        print("Launing Multi-GPU runtime...")
        print("Selecting devices: {}".format(device_list[0:num_gpus]))
        self._num_gpus = num_gpus

        super(MultiGpuRuntime, self).__init__()

    @tt.utils.attr.override
    def _build_internal(self, x, y_):
        # Variables that affect learning rate
        batch_size = self.datasets.train.batch_size
        num_batches_per_epoch = self.datasets.train.size / batch_size
        decay_steps = num_batches_per_epoch * NUM_EPOCHS_PER_DECAY

        if FIXED_NUM_STEPS_PER_DECAY is not None:
            decay_steps = FIXED_NUM_STEPS_PER_DECAY

        # Decay the learning rate exponentially based on the number of steps
        lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE,
                                        self._global_step,
                                        decay_steps,
                                        LEARNING_RATE_DECAY_FACTOR,
                                        staircase=True)
        tf.scalar_summary('learning_rate', lr)

        # Compute gradients
        opt = tf.train.AdamOptimizer(lr) # TODO: make optimizer parameterizable? or spcify in model?
        # Calculate the gradients for each model tower.
        tower_grads = []
        tower_total_losses = []
        tower_losses = []
        batch_size_per_gpu = (batch_size // self.num_gpus)
        for i in xrange(self.num_gpus):
            with tf.device('/gpu:%d' % i, ):
                with tf.name_scope('%s_%d' % ('tower', i)) as scope:
                    this_inputs = x[i*batch_size_per_gpu:(i+1)*batch_size_per_gpu, :, :, :, :]
                    this_targets = y_[i*batch_size_per_gpu:(i+1)*batch_size_per_gpu, :, :, :, :]

                    # Build inference Graph.This function constructs
                    # the entire model but shares the variables across all towers.
                    inference = self._model.inference(this_inputs, this_targets,
                                                      is_training=self._is_training,
                                                      device_scope=scope,
                                                      memory_device='/cpu:0')
                    self._inferences.append(inference)

                    loss = self._model.loss(inference, this_targets)
                    total_loss = self._model.total_loss(loss, inference, this_targets)

                    # Calculate the moving averages of the loss for one tower of the model
                    loss_averages_op = tt.board.loss_summary([total_loss, loss] +
                                                             tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) + # FIXME: redundant!??! Is the list empty???
                                                             tf.get_collection('intermediate_losses', scope),
                                                             decay=0.9)

                    with tf.control_dependencies([loss_averages_op]):
                        this_total_loss = tf.identity(total_loss)
                        this_loss = tf.identity(loss)

                    tower_total_losses.append(this_total_loss)
                    tower_losses.append(this_loss)

                    # Reuse variables for the next tower.
                    tf.get_variable_scope().reuse_variables()

                    # Retain the summaries from the final tower.
                    summaries = tf.get_collection(tf.GraphKeys.SUMMARIES, scope)

                    # Calculate the gradients for the batch of data on this tower.
                    grads = opt.compute_gradients(this_total_loss)

                    # Keep track of the gradients ackeep_probross all towers.
                    tower_grads.append(grads)

        # We must calculate the mean of each gradient.
        # This is also the synchronization point across all towers.
        grads = tt.training.average_gradients(tower_grads)

        total_loss = tf.reduce_mean(tower_total_losses)
        summaries.append(tf.scalar_summary('mean_total_loss', total_loss)) # TODO: really needed?

        loss = tf.reduce_mean(tower_losses)
        summaries.append(tf.scalar_summary('mean_loss', loss)) # TODO: really needed?

        # Add a summary to track the learning rate.
        summaries.append(tf.scalar_summary('learning_rate', lr))

        # Add histograms for gradients
        summaries.extend(tt.board.gradients_histogram_summary(grads))

        # Apply the gradients to adjust the shared variables and increment the step counter
        apply_gradient_op = opt.apply_gradients(grads, global_step=self._global_step)

        summaries.extend(tt.board.variables_histogram_summary())

        # Track the moving averages of all trainable variables
        variable_averages = tf.train.ExponentialMovingAverage(0.9999, self._global_step)
        variables_averages_op = variable_averages.apply(tf.trainable_variables())

        # Group all updates to into a single train op.
        train_op = tf.group(apply_gradient_op, variables_averages_op, name="train_op")
        return train_op, total_loss, loss, summaries

    def start_training(self):
        #with self.graph.as_default(), tf.device('/cpu:0'):
        with tf.device('/cpu:0'):
            return super(MultiGpuRuntime, self).start_training()

    @property
    def num_gpus(self):
        return self._num_gpus
            <|endoftext|>


<commit_before>import os.path
import shutil
import subprocess
from os.path import join
from sys import argv
from tempfile import mkdtemp

from plash.utils import hashstr, run

TMP_DIR = '/tmp'

class Container:

    def __init__(self, container_id):
        self._layer_ids = container_id.split(':')

    def get_layer_paths(self):
        lp = ["/var/lib/plash/builds/{}".format(self._layer_ids[0])]
        for ci in self._layer_ids[1:]:
            lp.append(lp[-1] + 'children/' + ci)
        return lp

    def invalidate(self):
        pass

    def mount_rootfs(self, *, mountpoint, write_dir=None, workdir=None):
        if write_dir == None:
            write_dir = mkdtemp(dir=TMP_DIR)
        if workdir == None:
            workdir = mkdtemp(dir=TMP_DIR)
        cmd = [
            'mount',
            '-t',
            'overlay',
            'overlay',
            '-o',
            'upperdir={write_dir},lowerdir={dirs},workdir={workdir}'.format(
                write_dir=write_dir,
                workdir=workdir,
                dirs=':'.join(join(p, 'payload') for p in self.get_layer_paths())),
            mountpoint]
        run(cmd)
        return mountpoint

    def prepare_chroot(self, mountpoint):
        run(['mount', '-t', 'proc', 'proc', join(mountpoint, 'proc')])
        run(['mount', '--bind', '/sys', join(mountpoint, 'sys')])
        run(['mount', '--bind', '/dev', join(mountpoint, 'dev')])
        run(['mount', '--bind', '/dev/pts', join(mountpoint, 'dev', 'pts')])
        run(['mount', '--bind', '/dev/shm', join(mountpoint, 'dev', 'shm')])
        run(['mount', '--bind', '/tmp', join(mountpoint, 'tmp')])

    def build_layer(self, cmd):
        print('*** plash: building layer')
        new_child = mkdtemp(dir=TMP_DIR)
        new_layer = join(new_child, 'payload')
        os.mkdir(new_layer)
        os.mkdir(join(new_child, 'children'))

        self.mount_rootfs(mountpoint=new_layer)

        self.prepare_chroot(new_layer)

        if os.fork():
            os.chroot(new_layer)
            asdf
            shell = 'sh'
            os.execvpe(shell, [shell, '-ce', cmd], os.environ) # maybe isolate envs better?
        exit = os.wait()
        assert False, exit
        run("umount", "--recursive", new_layer)
        assert exit == 0, 'Building returned non zero exit status'
        if exit != 0:
            from time import sleep
            sleep(3)
            print('non zero exit status code when building')
            shutil.rmtree(new_child) # cleanup

        last_layer = self.get_layer_paths()[-1]
        layer_hash = hashstr(cmd.encode())
        final_child_dst = join(last_layer, 'children', layer_hash)
        try:
            os.rename(new_child, final_child_dst)
        except OSError as exc:
            if exc.errno == errno.ENOTEMPTY:
                print('*** plash: this layer already exists builded and will not be replaced (layer: {})'.format(layer_hash))
            else:
                raise

        return self._layer_ids.append(layer_hash)

    def is_builded(self, cmd):
        layer_hash = hashstr(cmd).encode()
        last_layer = self.get_layer_paths()[-1]
        final_child_dst = join(last_layer, 'children', layer_hash)
        return os.path.exists(final_child_dst)

    def add_layer(self, cmd):
        if not self.is_builded(cmd):
            return self.build_layer(cmd)
        else:
            print('*** plash: cached layer')

    def create_runnable(self, file, layers_cmd, executable, *, verbose_flag=False):
        for cmd in layers_cmd:
            self.build_layer(cmd)

        mp = mkdtemp(dir=TMP_DIR)
        self.mount_rootfs(mountpoint=mp)

        assert False, mp # create runnable from here

c = Container('zesty')
c.create_runnable('/tmp/mytest', ['touch a'], "python3")
<commit_msg>core rewrite lg<commit_after>import errno
import os.path
import shutil
import subprocess
from os.path import join
from sys import argv
from tempfile import mkdtemp

from plash.utils import hashstr, run

TMP_DIR = '/tmp'

class Container:

    def __init__(self, container_id):
        self._layer_ids = container_id.split(':')

    def hash_cmd(self, cmd):
        return hashstr(cmd)[:12]

    def get_layer_paths(self):
        lp = ["/var/lib/plash/builds/{}".format(self._layer_ids[0])]
        for ci in self._layer_ids[1:]:
            lp.append(lp[-1] + '/children/' + ci)
        # print(lp)
        return lp

    def invalidate(self):
        pass

    def mount_rootfs(self, *, mountpoint, write_dir=None, workdir=None):
        if write_dir == None:
            write_dir = mkdtemp(dir=TMP_DIR)
        if workdir == None:
            workdir = mkdtemp(dir=TMP_DIR)
        cmd = [
            'mount',
            '-t',
            'overlay',
            'overlay',
            '-o',
            'upperdir={write_dir},lowerdir={dirs},workdir={workdir}'.format(
                write_dir=write_dir,
                workdir=workdir,
                dirs=':'.join(join(p, 'payload') for p in self.get_layer_paths())),
            mountpoint]
        run(cmd)
        return mountpoint

    def prepare_chroot(self, mountpoint):
        run(['mount', '-t', 'proc', 'proc', join(mountpoint, 'proc')])
        run(['mount', '--bind', '/sys', join(mountpoint, 'sys')])
        run(['mount', '--bind', '/dev', join(mountpoint, 'dev')])
        run(['mount', '--bind', '/dev/pts', join(mountpoint, 'dev', 'pts')])
        run(['mount', '--bind', '/dev/shm', join(mountpoint, 'dev', 'shm')])
        run(['mount', '--bind', '/tmp', join(mountpoint, 'tmp')])

    def build_layer(self, cmd):
        print('*** plash: building layer')
        new_child = mkdtemp(dir=TMP_DIR)
        new_layer = join(new_child, 'payload')
        os.mkdir(new_layer)
        os.mkdir(join(new_child, 'children'))

        self.mount_rootfs(mountpoint=new_layer)

        self.prepare_chroot(new_layer)

        if not os.fork():
            os.chroot(new_layer)
            # from time import sleep
            # sleep(5)
            shell = 'sh'
            os.execvpe(shell, [shell, '-ce', cmd], os.environ) # maybe isolate envs better?
        child_pid, child_exit = os.wait()

        run(["umount", "--recursive", new_layer])

        if child_exit != 0:
            print("*** plash: build failed with exit status: {}".format(child_exit))
            shutil.rmtree(new_child)
            sys.exit(1)

        last_layer = self.get_layer_paths()[-1]
        layer_hash = self.hash_cmd(cmd.encode())
        final_child_dst = join(last_layer, 'children', layer_hash)
        try:
            os.rename(new_child, final_child_dst)
        except OSError as exc:
            if exc.errno == errno.ENOTEMPTY:
                print('*** plash: this layer already exists builded and will not be replaced (layer: {})'.format(layer_hash))
            else:
                raise

        return self._layer_ids.append(layer_hash)

    def is_builded(self, cmd):
        layer_hash = self.hash_cmd(cmd.encode())
        last_layer = self.get_layer_paths()[-1]
        final_child_dst = join(last_layer, 'children', layer_hash)
        return os.path.exists(final_child_dst)

    def add_layer(self, cmd):
        if not self.is_builded(cmd):
            return self.build_layer(cmd)
        else:
            print('*** plash: cached layer')

    def create_runnable(self, file, layers_cmd, executable, *, verbose_flag=False):
        for cmd in layers_cmd:
            self.add_layer(cmd)

        mp = mkdtemp(dir=TMP_DIR)
        self.mount_rootfs(mountpoint=mp)

        assert False, mp # create runnable from here

c = Container('zesty')
c.create_runnable('/tmp/mytest', ['touch a', 'touch b'], "python3")
<|endoftext|>


<commit_before>from django.core.management import setup_environ
import settings
import datetime
setup_environ(settings)
# Spenglr
from questions.utils import *
from education.utils import *
from challenge.utils import *
from profiles.utils import *
from network.utils import *

def cron():
    print datetime.now().strftime('%H:%M:%S') + ": Update SQ values..."
    batch_question_update_sq()

    batch_module_update_sq()
    batch_concept_update_sq()

    print datetime.now().strftime('%H:%M:%S') + ": Update user SQ values..."

    batch_usermodule_update_sq()
    batch_userconcept_update_sq()
    batch_userconcept_update_focus()

    batch_user_update_sq()
    batch_network_update_sq()

    print datetime.now().strftime('%H:%M:%S') + ": Generating challenges..."
    batch_generate_user_challenges()

    print datetime.now().strftime('%H:%M:%S') + ": Done."
cron()
<commit_msg>Cron messages<commit_after>from django.core.management import setup_environ
import settings
import datetime
setup_environ(settings)
# Spenglr
from questions.utils import *
from education.utils import *
from challenge.utils import *
from profiles.utils import *
from network.utils import *

def cron():
    print datetime.now().strftime('\t %H:%M:%S') + ": Update SQ values..."
    batch_question_update_sq()

    batch_module_update_sq()
    batch_concept_update_sq()

    print datetime.now().strftime('\t %H:%M:%S') + ": Update user SQ values..."

    batch_usermodule_update_sq()
    batch_userconcept_update_sq()
    batch_userconcept_update_focus()

    batch_user_update_sq()
    batch_network_update_sq()

    print datetime.now().strftime('\t %H:%M:%S') + ": Generating challenges..."
    batch_generate_user_challenges()

    print datetime.now().strftime('\t %H:%M:%S') + ": Done."
cron()
<|endoftext|>


<commit_before>'''
generic CRUD oparations for the gitlab's objects
'''

import http

class Crud():
	def __init__(self, path, keyField = 'name'):
		self.path = path
		self.cache = {}
		self.key_field = keyField

	'''
	get an object by system's name and id
	'''
	def by_id(self, sysNam, id):
		return http.get(sysNam, '%s/%d' % (self.path, id))

	'''
	add a new instance of an object
	'''
	def add(self, sysNam, data):
		return http.post(sysNam, self.path, data)

	'''
	delete an instcnce by id
	'''
	def delete(self, sysNam, id):
		return http.delete(sysNam, '%s/%d' % (self.path, id))

	'''
	create a cache of objects
	'''
	def mk_cache(self, sysNam):
		if sysNam not in self.cache or not self.cache[sysNam]:
			self.cache[sysNam] = dict(map(lambda e: (e[self.key_field], e), http.get(sysNam, self.path)))
		return self.cache[sysNam]

	'''
	clear cache for a given system
	'''
	def clr_cache(self, sysNam):
		self.cache[sysNam] = {}

	'''
	get an object by system's name and an object's name
	'''
	def by_name(self, sysNam, objNam):
		return self.mk_cache(sysNam)[objNam]
<commit_msg>replace key_field in the Crud class by key_func in order make more sophisticated key expressions possible<commit_after>'''
generic CRUD oparations for the gitlab's objects
'''

import http

class Crud():
	def __init__(self, path, keyFunc = lambda x: x['name']):
		self.path = path
		self.cache = {}
		self.key_func = keyFunc

	'''
	get an object by system's name and id
	'''
	def by_id(self, sysNam, id):
		return http.get(sysNam, '%s/%d' % (self.path, id))

	'''
	add a new instance of an object
	'''
	def add(self, sysNam, data):
		return http.post(sysNam, self.path, data)

	'''
	delete an instcnce by id
	'''
	def delete(self, sysNam, id):
		return http.delete(sysNam, '%s/%d' % (self.path, id))

	'''
	create a cache of objects
	'''
	def mk_cache(self, sysNam):
		if sysNam not in self.cache or not self.cache[sysNam]:
			self.cache[sysNam] = dict(map(lambda e: (self.key_func(e), e), http.get(sysNam, self.path)))
		return self.cache[sysNam]

	'''
	clear cache for a given system
	'''
	def clr_cache(self, sysNam):
		self.cache[sysNam] = {}

	'''
	get an object by system's name and an object's name
	'''
	def by_name(self, sysNam, objNam):
		return self.mk_cache(sysNam)[objNam]
<|endoftext|>


<commit_before>import json
import random
from pathlib import Path

import numpy as np
from util import read_img

def image_generator(video_dirs, n_samples, batch_size):
    # Get all paths
    x_all = []
    y_all = []
    for video_dir in video_dirs:
        imgs = sorted((video_dir / 'frames/').iterdir())
        label = json.load((video_dir / 'label.json').open())['label']
        x_all.extend(imgs)
        y_all.extend(label)

    indices = np.random.permutation(len(x_all))[:n_samples]
    x_use = [x_all[i] for i in indices]
    y_use = [y_all[i] for i in indices]

    x_batch = np.zeros((batch_size, 224, 224, 3), dtype=np.float32)
    y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

    while True:
        for i, img_path in enumerate(x_use):
            idx = i % batch_size
            x_batch[idx] = read_img(img_path)
            y_batch[idx] = y_use[i]

            if idx == batch_size - 1:
                yield (x_batch, y_batch)

def window_generator(video_dirs, n_samples, batch_size, timesteps):
    x_batch = np.zeros((batch_size, timesteps, 224, 224, 3), dtype=np.float32)
    y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

    idx = 0
    while True:
        for video_dir in video_dirs:
            xs = sorted((video_dir / 'frames/').iterdir())
            ys = json.load((video_dir / 'label.json').open())['label']

            for i, path in enumerate(xs):
                xs[i] = read_img(path)

            # [i, i + timesteps)
            windows = [(i, i + timesteps) for i in range(len(xs) - timesteps + 1)]
            random.shuffle(windows)

            for (s, e) in windows:
                for f in range(s, e):
                    x_batch[idx][f - s] = xs[f]
                y_batch[idx] = ys[e - 1]

                if idx == batch_size - 1:
                    yield x_batch, y_batch
                idx = (idx + 1) % batch_size

            xs = None

<commit_msg>Update at 2017-07-14 22-59-37<commit_after>import json
import random
from pathlib import Path

import numpy as np
from util import read_img

def image_generator(video_dirs, n_samples, batch_size):
    # Get all paths
    x_all = []
    y_all = []
    for video_dir in video_dirs:
        imgs = sorted((video_dir / 'frames/').iterdir())
        label = json.load((video_dir / 'label.json').open())['label']
        x_all.extend(imgs)
        y_all.extend(label)

    indices = np.random.permutation(len(x_all))[:n_samples]
    x_use = [x_all[i] for i in indices]
    y_use = [y_all[i] for i in indices]

    x_batch = np.zeros((batch_size, 224, 224, 3), dtype=np.float32)
    y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

    while True:
        for i, img_path in enumerate(x_use):
            idx = i % batch_size
            x_batch[idx] = read_img(img_path)
            y_batch[idx] = y_use[i]

            if idx == batch_size - 1:
                yield (x_batch, y_batch)

def window_generator(video_dirs, n_samples, batch_size, timesteps):
    x_batch = np.zeros((batch_size, timesteps, 224, 224, 3), dtype=np.float32)
    y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

    idx = 0
    while True:
        for video_dir in video_dirs:
            xs = np.load(str(video_dir / 'imgs.npy'))
            ys = json.load((video_dir / 'label.json').open())['label']

            # [i, i + timesteps)
            windows = [(i, i + timesteps) for i in range(len(xs) - timesteps + 1)]
            random.shuffle(windows)[:samples]

            for (s, e) in windows:
                for f in range(s, e):
                    x_batch[idx][f - s] = xs[f]
                y_batch[idx] = ys[e - 1]

                if idx == batch_size - 1:
                    yield x_batch, y_batch
                idx = (idx + 1) % batch_size

            xs = None

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Student Organisation for Aerospace Reasearch
University of Calgary
Canada

Developers:
    Nathan Meulenbroek

Description:

"""
import datetime

class Data():
    """
    Data from rocket.

    """
    data_file = ''
    altitude = []
    pressure = []
    acceleration = []
    ang_acceleration = []
    magnetic_field = []
    pitch = []
    yaw = []
    roll = []
    temperature = []

    def __init__(self):
        self.data_file = open('data_file.txt', 'w')

        self.data_file.write('# Data file initialised: %s' % datetime.datetime.now() + '\n#\n')

        self.data_file.write("""
# This data file is free to use for the general public. All data found in this
# file is raw data from the rocket. If there was no data available from the
# sensors when the rocket was attempting to send the data, the data received
# is repeated from the last measurement. The pitch, roll and yaw columns were
# all calculated on board the rocket from the magnetometer, gyroscope and
# accelerometer data.
#
# Sensors were calibrated at the time when the first data was received.
#
# Units:
#
                             """)
        return

    def update_all(self, data):
        """
        Updates all data at once

        Params:
            data - (string) data, separated with vertical lines
        """

        self.data_file.write('%s' % datetime.datetime.now())

        for item in data:
            self.data_file.write('{:30}'.format(item))

        self.data_file.write('\n')

        self.altitude.append(data[0])
        self.pressure.append(data[1])
        self.acceleration.append(data[2])
        self.ang_acceleration.append(data[3])
        self.magnetic_field.append(data[4])
        self.pitch.append(data[5])
        self.yaw.append(data[6])
        self.roll.append(data[7])
        self.temperature.append(data[8])

        return <commit_msg>Here's what we have for now, is this what you're looking for???<commit_after># -*- coding: utf-8 -*-
"""
Student Organisation for Aerospace Reasearch
University of Calgary
Canada

Developers:
    Nathan Meulenbroek

Description:

"""
import datetime
from tkinter import *
from tkinter.filedialog import askopenfilename

class Data():
    """
    Data from rocket.

    """
    data_file = ''
    altitude = []
    pressure = []
    acceleration = []
    ang_acceleration = []
    magnetic_field = []
    pitch = []
    yaw = []
    roll = []
    temperature = []

    def __init__(self):
        self.data_file = open('data_file.txt', 'w')

        self.data_file.write('# Data file initialised: %s' % datetime.datetime.now() + '\n#\n')

        self.data_file.write("""
# This data file is free to use for the general public. All data found in this
# file is raw data from the rocket. If there was no data available from the
# sensors when the rocket was attempting to send the data, the data received
# is repeated from the last measurement. The pitch, roll and yaw columns were
# all calculated on board the rocket from the magnetometer, gyroscope and
# accelerometer data.
#
# Sensors were calibrated at the time when the first data was received.
#
# Units:
#
                             """)
        return

    def update_all(self, data):
        """
        Updates all data at once

        Params:
            data - (string) data, separated with vertical lines
        """

        self.data_file.write('%s' % datetime.datetime.now())

        for item in data:
            self.data_file.write('{:30}'.format(item))

        self.data_file.write('\n')

        self.altitude.append(data[0])
        self.pressure.append(data[1])
        self.acceleration.append(data[2])
        self.ang_acceleration.append(data[3])
        self.magnetic_field.append(data[4])
        self.pitch.append(data[5])
        self.yaw.append(data[6])
        self.roll.append(data[7])
        self.temperature.append(data[8])

        return

#Preliminary file reading function???
#Not really clear on what we're doing
#Authors: John and Courtney
#Status: Confused
	def getData(self):
		root = Tk()
		root.filename =  askopenfilename(initialdir = "E:/Images",title = "choose your file",filetypes = (("Text Files","*.txt"),("all files","*.*")))
		fileName = root.filename
		root.withdraw()
		textFile = open(fileName, "r")
		data = textFile.read()
		return data

		<|endoftext|>


<commit_before>import discord, feedparser, datetime
from discord.ext import commands

now = datetime.datetime.now()
h = now.hour
description = "This is a WIP bot to replace newsreaders."
bot_prefix = "!"
client = commands.Bot(description=description, command_prefix=bot_prefix)

@client.event
async def on_ready():
    print("Login Successful")
    print("Name : {}" .format(client.user.name))
    print("ID : {}" .format(client.user.id))

@client.command(pass_context=True)
async def helpnews(ctx):
    print("!helpnews")
    await client.say("\nAvailible commands:\n"
                     "!helpnews -- What you're seeing now.\n"
                     "!news         -- See top news stories now.\n")

@client.command(pass_context=True)
async def news(ctx):
    print("!news")
    bbc = feedparser.parse("https://www.reddit.com/domain/bbc.com/"
                           "top/.rss")
    game = feedparser.parse("https://www.gameinformer.com/b/mainfeed"
                            ".aspx?Tags=feature")
    await client.say(bbc.entries[0]['link'])
    await client.say(game.entries[0]['link'])

#@client.event
#async def timer():
#    while True:
#        if(h == 9) or (h == 18) or (h == 0) or (h == 3):
#            bbc = feedparser.parse("https://www.reddit.com/domain/bbc.com/"
#                                   "top/.rss")
#            game = feedparser.parse("https://www.gameinformer.com/b/mainfeed"
#                                    ".aspx?Tags=feature")
#            await client.say(bbc.entries[0]['link'])
#            await client.say(game.entries[0]['link'])

client.run("")
<commit_msg>Update dave.py<commit_after>import discord, feedparser, datetime, praw,
from discord.ext import commands

now = datetime.datetime.now()
h = now.hour
description = "This is a WIP bot to work discord."
bot_prefix = "!"
client = commands.Bot(description=description, command_prefix=bot_prefix)

reddit = praw.Reddit('prequelbot',
                     user_agent='prequelbot:v{}:t3rr0r_f3rr3t'.format(__version__))
subreddit = reddit.subreddit("prequelmemes")
global a
a = 0
global topattr
topattr = {"title": "", "img": "", "id": ""}

def prawin():
    topsub = subreddit.top("day", limit=1)
    oldsub = {"title": "", "img": "", "id": ""}
    topattr = {"title": "", "img": "", "id": ""}
    global a
    if a == 0:
        for submission in topsub:
            oldsub["title"] = str(submission.title)
            oldsub["img"] = str(submission.url)
            oldsub["id"] = str(submission.id)
        logger("init, returning top now")
        print("init, returning top now")
        return oldsub

def discout():
    @client.event
    async def on_ready():
        print("Login Successful")
        print("Name : {}" .format(client.user.name))
        print("ID : {}" .format(client.user.id))

    @client.command(pass_context=True)
    async def help(ctx):
        print("!help")
        await client.say("\nAvailible commands:\n"
                         "!help -- What you're seeing now.\n"
                         "!news -- See top news stories now.\n"
                         "!prequel -- See top post from /r/prequelmemes.\n")

    @client.command(pass_context=True)
    async def news(ctx):
        print("!news")
        bbc = feedparser.parse("https://www.reddit.com/domain/bbc.com/"
                               "top/.rss")
        game = feedparser.parse("https://www.gameinformer.com/b/mainfeed"
                                ".aspx?Tags=feature")
        await client.say(bbc.entries[0]['link'])
        await client.say(game.entries[0]['link'])

    @client.command(pass_context=True)
    async def prequel(ctx):
        print("!prequel")
        # Always get top post at time.
        global a
        a = 0
        topbot = prawin()
        await client.say(" \nImage: {}\nTitle = {}\nComments = "
                         "https://redd.it/{}\n".format(
                         topbot["img"],topbot["title"],topbot["id"]))

    client.run("inserthere")

if __name__ == "__main__":
    logger("Main File; discout is being called.")
    print("Main File; discout is being called.")
    try:
        discout()
    except:
        logger("Error while calling/in main(); exiting.")
        print("Error while calling/in main(); exiting.")
        sys.exit()
<|endoftext|>


<commit_before>with open("day1.txt") as f:
  north = 0
  east = 0
  for line in f:
    moves = line.strip().split(',')
    print(moves)
<commit_msg>Day 1: First attempt<commit_after>turns = { 'N' : { 'L' : 'W', 'R' : 'E' }
        , 'S' : { 'L' : 'E', 'R' : 'W' }
        , 'E' : { 'L' : 'N', 'R' : 'S' }
        , 'W' : { 'L' : 'S', 'R' : 'N' }
        }

with open("day1.txt") as f:
  north = 0
  east = 0
  currDir = 'N'
  for line in f:
    moves = line.strip().split(',')
    for move in moves:
      move = move.strip()
      currDir = turns[currDir][move[0]]
      blocks = int(move[1])
      if currDir == 'N':
        north += blocks
      elif currDir == 'S':
        north -= blocks
      elif currDir == 'E':
        east += blocks
      elif currDir == 'W':
        east += blocks

  print(abs(north) + abs(east))
<|endoftext|>