<commit_before>"""
 Mine grid point extracted values for our good and the good of the IEM
 Use Unidata's motherlode server :)

"""
import sys
import network
table = network.Table( ['AWOS', 'IA_ASOS'] )
import psycopg2
MOS = psycopg2.connect(database='mos', host='iemdb')
mcursor = MOS.cursor()

import csv
import urllib2
import datetime
import pytz

BASE_URL = "http://thredds.ucar.edu/thredds/ncss/grib/NCEP/"
URLS = {
 'NAM' : "NAM/CONUS_12km/conduit/NAM_CONUS_12km_conduit_%Y%m%d_%H00.grib2/GC",
 'GFS' : "GFS/Global_0p5deg/GFS_Global_0p5deg_%Y%m%d_%H00.grib2/GC",
 'RAP' : "RAP/CONUS_13km/RR_CONUS_13km_%Y%m%d_%H00.grib2/GC",
}
VLOOKUP = {
 'sbcape': {'NAM': 'Convective_available_potential_energy_surface',
            'GFS': 'Convective_available_potential_energy_surface',
            'RAP': 'Convective_available_potential_energy_surface'},
 'sbcin': {'NAM': 'Convective_inhibition_surface',
           'GFS': 'Convective_inhibition_surface',
           'RAP': 'Convective_inhibition_surface'},
 'pwater': {'NAM': 'Precipitable_water_entire_atmosphere',
            'GFS': 'Precipitable_water_entire_atmosphere',
            'RAP': 'Precipitable_water_entire_atmosphere'},
 'precipcon': {'RAP':
               'Convective_precipitation_surface_Mixed_Intervals_Accumulation',
            'NAM': 'Convective_precipitation_surface_Mixed_Intervals_Accumulation',
            'GFS':
            'Convective_precipitation_surface_Mixed_Intervals_Accumulation',
           },
 'precipnon': {'RAP': None,
            'NAM': None,
            'GFS': None
           },
 'precip': {'RAP':
'Large-scale_precipitation_non-convective_surface_Mixed_Intervals_Accumulation',
            'NAM': 'Total_precipitation_surface_Mixed_Intervals_Accumulation',
            'GFS': 'Total_precipitation_surface_Mixed_Intervals_Accumulation',
           }
}

def run(model, station, lon, lat, ts):
    """
    Ingest the model data for a given Model, stationid and timestamp
    """

    vstring = ""
    for v in VLOOKUP:
        if VLOOKUP[v][model] is not None:
            vstring += "var=%s&" % (VLOOKUP[v][model],)

    url = ("%s%s?%slatitude=%s&longitude=%s&temporal=all&vertCoord="
           +"&accept=csv&point=true") % (BASE_URL, ts.strftime( URLS[model] ),
                                        vstring, lat, lon)
    try:
        fp = urllib2.urlopen( url , timeout=60)
    except Exception, exp:
        print exp, url
        print 'FAIL ts: %s station: %s model: %s' % (ts.strftime("%Y-%m-%d %H"),
                                                     station, model)
        return

    table = "model_gridpoint_%s" % (ts.year,)
    sql = """DELETE from """+ table +""" WHERE
        station = %s and
          model = %s and runtime = %s """
    args = (station, model, ts)
    mcursor.execute( sql, args )
    if mcursor.rowcount > 0:
        print 'Deleted %s rows for ts: %s station: %s model: %s' % (
                        mcursor.rowcount, ts, station, model)

    count = 0
    r = csv.DictReader( fp )
    for row in r:
        for k in row.keys():
            row[ k[:k.find("[")] ] = row[k]
        sbcape = row[ VLOOKUP['sbcape'][model] ]
        sbcin = row[ VLOOKUP['sbcin'][model] ]
        pwater = row[ VLOOKUP['pwater'][model] ]
        precipcon = row[ VLOOKUP['precipcon'][model] ]
        if model == "RAP22":
            precip = (float(row[ VLOOKUP['precipcon'][model] ]) +
                      float(row[ VLOOKUP['precipnon'][model] ]))
        else:
            precip = row[ VLOOKUP['precip'][model] ]
        if precip < 0:
            precip = None
        if precipcon < 0:
            precipcon = None
        fts = datetime.datetime.strptime(row['date'], '%Y-%m-%dT%H:%M:%SZ')
        fts = fts.replace(tzinfo=pytz.timezone("UTC"))
        sql = """INSERT into """+table+""" (station, model, runtime,
              ftime, sbcape, sbcin, pwater, precipcon, precip)
              VALUES (%s, %s , %s,
              %s, %s, %s, %s, %s, %s )"""
        args = ( station, model, ts, fts, sbcape, sbcin, pwater, precipcon,
                 precip)
        mcursor.execute( sql, args )
        count += 1
    return count

if __name__ == '__main__':
    ''' Go Go gadget '''
    gts = datetime.datetime.utcnow()
    if len(sys.argv) == 5:
        gts = datetime.datetime(int(sys.argv[1]), int(sys.argv[2]),
                                int(sys.argv[3]), int(sys.argv[4]))
    gts = gts.replace(tzinfo=pytz.timezone("UTC"), minute=0, second=0,
                      microsecond=0)

    if gts.hour % 6 == 0:
        ts = gts - datetime.timedelta(hours=6)
        for sid in table.sts.keys():
            cnt = run("GFS", "K"+sid, table.sts[sid]['lon'],
                      table.sts[sid]['lat'], ts)
            if cnt == 0:
                print 'No data', "K"+sid, ts, 'GFS'
            cnt = run("NAM", "K"+sid, table.sts[sid]['lon'],
                      table.sts[sid]['lat'], ts)
            if cnt == 0:
                print 'No data', "K"+sid, ts, 'GFS'
    for sid in table.sts.keys():
        ts = gts - datetime.timedelta(hours=2)
        run("RAP", "K"+sid, table.sts[sid]['lon'], table.sts[sid]['lat'], ts)
    mcursor.close()
    MOS.commit()
    MOS.close()
<commit_msg>add check for 7 day old data and rerun if lots is missing<commit_after>"""
 Mine grid point extracted values for our good and the good of the IEM
 Use Unidata's motherlode server :)

"""
import sys
import network
table = network.Table( ['AWOS', 'IA_ASOS'] )
import psycopg2
MOS = psycopg2.connect(database='mos', host='iemdb')
mcursor = MOS.cursor()

import csv
import urllib2
import datetime
import pytz

BASE_URL = "http://thredds.ucar.edu/thredds/ncss/grib/NCEP/"
URLS = {
 'NAM' : "NAM/CONUS_12km/conduit/NAM_CONUS_12km_conduit_%Y%m%d_%H00.grib2/GC",
 'GFS' : "GFS/Global_0p5deg/GFS_Global_0p5deg_%Y%m%d_%H00.grib2/GC",
 'RAP' : "RAP/CONUS_13km/RR_CONUS_13km_%Y%m%d_%H00.grib2/GC",
}
VLOOKUP = {
 'sbcape': {'NAM': 'Convective_available_potential_energy_surface',
            'GFS': 'Convective_available_potential_energy_surface',
            'RAP': 'Convective_available_potential_energy_surface'},
 'sbcin': {'NAM': 'Convective_inhibition_surface',
           'GFS': 'Convective_inhibition_surface',
           'RAP': 'Convective_inhibition_surface'},
 'pwater': {'NAM': 'Precipitable_water_entire_atmosphere',
            'GFS': 'Precipitable_water_entire_atmosphere',
            'RAP': 'Precipitable_water_entire_atmosphere'},
 'precipcon': {'RAP':
               'Convective_precipitation_surface_Mixed_Intervals_Accumulation',
            'NAM': 'Convective_precipitation_surface_Mixed_Intervals_Accumulation',
            'GFS':
            'Convective_precipitation_surface_Mixed_Intervals_Accumulation',
           },
 'precipnon': {'RAP': None,
            'NAM': None,
            'GFS': None
           },
 'precip': {'RAP':
'Large-scale_precipitation_non-convective_surface_Mixed_Intervals_Accumulation',
            'NAM': 'Total_precipitation_surface_Mixed_Intervals_Accumulation',
            'GFS': 'Total_precipitation_surface_Mixed_Intervals_Accumulation',
           }
}

def run(model, station, lon, lat, ts):
    """
    Ingest the model data for a given Model, stationid and timestamp
    """

    vstring = ""
    for v in VLOOKUP:
        if VLOOKUP[v][model] is not None:
            vstring += "var=%s&" % (VLOOKUP[v][model],)

    url = ("%s%s?%slatitude=%s&longitude=%s&temporal=all&vertCoord="
           +"&accept=csv&point=true") % (BASE_URL, ts.strftime( URLS[model] ),
                                        vstring, lat, lon)
    try:
        fp = urllib2.urlopen( url , timeout=60)
    except Exception, exp:
        print exp, url
        print 'FAIL ts: %s station: %s model: %s' % (ts.strftime("%Y-%m-%d %H"),
                                                     station, model)
        return

    table = "model_gridpoint_%s" % (ts.year,)
    sql = """DELETE from """+ table +""" WHERE
        station = %s and
          model = %s and runtime = %s """
    args = (station, model, ts)
    mcursor.execute( sql, args )
    if mcursor.rowcount > 0:
        print 'Deleted %s rows for ts: %s station: %s model: %s' % (
                        mcursor.rowcount, ts, station, model)

    count = 0
    r = csv.DictReader( fp )
    for row in r:
        for k in row.keys():
            row[ k[:k.find("[")] ] = row[k]
        sbcape = row[ VLOOKUP['sbcape'][model] ]
        sbcin = row[ VLOOKUP['sbcin'][model] ]
        pwater = row[ VLOOKUP['pwater'][model] ]
        precipcon = row[ VLOOKUP['precipcon'][model] ]
        if model == "RAP22":
            precip = (float(row[ VLOOKUP['precipcon'][model] ]) +
                      float(row[ VLOOKUP['precipnon'][model] ]))
        else:
            precip = row[ VLOOKUP['precip'][model] ]
        if precip < 0:
            precip = None
        if precipcon < 0:
            precipcon = None
        fts = datetime.datetime.strptime(row['date'], '%Y-%m-%dT%H:%M:%SZ')
        fts = fts.replace(tzinfo=pytz.timezone("UTC"))
        sql = """INSERT into """+table+""" (station, model, runtime,
              ftime, sbcape, sbcin, pwater, precipcon, precip)
              VALUES (%s, %s , %s,
              %s, %s, %s, %s, %s, %s )"""
        args = ( station, model, ts, fts, sbcape, sbcin, pwater, precipcon,
                 precip)
        mcursor.execute( sql, args )
        count += 1
    return count

def run_model(model, ts):
    ''' Actually do a model and timestamp '''
    for sid in table.sts.keys():
        cnt = run(model, "K"+sid, table.sts[sid]['lon'],
                  table.sts[sid]['lat'], ts)
        if cnt == 0:
            print 'No data', "K"+sid, ts, model

def check_and_run(model, ts):
    ''' Check the database for missing data '''
    table = "model_gridpoint_%s" % (ts.year,)
    mcursor.execute("""SELECT count(*) from """+table+""" WHERE
    runtime = %s and model = %s""", (ts, model))
    if mcursor.rowcount < 10:
        print 'Rerunning %s [runtime=%s] due to rowcount %s' % (model, ts,
                                                      mcursor.rowcount)
        run_model(model, ts)

if __name__ == '__main__':
    ''' Go Go gadget '''
    gts = datetime.datetime.utcnow()
    if len(sys.argv) == 5:
        gts = datetime.datetime(int(sys.argv[1]), int(sys.argv[2]),
                                int(sys.argv[3]), int(sys.argv[4]))
    gts = gts.replace(tzinfo=pytz.timezone("UTC"), minute=0, second=0,
                      microsecond=0)

    if gts.hour % 6 == 0:
        ts = gts - datetime.timedelta(hours=6)
        for model in ['GFS', 'NAM']:
            run_model(model, ts)
            check_and_run(model, ts - datetime.timedelta(days=7))

    ts = gts - datetime.timedelta(hours=2)
    run_model("RAP", ts)
    check_and_run('RAP', ts - datetime.timedelta(days=7))
    mcursor.close()
    MOS.commit()
    MOS.close()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Python 101 documentation build configuration file, created by
# sphinx-quickstart on Tue Sep 16 15:47:34 2014.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.doctest',
    'sphinx.ext.pngmath',
    'sphinx.ext.viewcode',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# the path to the latex executable
pngmath_latex = "/usr/bin/latex"
# the path to the dvipng executable
pngmath_dvipng = "/usr/bin/dvipng"

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Python 101'
copyright = u'2014, Centrum Edukacji Obywatelskiej'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.5'
# The full version, including alpha/beta/rc tags.
release = '0.5'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
show_authors = True

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'Python101doc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    ('index', 'Python101.tex', u'Python 101 Documentation',
     u'Centrum Edukacji Obywatelskiej', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'python101', u'Python 101 Documentation',
     [u'Centrum Edukacji Obywatelskiej'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    ('index', 'Python101', u'Python 101 Documentation',
     u'Centrum Edukacji Obywatelskiej', 'Python101', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

import sphinx

rst_prolog = u"""
.. raw:: html

    <!-- Domyślnie ładowane fonty nie zawierają Latin Ext -->
    <link href='http://fonts.googleapis.com/css?family=Lato:400,700,400italic,700italic|Roboto+Slab:400,700&subset=latin,latin-ext' rel='stylesheet' type='text/css'>

"""

rst_epilog = u"""
.. include:: /copyright.rst

.. raw:: html

    <style>
        div.code_no {{ text-align: right; background: #e3e3e3; padding: 6px 12px; }}
        div.highlight, div.highlight-python {{ margin-top: 0px; }}
        .rst-content code {{ color: #E74C3C; }}
    </style>

.. |date| date::
.. |time| date:: %H:%M
.. |sphinx_version| replace:: {}

:Utworzony: |date| o |time| w Sphinx |sphinx_version|
:Autorzy: :doc:`Patrz plik "Autorzy" </authors>`

""".format(sphinx.__version__)

try:
    import sphinx_rtd_theme
    html_theme = "sphinx_rtd_theme"
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
    html_style = 'css/python101.css'
except:
    pass

# def setup(app):
<commit_msg>Poprawka pngmath na imgmath w conf.py ze względu na Sphinksa 1.4.x<commit_after># -*- coding: utf-8 -*-
#
# Python 101 documentation build configuration file, created by
# sphinx-quickstart on Tue Sep 16 15:47:34 2014.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.doctest',
    'sphinx.ext.imgmath',
    'sphinx.ext.viewcode',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# the path to the latex executable
pngmath_latex = "/usr/bin/latex"
# the path to the dvipng executable
pngmath_dvipng = "/usr/bin/dvipng"

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Python 101'
copyright = u'2014, Centrum Edukacji Obywatelskiej'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.5'
# The full version, including alpha/beta/rc tags.
release = '0.5'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
show_authors = True

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'Python101doc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    ('index', 'Python101.tex', u'Python 101 Documentation',
     u'Centrum Edukacji Obywatelskiej', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'python101', u'Python 101 Documentation',
     [u'Centrum Edukacji Obywatelskiej'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    ('index', 'Python101', u'Python 101 Documentation',
     u'Centrum Edukacji Obywatelskiej', 'Python101', 'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

import sphinx

rst_prolog = u"""
.. raw:: html

    <!-- Domyślnie ładowane fonty nie zawierają Latin Ext -->
    <link href='http://fonts.googleapis.com/css?family=Lato:400,700,400italic,700italic|Roboto+Slab:400,700&subset=latin,latin-ext' rel='stylesheet' type='text/css'>

"""

rst_epilog = u"""
.. include:: /copyright.rst

.. raw:: html

    <style>
        div.code_no {{ text-align: right; background: #e3e3e3; padding: 6px 12px; }}
        div.highlight, div.highlight-python {{ margin-top: 0px; }}
        .rst-content code {{ color: #E74C3C; }}
    </style>

.. |date| date::
.. |time| date:: %H:%M
.. |sphinx_version| replace:: {}

:Utworzony: |date| o |time| w Sphinx |sphinx_version|
:Autorzy: :doc:`Patrz plik "Autorzy" </authors>`

""".format(sphinx.__version__)

try:
    import sphinx_rtd_theme
    html_theme = "sphinx_rtd_theme"
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
    html_style = 'css/python101.css'
except:
    pass

# def setup(app):
<|endoftext|>


<commit_before>import time
from typing import Tuple

def nagios_from_file(results_file: str) -> Tuple[int, str]:
    """Returns a nagios-appropriate string and return code obtained by
    parsing the desired file on disk. The file on disk should be of format

    %s|%s % (timestamp, nagios_string)

    This file is created by various nagios checking cron jobs such as
    check-rabbitmq-queues and check-rabbitmq-consumers"""

    with open(results_file) as f:
        data = f.read().strip()
    pieces = data.split('|')

    if not len(pieces) == 4:
        state = 'UNKNOWN'
        ret = 3
        data = "Results file malformed"
    else:
        timestamp = int(pieces[0])

        time_diff = time.time() - timestamp
        if time_diff > 60 * 2:
            ret = 3
            state = 'UNKNOWN'
            data = "Results file is stale"
        else:
            ret = int(pieces[1])
            state = pieces[2]
            data = pieces[3]

    return (ret, f"{state}: {data}")
<commit_msg>nagios: Don’t crash on missing cron file.<commit_after>import time
from typing import Tuple

def nagios_from_file(results_file: str) -> Tuple[int, str]:
    """Returns a nagios-appropriate string and return code obtained by
    parsing the desired file on disk. The file on disk should be of format

    %s|%s % (timestamp, nagios_string)

    This file is created by various nagios checking cron jobs such as
    check-rabbitmq-queues and check-rabbitmq-consumers"""

    try:
        with open(results_file) as f:
            data = f.read().strip()
    except FileNotFoundError:
        state = 'UNKNOWN'
        ret = 3
        data = "Results file is missing"
    else:
        pieces = data.split('|')

        if not len(pieces) == 4:
            state = 'UNKNOWN'
            ret = 3
            data = "Results file malformed"
        else:
            timestamp = int(pieces[0])

            time_diff = time.time() - timestamp
            if time_diff > 60 * 2:
                ret = 3
                state = 'UNKNOWN'
                data = "Results file is stale"
            else:
                ret = int(pieces[1])
                state = pieces[2]
                data = pieces[3]

    return (ret, f"{state}: {data}")
<|endoftext|>


<commit_before>"""
pghoard - compressor threads

Copyright (c) 2016 Ohmu Ltd
See LICENSE for details
"""
from pghoard import wal
from pghoard.common import write_json_file
from pghoard.rohmu.compressor import Compressor
from queue import Empty
from threading import Thread
import logging
import os

class CompressorThread(Thread, Compressor):
    def __init__(self, config, compression_queue, transfer_queue):
        super().__init__()
        self.log = logging.getLogger("Compressor")
        self.config = config
        self.state = {}
        self.compression_queue = compression_queue
        self.transfer_queue = transfer_queue
        self.running = True
        self.log.debug("Compressor initialized")

    def get_compressed_file_path(self, site, filetype, original_path):
        if filetype == "basebackup":
            rest, _ = os.path.split(original_path)
            rest, backupname = os.path.split(rest)
        else:
            backupname = os.path.basename(original_path)

        cfp = os.path.join(self.config["backup_location"], self.config["path_prefix"], site, filetype, backupname)
        self.log.debug("compressed_file_path for %r is %r", original_path, cfp)
        return cfp

    def find_site_for_file(self, filepath):
        # Formats like:
        # /home/foo/t/default/xlog/000000010000000000000014
        # /home/foo/t/default/basebackup/2015-02-06_3/base.tar
        if os.path.basename(filepath) == "base.tar":
            return filepath.split("/")[-4]
        return filepath.split("/")[-3]

    def compression_algorithm(self):
        return self.config["compression"]["algorithm"]

    def run(self):
        while self.running:
            try:
                event = self.compression_queue.get(timeout=1.0)
            except Empty:
                continue
            try:
                if event["type"] == "QUIT":
                    break
                if event["type"] == "DECOMPRESSION":
                    self.handle_decompression_event(event)
                else:
                    filetype = self.get_event_filetype(event)
                    if not filetype:
                        if "callback_queue" in event and event["callback_queue"]:
                            self.log.debug("Returning success for event: %r, even though we did nothing for it", event)
                            event["callback_queue"].put({"success": True, "opaque": event.get("opaque")})
                        continue
                    else:
                        self.handle_event(event, filetype)
            except Exception as ex:
                if "blob" in event:
                    log_event = dict(event, blob="<{} bytes>".format(len(event["blob"])))
                else:
                    log_event = event
                self.log.exception("Problem handling: %r: %s: %s",
                                   log_event, ex.__class__.__name__, ex)
                raise
        self.log.debug("Quitting Compressor")

    def get_event_filetype(self, event):
        if event["type"] == "CLOSE_WRITE" and os.path.basename(event["full_path"]) == "base.tar":
            return "basebackup"

        if event["type"] == "CLOSE_WRITE" or (event["type"] == "MOVE" and event["src_path"].endswith(".partial")):
            if wal.XLOG_RE.match(os.path.basename(event["full_path"])):
                return "xlog"
            if wal.TIMELINE_RE.match(os.path.basename(event["full_path"])):
                return "timeline"

        return None

    def handle_decompression_event(self, event):
        rsa_private_key = None
        if "metadata" in event and "encryption-key-id" in event["metadata"]:
            key_id = event["metadata"]["encryption-key-id"]
            rsa_private_key = self.config["backup_sites"][event["site"]]["encryption_keys"][key_id]["private"]

        self.decompress_to_filepath(event, rsa_private_key)
        if "callback_queue" in event:
            event["callback_queue"].put({"success": True, "opaque": event.get("opaque")})

    def handle_event(self, event, filetype):
        rsa_public_key = None
        site = event.get("site", self.find_site_for_file(event["full_path"]))
        encryption_key_id = self.config["backup_sites"][site]["encryption_key_id"]
        if encryption_key_id:
            rsa_public_key = self.config["backup_sites"][site]["encryption_keys"][encryption_key_id]["public"]

        if event.get("compress_to_memory", False):
            original_file_size, compressed_blob = self.compress_filepath_to_memory(
                filepath=event["full_path"],
                compression_algorithm=self.config["compression"]["algorithm"],
                compression_level=self.config["compression"]["level"],
                rsa_public_key=rsa_public_key)
            compressed_file_size = len(compressed_blob)
            compressed_filepath = None
        else:
            compressed_blob = None
            compressed_filepath = self.get_compressed_file_path(site, filetype, event["full_path"])
            original_file_size, compressed_file_size = self.compress_filepath(
                filepath=event["full_path"],
                compressed_filepath=compressed_filepath,
                compression_algorithm=self.config["compression"]["algorithm"],
                compression_level=self.config["compression"]["level"],
                rsa_public_key=rsa_public_key)

        if event.get("delete_file_after_compression", True):
            os.unlink(event["full_path"])

        metadata = event.get("metadata", {})
        metadata.update({
            "pg-version": self.config["backup_sites"][site].get("pg_version", 90500),
            "compression-algorithm": self.config["compression"]["algorithm"],
            "compression-level": self.config["compression"]["level"],
            "original-file-size": original_file_size,
        })
        if encryption_key_id:
            metadata.update({"encryption-key-id": encryption_key_id})
        if compressed_filepath:
            metadata_path = compressed_filepath + ".metadata"
            write_json_file(metadata_path, metadata)

        self.set_state_defaults_for_site(site)
        self.state[site][filetype]["original_data"] += original_file_size
        self.state[site][filetype]["compressed_data"] += compressed_file_size
        self.state[site][filetype]["count"] += 1
        transfer_object = {
            "callback_queue": event.get("callback_queue"),
            "file_size": compressed_file_size,
            "filetype": filetype,
            "metadata": metadata,
            "opaque": event.get("opaque"),
            "site": site,
            "type": "UPLOAD",
        }
        if event.get("compress_to_memory", False):
            transfer_object["blob"] = compressed_blob
            transfer_object["local_path"] = event["full_path"]
        else:
            transfer_object["local_path"] = compressed_filepath
        self.transfer_queue.put(transfer_object)
        return True

    def set_state_defaults_for_site(self, site):
        if site not in self.state:
            self.state[site] = {
                "basebackup": {"original_data": 0, "compressed_data": 0, "count": 0},
                "xlog": {"original_data": 0, "compressed_data": 0, "count": 0},
                "timeline": {"original_data": 0, "compressed_data": 0, "count": 0},
            }
<commit_msg>compressor: drop default pg version from metadata<commit_after>"""
pghoard - compressor threads

Copyright (c) 2016 Ohmu Ltd
See LICENSE for details
"""
from pghoard import wal
from pghoard.common import write_json_file
from pghoard.rohmu.compressor import Compressor
from queue import Empty
from threading import Thread
import logging
import os

class CompressorThread(Thread, Compressor):
    def __init__(self, config, compression_queue, transfer_queue):
        super().__init__()
        self.log = logging.getLogger("Compressor")
        self.config = config
        self.state = {}
        self.compression_queue = compression_queue
        self.transfer_queue = transfer_queue
        self.running = True
        self.log.debug("Compressor initialized")

    def get_compressed_file_path(self, site, filetype, original_path):
        if filetype == "basebackup":
            rest, _ = os.path.split(original_path)
            rest, backupname = os.path.split(rest)
        else:
            backupname = os.path.basename(original_path)

        cfp = os.path.join(self.config["backup_location"], self.config["path_prefix"], site, filetype, backupname)
        self.log.debug("compressed_file_path for %r is %r", original_path, cfp)
        return cfp

    def find_site_for_file(self, filepath):
        # Formats like:
        # /home/foo/t/default/xlog/000000010000000000000014
        # /home/foo/t/default/basebackup/2015-02-06_3/base.tar
        if os.path.basename(filepath) == "base.tar":
            return filepath.split("/")[-4]
        return filepath.split("/")[-3]

    def compression_algorithm(self):
        return self.config["compression"]["algorithm"]

    def run(self):
        while self.running:
            try:
                event = self.compression_queue.get(timeout=1.0)
            except Empty:
                continue
            try:
                if event["type"] == "QUIT":
                    break
                if event["type"] == "DECOMPRESSION":
                    self.handle_decompression_event(event)
                else:
                    filetype = self.get_event_filetype(event)
                    if not filetype:
                        if "callback_queue" in event and event["callback_queue"]:
                            self.log.debug("Returning success for event: %r, even though we did nothing for it", event)
                            event["callback_queue"].put({"success": True, "opaque": event.get("opaque")})
                        continue
                    else:
                        self.handle_event(event, filetype)
            except Exception as ex:
                if "blob" in event:
                    log_event = dict(event, blob="<{} bytes>".format(len(event["blob"])))
                else:
                    log_event = event
                self.log.exception("Problem handling: %r: %s: %s",
                                   log_event, ex.__class__.__name__, ex)
                raise
        self.log.debug("Quitting Compressor")

    def get_event_filetype(self, event):
        if event["type"] == "CLOSE_WRITE" and os.path.basename(event["full_path"]) == "base.tar":
            return "basebackup"

        if event["type"] == "CLOSE_WRITE" or (event["type"] == "MOVE" and event["src_path"].endswith(".partial")):
            if wal.XLOG_RE.match(os.path.basename(event["full_path"])):
                return "xlog"
            if wal.TIMELINE_RE.match(os.path.basename(event["full_path"])):
                return "timeline"

        return None

    def handle_decompression_event(self, event):
        rsa_private_key = None
        if "metadata" in event and "encryption-key-id" in event["metadata"]:
            key_id = event["metadata"]["encryption-key-id"]
            rsa_private_key = self.config["backup_sites"][event["site"]]["encryption_keys"][key_id]["private"]

        self.decompress_to_filepath(event, rsa_private_key)
        if "callback_queue" in event:
            event["callback_queue"].put({"success": True, "opaque": event.get("opaque")})

    def handle_event(self, event, filetype):
        rsa_public_key = None
        site = event.get("site", self.find_site_for_file(event["full_path"]))
        encryption_key_id = self.config["backup_sites"][site]["encryption_key_id"]
        if encryption_key_id:
            rsa_public_key = self.config["backup_sites"][site]["encryption_keys"][encryption_key_id]["public"]

        if event.get("compress_to_memory", False):
            original_file_size, compressed_blob = self.compress_filepath_to_memory(
                filepath=event["full_path"],
                compression_algorithm=self.config["compression"]["algorithm"],
                compression_level=self.config["compression"]["level"],
                rsa_public_key=rsa_public_key)
            compressed_file_size = len(compressed_blob)
            compressed_filepath = None
        else:
            compressed_blob = None
            compressed_filepath = self.get_compressed_file_path(site, filetype, event["full_path"])
            original_file_size, compressed_file_size = self.compress_filepath(
                filepath=event["full_path"],
                compressed_filepath=compressed_filepath,
                compression_algorithm=self.config["compression"]["algorithm"],
                compression_level=self.config["compression"]["level"],
                rsa_public_key=rsa_public_key)

        if event.get("delete_file_after_compression", True):
            os.unlink(event["full_path"])

        metadata = event.get("metadata", {})
        metadata.update({
            "pg-version": self.config["backup_sites"][site].get("pg_version"),
            "compression-algorithm": self.config["compression"]["algorithm"],
            "compression-level": self.config["compression"]["level"],
            "original-file-size": original_file_size,
        })
        if encryption_key_id:
            metadata.update({"encryption-key-id": encryption_key_id})
        if compressed_filepath:
            metadata_path = compressed_filepath + ".metadata"
            write_json_file(metadata_path, metadata)

        self.set_state_defaults_for_site(site)
        self.state[site][filetype]["original_data"] += original_file_size
        self.state[site][filetype]["compressed_data"] += compressed_file_size
        self.state[site][filetype]["count"] += 1
        transfer_object = {
            "callback_queue": event.get("callback_queue"),
            "file_size": compressed_file_size,
            "filetype": filetype,
            "metadata": metadata,
            "opaque": event.get("opaque"),
            "site": site,
            "type": "UPLOAD",
        }
        if event.get("compress_to_memory", False):
            transfer_object["blob"] = compressed_blob
            transfer_object["local_path"] = event["full_path"]
        else:
            transfer_object["local_path"] = compressed_filepath
        self.transfer_queue.put(transfer_object)
        return True

    def set_state_defaults_for_site(self, site):
        if site not in self.state:
            self.state[site] = {
                "basebackup": {"original_data": 0, "compressed_data": 0, "count": 0},
                "xlog": {"original_data": 0, "compressed_data": 0, "count": 0},
                "timeline": {"original_data": 0, "compressed_data": 0, "count": 0},
            }
<|endoftext|>


<commit_before>import time

source_suffix = ['.rst']

master_doc = 'index'

project = 'Cambion'
copyright = time.localtime().tm_year + 'Whitestone'
author = 'Whitestone'

highlight_language = 'csharp'<commit_msg>3: test<commit_after>import time

source_suffix = ['.rst']

master_doc = 'index'

project = 'Cambion'
copyright = time.localtime().tm_year, 'Whitestone'
author = 'Whitestone'

highlight_language = 'csharp'<|endoftext|>


<commit_before>__all__ = []
__version__ = '1.7.0'

def export(defn):
    """Decorator to explicitly mark functions that are exposed in a lib."""
    globals()[defn.__name__] = defn
    __all__.append(defn.__name__)
    return defn

from . import dbcommands
from . import iocommands
<commit_msg>Releasing version 1.8.0<commit_after>__all__ = []
__version__ = '1.8.0'

def export(defn):
    """Decorator to explicitly mark functions that are exposed in a lib."""
    globals()[defn.__name__] = defn
    __all__.append(defn.__name__)
    return defn

from . import dbcommands
from . import iocommands
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# pycortex documentation build configuration file, created by
# sphinx-quickstart on Thu Sep 13 19:15:55 2012.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('sphinxext'))
sys.path.insert(0, os.path.abspath('..'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc',
              'sphinx.ext.todo',
              'sphinx.ext.mathjax',
              'sphinx.ext.viewcode',
              'sphinx.ext.autosummary',
              'numpydoc',
              'sphinx.ext.githubpages',
              'sphinx_gallery.gen_gallery']

#autosummary_generate = True

# Sphinx-gallery
sphinx_gallery_conf = {
    # path to your examples scripts
    'examples_dirs' : '../examples',
    # path where to save gallery generated examples
    'gallery_dirs'  : 'auto_examples'}

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'pycortex'
copyright = u'2012, James Gao'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.1'
# The full version, including alpha/beta/rc tags.
release = '0.1'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
html_theme_options = {
    # 'logo': 'logo.png',
    'github_user': 'gallantlab',
    'github_repo': 'pycortex',
}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
html_sidebars = {
    '**': [
        'about.html',
        'navigation.html',
        'relations.html',
        'searchbox.html',
        'donate.html',
    ]
}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'pycortexdoc'

# -- Options for LaTeX output --------------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'pycortex.tex', u'pycortex Documentation',
   u'James Gao', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'pycortex', u'pycortex Documentation',
     [u'James Gao'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'pycortex', u'pycortex Documentation',
   u'James Gao', 'pycortex', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# -- Options for Epub output ---------------------------------------------------

# Bibliographic Dublin Core info.
epub_title = u'pycortex'
epub_author = u'James Gao'
epub_publisher = u'James Gao'
epub_copyright = u'2012, James Gao'

# The language of the text. It defaults to the language option
# or en if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
#epub_cover = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files shat should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
#epub_exclude_files = []

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

# Allow duplicate toc entries.
#epub_tocdup = True
<commit_msg>DOC: made sphinx-gallery run ALL py files, not just plot_\*<commit_after># -*- coding: utf-8 -*-
#
# pycortex documentation build configuration file, created by
# sphinx-quickstart on Thu Sep 13 19:15:55 2012.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('sphinxext'))
sys.path.insert(0, os.path.abspath('..'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc',
              'sphinx.ext.todo',
              'sphinx.ext.mathjax',
              'sphinx.ext.viewcode',
              'sphinx.ext.autosummary',
              'numpydoc',
              'sphinx.ext.githubpages',
              'sphinx_gallery.gen_gallery']

#autosummary_generate = True

# Sphinx-gallery
sphinx_gallery_conf = {
    # path to your examples scripts
    'examples_dirs' : '../examples',
    # path where to save gallery generated examples
    'gallery_dirs'  : 'auto_examples',
    # which files to execute? all of them
    'filename_pattern' : '/*py'}

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'pycortex'
copyright = u'2012, James Gao'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.1'
# The full version, including alpha/beta/rc tags.
release = '0.1'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
html_theme_options = {
    # 'logo': 'logo.png',
    'github_user': 'gallantlab',
    'github_repo': 'pycortex',
}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
html_sidebars = {
    '**': [
        'about.html',
        'navigation.html',
        'relations.html',
        'searchbox.html',
        'donate.html',
    ]
}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'pycortexdoc'

# -- Options for LaTeX output --------------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'pycortex.tex', u'pycortex Documentation',
   u'James Gao', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'pycortex', u'pycortex Documentation',
     [u'James Gao'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'pycortex', u'pycortex Documentation',
   u'James Gao', 'pycortex', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# -- Options for Epub output ---------------------------------------------------

# Bibliographic Dublin Core info.
epub_title = u'pycortex'
epub_author = u'James Gao'
epub_publisher = u'James Gao'
epub_copyright = u'2012, James Gao'

# The language of the text. It defaults to the language option
# or en if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
#epub_cover = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files shat should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
#epub_exclude_files = []

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

# Allow duplicate toc entries.
#epub_tocdup = True
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os
import sys
from unittest.mock import MagicMock

# on_rtd is whether we are on readthedocs.org
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'

if not on_rtd:  # only set the theme if we're building docs locally
    html_theme = 'sphinx_rtd_theme'

# Need to mock modules using MagicMock, or else they won't be able to
# be installed on readthedocs
class Mock(MagicMock):
    @classmethod
    def __getattr__(cls, name):
        return MagicMock()

MOCK_MODULES = [
    'matplotlib', 'matplotlib.pyplot', 'matplotlib.ticker', 'numpy', 'pandas',
    'pandas.util', 'scipy', 'scipy.constants', 'scipy.stats', 'scipy.optimize',
    'scipy.interpolate'
]
sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)

# -- General configuration -----------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
    'nbsphinx',
    'sphinx.ext.autodoc',
    'sphinx.ext.autosummary',
    'sphinx.ext.coverage',
    'sphinx.ext.doctest',
    'sphinx.ext.extlinks',
    'sphinx.ext.ifconfig',
    'sphinx.ext.napoleon',
    'sphinx.ext.todo',
    'sphinx.ext.viewcode',
    'sphinx.ext.mathjax',
]
# If currently spellchecking
if os.getenv('SPELLCHECK'):
    extensions += 'sphinxcontrib.spelling',
    spelling_show_suggestions = True
    spelling_lang = 'en_US'

# The suffix of source filenames.
source_suffix = '.rst'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'pygaps'
year = '2019'
author = 'Paul Iacomi'
copyright = '{0}, {1}'.format(year, author)
version = release = '2.0.2'

# Add any paths that contain templates here, relative to this directory.
templates_path = ['.']

# Needed for jupyter notebook compilation by nbsphinx
exclude_patterns = ['_build', '**.ipynb_checkpoints']

# Needed for jupyter notebook compilation by nbsphinx
pygments_style = 'trac'

# Suppressing the nonlocal_uri image warning, as it appears due to
# github badges being stored on another server
suppress_warnings = ['image.nonlocal_uri']

# External links
extlinks = {
    'issue': ('https://github.com/pauliacomi/pygaps/issues/%s', '#'),
    'pr': ('https://github.com/pauliacomi/pygaps/pull/%s', 'PR #'),
}

linkcheck_ignore = [
    r'https://github.com/pauliacomi/pygaps/compare/.+',
    r'https://requires.io/.+',
]

# Checking for internal links
nitpicky = True

# -- Options for HTML output ---------------------------------------------------

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = "logo.png"

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
html_favicon = 'favicon.ico'

html_last_updated_fmt = '%b %d, %Y'
html_split_index = False
html_sidebars = {
    '**': ['searchbox.html', 'globaltoc.html', 'sourcelink.html'],
}
html_short_title = '%s-%s' % (project, version)

# -- napoleon configuration -----------------------------------------------------

napoleon_use_ivar = True
napoleon_use_rtype = False
napoleon_use_param = False

# -- autodoc configuration -----------------------------------------------------

autodoc_member_order = 'bysource'
autodoc_mock_imports = ['_tkinter']
<commit_msg>linkcheck params<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os
import sys
from unittest.mock import MagicMock

# on_rtd is whether we are on readthedocs.org
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'

if not on_rtd:  # only set the theme if we're building docs locally
    html_theme = 'sphinx_rtd_theme'

# Need to mock modules using MagicMock, or else they won't be able to
# be installed on readthedocs
class Mock(MagicMock):
    @classmethod
    def __getattr__(cls, name):
        return MagicMock()

MOCK_MODULES = [
    'matplotlib', 'matplotlib.pyplot', 'matplotlib.ticker', 'numpy', 'pandas',
    'pandas.util', 'scipy', 'scipy.constants', 'scipy.stats', 'scipy.optimize',
    'scipy.interpolate'
]
sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)

# -- General configuration -----------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
    'nbsphinx',
    'sphinx.ext.autodoc',
    'sphinx.ext.autosummary',
    'sphinx.ext.coverage',
    'sphinx.ext.doctest',
    'sphinx.ext.extlinks',
    'sphinx.ext.ifconfig',
    'sphinx.ext.napoleon',
    'sphinx.ext.todo',
    'sphinx.ext.viewcode',
    'sphinx.ext.mathjax',
]
# If currently spellchecking
if os.getenv('SPELLCHECK'):
    extensions += 'sphinxcontrib.spelling',
    spelling_show_suggestions = True
    spelling_lang = 'en_US'

# The suffix of source filenames.
source_suffix = '.rst'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'pygaps'
year = '2019'
author = 'Paul Iacomi'
copyright = '{0}, {1}'.format(year, author)
version = release = '2.0.2'

# Add any paths that contain templates here, relative to this directory.
templates_path = ['.']

# Needed for jupyter notebook compilation by nbsphinx
exclude_patterns = ['_build', '**.ipynb_checkpoints']

# Needed for jupyter notebook compilation by nbsphinx
pygments_style = 'trac'

# Suppressing the nonlocal_uri image warning, as it appears due to
# github badges being stored on another server
suppress_warnings = ['image.nonlocal_uri']

# External links
extlinks = {
    'issue': ('https://github.com/pauliacomi/pygaps/issues/%s', '#'),
    'pr': ('https://github.com/pauliacomi/pygaps/pull/%s', 'PR #'),
}

linkcheck_timeout = 3
linkcheck_retries = 2
linkcheck_ignore = [
    r'https://github.com/pauliacomi/pygaps/compare/.+',
    r'https://requires.io/.+',
]

# Checking for internal links
nitpicky = True

# -- Options for HTML output ---------------------------------------------------

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = "logo.png"

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
html_favicon = 'favicon.ico'

html_last_updated_fmt = '%b %d, %Y'
html_split_index = False
html_sidebars = {
    '**': ['searchbox.html', 'globaltoc.html', 'sourcelink.html'],
}
html_short_title = '%s-%s' % (project, version)

# -- napoleon configuration -----------------------------------------------------

napoleon_use_ivar = True
napoleon_use_rtype = False
napoleon_use_param = False

# -- autodoc configuration -----------------------------------------------------

autodoc_member_order = 'bysource'
autodoc_mock_imports = ['_tkinter']
<|endoftext|>


<commit_before>#!/usr/bin/env python
# --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------

# This script is interactive as you need to log in to 'az'.

from __future__ import print_function

import os
import sys
import time
from datetime import datetime
from six import StringIO
from sh import az, ssh

script_env = {}

def add_script_env(name):
    script_env[name] = os.environ.get(name)

add_script_env('REPO_NAME')
add_script_env('CLI_VERSION')
add_script_env('CLI_DOWNLOAD_SHA256')
add_script_env('AZURE_STORAGE_CONNECTION_STRING')
add_script_env('DEBIAN_REPO_ID')
add_script_env('DEBIAN_REPO_URL')
add_script_env('DEBIAN_REPO_USERNAME')
add_script_env('DEBIAN_REPO_PASSWORD')

assert (all(script_env[n] != None for n in script_env)), "Not all required environment variables have been set.  {}".format(script_env)

REPO_UPLOAD_SCRIPT_TMPL = """
import os, requests
payload = {{'name': 'azure-cli', 'version': '{cli_version}-1', 'repositoryId': '{repo_id}', 'sourceUrl': '{source_url}'}}
r = requests.post('{repo_package_url}', verify=False, auth=('{repo_user}', '{repo_pass}'), json=payload)
print("Status Code")
print(r.status_code)
print("Query with a GET to the following:")
print(r.headers['Location'])
"""

def print_env_vars():
    for n in script_env:
        print('{} = {}'.format(n, script_env[n]))

def print_status(msg=''):
    print('-- '+msg)

def print_heading(heading):
    print('{0}\n{1}\n{0}'.format('=' * len(heading), heading))

def give_chance_to_cancel(msg_prefix=''):
    cancel_time_secs = 10
    msg_tmpl = '{}: Starting in {} seconds.'
    for i in range(cancel_time_secs, 0, -1):
        print_status(msg_tmpl.format(msg_prefix, i))
        time.sleep(1)

def main():
    print_env_vars()
    time_str = datetime.utcnow().strftime('%Y%m%d%H%M%S')
    az(["login"], _out=sys.stdout, _err=sys.stdout)
    resource_group = 'azurecli-release-debian-' + time_str
    vm_name = 'vm-debian-' + time_str
    print_status('Creating resource group.')
    az(['group', 'create', '-l', 'westus', '-n', resource_group], _out=sys.stdout, _err=sys.stdout)
    print_status('Creating VM.')
    az(['vm', 'create', '-g', resource_group, '-n', vm_name, '--generate-ssh-keys', '--authentication-type', 'ssh',
        '--image', 'Canonical:UbuntuServer:14.04.4-LTS:latest', '--admin-username', 'ubuntu'],
       _out=sys.stdout, _err=sys.stdout)
    io = StringIO()
    print_status('Getting VM IP address.')
    az(['vm', 'list-ip-addresses', '--resource-group', resource_group, '--name', vm_name,
        '--query', '[0].virtualMachine.network.publicIpAddresses[0].ipAddress'], _out=io)
    ip_address = io.getvalue().strip().replace('"', '')
    print_status('VM IP address is {}'.format(ip_address))
    io.close()
    vm_connect_str = "ubuntu@{}".format(ip_address)
    my_vm = ssh.bake(['-oStrictHostKeyChecking=no', vm_connect_str])
    print_status('Installing git.')
    my_vm(['sudo', 'apt-get', 'update', '&&', 'sudo', 'apt-get', 'install', '-y', 'git'],
          _out=sys.stdout, _err=sys.stdout)
    io = StringIO()
    my_vm(['mktemp', '-d'], _out=io)
    repo_dir = io.getvalue().strip()
    io.close()
    print_status('Cloning repo.')
    my_vm(['git', 'clone', 'https://github.com/{}'.format(script_env.get('REPO_NAME')), repo_dir], _out=sys.stdout, _err=sys.stdout)
    path_to_debian_build_script = os.path.join(repo_dir, 'packaged_releases', 'debian', 'debian_build.sh')
    path_to_dir_creator = os.path.join(repo_dir, 'packaged_releases', 'debian', 'debian_dir_creator.sh')
    io = StringIO()
    my_vm(['mktemp', '-d'], _out=io)
    build_artifact_dir = io.getvalue().strip()
    io.close()
    print_status('Running debian build scripts.')
    my_vm(['chmod', '+x', path_to_debian_build_script, path_to_dir_creator], _out=sys.stdout, _err=sys.stdout)
    my_vm(['export', 'CLI_VERSION={}'.format(script_env.get('CLI_VERSION')), '&&',
           'export', 'CLI_DOWNLOAD_SHA256={}'.format(script_env.get('CLI_DOWNLOAD_SHA256')), '&&',
           'export', 'BUILD_ARTIFACT_DIR={}'.format(build_artifact_dir), '&&',
           path_to_debian_build_script, path_to_dir_creator],
          _out=sys.stdout, _err=sys.stdout)
    print_status('Debian build complete.')
    io = StringIO()
    my_vm(['ls', build_artifact_dir], _out=io)
    artifact_name = io.getvalue().strip()
    io.close()
    deb_file_path = os.path.join(build_artifact_dir, artifact_name)
    print_status('Installing the .deb on the build machine')
    my_vm(['sudo', 'dpkg', '-i', deb_file_path], _out=sys.stdout, _err=sys.stdout)
    # Upload to Azure Storage
    print_status('Uploading .deb to Azure storage.')
    my_vm(['az', 'storage', 'container', 'create', '--name', 'repos', '--public-access', 'blob',
           '--connection-string', '"{}"'.format(script_env.get('AZURE_STORAGE_CONNECTION_STRING'))],
          _out=sys.stdout, _err=sys.stdout)
    my_vm(['az', 'storage', 'blob', 'upload', '-f', deb_file_path,
           '-n', artifact_name, '-c', 'repos', '--connection-string', '"{}"'.format(script_env.get('AZURE_STORAGE_CONNECTION_STRING'))],
          _out=sys.stdout, _err=sys.stdout)
    io = StringIO()
    my_vm(['az', 'storage', 'blob', 'url', '-n', artifact_name, '-c', 'repos', '--output', 'tsv',
           '--connection-string', '"{}"'.format(script_env.get('AZURE_STORAGE_CONNECTION_STRING'))], _out=io)
    deb_url = io.getvalue().strip()
    io.close()
    print_status('Debian file uploaded to the following URL.')
    print_status(deb_url)
    # Publish to apt service
    my_vm(['wget', '-q', 'https://bootstrap.pypa.io/get-pip.py'], _out=sys.stdout, _err=sys.stdout)
    my_vm(['sudo', 'python', 'get-pip.py'], _out=sys.stdout, _err=sys.stdout)
    my_vm(['sudo', 'pip', 'install', '--upgrade', 'requests'], _out=sys.stdout, _err=sys.stdout)
    upload_script = REPO_UPLOAD_SCRIPT_TMPL.format(cli_version=script_env.get('CLI_VERSION'),
                                                   repo_id=script_env.get('DEBIAN_REPO_ID'),
                                                   source_url=deb_url,
                                                   repo_package_url=script_env.get('DEBIAN_REPO_URL'),
                                                   repo_user=script_env.get('DEBIAN_REPO_USERNAME'),
                                                   repo_pass=script_env.get('DEBIAN_REPO_PASSWORD'))
    my_vm(['echo', '-e', '"{}"'.format(upload_script), '>>', 'repo_upload.py'], _out=sys.stdout, _err=sys.stdout)
    my_vm(['python', 'repo_upload.py'], _out=sys.stdout, _err=sys.stdout)
    print_status('Done. :)')
    give_chance_to_cancel('Delete resource group (in background)')
    az(['group', 'delete', '--name', resource_group, '--yes', '--no-wait'], _out=sys.stdout, _err=sys.stdout)
    print_status('Finished. :)')

if __name__ == '__main__':
    main()
<commit_msg>Fix deb release script - Single instead of double quotes (#4204)<commit_after>#!/usr/bin/env python
# --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------

# This script is interactive as you need to log in to 'az'.

from __future__ import print_function

import os
import sys
import time
from datetime import datetime
from six import StringIO
from sh import az, ssh

script_env = {}

def add_script_env(name):
    script_env[name] = os.environ.get(name)

add_script_env('REPO_NAME')
add_script_env('CLI_VERSION')
add_script_env('CLI_DOWNLOAD_SHA256')
add_script_env('AZURE_STORAGE_CONNECTION_STRING')
add_script_env('DEBIAN_REPO_ID')
add_script_env('DEBIAN_REPO_URL')
add_script_env('DEBIAN_REPO_USERNAME')
add_script_env('DEBIAN_REPO_PASSWORD')

assert (all(script_env[n] != None for n in script_env)), "Not all required environment variables have been set.  {}".format(script_env)

REPO_UPLOAD_SCRIPT_TMPL = """
import os, requests
payload = {{'name': 'azure-cli', 'version': '{cli_version}-1', 'repositoryId': '{repo_id}', 'sourceUrl': '{source_url}'}}
r = requests.post('{repo_package_url}', verify=False, auth=('{repo_user}', '{repo_pass}'), json=payload)
print('Status Code')
print(r.status_code)
print('Query with a GET to the following:')
print(r.headers['Location'])
"""

def print_env_vars():
    for n in script_env:
        print('{} = {}'.format(n, script_env[n]))

def print_status(msg=''):
    print('-- '+msg)

def print_heading(heading):
    print('{0}\n{1}\n{0}'.format('=' * len(heading), heading))

def give_chance_to_cancel(msg_prefix=''):
    cancel_time_secs = 10
    msg_tmpl = '{}: Starting in {} seconds.'
    for i in range(cancel_time_secs, 0, -1):
        print_status(msg_tmpl.format(msg_prefix, i))
        time.sleep(1)

def main():
    print_env_vars()
    time_str = datetime.utcnow().strftime('%Y%m%d%H%M%S')
    az(["login"], _out=sys.stdout, _err=sys.stdout)
    resource_group = 'azurecli-release-debian-' + time_str
    vm_name = 'vm-debian-' + time_str
    print_status('Creating resource group.')
    az(['group', 'create', '-l', 'westus', '-n', resource_group], _out=sys.stdout, _err=sys.stdout)
    print_status('Creating VM.')
    az(['vm', 'create', '-g', resource_group, '-n', vm_name, '--generate-ssh-keys', '--authentication-type', 'ssh',
        '--image', 'Canonical:UbuntuServer:14.04.4-LTS:latest', '--admin-username', 'ubuntu'],
       _out=sys.stdout, _err=sys.stdout)
    io = StringIO()
    print_status('Getting VM IP address.')
    az(['vm', 'list-ip-addresses', '--resource-group', resource_group, '--name', vm_name,
        '--query', '[0].virtualMachine.network.publicIpAddresses[0].ipAddress'], _out=io)
    ip_address = io.getvalue().strip().replace('"', '')
    print_status('VM IP address is {}'.format(ip_address))
    io.close()
    vm_connect_str = "ubuntu@{}".format(ip_address)
    my_vm = ssh.bake(['-oStrictHostKeyChecking=no', vm_connect_str])
    print_status('Installing git.')
    my_vm(['sudo', 'apt-get', 'update', '&&', 'sudo', 'apt-get', 'install', '-y', 'git'],
          _out=sys.stdout, _err=sys.stdout)
    io = StringIO()
    my_vm(['mktemp', '-d'], _out=io)
    repo_dir = io.getvalue().strip()
    io.close()
    print_status('Cloning repo.')
    my_vm(['git', 'clone', 'https://github.com/{}'.format(script_env.get('REPO_NAME')), repo_dir], _out=sys.stdout, _err=sys.stdout)
    path_to_debian_build_script = os.path.join(repo_dir, 'packaged_releases', 'debian', 'debian_build.sh')
    path_to_dir_creator = os.path.join(repo_dir, 'packaged_releases', 'debian', 'debian_dir_creator.sh')
    io = StringIO()
    my_vm(['mktemp', '-d'], _out=io)
    build_artifact_dir = io.getvalue().strip()
    io.close()
    print_status('Running debian build scripts.')
    my_vm(['chmod', '+x', path_to_debian_build_script, path_to_dir_creator], _out=sys.stdout, _err=sys.stdout)
    my_vm(['export', 'CLI_VERSION={}'.format(script_env.get('CLI_VERSION')), '&&',
           'export', 'CLI_DOWNLOAD_SHA256={}'.format(script_env.get('CLI_DOWNLOAD_SHA256')), '&&',
           'export', 'BUILD_ARTIFACT_DIR={}'.format(build_artifact_dir), '&&',
           path_to_debian_build_script, path_to_dir_creator],
          _out=sys.stdout, _err=sys.stdout)
    print_status('Debian build complete.')
    io = StringIO()
    my_vm(['ls', build_artifact_dir], _out=io)
    artifact_name = io.getvalue().strip()
    io.close()
    deb_file_path = os.path.join(build_artifact_dir, artifact_name)
    print_status('Installing the .deb on the build machine')
    my_vm(['sudo', 'dpkg', '-i', deb_file_path], _out=sys.stdout, _err=sys.stdout)
    # Upload to Azure Storage
    print_status('Uploading .deb to Azure storage.')
    my_vm(['az', 'storage', 'container', 'create', '--name', 'repos', '--public-access', 'blob',
           '--connection-string', '"{}"'.format(script_env.get('AZURE_STORAGE_CONNECTION_STRING'))],
          _out=sys.stdout, _err=sys.stdout)
    my_vm(['az', 'storage', 'blob', 'upload', '-f', deb_file_path,
           '-n', artifact_name, '-c', 'repos', '--connection-string', '"{}"'.format(script_env.get('AZURE_STORAGE_CONNECTION_STRING'))],
          _out=sys.stdout, _err=sys.stdout)
    io = StringIO()
    my_vm(['az', 'storage', 'blob', 'url', '-n', artifact_name, '-c', 'repos', '--output', 'tsv',
           '--connection-string', '"{}"'.format(script_env.get('AZURE_STORAGE_CONNECTION_STRING'))], _out=io)
    deb_url = io.getvalue().strip()
    io.close()
    print_status('Debian file uploaded to the following URL.')
    print_status(deb_url)
    # Publish to apt service
    my_vm(['wget', '-q', 'https://bootstrap.pypa.io/get-pip.py'], _out=sys.stdout, _err=sys.stdout)
    my_vm(['sudo', 'python', 'get-pip.py'], _out=sys.stdout, _err=sys.stdout)
    my_vm(['sudo', 'pip', 'install', '--upgrade', 'requests'], _out=sys.stdout, _err=sys.stdout)
    upload_script = REPO_UPLOAD_SCRIPT_TMPL.format(cli_version=script_env.get('CLI_VERSION'),
                                                   repo_id=script_env.get('DEBIAN_REPO_ID'),
                                                   source_url=deb_url,
                                                   repo_package_url=script_env.get('DEBIAN_REPO_URL'),
                                                   repo_user=script_env.get('DEBIAN_REPO_USERNAME'),
                                                   repo_pass=script_env.get('DEBIAN_REPO_PASSWORD'))
    my_vm(['echo', '-e', '"{}"'.format(upload_script), '>>', 'repo_upload.py'], _out=sys.stdout, _err=sys.stdout)
    my_vm(['python', 'repo_upload.py'], _out=sys.stdout, _err=sys.stdout)
    print_status('Done. :)')
    give_chance_to_cancel('Delete resource group (in background)')
    az(['group', 'delete', '--name', resource_group, '--yes', '--no-wait'], _out=sys.stdout, _err=sys.stdout)
    print_status('Finished. :)')

if __name__ == '__main__':
    main()
<|endoftext|>