<commit_before># -*- coding: utf-8 -*-
#
# tephi documentation build configuration file, created by
# sphinx-quickstart on Thu Jun 26 15:24:09 2014.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

from datetime import datetime
import os
import sys

import tephi

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.append(os.path.abspath('sphinxext'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc',
              'sphinx.ext.coverage',
              'sphinx.ext.doctest',
              'sphinx.ext.imgmath',
              'sphinx.ext.intersphinx',
              'matplotlib.sphinxext.mathmpl',
              'matplotlib.sphinxext.plot_directive',
              'custom_class_autodoc',
              'generate_package_rst',
              ]

# List of packages to document
autopackage_name = ['tephi']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'tephi'
copyright = f'2014-{datetime.now().year}, British Crown Copyright, Met Office'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = tephi.__version__
# The full version, including alpha/beta/rc tags.
release = tephi.__version__

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
# language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['sphinxext', 'build']

# The reST default role (used for this markup: `text`) to use for all documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'sphinx_rtd_theme'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = dict(logo_only=True)

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_domain_indices = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'tephidoc'

# -- Options for LaTeX output --------------------------------------------------

latex_elements = {}
# The paper size ('letterpaper' or 'a4paper').
# 'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
# 'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
# 'preamble': '',

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
    ('index', 'tephi.tex', u'tephi Documentation',
     copyright, 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False

# If true, show page references after internal links.
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'tephi', u'tephi Documentation',
     [copyright], 1)
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    ('index', 'tephi', u'tephi Documentation',
     copyright, 'tephi',
     'One line description of project.', 'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
# texinfo_appendices = []

# If false, no module index is generated.
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# texinfo_show_urls = 'footnote'

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {
    'python': ('http://docs.python.org/3.8', None),
    'numpy': ('http://docs.scipy.org/doc/numpy/', None),
    'matplotlib': ('http://matplotlib.sourceforge.net/', None),
}
<commit_msg>fix doc build for rtd builder environment (#33)<commit_after># -*- coding: utf-8 -*-
#
# tephi documentation build configuration file, created by
# sphinx-quickstart on Thu Jun 26 15:24:09 2014.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

from datetime import datetime
import os
import pathlib
import sys

# ensure tephi is discoverable by rtd build environment
root = pathlib.Path(__file__).absolute().parent.parent.parent.parent
os.environ["PYTHONPATH"] = str(root)
sys.path.insert(0, str(root))

import tephi

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.append(os.path.abspath('sphinxext'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc',
              'sphinx.ext.coverage',
              'sphinx.ext.doctest',
              'sphinx.ext.imgmath',
              'sphinx.ext.intersphinx',
              'matplotlib.sphinxext.mathmpl',
              'matplotlib.sphinxext.plot_directive',
              'custom_class_autodoc',
              'generate_package_rst',
              ]

# List of packages to document
autopackage_name = ['tephi']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'tephi'
copyright = f'2014-{datetime.now().year}, British Crown Copyright, Met Office'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = tephi.__version__
# The full version, including alpha/beta/rc tags.
release = tephi.__version__

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
# language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['sphinxext', 'build']

# The reST default role (used for this markup: `text`) to use for all documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'sphinx_rtd_theme'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = dict(logo_only=True)

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_domain_indices = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'tephidoc'

# -- Options for LaTeX output --------------------------------------------------

latex_elements = {}
# The paper size ('letterpaper' or 'a4paper').
# 'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
# 'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
# 'preamble': '',

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
    ('index', 'tephi.tex', u'tephi Documentation',
     copyright, 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False

# If true, show page references after internal links.
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'tephi', u'tephi Documentation',
     [copyright], 1)
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    ('index', 'tephi', u'tephi Documentation',
     copyright, 'tephi',
     'One line description of project.', 'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
# texinfo_appendices = []

# If false, no module index is generated.
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# texinfo_show_urls = 'footnote'

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {
    'python': ('http://docs.python.org/3.8', None),
    'numpy': ('http://docs.scipy.org/doc/numpy/', None),
    'matplotlib': ('http://matplotlib.sourceforge.net/', None),
}
<|endoftext|>


<commit_before>__author__ = "aemerick <emerick@astro.columbia.edu>"

# --- external ---
from collections import OrderedDict

# --- internal ---
from constants import CONST as const
import imf as imf

#
# --------- Superclass for all parameters -------
#
class _parameters:

    def __init__(self):
        pass

    def help(self):
        print self.__doc__

    def reset_parameters_to_default(self):
        self.__init__()

#
# ----------- Units ----------------------
#
class _units(_parameters):
    """
    Units:

        Set the conversions between code units and 'base'
        quantities. For the sake of the onezone code,
        the base unit of time is in seconds and the base unit
        of mass is in solar masses. Therefore, in order to
        set the time units to Myr (default) and mass units
        to solar masses (default), one would just do:

            >>> units.time = 3.1536E13
            >>> units.mass = 1.0

        Output masses are always in solar, while everything else (e.g.
        luminosity) is in cgs, regardless of what the code units are
        set to.
    """

    def __init__(self):
        self.time      = const.yr_to_s * 1.0E6
        self.mass      = 1.0

        return

units = _units()
#
# -------- Global Zone Parameters -------
#
class _zone_parameters(_parameters):
    """
    Zone Parameters:

        The below is a list of all parameters that are set
        by the zone, including default values. Parameters
        are listed in order of : 1) required, 2) not required but
        highly suggested, or 3) optional

        Required Parameters:

        initial_gas_mass (float) : initial gas mass in solar masses
        initial_dark_matter_mass (float) : DM mass of halo in solar
        initial_metallicity (float)      : initial gas metal fraction
        dt (float)                       : constant timestep size in code time
        t_final (float)                  : time to end simulation

        Suggested Parameters:

        imf (function) : IMF function used to generate stars. Currently
            function must be one of the imf functional forms defined
            in the ``imf'' module, but user can supply their own.
            Default imf.salpeter()

        star_formation_method (int) : switch between star formation
            schemes:

            1) constant, uniform SFR throughout evolution
            2) SFR computed cosmologically
            3) SFH table provided using SFH_filename parameter where
               either two columns are provided, time and SFR, or
               time and stellar mass. Column headers must be named
               appropriately as ("time" or "SFR" or "mass").

         use_SF_mass_reservoir (bool , optional) : One of two ways to deal with low
             SFR's to ensure accurate sampling of the IMF (see the second
             below). Make sure you understand these parameters and their
             implications before setting -- know also that they may need to
             be set in certain situations. This parameter turns on the
             reservoir method whereby M_sf = dt * SFR is added to a
             reservoir each timestep. If the reservoir exceeds the
             mass threshold ``SF_mass_reservoir_size'', SF occurs in that timestep
             using up all of the reservoir. This may lead to bursty, intermittent SF
             depnding on size of resivoir. Default = False

         use_stochastic_mass_sampling (bool, optional) : Second and preferred method to deal
             with low SFR's. Prevent normal SF if M_sf = dt * SFR is below some threshold
             value, ``stochastic_sample_mass''. Instead of shutting off SF that
             timestep completely, however, and instead of accumulating mass in a
             reseroir, compute the probablility that a chunk of gas of mass
             ``stochastic_sample_mass'' is made into stars that timestep as
             P = M_sf / stochastic_sample_mass . That chunk is then formed
             completely into stars using a random number draw. Default is True

         SF_mass_reservoir_size (float, optional) : Size of accumulation reservoir used with
         ``use_SF_mass_reservoir''. Default is 1000.0

         stochastic_sample_mass (float, optional) : Size of mass chunk allowed to form
             stochastically when SFR is low. Be careful setting this to too large
             a value. Not recommended to set below ~200.0 Msun depending
             on one's choice of maximum star particle mass. Default is 250.0.

         inflow_factor  (float, optional) : Sets the mass inflow rate as a function of
             the star formation rate. Default 0.05

         mass_loading_factor (float, optional) : Sets the mass outlflow rate as a function of
             the star formation rate. Default 0.1

         SFR_efficiency (float, optional) : For cosmologically derived SFR's, sets the
             star formation rate efficiency of converging gas to stars in a free fall time
             Default is 0.01

         Optional:

         t_o     (float, optional) : initial time. Default is 0.0
         t_final (float, optional) : simulation end time. Default is 10 Gyr

    """

    def __init__(self):
        self.initial_gas_mass         = 0.0
        self.initial_dark_matter_mass = 0.0
        self.initial_metallicity      = 0.0
        self.initial_stellar_mass     = 0.0
        self.species_to_track         = OrderedDict()
        self.initial_abundances       = None

        self.imf                      = imf.salpeter()
        self.star_formation_method    = 1           # 0, 1, 2, 3
        self.SFH_filename             = None
        self.constant_SFR             = 10.0        # code mass / code time

        self.cosmological_evolution   = False       # on or off

        self.use_SF_mass_reservoir     = False
        self.SF_mass_reservoir_size    = 1000.0

        self.use_stochastic_mass_sampling = True
        self.stochastic_sample_mass   = 250.0

        # - inflow, outflow, and efficiency parameters
        self.inflow_factor            = 0.05
        self.mass_loading_factor      = 0.1
        self.SFR_efficiency           = 0.01

        self.t_o                      = 0.0             # Myr
        self.t_final                  = 1.0E4           # Myr
        self.dt                       = 1.0             # Myr
        self.adaptive_timestep        = True
        self.timestep_safety_factor   = 4

        # assert time units here

zone = _zone_parameters()

#
# ----------------- Stars and Stellar Evolution ------------------
#
class _star_particle_parameters(_parameters):
    """
    Star and Stellar Physics Parameters:

        The below is a list of all parameters that are set to be
        used in evolving stars and controlling the underlying stellar
        physics properties.

        SNII_mass_threshold (float) : Lower mass limit for stars to
            explode as a Type II supernovae at the end of their life.
            Default is 8.0

        SNIa_candidate_mass_bounds (list or array of floats) : Size
            two (lower and upper bound) boundaries for mass range where
            stars turn into WD's that are candidates for exploding as
            Type 1a. Default [3.0, 8.0]

        DTD_slope (float) : Slope of the delay time distribution (DTD)
            model used to compute probability of SNIa candidates
            exploding as SNIa in a given timestep. Slope is beta,
            where probability goes as t^(-beta). Default 1.0

        NSNIa (float) : Fraction of SNIa candidates that will explode
            as Type Ia supernovae in a hubble time. Default 0.043

    """

    def __init__(self):

        self.SNII_mass_threshold           = 8.0
        self.SNIa_candidate_mass_bounds    = [3.0, 8.0]

        self.DTD_slope                     = 1.0
        self.NSNIa                         = 0.043

        self.use_AGB_wind_phase            = True
        self.AGB_wind_phase_mass_threshold = 8.0

        self.normalize_black_body_to_OSTAR = True
        self.black_body_correction_mass    = 20.0
        self.black_body_q0_factors         = const.black_body_q0
        self.black_body_q1_factors         = const.black_body_q1
        self.black_body_FUV_factors        = const.black_body_fuv

stars = _star_particle_parameters()
#
# ----------------- Input and Output --------------
#
class _io_parameters(_parameters):

    def __init__(self):
        self.dump_output_basename     = 'dump'
        self.dt_dump                  = 0.0
        self.cycle_dump               = 0

        self.summary_output_filename  = 'summary_output.txt'
        self.dt_summary               = 0.0
        self.cycle_summary            = 0

io  = _io_parameters()

#
# ------------------ Data Table ----------------------
#
class _data_table(_parameters):

    def __init__(self):

        self.yields_mass_limits = [1.0, 25.0]

data = _data_table()
#
# ------------- Helper Functions -------------
#

def information():
    """
    Welcome to the configuration parameters for the onezone
    chemical enrichment model. Parameters are classified by
    whether or not they belong to the more global onezone gas
    reservoir, the stars (and stellar physics) itself, or
    input/output. The parameters can be accessed and modified
    as attributes of the following objects:

        Zone            :    zone
        Stars           :    stars
        Input / Output  :    io

    More information about these parameters can be found by
    calling the help method on a given object, e.g.:
         zone.help()
    which will print an explanation about each parameter, whether or
    not it requires user to set (i.e. will fail if defaults are kept),
    and its default value.

    Call the 'help' function in this module to see a description of
    all parameters at once.
    """

    print information.__doc__

def help():
    """
    Print the docstrings for all config parameter types.
    """

    for obj in [zone, stars, io]:
        obj.help()

    return

def reset_all_parameters():
    """
    Reset all parameter types to their default values
    """

    for obj in [zone, stars, io]:
        obj.reset_parameters_to_default()

    print "ALL parameters reset to default"

    return
<commit_msg>change imf properties<commit_after>__author__ = "aemerick <emerick@astro.columbia.edu>"

# --- external ---
from collections import OrderedDict

# --- internal ---
from constants import CONST as const
import imf as imf

#
# --------- Superclass for all parameters -------
#
class _parameters:

    def __init__(self):
        pass

    def help(self):
        print self.__doc__

    def reset_parameters_to_default(self):
        self.__init__()

#
# ----------- Units ----------------------
#
class _units(_parameters):
    """
    Units:

        Set the conversions between code units and 'base'
        quantities. For the sake of the onezone code,
        the base unit of time is in seconds and the base unit
        of mass is in solar masses. Therefore, in order to
        set the time units to Myr (default) and mass units
        to solar masses (default), one would just do:

            >>> units.time = 3.1536E13
            >>> units.mass = 1.0

        Output masses are always in solar, while everything else (e.g.
        luminosity) is in cgs, regardless of what the code units are
        set to.
    """

    def __init__(self):
        self.time      = const.yr_to_s * 1.0E6
        self.mass      = 1.0

        return

units = _units()
#
# -------- Global Zone Parameters -------
#
class _zone_parameters(_parameters):
    """
    Zone Parameters:

        The below is a list of all parameters that are set
        by the zone, including default values. Parameters
        are listed in order of : 1) required, 2) not required but
        highly suggested, or 3) optional

        Required Parameters:

        initial_gas_mass (float) : initial gas mass in solar masses
        initial_dark_matter_mass (float) : DM mass of halo in solar
        initial_metallicity (float)      : initial gas metal fraction
        dt (float)                       : constant timestep size in code time
        t_final (float)                  : time to end simulation

        Suggested Parameters:

        imf (function) : IMF function used to generate stars. Currently
            function must be one of the imf functional forms defined
            in the ``imf'' module, but user can supply their own.
            Default imf.salpeter()

        star_formation_method (int) : switch between star formation
            schemes:

            1) constant, uniform SFR throughout evolution
            2) SFR computed cosmologically
            3) SFH table provided using SFH_filename parameter where
               either two columns are provided, time and SFR, or
               time and stellar mass. Column headers must be named
               appropriately as ("time" or "SFR" or "mass").

         use_SF_mass_reservoir (bool , optional) : One of two ways to deal with low
             SFR's to ensure accurate sampling of the IMF (see the second
             below). Make sure you understand these parameters and their
             implications before setting -- know also that they may need to
             be set in certain situations. This parameter turns on the
             reservoir method whereby M_sf = dt * SFR is added to a
             reservoir each timestep. If the reservoir exceeds the
             mass threshold ``SF_mass_reservoir_size'', SF occurs in that timestep
             using up all of the reservoir. This may lead to bursty, intermittent SF
             depnding on size of resivoir. Default = False

         use_stochastic_mass_sampling (bool, optional) : Second and preferred method to deal
             with low SFR's. Prevent normal SF if M_sf = dt * SFR is below some threshold
             value, ``stochastic_sample_mass''. Instead of shutting off SF that
             timestep completely, however, and instead of accumulating mass in a
             reseroir, compute the probablility that a chunk of gas of mass
             ``stochastic_sample_mass'' is made into stars that timestep as
             P = M_sf / stochastic_sample_mass . That chunk is then formed
             completely into stars using a random number draw. Default is True

         SF_mass_reservoir_size (float, optional) : Size of accumulation reservoir used with
         ``use_SF_mass_reservoir''. Default is 1000.0

         stochastic_sample_mass (float, optional) : Size of mass chunk allowed to form
             stochastically when SFR is low. Be careful setting this to too large
             a value. Not recommended to set below ~200.0 Msun depending
             on one's choice of maximum star particle mass. Default is 250.0.

         inflow_factor  (float, optional) : Sets the mass inflow rate as a function of
             the star formation rate. Default 0.05

         mass_loading_factor (float, optional) : Sets the mass outlflow rate as a function of
             the star formation rate. Default 0.1

         SFR_efficiency (float, optional) : For cosmologically derived SFR's, sets the
             star formation rate efficiency of converging gas to stars in a free fall time
             Default is 0.01

         Optional:

         t_o     (float, optional) : initial time. Default is 0.0
         t_final (float, optional) : simulation end time. Default is 10 Gyr

    """

    def __init__(self):
        self.initial_gas_mass         = 0.0
        self.initial_dark_matter_mass = 0.0
        self.initial_metallicity      = 0.0
        self.initial_stellar_mass     = 0.0
        self.species_to_track         = OrderedDict()
        self.initial_abundances       = None

        self.imf                      = imf.salpeter()
        self.M_min                   = self.imf.M_min
        self.M_max                   = self.imf.M_max
        self.alpha                   = self.imf.alpha

        self.star_formation_method    = 1           # 0, 1, 2, 3
        self.SFH_filename             = None
        self.constant_SFR             = 10.0        # code mass / code time

        self.cosmological_evolution   = False       # on or off

        self.use_SF_mass_reservoir     = False
        self.SF_mass_reservoir_size    = 1000.0

        self.use_stochastic_mass_sampling = True
        self.stochastic_sample_mass   = 250.0

        # - inflow, outflow, and efficiency parameters
        self.inflow_factor            = 0.05
        self.mass_loading_factor      = 0.1
        self.SFR_efficiency           = 0.01

        self.t_o                      = 0.0             # Myr
        self.t_final                  = 1.0E4           # Myr
        self.dt                       = 1.0             # Myr
        self.adaptive_timestep        = True
        self.timestep_safety_factor   = 4

        # assert time units here

    @property
    def M_min(self):
        return self.__M_min

    @property
    def M_max(self):
        return self.__M_max

    @property
    def alpha(self):
        return self.__alpha

    @M_min.setter
    def M_min(self, value):
        self.__M_min = value
        self.imf.M_min = self.__M_min
        return

    @M_max.setter
    def M_max(self, value):
        self.__M_max = value
        self.imf.M_max = self.__M_max
        return

    @alpha.setter
    def alpha(self, value):
        self.__alpha = value
        self.imf.alpha = self.__alpha
        return

zone = _zone_parameters()

#
# ----------------- Stars and Stellar Evolution ------------------
#
class _star_particle_parameters(_parameters):
    """
    Star and Stellar Physics Parameters:

        The below is a list of all parameters that are set to be
        used in evolving stars and controlling the underlying stellar
        physics properties.

        SNII_mass_threshold (float) : Lower mass limit for stars to
            explode as a Type II supernovae at the end of their life.
            Default is 8.0

        SNIa_candidate_mass_bounds (list or array of floats) : Size
            two (lower and upper bound) boundaries for mass range where
            stars turn into WD's that are candidates for exploding as
            Type 1a. Default [3.0, 8.0]

        DTD_slope (float) : Slope of the delay time distribution (DTD)
            model used to compute probability of SNIa candidates
            exploding as SNIa in a given timestep. Slope is beta,
            where probability goes as t^(-beta). Default 1.0

        NSNIa (float) : Fraction of SNIa candidates that will explode
            as Type Ia supernovae in a hubble time. Default 0.043

    """

    def __init__(self):

        self.SNII_mass_threshold           = 8.0
        self.SNIa_candidate_mass_bounds    = [3.0, 8.0]

        self.DTD_slope                     = 1.0
        self.NSNIa                         = 0.043

        self.use_AGB_wind_phase            = True
        self.AGB_wind_phase_mass_threshold = 8.0

        self.normalize_black_body_to_OSTAR = True
        self.black_body_correction_mass    = 20.0
        self.black_body_q0_factors         = const.black_body_q0
        self.black_body_q1_factors         = const.black_body_q1
        self.black_body_FUV_factors        = const.black_body_fuv

stars = _star_particle_parameters()
#
# ----------------- Input and Output --------------
#
class _io_parameters(_parameters):

    def __init__(self):
        self.dump_output_basename     = 'dump'
        self.dt_dump                  = 0.0
        self.cycle_dump               = 0

        self.summary_output_filename  = 'summary_output.txt'
        self.dt_summary               = 0.0
        self.cycle_summary            = 0

io  = _io_parameters()

#
# ------------------ Data Table ----------------------
#
class _data_table(_parameters):

    def __init__(self):

        self.yields_mass_limits = [1.0, 25.0]

data = _data_table()
#
# ------------- Helper Functions -------------
#

def information():
    """
    Welcome to the configuration parameters for the onezone
    chemical enrichment model. Parameters are classified by
    whether or not they belong to the more global onezone gas
    reservoir, the stars (and stellar physics) itself, or
    input/output. The parameters can be accessed and modified
    as attributes of the following objects:

        Zone            :    zone
        Stars           :    stars
        Input / Output  :    io

    More information about these parameters can be found by
    calling the help method on a given object, e.g.:
         zone.help()
    which will print an explanation about each parameter, whether or
    not it requires user to set (i.e. will fail if defaults are kept),
    and its default value.

    Call the 'help' function in this module to see a description of
    all parameters at once.
    """

    print information.__doc__

def help():
    """
    Print the docstrings for all config parameter types.
    """

    for obj in [zone, stars, io]:
        obj.help()

    return

def reset_all_parameters():
    """
    Reset all parameter types to their default values
    """

    for obj in [zone, stars, io]:
        obj.reset_parameters_to_default()

    print "ALL parameters reset to default"

    return
<|endoftext|>


<commit_before>"""
Persist messages to a database server, while also pushing to live view
on html page in browser Utilizing Websockets, Redis and Flask.

Best enjoyed with a big, cold glass of anything.
"""

import json
import sys
import os

from flask import Flask, redirect, render_template, request
from flask_socketio import SocketIO

from app.consumer import consumer
from app.helpers import get_redis_connection, broadcast, simple_logger, CreateOrder
from app.members import db, Order, prepare_demo_data

def config_app():
    """Create app and set configuration.
    """
    if not os.path.exists('data'):
        os.makedirs('data')
    app = Flask(__name__)
    app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///data/transact_data.db'
    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
    db.init_app(app)
    db.app = app
    app.db = db
    app.redis = get_redis_connection(decode_responses = True)
    app.socketio = SocketIO(app)
    return app

app = config_app()

# Orders API
@app.route('/api/v1/orders/', methods=['GET'])
def get_orders():
    """Get json of all orders"""
    all_orders = Order.query.all()
    as_dicts = [order.make_as_dict for order in all_orders]
    return json.dumps(as_dicts)

@app.route('/api/v1/orders/<order_id>', methods=['GET'])
def get_order(order_id):
    """Get json of single order by passing order_number"""
    try:
        order_id = int(order_id)
    except ValueError:
        return ('Error: order_id must be a number', 422)
    one_order = Order.query.filter(Order.order_id == order_id).first()
    if one_order is None:
        return ('No such record', 404)
    return one_order.make_as_json

@app.route('/api/v1/orders/', methods=['POST'])
def post_order(json_data):
    """Pass one or more orders as a json object"""
    handler = CreateOrder(app.redis)
    handler.perform(json_data)

@app.route('/new', methods=['POST'])
def receive_new_order():
    """Endpoind for ordering through web interface"""
    message = request.form.get('message', '')
    drink = {
        'g&t': 'Gin & Tonic',
        'espresso-martini': 'Espresso Martini',
        'negroni': 'Negroni',
        'beer': 'Beer',
        'other': request.form.get('other')
    }.get(request.form.get('drink'))
    if drink is None:
        return (
            'Unable to process Entry: Something\'s wrong with '
            'your order, perhaps you meant to select "Other".',
            422)
    broadcast(app.socketio, drink, message)
    order = Order(drink, message)
    json_data = order.make_as_json
    post_order(json_data)
    return ('Order created', 201)

@app.route('/', methods=['GET'])
def get_index_page():
    """Redirect to the orders page"""
    return redirect('/orders', code=302)

@app.route('/orders', methods=['GET'])
def get_orders_page():
    """Render the orders page"""
    all_orders = Order.query.all()
    return render_template('/orders/index.html', orders = all_orders)

# Live orders page
@app.route('/live', methods=['GET'])
def get_live_orders():
    """Render live view of orders page"""
    return render_template('/orders/live-orders.html')

# Page: New order form
@app.route('/new', methods=['GET'])
def get_new_order():
    """Render page for posting new orders"""
    return render_template('orders/new-order.html')

if __name__ == '__main__':
    db.create_all()
    logger = simple_logger('transact.log', 'input_log')
    try:
        # If 'dbwriter' is passed as an argument, launch
        # the consumer, else launch the main app.
        # At least one of each is required.
        if 'dbwriter' in sys.argv:
            if len(Order.query.all()) == 0:
                print('Preparing demo data...')
                json_data = prepare_demo_data()
                post_order(json_data)
            consumer(app.redis, db, Order)
        else:
            app.socketio.run(app, host='0.0.0.0')
    except KeyboardInterrupt:
        logger.info('Server shut down by user')
<commit_msg>Move db.create_all to dbwriter<commit_after>"""
Persist messages to a database server, while also pushing to live view
on html page in browser Utilizing Websockets, Redis and Flask.

Best enjoyed with a big, cold glass of anything.
"""

import json
import sys
import os

from flask import Flask, redirect, render_template, request
from flask_socketio import SocketIO

from app.consumer import consumer
from app.helpers import get_redis_connection, broadcast, simple_logger, CreateOrder
from app.members import db, Order, prepare_demo_data

def config_app():
    """Create app and set configuration.
    """
    if not os.path.exists('data'):
        os.makedirs('data')
    app = Flask(__name__)
    app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///data/transact_data.db'
    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
    db.init_app(app)
    db.app = app
    app.db = db
    app.redis = get_redis_connection(decode_responses = True)
    app.socketio = SocketIO(app)
    return app

app = config_app()

# Orders API
@app.route('/api/v1/orders/', methods=['GET'])
def get_orders():
    """Get json of all orders"""
    all_orders = Order.query.all()
    as_dicts = [order.make_as_dict for order in all_orders]
    return json.dumps(as_dicts)

@app.route('/api/v1/orders/<order_id>', methods=['GET'])
def get_order(order_id):
    """Get json of single order by passing order_number"""
    try:
        order_id = int(order_id)
    except ValueError:
        return ('Error: order_id must be a number', 422)
    one_order = Order.query.filter(Order.order_id == order_id).first()
    if one_order is None:
        return ('No such record', 404)
    return one_order.make_as_json

@app.route('/api/v1/orders/', methods=['POST'])
def post_order(json_data):
    """Pass one or more orders as a json object"""
    handler = CreateOrder(app.redis)
    handler.perform(json_data)

@app.route('/new', methods=['POST'])
def receive_new_order():
    """Endpoind for ordering through web interface"""
    message = request.form.get('message', '')
    drink = {
        'g&t': 'Gin & Tonic',
        'espresso-martini': 'Espresso Martini',
        'negroni': 'Negroni',
        'beer': 'Beer',
        'other': request.form.get('other')
    }.get(request.form.get('drink'))
    if drink is None:
        return (
            'Unable to process Entry: Something\'s wrong with '
            'your order, perhaps you meant to select "Other".',
            422)
    broadcast(app.socketio, drink, message)
    order = Order(drink, message)
    json_data = order.make_as_json
    post_order(json_data)
    return ('Order created', 201)

@app.route('/', methods=['GET'])
def get_index_page():
    """Redirect to the orders page"""
    return redirect('/orders', code=302)

@app.route('/orders', methods=['GET'])
def get_orders_page():
    """Render the orders page"""
    all_orders = Order.query.all()
    return render_template('/orders/index.html', orders = all_orders)

# Live orders page
@app.route('/live', methods=['GET'])
def get_live_orders():
    """Render live view of orders page"""
    return render_template('/orders/live-orders.html')

# Page: New order form
@app.route('/new', methods=['GET'])
def get_new_order():
    """Render page for posting new orders"""
    return render_template('orders/new-order.html')

if __name__ == '__main__':
    logger = simple_logger('transact.log', 'input_log')
    try:
        # If 'dbwriter' is passed as an argument, launch
        # the consumer, else launch the main app.
        # At least one of each is required.
        if 'dbwriter' in sys.argv:
            db.create_all()
            if len(Order.query.all()) == 0:
                print('Preparing demo data...')
                json_data = prepare_demo_data()
                post_order(json_data)
            consumer(app.redis, db, Order)
        else:
            app.socketio.run(app, host='0.0.0.0')
    except KeyboardInterrupt:
        logger.info('Server shut down by user')
<|endoftext|>


<commit_before>#!/usr/bin/python2.3
# -*- mode: python -*-
#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#

import branch
import commit
import cook
import cscmd
from local import database
import display
import helper
import importrpm
import log
import os
import queryrep
import repository
import rollbacks
import srscfg
import srcctl
import sys
import updatecmd
import util

if sys.version_info < (2, 3):
    print "error: python 2.3 or greater is requried"
    sys.exit(1)

sys.excepthook = util.excepthook

try:
    cfg = srscfg.SrsConfiguration()
except srscfg.ParseError, e:
    log.error(str(e))
    sys.exit(1)

def usage(rc = 1):
    print "usage: srs branch <newbranch> <branchfrom> [<trove>]"
    print "       srs changeset <pkg> [<oldver>] <newver> <outfile>"
    print "       srs cook    [--prep] [--macros file] <first.recipe> <second.recipe> ..."
    print "       srs commit       <changeset>"
    print "       srs erase        <pkgname> [<version>]"
    print "       srs localcs      <pkg> <outfile>"
    print "       srs localcommit  <changeset>"
    print "       srs pkglist      <pkgname> [<version>]"
    print "       srs remove       <path>"
    print "       srs replist      <pkgname> [<version>]"
    print "       srs rblist"
    print "       srs rollback     <rollback>"
    print "       srs source       [usage]"
    print "       srs update	   <pkgname> <version>"
    print "           update       <changeset>"
    print "       srs usage"
    print ""
    print "commit flags:  --target-branch <branch>"
    print ""
    print "common flags:  --branch <defaultbranch>"
    print '               --config "<item> <value>"'
    print "               --reppath <repository-path>"
    print "               --root <root>"
    print ""
    print "pkglist flags: --sha1s"
    print "               --ids"
    print "               --ls"
    print ""
    print "replist flags: --all"
    print "               --sha1s"
    print "               --ids"
    print "               --ls"
    print ""
    print "update flags: --replace-files"
    return rc

def openRepository(path):
    try:
        return helper.openRepository(path)
    except repository.repository.OpenError, e:
	log.error('Unable to open repository %s: %s', path, str(e))
	sys.exit(1)

def openDatabase(root, path):
    try:
        db = database.Database(root, path)
    except repository.repository.OpenError, e:
        log.error('Unable to open database %s%s%s: %s', root, os.sep, path, str(e))
        sys.exit(1)
    return db

def main():
    otherArgs = [ sys.argv[0] ]
    argSet = {}
    argDef = {}
    cfgMap = {}
    # 0 - arg may occur, no parameter
    # 1 - arg may occur once, w/ parameter
    # 2 - arg may occur N times, w/ parameter

    cfgMap["branch"] = "defaultbranch"
    cfgMap["reppath"] = "reppath"
    cfgMap["root"] = "root"

    argDef["all"] = 0
    argDef["config"] = 2
    argDef["debug"] = 0
    argDef["ids"] = 0
    argDef["ls"] = 0
    argDef["macros"] = 1
    argDef["prep"] = 0
    argDef["profile"] = 0
    argDef["replace-files"] = 0
    argDef["sha1s"] = 0
    argDef["target-branch"] = 1

    argDef.update(srcctl.argDef)

    for arg in cfgMap.keys():
	argDef[arg] = 1

    i = 1
    while i < len(sys.argv):
	if sys.argv[i][:2] != "--":
	    otherArgs.append(sys.argv[i])
	else:
	    arg = sys.argv[i][2:]
	    if not argDef.has_key(arg): return usage()

	    if not argDef[arg]:
		argSet[arg] = 1
	    else:
		# the argument takes a parameter
		i = i + 1
		if i >= len(sys.argv): return usage()

		if argDef[arg] == 1:
		    # exactly one parameter is allowd
		    if argSet.has_key(arg): return usage()
		    argSet[arg] = sys.argv[i]
		else:
		    # multiple parameters may occur
		    if argSet.has_key(arg):
			argSet[arg].append(sys.argv[i])
		    else:
			argSet[arg] = [sys.argv[i]]

	i = i + 1

    if '-v' in otherArgs:
	otherArgs.remove('-v')
	log.setVerbosity(1)
    else:
	log.setVerbosity(0)

    if argSet.has_key('debug'):
	del argSet['debug']
	import pdb
	pdb.set_trace()

    profile = False
    if argSet.has_key('profile'):
	import hotshot
	prof = hotshot.Profile('srs.prof')
	prof.start()
	profile = True
	del argSet['profile']

    if argSet.has_key('config'):
	for param in argSet['config']:
	    cfg.configLine(param)

	del argSet['config']

    for (arg, name) in cfgMap.items():
	if argSet.has_key(arg):
	    cfg.configLine("%s %s" % (name, argSet[arg]))
	    del argSet[arg]

    if (len(otherArgs) < 2):
	return usage()
    elif (otherArgs[1] == "branch"):
	if argSet: return usage
	if len(otherArgs) < 4 or len(otherArgs) > 5: return usage()
	repos = openRepository(cfg.reppath)

	args = [repos, ] + otherArgs[2:]
	branch.branch(*args)
    elif (otherArgs[1] == "changeset"):
	# current usage is "package file oldversion newversion"
	if len(otherArgs) != 5 and len(otherArgs) != 6:
	    return usage()

	name = otherArgs[2]
	if len(otherArgs) == 6:
	    (old, new) = (otherArgs[3], otherArgs[4])
	    outFile = otherArgs[5]
	else:
	    (old, new) = (None, otherArgs[3])
	    outFile = otherArgs[4]

	repos = openRepository(cfg.reppath)

	cscmd.ChangeSetCommand(repos, cfg, name, outFile, old, new)
    elif (otherArgs[1] == "commit"):
	targetBranch = None
	if argSet.has_key('target-branch'):
	    targetBranch  = argSet['target-branch']
	    del argSet['target-branch']
	if len(otherArgs) < 3: return usage()
	repos = openRepository(cfg.reppath)
	for changeSet in otherArgs[2:]:
	    commit.doCommit(repos, changeSet, targetBranch)
    elif (otherArgs[1] == "config"):
	if argSet: return usage
	if (len(otherArgs) > 2):
	    return usage()
	else:
	    cfg.display()
    elif (otherArgs[1] == "cook"):
	log.setVerbosity(1)
	macros = {}
	prep = 0
	if argSet.has_key('prep'):
	    del argSet['prep']
	    prep = 1
	if argSet.has_key('macros'):
	    argSet['macros']
	    f = open(argSet['macros'])
	    # XXX sick hack
	    macroSrc = "macros =" + f.read()
	    exec macroSrc
	    del f
	    del argSet['macros']
	if argSet: return usage()

	cook.cookCommand(cfg, otherArgs[2:], prep, macros)
    elif (otherArgs[1] == "erase"):
	if argSet: return usage
	if len(otherArgs) >= 3 and len(otherArgs) <=4:
	    db = openDatabase(cfg.root, cfg.dbpath)

	    args = [db, cfg] + otherArgs[2:]
	    updatecmd.doErase(*args)
	else:
	    return usage()
    elif (otherArgs[1] == "import"):
	if len(otherArgs) != 3 and len(otherArgs) != 3:
	    return usage()

	repos = openRepository(cfg.reppath)
	importrpm.doImport(repos, cfg, otherArgs[2])
    elif (otherArgs[1] == "localcs"):
	if len(otherArgs) != 4 and len(otherArgs) != 4:
	    return usage()

	name = otherArgs[2]
	outFile = otherArgs[3]

	db = database.Database(cfg.root, cfg.dbpath, "r")
	cscmd.LocalChangeSetCommand(db, cfg, name, outFile)
    elif (otherArgs[1] == "localcommit"):
	if len(otherArgs) < 3: return usage()
	db = database.Database(cfg.root, cfg.dbpath, "c")
	for changeSet in otherArgs[2:]:
	    commit.doLocalCommit(db, changeSet)
    elif (otherArgs[1] == "pkglist"):
	ls = argSet.has_key('ls')
	if ls: del argSet['ls']

	ids = argSet.has_key('ids')
	if ids: del argSet['ids']

	sha1s = argSet.has_key('sha1s')
	if sha1s: del argSet['sha1s']

	db = openDatabase(cfg.root, cfg.dbpath)

	if argSet: return usage()

	if len(otherArgs) >= 2 and len(otherArgs) <= 4:
	    args = [db, cfg, ls, ids, sha1s] + otherArgs[2:]
	    try:
		display.displayTroves(*args)
	    except IOError, msg:
		sys.stderr.write(msg.strerror + '\n')
		sys.exit(1)
	else:
	    return usage()
    elif (otherArgs[1] == "replist"):
	all = argSet.has_key('all')
	if all: del argSet['all']

	ls = argSet.has_key('ls')
	if ls: del argSet['ls']

	ids = argSet.has_key('ids')
	if ids: del argSet['ids']

	sha1s = argSet.has_key('sha1s')
	if sha1s: del argSet['sha1s']

	repos = openRepository(cfg.reppath)

	if argSet: return usage()

	if len(otherArgs) >= 2 and len(otherArgs) <= 4:
	    args = [repos, cfg, all, ls, ids, sha1s] + otherArgs[2:]
	    try:
		queryrep.displayTroves(*args)
	    except IOError, msg:
		sys.stderr.write(msg.strerror + '\n')
		sys.exit(1)
	else:
	    return usage()
    elif (otherArgs[1] == "rblist"):
	if argSet: return usage
	db = openDatabase(cfg.root, cfg.dbpath)
	rollbacks.listRollbacks(db, cfg)
    elif (otherArgs[1] == "remove"):
	if len(otherArgs) != 3: return usage()
	if argSet: return usage
	db = openDatabase(cfg.root, cfg.dbpath)
	fullPath = util.joinPaths(cfg.root, otherArgs[2])
	if os.path.exists(fullPath):
	    os.unlink(fullPath)
	else:
	    log.warning("%s has already been removed", fullPath)
	db.removeFile(otherArgs[2])
    elif (otherArgs[1] == "rollback"):
	if argSet: return usage
	db = openDatabase(cfg.root, cfg.dbpath)
	args = [db, cfg] + otherArgs[2:]
	rollbacks.apply(*args)
    elif (otherArgs[1] == "source" or otherArgs[1] == "src"):
	return srcctl.sourceCommand(cfg, otherArgs[2:], argSet)
    elif (otherArgs[1] == "update"):
	kwargs = {}
	replaceFiles = argSet.has_key('replace-files')
	if replaceFiles:
	    kwargs['replaceFiles'] = True
	    del argSet['replace-files']
	if argSet: return usage
	if len(otherArgs) >=3 and len(otherArgs) <= 4:
	    repos = openRepository(cfg.reppath)
	    db = openDatabase(cfg.root, cfg.dbpath)

	    args = [repos, db, cfg] + otherArgs[2:]
	    updatecmd.doUpdate(*args, **kwargs)
	else:
	    return usage()
    elif (otherArgs[1] == "return usage"):
	return usage(rc = 0)
    else:
	return usage()

    if profile:
	prof.stop()

    if log.errorOccurred():
	sys.exit(1)

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>add a place for final exception handling, give nice "permission denied" messages<commit_after>#!/usr/bin/python2.3
# -*- mode: python -*-
#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#

import branch
import commit
import cook
import cscmd
from local import database
import display
import helper
import importrpm
import log
import os
import queryrep
import repository
import rollbacks
import srscfg
import srcctl
import sys
import updatecmd
import util
import xmlrpclib

if sys.version_info < (2, 3):
    print "error: python 2.3 or greater is requried"
    sys.exit(1)

sys.excepthook = util.excepthook

try:
    cfg = srscfg.SrsConfiguration()
except srscfg.ParseError, e:
    log.error(str(e))
    sys.exit(1)

def usage(rc = 1):
    print "usage: srs branch <newbranch> <branchfrom> [<trove>]"
    print "       srs changeset <pkg> [<oldver>] <newver> <outfile>"
    print "       srs cook    [--prep] [--macros file] <first.recipe> <second.recipe> ..."
    print "       srs commit       <changeset>"
    print "       srs erase        <pkgname> [<version>]"
    print "       srs localcs      <pkg> <outfile>"
    print "       srs localcommit  <changeset>"
    print "       srs pkglist      <pkgname> [<version>]"
    print "       srs remove       <path>"
    print "       srs replist      <pkgname> [<version>]"
    print "       srs rblist"
    print "       srs rollback     <rollback>"
    print "       srs source       [usage]"
    print "       srs update	   <pkgname> <version>"
    print "           update       <changeset>"
    print "       srs usage"
    print ""
    print "commit flags:  --target-branch <branch>"
    print ""
    print "common flags:  --branch <defaultbranch>"
    print '               --config "<item> <value>"'
    print "               --reppath <repository-path>"
    print "               --root <root>"
    print ""
    print "pkglist flags: --sha1s"
    print "               --ids"
    print "               --ls"
    print ""
    print "replist flags: --all"
    print "               --sha1s"
    print "               --ids"
    print "               --ls"
    print ""
    print "update flags: --replace-files"
    return rc

def openRepository(path):
    try:
        return helper.openRepository(path)
    except repository.repository.OpenError, e:
	log.error('Unable to open repository %s: %s', path, str(e))
	sys.exit(1)

def openDatabase(root, path):
    try:
        db = database.Database(root, path)
    except repository.repository.OpenError, e:
        log.error('Unable to open database %s%s%s: %s', root, os.sep, path, str(e))
        sys.exit(1)
    return db

def realMain():
    otherArgs = [ sys.argv[0] ]
    argSet = {}
    argDef = {}
    cfgMap = {}
    # 0 - arg may occur, no parameter
    # 1 - arg may occur once, w/ parameter
    # 2 - arg may occur N times, w/ parameter

    cfgMap["branch"] = "defaultbranch"
    cfgMap["reppath"] = "reppath"
    cfgMap["root"] = "root"

    argDef["all"] = 0
    argDef["config"] = 2
    argDef["debug"] = 0
    argDef["ids"] = 0
    argDef["ls"] = 0
    argDef["macros"] = 1
    argDef["prep"] = 0
    argDef["profile"] = 0
    argDef["replace-files"] = 0
    argDef["sha1s"] = 0
    argDef["target-branch"] = 1

    argDef.update(srcctl.argDef)

    for arg in cfgMap.keys():
	argDef[arg] = 1

    i = 1
    while i < len(sys.argv):
	if sys.argv[i][:2] != "--":
	    otherArgs.append(sys.argv[i])
	else:
	    arg = sys.argv[i][2:]
	    if not argDef.has_key(arg): return usage()

	    if not argDef[arg]:
		argSet[arg] = 1
	    else:
		# the argument takes a parameter
		i = i + 1
		if i >= len(sys.argv): return usage()

		if argDef[arg] == 1:
		    # exactly one parameter is allowd
		    if argSet.has_key(arg): return usage()
		    argSet[arg] = sys.argv[i]
		else:
		    # multiple parameters may occur
		    if argSet.has_key(arg):
			argSet[arg].append(sys.argv[i])
		    else:
			argSet[arg] = [sys.argv[i]]

	i = i + 1

    if '-v' in otherArgs:
	otherArgs.remove('-v')
	log.setVerbosity(1)
    else:
	log.setVerbosity(0)

    if argSet.has_key('debug'):
	del argSet['debug']
	import pdb
	pdb.set_trace()

    profile = False
    if argSet.has_key('profile'):
	import hotshot
	prof = hotshot.Profile('srs.prof')
	prof.start()
	profile = True
	del argSet['profile']

    if argSet.has_key('config'):
	for param in argSet['config']:
	    cfg.configLine(param)

	del argSet['config']

    for (arg, name) in cfgMap.items():
	if argSet.has_key(arg):
	    cfg.configLine("%s %s" % (name, argSet[arg]))
	    del argSet[arg]

    if (len(otherArgs) < 2):
	return usage()
    elif (otherArgs[1] == "branch"):
	if argSet: return usage
	if len(otherArgs) < 4 or len(otherArgs) > 5: return usage()
	repos = openRepository(cfg.reppath)

	args = [repos, ] + otherArgs[2:]
	branch.branch(*args)
    elif (otherArgs[1] == "changeset"):
	# current usage is "package file oldversion newversion"
	if len(otherArgs) != 5 and len(otherArgs) != 6:
	    return usage()

	name = otherArgs[2]
	if len(otherArgs) == 6:
	    (old, new) = (otherArgs[3], otherArgs[4])
	    outFile = otherArgs[5]
	else:
	    (old, new) = (None, otherArgs[3])
	    outFile = otherArgs[4]

	repos = openRepository(cfg.reppath)

	cscmd.ChangeSetCommand(repos, cfg, name, outFile, old, new)
    elif (otherArgs[1] == "commit"):
	targetBranch = None
	if argSet.has_key('target-branch'):
	    targetBranch  = argSet['target-branch']
	    del argSet['target-branch']
	if len(otherArgs) < 3: return usage()
	repos = openRepository(cfg.reppath)
	for changeSet in otherArgs[2:]:
	    commit.doCommit(repos, changeSet, targetBranch)
    elif (otherArgs[1] == "config"):
	if argSet: return usage
	if (len(otherArgs) > 2):
	    return usage()
	else:
	    cfg.display()
    elif (otherArgs[1] == "cook"):
	log.setVerbosity(1)
	macros = {}
	prep = 0
	if argSet.has_key('prep'):
	    del argSet['prep']
	    prep = 1
	if argSet.has_key('macros'):
	    argSet['macros']
	    f = open(argSet['macros'])
	    # XXX sick hack
	    macroSrc = "macros =" + f.read()
	    exec macroSrc
	    del f
	    del argSet['macros']
	if argSet: return usage()

	cook.cookCommand(cfg, otherArgs[2:], prep, macros)
    elif (otherArgs[1] == "erase"):
	if argSet: return usage
	if len(otherArgs) >= 3 and len(otherArgs) <=4:
	    db = openDatabase(cfg.root, cfg.dbpath)

	    args = [db, cfg] + otherArgs[2:]
	    updatecmd.doErase(*args)
	else:
	    return usage()
    elif (otherArgs[1] == "import"):
	if len(otherArgs) != 3 and len(otherArgs) != 3:
	    return usage()

	repos = openRepository(cfg.reppath)
	importrpm.doImport(repos, cfg, otherArgs[2])
    elif (otherArgs[1] == "localcs"):
	if len(otherArgs) != 4 and len(otherArgs) != 4:
	    return usage()

	name = otherArgs[2]
	outFile = otherArgs[3]

	db = database.Database(cfg.root, cfg.dbpath, "r")
	cscmd.LocalChangeSetCommand(db, cfg, name, outFile)
    elif (otherArgs[1] == "localcommit"):
	if len(otherArgs) < 3: return usage()
	db = database.Database(cfg.root, cfg.dbpath, "c")
	for changeSet in otherArgs[2:]:
	    commit.doLocalCommit(db, changeSet)
    elif (otherArgs[1] == "pkglist"):
	ls = argSet.has_key('ls')
	if ls: del argSet['ls']

	ids = argSet.has_key('ids')
	if ids: del argSet['ids']

	sha1s = argSet.has_key('sha1s')
	if sha1s: del argSet['sha1s']

	db = openDatabase(cfg.root, cfg.dbpath)

	if argSet: return usage()

	if len(otherArgs) >= 2 and len(otherArgs) <= 4:
	    args = [db, cfg, ls, ids, sha1s] + otherArgs[2:]
	    try:
		display.displayTroves(*args)
	    except IOError, msg:
		sys.stderr.write(msg.strerror + '\n')
		sys.exit(1)
	else:
	    return usage()
    elif (otherArgs[1] == "replist"):
	all = argSet.has_key('all')
	if all: del argSet['all']

	ls = argSet.has_key('ls')
	if ls: del argSet['ls']

	ids = argSet.has_key('ids')
	if ids: del argSet['ids']

	sha1s = argSet.has_key('sha1s')
	if sha1s: del argSet['sha1s']

	repos = openRepository(cfg.reppath)

	if argSet: return usage()

	if len(otherArgs) >= 2 and len(otherArgs) <= 4:
	    args = [repos, cfg, all, ls, ids, sha1s] + otherArgs[2:]
	    try:
		queryrep.displayTroves(*args)
	    except IOError, msg:
		sys.stderr.write(msg.strerror + '\n')
		sys.exit(1)
	else:
	    return usage()
    elif (otherArgs[1] == "rblist"):
	if argSet: return usage
	db = openDatabase(cfg.root, cfg.dbpath)
	rollbacks.listRollbacks(db, cfg)
    elif (otherArgs[1] == "remove"):
	if len(otherArgs) != 3: return usage()
	if argSet: return usage
	db = openDatabase(cfg.root, cfg.dbpath)
	fullPath = util.joinPaths(cfg.root, otherArgs[2])
	if os.path.exists(fullPath):
	    os.unlink(fullPath)
	else:
	    log.warning("%s has already been removed", fullPath)
	db.removeFile(otherArgs[2])
    elif (otherArgs[1] == "rollback"):
	if argSet: return usage
	db = openDatabase(cfg.root, cfg.dbpath)
	args = [db, cfg] + otherArgs[2:]
	rollbacks.apply(*args)
    elif (otherArgs[1] == "source" or otherArgs[1] == "src"):
	return srcctl.sourceCommand(cfg, otherArgs[2:], argSet)
    elif (otherArgs[1] == "update"):
	kwargs = {}
	replaceFiles = argSet.has_key('replace-files')
	if replaceFiles:
	    kwargs['replaceFiles'] = True
	    del argSet['replace-files']
	if argSet: return usage
	if len(otherArgs) >=3 and len(otherArgs) <= 4:
	    repos = openRepository(cfg.reppath)
	    db = openDatabase(cfg.root, cfg.dbpath)

	    args = [repos, db, cfg] + otherArgs[2:]
	    updatecmd.doUpdate(*args, **kwargs)
	else:
	    return usage()
    elif (otherArgs[1] == "return usage"):
	return usage(rc = 0)
    else:
	return usage()

    if profile:
	prof.stop()

    if log.errorOccurred():
	sys.exit(1)

def main():
    try:
	realMain()
    except xmlrpclib.ProtocolError, e:
	if e.errcode == 403:
	    print >> sys.stderr, \
		"remote server denied permission for the requested operation"
	else:
	    raise

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before>"""
python3 concat.py [-h] [--generate] [--merge] [--both] [--max-gap MAX_GAP]
                 [-e EXT]
                 [TARGET_DIR]

Generates concat files for merging. Creates separate videos for separate
broadcasts and incompatible resolutions, ignores very broken videos.

positional arguments:
  TARGET_DIR         Optional. defaults to the current working directory.

optional arguments:
  -h, --help         show this help message and exit
  --generate         generates concat files in TARGET_DIR, runs by default
  --merge            merges videos in TARGET_DIR according to existing concat
                     files
  --both             both generates concat files and merges videos
  --max-gap MAX_GAP  maximum gap between merged videos, in seconds. anything
                     larger is treated as a separate broadcast
  -e EXT             extension to merge, defaults to mp4

When merging, watch the output for "Non-monotonous DTS in output stream" -- A
few of these are harmless but a wall of them means that video is probably
corrupted.

TODO:
    Check output of ffmpeg for "Non-monotonous DTS in output stream" and "DTS discontinuity in stream",
    notify user of the problem as well as suggest fixes if fixes are possible (are they?)
    Alternatively log ffmpeg's output the way showroom.py does
"""

import os
import glob
import argparse
from subprocess import check_output, run
import json
from math import floor

"""
{
    "member1" : [
        video1,
        video2,
        video3,
        video4
    ],
    "member2" : [

    ]
}

video: {
    "start_time" : parsed from file name; in seconds,
    "duration"   : parsed from ffprobe,
    "height"     : parsed from ffprobe,
    "valid"      : true or false (false for stuff with no video content),
    "file"       : location of file
}

sample ffprobe output:
{
    "programs": [

    ],
    "streams": [
        {
            "height": 198,
            "duration": "499.654000"
        }
    ]
}

"""
def generate_concat_files(target_dir, target_ext, max_gap):
    oldcwd = os.getcwd()
    os.chdir(target_dir)

    max_gap = float(max_gap)

    # TODO: deal with leftovers (from after 24:00)
    files = sorted(glob.glob('{}/*.{}'.format(target_dir, target_ext)))

    def get_start_seconds(file):
        time_str = file.rsplit(' ', 1)[1].split('.')[0]
        hours, minutes, seconds = int(time_str[:2]), int(time_str[2:4]), int(time_str[4:])
        return float(hours*60*60 + minutes*60 + seconds)

    def get_start_hhmm(seconds):
        hours = seconds/(60*60)
        minutes = (hours - floor(hours))*60
        return '{:02d}{:02d}'.format(floor(hours), floor(minutes))

    member_dict = {}
    for file in files:
        member_name = file.rsplit(' ', 1)[0]
        if member_name not in member_dict:
            member_dict[member_name] = []
        new_video = {"start_time": get_start_seconds(file)}
        results = check_output([
                "ffprobe",
                '-loglevel', '16',
                '-show_entries', 'stream=duration,height',
                '-select_streams', 'v',
                '-i', file,
                '-of', 'json'
            ],
            universal_newlines=True
        )
        try:
            stream = json.loads(results)['streams'][0]
        except IndexError:
            new_video['valid']  = False
        else:
            new_video['file']     = file
            new_video['duration'] = float(stream['duration'])
            new_video['height']   = int(stream['height'])
            if new_video['duration'] >= 0.001:
                new_video['valid']  = True
            else:
                new_video['valid']  = False

        if new_video['valid']:
            member_dict[member_name].append(new_video)

    concat_files = {}

    def new_concat_file(member, first_video):
        filename = '{} {}.{}.concat'.format(member, get_start_hhmm(first_video['start_time']), target_ext)
        info = {
                'files': []
        }
        info['height'] = first_video['height']
        info['last_time'] = first_video['start_time'] + first_video['duration']
        info['files'].append(first_video['file'])
        return filename, info

    for member in member_dict.keys():
        """
        file_specifier (name + hhmm) : {
            height : 360 or 198,
            last_time : start_time + duration of most recently processed video
            files: [
                list of files
            ]
        }
        """
        filename, working = new_concat_file(member, member_dict[member][0])
        for item in member_dict[member][1:]:
            if (item['start_time'] >= working['last_time'] + max_gap
                    or item['height'] != working['height']):
                if filename in working:
                    # This needs to be dealt with by hand for now
                    print('Tried to add duplicate concat file name: {}'.format(filename))
                    raise FileExistsError
                concat_files[filename] = working
                filename, working = new_concat_file(member, item)
            else:
                working['files'].append(item['file'])
                working['last_time'] = item['start_time'] + item['duration']
        concat_files[filename] = working

    for file in concat_files.keys():
        # skip singleton videos
        #if len(concat_files[file]['files']) == 1:
        #    continue
        text = ""
        for item in concat_files[file]['files']:
            text += "file '" + item + "'\n"
        with open(file, 'w', encoding='utf8') as outfp:
            outfp.write(text)

    os.chdir(oldcwd)

"""
#!/bin/bash

# for f in ./*.mp4; do echo "file '$f'" >> mylist.txt; done
# for f in ./*.concat; do echo "$( basename $f )"; done
# for f in ./*.concat; do g="\"$( basename "$f" .mp4)\""; echo $f; echo $g; done
# echo "\"$( basename ./160612\ Showroom\ -\ AKB48\ Team\ K\ Tano\ Yuka\ 124028.mp4 .mp4)\""
for f in ./*.concat; do
    g=$( basename "$f" .concat);
    #ffmpeg -copytb 1 -f concat -i "$f" -vf "pad=width=640:height=360:(ow-iw)/2:(oh-ih)/2:color=black" -movflags +faststart "$g";
    ffmpeg -copytb 1 -f concat -i "$f" -movflags +faststart -c copy "$g";
done
"""

def merge_videos(target_dir):
    oldcwd = os.getcwd()
    os.chdir(target_dir)

    for concat_file in glob.glob('*.concat'):
        outfile = os.path.splitext(concat_file)[0]
        run(['ffmpeg',
            '-copytb', '1',
            '-f', 'concat',
            '-i', concat_file,
            '-movflags', '+faststart',
            '-c', 'copy', outfile])

    os.chdir(oldcwd)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Generates concat files for merging. Creates separate videos for separate broadcasts and incompatible resolutions, ignores very broken videos.", epilog="When merging, watch the output for \"Non-monotonous DTS in output stream\" -- A few of these are harmless but a wall of them means that video is probably corrupted.")
    parser.add_argument("--generate", action='store_true', help='generates concat files in TARGET_DIR, runs by default')
    parser.add_argument("--merge", action='store_true', help='merges videos in TARGET_DIR according to existing concat files')
    parser.add_argument("--both", action='store_true', help='both generates concat files and merges videos')
    parser.add_argument("target_dir", nargs='?', default='.', help='Optional. defaults to the current working directory.', metavar='TARGET_DIR')
    parser.add_argument("--max-gap", type=float, default=300.0, help='maximum gap between merged videos, in seconds. anything larger is treated as a separate broadcast. default = 300.0')
    parser.add_argument('-e', dest='ext', default='mp4', help='extension to merge, defaults to mp4')
    args = parser.parse_args()

    if not args.merge:
        generate_concat_files(args.target_dir, args.ext, args.max_gap)
    if args.merge or args.both:
        merge_videos(args.target_dir)

<commit_msg>Fixed target_dir<commit_after>#!/usr/bin/env python3

"""
python3 concat.py [-h] [--generate] [--merge] [--both] [--max-gap MAX_GAP]
                 [-e EXT]
                 [TARGET_DIR]

Generates concat files for merging. Creates separate videos for separate
broadcasts and incompatible resolutions, ignores very broken videos.

positional arguments:
  TARGET_DIR         Optional. defaults to the current working directory.

optional arguments:
  -h, --help         show this help message and exit
  --generate         generates concat files in TARGET_DIR, runs by default
  --merge            merges videos in TARGET_DIR according to existing concat
                     files
  --both             both generates concat files and merges videos
  --max-gap MAX_GAP  maximum gap between merged videos, in seconds. anything
                     larger is treated as a separate broadcast
  -e EXT             extension to merge, defaults to mp4

When merging, watch the output for "Non-monotonous DTS in output stream" -- A
few of these are harmless but a wall of them means that video is probably
corrupted.

"""

import os
import glob
import argparse
from subprocess import check_output, run
import json
from math import floor

# old version
def create_concat_files(target_dir, target_ext):
    oldcwd = os.getcwd()
    os.chdir(target_dir)

    # TODO: use ffprobe to separate files with incompatible resolutions and those with a gap greater than ~10 minutes

    files = sorted(glob.glob('{}/*.{}'.format(target_dir, target_ext)))

    member_dict = {}
    for file in files:
        member_name = file.rsplit(' ', 1)[0]
        if member_name not in member_dict:
            member_dict[member_name] = []
        member_dict[member_name].append(file)

    concat_files = {}
    for key in member_dict.keys():
        filename = key +' ' + member_dict[key][0].rsplit(' ', 1)[1][:4] + '.mp4.concat'
        text = ""
        for item in member_dict[key]:
            text += "file '" + item + "'\n"
        concat_files.update({filename:text})

    for key in concat_files.keys():
        with open(key, 'w', encoding='utf8') as outfp:
            _ = outfp.write(concat_files[key])

    os.chdir(oldcwd)

"""
{
    "member1" : [
        video1,
        video2,
        video3,
        video4
    ],
    "member2" : [

    ]
}

video: {
    "start_time" : parsed from file name; in seconds,
    "duration"   : parsed from ffprobe,
    "height"     : parsed from ffprobe,
    "valid"      : true or false (false for stuff with no video content),
    "file"       : location of file
}
"duration"
"height"

sample ffprobe output:
{
    "programs": [

    ],
    "streams": [
        {
            "height": 198,
            "duration": "499.654000"
        }
    ]
}
for member in members:

"""
def generate_concat_files(target_dir, target_ext, max_gap):
    oldcwd = os.getcwd()
    os.chdir(target_dir)

    max_gap = float(max_gap)

    # TODO: deal with leftovers (from after 24:00)
    files = sorted(glob.glob('*.{}'.format(target_ext)))

    def get_start_seconds(file):
        time_str = file.rsplit(' ', 1)[1].split('.')[0]
        hours, minutes, seconds = int(time_str[:2]), int(time_str[2:4]), int(time_str[4:])
        return float(hours*60*60 + minutes*60 + seconds)

    def get_start_hhmm(seconds):
        hours = seconds/(60*60)
        minutes = (hours - floor(hours))*60
        return '{:02d}{:02d}'.format(floor(hours), floor(minutes))

    member_dict = {}
    for file in files:
        member_name = file.rsplit(' ', 1)[0]
        if member_name not in member_dict:
            member_dict[member_name] = []
        new_video = {"start_time": get_start_seconds(file)}
        results = check_output([
                "ffprobe",
                '-loglevel', '16',
                '-show_entries', 'stream=duration,height',
                '-select_streams', 'v',
                '-i', file,
                '-of', 'json'
            ],
            universal_newlines=True
        )
        try:
            stream = json.loads(results)['streams'][0]
        except IndexError:
            new_video['valid']  = False
        else:
            new_video['file']     = file
            new_video['duration'] = float(stream['duration'])
            new_video['height']   = int(stream['height'])
            if new_video['duration'] >= 0.001 and 197 < new_video['height'] < 500:
                new_video['valid']  = True
            else:
                new_video['valid']  = False

        if new_video['valid']:
            member_dict[member_name].append(new_video)

    concat_files = {}

    def new_concat_file(member, first_video):
        filename = '{} {}.{}.concat'.format(member, get_start_hhmm(first_video['start_time']), target_ext)
        info = {
                'files': []
        }
        info['height'] = first_video['height']
        info['last_time'] = first_video['start_time'] + first_video['duration']
        info['files'].append(first_video['file'])
        return filename, info

    for member in member_dict.keys():
        """
        file_specifier (name + hhmm) : {
            height : 360 or 198,
            last_time : start_time + duration of most recently processed video
            files: [
                list of files
            ]
        }
        """
        filename, working = new_concat_file(member, member_dict[member][0])
        for item in member_dict[member][1:]:
            if (item['start_time'] >= working['last_time'] + max_gap
                    or item['height'] != working['height']):
                if filename in working:
                    # This needs to be dealt with by hand for now
                    print('Tried to add duplicate concat file name: {}'.format(filename))
                    raise FileExistsError
                concat_files[filename] = working
                filename, working = new_concat_file(member, item)
            else:
                working['files'].append(item['file'])
                working['last_time'] = item['start_time'] + item['duration']
        concat_files[filename] = working

    for file in concat_files.keys():
        # skip singleton videos
        #if len(concat_files[file]['files']) == 1:
        #    continue
        text = ""
        for item in concat_files[file]['files']:
            text += "file '" + item + "'\n"
        with open(file, 'w', encoding='utf8') as outfp:
            outfp.write(text)

    os.chdir(oldcwd)

"""
#!/bin/bash

# for f in ./*.mp4; do echo "file '$f'" >> mylist.txt; done
# for f in ./*.concat; do echo "$( basename $f )"; done
# for f in ./*.concat; do g="\"$( basename "$f" .mp4)\""; echo $f; echo $g; done
# echo "\"$( basename ./160612\ Showroom\ -\ AKB48\ Team\ K\ Tano\ Yuka\ 124028.mp4 .mp4)\""
for f in ./*.concat; do
    g=$( basename "$f" .concat);
    #ffmpeg -copytb 1 -f concat -i "$f" -vf "pad=width=640:height=360:(ow-iw)/2:(oh-ih)/2:color=black" -movflags +faststart "$g";
    ffmpeg -copytb 1 -f concat -i "$f" -movflags +faststart -c copy "$g";
done
"""

def merge_videos(target_dir):
    oldcwd = os.getcwd()
    os.chdir(target_dir)

    for concat_file in glob.glob('*.concat'):
        outfile = os.path.splitext(concat_file)[0]

        # wish i didn't have to read it twice...
        with open(concat_file, encoding='utf8') as infp:
            data = infp.read()
        if data.count('file \'') == 1:
            src = data[5:].strip('\'\n./')
            instructions = ['-i', src]
        elif data.count('file \'') > 1:
            instructions = ['-f', 'concat', '-i', concat_file]
        else:
            print("Empty concat file")
            raise FileNotFoundError

        run(['ffmpeg',
            '-copytb', '1',
            *instructions,
            '-movflags', '+faststart',
            '-c', 'copy', outfile])

    os.chdir(oldcwd)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Generates concat files for merging. Creates separate videos for separate broadcasts and incompatible resolutions, ignores very broken videos.", epilog="When merging, watch the output for \"Non-monotonous DTS in output stream\" -- A few of these are harmless but a wall of them means that video is probably corrupted.")
    parser.add_argument("--generate", action='store_true', help='generates concat files in TARGET_DIR, runs by default')
    parser.add_argument("--merge", action='store_true', help='merges videos in TARGET_DIR according to existing concat files')
    parser.add_argument("--both", action='store_true', help='both generates concat files and merges videos')
    parser.add_argument("target_dir", nargs='?', default='.', help='Optional. defaults to the current working directory.', metavar='TARGET_DIR')
    parser.add_argument("--max-gap", type=float, default=300.0, help='maximum gap between merged videos, in seconds. anything larger is treated as a separate broadcast. default = 300.0')
    parser.add_argument('-e', dest='ext', default='mp4', help='extension to merge, defaults to mp4')
    args = parser.parse_args()

    if not args.merge:
        generate_concat_files(args.target_dir, args.ext, args.max_gap)
    if args.merge or args.both:
        merge_videos(args.target_dir)

<|endoftext|>