<commit_before>"""Metadata and operational constants."""
from typing import Any, Dict

from pudl.metadata.constants import KEYWORDS, CONTRIBUTORS
from pudl.metadata.enums import EPACEMS_STATES

SOURCES: Dict[str, Any] = {
    "censusdp1tract": {
        "title": "Census DP1",
        "path": "https://www.census.gov/geographies/mapping-files/2010/geo/tiger-data.html",
        "description": (
            "US Census Demographic Profile 1 (DP1) County and Tract "
            "GeoDatabase."
        ),
        "working_partitions": {},  # Census DP1 is monolithic.
    },
    "eia860": {
        "title": "EIA Form 860",
        "path": "https://www.eia.gov/electricity/data/eia860",
        "description": (
            "US Energy Information Administration (EIA) Form 860 data for "
            "electric power plants with 1 megawatt or greater combined nameplate "
            "capacity."
        ),
        "field_namespace": "eia",
        "contributors": [
            CONTRIBUTORS["catalyst-cooperative"],
            CONTRIBUTORS["zane-selvans"],
            CONTRIBUTORS["christina-gosnell"],
            CONTRIBUTORS["steven-winter"],
            CONTRIBUTORS["alana-wilson"],
        ],
        "working_partitions": {
            "years": sorted(set(range(2001, 2021))),
        },
        "keywords": sorted(set(
            [
                "eia860",
                "form 860",
                "ownership",
                "annual",
                "yearly",
            ]
            + KEYWORDS["eia"]
            + KEYWORDS["us_govt"]
            + KEYWORDS["electricity"]
            + KEYWORDS["fuels"]
            + KEYWORDS["plants"]
            + KEYWORDS["environment"]
        )),
    },
    "eia860m": {
        "title": "EIA Form 860m",
        "path": "https://www.eia.gov/electricity/data/eia860m",
        "description": (
            "US Energy Information Administration (EIA) Form 860 M data for "
            "electric power plants with 1 megawatt or greater combined nameplate "
            "capacity. Preliminary Monthly Electric Generator Inventory (based on "
            "Form EIA-860M as a supplement to Form EIA-860)"
        ),
        "field_namespace": "eia",
        "contributors": [],
        "working_partitions": {
            "year_month": "2021-08",
        },
        "keywords": sorted(set(
            [
                "eia860m",
                "form 860m"
                "monthly"
            ]
        ))
    },
    "eia861": {
        "title": "EIA Form 861: Annual Electric Power Industry Report",
        "path": "https://www.eia.gov/electricity/data/eia861",
        "description": (
            "EIA Form 861 Annual Electric Power Industry Report, detailed "
            "data files."
        ),
        "field_namespace": "eia",
        "working_partitions": {
            "years": sorted(set(range(2001, 2021))),
        },
        "contributors": [],
        "keywords": sorted(set(
            [
                "eia861",
                "form 861",
                "balancing authority",
                "advanced metering infrastructure",
                "customer class",
                "green pricing",
                "energy efficiency",
                "demand side management",
                "demand response",
                "net metering",
                "business model",
                "service territory",
                "annual",
                "yearly"
            ]
            + KEYWORDS["eia"]
            + KEYWORDS["us_govt"]
            + KEYWORDS["electricity"]
        ))
    },
    "eia923": {
        "title": "EIA Form 923",
        "path": "https://www.eia.gov/electricity/data/eia923",
        "description": (
            "The EIA Form 923 collects detailed monthly and annual electric "
            "power data on electricity generation, fuel consumption, fossil fuel "
            "stocks, and receipts at the power plant and prime mover level."
        ),
        "field_namespace": "eia",
        "working_partitions": {
            "years": sorted(set(range(2001, 2021))),
        },
        "contributors": [
            CONTRIBUTORS["catalyst-cooperative"],
            CONTRIBUTORS["zane-selvans"],
            CONTRIBUTORS["christina-gosnell"],
            CONTRIBUTORS["steven-winter"],
            CONTRIBUTORS["katherine-lamb"],
        ],
        "keywords": sorted(set(
            [
                "eia923",
                "form 923",
                "monthly",
            ]
            + KEYWORDS["eia"]
            + KEYWORDS["us_govt"]
            + KEYWORDS["electricity"]
            + KEYWORDS["fuels"]
            + KEYWORDS["plants"]
            + KEYWORDS["environment"]
        )),
    },
    "eiawater": {
        "title": "EIA Thermoelectric Cooling Water",
        "path": "https://www.eia.gov/electricity/data/water",
        "description": "",
    },
    "epacems": {
        "title": "EPA Hourly Continuous Emission Monitoring System (CEMS)",
        "path": "https://ampd.epa.gov/ampd",
        "description": (
            "US EPA hourly Continuous Emissions Monitoring System (CEMS) data."
        ),
        "field_namespace": "epacems",
        "working_partitions": {
            "years": sorted(set(range(1995, 2021))),
            "states": sorted(set(EPACEMS_STATES)),
        },
        "contributors": [
            CONTRIBUTORS["catalyst-cooperative"],
            CONTRIBUTORS["karl-dunkle-werner"],
            CONTRIBUTORS["zane-selvans"],
        ],
        "keywords": sorted(set(
            [
                "continuous emissions monitoring system",
                "cems",
                "air markets program data",
                "ampd",
                "hourly"
            ]
            + KEYWORDS["epa"]
            + KEYWORDS["us_govt"]
            + KEYWORDS["electricity"]
            + KEYWORDS["environment"]
        )),
    },
    "ferc1": {
        "title": "FERC Form 1",
        "path": "https://www.ferc.gov/industries-data/electric/general-information/electric-industry-forms/form-1-electric-utility-annual",
        "description": (
            "The Federal Energy Regulatory Commission (FERC) Form 1 is a "
            "comprehensive financial and operating report submitted annually for "
            "electric rate regulation, market oversight analysis, and financial audits "
            "by Major electric utilities, licensees and others."
        ),
        "field_namespace": "ferc1",
        "working_partitions": {
            "years": sorted(set(range(1994, 2021))),
        },
        "contributors": [
            CONTRIBUTORS["catalyst-cooperative"],
            CONTRIBUTORS["zane-selvans"],
            CONTRIBUTORS["christina-gosnell"],
            CONTRIBUTORS["steven-winter"],
            CONTRIBUTORS["alana-wilson"],
            CONTRIBUTORS["austen-sharpe"],
        ],
        "keywords": sorted(set(
            [
                "form 1",
                "ferc1",
            ]
            + KEYWORDS["ferc"]
            + KEYWORDS["us_govt"]
            + KEYWORDS["finance"]
            + KEYWORDS["electricity"]
            + KEYWORDS["plants"]
            + KEYWORDS["fuels"]
        )),
    },
    "ferc2": {
        "title": "FERC Form 2",
        "path": "https://www.ferc.gov/industries-data/natural-gas/industry-forms/form-2-2a-3-q-gas-historical-vfp-data",
        "description": (
            "The Federal Energy Regulatory Commission (FERC) Form 2 is a "
            "comprehensive financial and operating report submitted for natural gas "
            "pipelines rate regulation and financial audits."
        ),
        "field_namespace": "ferc2",
        "working_partitions": {
            "years": [],  # Not yet working!
        }
    },
    "ferc714": {
        "title": "FERC Form 714: Annual Electric Balancing Authority Area and Planning Area Report",
        "path": "https://www.ferc.gov/industries-data/electric/general-information/electric-industry-forms/form-no-714-annual-electric",
        "description": (
            "Electric transmitting utilities operating balancing authority "
            "areas and planning areas with annual peak demand over 200MW are required "
            "to file Form 714 with the Federal Energy Regulatory Commission (FERC), "
            "reporting balancing authority area generation, actual and scheduled "
            "inter-balancing authority area power transfers, and net energy for load, "
            "summer-winter generation peaks and system lambda."
        ),
        "field_namespace": "ferc714",
        "working_partitions": {},  # Data is monolitic, one file with all years.
    },
    "ferceqr": {
        "title": "FERC Form 920: Electric Quarterly Report (EQR)",
        "path": "https://www.ferc.gov/industries-data/electric/power-sales-and-markets/electric-quarterly-reports-eqr",
        "description": "",
    },
    "msha": {
        "title": "Mine Safety and Health Administration (MSHA)",
        "path": "https://arlweb.msha.gov/OpenGovernmentData/OGIMSHA.asp",
        "description": "",
    },
    "phmsa": {
        "title": "Pipelines and Hazardous Materials Safety Administration (PHMSA)",
        "path": "https://www.phmsa.dot.gov/data-and-statistics/pipeline/data-and-statistics-overview",
        "description": "",
    },
    "pudl": {
        "title": "The Public Utility Data Liberation (PUDL) Project",
        "path": "https://catalyst.coop/pudl",
        "description": "PUDL is a data processing pipeline created by Catalyst"
        "Cooperative that cleans, integrates, and standardizes some of the most"
        "widely used public energy datasets in the US. The data serve researchers,"
        "activists, journalists, and policy makers that might not have the"
        "technical expertise to access it in its raw form, the time to clean and"
        "prepare the data for bulk analysis, or the means to purchase it from"
        "existing commercial providers.",
        "contributors": [
            CONTRIBUTORS["catalyst-cooperative"],
            CONTRIBUTORS["zane-selvans"],
            CONTRIBUTORS["christina-gosnell"],
            CONTRIBUTORS["steven-winter"],
        ],
        "keywords": ["us", "electricity", "open data", "open source"],
    },
}
"""
Data source attributes by PUDL identifier.
"""
<commit_msg>Add explicit licenses to all data source metadata.<commit_after>"""Metadata and operational constants."""
from typing import Any, Dict

from pudl.metadata.constants import CONTRIBUTORS, KEYWORDS, LICENSES
from pudl.metadata.enums import EPACEMS_STATES

SOURCES: Dict[str, Any] = {
    "censusdp1tract": {
        "title": "Census DP1",
        "path": "https://www.census.gov/geographies/mapping-files/2010/geo/tiger-data.html",
        "description": (
            "US Census Demographic Profile 1 (DP1) County and Tract "
            "GeoDatabase."
        ),
        "working_partitions": {},  # Census DP1 is monolithic.
        "license_raw": LICENSES["us-govt"],
        "license_pudl": LICENSES["cc-by-4.0"],
    },
    "eia860": {
        "title": "EIA Form 860",
        "path": "https://www.eia.gov/electricity/data/eia860",
        "description": (
            "US Energy Information Administration (EIA) Form 860 data for "
            "electric power plants with 1 megawatt or greater combined nameplate "
            "capacity."
        ),
        "field_namespace": "eia",
        "contributors": [
            CONTRIBUTORS["catalyst-cooperative"],
            CONTRIBUTORS["zane-selvans"],
            CONTRIBUTORS["christina-gosnell"],
            CONTRIBUTORS["steven-winter"],
            CONTRIBUTORS["alana-wilson"],
        ],
        "working_partitions": {
            "years": sorted(set(range(2001, 2021))),
        },
        "keywords": sorted(set(
            [
                "eia860",
                "form 860",
                "ownership",
                "annual",
                "yearly",
            ]
            + KEYWORDS["eia"]
            + KEYWORDS["us_govt"]
            + KEYWORDS["electricity"]
            + KEYWORDS["fuels"]
            + KEYWORDS["plants"]
            + KEYWORDS["environment"]
        )),
        "license_raw": LICENSES["us-govt"],
        "license_pudl": LICENSES["cc-by-4.0"],
    },
    "eia860m": {
        "title": "EIA Form 860m",
        "path": "https://www.eia.gov/electricity/data/eia860m",
        "description": (
            "US Energy Information Administration (EIA) Form 860 M data for "
            "electric power plants with 1 megawatt or greater combined nameplate "
            "capacity. Preliminary Monthly Electric Generator Inventory (based on "
            "Form EIA-860M as a supplement to Form EIA-860)"
        ),
        "field_namespace": "eia",
        "contributors": [],
        "working_partitions": {
            "year_month": "2021-08",
        },
        "keywords": sorted(set(
            [
                "eia860m",
                "form 860m"
                "monthly"
            ]
        )),
        "license_raw": LICENSES["us-govt"],
        "license_pudl": LICENSES["cc-by-4.0"],
    },
    "eia861": {
        "title": "EIA Form 861: Annual Electric Power Industry Report",
        "path": "https://www.eia.gov/electricity/data/eia861",
        "description": (
            "EIA Form 861 Annual Electric Power Industry Report, detailed "
            "data files."
        ),
        "field_namespace": "eia",
        "working_partitions": {
            "years": sorted(set(range(2001, 2021))),
        },
        "contributors": [],
        "keywords": sorted(set(
            [
                "eia861",
                "form 861",
                "balancing authority",
                "advanced metering infrastructure",
                "customer class",
                "green pricing",
                "energy efficiency",
                "demand side management",
                "demand response",
                "net metering",
                "business model",
                "service territory",
                "annual",
                "yearly"
            ]
            + KEYWORDS["eia"]
            + KEYWORDS["us_govt"]
            + KEYWORDS["electricity"]
        )),
        "license_raw": LICENSES["us-govt"],
        "license_pudl": LICENSES["cc-by-4.0"],
    },
    "eia923": {
        "title": "EIA Form 923",
        "path": "https://www.eia.gov/electricity/data/eia923",
        "description": (
            "The EIA Form 923 collects detailed monthly and annual electric "
            "power data on electricity generation, fuel consumption, fossil fuel "
            "stocks, and receipts at the power plant and prime mover level."
        ),
        "field_namespace": "eia",
        "working_partitions": {
            "years": sorted(set(range(2001, 2021))),
        },
        "contributors": [
            CONTRIBUTORS["catalyst-cooperative"],
            CONTRIBUTORS["zane-selvans"],
            CONTRIBUTORS["christina-gosnell"],
            CONTRIBUTORS["steven-winter"],
            CONTRIBUTORS["katherine-lamb"],
        ],
        "keywords": sorted(set(
            [
                "eia923",
                "form 923",
                "monthly",
            ]
            + KEYWORDS["eia"]
            + KEYWORDS["us_govt"]
            + KEYWORDS["electricity"]
            + KEYWORDS["fuels"]
            + KEYWORDS["plants"]
            + KEYWORDS["environment"]
        )),
        "license_raw": LICENSES["us-govt"],
        "license_pudl": LICENSES["cc-by-4.0"],
    },
    "eiawater": {
        "title": "EIA Thermoelectric Cooling Water",
        "path": "https://www.eia.gov/electricity/data/water",
        "description": "",
        "license_raw": LICENSES["us-govt"],
        "license_pudl": LICENSES["cc-by-4.0"],
    },
    "epacems": {
        "title": "EPA Hourly Continuous Emission Monitoring System (CEMS)",
        "path": "https://ampd.epa.gov/ampd",
        "description": (
            "US EPA hourly Continuous Emissions Monitoring System (CEMS) data."
        ),
        "field_namespace": "epacems",
        "working_partitions": {
            "years": sorted(set(range(1995, 2021))),
            "states": sorted(set(EPACEMS_STATES)),
        },
        "contributors": [
            CONTRIBUTORS["catalyst-cooperative"],
            CONTRIBUTORS["karl-dunkle-werner"],
            CONTRIBUTORS["zane-selvans"],
        ],
        "keywords": sorted(set(
            [
                "continuous emissions monitoring system",
                "cems",
                "air markets program data",
                "ampd",
                "hourly"
            ]
            + KEYWORDS["epa"]
            + KEYWORDS["us_govt"]
            + KEYWORDS["electricity"]
            + KEYWORDS["environment"]
        )),
        "license_raw": LICENSES["us-govt"],
        "license_pudl": LICENSES["cc-by-4.0"],
    },
    "ferc1": {
        "title": "FERC Form 1",
        "path": "https://www.ferc.gov/industries-data/electric/general-information/electric-industry-forms/form-1-electric-utility-annual",
        "description": (
            "The Federal Energy Regulatory Commission (FERC) Form 1 is a "
            "comprehensive financial and operating report submitted annually for "
            "electric rate regulation, market oversight analysis, and financial audits "
            "by Major electric utilities, licensees and others."
        ),
        "field_namespace": "ferc1",
        "working_partitions": {
            "years": sorted(set(range(1994, 2021))),
        },
        "contributors": [
            CONTRIBUTORS["catalyst-cooperative"],
            CONTRIBUTORS["zane-selvans"],
            CONTRIBUTORS["christina-gosnell"],
            CONTRIBUTORS["steven-winter"],
            CONTRIBUTORS["alana-wilson"],
            CONTRIBUTORS["austen-sharpe"],
        ],
        "keywords": sorted(set(
            [
                "form 1",
                "ferc1",
            ]
            + KEYWORDS["ferc"]
            + KEYWORDS["us_govt"]
            + KEYWORDS["finance"]
            + KEYWORDS["electricity"]
            + KEYWORDS["plants"]
            + KEYWORDS["fuels"]
        )),
        "license_raw": LICENSES["us-govt"],
        "license_pudl": LICENSES["cc-by-4.0"],
    },
    "ferc2": {
        "title": "FERC Form 2",
        "path": "https://www.ferc.gov/industries-data/natural-gas/industry-forms/form-2-2a-3-q-gas-historical-vfp-data",
        "description": (
            "The Federal Energy Regulatory Commission (FERC) Form 2 is a "
            "comprehensive financial and operating report submitted for natural gas "
            "pipelines rate regulation and financial audits."
        ),
        "field_namespace": "ferc2",
        "working_partitions": {
            "years": [],  # Not yet working!
        },
        "license_raw": LICENSES["us-govt"],
        "license_pudl": LICENSES["cc-by-4.0"],
    },
    "ferc714": {
        "title": "FERC Form 714: Annual Electric Balancing Authority Area and Planning Area Report",
        "path": "https://www.ferc.gov/industries-data/electric/general-information/electric-industry-forms/form-no-714-annual-electric",
        "description": (
            "Electric transmitting utilities operating balancing authority "
            "areas and planning areas with annual peak demand over 200MW are required "
            "to file Form 714 with the Federal Energy Regulatory Commission (FERC), "
            "reporting balancing authority area generation, actual and scheduled "
            "inter-balancing authority area power transfers, and net energy for load, "
            "summer-winter generation peaks and system lambda."
        ),
        "field_namespace": "ferc714",
        "working_partitions": {},  # Data is monolitic, one file with all years.
        "license_raw": LICENSES["us-govt"],
        "license_pudl": LICENSES["cc-by-4.0"],
    },
    "ferceqr": {
        "title": "FERC Form 920: Electric Quarterly Report (EQR)",
        "path": "https://www.ferc.gov/industries-data/electric/power-sales-and-markets/electric-quarterly-reports-eqr",
        "description": "",
        "license_raw": LICENSES["us-govt"],
        "license_pudl": LICENSES["cc-by-4.0"],
    },
    "msha": {
        "title": "Mine Safety and Health Administration (MSHA)",
        "path": "https://arlweb.msha.gov/OpenGovernmentData/OGIMSHA.asp",
        "description": "",
        "license_raw": LICENSES["us-govt"],
        "license_pudl": LICENSES["cc-by-4.0"],
    },
    "phmsa": {
        "title": "Pipelines and Hazardous Materials Safety Administration (PHMSA)",
        "path": "https://www.phmsa.dot.gov/data-and-statistics/pipeline/data-and-statistics-overview",
        "description": "",
        "license_raw": LICENSES["us-govt"],
        "license_pudl": LICENSES["cc-by-4.0"],
    },
    "pudl": {
        "title": "The Public Utility Data Liberation (PUDL) Project",
        "path": "https://catalyst.coop/pudl",
        "description": "PUDL is a data processing pipeline created by Catalyst"
        "Cooperative that cleans, integrates, and standardizes some of the most"
        "widely used public energy datasets in the US. The data serve researchers,"
        "activists, journalists, and policy makers that might not have the"
        "technical expertise to access it in its raw form, the time to clean and"
        "prepare the data for bulk analysis, or the means to purchase it from"
        "existing commercial providers.",
        "contributors": [
            CONTRIBUTORS["catalyst-cooperative"],
            CONTRIBUTORS["zane-selvans"],
            CONTRIBUTORS["christina-gosnell"],
            CONTRIBUTORS["steven-winter"],
        ],
        "keywords": ["us", "electricity", "open data", "open source"],
        "license_pudl": LICENSES["cc-by-4.0"],
    },
}
"""
Data source attributes by PUDL identifier.
"""
<|endoftext|>


<commit_before>"""
Use orca_train with a parser.

Usage:
    parser_orcatrain.py FOLDER LIST CONFIG MODEL
    parser_orcatrain.py (-h | --help)

Arguments:
    FOLDER  Path to the folder where everything gets saved to, e.g. the summary.txt, the plots, the trained models, etc.
    LIST    A .toml file which contains the pathes of the training and validation files.
            An example can be found in config/lists/example_list.toml
    CONFIG  A .toml file which sets up the training.
            An example can be found in config/models/example_config.toml. The possible parameters are listed in
            utilities/input_output_utilities.py in the class Configuration.
    MODEL   Path to a .toml file with infos about a model.

Options:
    -h --help                       Show this screen.

"""
from docopt import docopt
from orcanet.core import orca_train, Configuration
from orcanet.model_archs.model_setup import build_nn_model

def run_train(main_folder, list_file, config_file, model_file):
    """
    Use orca_train with a parser.

    Parameters
    ----------
    main_folder : str
        Path to the folder where everything gets saved to, e.g. the summary log file, the plots, the trained models, etc.
    list_file : str
        Path to a list file which contains pathes to all the h5 files that should be used for training and validation.
    config_file : str
        Path to a .toml file which overwrite some of the default settings for training and validating a model.
    model_file : str
        Path to a file with parameters to build a model of a predefined architecture with OrcaNet.

    """
    # Set up the cfg object with the input data
    cfg = Configuration(main_folder, list_file, config_file)
    # If this is the start of the training, a compiled model needs to be handed to the orca_train function
    if cfg.get_latest_epoch() == (0, 1):
        # Add Info for building a model with OrcaNet to the cfg object
        cfg.set_from_model_file(model_file)
        # Build it
        initial_model = build_nn_model(cfg)
    else:
        # No model is required if the training is continued, as it will be loaded automatically
        initial_model = None
    orca_train(cfg, initial_model)

def parse_input():
    """ Run the orca_train function with a parser. """
    args = docopt(__doc__)
    main_folder = args['FOLDER']
    list_file = args['LIST']
    config_file = args['CONFIG']
    model_file = args['MODEL']
    run_train(main_folder, list_file, config_file, model_file)

if __name__ == '__main__':
    parse_input()
<commit_msg>Fix again.<commit_after>"""
Use orca_train with a parser.

Usage:
    parser_orcatrain.py FOLDER LIST CONFIG MODEL
    parser_orcatrain.py (-h | --help)

Arguments:
    FOLDER  Path to the folder where everything gets saved to, e.g. the summary.txt, the plots, the trained models, etc.
    LIST    A .toml file which contains the pathes of the training and validation files.
            An example can be found in config/lists/example_list.toml
    CONFIG  A .toml file which sets up the training.
            An example can be found in config/models/example_config.toml. The possible parameters are listed in
            utilities/input_output_utilities.py in the class Configuration.
    MODEL   Path to a .toml file with infos about a model.

Options:
    -h --help                       Show this screen.

"""
from docopt import docopt
from orcanet.core import orca_train, Configuration
from orcanet.model_archs.model_setup import build_nn_model

def run_train(main_folder, list_file, config_file, model_file):
    """
    Use orca_train with a parser.

    Parameters
    ----------
    main_folder : str
        Path to the folder where everything gets saved to, e.g. the summary log file, the plots, the trained models, etc.
    list_file : str
        Path to a list file which contains pathes to all the h5 files that should be used for training and validation.
    config_file : str
        Path to a .toml file which overwrite some of the default settings for training and validating a model.
    model_file : str
        Path to a file with parameters to build a model of a predefined architecture with OrcaNet.

    """
    # Set up the cfg object with the input data
    cfg = Configuration(main_folder, list_file, config_file)
    # If this is the start of the training, a compiled model needs to be handed to the orca_train function
    if cfg.get_latest_epoch() == (0, 0):
        # Add Info for building a model with OrcaNet to the cfg object
        cfg.set_from_model_file(model_file)
        # Build it
        initial_model = build_nn_model(cfg)
    else:
        # No model is required if the training is continued, as it will be loaded automatically
        initial_model = None
    orca_train(cfg, initial_model)

def parse_input():
    """ Run the orca_train function with a parser. """
    args = docopt(__doc__)
    main_folder = args['FOLDER']
    list_file = args['LIST']
    config_file = args['CONFIG']
    model_file = args['MODEL']
    run_train(main_folder, list_file, config_file, model_file)

if __name__ == '__main__':
    parse_input()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import cPickle
import datetime
import threading
import time
import json
import random
import re
from bottle import route, hook, response, run, static_file

database = cPickle.load(open('../data/spring15.bin', 'rb'))

def updateInformation():
    weekday = datetime.datetime.now().weekday()
    time_later = (datetime.datetime.now() + datetime.timedelta(minutes = 15)).time()
    new_information = []
    for meeting in database:
        if meeting['weekday'] != weekday:
            continue
        if meeting['start'] < time_later < meeting['end']:
            new_meeting = meeting.copy()
            del new_meeting['start']
            del new_meeting['end']
            new_meeting['course'] = ', '.join(new_meeting['course'])
            new_information.append(new_meeting)
    if len(new_information) < 2:
        empty_meeting = {}
        empty_meeting['course'] = 'n/a'
        new_information.append(empty_meeting)
    globals()['information_string'] = json.dumps(new_information)
    globals()['information_list'] = map(json.dumps, new_information)

with open('index.html') as f:
    # this is absolutely horrible and we should use nginx
    globals()['index_html_content'] = f.read()

with open('index.css') as f:
    # this is absolutely horrible and we should use nginx
    globals()['index_css_content'] = f.read()

with open('index.js') as f:
    # this is absolutely horrible and we should use nginx
    globals()['index_js_content'] = f.read()

with open('quojs.js') as f:
    # this is absolutely horrible and we should use nginx
    globals()['quo_js_content'] = f.read()

def updateThread():
    while True:
        updateInformation()
        time.sleep(60)

updateInformation()
looper = threading.Thread(target = updateThread)
looper.daemon = True
looper.start()

@route('/')
def index():
    return static_file('index.html', root = '.')

@route('/index.css')
def index_css():
    return static_file('index.css', root = '.')

@route('/index.js')
def index_js():
    return static_file('index.js', root = '.')

@route('/quojs.js')
def index_js():
    return static_file('quojs.js', root = '.')

@route('/favicon.ico')
def index_js():
    return static_file('favicon.ico', root = '.')

@route('/all')
def all_data():
    return globals()['information_string']

@route('/random')
def select_data():
    return random.choice(globals()['information_list'])

run(host = '0.0.0.0', port = 8080, server = 'tornado')
<commit_msg>ugh was running on 8080<commit_after>#!/usr/bin/env python

import cPickle
import datetime
import threading
import time
import json
import random
import re
from bottle import route, hook, response, run, static_file

database = cPickle.load(open('../data/spring15.bin', 'rb'))

def updateInformation():
    weekday = datetime.datetime.now().weekday()
    time_later = (datetime.datetime.now() + datetime.timedelta(minutes = 15)).time()
    new_information = []
    for meeting in database:
        if meeting['weekday'] != weekday:
            continue
        if meeting['start'] < time_later < meeting['end']:
            new_meeting = meeting.copy()
            del new_meeting['start']
            del new_meeting['end']
            new_meeting['course'] = ', '.join(new_meeting['course'])
            new_information.append(new_meeting)
    if len(new_information) < 2:
        empty_meeting = {}
        empty_meeting['course'] = 'n/a'
        new_information.append(empty_meeting)
    globals()['information_string'] = json.dumps(new_information)
    globals()['information_list'] = map(json.dumps, new_information)

with open('index.html') as f:
    # this is absolutely horrible and we should use nginx
    globals()['index_html_content'] = f.read()

with open('index.css') as f:
    # this is absolutely horrible and we should use nginx
    globals()['index_css_content'] = f.read()

with open('index.js') as f:
    # this is absolutely horrible and we should use nginx
    globals()['index_js_content'] = f.read()

with open('quojs.js') as f:
    # this is absolutely horrible and we should use nginx
    globals()['quo_js_content'] = f.read()

def updateThread():
    while True:
        updateInformation()
        time.sleep(60)

updateInformation()
looper = threading.Thread(target = updateThread)
looper.daemon = True
looper.start()

@route('/')
def index():
    return static_file('index.html', root = '.')

@route('/index.css')
def index_css():
    return static_file('index.css', root = '.')

@route('/index.js')
def index_js():
    return static_file('index.js', root = '.')

@route('/quojs.js')
def index_js():
    return static_file('quojs.js', root = '.')

@route('/favicon.ico')
def index_js():
    return static_file('favicon.ico', root = '.')

@route('/all')
def all_data():
    return globals()['information_string']

@route('/random')
def select_data():
    return random.choice(globals()['information_list'])

run(host = '0.0.0.0', port = 80, server = 'tornado')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import division, print_function
from multiprocessing import Pool, Lock
import warnings
import six

import numpy as np
import pandas
import h5py

# global lock because libhdf5 builds are not thread-safe by default :(
lock = Lock()

class Worker(object):
    """
    Worker to do partial marginalization of a sparse heatmap in Cooler format.

    A worker fetches a set of records representing matrix elements (pixels) and
    uses a weighted bincount algorithm to compute the partial marginal sum of
    the matrix coming from that set.

    The initial weight assigned to each pixel in a chunk is its value. The
    initializer accepts a sequence of "filters" or transformations to
    sequentially process the pixel weights before aggregating/marginalizing,
    e.g., to remove unwanted elements by setting their weights to zero.

    Parameters
    ----------
    cooler_path : str
        Path to Cooler HDF5 file. Must be a string and not a file-like object
        when running in parallel because the latter cannot be pickled.

    filters : sequence of callables
        Transformations that the worker will apply on the pixel weights prior
        to bincount. The required signature is ``f(chunk, data_weights)``,
        where ``chunk`` is a dictionary containing the data chunk and bin
        information, and ``data_weights`` is the 1D-array of current pixel
        weights of the chunk. The filter must return the updated array of pixel
        weights.

    Example
    -------
    # Create a worker and register filters f1, f2, f3
    >>> w = Worker('mycooler.coo', [f1, f2, f3])

    # Map to three consecutive chunks
    >>> marg1 = w((0, 1000))
    >>> marg2 = w((1000, 2000))
    >>> marg3 = w((2000, 3000))

    # Reduce the partial marginals into one
    >>> marg = np.sum([marg1, marg2, marg3], axis=0)

    """
    def __init__(self, cooler_path, filters=None):
        self.filepath = cooler_path
        self.filters = filters if filters is not None else []

    def __call__(self, span):
        lo, hi = span
        import numpy as np
        import h5py

        # prepare chunk dict
        chunk = {}
        lock.acquire()
        with h5py.File(self.filepath, 'r') as h5:
            n_bins_total = h5['bins/chrom'].shape[0]
            chunk['bin1_id'] = h5['pixels/bin1_id'][lo:hi]
            chunk['bin2_id'] = h5['pixels/bin2_id'][lo:hi]
            chunk['count'] = h5['pixels/count'][lo:hi]
            chunk['bintable'] = {
                'chrom_id':  h5['bins/chrom'][:],
                'start':  h5['bins/start'][:],
                'end':   h5['bins/end'][:],
            }
        lock.release()

        # apply filters to chunk
        data_weights = chunk['count']
        for filter_ in self.filters:
            data_weights = filter_(chunk, data_weights)

        # marginalize
        marg = np.zeros(n_bins_total, dtype=float)
        marg += np.bincount(
            chunk['bin1_id'], weights=data_weights, minlength=n_bins_total)
        marg += np.bincount(
            chunk['bin2_id'], weights=data_weights, minlength=n_bins_total)
        return marg

class BinarizeFilter(object):
    def __call__(self, chunk, data_weights):
        data_weights[data_weights != 0] = 1
        return data_weights

class DropDiagFilter(object):
    def __init__(self, n_diags):
        self.n_diags = n_diags
    def __call__(self, chunk, data_weights):
        mask = np.abs(chunk['bin1_id'] - chunk['bin2_id']) < self.n_diags
        data_weights[mask] = 0
        return data_weights

class CisOnlyFilter(object):
    def __call__(self, chunk, data_weights):
        chrom_ids = chunk['bintable']['chrom_id']
        mask = chrom_ids[chunk['bin1_id']] != chrom_ids[chunk['bin2_id']]
        data_weights[mask] = 0
        return data_weights

class TimesOuterProductFilter(object):
    def __init__(self, vec):
        self.vec = vec
    def __call__(self, chunk, data_weights):
        data_weights = (self.vec[chunk['bin1_id']]
                            * self.vec[chunk['bin2_id']]
                            * data_weights)
        return data_weights

def mad(data, axis=None):
    return np.median(np.abs(data - np.median(data, axis)), axis)

def iterative_correction(coo, chunksize=None, map=map, tol=1e-5,
                         min_nnz=0, min_count=0, mad_max=0,
                         cis_only=False, ignore_diags=False,
                         max_iters=200):
    """
    Iterative correction or matrix balancing of a sparse Hi-C contact map in
    Cooler HDF5 format.

    Parameters
    ----------
    coo : h5py.File object
        Cooler file
    chunksize : int, optional
        Split the contact matrix pixel records into equally sized chunks to
        save memory and/or parallelize. Default is to use all the pixels at
        once.
    map : callable, optional
        Map function to dispatch the matrix chunks to workers.
        Default is the builtin ``map``, but alternatives include parallel map
        implementations from a multiprocessing pool.
    tol : float, optional
        Convergence criterion is the variance of the marginal (row/col) sum
        vector.
    min_nnz : int, optional
        Pre-processing bin-level filter. Drop bins with fewer nonzero elements
        than this value.
    min_count : int, optional
        Pre-processing bin-level filter. Drop bins with lower marginal sum than
        this value.
    mad_max : int, optional
        Pre-processing bin-level filter. Drop bins whose log marginal sum is
        less than ``mad_max`` mean absolute deviations below the median log
        marginal sum.
    cis_only: bool, optional
        Do iterative correction on intra-chromosomal data only.
        Inter-chromosomal data is ignored.
    ignore_diags : int or False, optional
        Drop elements occurring on the first ``ignore_diags`` diagonals of the
        matrix.
    max_iters : int, optional
        Iteration limit.

    Returns
    -------
    bias : 1D array, whose shape is the number of bins in ``coo``.
        Vector of bin bias weights to normalize the observed contact map.
        Dropped bins will be assigned the value NaN.
        N[i, j] = O[i, j] * bias[i] * bias[j]

    """

    # Divide the number of elements into non-overlapping chunks
    nnz = coo.attrs['nnz']
    if chunksize is None:
        spans = [(0, nnz)]
    else:
        edges = np.arange(0, nnz+chunksize, chunksize)
        spans = list(zip(edges[:-1], edges[1:]))

    # List of pre-marginalization data transformations
    base_filters = []
    if cis_only:
        base_filters.append(CisOnlyFilter())
    if ignore_diags:
        base_filters.append(DropDiagFilter(ignore_diags))

    # Initialize the bias weights
    n_bins = coo.attrs['nbins']
    bias = np.ones(n_bins, dtype=float)

    # Drop bins with too few nonzeros from bias
    if min_nnz > 0:
        filters = [BinarizeFilter()] + base_filters
        marg_partials = map(Worker(coo.filename, filters), spans)
        marg_nnz = np.sum(list(marg_partials), axis=0)
        bias[marg_nnz < min_nnz] = 0

    filters = base_filters
    marg_partials = map(Worker(coo.filename, filters), spans)
    marg = np.sum(list(marg_partials), axis=0)

    # Drop bins with too few total counts from bias
    if min_count:
        bias[marg < min_count] = 0

    # MAD-max filter on the marginals
    if mad_max > 0:
        offsets = coo['indexes']['chrom_offset'][:]
        for lo, hi in zip(offsets[:-1], offsets[1:]):
            c_marg = marg[lo:hi]
            marg[lo:hi] /= np.median(c_marg[c_marg > 0])
        logNzMarg = np.log(marg[marg>0])
        logMedMarg = np.median(logNzMarg)
        madSigma = mad(logNzMarg) / 0.6745
        cutoff = np.exp(logMedMarg - mad_max * madSigma)
        bias[marg < cutoff] = 0

    # Do balancing
    for _ in range(max_iters):
        filters = base_filters + [TimesOuterProductFilter(bias)]
        worker = Worker(coo.filename, filters)
        marg_partials = map(worker, spans)
        marg = np.sum(list(marg_partials), axis=0)

        marg_ = marg[marg != 0]
        m = marg / marg_.mean()
        m[m == 0] = 1
        bias /= m

        var = marg_.var()
        print("variance is", var)
        if var < tol:
            bias[bias==0] = np.nan
            break
    else:
        warnings.warn('Iteration limit reached without convergence.')

    return bias
<commit_msg>Make iterative correction compatible with nested Cooler tree<commit_after># -*- coding: utf-8 -*-
from __future__ import division, print_function
from multiprocessing import Pool, Lock
import warnings
import six

import numpy as np
import pandas
import h5py

# Lock prevents race condition if HDF5 file is already open before forking.
# See discussion: <https://groups.google.com/forum/#!topic/h5py/bJVtWdFtZQM>
lock = Lock()

class Worker(object):
    """
    Worker to do partial marginalization of a sparse heatmap in Cooler format.

    A worker fetches a set of records representing matrix elements (pixels) and
    uses a weighted bincount algorithm to compute the partial marginal sum of
    the matrix coming from that set.

    The initial weight assigned to each pixel in a chunk is its value. The
    initializer accepts a sequence of "filters" or transformations to
    sequentially process the pixel weights before aggregating/marginalizing,
    e.g., to remove unwanted elements by setting their weights to zero.

    Parameters
    ----------
    cooler_path : str
        Path to Cooler HDF5 file. Must be a string and not a file-like object
        when running in parallel because the latter cannot be pickled.

    cooler_root : str
        HDF5 path to root group of a Cooler tree.

    filters : sequence of callables
        Transformations that the worker will apply on the pixel weights prior
        to bincount. The required signature is ``f(chunk, data_weights)``,
        where ``chunk`` is a dictionary containing the data chunk and bin
        information, and ``data_weights`` is the 1D-array of current pixel
        weights of the chunk. The filter must return the updated array of pixel
        weights.

    Example
    -------
    # Create a worker and register filters f1, f2, f3
    >>> w = Worker('mycooler.coo', [f1, f2, f3])

    # Map to three consecutive chunks
    >>> marg1 = w((0, 1000))
    >>> marg2 = w((1000, 2000))
    >>> marg3 = w((2000, 3000))

    # Reduce the partial marginals into one
    >>> marg = np.sum([marg1, marg2, marg3], axis=0)

    """
    def __init__(self, cooler_path, cooler_root, filters=None):
        self.filepath = cooler_path
        self.root = cooler_root
        self.filters = filters if filters is not None else []

    def __call__(self, span):
        lo, hi = span
        import numpy as np
        import h5py

        # prepare chunk dict
        chunk = {}
        lock.acquire()
        with h5py.File(self.filepath, 'r') as h5:
            coo = h5[self.root]
            n_bins_total = coo['bins/chrom'].shape[0]
            chunk['bin1_id'] = coo['pixels/bin1_id'][lo:hi]
            chunk['bin2_id'] = coo['pixels/bin2_id'][lo:hi]
            chunk['count']   = coo['pixels/count'][lo:hi]
            chunk['bintable'] = {
                'chrom_id':  coo['bins/chrom'][:],
                'start':     coo['bins/start'][:],
                'end':       coo['bins/end'][:],
            }
        lock.release()

        # apply filters to chunk
        data_weights = chunk['count']
        for filter_ in self.filters:
            data_weights = filter_(chunk, data_weights)

        # marginalize
        marg = np.zeros(n_bins_total, dtype=float)
        marg += np.bincount(
            chunk['bin1_id'], weights=data_weights, minlength=n_bins_total)
        marg += np.bincount(
            chunk['bin2_id'], weights=data_weights, minlength=n_bins_total)
        return marg

class BinarizeFilter(object):
    def __call__(self, chunk, data_weights):
        data_weights[data_weights != 0] = 1
        return data_weights

class DropDiagFilter(object):
    def __init__(self, n_diags):
        self.n_diags = n_diags
    def __call__(self, chunk, data_weights):
        mask = np.abs(chunk['bin1_id'] - chunk['bin2_id']) < self.n_diags
        data_weights[mask] = 0
        return data_weights

class CisOnlyFilter(object):
    def __call__(self, chunk, data_weights):
        chrom_ids = chunk['bintable']['chrom_id']
        mask = chrom_ids[chunk['bin1_id']] != chrom_ids[chunk['bin2_id']]
        data_weights[mask] = 0
        return data_weights

class TimesOuterProductFilter(object):
    def __init__(self, vec):
        self.vec = vec
    def __call__(self, chunk, data_weights):
        data_weights = (self.vec[chunk['bin1_id']]
                            * self.vec[chunk['bin2_id']]
                            * data_weights)
        return data_weights

def mad(data, axis=None):
    return np.median(np.abs(data - np.median(data, axis)), axis)

def iterative_correction(h5, cooler_root='/', chunksize=None, map=map, tol=1e-5,
                         min_nnz=0, min_count=0, mad_max=0,
                         cis_only=False, ignore_diags=False,
                         max_iters=200):
    """
    Iterative correction or matrix balancing of a sparse Hi-C contact map in
    Cooler HDF5 format.

    Parameters
    ----------
    h5 : h5py.File object
        Cooler file
    cooler_root : str, optional
        Path of the root node of a cooler tree. Default is the file root, '/'.
    chunksize : int, optional
        Split the contact matrix pixel records into equally sized chunks to
        save memory and/or parallelize. Default is to use all the pixels at
        once.
    map : callable, optional
        Map function to dispatch the matrix chunks to workers.
        Default is the builtin ``map``, but alternatives include parallel map
        implementations from a multiprocessing pool.
    tol : float, optional
        Convergence criterion is the variance of the marginal (row/col) sum
        vector.
    min_nnz : int, optional
        Pre-processing bin-level filter. Drop bins with fewer nonzero elements
        than this value.
    min_count : int, optional
        Pre-processing bin-level filter. Drop bins with lower marginal sum than
        this value.
    mad_max : int, optional
        Pre-processing bin-level filter. Drop bins whose log marginal sum is
        less than ``mad_max`` mean absolute deviations below the median log
        marginal sum.
    cis_only: bool, optional
        Do iterative correction on intra-chromosomal data only.
        Inter-chromosomal data is ignored.
    ignore_diags : int or False, optional
        Drop elements occurring on the first ``ignore_diags`` diagonals of the
        matrix (including the main diagonal).
    max_iters : int, optional
        Iteration limit.

    Returns
    -------
    bias : 1D array, whose shape is the number of bins in ``h5``.
        Vector of bin bias weights to normalize the observed contact map.
        Dropped bins will be assigned the value NaN.
        N[i, j] = O[i, j] * bias[i] * bias[j]

    """
    filepath = h5.file.filename

    # Divide the number of elements into non-overlapping chunks
    nnz = h5[cooler_root].attrs['nnz']
    if chunksize is None:
        spans = [(0, nnz)]
    else:
        edges = np.arange(0, nnz+chunksize, chunksize)
        spans = list(zip(edges[:-1], edges[1:]))

    # List of pre-marginalization data transformations
    base_filters = []
    if cis_only:
        base_filters.append(CisOnlyFilter())
    if ignore_diags:
        base_filters.append(DropDiagFilter(ignore_diags))

    # Initialize the bias weights
    n_bins = h5[cooler_root].attrs['nbins']
    bias = np.ones(n_bins, dtype=float)

    # Drop bins with too few nonzeros from bias
    if min_nnz > 0:
        filters = [BinarizeFilter()] + base_filters
        marg_partials = map(Worker(filepath, cooler_root, filters), spans)
        marg_nnz = np.sum(list(marg_partials), axis=0)
        bias[marg_nnz < min_nnz] = 0

    filters = base_filters
    marg_partials = map(Worker(filepath, cooler_root, filters), spans)
    marg = np.sum(list(marg_partials), axis=0)

    # Drop bins with too few total counts from bias
    if min_count:
        bias[marg < min_count] = 0

    # MAD-max filter on the marginals
    if mad_max > 0:
        offsets = h5[cooler_root]['indexes']['chrom_offset'][:]
        for lo, hi in zip(offsets[:-1], offsets[1:]):
            c_marg = marg[lo:hi]
            marg[lo:hi] /= np.median(c_marg[c_marg > 0])
        logNzMarg = np.log(marg[marg>0])
        logMedMarg = np.median(logNzMarg)
        madSigma = mad(logNzMarg) / 0.6745
        cutoff = np.exp(logMedMarg - mad_max * madSigma)
        bias[marg < cutoff] = 0

    # Do balancing
    for _ in range(max_iters):
        filters = base_filters + [TimesOuterProductFilter(bias)]
        worker = Worker(filepath, cooler_root, filters)
        marg_partials = map(worker, spans)
        marg = np.sum(list(marg_partials), axis=0)

        marg_ = marg[marg != 0]
        m = marg / marg_.mean()
        m[m == 0] = 1
        bias /= m

        var = marg_.var()
        print("variance is", var)
        if var < tol:
            bias[bias==0] = np.nan
            break
    else:
        warnings.warn('Iteration limit reached without convergence.')

    return bias
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
from __future__ import division

import os
import sys
import glob
import subprocess
import re
import collections
import socket
from itertools import *
import pprint; pp = pprint.pprint

# this shit needs astats, which didn't exist in 2013
if socket.gethostname() == 'video-main':
	ffmpeg = '/home/crackwitz/portalhome/.local/bin/ffmpeg'
else:
	ffmpeg = 'ffmpeg'

def common_prefix(strings):
	def sub(a, b):
		diff = [i for i,(u,v) in enumerate(zip(a,b)) if u != v]
		return a[:min(diff)]

	return reduce(sub, strings)

def get_astats(fname):
	output = subprocess.check_output([ffmpeg, '-i', fname, '-filter:a', 'astats', '-vn', '-f', 'null', 'dummy'], stderr=subprocess.STDOUT)

	lines = (line.rstrip() for line in output.split('\n') if 'Parsed_astats_0' in line)

	lines = dropwhile(lambda line: "Overall" not in line, lines)

	lines = (re.match(r'\[[^]]*\] ([^:]+): (.*)$', line) for line in lines)

	lines = (line.groups() for line in lines if line)

	return dict(lines)

def get_rms_peak(fname):
	astats = get_astats(fname)
	rmspeak = float(astats['RMS peak dB'])
	return rmspeak

def transcode(inputs, output, dar, gain=0):
	command = [ffmpeg, '-async', '1']

	for i in inputs: command += ['-i', i]

	chains = []

	chains += [
		'[{0}:v] setdar=dar={1}/{2} [{0}aspect]'.format(i, dar[0], dar[1])
		for i in xrange(len(inputs))
	]

	# assuming 1 video and 1 audio stream per file
	chains.append(
		# inputs
		' '.join("[{0}aspect][{0}:a] ".format(i) for i,inp in enumerate(inputs)) +
		# body
		'concat=n={0}:v=1:a=1'.format(len(inputs)) +
		# outputs
		' [vj][aj]'
	)

	chains.append(
		'[aj] volume=volume={0:+f}dB [ag]'.format(gain)
	)

	command += ['-filter_complex', '; '.join(chains)]

	command += ['-map', '[vj]']
	command += ['-map', '[ag]']

	command += ['-c:a', 'libvo_aacenc', '-b:a', '64k']
	command += ['-c:v', 'libx264', '-profile:v', 'main', '-crf', '23', '-g', '125']
	command += ['-aspect:v', '{0}:{1}'.format(dar[0], dar[1])]
	command += ['-movflags', 'faststart']
	#command += ['-threads', '4']
	command += [output]

	print command
	rv = subprocess.call(command)

	return rv

def group_files(inpattern, outpattern, files):
	buckets = collections.defaultdict(lambda: [])

	for f in files:
		m = re.match(inpattern, f)
		assert m, (inpattern, f)
		groups = m.groups()
		outkey = outpattern.format(f, *groups)
		buckets[outkey].append(f)

	return dict(buckets)

if __name__ == '__main__':
	settings = {
		'target': [-9.0,   lambda s: float(s)],
		'dar':    [(4,3),  lambda s: tuple(map(int, s.split(':')))],
		'in':     [None,   lambda s: s],
		'out':    [None,   lambda s: s],
		'skip':   [None,   lambda s: eval(s)],
	}

	while sys.argv[1:] and '=' in sys.argv[1]:
		arg = sys.argv.pop(1)
		(k,v) = arg.split('=', 1)
		assert k in settings
		settings[k][0] = settings[k][1](v)

	assert settings['in'][0] is not None
	assert settings['out'][0] is not None

	infiles = []
	for arg in sys.argv[1:]:
		infiles += glob.glob(arg) or [arg]

	groups = group_files(settings['in'][0], settings['out'][0], infiles)

	pp(groups)

	if raw_input("proceed? ").strip() not in ('y', ''):
		sys.exit(-1)

	for outfile in sorted(groups):
		print

		if os.path.exists(outfile):
			if settings['skip'][0] is None:
				print "<- {0}".format(outfile)
				if raw_input("file exists. skip? (y/n) ") in ("", 'y'):
					continue
			elif settings['skip'][0]:
				continue
			elif not settings['skip'][0]:
				pass

		infiles = groups[outfile]

		peaks = []
		for f in infiles:
			peak = get_rms_peak(f)
			peaks.append(peak)
			print "-> {0}, peak {1:+.3f} dB".format(f, peak)

		peak = max(peaks)
		print "<- {0} peak, {1:+.3f} dB".format(outfile, peak)

		gain  = settings['target'][0] - peak
		if gain <= 0: gain = 0
		print "   gain {1:+.3f} dB".format(peak, gain)

		rv = transcode(infiles, outfile, gain=gain, dar=settings['dar'][0])
		assert rv == 0

# sinf-transcode.py -9.0 'source/2003-SS-CG1.([VU]\d+)([ab]?).20(\d{2})-(\d{2})-(\d{2}).rmvb' '03ss-cg1-{3}{4}{5}-{1}-sd.mp4' source/*.rmvb
<commit_msg>sinf-transcode: back to crf 20, close enough to 1M for 720p<commit_after>#!/usr/bin/env python2.7
from __future__ import division

import os
import sys
import glob
import subprocess
import re
import collections
import socket
from itertools import *
import pprint; pp = pprint.pprint

# this shit needs astats, which didn't exist in 2013
if socket.gethostname() == 'video-main':
	ffmpeg = '/home/crackwitz/portalhome/.local/bin/ffmpeg'
else:
	ffmpeg = 'ffmpeg'

def common_prefix(strings):
	def sub(a, b):
		diff = [i for i,(u,v) in enumerate(zip(a,b)) if u != v]
		return a[:min(diff)]

	return reduce(sub, strings)

def get_astats(fname):
	output = subprocess.check_output([ffmpeg, '-i', fname, '-filter:a', 'astats', '-vn', '-f', 'null', 'dummy'], stderr=subprocess.STDOUT)

	lines = (line.rstrip() for line in output.split('\n') if 'Parsed_astats_0' in line)

	lines = dropwhile(lambda line: "Overall" not in line, lines)

	lines = (re.match(r'\[[^]]*\] ([^:]+): (.*)$', line) for line in lines)

	lines = (line.groups() for line in lines if line)

	return dict(lines)

def get_rms_peak(fname):
	astats = get_astats(fname)
	rmspeak = float(astats['RMS peak dB'])
	return rmspeak

def transcode(inputs, output, dar, gain=0):
	command = [ffmpeg, '-async', '1']

	for i in inputs: command += ['-i', i]

	chains = []

	chains += [
		'[{0}:v] setdar=dar={1}/{2} [{0}aspect]'.format(i, dar[0], dar[1])
		for i in xrange(len(inputs))
	]

	# assuming 1 video and 1 audio stream per file
	chains.append(
		# inputs
		' '.join("[{0}aspect][{0}:a] ".format(i) for i,inp in enumerate(inputs)) +
		# body
		'concat=n={0}:v=1:a=1'.format(len(inputs)) +
		# outputs
		' [vj][aj]'
	)

	chains.append(
		'[aj] volume=volume={0:+f}dB [ag]'.format(gain)
	)

	command += ['-filter_complex', '; '.join(chains)]

	command += ['-map', '[vj]']
	command += ['-map', '[ag]']

	command += ['-c:a', 'libvo_aacenc', '-b:a', '64k']
	command += ['-c:v', 'libx264', '-profile:v', 'main', '-crf', '20', '-g', '125']
	command += ['-aspect:v', '{0}:{1}'.format(dar[0], dar[1])]
	command += ['-movflags', 'faststart']
	#command += ['-threads', '4']
	command += [output]

	print command
	rv = subprocess.call(command)

	return rv

def group_files(inpattern, outpattern, files):
	buckets = collections.defaultdict(lambda: [])

	for f in files:
		m = re.match(inpattern, f)
		assert m, (inpattern, f)
		groups = m.groups()
		outkey = outpattern.format(f, *groups)
		buckets[outkey].append(f)

	return dict(buckets)

if __name__ == '__main__':
	settings = {
		'target': [-9.0,   lambda s: float(s)],
		'dar':    [(4,3),  lambda s: tuple(map(int, s.split(':')))],
		'in':     [None,   lambda s: s],
		'out':    [None,   lambda s: s],
		'skip':   [None,   lambda s: eval(s)],
	}

	while sys.argv[1:] and '=' in sys.argv[1]:
		arg = sys.argv.pop(1)
		(k,v) = arg.split('=', 1)
		assert k in settings
		settings[k][0] = settings[k][1](v)

	assert settings['in'][0] is not None
	assert settings['out'][0] is not None

	infiles = []
	for arg in sys.argv[1:]:
		infiles += glob.glob(arg) or [arg]

	groups = group_files(settings['in'][0], settings['out'][0], infiles)

	pp(groups)

	if raw_input("proceed? ").strip() not in ('y', ''):
		sys.exit(-1)

	for outfile in sorted(groups):
		print

		if os.path.exists(outfile):
			if settings['skip'][0] is None:
				print "<- {0}".format(outfile)
				if raw_input("file exists. skip? (y/n) ") in ("", 'y'):
					continue
			elif settings['skip'][0]:
				continue
			elif not settings['skip'][0]:
				pass

		infiles = groups[outfile]

		peaks = []
		for f in infiles:
			peak = get_rms_peak(f)
			peaks.append(peak)
			print "-> {0}, peak {1:+.3f} dB".format(f, peak)

		peak = max(peaks)
		print "<- {0} peak, {1:+.3f} dB".format(outfile, peak)

		gain  = settings['target'][0] - peak
		if gain <= 0: gain = 0
		print "   gain {1:+.3f} dB".format(peak, gain)

		rv = transcode(infiles, outfile, gain=gain, dar=settings['dar'][0])
		assert rv == 0

# sinf-transcode.py -9.0 'source/2003-SS-CG1.([VU]\d+)([ab]?).20(\d{2})-(\d{2})-(\d{2}).rmvb' '03ss-cg1-{3}{4}{5}-{1}-sd.mp4' source/*.rmvb
<|endoftext|>


<commit_before>from collections import OrderedDict
from collections import defaultdict

from flask import render_template
from sqlalchemy import and_
from sqlalchemy import func

from tracker import db
from tracker import tracker
from tracker.model import CVE
from tracker.model import Advisory
from tracker.model import CVEGroup
from tracker.model import CVEGroupEntry
from tracker.model import CVEGroupPackage
from tracker.model import Package
from tracker.model.enum import Publication
from tracker.model.enum import Status
from tracker.util import json_response

def get_index_data(only_vulnerable=False, only_in_repo=True):
    select = (db.session.query(CVEGroup, CVE, func.group_concat(CVEGroupPackage.pkgname, ' '),
                               func.group_concat(Advisory.id, ' '))
                        .join(CVEGroupEntry, CVEGroup.issues)
                        .join(CVE, CVEGroupEntry.cve)
                        .join(CVEGroupPackage, CVEGroup.packages)
                        .outerjoin(Advisory, and_(Advisory.group_package_id == CVEGroupPackage.id,
                                                  Advisory.publication == Publication.published)))
    if only_vulnerable:
        select = select.filter(CVEGroup.status.in_([Status.unknown, Status.vulnerable, Status.testing]))
    if only_in_repo:
        select = select.join(Package, Package.name == CVEGroupPackage.pkgname)

    entries = (select.group_by(CVEGroup.id).group_by(CVE.id)
                     .order_by(CVEGroup.status.desc())
                     .order_by(CVEGroup.created.desc())).all()

    groups = defaultdict(defaultdict)
    for group, cve, pkgs, advisories in entries:
        group_entry = groups.setdefault(group.id, {})
        group_entry['group'] = group
        group_entry['pkgs'] = list(set(pkgs.split(' ')))
        group_entry['advisories'] = advisories.split(' ') if advisories else []
        group_entry.setdefault('issues', []).append(cve)

    for key, group in groups.items():
        group['issues'] = sorted(group['issues'], key=lambda item: item.id, reverse=True)

    groups = groups.values()
    groups = sorted(groups, key=lambda item: item['group'].created, reverse=True)
    groups = sorted(groups, key=lambda item: item['group'].severity)
    groups = sorted(groups, key=lambda item: item['group'].status)
    return groups

@tracker.route('/', defaults={'path': '', 'only_vulnerable': True}, methods=['GET'])
def index(only_vulnerable=True, path=None):
    groups = get_index_data(only_vulnerable)
    return render_template('index.html',
                           title='Issues' if not only_vulnerable else 'Vulnerable issues',
                           entries=groups,
                           only_vulnerable=only_vulnerable)

@tracker.route('/<regex("(issues?/)?(open|vulnerable)?"):path>', defaults={'path': ''}, methods=['GET'])
def index_vulnerable(path=None):
    return index(only_vulnerable=True)

@tracker.route('/<regex("(issues/)?(all)?"):path>', defaults={'path': ''}, methods=['GET'])
def index_all(path=None):
    return index(only_vulnerable=False)

# TODO: temporarily keep /json this way until tools adopted new endpoint
@tracker.route('/json', defaults={'path': 'json', 'only_vulnerable': False}, methods=['GET'])
@tracker.route('/<regex("(issues/?)?(all)?.json"):path>', defaults={'path': 'all.json', 'only_vulnerable': False}, methods=['GET'])
@json_response
def index_json(only_vulnerable=False, path=None):
    entries = get_index_data(only_vulnerable)
    json_data = []
    for entry in entries:
        group = entry['group']
        types = list(set([cve.issue_type for cve in entry['issues']]))

        json_entry = OrderedDict()
        json_entry['name'] = group.name
        json_entry['packages'] = entry['pkgs']
        json_entry['status'] = group.status.label
        json_entry['severity'] = group.severity.label
        json_entry['type'] = 'multiple issues' if len(types) > 1 else types[0]
        json_entry['affected'] = group.affected
        json_entry['fixed'] = group.fixed if group.fixed else None
        json_entry['ticket'] = group.bug_ticket if group.bug_ticket else None
        json_entry['issues'] = [str(cve) for cve in entry['issues']]
        json_entry['advisories'] = entry['advisories']
        json_data.append(json_entry)
    return json_data

@tracker.route('/<regex("(issues/?)?(open|vulnerable).json"):path>', methods=['GET'])
def index_vulnerable_json(path=None):
    return index_json(only_vulnerable=True)
<commit_msg>index: order by changed date to put updated groups on top<commit_after>from collections import OrderedDict
from collections import defaultdict

from flask import render_template
from sqlalchemy import and_
from sqlalchemy import func

from tracker import db
from tracker import tracker
from tracker.model import CVE
from tracker.model import Advisory
from tracker.model import CVEGroup
from tracker.model import CVEGroupEntry
from tracker.model import CVEGroupPackage
from tracker.model import Package
from tracker.model.enum import Publication
from tracker.model.enum import Status
from tracker.util import json_response

def get_index_data(only_vulnerable=False, only_in_repo=True):
    select = (db.session.query(CVEGroup, CVE, func.group_concat(CVEGroupPackage.pkgname, ' '),
                               func.group_concat(Advisory.id, ' '))
                        .join(CVEGroupEntry, CVEGroup.issues)
                        .join(CVE, CVEGroupEntry.cve)
                        .join(CVEGroupPackage, CVEGroup.packages)
                        .outerjoin(Advisory, and_(Advisory.group_package_id == CVEGroupPackage.id,
                                                  Advisory.publication == Publication.published)))
    if only_vulnerable:
        select = select.filter(CVEGroup.status.in_([Status.unknown, Status.vulnerable, Status.testing]))
    if only_in_repo:
        select = select.join(Package, Package.name == CVEGroupPackage.pkgname)

    entries = (select.group_by(CVEGroup.id).group_by(CVE.id)
                     .order_by(CVEGroup.status.desc())
                     .order_by(CVEGroup.changed.desc())).all()

    groups = defaultdict(defaultdict)
    for group, cve, pkgs, advisories in entries:
        group_entry = groups.setdefault(group.id, {})
        group_entry['group'] = group
        group_entry['pkgs'] = list(set(pkgs.split(' ')))
        group_entry['advisories'] = advisories.split(' ') if advisories else []
        group_entry.setdefault('issues', []).append(cve)

    for key, group in groups.items():
        group['issues'] = sorted(group['issues'], key=lambda item: item.id, reverse=True)

    groups = groups.values()
    groups = sorted(groups, key=lambda item: item['group'].changed, reverse=True)
    groups = sorted(groups, key=lambda item: item['group'].severity)
    groups = sorted(groups, key=lambda item: item['group'].status)
    return groups

@tracker.route('/', defaults={'path': '', 'only_vulnerable': True}, methods=['GET'])
def index(only_vulnerable=True, path=None):
    groups = get_index_data(only_vulnerable)
    return render_template('index.html',
                           title='Issues' if not only_vulnerable else 'Vulnerable issues',
                           entries=groups,
                           only_vulnerable=only_vulnerable)

@tracker.route('/<regex("(issues?/)?(open|vulnerable)?"):path>', defaults={'path': ''}, methods=['GET'])
def index_vulnerable(path=None):
    return index(only_vulnerable=True)

@tracker.route('/<regex("(issues/)?(all)?"):path>', defaults={'path': ''}, methods=['GET'])
def index_all(path=None):
    return index(only_vulnerable=False)

# TODO: temporarily keep /json this way until tools adopted new endpoint
@tracker.route('/json', defaults={'path': 'json', 'only_vulnerable': False}, methods=['GET'])
@tracker.route('/<regex("(issues/?)?(all)?.json"):path>', defaults={'path': 'all.json', 'only_vulnerable': False}, methods=['GET'])
@json_response
def index_json(only_vulnerable=False, path=None):
    entries = get_index_data(only_vulnerable)
    json_data = []
    for entry in entries:
        group = entry['group']
        types = list(set([cve.issue_type for cve in entry['issues']]))

        json_entry = OrderedDict()
        json_entry['name'] = group.name
        json_entry['packages'] = entry['pkgs']
        json_entry['status'] = group.status.label
        json_entry['severity'] = group.severity.label
        json_entry['type'] = 'multiple issues' if len(types) > 1 else types[0]
        json_entry['affected'] = group.affected
        json_entry['fixed'] = group.fixed if group.fixed else None
        json_entry['ticket'] = group.bug_ticket if group.bug_ticket else None
        json_entry['issues'] = [str(cve) for cve in entry['issues']]
        json_entry['advisories'] = entry['advisories']
        json_data.append(json_entry)
    return json_data

@tracker.route('/<regex("(issues/?)?(open|vulnerable).json"):path>', methods=['GET'])
def index_vulnerable_json(path=None):
    return index_json(only_vulnerable=True)
<|endoftext|>


<commit_before>"""
Display Volume Data
===================

PySurfer provides a function to sample a volume-encoded file
to the surface for improved visualization. At the moment, this
uses Freesurfer's mri_vol2surf routine.

"""
print __doc__

from surfer import Brain, io

"""Bring up the visualization"""
brain = Brain("fsaverage", "lh", "inflated",
              config_opts=dict(background="white"))

"""Project the volume file and return as an array"""
mri_file = "auto_examples/data/resting_corr.nii.gz"
reg_file = "auto_examples/data/register.dat"
surf_data = io.project_volume_data(mri_file, "lh", reg_file)

"""
You can pass this array to the add_overlay method for
a typical activation overlay (with thresholding, etc.)
"""
brain.add_overlay(surf_data, min=.3, max=.7, name="ang_corr")

"""
You can also pass it to add_data for more control
over the visualzation. Here we'll plot the whole
range of correlations
"""
brain.overlays["ang_corr"].remove()
brain.add_data(surf_data, -.7, .7, "jet", alpha=.7)

"""
This overlay represents resting-state correlations with a
seed in left angular gyrus. Let's plot that seed.
"""
seed_coords = (-45, -67, 36)
brain.add_foci(seed_coords, map_surface="white", scale_factor=2)
<commit_msg>DOC change to volume plot to reflect foci changes<commit_after>"""
Display Volume Data
===================

PySurfer provides a function to sample a volume-encoded file
to the surface for improved visualization. At the moment, this
uses Freesurfer's mri_vol2surf routine.

"""
print __doc__

from surfer import Brain, io

"""Bring up the visualization"""
brain = Brain("fsaverage", "lh", "inflated",
              config_opts=dict(background="white"))

"""Project the volume file and return as an array"""
mri_file = "auto_examples/data/resting_corr.nii.gz"
reg_file = "auto_examples/data/register.dat"
surf_data = io.project_volume_data(mri_file, "lh", reg_file)

"""
You can pass this array to the add_overlay method for
a typical activation overlay (with thresholding, etc.)
"""
brain.add_overlay(surf_data, min=.3, max=.7, name="ang_corr")

"""
You can also pass it to add_data for more control
over the visualzation. Here we'll plot the whole
range of correlations
"""
brain.overlays["ang_corr"].remove()
brain.add_data(surf_data, -.7, .7, "jet", alpha=.7)

"""
This overlay represents resting-state correlations with a
seed in left angular gyrus. Let's plot that seed.
"""
seed_coords = (-45, -67, 36)
brain.add_foci(seed_coords, map_surface="white")
<|endoftext|>


<commit_before>import json
import logging
import queue
import socket
import threading

"""
Exception thrown if the peer sends an error.
"""
class RpcException(Exception):
    pass

"""
Connects to a remote RPC peer.
"""
def connect(address, port, handler = {}):
    connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    connection.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
    connection.connect((address, port))

    listener = Listener(connection, handler)
    return Proxy(listener)

"""
Object that acts as a stub for remote methods.

A proxy object is usually created by calling connect() to connect to a remote
peer.

The proxy also acts as a state object for the current connection, since the
object lasts as long as the connection does.
"""
class Proxy:
    """
    Creates a new proxy object around the given listener.
    """
    def __init__(self, listener):
        self.listener = listener
        self.listener.start()

    """
    Gets the name of the remote peer as a tuple of the address and port.
    """
    def get_peer_name(self):
        return self.listener.address

    """
    Calls a method on the peer.
    """
    def invoke(self, name, params = {}):
        request = {
            "jsonrpc": "2.0",
            "method": name,
            "params": params
        }

        # Send the request and then read the response.
        self.listener.send(request)
        response = self.listener.receive_and_wait()

        if "error" in response:
            raise RpcException(response["error"]["code"], response["error"]["message"])

        return response["result"]

    """
    Informs the peer that the connection is closing and then closes the socket.
    """
    def close(self):
        self.invoke("close")
        self.listener.close()

    """
    Provides syntax sugar for calling remote methods.
    """
    def __getattr__(self, name):
        def closure(**kwargs):
            return self.invoke(name, kwargs)

        return closure

"""
Wrapper around a socket that provides message parsing, writing, and background
reading.

Uses a dedicated thread for reading and handling incoming messages from the
remote peer.
"""
class Listener(threading.Thread):
    """
    Creates a new listener for a given socket.
    """
    def __init__(self, socket, handler, timeout = 1.0, bufsize = 4096):
        threading.Thread.__init__(self)

        self.message_queue = queue.Queue()
        self.open = False
        self.socket = socket
        self.handler = handler() # Instantiate the handler class
        self.bufsize = bufsize

        socket.settimeout(timeout)
        self.address = socket.getpeername()

    """
    Receives the next message if available, or raises an Empty exception.

    Returns the message as a dictionary.
    """
    def receive(self):
        # Attempt to get a message.
        message = self.message_queue.get(False)
        # If no exception was thrown, we can use the result.
        self.message_queue.task_done()
        return message

    """
    Receives the next message, waiting until it arrives.

    Returns the message as a dictionary.
    """
    def receive_and_wait(self):
        # Block until the item in the queue is accessible.
        message = self.message_queue.get(True)
        # Mark it as received.
        self.message_queue.task_done()
        return message

    """
    Sends a message to the peer and waits for the response.
    """
    def send(self, message):
        # Serialize the message.
        serialized = json.dumps(message)
        # Send a packet ending in a magic byte package separator.
        self.socket.sendall(serialized.encode() + b"\0\0\0\0")

    """
    Runs the listener, which reads from the peer forever until close() is called.
    """
    def run(self):
        # Set up a local byte buffer.
        buffer = bytearray()

        # Read forever until we are told to close.
        self.open = True
        while self.open:
            # Read some data from the peer when it arrives.
            try:
                packet = self.socket.recv(self.bufsize)
                buffer.extend(packet)
            except socket.timeout:
                continue
            except (ConnectionError, OSError):
                self.close()
                break

            if len(packet) == 0:
                self.close()
                break

            # Look for any complete messages in the buffer.
            while True:
                # Look for the "magic string" at the end of each message.
                pos = buffer.find(b"\0\0\0\0")
                if pos < 0:
                    break

                # Slice the correct portion out of the buffer.
                string = buffer[:pos].decode()
                del buffer[:pos + 4]

                # Parse the message as JSON.
                message = json.loads(string)
                logging.info("Received message from %s", self.address)

                if not "method" in message:
                    # If the message is a response, put it into the shared queue to alert consumers.
                    self.message_queue.put(message)
                else:
                    # The message is a request, so handle it now.
                    self._handle_request(message)

        # Make sure all consumer threads pick out their responses before we clean up.
        self.message_queue.join()

        # Make sure the socket gets closed if we were asked politely to close.
        self.close()

    """
    Closes the listener politely whenever the loop runs around next.
    """
    def close_later(self):
        self.open = False

    """
    Closes the listener immediately.
    """
    def close(self):
        self.open = False
        try:
            self.socket.shutdown(socket.SHUT_RDWR)
            self.socket.close()
        except OSError:
            pass

    def _handle_request(self, request):
        response = {
            "jsonrpc": "2.0"
        }

        # Attempt to invoke the requested method.
        try:
            # If the method is "close", close the connection.
            if request["method"] == "close":
                self.close_later()
                response["result"] = True

            # For any other method, invoke it on the handler.
            else:
                func = getattr(self.handler, request["method"])
                args = request["params"]
                return_val = func(**args)

                response["result"] = return_val
        except Exception as e:
            # Method errored, so respond with an error instead.
            response["error"] = {
                "code": 500,
                "message": str(e)
            }

        # Send back a response to the peer.
        self.send(response)
<commit_msg>Use connection timeout<commit_after>import json
import logging
import queue
import socket
import threading

"""
Exception thrown if the peer sends an error.
"""
class RpcException(Exception):
    pass

"""
Connects to a remote RPC peer.
"""
def connect(address, port, handler = {}, timeout=5):
    connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    connection.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
    connection.settimeout(timeout)
    connection.connect((address, port))

    listener = Listener(connection, handler)
    return Proxy(listener)

"""
Object that acts as a stub for remote methods.

A proxy object is usually created by calling connect() to connect to a remote
peer.

The proxy also acts as a state object for the current connection, since the
object lasts as long as the connection does.
"""
class Proxy:
    """
    Creates a new proxy object around the given listener.
    """
    def __init__(self, listener):
        self.listener = listener
        self.listener.start()

    """
    Gets the name of the remote peer as a tuple of the address and port.
    """
    def get_peer_name(self):
        return self.listener.address

    """
    Calls a method on the peer.
    """
    def invoke(self, name, params = {}):
        request = {
            "jsonrpc": "2.0",
            "method": name,
            "params": params
        }

        # Send the request and then read the response.
        self.listener.send(request)
        response = self.listener.receive_and_wait()

        if "error" in response:
            raise RpcException(response["error"]["code"], response["error"]["message"])

        return response["result"]

    """
    Informs the peer that the connection is closing and then closes the socket.
    """
    def close(self):
        self.invoke("close")
        self.listener.close()

    """
    Provides syntax sugar for calling remote methods.
    """
    def __getattr__(self, name):
        def closure(**kwargs):
            return self.invoke(name, kwargs)

        return closure

"""
Wrapper around a socket that provides message parsing, writing, and background
reading.

Uses a dedicated thread for reading and handling incoming messages from the
remote peer.
"""
class Listener(threading.Thread):
    """
    Creates a new listener for a given socket.
    """
    def __init__(self, socket, handler, timeout = 1.0, bufsize = 4096):
        threading.Thread.__init__(self)

        self.message_queue = queue.Queue()
        self.open = False
        self.socket = socket
        self.handler = handler() # Instantiate the handler class
        self.bufsize = bufsize

        socket.settimeout(timeout)
        self.address = socket.getpeername()

    """
    Receives the next message if available, or raises an Empty exception.

    Returns the message as a dictionary.
    """
    def receive(self):
        # Attempt to get a message.
        message = self.message_queue.get(False)
        # If no exception was thrown, we can use the result.
        self.message_queue.task_done()
        return message

    """
    Receives the next message, waiting until it arrives.

    Returns the message as a dictionary.
    """
    def receive_and_wait(self):
        # Block until the item in the queue is accessible.
        message = self.message_queue.get(True)
        # Mark it as received.
        self.message_queue.task_done()
        return message

    """
    Sends a message to the peer and waits for the response.
    """
    def send(self, message):
        # Serialize the message.
        serialized = json.dumps(message)
        # Send a packet ending in a magic byte package separator.
        self.socket.sendall(serialized.encode() + b"\0\0\0\0")

    """
    Runs the listener, which reads from the peer forever until close() is called.
    """
    def run(self):
        # Set up a local byte buffer.
        buffer = bytearray()

        # Read forever until we are told to close.
        self.open = True
        while self.open:
            # Read some data from the peer when it arrives.
            try:
                packet = self.socket.recv(self.bufsize)
                buffer.extend(packet)
            except socket.timeout:
                continue
            except (ConnectionError, OSError):
                self.close()
                break

            if len(packet) == 0:
                self.close()
                break

            # Look for any complete messages in the buffer.
            while True:
                # Look for the "magic string" at the end of each message.
                pos = buffer.find(b"\0\0\0\0")
                if pos < 0:
                    break

                # Slice the correct portion out of the buffer.
                string = buffer[:pos].decode()
                del buffer[:pos + 4]

                # Parse the message as JSON.
                message = json.loads(string)
                logging.info("Received message from %s", self.address)

                if not "method" in message:
                    # If the message is a response, put it into the shared queue to alert consumers.
                    self.message_queue.put(message)
                else:
                    # The message is a request, so handle it now.
                    self._handle_request(message)

        # Make sure all consumer threads pick out their responses before we clean up.
        self.message_queue.join()

        # Make sure the socket gets closed if we were asked politely to close.
        self.close()

    """
    Closes the listener politely whenever the loop runs around next.
    """
    def close_later(self):
        self.open = False

    """
    Closes the listener immediately.
    """
    def close(self):
        self.open = False
        try:
            self.socket.shutdown(socket.SHUT_RDWR)
            self.socket.close()
        except OSError:
            pass

    def _handle_request(self, request):
        response = {
            "jsonrpc": "2.0"
        }

        # Attempt to invoke the requested method.
        try:
            # If the method is "close", close the connection.
            if request["method"] == "close":
                self.close_later()
                response["result"] = True

            # For any other method, invoke it on the handler.
            else:
                func = getattr(self.handler, request["method"])
                args = request["params"]
                return_val = func(**args)

                response["result"] = return_val
        except Exception as e:
            # Method errored, so respond with an error instead.
            response["error"] = {
                "code": 500,
                "message": str(e)
            }

        # Send back a response to the peer.
        self.send(response)
<|endoftext|>


<commit_before>#!/usr/bin/python

import struct,sys
import traceback

from PIL import Image

class OpenTTDFileParser():
    def __init__(self,filen):
        self.fileName = filen
        self.filePt = open(filen)
        self.header = None
        self.version = None
        self.chunks = []
        self._readHeaders()
        try:
            self._readAllChunks()
        except:
            traceback.print_exc()

    def _readHeaders(self):
        self.header = self.filePt.read(4)
        self.version = struct.unpack(">HH",self.filePt.read(4))

    def _readByte(self):
        n = struct.unpack(">B",self.filePt.read(1))[0]
        return n

    def _readGamma(self):
        i = self._readByte()
        if (i >> 7) & 1:
            i &= ~0x80
            if (i >> 6) & 1:
                i &= ~0x40
                if (i >> 5) & 1:
                    i &= ~0x20
                    if (i >> 4) & 1:
                        i &= ~0x10
                        if (i >> 3) & 1:
                            assert False
                        i = self._readByte()
                    i = (i << 8) | self._readByte()
                i = (i << 8) | self._readByte()
            i = (i << 8) | self._readByte()
        return i

    def _readSparseChunk(self):
        length = self._readGamma()
        parts = []
        while length != 0:
            payload = self.filePt.read(length-1)
            parts.append(payload)
            length = self._readGamma()
        return parts

    def _readAllChunks(self):
        cTyp = self.filePt.read(4)

        while cTyp != "\x00\x00\x00\x00":
            cMod = self.filePt.read(1)
            if cMod == '\x02' or cMod == '\x01':
                payload = self._readSparseChunk()
            else:
                rl = self.filePt.read(3)
                length = struct.unpack(">I",cMod+rl)[0]
                payload = self.filePt.read(length)
            getattr(self,"_parse_"+cTyp,self._noparse)(cTyp,payload)
            cTyp = self.filePt.read(4)

    def _noparse(self,block,payload):
        print "unhandled chunk",block,
        if isinstance(payload,str) or len(payload) == 0:
            print len(payload)
        else:
            print len(payload),"of",len(payload[0])
        self.chunks.append((block,payload,))

    def _parse_MAPS(self,block,payload):
        self.size = struct.unpack(">II",payload)
        print "SIZE",self.size

    #Tile type & Height map
    def _parse_MAPT(self,block,payload):
        width,height = self.size
        x,y = 0,0
        self.heightMap = [[-1 for j in xrange(height)] for i in xrange(width)]
        self.tileMap = [[-1 for j in xrange(height)] for i in xrange(width)]
        for c in payload:
            n = ord(c)
            h = n & 0xF
            t = n >> 4
            self.heightMap[y][x] = h
            self.tileMap[y][x] = t
            x += 1
            if x == width:
                x=0
                y+=1

    def _parse_MAPO(self,block,payload):
        width,height = self.size
        x,y = 0,0
        self.mapO = [[-1 for j in xrange(height)] for i in xrange(width)]
        for c in payload:
            n = ord(c)
            self.mapO[y][x] = n
            x += 1
            if x == width:
                x=0
                y+=1

    def _parse_MAP5(self,block,payload):
        width,height = self.size
        x,y = 0,0
        self.map5 = [[-1 for j in xrange(height)] for i in xrange(width)]
        for c in payload:
            n = ord(c)
            self.map5[y][x] = n
            x += 1
            if x == width:
                x=0
                y+=1

if __name__ == "__main__":
    f = OpenTTDFileParser(sys.argv[1])

    cols = [
        (64,255,64),    #Clear
        (64,64,64),     #Railway
        (16,16,16),     #Road
        (8,8,8),        #House
        (0,180,180),    #Tree
        (32,32,32),     #Station
        (0,0,128),      #Water
        (255,0,255),    #Void
        (8,8,8),        #Industry
        (255,127,0),    #Tunnel
        (180,180,180)   #Object
    ]

    w,h=f.size
    img = Image.new("RGB",(w,h))
    pix = img.load()
    for x in xrange(w):
        for y in xrange(h):
            t = f.tileMap[x][y]
            pix[x,y] = cols[t]
            #print x,y,t,cols[t]
    img.resize((w*10,h*10)).save("ottdmaptest.png")

    img = Image.new("RGB",(w,h))
    pix = img.load()
    for x in xrange(w):
        for y in xrange(h):
            t = f.mapO[x][y]
            pix[x,y] = (t,t,t)
    img.resize((w*10,h*10)).save("ottdmaptestO.png")

    img = Image.new("RGB",(w,h))
    pix = img.load()
    for x in xrange(w):
        for y in xrange(h):
            t = f.map5[x][y]
            pix[x,y] = (t,t,t)
    img.save("ottdmaptest5.png")
<commit_msg>	modified:   openttdMap.py<commit_after>#!/usr/bin/python

import struct,sys
import traceback

from PIL import Image

class OpenTTDFileParser():
    def __init__(self,filen):
        self.fileName = filen
        self.filePt = open(filen)
        self.header = None
        self.version = None
        self.chunks = []
        self._readHeaders()
        if self.header == 'OTTN':
            try:
                self._readAllChunks()
            except:
                traceback.print_exc()
        else:
            print 'Unsupported save encryption'
            sys.exit()

    def _readHeaders(self):
        self.header = self.filePt.read(4)
        self.version = struct.unpack(">HH",self.filePt.read(4))

    def _readByte(self):
        n = struct.unpack(">B",self.filePt.read(1))[0]
        return n

    def _readGamma(self):
        i = self._readByte()
        if (i >> 7) & 1:
            i &= ~0x80
            if (i >> 6) & 1:
                i &= ~0x40
                if (i >> 5) & 1:
                    i &= ~0x20
                    if (i >> 4) & 1:
                        i &= ~0x10
                        if (i >> 3) & 1:
                            assert False
                        i = self._readByte()
                    i = (i << 8) | self._readByte()
                i = (i << 8) | self._readByte()
            i = (i << 8) | self._readByte()
        return i

    def _readSparseChunk(self):
        length = self._readGamma()
        parts = []
        while length != 0:
            payload = self.filePt.read(length-1)
            parts.append(payload)
            length = self._readGamma()
        return parts

    def _readAllChunks(self):
        cTyp = self.filePt.read(4)

        while cTyp != "\x00\x00\x00\x00":
            cMod = self.filePt.read(1)
            if cMod == '\x02' or cMod == '\x01':
                payload = self._readSparseChunk()
            else:
                rl = self.filePt.read(3)
                length = struct.unpack(">I",cMod+rl)[0]
                payload = self.filePt.read(length)
            getattr(self,"_parse_"+cTyp,self._noparse)(cTyp,payload)
            cTyp = self.filePt.read(4)

    def _noparse(self,block,payload):
        print "unhandled chunk",block,
        if isinstance(payload,str) or len(payload) == 0:
            print len(payload)
        else:
            print len(payload),"of",len(payload[0])
        self.chunks.append((block,payload,))

    def _parse_MAPS(self,block,payload):
        self.size = struct.unpack(">II",payload)
        print "SIZE",self.size

    #Tile type & Height map
    def _parse_MAPT(self,block,payload):
        width,height = self.size
        x,y = 0,0
        self.heightMap = [[-1 for j in xrange(height)] for i in xrange(width)]
        self.tileMap = [[-1 for j in xrange(height)] for i in xrange(width)]
        for c in payload:
            n = ord(c)
            h = n & 0xF
            t = n >> 4
            self.heightMap[y][x] = h
            self.tileMap[y][x] = t
            x += 1
            if x == width:
                x=0
                y+=1

    def _parse_MAPO(self,block,payload):
        width,height = self.size
        x,y = 0,0
        self.mapO = [[-1 for j in xrange(height)] for i in xrange(width)]
        for c in payload:
            n = ord(c)
            self.mapO[y][x] = n
            x += 1
            if x == width:
                x=0
                y+=1

    def _parse_MAP5(self,block,payload):
        width,height = self.size
        x,y = 0,0
        self.map5 = [[-1 for j in xrange(height)] for i in xrange(width)]
        for c in payload:
            n = ord(c)
            self.map5[y][x] = n
            x += 1
            if x == width:
                x=0
                y+=1

if __name__ == "__main__":
    f = OpenTTDFileParser(sys.argv[1])

    cols = [
        (64,255,64),    #Clear
        (64,64,64),     #Railway
        (16,16,16),     #Road
        (8,8,8),        #House
        (0,180,180),    #Tree
        (32,32,32),     #Station
        (0,0,128),      #Water
        (255,0,255),    #Void
        (8,8,8),        #Industry
        (255,127,0),    #Tunnel
        (180,180,180)   #Object
    ]

    w,h=f.size
    img = Image.new("RGB",(w,h))
    pix = img.load()
    for x in xrange(w):
        for y in xrange(h):
            t = f.tileMap[x][y]
            pix[x,y] = cols[t]
            #print x,y,t,cols[t]
    img.resize((w*10,h*10)).save("ottdmaptest.png")

    img = Image.new("RGB",(w,h))
    pix = img.load()
    for x in xrange(w):
        for y in xrange(h):
            t = f.mapO[x][y]
            pix[x,y] = (t,t,t)
    img.resize((w*10,h*10)).save("ottdmaptestO.png")

    img = Image.new("RGB",(w,h))
    pix = img.load()
    for x in xrange(w):
        for y in xrange(h):
            t = f.map5[x][y]
            pix[x,y] = (t,t,t)
    img.save("ottdmaptest5.png")
<|endoftext|>


<commit_before>from __future__ import print_function

import logging

from argh import *
from ghtools.api import GithubAPIClient, ClientError, APIError

log = logging.getLogger(__name__)

def extract_keys(obj, keys):
    newobj = {}
    for k in keys:
        newobj[k] = obj[k]
    return newobj

def sync_repo(gh_client, org, repo):
    log.debug("Checking if %s/%s exists on %s", org, repo['name'], gh_client)

    try:
        r = gh_client.repo(org, repo['name'])
    except requests.exceptions.HTTPError as exc:
        if exc.response.status_code == 404:
            log.debug("%s/%s doesn't exist on %s, creating...", org, repo['name'], gh_client)
            create_repo(gh_client, org, repo)
        else:
            raise
    else:
        log.debug("%s/%s already exists on %s, skipping...", org, repo['name'], gh_client)

def create_repo(gh_client, org, repo):
    data = extract_keys(repo, [
        'name',
        'description',
        'homepage',
        'private',
        'has_issues',
        'has_wiki',
        'has_downloads'
    ])
    gh_client.org_repo_create(org, repo)
    log.info("Created %s/%s on %s", org, repo['name'], gh_client)

@arg('src', help='Sync source')
@arg('dst', help='Sync destination')
def sync_org(args):
    """
    Sync an organisation between two GitHub instances.

    The source and destination should be given in the format

       <nickname>:<org>

    For example, to sync the 'foocorp' organisation on the public GitHub to
    the 'foo' organisation on the GitHub with nickname 'internal', you would
    run

       %(prog)s public:foocorp internal:foo
    """

    src_nickname, src_org = args.src.rsplit(':', 2)
    dst_nickname, dst_org = args.dst.rsplit(':', 2)

    src = GithubAPIClient(nickname=src_nickname)
    dst = GithubAPIClient(nickname=dst_nickname)

    for repo in src.org_repos(src_org):
        sync_repo(dst, dst_org, repo)

def main():
    dispatch_command(sync_org)

if __name__ == '__main__':
    main()
<commit_msg>Update sync-org command<commit_after>from __future__ import print_function

import logging

from argh import *
from ghtools.api import GithubAPIClient, ClientError, APIError

log = logging.getLogger(__name__)

def extract_keys(obj, keys):
    newobj = {}
    for k in keys:
        newobj[k] = obj[k]
    return newobj

def sync_repo(gh_client, org, repo):
    log.debug("Checking if %s/%s exists on %s", org, repo['name'], gh_client)

    try:
        r = gh_client.get('/repos/{0}/{1}'.format(org, repo['name'])).json
    except APIError as exc:
        if exc.response.status_code == 404:
            log.debug("%s/%s doesn't exist on %s, creating...", org, repo['name'], gh_client)
            create_repo(gh_client, org, repo)
        else:
            raise
    else:
        log.debug("%s/%s already exists on %s, skipping...", org, repo['name'], gh_client)

def create_repo(gh_client, org, repo):
    data = extract_keys(repo, [
        'name',
        'description',
        'homepage',
        'private',
        'has_issues',
        'has_wiki',
        'has_downloads'
    ])
    gh_client.post('/orgs/{0}/repos'.format(org), data)
    log.info("Created %s/%s on %s", org, repo['name'], gh_client)

@arg('src', help='Sync source')
@arg('dst', help='Sync destination')
def sync_org(args):
    """
    Sync an organisation between two GitHub instances.

    The source and destination should be given in the format

       <nickname>:<org>

    For example, to sync the 'foocorp' organisation on the public GitHub to
    the 'foo' organisation on the GitHub with nickname 'internal', you would
    run

       %(prog)s public:foocorp internal:foo
    """

    src_nickname, src_org = args.src.rsplit(':', 2)
    dst_nickname, dst_org = args.dst.rsplit(':', 2)

    src = GithubAPIClient(nickname=src_nickname)
    dst = GithubAPIClient(nickname=dst_nickname)

    for repo in src.paged_get('/orgs/{0}/repos'.format(src_org)):
        sync_repo(dst, dst_org, repo)

def main():
    dispatch_command(sync_org)

if __name__ == '__main__':
    main()
<|endoftext|>