<commit_before># Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import os
from pathlib import Path
import sys
from typing import Callable, Dict, List, Optional

import nox

# WARNING - WARNING - WARNING - WARNING - WARNING
# WARNING - WARNING - WARNING - WARNING - WARNING
#           DO NOT EDIT THIS FILE EVER!
# WARNING - WARNING - WARNING - WARNING - WARNING
# WARNING - WARNING - WARNING - WARNING - WARNING

BLACK_VERSION = "black==19.10b0"

# Copy `noxfile_config.py` to your directory and modify it instead.

# `TEST_CONFIG` dict is a configuration hook that allows users to
# modify the test configurations. The values here should be in sync
# with `noxfile_config.py`. Users will copy `noxfile_config.py` into
# their directory and modify it.

TEST_CONFIG = {
    # You can opt out from the test for specific Python versions.
    'ignored_versions': ["2.7"],

    # Old samples are opted out of enforcing Python type hints
    # All new samples should feature them
    'enforce_type_hints': False,

    # An envvar key for determining the project id to use. Change it
    # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a
    # build specific Cloud project. You can also use your own string
    # to use your own Cloud project.
    'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT',
    # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT',
    # If you need to use a specific version of pip,
    # change pip_version_override to the string representation
    # of the version number, for example, "20.2.4"
    "pip_version_override": None,
    # A dictionary you want to inject into your test. Don't put any
    # secrets here. These values will override predefined values.
    'envs': {},
}

try:
    # Ensure we can import noxfile_config in the project's directory.
    sys.path.append('.')
    from noxfile_config import TEST_CONFIG_OVERRIDE
except ImportError as e:
    print("No user noxfile_config found: detail: {}".format(e))
    TEST_CONFIG_OVERRIDE = {}

# Update the TEST_CONFIG with the user supplied values.
TEST_CONFIG.update(TEST_CONFIG_OVERRIDE)

def get_pytest_env_vars() -> Dict[str, str]:
    """Returns a dict for pytest invocation."""
    ret = {}

    # Override the GCLOUD_PROJECT and the alias.
    env_key = TEST_CONFIG['gcloud_project_env']
    # This should error out if not set.
    ret['GOOGLE_CLOUD_PROJECT'] = os.environ[env_key]

    # Apply user supplied envs.
    ret.update(TEST_CONFIG['envs'])
    return ret

# DO NOT EDIT - automatically generated.
# All versions used to tested samples.
ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8", "3.9"]

# Any default versions that should be ignored.
IGNORED_VERSIONS = TEST_CONFIG['ignored_versions']

TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS])

INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ("True", "true")
#
# Style Checks
#

def _determine_local_import_names(start_dir: str) -> List[str]:
    """Determines all import names that should be considered "local".

    This is used when running the linter to insure that import order is
    properly checked.
    """
    file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)]
    return [
        basename
        for basename, extension in file_ext_pairs
        if extension == ".py"
        or os.path.isdir(os.path.join(start_dir, basename))
        and basename not in ("__pycache__")
    ]

# Linting with flake8.
#
# We ignore the following rules:
#   E203: whitespace before ‘:’
#   E266: too many leading ‘#’ for block comment
#   E501: line too long
#   I202: Additional newline in a section of imports
#
# We also need to specify the rules which are ignored by default:
# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121']
FLAKE8_COMMON_ARGS = [
    "--show-source",
    "--builtin=gettext",
    "--max-complexity=20",
    "--import-order-style=google",
    "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py",
    "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202",
    "--max-line-length=88",
]

@nox.session
def lint(session: nox.sessions.Session) -> None:
    if not TEST_CONFIG['enforce_type_hints']:
        session.install("flake8", "flake8-import-order")
    else:
        session.install("flake8", "flake8-import-order", "flake8-annotations")

    local_names = _determine_local_import_names(".")
    args = FLAKE8_COMMON_ARGS + [
        "--application-import-names",
        ",".join(local_names),
        "."
    ]
    session.run("flake8", *args)
#
# Black
#

@nox.session
def blacken(session: nox.sessions.Session) -> None:
    session.install(BLACK_VERSION)
    python_files = [path for path in os.listdir(".") if path.endswith(".py")]

    session.run("black", *python_files)

#
# Sample Tests
#

PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"]

def _session_tests(session: nox.sessions.Session, post_install: Callable = None) -> None:
    if TEST_CONFIG["pip_version_override"]:
        pip_version = TEST_CONFIG["pip_version_override"]
        session.install(f"pip=={pip_version}")
    """Runs py.test for a particular project."""
    if os.path.exists("requirements.txt"):
        if os.path.exists("constraints.txt"):
            session.install("-r", "requirements.txt", "-c", "constraints.txt")
        else:
            session.install("-r", "requirements.txt")

    if os.path.exists("requirements-test.txt"):
        if os.path.exists("constraints-test.txt"):
            session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt")
        else:
            session.install("-r", "requirements-test.txt")

    if INSTALL_LIBRARY_FROM_SOURCE:
        session.install("-e", _get_repo_root())

    if post_install:
        post_install(session)

    session.run(
        "pytest",
        *(PYTEST_COMMON_ARGS + session.posargs),
        # Pytest will return 5 when no tests are collected. This can happen
        # on travis where slow and flaky tests are excluded.
        # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html
        success_codes=[0, 5],
        env=get_pytest_env_vars()
    )

@nox.session(python=ALL_VERSIONS)
def py(session: nox.sessions.Session) -> None:
    """Runs py.test for a sample using the specified version of Python."""
    if session.python in TESTED_VERSIONS:
        _session_tests(session)
    else:
        session.skip("SKIPPED: {} tests are disabled for this sample.".format(
            session.python
        ))

#
# Readmegen
#

def _get_repo_root() -> Optional[str]:
    """ Returns the root folder of the project. """
    # Get root of this repository. Assume we don't have directories nested deeper than 10 items.
    p = Path(os.getcwd())
    for i in range(10):
        if p is None:
            break
        if Path(p / ".git").exists():
            return str(p)
        # .git is not available in repos cloned via Cloud Build
        # setup.py is always in the library's root, so use that instead
        # https://github.com/googleapis/synthtool/issues/792
        if Path(p / "setup.py").exists():
            return str(p)
        p = p.parent
    raise Exception("Unable to detect repository root.")

GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")])

@nox.session
@nox.parametrize("path", GENERATED_READMES)
def readmegen(session: nox.sessions.Session, path: str) -> None:
    """(Re-)generates the readme for a sample."""
    session.install("jinja2", "pyyaml")
    dir_ = os.path.dirname(path)

    if os.path.exists(os.path.join(dir_, "requirements.txt")):
        session.install("-r", os.path.join(dir_, "requirements.txt"))

    in_file = os.path.join(dir_, "README.rst.in")
    session.run(
        "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file
    )
<commit_msg>chore: drop mention of Python 2.7 from templates (#147)<commit_after># Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import os
from pathlib import Path
import sys
from typing import Callable, Dict, List, Optional

import nox

# WARNING - WARNING - WARNING - WARNING - WARNING
# WARNING - WARNING - WARNING - WARNING - WARNING
#           DO NOT EDIT THIS FILE EVER!
# WARNING - WARNING - WARNING - WARNING - WARNING
# WARNING - WARNING - WARNING - WARNING - WARNING

BLACK_VERSION = "black==19.10b0"

# Copy `noxfile_config.py` to your directory and modify it instead.

# `TEST_CONFIG` dict is a configuration hook that allows users to
# modify the test configurations. The values here should be in sync
# with `noxfile_config.py`. Users will copy `noxfile_config.py` into
# their directory and modify it.

TEST_CONFIG = {
    # You can opt out from the test for specific Python versions.
    'ignored_versions': [],

    # Old samples are opted out of enforcing Python type hints
    # All new samples should feature them
    'enforce_type_hints': False,

    # An envvar key for determining the project id to use. Change it
    # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a
    # build specific Cloud project. You can also use your own string
    # to use your own Cloud project.
    'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT',
    # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT',
    # If you need to use a specific version of pip,
    # change pip_version_override to the string representation
    # of the version number, for example, "20.2.4"
    "pip_version_override": None,
    # A dictionary you want to inject into your test. Don't put any
    # secrets here. These values will override predefined values.
    'envs': {},
}

try:
    # Ensure we can import noxfile_config in the project's directory.
    sys.path.append('.')
    from noxfile_config import TEST_CONFIG_OVERRIDE
except ImportError as e:
    print("No user noxfile_config found: detail: {}".format(e))
    TEST_CONFIG_OVERRIDE = {}

# Update the TEST_CONFIG with the user supplied values.
TEST_CONFIG.update(TEST_CONFIG_OVERRIDE)

def get_pytest_env_vars() -> Dict[str, str]:
    """Returns a dict for pytest invocation."""
    ret = {}

    # Override the GCLOUD_PROJECT and the alias.
    env_key = TEST_CONFIG['gcloud_project_env']
    # This should error out if not set.
    ret['GOOGLE_CLOUD_PROJECT'] = os.environ[env_key]

    # Apply user supplied envs.
    ret.update(TEST_CONFIG['envs'])
    return ret

# DO NOT EDIT - automatically generated.
# All versions used to test samples.
ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9"]

# Any default versions that should be ignored.
IGNORED_VERSIONS = TEST_CONFIG['ignored_versions']

TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS])

INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ("True", "true")
#
# Style Checks
#

def _determine_local_import_names(start_dir: str) -> List[str]:
    """Determines all import names that should be considered "local".

    This is used when running the linter to insure that import order is
    properly checked.
    """
    file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)]
    return [
        basename
        for basename, extension in file_ext_pairs
        if extension == ".py"
        or os.path.isdir(os.path.join(start_dir, basename))
        and basename not in ("__pycache__")
    ]

# Linting with flake8.
#
# We ignore the following rules:
#   E203: whitespace before ‘:’
#   E266: too many leading ‘#’ for block comment
#   E501: line too long
#   I202: Additional newline in a section of imports
#
# We also need to specify the rules which are ignored by default:
# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121']
FLAKE8_COMMON_ARGS = [
    "--show-source",
    "--builtin=gettext",
    "--max-complexity=20",
    "--import-order-style=google",
    "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py",
    "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202",
    "--max-line-length=88",
]

@nox.session
def lint(session: nox.sessions.Session) -> None:
    if not TEST_CONFIG['enforce_type_hints']:
        session.install("flake8", "flake8-import-order")
    else:
        session.install("flake8", "flake8-import-order", "flake8-annotations")

    local_names = _determine_local_import_names(".")
    args = FLAKE8_COMMON_ARGS + [
        "--application-import-names",
        ",".join(local_names),
        "."
    ]
    session.run("flake8", *args)
#
# Black
#

@nox.session
def blacken(session: nox.sessions.Session) -> None:
    session.install(BLACK_VERSION)
    python_files = [path for path in os.listdir(".") if path.endswith(".py")]

    session.run("black", *python_files)

#
# Sample Tests
#

PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"]

def _session_tests(session: nox.sessions.Session, post_install: Callable = None) -> None:
    if TEST_CONFIG["pip_version_override"]:
        pip_version = TEST_CONFIG["pip_version_override"]
        session.install(f"pip=={pip_version}")
    """Runs py.test for a particular project."""
    if os.path.exists("requirements.txt"):
        if os.path.exists("constraints.txt"):
            session.install("-r", "requirements.txt", "-c", "constraints.txt")
        else:
            session.install("-r", "requirements.txt")

    if os.path.exists("requirements-test.txt"):
        if os.path.exists("constraints-test.txt"):
            session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt")
        else:
            session.install("-r", "requirements-test.txt")

    if INSTALL_LIBRARY_FROM_SOURCE:
        session.install("-e", _get_repo_root())

    if post_install:
        post_install(session)

    session.run(
        "pytest",
        *(PYTEST_COMMON_ARGS + session.posargs),
        # Pytest will return 5 when no tests are collected. This can happen
        # on travis where slow and flaky tests are excluded.
        # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html
        success_codes=[0, 5],
        env=get_pytest_env_vars()
    )

@nox.session(python=ALL_VERSIONS)
def py(session: nox.sessions.Session) -> None:
    """Runs py.test for a sample using the specified version of Python."""
    if session.python in TESTED_VERSIONS:
        _session_tests(session)
    else:
        session.skip("SKIPPED: {} tests are disabled for this sample.".format(
            session.python
        ))

#
# Readmegen
#

def _get_repo_root() -> Optional[str]:
    """ Returns the root folder of the project. """
    # Get root of this repository. Assume we don't have directories nested deeper than 10 items.
    p = Path(os.getcwd())
    for i in range(10):
        if p is None:
            break
        if Path(p / ".git").exists():
            return str(p)
        # .git is not available in repos cloned via Cloud Build
        # setup.py is always in the library's root, so use that instead
        # https://github.com/googleapis/synthtool/issues/792
        if Path(p / "setup.py").exists():
            return str(p)
        p = p.parent
    raise Exception("Unable to detect repository root.")

GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")])

@nox.session
@nox.parametrize("path", GENERATED_READMES)
def readmegen(session: nox.sessions.Session, path: str) -> None:
    """(Re-)generates the readme for a sample."""
    session.install("jinja2", "pyyaml")
    dir_ = os.path.dirname(path)

    if os.path.exists(os.path.join(dir_, "requirements.txt")):
        session.install("-r", os.path.join(dir_, "requirements.txt"))

    in_file = os.path.join(dir_, "README.rst.in")
    session.run(
        "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file
    )
<|endoftext|>


<commit_before>''' Tank structure calculations.

Matt Vernacchia
proptools
2015 April 17
'''

from math import pi

def crown_thickness(p_t, R, stress, weld_eff):
    ''' Crown thickness of a spherical or ellipsoidal tank end.

    Implements eqn 8-16 from Huzel and Huang. The crown is the center of the
    tank end, see figure 8-6 in Huzel and Huang.

    Arguments:
        p_t: Tank internal pressure (less the ambient pressure) [units: pascal].
        R: Crown radius [units: meter].
        stress: Max allowable stress in thank wall material [units: pascal].
        weld_eff: Weld efficiency, scalar in [0, 1] [units: none].

    Returns:
        crown thickness [units: meter].
    '''
    return p_t * R / (2 * stress * weld_eff)

def knuckle_thickness(p_t, a, stress, weld_eff):
    ''' Knuckle thickness of a spherical tank end.

    Implements eqn 8-15 from Huzel and Huang. The knuckle is the transition from
    the cylindrical section to the tank end, see figure 8-6 in Huzel and Huang.

    Arguments:
        p_t: Tank internal pressure (less the ambient pressure) [units: pascal].
        a: Tank radius [units: meter].
        stress: Max allowable stress in thank wall material [units: pascal].
        weld_eff: Weld efficiency, scalar in [0, 1] [units: none].

    Returns:
        knuckle thickness [units: meter].
    '''
    # K is the stress factor, see figure 8-7 in Huzel and Huang. Its value is
    # 0.67 for spherical ends, and increases for ellipsoidal ends.
    K = 0.67
    return K * p_t * a / (stress * weld_eff)

def cylinder_thickness(p_t, a, stress, weld_eff):
    ''' Thickness of a cylindrical tank section.

    Implements eqn 8-28 from Huzel and Huang.

    Arguments:
        p_t: Tank internal pressure (less the ambient pressure) [units: pascal].
        a: Tank radius [units: meter].
        stress: Max allowable stress in thank wall material [units: pascal].
        weld_eff: Weld efficiency, scalar in [0, 1] [units: none].

    Returns:
        cylinder thickness [units: meter].
    '''
    return p_t * a / (stress * weld_eff)

def sphere_thickness(p_t, a, stress, weld_eff):
    ''' Thickness of a spherical tank.

    Implements eqn 8-9 from Huzel and Huang.

    Arguments:
        p_t: Tank internal pressure (less the ambient pressure) [units: pascal].
        a: Tank radius [units: meter].
        stress: Max allowable stress in thank wall material [units: pascal].
        weld_eff: Weld efficiency, scalar in [0, 1] [units: none].

    Returns:
        sphere thickness [units: meter].
    '''
    return p_t * a / (2 * stress * weld_eff)

def max_axial_load(p_t, a, t_c, l_c, E):
    '''Maximum compressive axial load that a cylindrical section can support.

    Implements eqn 8-33 from Huzel and Huang.

    Arguments:
        p_t: Tank internal pressure (less the ambient pressure) [units: pascal].
        a: Tank radius [units: meter].
        t_c: Cylinder wall thickness [units: meter].
        l_c: Cylinder length [units: meter].
        E: Wall material modulus of elasticity [units: pascal].

    Returns:
        Critical compressive axial load [units: newtons].
    '''
    # Critical axial stress (unpressurized) [units: pascal].
    S_c = (9 * (t_c / a)**1.6 + 0.16 * (t_c / l_c)**1.3) * E
    # Axial cross section area [units: meter**2].
    A_section = 2 * pi * a * t_c
    # Critical axial compression load (unpressurized) [units: newton].
    F_c = S_c *  A_section
    # Axial tension load due to internal pressure [units: newton].
    F_a = pi * a**2 * p_t
    # The total critical compressive load is the load supported by the tank
    # walls plus the load supported by the internal pressure.
    return F_c + F_a

def cylinder_mass(a, t_c, l_c, rho):
    ''' Mass of a cylindrical tank section.

    Arguments:
        a: Tank radius [units: meter].
        t_c: Tank wall thickness [units: meter].
        l_c: Cylinder length [units: meter].
        rho: Tank material density [units: kilogram meter**-3].

    Returns:
        mass of spherical tank [kilogram].
    '''
    return 2 * pi * a * l_c * t_c * rho

def sphere_mass(a, t, rho):
    ''' Mass of a spherical tank.

    Arguments:
        a: Tank radius [units: meter].
        t: Tank wall thickness [units: meter].
        rho: Tank material density [units: kilogram meter**-3].

    Returns:
        mass of spherical tank [kilogram].
    '''
    return 4 * pi * a**2 * t * rho

<commit_msg>Tank buckling calculations.<commit_after>''' Tank structure calculations.

Matt Vernacchia
proptools
2015 April 17
'''

from math import pi

def crown_thickness(p_t, R, stress, weld_eff):
    ''' Crown thickness of a spherical or ellipsoidal tank end.

    Implements eqn 8-16 from Huzel and Huang. The crown is the center of the
    tank end, see figure 8-6 in Huzel and Huang.

    Arguments:
        p_t: Tank internal pressure (less the ambient pressure) [units: pascal].
        R: Crown radius [units: meter].
        stress: Max allowable stress in thank wall material [units: pascal].
        weld_eff: Weld efficiency, scalar in [0, 1] [units: none].

    Returns:
        crown thickness [units: meter].
    '''
    return p_t * R / (2 * stress * weld_eff)

def knuckle_thickness(p_t, a, stress, weld_eff):
    ''' Knuckle thickness of a spherical tank end.

    Implements eqn 8-15 from Huzel and Huang. The knuckle is the transition from
    the cylindrical section to the tank end, see figure 8-6 in Huzel and Huang.

    Arguments:
        p_t: Tank internal pressure (less the ambient pressure) [units: pascal].
        a: Tank radius [units: meter].
        stress: Max allowable stress in thank wall material [units: pascal].
        weld_eff: Weld efficiency, scalar in [0, 1] [units: none].

    Returns:
        knuckle thickness [units: meter].
    '''
    # K is the stress factor, see figure 8-7 in Huzel and Huang. Its value is
    # 0.67 for spherical ends, and increases for ellipsoidal ends.
    K = 0.67
    return K * p_t * a / (stress * weld_eff)

def cylinder_thickness(p_t, a, stress, weld_eff):
    ''' Thickness of a cylindrical tank section.

    Implements eqn 8-28 from Huzel and Huang.

    Arguments:
        p_t: Tank internal pressure (less the ambient pressure) [units: pascal].
        a: Tank radius [units: meter].
        stress: Max allowable stress in thank wall material [units: pascal].
        weld_eff: Weld efficiency, scalar in [0, 1] [units: none].

    Returns:
        cylinder thickness [units: meter].
    '''
    return p_t * a / (stress * weld_eff)

def sphere_thickness(p_t, a, stress, weld_eff):
    ''' Thickness of a spherical tank.

    Implements eqn 8-9 from Huzel and Huang.

    Arguments:
        p_t: Tank internal pressure (less the ambient pressure) [units: pascal].
        a: Tank radius [units: meter].
        stress: Max allowable stress in thank wall material [units: pascal].
        weld_eff: Weld efficiency, scalar in [0, 1] [units: none].

    Returns:
        sphere thickness [units: meter].
    '''
    return p_t * a / (2 * stress * weld_eff)

def max_axial_load(p_t, a, t_c, l_c, E):
    '''Maximum compressive axial load that a cylindrical section can support.

    Implements eqn 8-33 from Huzel and Huang.

    Arguments:
        p_t: Tank internal pressure (less the ambient pressure) [units: pascal].
        a: Tank radius [units: meter].
        t_c: Cylinder wall thickness [units: meter].
        l_c: Cylinder length [units: meter].
        E: Wall material modulus of elasticity [units: pascal].

    Returns:
        Critical compressive axial load [units: newtons].
    '''
    # Critical axial stress (unpressurized) [units: pascal].
    S_c = (9 * (t_c / a)**1.6 + 0.16 * (t_c / l_c)**1.3) * E
    # Axial cross section area [units: meter**2].
    A_section = 2 * pi * a * t_c
    # Critical axial compression load (unpressurized) [units: newton].
    F_c = S_c *  A_section
    # Axial tension load due to internal pressure [units: newton].
    F_a = pi * a**2 * p_t
    # The total critical compressive load is the load supported by the tank
    # walls plus the load supported by the internal pressure.
    return F_c + F_a

def cylinder_mass(a, t_c, l_c, rho):
    ''' Mass of a cylindrical tank section.

    Arguments:
        a: Tank radius [units: meter].
        t_c: Tank wall thickness [units: meter].
        l_c: Cylinder length [units: meter].
        rho: Tank material density [units: kilogram meter**-3].

    Returns:
        mass of spherical tank [kilogram].
    '''
    return 2 * pi * a * l_c * t_c * rho

def sphere_mass(a, t, rho):
    ''' Mass of a spherical tank.

    Arguments:
        a: Tank radius [units: meter].
        t: Tank wall thickness [units: meter].
        rho: Tank material density [units: kilogram meter**-3].

    Returns:
        mass of spherical tank [kilogram].
    '''
    return 4 * pi * a**2 * t * rho

def cr_ex_press_sphere(a, t, E, v):
    ''' Critical external pressure difference to buckle a spherical tank.

    Implements eqn 8-12 in Huzel and Huang.

    Arguments:
        a: Tank radius [units: meter].
        t: Wall thickness [units: meter].
        E: Wall material modulus of elasticity [units: pascal].
        v: Wall material Poisson's ratio [units: none].

    Returns:
        Critical external pressure for buckling [units: pascal].
    '''
    return 2 * E * t**2 / a**2 * (3 * (1 - v**2))**0.5

def cr_ex_press_sphere_end(a, t, E):
    ''' Critical external pressure difference to buckle a spherical tank end.

    Implements eqn 8-16 in Huzel and Huang.

    Arguments:
        a: Tank radius [units: meter].
        t: Wall thickness [units: meter].
        E: Wall material modulus of elasticity [units: pascal].

    Returns:
        Critical external pressure for buckling [units: pascal].
    '''
    return 0.342 * E * t**2 / a**2

def cr_ex_press_cylinder(a, t_c, l_c, E, v):
    ''' Critical external pressure difference to buckle a cylindrical tank section.

    Implements eqn 8-12 in Huzel and Huang.

    Arguments:
        a: Tank radius [units: meter].
        t_c: Tank wall thickness [units: meter].
        l_c: Cylinder length [units: meter].
        E: Wall material modulus of elasticity [units: pascal].
        v: Wall material Poisson's ratio [units: none].

    Returns:
        Critical external pressure for buckling [units: pascal].
    '''
    if l_c < 4.9 * a * (a / t_c)**0.5:
        # Short tank.
        return 0.807 * E * t_c**2 / (l_c * a) \
            * ( (1 / (1 - v**2))**3 * t_c**2 / a**2)**0.25
    else:
        # Long tank.
        return E * t_c**3 / (4 * (1 - v**2) * a**3)

<|endoftext|>


<commit_before>from wqflask import app

# Please note, running with host set externally below combined with debug mode
# is a big security no-no
# Unless you have a firewall setup
#
# Something like /sbin/iptables -A INPUT -p tcp -i eth0 -s ! 71.236.239.43 --dport 5000 -j DROP
# should do the trick
#
# You'll probably have to firewall the main port and the
#
# For more info see: http://www.cyberciti.biz/faq/iptables-block-port/

#import logging
#logging.basicConfig(filename="/tmp/flask_gn_log", level=logging.INFO)
#
#_log = logging.getLogger("search")
#_ch = logging.StreamHandler()
#_log.addHandler(_ch)

import logging
file_handler = logging.FileHandler(app.config['LOGFILE'])
file_handler.setLevel(logging.DEBUG)
app.logger.addHandler(file_handler)

import logging_tree
logging_tree.printout()

app.run(host='0.0.0.0',
        port=app.config['SERVER_PORT'],
        use_debugger=False,
        threaded=True,
        use_reloader=True)
<commit_msg>Output configi<commit_after>from wqflask import app

# Please note, running with host set externally below combined with debug mode
# is a big security no-no
# Unless you have a firewall setup
#
# Something like /sbin/iptables -A INPUT -p tcp -i eth0 -s ! 71.236.239.43 --dport 5000 -j DROP
# should do the trick
#
# You'll probably have to firewall the main port and the
#
# For more info see: http://www.cyberciti.biz/faq/iptables-block-port/

#import logging
#logging.basicConfig(filename="/tmp/flask_gn_log", level=logging.INFO)
#
#_log = logging.getLogger("search")
#_ch = logging.StreamHandler()
#_log.addHandler(_ch)

print app.config

import logging
file_handler = logging.FileHandler(app.config['LOGFILE'])
file_handler.setLevel(logging.DEBUG)
app.logger.addHandler(file_handler)

import logging_tree
logging_tree.printout()

app.run(host='0.0.0.0',
        port=app.config['SERVER_PORT'],
        use_debugger=False,
        threaded=True,
        use_reloader=True)
<|endoftext|>


<commit_before>import yaml
import os
import re
from collections import OrderedDict

# codes form stackoverflow.com
# http://stackoverflow.com/questions/6432605/any-yaml-libraries-in-python-that-support-dumping-of-long-strings-as-block-liter
# http://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data

class folded_unicode(unicode): pass
class literal_unicode(unicode): pass
class quoted(str): pass
class literal(str): pass

def folded_unicode_representer(dumper, data):
    return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='>')
def literal_unicode_representer(dumper, data):
    return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|')
def quoted_presenter(dumper, data):
    return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"')
def literal_presenter(dumper, data):
    return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
def ordered_dict_presenter(dumper, data):
    return dumper.represent_dict(data.items())

yaml.add_representer(folded_unicode, folded_unicode_representer)
yaml.add_representer(literal_unicode, literal_unicode_representer)
yaml.add_representer(quoted, quoted_presenter)
yaml.add_representer(literal, literal_presenter)
yaml.add_representer(OrderedDict, ordered_dict_presenter)

def include_constructor(loader, node):
    filename = loader.construct_scalar(node)
    if not os.path.exists(filename):
        stream_filename = os.path.abspath(loader.stream.name)
        stream_dirname = os.path.dirname(stream_filename)
        filename = os.path.join(stream_dirname, filename)

    with open(filename) as f:
        data = yaml.load(f)
    return data

yaml.add_constructor(u'!include', include_constructor)

def range_constructor(loader, node):
    data = loader.construct_scalar(node)
    match = range_regex.match(data)
    if match:
        start, stop = map(int, match.groups())
        return range(start, stop + 1)
    else:
        raise ValueError('Invalid range: %s' % data)

range_regex = re.compile(r'^\s*(\d+)\s*\.\.\s*(\d+)\s*$')
yaml.add_constructor(u'!range', range_constructor)
yaml.add_implicit_resolver(u'!range', range_regex)
<commit_msg>add refer_constructor, which values load from other position in the same yaml<commit_after>import yaml
import os
import re
from collections import OrderedDict

# codes form stackoverflow.com
# http://stackoverflow.com/questions/6432605/any-yaml-libraries-in-python-that-support-dumping-of-long-strings-as-block-liter
# http://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data

class folded_unicode(unicode): pass
class literal_unicode(unicode): pass
class quoted(str): pass
class literal(str): pass

def folded_unicode_representer(dumper, data):
    return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='>')
def literal_unicode_representer(dumper, data):
    return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|')
def quoted_presenter(dumper, data):
    return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"')
def literal_presenter(dumper, data):
    return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
def ordered_dict_presenter(dumper, data):
    return dumper.represent_dict(data.items())

yaml.add_representer(folded_unicode, folded_unicode_representer)
yaml.add_representer(literal_unicode, literal_unicode_representer)
yaml.add_representer(quoted, quoted_presenter)
yaml.add_representer(literal, literal_presenter)
yaml.add_representer(OrderedDict, ordered_dict_presenter)

def include_constructor(loader, node):
    filename = loader.construct_scalar(node)
    if not os.path.exists(filename):
        stream_filename = os.path.abspath(loader.stream.name)
        stream_dirname = os.path.dirname(stream_filename)
        filename = os.path.join(stream_dirname, filename)

    with open(filename) as f:
        data = yaml.load(f)
    return data

yaml.add_constructor(u'!include', include_constructor)

def range_constructor(loader, node):
    data = loader.construct_scalar(node)
    match = range_regex.match(data)
    if match:
        start, stop = map(int, match.groups())
        return range(start, stop + 1)
    else:
        raise ValueError('Invalid range: %s' % data)

range_regex = re.compile(r'^\s*(\d+)\s*\.\.\s*(\d+)\s*$')
yaml.add_constructor(u'!range', range_constructor)
yaml.add_implicit_resolver(u'!range', range_regex)

def refer_constructor(loader, node):
    def getRefer(content, keys):
        content = content.replace('!refer ', '')
        ref = yaml.load(content)
        for k in keys:
            ref = ref.get(k)
        return ref

    data = loader.construct_scalar(node)
    keys = data.split('.')
    offset = loader.stream.tell()
    loader.stream.seek(0)
    content = loader.stream.read()
    loader.stream.seek(offset)
    refer = getRefer(content, keys)
    return refer

yaml.add_constructor(u'!refer', refer_constructor)
<|endoftext|>


<commit_before>"""

Actor Runtime
=============

The runtime provides the functions to create an actor from a behavior,
and to report an error.  Subclass from `AbstractRuntime` to create a
runtime class.

`SimpleRuntime.create` just creates the actor, and
`SimpleRuntime.throw` prints the error message to stdout.

The behaviors are defined as::

    @behavior
    def my_beh(arg1, ..., self, msg):
        ...

where `arg1, ...` are 0 or more arguments to be passed at creation
time. `self` refers to the actor in context, and `msg` is the message
being passed.

To create or change behavior, `self` provides the methods `create` and
`become`. To send a message to an actor `x` use `x << msg`.  For
example::

    @behavior
    def my_beh(n, self, msg):
        if n > 0:
            another = self.create(my_beh, n-1)
            another << 'a message'
        self.become(sink_beh)

    @behavior
    def sink_beh(self, msg):
        pass

"""

from functools import wraps, partial
import pprint
import sys
import traceback

from .singleton import Singleton
from .eventloop import EventLoop

class AbstractRuntime(object, metaclass=Singleton):

    def create(self, behavior, *args):
        raise NotImplementedError()

    def throw(self, message):
        raise NotImplementedError()

class SimpleRuntime(AbstractRuntime):

    def create(self, behavior, *args):
        return Actor(self, behavior, *args)

    def throw(self, message):
        print('ERROR: {0}'.format(pprint.pformat(message)))

class Runtime(SimpleRuntime):

    def throw(self, message):
        super().throw(message)
        # also display human readable traceback, if possible
        try:
            print('\n' + ''.join(message['traceback']))
        except (TypeError, KeyError):
            pass

class ThreadedRuntime(Runtime):

    def __init__(self):
        super().__init__()
        self.restart()

    def pause(self):
        self.evloop.stop()

    def restart(self):
        self.evloop = EventLoop()
        self.evloop.run_in_thread()

def exception_message():
    """Create a message with details on the exception."""
    exc_type, exc_value, exc_tb = exc_info = sys.exc_info()
    return {'exception': {'type': exc_type,
                          'value': exc_value,
                          'traceback': exc_tb},
            'traceback': traceback.format_exception(*exc_info)}

class Actor(object):

    def __init__(self, runtime, behavior, *args):
        self._runtime = runtime
        self.become(behavior, *args)
        self._ev_loop = EventLoop()

    def become(self, behavior, *args):
        self._behavior = partial(behavior, *args)

    def send(self, msg):
        def event():
            try:
                self._behavior(self, msg)
            except Exception as exc:
                self.throw(exception_message())
        self._ev_loop.schedule(self, event)

    def create(self, behavior, *args):
        return self._runtime.create(behavior, *args)

    def throw(self, message):
        self._runtime.throw(message)

    def __lshift__(self, msg):
        """Syntax sugar for sending a message.

        Use as `x << msg`.

        """
        self.send(msg)

    def __call__(self, msg):
        self.send(msg)

def behavior(f):
    """Decorator for declaring a function as behavior.

    Use as::

        @behavior
        def fun(x, y, self, msg):
            ...

    And create or become this behavior by passing the two arguments
    `x` and `y`.

    """
    return f

<commit_msg>Actor retrieves eventloop from runtime<commit_after>"""

Actor Runtime
=============

The runtime provides the functions to create an actor from a behavior,
and to report an error.  Subclass from `AbstractRuntime` to create a
runtime class.

`SimpleRuntime.create` just creates the actor, and
`SimpleRuntime.throw` prints the error message to stdout.

The behaviors are defined as::

    @behavior
    def my_beh(arg1, ..., self, msg):
        ...

where `arg1, ...` are 0 or more arguments to be passed at creation
time. `self` refers to the actor in context, and `msg` is the message
being passed.

To create or change behavior, `self` provides the methods `create` and
`become`. To send a message to an actor `x` use `x << msg`.  For
example::

    @behavior
    def my_beh(n, self, msg):
        if n > 0:
            another = self.create(my_beh, n-1)
            another << 'a message'
        self.become(sink_beh)

    @behavior
    def sink_beh(self, msg):
        pass

"""

from functools import wraps, partial
import pprint
import sys
import traceback

from .singleton import Singleton
from .eventloop import EventLoop

class AbstractRuntime(object, metaclass=Singleton):

    def create(self, behavior, *args):
        raise NotImplementedError()

    def throw(self, message):
        raise NotImplementedError()

class SimpleRuntime(AbstractRuntime):

    def __init__(self):
        super().__init__()
        self.loop = EventLoop()

    def create(self, behavior, *args):
        return Actor(self, behavior, *args)

    def throw(self, message):
        print('ERROR: {0}'.format(pprint.pformat(message)))

class Runtime(SimpleRuntime):

    def throw(self, message):
        super().throw(message)
        # also display human readable traceback, if possible
        try:
            print('\n' + ''.join(message['traceback']))
        except (TypeError, KeyError):
            pass

class ThreadedRuntime(Runtime):

    def __init__(self):
        super().__init__()
        self.restart()

    def pause(self):
        self.evloop.stop()

    def restart(self):
        self.evloop = EventLoop()
        self.evloop.run_in_thread()

def exception_message():
    """Create a message with details on the exception."""
    exc_type, exc_value, exc_tb = exc_info = sys.exc_info()
    return {'exception': {'type': exc_type,
                          'value': exc_value,
                          'traceback': exc_tb},
            'traceback': traceback.format_exception(*exc_info)}

class Actor(object):

    def __init__(self, runtime, behavior, *args):
        self._runtime = runtime
        self.become(behavior, *args)
        self._loop = self._runtime.loop

    def become(self, behavior, *args):
        self._behavior = partial(behavior, *args)

    def send(self, msg):
        def event():
            try:
                self._behavior(self, msg)
            except Exception as exc:
                self.throw(exception_message())
        self._loop.schedule(self, event)

    def create(self, behavior, *args):
        return self._runtime.create(behavior, *args)

    def throw(self, message):
        self._runtime.throw(message)

    def __lshift__(self, msg):
        """Syntax sugar for sending a message.

        Use as `x << msg`.

        """
        self.send(msg)

    def __call__(self, msg):
        self.send(msg)

def behavior(f):
    """Decorator for declaring a function as behavior.

    Use as::

        @behavior
        def fun(x, y, self, msg):
            ...

    And create or become this behavior by passing the two arguments
    `x` and `y`.

    """
    return f

<|endoftext|>


<commit_before>from .plugins import *
import time
import pickle
import os
import logging
import ws.bad as bad
import urllib.error
import http.client
from .tools import misc
import multiprocessing
import pandas as pd
import datetime
import numpy as np

def generate_forecast_filepath(pname, city, basepath=''):
    """Generate forecast filepath.

    basepath/city/pname
    """
    posix_time = time.time()
    utc_posix_time = posix_time + time.timezone

    forecast_dir = os.path.join(basepath, city, pname)
    # exist_ok=True to make the function thread safe.
    os.makedirs(forecast_dir, exist_ok=True)

    filename = str(utc_posix_time).replace('.', 's', 1) + '.forecast'
    forecast_path = os.path.join(forecast_dir, filename)

    return forecast_path

def get_citylist():
    """Return list with all city
    names."""
    fp = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'citylist.dump')
    citylist = pickle.load(open(fp, 'rb'))
    citylist = [str(i) for i in citylist]

    return citylist

def store_forecast(city, pname, basepath=''):
    """Store forecast for city from plugin pname."""
    logging.debug('args city=%s pname=%s basepath=%s', city, pname,
                 basepath)
    forecast_data = None
    p = load_plugin(str(pname))

    try:
        url = p.build_url(str(city))
    except bad.City:
        logging.error('plugin %s cannot deal with city %s', pname, city)
        return -1

    failcounter = 0
    continue_loop = True
    while continue_loop:
        try:
            try:
                url = p.build_url(str(city))
            except bad.City:
                logging.error('plugin %s cannot deal with city %s', pname, city)
                return -1
            forecast_data = misc.download_from_url(url)
            continue_loop = False
            if failcounter == 0:
                logging.info('Queried %s for %s successfully', pname, city)
        except urllib.error.HTTPError as err:
            failcounter += 1
            logging.error("%s for url %s", err, url)
            logging.info("Trying again...")
            logging.info("This was attempt number " +str(failcounter))
        except http.client.IncompleteRead as err:
            logging.error("%s", err)

        if failcounter > 0 and continue_loop == False:
            logging.info("SUCCESS! This time querying %s worked", pname)

        if continue_loop == False:
            break

    # save the data to disk
    filepath = generate_forecast_filepath(pname, city, basepath)
    misc.save_to_disk(forecast_data, filepath)

    # save all the data also into a temporary directory
    basepath_temp = os.path.join(basepath, 'temp')
    filepath_temp = generate_forecast_filepath(pname, city, basepath_temp)
    misc.save_to_disk(forecast_data, filepath_temp)

    return forecast_data

def store_forecasts_loop(cities, pname, basepath=''):
    try:
        for city in list(cities):
            store_forecast(city, pname, basepath)
    except KeyboardInterrupt as err:
        raise KeyboardInterrupt(err)

def store_forecasts(cities, pnames, basepath=''):
    """store_forecast but takes list of cities and plugin names.

    Each plugin gets its own process. This way a plugin can rate limit without
    blocking the others.
    """
    # delete the old temporary directory that stores forecasts to be pandized later
    basepath_temp = os.path.join(basepath, 'temp')
    os.remove(basepath_temp)

    for pname in list(pnames):
        p = multiprocessing.Process(target=store_forecasts_loop,
                                    args=(cities, pname, basepath))
        p.start()

def forecasts_newer_than(newer_than, basepath=''):
    forecast_lists = {}
    for city in os.listdir(basepath):
        for provider in os.listdir(os.path.join(basepath, city)):
            if provider not in forecast_lists:
                forecast_lists[provider] = []
            for forecast in os.listdir(os.path.join(basepath, city, provider)):
                utc_posix_time = float(forecast.replace('s', '.',
                                                        1)[:-len('.forecast')])
                if utc_posix_time > newer_than:
                    filepath = os.path.join(basepath, city, provider, forecast)
                    with open(filepath, 'r') as fd:
                        forecast_lists[provider].append([fd.read(), city,
                                                         datetime.datetime.fromtimestamp(utc_posix_time)])

    return forecast_lists

def pandize_plugin_forecasts(forecast_lists, pname, database_filepath):
    p = load_plugin(str(pname))
    for forecast_list in forecast_lists:
        logging.debug('pname %s city %s date %s', pname, forecast_list[1],
                      forecast_list[2])
        try:
            pandas_table = p.pandize(*forecast_list)
            insert_into_master_frame(pandas_table)
        except Exception:
            logging.error("Error while parsing one forecast in " + str(pname))

# this is the overall pandize function that will loop over all plugins
# and then call pandize_plugin_forecasts separately for each plugin
def pandize_forecasts(pnames, database_filepath='', basepath='', newer_than=0):
    global master_frame
    forecast_lists = forecasts_newer_than(newer_than, basepath)
    for pname in list(pnames):
        pandize_plugin_forecasts(forecast_lists[pname], pname,
                                 database_filepath)

    # save the master pandas dataframe
    # this ist just TEMPORARY
    pickle.dump(master_frame, open("master_pandas_file.dump", "wb"))

def insert_into_master_frame(pandas_part):
    global master_frame

    if type(pandas_part) == pd.core.frame.DataFrame:
       master_frame = master_frame.append(pandas_part)
       logging.info("Pandized successfully")
    else:
       logging.info("One pandized row not entered into master frame because" + \
               " received -1 from pandize function. This an expected problem" + \
               " unless you would see it happening a lots (hundreds) of times")

# yes, this solution  of having the master frame as a global variable is terrible,
# but there's no way around it (that's not extremely
# inconvenient).
master_frame = pd.DataFrame(columns=
    np.array(['Provider','ref_date','city','pred_offset','Station ID', 'Date', \
        'Quality Level', 'Air Temperature', \
        'Vapor Pressure', 'Degree of Coverage', 'Air Pressure', 'Rel Humidity', \
        'Wind Speed', 'Max Air Temp', 'Min Air Temp', 'Min Groundlvl Temp', \
        'Max Wind Speed', 'Precipitation', 'Precipitation Ind', 'Hrs of Sun', \
        'Snow Depth']))<commit_msg>minor edit to avoid crash<commit_after>from .plugins import *
import time
import pickle
import os
import logging
import ws.bad as bad
import urllib.error
import http.client
from .tools import misc
import multiprocessing
import pandas as pd
import datetime
import numpy as np

def generate_forecast_filepath(pname, city, basepath=''):
    """Generate forecast filepath.

    basepath/city/pname
    """
    posix_time = time.time()
    utc_posix_time = posix_time + time.timezone

    forecast_dir = os.path.join(basepath, city, pname)
    # exist_ok=True to make the function thread safe.
    os.makedirs(forecast_dir, exist_ok=True)

    filename = str(utc_posix_time).replace('.', 's', 1) + '.forecast'
    forecast_path = os.path.join(forecast_dir, filename)

    return forecast_path

def get_citylist():
    """Return list with all city
    names."""
    fp = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'citylist.dump')
    citylist = pickle.load(open(fp, 'rb'))
    citylist = [str(i) for i in citylist]

    return citylist

def store_forecast(city, pname, basepath=''):
    """Store forecast for city from plugin pname."""
    logging.debug('args city=%s pname=%s basepath=%s', city, pname,
                 basepath)
    forecast_data = None
    p = load_plugin(str(pname))

    try:
        url = p.build_url(str(city))
    except bad.City:
        logging.error('plugin %s cannot deal with city %s', pname, city)
        return -1

    failcounter = 0
    continue_loop = True
    while continue_loop:
        try:
            try:
                url = p.build_url(str(city))
            except bad.City:
                logging.error('plugin %s cannot deal with city %s', pname, city)
                return -1
            forecast_data = misc.download_from_url(url)
            continue_loop = False
            if failcounter == 0:
                logging.info('Queried %s for %s successfully', pname, city)
        except urllib.error.HTTPError as err:
            failcounter += 1
            logging.error("%s for url %s", err, url)
            logging.info("Trying again...")
            logging.info("This was attempt number " +str(failcounter))
        except http.client.IncompleteRead as err:
            logging.error("%s", err)

        if failcounter > 0 and continue_loop == False:
            logging.info("SUCCESS! This time querying %s worked", pname)

        if continue_loop == False:
            break

    # save the data to disk
    filepath = generate_forecast_filepath(pname, city, basepath)
    misc.save_to_disk(forecast_data, filepath)

    # save all the data also into a temporary directory
    basepath_temp = os.path.join(basepath, 'temp')
    filepath_temp = generate_forecast_filepath(pname, city, basepath_temp)
    misc.save_to_disk(forecast_data, filepath_temp)

    return forecast_data

def store_forecasts_loop(cities, pname, basepath=''):
    try:
        for city in list(cities):
            store_forecast(city, pname, basepath)
    except KeyboardInterrupt as err:
        raise KeyboardInterrupt(err)

def store_forecasts(cities, pnames, basepath=''):
    """store_forecast but takes list of cities and plugin names.

    Each plugin gets its own process. This way a plugin can rate limit without
    blocking the others.
    """
    # delete the old temporary directory that stores forecasts to be pandized later
    basepath_temp = os.path.join(basepath, 'temp')
    if os.path.exists("/home/el/myfile.txt"):
        os.remove(basepath_temp)

    for pname in list(pnames):
        p = multiprocessing.Process(target=store_forecasts_loop,
                                    args=(cities, pname, basepath))
        p.start()

def forecasts_newer_than(newer_than, basepath=''):
    forecast_lists = {}
    for city in os.listdir(basepath):
        for provider in os.listdir(os.path.join(basepath, city)):
            if provider not in forecast_lists:
                forecast_lists[provider] = []
            for forecast in os.listdir(os.path.join(basepath, city, provider)):
                utc_posix_time = float(forecast.replace('s', '.',
                                                        1)[:-len('.forecast')])
                if utc_posix_time > newer_than:
                    filepath = os.path.join(basepath, city, provider, forecast)
                    with open(filepath, 'r') as fd:
                        forecast_lists[provider].append([fd.read(), city,
                                                         datetime.datetime.fromtimestamp(utc_posix_time)])

    return forecast_lists

def pandize_plugin_forecasts(forecast_lists, pname, database_filepath):
    p = load_plugin(str(pname))
    for forecast_list in forecast_lists:
        logging.debug('pname %s city %s date %s', pname, forecast_list[1],
                      forecast_list[2])
        try:
            pandas_table = p.pandize(*forecast_list)
            insert_into_master_frame(pandas_table)
        except Exception:
            logging.error("Error while parsing one forecast in " + str(pname))

# this is the overall pandize function that will loop over all plugins
# and then call pandize_plugin_forecasts separately for each plugin
def pandize_forecasts(pnames, database_filepath='', basepath='', newer_than=0):
    global master_frame
    forecast_lists = forecasts_newer_than(newer_than, basepath)
    for pname in list(pnames):
        pandize_plugin_forecasts(forecast_lists[pname], pname,
                                 database_filepath)

    # save the master pandas dataframe
    # this ist just TEMPORARY
    pickle.dump(master_frame, open("master_pandas_file.dump", "wb"))

def insert_into_master_frame(pandas_part):
    global master_frame

    if type(pandas_part) == pd.core.frame.DataFrame:
       master_frame = master_frame.append(pandas_part)
       logging.info("Pandized successfully")
    else:
       logging.info("One pandized row not entered into master frame because" + \
               " received -1 from pandize function. This an expected problem" + \
               " unless you would see it happening a lots (hundreds) of times")

# yes, this solution  of having the master frame as a global variable is terrible,
# but there's no way around it (that's not extremely
# inconvenient).
master_frame = pd.DataFrame(columns=
    np.array(['Provider','ref_date','city','pred_offset','Station ID', 'Date', \
        'Quality Level', 'Air Temperature', \
        'Vapor Pressure', 'Degree of Coverage', 'Air Pressure', 'Rel Humidity', \
        'Wind Speed', 'Max Air Temp', 'Min Air Temp', 'Min Groundlvl Temp', \
        'Max Wind Speed', 'Precipitation', 'Precipitation Ind', 'Hrs of Sun', \
        'Snow Depth']))<|endoftext|>


<commit_before>import time

import django.core.exceptions
import subprocess
import random
from core.common.models import AllRequests

from django.utils import timezone
from datetime import timedelta
from django.db.models import Count, Sum
from django.http import HttpResponse
import json
import psutil
from django.db import connection

# We postpone JSON requests is server is overloaded
# Done for protection from bunch of requests from JSON

class DDOSMiddleware(object):

    sleepInterval = 5 #sec
    maxAllowedJSONRequstesPerHour = 400
    notcachedRemoteAddress = ['188.184.185.129', '188.185.80.72', '188.184.116.46', '188.184.28.86']
    blacklist = ['130.132.21.90','192.170.227.149']

    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):

        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
        try:
            x_referer = request.META.get('HTTP_REFERER')
        except:
            x_referer = ''

        dbtotalsess, dbactivesess = 0, 0
        cursor = connection.cursor()
        cursor.execute("SELECT SUM(NUM_ACTIVE_SESS), SUM(NUM_SESS) FROM ATLAS_DBA.COUNT_PANDAMON_SESSIONS")
        rows = cursor.fetchall()
        for row in rows:
            dbactivesess = row[0]
            dbtotalsess = row[1]
            break
        cursor.close()

        reqs = AllRequests(
            server = request.META.get('HTTP_HOST'),
            remote = x_forwarded_for,
            qtime = timezone.now(),
            url= request.META.get('QUERY_STRING'),
            urlview = request.path,
            referrer = x_referer[:3900] if x_referer and len(x_referer) > 3900 else x_referer,
            useragent = request.META.get('HTTP_USER_AGENT'),
            is_rejected = 0,
            load=psutil.cpu_percent(interval=1),
            mem = psutil.virtual_memory().percent,
            dbtotalsess = dbtotalsess,
            dbactivesess = dbactivesess
        )

        # we limit number of requests per hour
        #if ('json' in request.GET):
        if (not x_forwarded_for is None) and x_forwarded_for not in self.notcachedRemoteAddress:
#                x_forwarded_for = '141.108.38.22'
            startdate = timezone.now() - timedelta(hours=2)
            enddate = timezone.now()
            query = {'remote':x_forwarded_for,
                     'qtime__range': [startdate, enddate],
                     }
            countRequest = []
            countRequest.extend(AllRequests.objects.filter(**query).values('remote').exclude(urlview='/grafana/').annotate(Count('remote')))
            if len(countRequest) > 0:
                if countRequest[0]['remote__count'] > self.maxAllowedJSONRequstesPerHour or x_forwarded_for in self.blacklist:
                    reqs.is_rejected = 1
                    reqs.save()
                    return HttpResponse(json.dumps({'message':'your IP produces too many requests per hour, please try later'}), content_type='application/json')

        response = self.get_response(request)
        reqs.rtime = timezone.now()
        reqs.save()
        return response

<commit_msg>Add temp more strict protection against EI monitor<commit_after>import time

import django.core.exceptions
import subprocess
import random
from core.common.models import AllRequests

from django.utils import timezone
from datetime import timedelta
from django.db.models import Count, Sum
from django.http import HttpResponse
import json
import psutil
from django.db import connection

# We postpone JSON requests is server is overloaded
# Done for protection from bunch of requests from JSON

class DDOSMiddleware(object):

    sleepInterval = 5 #sec
    maxAllowedJSONRequstesPerHour = 400
    notcachedRemoteAddress = ['188.184.185.129', '188.185.80.72', '188.184.116.46', '188.184.28.86']
    blacklist = ['130.132.21.90','192.170.227.149']
    maxAllowedJSONRequstesPerHourEI = 100

    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):

        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
        try:
            x_referer = request.META.get('HTTP_REFERER')
        except:
            x_referer = ''

        dbtotalsess, dbactivesess = 0, 0
        cursor = connection.cursor()
        cursor.execute("SELECT SUM(NUM_ACTIVE_SESS), SUM(NUM_SESS) FROM ATLAS_DBA.COUNT_PANDAMON_SESSIONS")
        rows = cursor.fetchall()
        for row in rows:
            dbactivesess = row[0]
            dbtotalsess = row[1]
            break
        cursor.close()

        reqs = AllRequests(
            server = request.META.get('HTTP_HOST'),
            remote = x_forwarded_for,
            qtime = timezone.now(),
            url= request.META.get('QUERY_STRING'),
            urlview = request.path,
            referrer = x_referer[:3900] if x_referer and len(x_referer) > 3900 else x_referer,
            useragent = request.META.get('HTTP_USER_AGENT'),
            is_rejected = 0,
            load=psutil.cpu_percent(interval=1),
            mem = psutil.virtual_memory().percent,
            dbtotalsess = dbtotalsess,
            dbactivesess = dbactivesess
        )

        # we limit number of requests per hour
        #if ('json' in request.GET):
        if (not x_forwarded_for is None) and x_forwarded_for not in self.notcachedRemoteAddress:
#                x_forwarded_for = '141.108.38.22'
            startdate = timezone.now() - timedelta(hours=2)
            enddate = timezone.now()
            query = {'remote':x_forwarded_for,
                     'qtime__range': [startdate, enddate],
                     }
            countRequest = []
            countRequest.extend(AllRequests.objects.filter(**query).values('remote').exclude(urlview='/grafana/').annotate(Count('remote')))
            if len(countRequest) > 0:
                if countRequest[0]['remote__count'] > self.maxAllowedJSONRequstesPerHour or x_forwarded_for in self.blacklist:
                    reqs.is_rejected = 1
                    reqs.save()
                    return HttpResponse(json.dumps({'message':'your IP produces too many requests per hour, please try later'}), status=429, content_type='application/json')

            # temporary protaction against EI monitor
            try:
                useragent = request.META.get('HTTP_USER_AGENT')
            except:
                useragent = None
                pass
            if useragent and useragent.startswith('EI-moniror'):
                countEIrequests = []
                query['useragent'] = useragent
                countEIrequests.extend(AllRequests.objects.filter(**query).values('remote').exclude(urlview='/grafana/').annotate(
                        Count('remote')))
                if len(countEIrequests) > 0:
                    if countEIrequests[0]['remote__count'] > self.maxAllowedJSONRequstesPerHourEI or x_forwarded_for in self.blacklist:
                        reqs.is_rejected = 1
                        reqs.save()
                        return HttpResponse(
                            json.dumps({'message': 'your IP produces too many requests per hour, please try later'}),
                            status=429,
                            content_type='application/json')

        response = self.get_response(request)
        reqs.rtime = timezone.now()
        reqs.save()
        return response

<|endoftext|>


<commit_before>import logging
from core.common.models import AllRequests

from django.utils import timezone
from datetime import timedelta, datetime
from django.db.models import Count, Sum
from django.http import HttpResponse
import json
import psutil
from django.db import connection

_logger = logging.getLogger('bigpandamon')
# We postpone JSON requests is server is overloaded
# Done for protection from bunch of requests from JSON

class DDOSMiddleware(object):

    sleepInterval = 5 #sec
    maxAllowedJSONRequstesPerHour = 400
    notcachedRemoteAddress = ['188.184.185.129', '188.185.80.72', '188.184.116.46', '188.184.28.86',
                              '188.184.90.172' # J..h M......n request
                              ]
    blacklist = ['130.132.21.90','192.170.227.149']
    maxAllowedJSONRequstesParallelEI = 3
    maxAllowedSimultaneousRequestsToFileBrowser = 5
    listOfServerBackendNodesIPs = ['188.184.93.101', '188.184.116.46', '188.184.104.150',
                                   '188.184.84.149', '188.184.108.134', '188.184.108.131']

    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):

        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
        try:
            x_referer = request.META.get('HTTP_REFERER')
        except:
            x_referer = ''
        dbtotalsess, dbactivesess = 0, 0
        cursor = connection.cursor()
        cursor.execute("SELECT SUM(NUM_ACTIVE_SESS), SUM(NUM_SESS) FROM ATLAS_DBA.COUNT_PANDAMON_SESSIONS")
        rows = cursor.fetchall()
        for row in rows:
            dbactivesess = row[0]
            dbtotalsess = row[1]
            break

        sqlRequest = "SELECT ATLAS_PANDABIGMON.ALL_REQUESTS_SEQ.NEXTVAL as my_req_token FROM dual;"
        cursor.execute(sqlRequest)
        requestToken = cursor.fetchall()
        requestToken = requestToken[0][0]
        cursor.close()

        reqs = AllRequests(
            id = requestToken,
            server = request.META.get('HTTP_HOST'),
            remote = x_forwarded_for,
            qtime = timezone.now(),
            url= request.META.get('QUERY_STRING'),
            urlview = request.path,
            referrer = x_referer[:3900] if x_referer and len(x_referer) > 3900 else x_referer,
            useragent = request.META.get('HTTP_USER_AGENT'),
            is_rejected = 0,
            load=psutil.cpu_percent(interval=1),
            mem = psutil.virtual_memory().percent,
            dbtotalsess = dbtotalsess,
            dbactivesess = dbactivesess
        )
        reqs.save()

        # we limit number of requests per hour
        # temporary protection against EI monitor
        try:
            useragent = request.META.get('HTTP_USER_AGENT')
        except:
            useragent = None

        if useragent and 'EI-monitor' in useragent:
            _logger.debug('[DDOS protection] got request from agent: {}'.format(useragent))
            countEIrequests = []
            startdate = datetime.utcnow() - timedelta(hours=1)
            enddate = datetime.utcnow()
            eiquery = {
                'qtime__range': [startdate, enddate],
                'useragent': useragent,
                'is_rejected': 0,
                'rtime': None,
            }
            countEIrequests.extend(
                AllRequests.objects.filter(**eiquery).values('remote').exclude(urlview='/grafana/').annotate(
                    Count('remote')))
            if len(countEIrequests) > 0 and 'remote__count' in countEIrequests[0]:
                _logger.debug('[DDOS protection] found number of non rejected request for last minute: {}'.format(countEIrequests[0]['remote__count']))
                if countEIrequests[0]['remote__count'] > self.maxAllowedJSONRequstesParallelEI:
                    reqs.is_rejected = 1
                    reqs.save()
                    return HttpResponse(
                        json.dumps({'message': 'your IP produces too many requests per hour, please try later'}),
                        status=429,
                        content_type='application/json')

            response = self.get_response(request)
            reqs.rtime = datetime.utcnow()
            reqs.save()
            return response

        #if (1==1):
        #if ('json' in request.GET):
        if (not x_forwarded_for is None) and x_forwarded_for not in self.notcachedRemoteAddress:
                # x_forwarded_for = '141.108.38.22'
            startdate = datetime.utcnow() - timedelta(hours=1)
            enddate = datetime.utcnow()
            query = {
                'remote':x_forwarded_for,
                'qtime__range': [startdate, enddate],
                'is_rejected': 0,
                 }
            countRequest = []
            countRequest.extend(AllRequests.objects.filter(**query).values('remote').exclude(urlview='/grafana/').annotate(Count('remote')))

            #Check against general number of request
            if len(countRequest) > 0:
                if countRequest[0]['remote__count'] > self.maxAllowedJSONRequstesPerHour or x_forwarded_for in self.blacklist:
                    reqs.is_rejected = 1
                    reqs.save()
                    return HttpResponse(json.dumps({'message':'your IP produces too many requests per hour, please try later'}), status=429, content_type='application/json')

        #Check against number of unprocessed requests to filebrowser from ART subsystem
        #if 1==1:
        if request.path == '/filebrowser/' and x_forwarded_for in self.listOfServerBackendNodesIPs:
            startdate = datetime.utcnow() - timedelta(minutes=20)
            enddate = datetime.utcnow()
            query = {
                'qtime__range': [startdate, enddate],
                'is_rejected': 0,
                'urlview': '/filebrowser/',
                'rtime': None,
                 }
            countRequest = []
            countRequest.extend(AllRequests.objects.filter(**query).values('remote').annotate(Count('urlview')))
            if len(countRequest) > 0:
                if countRequest[0]['urlview__count'] > self.maxAllowedSimultaneousRequestsToFileBrowser or x_forwarded_for in self.blacklist:
                    reqs.is_rejected = 1
                    reqs.save()
                    return HttpResponse(
                        json.dumps({'message': 'your IP produces too many requests per hour, please try later'}),
                        status=429, content_type='application/json')

        response = self.get_response(request)
        reqs.rtime = datetime.utcnow()
        reqs.save()
        return response

<commit_msg>applied restriction for a particular host<commit_after>import logging
from core.common.models import AllRequests

from django.utils import timezone
from datetime import timedelta, datetime
from django.db.models import Count, Sum
from django.http import HttpResponse
import json
import psutil
from django.db import connection

_logger = logging.getLogger('bigpandamon')
# We postpone JSON requests is server is overloaded
# Done for protection from bunch of requests from JSON

class DDOSMiddleware(object):

    sleepInterval = 5 #sec
    maxAllowedJSONRequstesPerHour = 400
    notcachedRemoteAddress = ['188.184.185.129', '188.185.80.72', '188.184.116.46', '188.184.28.86',
                              '188.184.90.172' # J..h M......n request
                              ]
    blacklist = ['130.132.21.90','192.170.227.149']
    maxAllowedJSONRequstesParallel = 1
    maxAllowedSimultaneousRequestsToFileBrowser = 5
    listOfServerBackendNodesIPs = ['188.184.93.101', '188.184.116.46', '188.184.104.150',
                                   '188.184.84.149', '188.184.108.134', '188.184.108.131']

    restrictedIPs = ['137.138.77.2', # Incident on 13-01-2020 14:30:00
                     '188.185.76.164', #EI Machine
                     '147.156.116.63', #EI Machine
                     '147.156.116.43', #EI Machine
                     '147.156.116.44', #EI Machine
                     '147.156.116.81', #EI Machine
                     '147.156.116.83', #EI Machine
                     ]

    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):

        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
        try:
            x_referer = request.META.get('HTTP_REFERER')
        except:
            x_referer = ''
        dbtotalsess, dbactivesess = 0, 0
        cursor = connection.cursor()
        cursor.execute("SELECT SUM(NUM_ACTIVE_SESS), SUM(NUM_SESS) FROM ATLAS_DBA.COUNT_PANDAMON_SESSIONS")
        rows = cursor.fetchall()
        for row in rows:
            dbactivesess = row[0]
            dbtotalsess = row[1]
            break

        sqlRequest = "SELECT ATLAS_PANDABIGMON.ALL_REQUESTS_SEQ.NEXTVAL as my_req_token FROM dual;"
        cursor.execute(sqlRequest)
        requestToken = cursor.fetchall()
        requestToken = requestToken[0][0]
        cursor.close()

        reqs = AllRequests(
            id = requestToken,
            server = request.META.get('HTTP_HOST'),
            remote = x_forwarded_for,
            qtime = timezone.now(),
            url= request.META.get('QUERY_STRING'),
            urlview = request.path,
            referrer = x_referer[:3900] if x_referer and len(x_referer) > 3900 else x_referer,
            useragent = request.META.get('HTTP_USER_AGENT'),
            is_rejected = 0,
            load=psutil.cpu_percent(interval=1),
            mem = psutil.virtual_memory().percent,
            dbtotalsess = dbtotalsess,
            dbactivesess = dbactivesess
        )
        reqs.save()

        # we limit number of requests per hour for a set of IPs
        try:
            useragent = request.META.get('HTTP_USER_AGENT')
        except:
            useragent = None

        if (not x_forwarded_for is None) and x_forwarded_for in self.restrictedIPs:
            _logger.debug('[DDOS protection] got request from agent: {}'.format(useragent))
            countRestictedrequests = []
            startdate = datetime.utcnow() - timedelta(hours=1)
            enddate = datetime.utcnow()
            eiquery = {
                'qtime__range': [startdate, enddate],
                'remote':x_forwarded_for,
                'is_rejected': 0,
                'rtime': None,
            }
            countRestictedrequests.extend(
                AllRequests.objects.filter(**eiquery).values('remote').exclude(urlview='/grafana/').annotate(
                    Count('remote')))
            if len(countRestictedrequests) > 0 and 'remote__count' in countRestictedrequests[0]:
                _logger.debug('[DDOS protection] found number of non rejected request for last minute: {}'.format(countRestictedrequests[0]['remote__count']))
                if countRestictedrequests[0]['remote__count'] > self.maxAllowedJSONRequstesParallel:
                    reqs.is_rejected = 1
                    reqs.save()
                    return HttpResponse(
                        json.dumps({'message': 'your IP produces too many requests per hour, please try later'}),
                        status=429,
                        content_type='application/json')

            response = self.get_response(request)
            reqs.rtime = datetime.utcnow()
            reqs.save()
            return response

        #We restrinct number of requets per hour
        if (not x_forwarded_for is None) and x_forwarded_for not in self.notcachedRemoteAddress:
                # x_forwarded_for = '141.108.38.22'
            startdate = datetime.utcnow() - timedelta(hours=1)
            enddate = datetime.utcnow()
            query = {
                'remote':x_forwarded_for,
                'qtime__range': [startdate, enddate],
                'is_rejected': 0,
                 }
            countRequest = []
            countRequest.extend(AllRequests.objects.filter(**query).values('remote').exclude(urlview='/grafana/').annotate(Count('remote')))

            #Check against general number of request
            if len(countRequest) > 0:
                if countRequest[0]['remote__count'] > self.maxAllowedJSONRequstesPerHour or x_forwarded_for in self.blacklist:
                    reqs.is_rejected = 1
                    reqs.save()
                    return HttpResponse(json.dumps({'message':'your IP produces too many requests per hour, please try later'}), status=429, content_type='application/json')

        #Check against number of unprocessed requests to filebrowser from ART subsystem
        #if 1==1:
        if request.path == '/filebrowser/' and x_forwarded_for in self.listOfServerBackendNodesIPs:
            startdate = datetime.utcnow() - timedelta(minutes=20)
            enddate = datetime.utcnow()
            query = {
                'qtime__range': [startdate, enddate],
                'is_rejected': 0,
                'urlview': '/filebrowser/',
                'rtime': None,
                 }
            countRequest = []
            countRequest.extend(AllRequests.objects.filter(**query).values('remote').annotate(Count('urlview')))
            if len(countRequest) > 0:
                if countRequest[0]['urlview__count'] > self.maxAllowedSimultaneousRequestsToFileBrowser or x_forwarded_for in self.blacklist:
                    reqs.is_rejected = 1
                    reqs.save()
                    return HttpResponse(
                        json.dumps({'message': 'your IP produces too many requests per hour, please try later'}),
                        status=429, content_type='application/json')

        response = self.get_response(request)
        reqs.rtime = datetime.utcnow()
        reqs.save()
        return response

<|endoftext|>


<commit_before>from core.messages import *
from collections import OrderedDict
from core.apistatus import *<commit_msg>update<commit_after>from core.messages import *
from collections import OrderedDict
from core.apistatus import *
from core.exceptions import ModuleError<|endoftext|>


<commit_before># Convert GPO Fdsys STATUTE metadata into bill files.
#
# GPO has the Statutes at Large from 1951 (the 65th
# volume, 82nd Congress) to the present, with metadata
# at the level of the law.
#
# The bill files have sort of made up action entries
# since we don't know the legislative history of the bill.
# We also assume all bills are enacted by being signed
# by the President for the sake of outputting status
# information.
#
# First download the Statutes at Large from GPO:
#
# ./run fdsys --collections=STATUTE --store=mods
#
# Then run this script:
#
# ./run statutes
# Processes all downloaded statutes files and saves
# bill files (e.g. data/82/bills/hr/hr1/data.json).
#
# ./run statutes --volume=65
# ./run statutes --volumes=65-86
# ./run statutes --year=1951
# ./run statutes --years=1951-1972
# Processes just the indicated volume or range of volumes.
# Starting with the 93rd Congress (1973-1974, corresponding
# to volume 78 of the Statutes of Large), we have bill
# data from THOMAS. Be careful not to overwrite those files.

import logging
import time, datetime
from lxml import etree
import glob

import utils
import bill_info

def run(options):
  root_dir = utils.data_dir() + '/fdsys/STATUTE'

  if "volume" in options:
    to_fetch = glob.glob(root_dir + "/*/STATUTE-" + str(int(options["volume"])))
  elif "volumes" in options:
    start, end = options["volumes"].split("-")
    to_fetch = []
    for v in xrange(int(start), int(end)+1):
      to_fetch.extend(glob.glob(root_dir + "/*/STATUTE-" + str(v)))
  elif "year" in options:
    to_fetch = glob.glob(root_dir + "/" + str(int(options["year"])) + "/STATUTE-*")
  elif "years" in options:
    start, end = options["years"].split("-")
    to_fetch = []
    for y in xrange(int(start), int(end)+1):
      to_fetch.extend(glob.glob(root_dir + "/" + str(y) + "/STATUTE-*"))
  else:
    to_fetch = sorted(glob.glob(root_dir + "/*/STATUTE-*"))

  logging.warn("Going to process %i volumes" % len(to_fetch))

  utils.process_set(to_fetch, proc_statute, options)

def proc_statute(path, options):
  mods = etree.parse(path + "/mods.xml")
  mods_ns = { "mods": "http://www.loc.gov/mods/v3" }

  # Load the THOMAS committee names for this Congress, which is our best
  # bet for normalizing committee names in the GPO data.
  congress = mods.find( "/mods:extension[2]/mods:congress", mods_ns ).text
  utils.fetch_committee_names(congress, options)

  logging.warn("Processing %s (Congress %s)" % (path, congress))

  for bill in mods.findall( "/mods:relatedItem", mods_ns ):
    titles = []

    titles.append( {
      "title": bill.find( "mods:titleInfo/mods:title", mods_ns ).text.replace( '""', '"' ),
      "as": "enacted",
      "type": "official",
    } )

    descriptor = bill.find( "mods:extension/mods:descriptor", mods_ns )

    if descriptor is not None:
      subject = descriptor.text
    else:
      subject = None

    # MODS files also contain information about:
    # ['BACKMATTER', 'FRONTMATTER', 'CONSTAMEND', 'PROCLAMATION', 'REORGPLAN']
    if bill.find( "mods:extension/mods:granuleClass", mods_ns ).text not in [ "PUBLICLAW", "PRIVATELAW", "HCONRES", "SCONRES" ]:
      continue

    committees = []

    cong_committee = bill.find( "mods:extension/mods:congCommittee", mods_ns )

    if cong_committee is not None:
      chambers = { "H": "House", "S": "Senate", "J": "Joint" }

      committee = chambers[cong_committee.attrib["chamber"]] + " " + cong_committee.find( "mods:name", mods_ns ).text

      committee_info = {
        "committee": committee,
        "activity": [], # XXX
        "committee_id": utils.committee_names[committee] if committee in utils.committee_names else None,
      }

      committees.append( committee_info )

    bill_elements = bill.findall( "mods:extension/mods:bill", mods_ns )

    if ( bill_elements is None ) or ( len( bill_elements ) != 1 ):
      logging.error("Could not get bill data for %s" % repr(titles) )
      continue
    else:
      bill_congress = bill_elements[0].attrib["congress"]
      bill_type = bill_elements[0].attrib["type"].lower()
      bill_number = bill_elements[0].attrib["number"]
      bill_id = "%s%s-%s" % (bill_type, bill_number, bill_congress)

    access_id = bill.find( "mods:extension/mods:accessId", mods_ns ).text
    granule_date = bill.find( "mods:extension/mods:granuleDate", mods_ns ).text

    actions = []

    law_elements = bill.findall( "mods:extension/mods:law", mods_ns )

    # XXX: If <law> is missing, this assumes it is a concurrent resolution.
    #      This may be a problem if the code is updated to accept joint resolutions for constitutional amendments.
    if ( law_elements is None ) or ( len( law_elements ) != 1 ):
      other_chamber = { "HOUSE": "s", "SENATE": "h" }

      action = {
        "type": "vote",
        "vote_type": "vote2",
        "where": other_chamber[bill.find( "mods:extension/mods:originChamber", mods_ns ).text],
        "result": "pass", # XXX
        "how": "unknown", # XXX
#        "text": "",
        "acted_at": granule_date, # XXX
        "status": "PASSED:CONCURRENTRES",
        "references": [], # XXX
        "sources": [ access_id ],
      }
    else:
      law_congress = law_elements[0].attrib["congress"]
      law_number = law_elements[0].attrib["number"]
      law_type = ( "private" if ( law_elements[0].attrib["isPrivate"] == "true" ) else "public" )

      action = {
        "congress": law_congress,
        "number": law_number,
        "type": "enacted",
        "law": law_type,
        "text": "Became %s Law No: %s-%s." % ( law_type.capitalize(), law_congress, law_number ),
        "acted_at": granule_date, # XXX
        "status": "ENACTED:SIGNED", # XXX: Check for overridden vetoes!
        "references": [], # XXX
        "sources": [ access_id ],
      }

    actions.append( action )

    # Check for typos in the metadata.
    if law_congress != bill_congress:
      logging.error("Congress mismatch for %s%s: %s or %s?" % ( bill_type, bill_number, bill_congress, law_congress ) )
      continue

    status, status_date = bill_info.latest_status( actions )

    bill_data = {
      'bill_id': bill_id,
      'bill_type': bill_type,
      'number': bill_number,
      'congress': bill_congress,

      'introduced_at': None, # XXX
      'sponsor': None, # XXX
      'cosponsors': [], # XXX

      'actions': actions, # XXX
      'history': bill_info.history_from_actions( actions ),
      'status': status,
      'status_at': status_date,
      'enacted_as': bill_info.slip_law_from( actions ),

      'titles': titles,
      'official_title': bill_info.current_title_for( titles, "official" ),
      'short_title': bill_info.current_title_for( titles, "short" ), # XXX
      'popular_title': bill_info.current_title_for( titles, "popular" ), # XXX

#      'summary': summary,
      'subjects_top_term': subject,
      'subjects': [],

      'related_bills': [], # XXX: <associatedBills> usually only lists the current bill.
      'committees': committees,
      'amendments': [], # XXX

      'updated_at': datetime.datetime.fromtimestamp(time.time()),
    }

    bill_info.output_bill( bill_data, options )

    # XXX: Can't use bill_versions.fetch_version() because it depends on fdsys.
    version_code = "enr"
    bill_version_id = "%s%s-%s-%s" % ( bill_type, bill_number, bill_congress, version_code )
    bill_version = {
      'bill_version_id': bill_version_id,
      'version_code': version_code,
      'issued_on': status_date,
      'urls': { "pdf": bill.find( "mods:location/mods:url[@displayLabel='PDF rendition']", mods_ns ).text },
    }
    import json, bill_versions
    utils.write(
      json.dumps(bill_version, sort_keys=True, indent=2, default=utils.format_datetime),
      bill_versions.output_for_bill_version(bill_version_id)
    )

  return {'ok': True, 'saved': True}
<commit_msg>Add more detailed information source.<commit_after># Convert GPO Fdsys STATUTE metadata into bill files.
#
# GPO has the Statutes at Large from 1951 (the 65th
# volume, 82nd Congress) to the present, with metadata
# at the level of the law.
#
# The bill files have sort of made up action entries
# since we don't know the legislative history of the bill.
# We also assume all bills are enacted by being signed
# by the President for the sake of outputting status
# information.
#
# First download the Statutes at Large from GPO:
#
# ./run fdsys --collections=STATUTE --store=mods
#
# Then run this script:
#
# ./run statutes
# Processes all downloaded statutes files and saves
# bill files (e.g. data/82/bills/hr/hr1/data.json).
#
# ./run statutes --volume=65
# ./run statutes --volumes=65-86
# ./run statutes --year=1951
# ./run statutes --years=1951-1972
# Processes just the indicated volume or range of volumes.
# Starting with the 93rd Congress (1973-1974, corresponding
# to volume 78 of the Statutes of Large), we have bill
# data from THOMAS. Be careful not to overwrite those files.

import logging
import time, datetime
from lxml import etree
import glob

import utils
import bill_info

def run(options):
  root_dir = utils.data_dir() + '/fdsys/STATUTE'

  if "volume" in options:
    to_fetch = glob.glob(root_dir + "/*/STATUTE-" + str(int(options["volume"])))
  elif "volumes" in options:
    start, end = options["volumes"].split("-")
    to_fetch = []
    for v in xrange(int(start), int(end)+1):
      to_fetch.extend(glob.glob(root_dir + "/*/STATUTE-" + str(v)))
  elif "year" in options:
    to_fetch = glob.glob(root_dir + "/" + str(int(options["year"])) + "/STATUTE-*")
  elif "years" in options:
    start, end = options["years"].split("-")
    to_fetch = []
    for y in xrange(int(start), int(end)+1):
      to_fetch.extend(glob.glob(root_dir + "/" + str(y) + "/STATUTE-*"))
  else:
    to_fetch = sorted(glob.glob(root_dir + "/*/STATUTE-*"))

  logging.warn("Going to process %i volumes" % len(to_fetch))

  utils.process_set(to_fetch, proc_statute, options)

def proc_statute(path, options):
  mods = etree.parse(path + "/mods.xml")
  mods_ns = { "mods": "http://www.loc.gov/mods/v3" }

  # Load the THOMAS committee names for this Congress, which is our best
  # bet for normalizing committee names in the GPO data.
  congress = mods.find( "/mods:extension[2]/mods:congress", mods_ns ).text
  utils.fetch_committee_names(congress, options)

  logging.warn("Processing %s (Congress %s)" % (path, congress))

  package_id = mods.find( "/mods:extension[2]/mods:accessId", mods_ns ).text

  for bill in mods.findall( "/mods:relatedItem", mods_ns ):
    titles = []

    titles.append( {
      "title": bill.find( "mods:titleInfo/mods:title", mods_ns ).text.replace( '""', '"' ),
      "as": "enacted",
      "type": "official",
    } )

    descriptor = bill.find( "mods:extension/mods:descriptor", mods_ns )

    if descriptor is not None:
      subject = descriptor.text
    else:
      subject = None

    # MODS files also contain information about:
    # ['BACKMATTER', 'FRONTMATTER', 'CONSTAMEND', 'PROCLAMATION', 'REORGPLAN']
    if bill.find( "mods:extension/mods:granuleClass", mods_ns ).text not in [ "PUBLICLAW", "PRIVATELAW", "HCONRES", "SCONRES" ]:
      continue

    committees = []

    cong_committee = bill.find( "mods:extension/mods:congCommittee", mods_ns )

    if cong_committee is not None:
      chambers = { "H": "House", "S": "Senate", "J": "Joint" }

      committee = chambers[cong_committee.attrib["chamber"]] + " " + cong_committee.find( "mods:name", mods_ns ).text

      committee_info = {
        "committee": committee,
        "activity": [], # XXX
        "committee_id": utils.committee_names[committee] if committee in utils.committee_names else None,
      }

      committees.append( committee_info )

    bill_elements = bill.findall( "mods:extension/mods:bill", mods_ns )

    if ( bill_elements is None ) or ( len( bill_elements ) != 1 ):
      logging.error("Could not get bill data for %s" % repr(titles) )
      continue
    else:
      bill_congress = bill_elements[0].attrib["congress"]
      bill_type = bill_elements[0].attrib["type"].lower()
      bill_number = bill_elements[0].attrib["number"]
      bill_id = "%s%s-%s" % (bill_type, bill_number, bill_congress)

    granule_date = bill.find( "mods:extension/mods:granuleDate", mods_ns ).text

    actions = []
    sources = []

    source = {
      "source": "statutes",
      "package_id": package_id,
      "access_id": bill.find( "mods:extension/mods:accessId", mods_ns ).text,
      "source_url": bill.find( "mods:location/mods:url[@displayLabel='Content Detail']", mods_ns ).text,
      "volume": bill.find( "mods:extension/mods:volume", mods_ns ).text,
      "page": bill.find( "mods:part[@type='article']/mods:extent[@unit='pages']/mods:start", mods_ns ).text,
      "position": bill.find( "mods:extension/mods:pagePosition", mods_ns ).text,
    }

    sources.append( source )

    law_elements = bill.findall( "mods:extension/mods:law", mods_ns )

    # XXX: If <law> is missing, this assumes it is a concurrent resolution.
    #      This may be a problem if the code is updated to accept joint resolutions for constitutional amendments.
    if ( law_elements is None ) or ( len( law_elements ) != 1 ):
      other_chamber = { "HOUSE": "s", "SENATE": "h" }

      action = {
        "type": "vote",
        "vote_type": "vote2",
        "where": other_chamber[bill.find( "mods:extension/mods:originChamber", mods_ns ).text],
        "result": "pass", # XXX
        "how": "unknown", # XXX
#        "text": "",
        "acted_at": granule_date, # XXX
        "status": "PASSED:CONCURRENTRES",
        "references": [], # XXX
        "sources": sources,
      }
    else:
      law_congress = law_elements[0].attrib["congress"]
      law_number = law_elements[0].attrib["number"]
      law_type = ( "private" if ( law_elements[0].attrib["isPrivate"] == "true" ) else "public" )

      action = {
        "congress": law_congress,
        "number": law_number,
        "type": "enacted",
        "law": law_type,
        "text": "Became %s Law No: %s-%s." % ( law_type.capitalize(), law_congress, law_number ),
        "acted_at": granule_date, # XXX
        "status": "ENACTED:SIGNED", # XXX: Check for overridden vetoes!
        "references": [], # XXX
        "sources": sources,
      }

    actions.append( action )

    # Check for typos in the metadata.
    if law_congress != bill_congress:
      logging.error("Congress mismatch for %s%s: %s or %s?" % ( bill_type, bill_number, bill_congress, law_congress ) )
      continue

    status, status_date = bill_info.latest_status( actions )

    bill_data = {
      'bill_id': bill_id,
      'bill_type': bill_type,
      'number': bill_number,
      'congress': bill_congress,

      'introduced_at': None, # XXX
      'sponsor': None, # XXX
      'cosponsors': [], # XXX

      'actions': actions, # XXX
      'history': bill_info.history_from_actions( actions ),
      'status': status,
      'status_at': status_date,
      'enacted_as': bill_info.slip_law_from( actions ),

      'titles': titles,
      'official_title': bill_info.current_title_for( titles, "official" ),
      'short_title': bill_info.current_title_for( titles, "short" ), # XXX
      'popular_title': bill_info.current_title_for( titles, "popular" ), # XXX

#      'summary': summary,
      'subjects_top_term': subject,
      'subjects': [],

      'related_bills': [], # XXX: <associatedBills> usually only lists the current bill.
      'committees': committees,
      'amendments': [], # XXX

      'updated_at': datetime.datetime.fromtimestamp(time.time()),
    }

    bill_info.output_bill( bill_data, options )

    # XXX: Can't use bill_versions.fetch_version() because it depends on fdsys.
    version_code = "enr"
    bill_version_id = "%s%s-%s-%s" % ( bill_type, bill_number, bill_congress, version_code )
    bill_version = {
      'bill_version_id': bill_version_id,
      'version_code': version_code,
      'issued_on': status_date,
      'urls': { "pdf": bill.find( "mods:location/mods:url[@displayLabel='PDF rendition']", mods_ns ).text },
    }
    import json, bill_versions
    utils.write(
      json.dumps(bill_version, sort_keys=True, indent=2, default=utils.format_datetime),
      bill_versions.output_for_bill_version(bill_version_id)
    )

  return {'ok': True, 'saved': True}
<|endoftext|>


<commit_before>from cumulusci.tasks.salesforce import BaseSalesforceTask

class TestTask(BaseSalesforceTask):
    def __call__(self):
        self.logger.info('testing')
<commit_msg>remove testtask<commit_after><|endoftext|>


<commit_before>from pillowtop.listener import AliasedElasticPillow
from dimagi.utils.decorators.memoized import memoized
from django.conf import settings

VALUE_TAG = '#value'

def map_types(item, mapping, override_root_keys=None):
    if isinstance(item, dict):
        return convert_property_dict(item, mapping, override_root_keys=override_root_keys)
    elif isinstance(item, list):
        return [map_types(x, mapping) for x in item]
    else:
        return {VALUE_TAG: item}

def convert_property_dict(sub_dict, mapping, override_root_keys=None):
    """
    For mapping out ALL nested properties on cases, convert everything to a dict so as to
    prevent string=>object and object=>string mapping errors.

    sub_dict: the doc dict you want to modify in place before sending to ES
    mapping: The mapping at the level of the properties you are at - originally passing as the default mapping of the pillow
    override_root_keys: a list of keys you want explicitly skipped at the root level and are not recursed down
    """
    mapping = mapping or {}
    override_root_keys = override_root_keys or []

    for k, v in sub_dict.items():
        if k in mapping.get('properties', {}) or k in override_root_keys:
            continue
        dynamic_mapping = mapping.get('dynamic', True)
        sub_mapping = mapping.get('properties', {}).get(k, {})
        if dynamic_mapping is not False:
            sub_dict[k] = map_types(v, sub_mapping, override_root_keys=override_root_keys)
    return sub_dict

def restore_property_dict(report_dict_item):
    """
    Revert a converted/retrieved document from Report<index> and deconvert all its properties
    back from {#value: <val>} to just <val>
    """
    restored = {}
    if not isinstance(report_dict_item, dict):
        return report_dict_item

    for k, v in report_dict_item.items():
        if isinstance(v, list):
            restored[k] = [restore_property_dict(x) for x in v]
        elif isinstance(v, dict):
            if VALUE_TAG in v:
                restored[k] = v[VALUE_TAG]
            else:
                restored[k] = restore_property_dict(v)
        else:
            restored[k] = v

    return restored

class HQPillow(AliasedElasticPillow):
    es_timeout = 60
    es_meta = {
        "settings": {
            "analysis": {
                "analyzer": {
                    "default": {
                        "type": "custom",
                        "tokenizer": "whitespace",
                        "filter": ["lowercase"]
                    },
                    "sortable_exact": {
                        "type": "custom",
                        "tokenizer": "keyword",
                        "filter": ["lowercase"]
                    }
                }
            }
        }
    }

    @classmethod
    @memoized
    def calc_meta(cls):
        """
        override of the meta calculator since we're separating out all the types,
        so we just do a hash of the "prototype" instead to determind md5
        """
        return cls.calc_mapping_hash(cls.default_mapping)
<commit_msg>pull out default meta to a constant<commit_after>from pillowtop.listener import AliasedElasticPillow
from dimagi.utils.decorators.memoized import memoized
from django.conf import settings

VALUE_TAG = '#value'
DEFAULT_META = {
    "settings": {
        "analysis": {
            "analyzer": {
                "default": {
                    "type": "custom",
                    "tokenizer": "whitespace",
                    "filter": ["lowercase"]
                },
                "sortable_exact": {
                    "type": "custom",
                    "tokenizer": "keyword",
                    "filter": ["lowercase"]
                }
            }
        }
    }
}

def map_types(item, mapping, override_root_keys=None):
    if isinstance(item, dict):
        return convert_property_dict(item, mapping, override_root_keys=override_root_keys)
    elif isinstance(item, list):
        return [map_types(x, mapping) for x in item]
    else:
        return {VALUE_TAG: item}

def convert_property_dict(sub_dict, mapping, override_root_keys=None):
    """
    For mapping out ALL nested properties on cases, convert everything to a dict so as to
    prevent string=>object and object=>string mapping errors.

    sub_dict: the doc dict you want to modify in place before sending to ES
    mapping: The mapping at the level of the properties you are at - originally passing as the default mapping of the pillow
    override_root_keys: a list of keys you want explicitly skipped at the root level and are not recursed down
    """
    mapping = mapping or {}
    override_root_keys = override_root_keys or []

    for k, v in sub_dict.items():
        if k in mapping.get('properties', {}) or k in override_root_keys:
            continue
        dynamic_mapping = mapping.get('dynamic', True)
        sub_mapping = mapping.get('properties', {}).get(k, {})
        if dynamic_mapping is not False:
            sub_dict[k] = map_types(v, sub_mapping, override_root_keys=override_root_keys)
    return sub_dict

def restore_property_dict(report_dict_item):
    """
    Revert a converted/retrieved document from Report<index> and deconvert all its properties
    back from {#value: <val>} to just <val>
    """
    restored = {}
    if not isinstance(report_dict_item, dict):
        return report_dict_item

    for k, v in report_dict_item.items():
        if isinstance(v, list):
            restored[k] = [restore_property_dict(x) for x in v]
        elif isinstance(v, dict):
            if VALUE_TAG in v:
                restored[k] = v[VALUE_TAG]
            else:
                restored[k] = restore_property_dict(v)
        else:
            restored[k] = v

    return restored

class HQPillow(AliasedElasticPillow):
    es_timeout = 60
    es_meta = DEFAULT_META

    @classmethod
    @memoized
    def calc_meta(cls):
        """
        override of the meta calculator since we're separating out all the types,
        so we just do a hash of the "prototype" instead to determind md5
        """
        return cls.calc_mapping_hash(cls.default_mapping)
<|endoftext|>