<commit_before>#!/usr/bin/env python3
"""
Generate starter kit downloads and a download page.
"""

import argparse
import glob
import json
import os
import zipfile

ENVIRONMENT_DIR_HELP = "Directory containing precompiled Halite environment " \
                       "executables, each named after their platform. "
VERSION_HELP = "The version string to embed in the downloads page."
IGNORED_EXTENSIONS = [".exe", ".class", ".pyc", ".obj"]
INCLUDED_EXTENSIONS = [".py", ".java", ".cpp", ".hpp", ".cs", ".csproj", ".scala", ".js", ".sh", ".bat", ".toml", ".rs",".go",".txt",".rb", ".kt", ".clj"]
INCLUDED_FILES = ["Makefile", "README"]
STARTER_KIT_DIR = "../airesources"
DOWNLOAD_DATA = "_data/downloads.json"
PLATFORM_AGNOSTIC = "None"

# Names of generated downloads
# Standard language + platform
OUTPUT_FILE_FORMAT = "assets/downloads/Halite2_{language}_{platform}.zip"

# Platform only
ENVIRONMENT_OUTPUT_FILE_FORMAT = "assets/downloads/Halite2_{platform}.zip"

# All languages + platform
ALL_LANGUAGES_OUTPUT_FILE_FORMAT = "assets/downloads/Halite2_all_{platform}.zip"
SOURCE_FILE = "assets/downloads/Halite2Source.zip"

versions =  {"Python3" : "1.0", "C++" : "1.0", "Java" : "1.0", "CSharp" : "0.9.0-beta" ,"JavaScript": "0.9-beta",
"ML-StarterBot-Python":"1.0", "Rust" : "0.9.0-beta", "Scala" : "0.9.0-beta", "Go" : "0.9.0-beta", "Ruby" : "0.9.0-beta" , "Kotlin" : "0.9.0-beta", "Clojure" : "0.9.0-beta" }

def detect_environments(directory):
    """Detect which platform binaries we have."""
    environments = [(PLATFORM_AGNOSTIC, None, None)]
    for filename in os.listdir(directory):
        platform, platform_ext = os.path.splitext(filename)

        if platform == ".DS_Store":
            # Dang it, MacOS
            continue

        print("Detected platform", platform)
        environments.append((platform,
                             os.path.join(directory, filename),
                             "halite" + platform_ext))

    return environments

def scan_directory(full_path):
    """Figure out what the starter kit files in a directory are."""

    included_files = []
    for containing_dir, _, possible_files in os.walk(full_path):
        for filename in possible_files:
            _, ext = os.path.splitext(filename)
            if ext.lower() in INCLUDED_EXTENSIONS or filename in INCLUDED_FILES:
                included_files.append(os.path.join(containing_dir, filename))

    included_files.append(os.path.join(STARTER_KIT_DIR, "README.MD"))
    return included_files

def make_archive(output, environment, base_path, included_files):
    """Create the output ZIP archive."""
    platform, source, target = environment
    # Get rid of duplicates
    included_files = list(set(included_files))
    with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as archive:
        if source is not None:
            # source is None <=> platform-agnostic archive
            zinfo = zipfile.ZipInfo.from_file(source, target)
            zinfo.compress_type = zipfile.ZIP_DEFLATED
            zinfo.external_attr = 0o0100755 << 16
            with open(source, 'rb') as source_file:
                archive.writestr(zinfo, source_file.read())

        for file in included_files:
            target_path = os.path.relpath(file, base_path)
            archive.write(file, target_path)

def make_source_download():
    included_files = []

    for directory, _, file_list in os.walk("../environment"):
        target_dir = os.path.relpath(directory, "../environment")
        for filename in file_list:
            _, ext = os.path.splitext(filename)
            if ext.lower() in {".cpp", ".c", ".hpp", ".h", ".bat"} or \
                    filename == "Makefile":
                source_path = os.path.join(directory, filename)
                target_path = os.path.normpath(
                    os.path.join("Halite/", target_dir, filename))

                included_files.append((source_path, target_path))

    with zipfile.ZipFile(SOURCE_FILE, "w", zipfile.ZIP_DEFLATED) as archive:
        for source_path, target_path in included_files:
            archive.write(source_path, target_path)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("version", help=VERSION_HELP)
    parser.add_argument("environment_dir", help=ENVIRONMENT_DIR_HELP)

    args = parser.parse_args()

    environments = detect_environments(args.environment_dir)
    generated_languages = []

    # Ensure output directories exists
    for location in [SOURCE_FILE, DOWNLOAD_DATA]:
        try:
            os.makedirs(os.path.dirname(location))
        except FileExistsError:
            pass

    make_source_download()

    # Keep paths of all source files around so we can make a single combined
    # download at the end
    all_files = []

    for directory in os.listdir(STARTER_KIT_DIR):
        full_path = os.path.join(STARTER_KIT_DIR, directory)
        if not os.path.isdir(full_path):
            continue

        if directory == "starterkitdocs":
            continue

        language = directory
        generated_languages.append(language)
        print("Language:", language)

        included_files = scan_directory(full_path)
        for file in included_files:
            print("\tIncluding:", file)

        print()

        all_files.extend(included_files)

        for (platform, source, target) in environments:
            output = "./" + OUTPUT_FILE_FORMAT.format(
                language=language, platform=platform)
            print("\tMaking:", output)
            make_archive(output, (platform, source, target),
                         full_path, included_files)

    panlanguage_kits = []
    for (platform, source, target) in environments:
        # Make downloads including all languages
        filename = ALL_LANGUAGES_OUTPUT_FILE_FORMAT.format(platform=platform)
        all_output = "./" + filename
        print("\tMaking:", all_output)
        make_archive(all_output, (platform, source, target), "../airesources", all_files)
        panlanguage_kits.append(filename)

        # Make downloads including no languages
        if source is None:
            continue
        output = "./" + ENVIRONMENT_OUTPUT_FILE_FORMAT.format(platform=platform)
        print("\tMaking:", output)
        make_archive(output, (platform, source, target), "", [])

    output = {
        "platforms": [environment[0] for environment in environments],
        "languages": [],
        "environments": [],
        "source": SOURCE_FILE,
        "version": args.version,
    }
    generated_languages.sort()
    for language in generated_languages:
        language_kits = []
        for (platform, _, _) in environments:
            language_kits.append(
                OUTPUT_FILE_FORMAT.format(
                    language=language, platform=platform))

        output["languages"].append({
            "language": language,
            "files": language_kits,
            "version": versions[language]
        })

    output["languages"].append({
        "language": "All Languages",
        "files": panlanguage_kits,
    })

    for (platform, source, _) in environments:
        if source is None:
            continue
        output["environments"].append(
            ENVIRONMENT_OUTPUT_FILE_FORMAT.format(platform=platform))

    with open(DOWNLOAD_DATA, "w") as output_file:
        json.dump(output, output_file, indent=2)

if __name__ == "__main__":
    main()
<commit_msg>Julia starter kit build<commit_after>#!/usr/bin/env python3
"""
Generate starter kit downloads and a download page.
"""

import argparse
import glob
import json
import os
import zipfile

ENVIRONMENT_DIR_HELP = "Directory containing precompiled Halite environment " \
                       "executables, each named after their platform. "
VERSION_HELP = "The version string to embed in the downloads page."
IGNORED_EXTENSIONS = [".exe", ".class", ".pyc", ".obj"]
INCLUDED_EXTENSIONS = [".py", ".java", ".cpp", ".hpp", ".cs", ".csproj", ".scala", ".js", ".sh", ".bat", ".toml", ".rs",".go",".txt",".rb", ".kt", ".clj",".jl"]
INCLUDED_FILES = ["Makefile", "README"]
STARTER_KIT_DIR = "../airesources"
DOWNLOAD_DATA = "_data/downloads.json"
PLATFORM_AGNOSTIC = "None"

# Names of generated downloads
# Standard language + platform
OUTPUT_FILE_FORMAT = "assets/downloads/Halite2_{language}_{platform}.zip"

# Platform only
ENVIRONMENT_OUTPUT_FILE_FORMAT = "assets/downloads/Halite2_{platform}.zip"

# All languages + platform
ALL_LANGUAGES_OUTPUT_FILE_FORMAT = "assets/downloads/Halite2_all_{platform}.zip"
SOURCE_FILE = "assets/downloads/Halite2Source.zip"

versions =  {"Python3" : "1.0", "C++" : "1.0", "Java" : "1.0", "CSharp" : "0.9.0-beta" ,"JavaScript": "0.9-beta",
"ML-StarterBot-Python":"1.0", "Rust" : "1.0", "Scala" : "1.0-beta", "Go" : "1.0-beta", "Ruby" : "1.0-beta" , "Kotlin" : "0.9.0-beta", "Clojure" : "0.9.0-beta", "Julia" : "0.9.0-beta"  }

def detect_environments(directory):
    """Detect which platform binaries we have."""
    environments = [(PLATFORM_AGNOSTIC, None, None)]
    for filename in os.listdir(directory):
        platform, platform_ext = os.path.splitext(filename)

        if platform == ".DS_Store":
            # Dang it, MacOS
            continue

        print("Detected platform", platform)
        environments.append((platform,
                             os.path.join(directory, filename),
                             "halite" + platform_ext))

    return environments

def scan_directory(full_path):
    """Figure out what the starter kit files in a directory are."""

    included_files = []
    for containing_dir, _, possible_files in os.walk(full_path):
        for filename in possible_files:
            _, ext = os.path.splitext(filename)
            if ext.lower() in INCLUDED_EXTENSIONS or filename in INCLUDED_FILES:
                included_files.append(os.path.join(containing_dir, filename))

    included_files.append(os.path.join(STARTER_KIT_DIR, "README.MD"))
    return included_files

def make_archive(output, environment, base_path, included_files):
    """Create the output ZIP archive."""
    platform, source, target = environment
    # Get rid of duplicates
    included_files = list(set(included_files))
    with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as archive:
        if source is not None:
            # source is None <=> platform-agnostic archive
            zinfo = zipfile.ZipInfo.from_file(source, target)
            zinfo.compress_type = zipfile.ZIP_DEFLATED
            zinfo.external_attr = 0o0100755 << 16
            with open(source, 'rb') as source_file:
                archive.writestr(zinfo, source_file.read())

        for file in included_files:
            target_path = os.path.relpath(file, base_path)
            archive.write(file, target_path)

def make_source_download():
    included_files = []

    for directory, _, file_list in os.walk("../environment"):
        target_dir = os.path.relpath(directory, "../environment")
        for filename in file_list:
            _, ext = os.path.splitext(filename)
            if ext.lower() in {".cpp", ".c", ".hpp", ".h", ".bat"} or \
                    filename == "Makefile":
                source_path = os.path.join(directory, filename)
                target_path = os.path.normpath(
                    os.path.join("Halite/", target_dir, filename))

                included_files.append((source_path, target_path))

    with zipfile.ZipFile(SOURCE_FILE, "w", zipfile.ZIP_DEFLATED) as archive:
        for source_path, target_path in included_files:
            archive.write(source_path, target_path)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("version", help=VERSION_HELP)
    parser.add_argument("environment_dir", help=ENVIRONMENT_DIR_HELP)

    args = parser.parse_args()

    environments = detect_environments(args.environment_dir)
    generated_languages = []

    # Ensure output directories exists
    for location in [SOURCE_FILE, DOWNLOAD_DATA]:
        try:
            os.makedirs(os.path.dirname(location))
        except FileExistsError:
            pass

    make_source_download()

    # Keep paths of all source files around so we can make a single combined
    # download at the end
    all_files = []

    for directory in os.listdir(STARTER_KIT_DIR):
        full_path = os.path.join(STARTER_KIT_DIR, directory)
        if not os.path.isdir(full_path):
            continue

        if directory == "starterkitdocs":
            continue

        language = directory
        generated_languages.append(language)
        print("Language:", language)

        included_files = scan_directory(full_path)
        for file in included_files:
            print("\tIncluding:", file)

        print()

        all_files.extend(included_files)

        for (platform, source, target) in environments:
            output = "./" + OUTPUT_FILE_FORMAT.format(
                language=language, platform=platform)
            print("\tMaking:", output)
            make_archive(output, (platform, source, target),
                         full_path, included_files)

    panlanguage_kits = []
    for (platform, source, target) in environments:
        # Make downloads including all languages
        filename = ALL_LANGUAGES_OUTPUT_FILE_FORMAT.format(platform=platform)
        all_output = "./" + filename
        print("\tMaking:", all_output)
        make_archive(all_output, (platform, source, target), "../airesources", all_files)
        panlanguage_kits.append(filename)

        # Make downloads including no languages
        if source is None:
            continue
        output = "./" + ENVIRONMENT_OUTPUT_FILE_FORMAT.format(platform=platform)
        print("\tMaking:", output)
        make_archive(output, (platform, source, target), "", [])

    output = {
        "platforms": [environment[0] for environment in environments],
        "languages": [],
        "environments": [],
        "source": SOURCE_FILE,
        "version": args.version,
    }
    generated_languages.sort()
    for language in generated_languages:
        language_kits = []
        for (platform, _, _) in environments:
            language_kits.append(
                OUTPUT_FILE_FORMAT.format(
                    language=language, platform=platform))

        output["languages"].append({
            "language": language,
            "files": language_kits,
            "version": versions[language]
        })

    output["languages"].append({
        "language": "All Languages",
        "files": panlanguage_kits,
    })

    for (platform, source, _) in environments:
        if source is None:
            continue
        output["environments"].append(
            ENVIRONMENT_OUTPUT_FILE_FORMAT.format(platform=platform))

    with open(DOWNLOAD_DATA, "w") as output_file:
        json.dump(output, output_file, indent=2)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.

import os
import re
import json
import time
from collections import defaultdict, OrderedDict

import requests
import demjson  # pylint: disable=import-error

LINUX_PRICING_URLS = [
    # Deprecated instances (JSON format)
    'https://aws.amazon.com/ec2/pricing/json/linux-od.json',
    # Previous generation instances (JavaScript file)
    'https://a0.awsstatic.com/pricing/1/ec2/previous-generation/linux-od.min.js',
    # New generation instances (JavaScript file)
    # Using other endpoint atm
    # 'https://a0.awsstatic.com/pricing/1/ec2/linux-od.min.js'
]

EC2_REGIONS = [
    'us-east-1',
    'us-east-2',
    'us-west-1',
    'us-west-2',
    'us-gov-west-1',
    'eu-west-1',
    'eu-west-2',
    'eu-west-3',
    'eu-north-1',
    'eu-central-1',
    'ca-central-1',
    'ap-southeast-1',
    'ap-southeast-2',
    'ap-northeast-1',
    'ap-northeast-2',
    'ap-south-1',
    'sa-east-1',
    'cn-north-1',
]

EC2_INSTANCE_TYPES = [
    't1.micro',
    'm1.small',
    'm1.medium',
    'm1.large',
    'm1.xlarge',
    'm2.xlarge',
    'm2.2xlarge',
    'm2.4xlarge',
    'm3.medium',
    'm3.large',
    'm3.xlarge',
    'm3.2xlarge',
    'c1.medium',
    'c1.xlarge',
    'cc1.4xlarge',
    'cc2.8xlarge',
    'c3.large',
    'c3.xlarge',
    'c3.2xlarge',
    'c3.4xlarge',
    'c3.8xlarge',
    'd2.xlarge',
    'd2.2xlarge',
    'd2.4xlarge',
    'd2.8xlarge',
    'cg1.4xlarge',
    'g2.2xlarge',
    'g2.8xlarge',
    'cr1.8xlarge',
    'hs1.4xlarge',
    'hs1.8xlarge',
    'i2.xlarge',
    'i2.2xlarge',
    'i2.4xlarge',
    'i2.8xlarge',
    'i3.large',
    'i3.xlarge',
    'i3.2xlarge',
    'i3.4xlarge',
    'i3.8xlarge',
    'i3.16large',
    'r3.large',
    'r3.xlarge',
    'r3.2xlarge',
    'r3.4xlarge',
    'r3.8xlarge',
    'r4.large',
    'r4.xlarge',
    'r4.2xlarge',
    'r4.4xlarge',
    'r4.8xlarge',
    'r4.16xlarge',
    't2.micro',
    't2.small',
    't2.medium',
    't2.large',
    'x1.32xlarge'
]

# Maps EC2 region name to region name used in the pricing file
REGION_NAME_MAP = {
    'us-east': 'ec2_us_east',
    'us-east-1': 'ec2_us_east',
    'us-east-2': 'ec2_us_east_ohio',
    'us-west': 'ec2_us_west',
    'us-west-1': 'ec2_us_west',
    'us-west-2': 'ec2_us_west_oregon',
    'eu-west-1': 'ec2_eu_west',
    'eu-west-2': 'ec2_eu_west_london',
    'eu-west-3': 'ec2_eu_west_3',
    'eu-ireland': 'ec2_eu_west',
    'eu-central-1': 'ec2_eu_central',
    'ca-central-1': 'ec2_ca_central_1',
    'apac-sin': 'ec2_ap_southeast',
    'ap-southeast-1': 'ec2_ap_southeast',
    'apac-syd': 'ec2_ap_southeast_2',
    'ap-southeast-2': 'ec2_ap_southeast_2',
    'apac-tokyo': 'ec2_ap_northeast',
    'ap-northeast-1': 'ec2_ap_northeast',
    'ap-northeast-2': 'ec2_ap_northeast',
    'ap-south-1': 'ec2_ap_south_1',
    'sa-east-1': 'ec2_sa_east',
    'us-gov-west-1': 'ec2_us_govwest',
    'cn-north-1': 'ec2_cn_north',
}

INSTANCE_SIZES = [
    'micro',
    'small',
    'medium',
    'large',
    'xlarge',
    'x-large',
    'extra-large'
]

RE_NUMERIC_OTHER = re.compile(r'(?:([0-9]+)|([-A-Z_a-z]+)|([^-0-9A-Z_a-z]+))')

BASE_PATH = os.path.dirname(os.path.abspath(__file__))
PRICING_FILE_PATH = os.path.join(BASE_PATH, '../libcloud/data/pricing.json')
PRICING_FILE_PATH = os.path.abspath(PRICING_FILE_PATH)

def scrape_ec2_pricing():
    result = defaultdict(OrderedDict)
    os_map = {'linux': 'ec2_linux', 'windows-std': 'ec2_windows'}
    for item in os_map.values():
        result[item] = {}
    for url in LINUX_PRICING_URLS:
        response = requests.get(url)

        if re.match(r'.*?\.json$', url):
            data = response.json()
        elif re.match(r'.*?\.js$', url):
            data = response.content.decode()
            match = re.match(r'^.*callback\((.*?)\);?$', data,
                             re.MULTILINE | re.DOTALL)
            data = match.group(1)
            # demjson supports non-strict mode and can parse unquoted objects
            data = demjson.decode(data)
        regions = data['config']['regions']

        for region_data in regions:
            region_name = region_data['region']
            instance_types = region_data['instanceTypes']

            for instance_type in instance_types:
                sizes = instance_type['sizes']
                for size in sizes:
                    if not result['ec2_linux'].get(size['size'], False):
                        result['ec2_linux'][size['size']] = {}
                    price = size['valueColumns'][0]['prices']['USD']
                    if str(price).lower() == 'n/a':
                        # Price not available
                        continue

                    result['ec2_linux'][size['size']][
                        region_name] = float(price)

    res = defaultdict(OrderedDict)
    url = ('https://calculator.aws/pricing/1.0/'
           'ec2/region/{}/ondemand/{}/index.json')
    instances = set()
    for OS in ['linux', 'windows-std']:
        res[os_map[OS]] = {}
        for region in EC2_REGIONS:
            res[os_map[OS]][region] = {}
            response = requests.get(url.format(region, OS))
            if response.status_code != 200:
                continue
            data = response.json()

            for entry in data['prices']:
                instance_type = entry['attributes'].get(
                    'aws:ec2:instanceType', "")
                instances.add(instance_type)
                price = entry['price'].get('USD', 0)
                res[os_map[OS]][region][instance_type] = price
    for item in os_map.values():
        for instance in instances:
            if not result[item].get(instance, False):
                result[item][instance] = {}
            for region in EC2_REGIONS:
                if res[item][region].get(instance, False):
                    result[item][instance][region] = float(res[
                        item][region][instance])

    return result

def update_pricing_file(pricing_file_path, pricing_data):
    with open(pricing_file_path, 'r') as fp:
        content = fp.read()

    data = json.loads(content)
    data['updated'] = int(time.time())
    data['compute'].update(pricing_data)

    # Always sort the pricing info
    data = sort_nested_dict(data)

    content = json.dumps(data, indent=4)
    lines = content.splitlines()
    lines = [line.rstrip() for line in lines]
    content = '\n'.join(lines)

    with open(pricing_file_path, 'w') as fp:
        fp.write(content)

def sort_nested_dict(value):
    """
    Recursively sort a nested dict.
    """
    result = OrderedDict()

    for key, value in sorted(value.items(), key=sort_key_by_numeric_other):
        if isinstance(value, (dict, OrderedDict)):
            result[key] = sort_nested_dict(value)
        else:
            result[key] = value

    return result

def sort_key_by_numeric_other(key_value):
    """
    Split key into numeric, alpha and other part and sort accordingly.
    """
    result = []

    for (numeric, alpha, other) in RE_NUMERIC_OTHER.findall(key_value[0]):
        numeric = int(numeric) if numeric else -1
        alpha = INSTANCE_SIZES.index(alpha) if alpha in INSTANCE_SIZES else alpha
        alpha = str(alpha)
        item = tuple([numeric, alpha, other])
        result.append(item)

    return tuple(result)

def main():
    print('Scraping EC2 pricing data (this may take up to 2 minutes)....')

    pricing_data = scrape_ec2_pricing()
    update_pricing_file(pricing_file_path=PRICING_FILE_PATH,
                        pricing_data=pricing_data)

    print('Pricing data updated')

if __name__ == '__main__':
    main()
<commit_msg>Bail early if pricing data hasn't changed.<commit_after>#!/usr/bin/env python
#
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.

import os
import re
import json
import copy
import time
from collections import defaultdict, OrderedDict

import requests
import demjson  # pylint: disable=import-error

LINUX_PRICING_URLS = [
    # Deprecated instances (JSON format)
    'https://aws.amazon.com/ec2/pricing/json/linux-od.json',
    # Previous generation instances (JavaScript file)
    'https://a0.awsstatic.com/pricing/1/ec2/previous-generation/linux-od.min.js',
    # New generation instances (JavaScript file)
    # Using other endpoint atm
    # 'https://a0.awsstatic.com/pricing/1/ec2/linux-od.min.js'
]

EC2_REGIONS = [
    'us-east-1',
    'us-east-2',
    'us-west-1',
    'us-west-2',
    'us-gov-west-1',
    'eu-west-1',
    'eu-west-2',
    'eu-west-3',
    'eu-north-1',
    'eu-central-1',
    'ca-central-1',
    'ap-southeast-1',
    'ap-southeast-2',
    'ap-northeast-1',
    'ap-northeast-2',
    'ap-south-1',
    'sa-east-1',
    'cn-north-1',
]

EC2_INSTANCE_TYPES = [
    't1.micro',
    'm1.small',
    'm1.medium',
    'm1.large',
    'm1.xlarge',
    'm2.xlarge',
    'm2.2xlarge',
    'm2.4xlarge',
    'm3.medium',
    'm3.large',
    'm3.xlarge',
    'm3.2xlarge',
    'c1.medium',
    'c1.xlarge',
    'cc1.4xlarge',
    'cc2.8xlarge',
    'c3.large',
    'c3.xlarge',
    'c3.2xlarge',
    'c3.4xlarge',
    'c3.8xlarge',
    'd2.xlarge',
    'd2.2xlarge',
    'd2.4xlarge',
    'd2.8xlarge',
    'cg1.4xlarge',
    'g2.2xlarge',
    'g2.8xlarge',
    'cr1.8xlarge',
    'hs1.4xlarge',
    'hs1.8xlarge',
    'i2.xlarge',
    'i2.2xlarge',
    'i2.4xlarge',
    'i2.8xlarge',
    'i3.large',
    'i3.xlarge',
    'i3.2xlarge',
    'i3.4xlarge',
    'i3.8xlarge',
    'i3.16large',
    'r3.large',
    'r3.xlarge',
    'r3.2xlarge',
    'r3.4xlarge',
    'r3.8xlarge',
    'r4.large',
    'r4.xlarge',
    'r4.2xlarge',
    'r4.4xlarge',
    'r4.8xlarge',
    'r4.16xlarge',
    't2.micro',
    't2.small',
    't2.medium',
    't2.large',
    'x1.32xlarge'
]

# Maps EC2 region name to region name used in the pricing file
REGION_NAME_MAP = {
    'us-east': 'ec2_us_east',
    'us-east-1': 'ec2_us_east',
    'us-east-2': 'ec2_us_east_ohio',
    'us-west': 'ec2_us_west',
    'us-west-1': 'ec2_us_west',
    'us-west-2': 'ec2_us_west_oregon',
    'eu-west-1': 'ec2_eu_west',
    'eu-west-2': 'ec2_eu_west_london',
    'eu-west-3': 'ec2_eu_west_3',
    'eu-ireland': 'ec2_eu_west',
    'eu-central-1': 'ec2_eu_central',
    'ca-central-1': 'ec2_ca_central_1',
    'apac-sin': 'ec2_ap_southeast',
    'ap-southeast-1': 'ec2_ap_southeast',
    'apac-syd': 'ec2_ap_southeast_2',
    'ap-southeast-2': 'ec2_ap_southeast_2',
    'apac-tokyo': 'ec2_ap_northeast',
    'ap-northeast-1': 'ec2_ap_northeast',
    'ap-northeast-2': 'ec2_ap_northeast',
    'ap-south-1': 'ec2_ap_south_1',
    'sa-east-1': 'ec2_sa_east',
    'us-gov-west-1': 'ec2_us_govwest',
    'cn-north-1': 'ec2_cn_north',
}

INSTANCE_SIZES = [
    'micro',
    'small',
    'medium',
    'large',
    'xlarge',
    'x-large',
    'extra-large'
]

RE_NUMERIC_OTHER = re.compile(r'(?:([0-9]+)|([-A-Z_a-z]+)|([^-0-9A-Z_a-z]+))')

BASE_PATH = os.path.dirname(os.path.abspath(__file__))
PRICING_FILE_PATH = os.path.join(BASE_PATH, '../libcloud/data/pricing.json')
PRICING_FILE_PATH = os.path.abspath(PRICING_FILE_PATH)

def scrape_ec2_pricing():
    result = defaultdict(OrderedDict)
    os_map = {'linux': 'ec2_linux', 'windows-std': 'ec2_windows'}
    for item in os_map.values():
        result[item] = {}
    for url in LINUX_PRICING_URLS:
        response = requests.get(url)

        if re.match(r'.*?\.json$', url):
            data = response.json()
        elif re.match(r'.*?\.js$', url):
            data = response.content.decode()
            match = re.match(r'^.*callback\((.*?)\);?$', data,
                             re.MULTILINE | re.DOTALL)
            data = match.group(1)
            # demjson supports non-strict mode and can parse unquoted objects
            data = demjson.decode(data)
        regions = data['config']['regions']

        for region_data in regions:
            region_name = region_data['region']
            instance_types = region_data['instanceTypes']

            for instance_type in instance_types:
                sizes = instance_type['sizes']
                for size in sizes:
                    if not result['ec2_linux'].get(size['size'], False):
                        result['ec2_linux'][size['size']] = {}
                    price = size['valueColumns'][0]['prices']['USD']
                    if str(price).lower() == 'n/a':
                        # Price not available
                        continue

                    result['ec2_linux'][size['size']][
                        region_name] = float(price)

    res = defaultdict(OrderedDict)
    url = ('https://calculator.aws/pricing/1.0/'
           'ec2/region/{}/ondemand/{}/index.json')
    instances = set()
    for OS in ['linux', 'windows-std']:
        res[os_map[OS]] = {}
        for region in EC2_REGIONS:
            res[os_map[OS]][region] = {}
            response = requests.get(url.format(region, OS))
            if response.status_code != 200:
                continue
            data = response.json()

            for entry in data['prices']:
                instance_type = entry['attributes'].get(
                    'aws:ec2:instanceType', "")
                instances.add(instance_type)
                price = entry['price'].get('USD', 0)
                res[os_map[OS]][region][instance_type] = price
    for item in os_map.values():
        for instance in instances:
            if not result[item].get(instance, False):
                result[item][instance] = {}
            for region in EC2_REGIONS:
                if res[item][region].get(instance, False):
                    result[item][instance][region] = float(res[
                        item][region][instance])

    return result

def update_pricing_file(pricing_file_path, pricing_data):
    with open(pricing_file_path, 'r') as fp:
        content = fp.read()

    data = json.loads(content)
    original_data = copy.deepcopy(data)

    data['compute'].update(pricing_data)

    if data == original_data:
        # Nothing has changed, bail out early and don't update "updated" attribute
        print("Nothing has changed, skipping update.")
        return

    data['updated'] = int(time.time())

    # Always sort the pricing info
    data = sort_nested_dict(data)

    content = json.dumps(data, indent=4)
    lines = content.splitlines()
    lines = [line.rstrip() for line in lines]
    content = '\n'.join(lines)

    with open(pricing_file_path, 'w') as fp:
        fp.write(content)

def sort_nested_dict(value):
    """
    Recursively sort a nested dict.
    """
    result = OrderedDict()

    for key, value in sorted(value.items(), key=sort_key_by_numeric_other):
        if isinstance(value, (dict, OrderedDict)):
            result[key] = sort_nested_dict(value)
        else:
            result[key] = value

    return result

def sort_key_by_numeric_other(key_value):
    """
    Split key into numeric, alpha and other part and sort accordingly.
    """
    result = []

    for (numeric, alpha, other) in RE_NUMERIC_OTHER.findall(key_value[0]):
        numeric = int(numeric) if numeric else -1
        alpha = INSTANCE_SIZES.index(alpha) if alpha in INSTANCE_SIZES else alpha
        alpha = str(alpha)
        item = tuple([numeric, alpha, other])
        result.append(item)

    return tuple(result)

def main():
    print('Scraping EC2 pricing data (this may take up to 2 minutes)....')

    pricing_data = scrape_ec2_pricing()
    update_pricing_file(pricing_file_path=PRICING_FILE_PATH,
                        pricing_data=pricing_data)

    print('Pricing data updated')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import httplib as http
import logging

from framework import User, request, get_current_user
from framework.auth import get_api_key
from framework.exceptions import HTTPError

from website.project.model import Node, NodeLog
from website.project.decorators import must_be_valid_project

logger = logging.getLogger(__name__)

def get_log(log_id):

    log = NodeLog.load(log_id)
    node_to_use = log.node
    user = get_current_user()
    api_key = get_api_key()

    if not node_to_use.can_edit(user, api_key) and not node_to_use.are_logs_public:
        raise HTTPError(http.FORBIDDEN)
    return {'log': log.serialize()}

#todo: hide private logs of children
@must_be_valid_project
def get_logs(*args, **kwargs):
    user = get_current_user()
    api_key = get_api_key()
    node_to_use = kwargs['node'] or kwargs['project']

    if not node_to_use.can_edit(user, api_key) and not node_to_use.are_logs_public:
        raise HTTPError(http.FORBIDDEN)
    if 'count' in request.args:
        count = int(request.args['count'])
    elif 'count' in kwargs:
        count = kwargs['count']
    elif request.json and 'count' in request.json.keys():
        count = request.json['count']
    else:
        count = 10
    # logs in reverse chronological order
    chrono_logs = reversed(node_to_use.logs)
    log_data = [
        log.serialize()
        for log in chrono_logs[:count]
        if log
    ]
    return {'logs': log_data}
<commit_msg>hide logs of inaccessible components<commit_after># -*- coding: utf-8 -*-
import httplib as http
import logging

from framework import User, request, get_current_user
from framework.auth import get_api_key
from framework.exceptions import HTTPError

from website.project.model import Node, NodeLog
from website.project.decorators import must_be_valid_project

logger = logging.getLogger(__name__)

def get_log(log_id):

    log = NodeLog.load(log_id)
    node_to_use = log.node
    user = get_current_user()
    api_key = get_api_key()

    if not node_to_use.can_edit(user, api_key) and not node_to_use.are_logs_public:
        raise HTTPError(http.FORBIDDEN)
    return {'log': log.serialize()}

# todo: test log visibility
@must_be_valid_project
def get_logs(*args, **kwargs):
    user = get_current_user()
    api_key = get_api_key()
    node_to_use = kwargs['node'] or kwargs['project']

    if not node_to_use.can_edit(user, api_key) and not node_to_use.are_logs_public:
        raise HTTPError(http.FORBIDDEN)
    if 'count' in request.args:
        count = int(request.args['count'])
    elif 'count' in kwargs:
        count = kwargs['count']
    elif request.json and 'count' in request.json.keys():
        count = request.json['count']
    else:
        count = 10

    # Serialize up to `count` logs in reverse chronological order; skip
    # logs that the current user / API key cannot access
    log_data = []
    chrono_logs = reversed(node_to_use.logs)
    for logidx in range(len(chrono_logs)):
        log = chrono_logs[logidx]
        if log and log.node.can_edit(user, api_key):
            log_data.append(log.serialize())
        if len(log_data) >= count:
            break

    return {'logs': log_data}
<|endoftext|>


<commit_before>import numpy as np
from scipy.optimize import curve_fit
import matplotlib.pyplot as plt
import scipy.signal as sig

plt.ioff()

def line(x, a, b):
    return a*x+b

class Calibration:
    step_to_dist = {
            'x': {'f': 35.71, 'b': 25.64},
            'y': {'f': 27.78, 'b': 37.04}
            }
    def __init__(self, settings_file, step_freq, cal_x_volt, cal_y_volt):
        self.settings = self.load_settings(settings_file)
        self.step_freq = step_freq
        self.cal_x_volt = np.loadtxt(cal_x_volt).astype(float)
        self.cal_y_volt = np.loadtxt(cal_y_volt).astype(float)
        self.ts = {
            'x': self.cal_x_volt[:,0],
            'y': self.cal_y_volt[:,1]
            }
        self.rate = self.settings['Sample Rate']

    def load_settings(self, settings):
        setdict = {}
        with open(settings) as s:
            for line in s:
                (key, val) = line.split(':')
                setdict[key] = float(val)
        return setdict

    @property
    def xplot(self):
        xfig = plt.figure()
        xplt = xfig.add_subplot(111)
        vdat = self.ts['x']
        xdat = np.arange(len(vdat)) / self.rate
        xplt.plot(xdat, vdat)
        xplt.set_xlabel("Time (s)")
        xplt.set_ylabel("Voltage (V)")
        xplt.set_title("X Sensitivity Calibration")
        return xfig

    @property
    def yplot(self):
        yfig = plt.figure()
        yplt = yfig.add_subplot(111)
        vdat = self.ts['y']
        ydat = np.arange(len(vdat)) / self.rate
        yplt.plot(ydat, vdat)
        yplt.set_xlabel("Time (s)")
        yplt.set_ylabel("Voltage (V)")
        yplt.set_title("Y Sensitivity Calibration")
        return yfig

    def sensitivity(self, axis, direction, start, stop, vdat=None):
        """
        Calculate sensitivity from voltage data.
        TODO: Automatically compute start and stop.
        Returns: Sensitivity in V/um and uncertainty
        """
        if vdat is None:
            vdat = self.ts[axis]
        stepdist = Calibration.step_to_dist[axis][direction]
        sampledist = stepdist / self.rate * self.step_freq
        params, cov = curve_fit(line, np.arange(len(vdat[start:stop])), vdat[start:stop])
        slope = params[0]
        uncertainty = np.sqrt(cov[0,0])/sampledist
        return (slope/sampledist, uncertainty)

    @property
    def x_sensitivity(self):
        pass

    @property
    def y_sensitivity(self):
        pass

    def band_stop(self, low_f, high_f, axis='x', order=6):
        f_nyq = self.rate / 2
        band_low = low_f / f_nyq
        band_high = high_f / f_nyq
        b, a = sig.butter(order, [band_low, band_high], btype='bandstop')
        w, h = sig.freqs(b, a)
        vdat = self.ts[axis]
        filtered = sig.lfilter(b,a,vdat)
        return filtered

    def plot_band_stop(self, low_f, high_f, axis='x', order=6):
        fig = plt.figure()
        xplt = fig.add_subplot(111)
        vdat = self.ts[axis]
        tdat = np.arange(len(vdat)) / self.rate
        xplt.plot(tdat, vdat)
        filtered = self.band_stop(low_f, high_f, axis, order)
        xplt.plot(tdat, filtered)
        xplt.set_xlabel("Time (s)")
        xplt.set_ylabel("Voltage (V)")
        xplt.set_title(
            "{order}-Order Butterworth Bandstop Filter ({low}Hz-{high}Hz)".format(
            order=order,low=low_f,high=high_f))
        fig.show()

    def psd(self, axis='x', vdat=None):
        if vdat is None:
            vdat = self.ts[axis]<commit_msg>plot psd<commit_after>import numpy as np
from scipy.optimize import curve_fit
import matplotlib.pyplot as plt
import scipy.signal as sig

plt.ioff()

def line(x, a, b):
    return a*x+b

class Calibration:
    step_to_dist = {
            'x': {'f': 35.71, 'b': 25.64},
            'y': {'f': 27.78, 'b': 37.04}
            }
    def __init__(self, settings_file, step_freq, cal_x_volt, cal_y_volt):
        self.settings = self.load_settings(settings_file)
        self.step_freq = step_freq
        self.cal_x_volt = np.loadtxt(cal_x_volt).astype(float)
        self.cal_y_volt = np.loadtxt(cal_y_volt).astype(float)
        self.ts = {
            'x': self.cal_x_volt[:,0],
            'y': self.cal_y_volt[:,1]
            }
        self.rate = self.settings['Sample Rate']

    @property
    def tdata(self):
        return np.arange(len(self.ts['x']))/self.rate

    def load_settings(self, settings):
        setdict = {}
        with open(settings) as s:
            for line in s:
                (key, val) = line.split(':')
                setdict[key] = float(val)
        return setdict

    @property
    def xplot(self):
        xfig = plt.figure()
        xplt = xfig.add_subplot(111)
        vdat = self.ts['x']
        xdat = self.tdata
        xplt.plot(xdat, vdat)
        xplt.set_xlabel("Time (s)")
        xplt.set_ylabel("Voltage (V)")
        xplt.set_title("X Sensitivity Calibration")
        return xfig

    @property
    def yplot(self):
        yfig = plt.figure()
        yplt = yfig.add_subplot(111)
        vdat = self.ts['y']
        ydat = self.tdata
        yplt.plot(ydat, vdat)
        yplt.set_xlabel("Time (s)")
        yplt.set_ylabel("Voltage (V)")
        yplt.set_title("Y Sensitivity Calibration")
        return yfig

    def sensitivity(self, axis, direction, start, stop, vdat=None):
        """
        Calculate sensitivity from voltage data.
        TODO: Automatically compute start and stop.
        Returns: Sensitivity in V/um and uncertainty
        """
        if vdat is None:
            vdat = self.ts[axis]
        stepdist = Calibration.step_to_dist[axis][direction]
        sampledist = stepdist / self.rate * self.step_freq
        params, cov = curve_fit(line, np.arange(len(vdat[start:stop])), vdat[start:stop])
        slope = params[0]
        uncertainty = np.sqrt(cov[0,0])/sampledist
        return (slope/sampledist, uncertainty)

    @property
    def x_sensitivity(self):
        pass

    @property
    def y_sensitivity(self):
        pass

    def band_stop(self, low_f, high_f, axis='x', order=6):
        f_nyq = self.rate / 2
        band_low = low_f / f_nyq
        band_high = high_f / f_nyq
        b, a = sig.butter(order, [band_low, band_high], btype='bandstop')
        w, h = sig.freqs(b, a)
        vdat = self.ts[axis]
        filtered = sig.lfilter(b,a,vdat)
        return filtered

    def plot_band_stop(self, low_f, high_f, axis='x', order=6, plot_orig=False):
        fig = plt.figure()
        xplt = fig.add_subplot(111)
        vdat = self.ts[axis]
        if plot_orig:
            xplt.plot(self.tdata, vdat)
        filtered = self.band_stop(low_f, high_f, axis, order)
        xplt.plot(self.tdata, filtered)
        xplt.set_xlabel("Time (s)")
        xplt.set_ylabel("Voltage (V)")
        xplt.set_title(
            "Order-{order} Butterworth Bandstop Filter ({low}Hz-{high}Hz)".format(
            order=order,low=low_f,high=high_f))
        fig.show()

    def plot_psd(self, axis='x', vdat=None, plot_orig=False):
        if vdat is None:
            vdat = self.ts[axis]
        f, psd = sig.periodogram(vdat, self.rate)
        fig = plt.figure()
        psdplt = fig.add_subplot(111)
        psdplt.loglog(f, psd, basey=np.e)
        if plot_orig:
            f, psd = sig.periodogram(self.ts[axis], self.rate)
            psdplt.loglog(f, psd)
        psdplt.set_title("PSD")
        psdplt.set_xlabel("Frequency ($Hz$)")
        psdplt.set_ylabel("PSD ($V^2/Hz$)")
        fig.show()
<|endoftext|>


<commit_before>"""

  paranoidsessions:  more comprehensive security checking for Django sessions

This module implements "paranoid sessions" for Django - sessions that work
considerably harder at security than the standard version.  Designed to make
session-stealing attacks as hard as possible, the extra measures that can be
employed are:

  * HTTP header fingerprinting (e.g. REMOTE_ADDR, HTTP_USER_AGENT)
  * per-request nonces (with configurable timeout and duplicate window)
  * periodic cycling of session keys

As always, there's a tradeoff here - these security measures involve increased
processing per request and more frequent writes to the session store.  But by
adjusting the various settings offered by this module, you should be able to
find a compromise that's suitable for your project.

To use this module, simply include it in MIDDLEWARE_CLASSES after the standard
session middleware:

MIDDLEWARE_CLASSES = (
    ...
    'django.contrib.sessions.middleware.SessionMiddleware',
    'paranoidsessions.ParanoidSessionMiddleware',
    ...
)

In keeping with the name, the default settings of this module are designed
to maximise security at the expense of performance and user convenience.
You'll probably want to adjust them to a compromise that's more suitable for
your application, but you should understand the tradeoffs before doing so.

In particular, the management of per-request nonces must account for the
asynchronous nature of the web.  To permit a user to have multiple overlapping
requests in progress, there must be a brief window in which old nonces are
accepted.  This also requires multiple Django processes/threads to coordinate
the nonces that they issue.  The nonces generated by this module are therefore
not truly random - instead they are generated as a "nonce stream" by repeatedly
hashing a random seed value with the server-side secret key.  This should be
sufficiently unforgeable for most applications (if your attacker can compromise
nonces generated in this manner, you should consider serving your entire site
over a secure channel!)

The following settings are available:

  PSESSION_CHECK_HEADERS:  List of headers to check on each request.  The
          session will be cleared if any of these headers vary from those
          sent when the session was initially created.
          Default:  ["REMOTE_ADDR","HTTP_X_FORWARDED_FOR"]

  PSESSION_NONCE_TIMEOUT:  Time (in seconds) after which a new nonce will be
          generated.  The client must return a valid nonce with each request
          or the session will be cleared. Setting this to a larger value will
          decrease security but reduce the frequency of writes to the session
          store.  Setting it to zero means a new nonce is generated on every
          request.  Setting it to None will disable the use of nonces entirely.
          Default:  0

  PSESSION_NONCE_WINDOW:  Number of nonces prior to the current nonce that will
          still be accepted as valid.  This corresponds to the number of nonce
          updates that could be "in flight" between the server and client at
          any given time.  If you need to set it to any more than 1, your site
          has some serious performance issues.
          Default:  1

  PSESSION_NONCE_WINDOW_TIMEOUT:  Time (in seconds) within which old nonces
          are accepted.  This window should be as small as possible, but is
          necessary if users will perform multiple overlapping requests.
          Default:  0.5

  PSESSION_KEY_TIMEOUT:  Time (in seconds) after which the session key will
          be cycled.  This should only be needed if nonces are not in use;
          the difference is that key cycling doesn't require the client to
          send a separate nonce cookie.  Set it to None to disable key cycling.
          Default:  None

  PSESSION_SESSION_KEY:  Session key under which the request-tracking and
          verification data for this module should be stored.
          Default:  "PARANOID_SESSION_DATA"

  PSESSION_COOKIE_NAME:  Name of cookie used for the per-request nonce.
          Default:  "sessionnonce"

  PSESSION_CLEAR_SESSION_FUNCTION:  Function called to clear the session if a
          potential attack is detected.  An importable name or callable object
          may be given, taking a request object as its only argument.
          Default:  lambda req: req.session.flush()

  PSESSION_REQUEST_FILTER_FUNCTION:  Function to filter requests to be checked.
          Any requests for which this function returns False will not be
          subjected to paranoid validation.  This may be helpful for reducing
          processing overhead on low-risk targets such as media files, but
          will give an attacker more opportunities to compromise a given nonce.
          Default:  lambda req: True

"""

__version__ = "0.1.0"

import time

from django.utils.http import cookie_date
from django.utils.hashcompat import md5_constructor
from django.core.urlresolvers import get_callable
from django.contrib.sessions.backends.base import randrange
from django.conf import settings

MAX_NONCE_SEED = 18446744073709551616L     # 2 << 63

if not hasattr(settings,"PSESSION_CHECK_HEADERS"):
    settings.PSESSION_CHECK_HEADERS = ("REMOTE_ADDR","HTTP_X_FORWARDED_FOR",)
if not hasattr(settings,"PSESSION_NONCE_TIMEOUT"):
    settings.PSESSION_NONCE_TIMEOUT = 0
if not hasattr(settings,"PSESSION_NONCE_WINDOW"):
    settings.PSESSION_NONCE_WINDOW = 1
if not hasattr(settings,"PSESSION_NONCE_WINDOW_TIMEOUT"):
    settings.PSESSION_NONCE_WINDOW_TIMEOUT = 0.5
if not hasattr(settings,"PSESSION_KEY_TIMEOUT"):
    settings.PSESSION_KEY_TIMEOUT = None
if not hasattr(settings,"PSESSION_SESSION_KEY"):
    settings.PSESSION_SESSION_KEY = "PARANOID_SESSION_DATA"
if not hasattr(settings,"PSESSION_COOKIE_NAME"):
    settings.PSESSION_COOKIE_NAME = "sessionnonce"
if hasattr(settings,"PSESSION_REQUEST_FILTER_FUNCTION"):
    request_filter = get_callable(settings.PSESSION_REQUEST_FILTER_FUNCTION)
    settings.PSESSION_REQUEST_FILTER_FUNCTION = request_filter
else:
    settings.PSESSION_REQUEST_FILTER_FUNCTION = lambda req: True
if hasattr(settings,"PSESSION_CLEAR_SESSION_FUNCTION"):
    clear_session = get_callable(settings.PSESSION_CLEAR_SESSION_FUNCTION)
    settings.PSESSION_CLEAR_SESSION_FUNCTION = clear_session
else:
    settings.PSESSION_CLEAR_SESSION_FUNCTION = lambda req: req.session.flush()

class NonceStream(object):
    """Generate an unpredictable stream of nonce values.

    This class simulates a cryptographic pseudo-random number generator
    using repeated hashing of an initial random seed.  There has been no
    formal cryptanalysis of such a scheme but:
        * I'm confident in its utility for this purpose
        * It's implementable using only python builtins

    If you're worried about the security of such a stream, you can replace
    this class with one based on a proper CSPRNG using e.g. PyCrypto. I'm
    open to suggestions for how to improve this using python builtins.
    """

    def __init__(self):
        seed = (randrange(0,MAX_NONCE_SEED),settings.SECRET_KEY)
        self.state = md5_constructor("%s%s" % seed).hexdigest()

    def nonces(self):
        """Generate producing sequence of nonce values."""
        state = self.state
        while True:
            yield state
            state = md5_constructor(state + settings.SECRET_KEY).hexdigest()

    def increment(self):
        """Increment the nonce stream, discarding initial nonce."""
        state = self.state
        state = self.state
        self.state = md5_constructor(state + settings.SECRET_KEY).hexdigest()
        return self.state

class SessionFingerprint(object):
    """Object representing a unique request fingerprint for a session.

    This class is used to check that incoming requests are presenting valid
    session tokens, by checking them against certain "fingerprint" information
    that is contained in the session.  The fingerprint may include:

        * the values of certain HTTP headers
        * a per-request nonce

    It also maintains ancilliarly data for managing timeouts etc.
    """

    def __init__(self,request):
        self.hash = self.request_hash(request)
        self.nonce_stream = NonceStream()
        now = time.time()
        self.last_nonce_time = 0
        self.last_key_time = now
        self.last_request_time = now

    def check_request(self,request):
        """Check that the given request is valid for this session.

        It's valid if its fingerprint hash matches the current hash, or
        if the duplicate nonce window is enabled and it matches a sufficiently
        recent hash.
        """
        hash = self.request_hash(request)
        if hash != self.hash:
            return False
        if settings.PSESSION_NONCE_TIMEOUT is not None:
            nonce = request.COOKIES[settings.PSESSION_COOKIE_NAME]
            if nonce not in self.get_valid_nonces():
                return False
        self.last_request_time = time.time()
        return True

    def process_response(self,request,response):
        """Process a response and mark session as modified if necessary.

        This method sets the nonce cookie and cycles the session key if
        necessary.  If any changes are made, the request session is marked
        as modified so that it will be saved appropriately.
        """
        now = time.time()
        nonce_timeout = settings.PSESSION_NONCE_TIMEOUT
        #  Generate a new nonce if necessary
        if nonce_timeout is not None:
            #  If this request took an unusually long time, other requests
            #  might have incremented the nonce more than once.  We need to
            #  resync our data from the session store.
            if self.last_request_time < now + nonce_timeout:
                self.refresh_from_session(request)
            if self.last_nonce_time + nonce_timeout < now:
                self.nonce_stream.increment()
                self.set_nonce_cookie(request,response)
                self.last_nonce_time = now
                request.session.modified = True
        #  Generate a new session key if necessary
        if settings.PSESSION_KEY_TIMEOUT is not None:
            if self.last_key_time + settings.PSESSION_KEY_TIMEOUT < now:
                request.session.cycle_key()
                self.last_key_time = now
                request.session.modified = True

    def request_hash(self,request):
        """Create a hash of the given request's fingerprint data.

        This hash will contain data that should be the same for every request
        in this session.
        """
        hash = md5_constructor()
        if settings.PSESSION_CHECK_HEADERS:
            for header in settings.PSESSION_CHECK_HEADERS:
               hash.update(request.META.get(header,""))
        return hash.digest()

    def get_valid_nonces(self):
        """Get a sequence of all currently valid nonces."""
        now = time.time()
        window = settings.PSESSION_NONCE_WINDOW
        nonces = self.nonce_stream.nonces()
        #  Yield or skip old nonces, depending on window timeout
        if now < self.last_nonce_time + settings.PSESSION_NONCE_WINDOW_TIMEOUT:
            for _ in xrange(window):
                yield nonces.next()
        else:
            for _ in xrange(window):
                nonces.next()
        #  Yield the current nonce
        yield nonces.next()
        #  I belive that races between Django processes could result in the
        #  client actually being one step ahead of the server.
        yield nonces.next()

    def set_nonce_cookie(self,request,response):
        """Set the nonce cookie on the given response.

        This duplicates the cookie-setting logic in the session middleware,
        so it will expire under the same rules as the session itself.
        """
        nonce = list(self.get_valid_nonces())[-1]
        if request.session.get_expire_at_browser_close():
            max_age = None
            expires = None
        else:
            max_age = request.session.get_expiry_age()
            expires_time = time.time() + max_age
            expires = cookie_date(expires_time)
        response.set_cookie(settings.PSESSION_COOKIE_NAME,nonce,
                            max_age=max_age,expires=expires,
                            domain=settings.SESSION_COOKIE_DOMAIN,
                            path=settings.SESSION_COOKIE_PATH,
                            secure=settings.SESSION_COOKIE_SECURE or None)

    def refresh_from_session(self,request):
        """Refresh internal data from the session store."""
        session_data = request.session.load()
        try:
            fingerprint = session_data[settings.PSESSION_SESSION_KEY]
        except KeyError:
            pass
        else:
            if fingerprint.last_nonce_time > self.last_nonce_time:
                self.nonce_stream = fingerprint.nonce_stream
                self.last_nonce_time = fingerprint.last_nonce_time
            if fingerprint.last_key_time > self.last_key_time:
                self.last_key_time = fingerprint.last_key_time
            if fingerprint.last_request_time > self.last_request_time:
                self.last_request_time = fingerprint.last_request_time

class ParanoidSessionMiddleware(object):
    """Middleware implementing paranoid session checking.

    This middleware ensures that each session contains a SessionFingerprint
    object, and asks that object to validate each incoming request.
    """

    def process_request(self,request):
        if settings.PSESSION_REQUEST_FILTER_FUNCTION(request):
            try:
                fingerprint = request.session[settings.PSESSION_SESSION_KEY]
            except KeyError:
                fingerprint = SessionFingerprint(request)
                request.session[settings.PSESSION_SESSION_KEY] = fingerprint
                request.session.save()
            else:
                if not fingerprint.check_request(request):
                    settings.PSESSION_CLEAR_SESSION_FUNCTION(request)
            request.session.paranoid = True
        else:
            request.session.paranoid = False

    def process_response(self,request,response):
        if request.session.paranoid:
            try:
                fingerprint = request.session[settings.PSESSION_SESSION_KEY]
            except KeyError:
                fingerprint = SessionFingerprint(request)
                request.session[settings.PSESSION_SESSION_KEY] = fingerprint
            fingerprint.process_response(request,response)
        return response

<commit_msg>add HTTP_USER_AGENT to default CHECK_HEADERS<commit_after>"""

  paranoidsessions:  more comprehensive security checking for Django sessions

This module implements "paranoid sessions" for Django - sessions that work
considerably harder at security than the standard version.  Designed to make
session-stealing attacks as hard as possible, the extra measures that can be
employed are:

  * HTTP header fingerprinting (e.g. REMOTE_ADDR, HTTP_USER_AGENT)
  * per-request nonces (with configurable timeout and duplicate window)
  * periodic cycling of session keys

As always, there's a tradeoff here - these security measures involve increased
processing per request and more frequent writes to the session store.  But by
adjusting the various settings offered by this module, you should be able to
find a compromise that's suitable for your project.

To use this module, simply include it in MIDDLEWARE_CLASSES after the standard
session middleware:

MIDDLEWARE_CLASSES = (
    ...
    'django.contrib.sessions.middleware.SessionMiddleware',
    'paranoidsessions.ParanoidSessionMiddleware',
    ...
)

In keeping with the name, the default settings of this module are designed
to maximise security at the expense of performance and user convenience.
You'll probably want to adjust them to a compromise that's more suitable for
your application, but you should understand the tradeoffs before doing so.

In particular, the management of per-request nonces must account for the
asynchronous nature of the web.  To permit a user to have multiple overlapping
requests in progress, there must be a brief window in which old nonces are
accepted.  This also requires multiple Django processes/threads to coordinate
the nonces that they issue.  The nonces generated by this module are therefore
not truly random - instead they are generated as a "nonce stream" by repeatedly
hashing a random seed value with the server-side secret key.  This should be
sufficiently unforgeable for most applications (if your attacker can compromise
nonces generated in this manner, you should consider serving your entire site
over a secure channel!)

The following settings are available:

  PSESSION_CHECK_HEADERS:  List of headers to check on each request.  The
          session will be cleared if any of these headers vary from those
          sent when the session was initially created.

          Default:  ["REMOTE_ADDR","HTTP_X_FORWARDED_FOR","HTTP_USER_AGENT"]

  PSESSION_NONCE_TIMEOUT:  Time (in seconds) after which a new nonce will be
          generated.  The client must return a valid nonce with each request
          or the session will be cleared. Setting this to a larger value will
          decrease security but reduce the frequency of writes to the session
          store.  Setting it to zero means a new nonce is generated on every
          request.  Setting it to None will disable the use of nonces entirely.
          Default:  0

  PSESSION_NONCE_WINDOW:  Number of nonces prior to the current nonce that will
          still be accepted as valid.  This corresponds to the number of nonce
          updates that could be "in flight" between the server and client at
          any given time.  If you need to set it to any more than 1, your site
          has some serious performance issues.
          Default:  1

  PSESSION_NONCE_WINDOW_TIMEOUT:  Time (in seconds) within which old nonces
          are accepted.  This window should be as small as possible, but is
          necessary if users will perform multiple overlapping requests.
          Default:  0.5

  PSESSION_KEY_TIMEOUT:  Time (in seconds) after which the session key will
          be cycled.  This should only be needed if nonces are not in use;
          the difference is that key cycling doesn't require the client to
          send a separate nonce cookie.  Set it to None to disable key cycling.
          Default:  None

  PSESSION_SESSION_KEY:  Session key under which the request-tracking and
          verification data for this module should be stored.
          Default:  "PARANOID_SESSION_DATA"

  PSESSION_COOKIE_NAME:  Name of cookie used for the per-request nonce.
          Default:  "sessionnonce"

  PSESSION_CLEAR_SESSION_FUNCTION:  Function called to clear the session if a
          potential attack is detected.  An importable name or callable object
          may be given, taking a request object as its only argument.
          Default:  lambda req: req.session.flush()

  PSESSION_REQUEST_FILTER_FUNCTION:  Function to filter requests to be checked.
          Any requests for which this function returns False will not be
          subjected to paranoid validation.  This may be helpful for reducing
          processing overhead on low-risk targets such as media files, but
          will give an attacker more opportunities to compromise a given nonce.
          Default:  lambda req: True

"""

__version__ = "0.1.0"

import time

from django.utils.http import cookie_date
from django.utils.hashcompat import md5_constructor
from django.core.urlresolvers import get_callable
from django.contrib.sessions.backends.base import randrange
from django.conf import settings

MAX_NONCE_SEED = 18446744073709551616L     # 2 << 63

if not hasattr(settings,"PSESSION_CHECK_HEADERS"):
    check_headers = ("REMOTE_ADDR","HTTP_X_FORWARDED_FOR","HTTP_USER_AGENT",)
    settings.PSESSION_CHECK_HEADERS = check_headers
if not hasattr(settings,"PSESSION_NONCE_TIMEOUT"):
    settings.PSESSION_NONCE_TIMEOUT = 0
if not hasattr(settings,"PSESSION_NONCE_WINDOW"):
    settings.PSESSION_NONCE_WINDOW = 1
if not hasattr(settings,"PSESSION_NONCE_WINDOW_TIMEOUT"):
    settings.PSESSION_NONCE_WINDOW_TIMEOUT = 0.5
if not hasattr(settings,"PSESSION_KEY_TIMEOUT"):
    settings.PSESSION_KEY_TIMEOUT = None
if not hasattr(settings,"PSESSION_SESSION_KEY"):
    settings.PSESSION_SESSION_KEY = "PARANOID_SESSION_DATA"
if not hasattr(settings,"PSESSION_COOKIE_NAME"):
    settings.PSESSION_COOKIE_NAME = "sessionnonce"
if hasattr(settings,"PSESSION_REQUEST_FILTER_FUNCTION"):
    request_filter = get_callable(settings.PSESSION_REQUEST_FILTER_FUNCTION)
    settings.PSESSION_REQUEST_FILTER_FUNCTION = request_filter
else:
    settings.PSESSION_REQUEST_FILTER_FUNCTION = lambda req: True
if hasattr(settings,"PSESSION_CLEAR_SESSION_FUNCTION"):
    clear_session = get_callable(settings.PSESSION_CLEAR_SESSION_FUNCTION)
    settings.PSESSION_CLEAR_SESSION_FUNCTION = clear_session
else:
    settings.PSESSION_CLEAR_SESSION_FUNCTION = lambda req: req.session.flush()

class NonceStream(object):
    """Generate an unpredictable stream of nonce values.

    This class simulates a cryptographic pseudo-random number generator
    using repeated hashing of an initial random seed.  There has been no
    formal cryptanalysis of such a scheme but:
        * I'm confident in its utility for this purpose
        * It's implementable using only python builtins

    If you're worried about the security of such a stream, you can replace
    this class with one based on a proper CSPRNG using e.g. PyCrypto. I'm
    open to suggestions for how to improve this using python builtins.
    """

    def __init__(self):
        seed = (randrange(0,MAX_NONCE_SEED),settings.SECRET_KEY)
        self.state = md5_constructor("%s%s" % seed).hexdigest()

    def nonces(self):
        """Generate producing sequence of nonce values."""
        state = self.state
        while True:
            yield state
            state = md5_constructor(state + settings.SECRET_KEY).hexdigest()

    def increment(self):
        """Increment the nonce stream, discarding initial nonce."""
        state = self.state
        state = self.state
        self.state = md5_constructor(state + settings.SECRET_KEY).hexdigest()
        return self.state

class SessionFingerprint(object):
    """Object representing a unique request fingerprint for a session.

    This class is used to check that incoming requests are presenting valid
    session tokens, by checking them against certain "fingerprint" information
    that is contained in the session.  The fingerprint may include:

        * the values of certain HTTP headers
        * a per-request nonce

    It also maintains ancilliarly data for managing timeouts etc.
    """

    def __init__(self,request):
        self.hash = self.request_hash(request)
        self.nonce_stream = NonceStream()
        now = time.time()
        self.last_nonce_time = 0
        self.last_key_time = now
        self.last_request_time = now

    def check_request(self,request):
        """Check that the given request is valid for this session.

        It's valid if its fingerprint hash matches the current hash, or
        if the duplicate nonce window is enabled and it matches a sufficiently
        recent hash.
        """
        hash = self.request_hash(request)
        if hash != self.hash:
            return False
        if settings.PSESSION_NONCE_TIMEOUT is not None:
            nonce = request.COOKIES[settings.PSESSION_COOKIE_NAME]
            if nonce not in self.get_valid_nonces():
                return False
        self.last_request_time = time.time()
        return True

    def process_response(self,request,response):
        """Process a response and mark session as modified if necessary.

        This method sets the nonce cookie and cycles the session key if
        necessary.  If any changes are made, the request session is marked
        as modified so that it will be saved appropriately.
        """
        now = time.time()
        nonce_timeout = settings.PSESSION_NONCE_TIMEOUT
        #  Generate a new nonce if necessary
        if nonce_timeout is not None:
            #  If this request took an unusually long time, other requests
            #  might have incremented the nonce more than once.  We need to
            #  resync our data from the session store.
            if self.last_request_time < now + nonce_timeout:
                self.refresh_from_session(request)
            if self.last_nonce_time + nonce_timeout < now:
                self.nonce_stream.increment()
                self.set_nonce_cookie(request,response)
                self.last_nonce_time = now
                request.session.modified = True
        #  Generate a new session key if necessary
        if settings.PSESSION_KEY_TIMEOUT is not None:
            if self.last_key_time + settings.PSESSION_KEY_TIMEOUT < now:
                request.session.cycle_key()
                self.last_key_time = now
                request.session.modified = True

    def request_hash(self,request):
        """Create a hash of the given request's fingerprint data.

        This hash will contain data that should be the same for every request
        in this session.
        """
        hash = md5_constructor()
        if settings.PSESSION_CHECK_HEADERS:
            for header in settings.PSESSION_CHECK_HEADERS:
               hash.update(request.META.get(header,""))
        return hash.digest()

    def get_valid_nonces(self):
        """Get a sequence of all currently valid nonces."""
        now = time.time()
        window = settings.PSESSION_NONCE_WINDOW
        nonces = self.nonce_stream.nonces()
        #  Yield or skip old nonces, depending on window timeout
        if now < self.last_nonce_time + settings.PSESSION_NONCE_WINDOW_TIMEOUT:
            for _ in xrange(window):
                yield nonces.next()
        else:
            for _ in xrange(window):
                nonces.next()
        #  Yield the current nonce
        yield nonces.next()
        #  I belive that races between Django processes could result in the
        #  client actually being one step ahead of the server.
        yield nonces.next()

    def set_nonce_cookie(self,request,response):
        """Set the nonce cookie on the given response.

        This duplicates the cookie-setting logic in the session middleware,
        so it will expire under the same rules as the session itself.
        """
        nonce = list(self.get_valid_nonces())[-1]
        if request.session.get_expire_at_browser_close():
            max_age = None
            expires = None
        else:
            max_age = request.session.get_expiry_age()
            expires_time = time.time() + max_age
            expires = cookie_date(expires_time)
        response.set_cookie(settings.PSESSION_COOKIE_NAME,nonce,
                            max_age=max_age,expires=expires,
                            domain=settings.SESSION_COOKIE_DOMAIN,
                            path=settings.SESSION_COOKIE_PATH,
                            secure=settings.SESSION_COOKIE_SECURE or None)

    def refresh_from_session(self,request):
        """Refresh internal data from the session store."""
        session_data = request.session.load()
        try:
            fingerprint = session_data[settings.PSESSION_SESSION_KEY]
        except KeyError:
            pass
        else:
            if fingerprint.last_nonce_time > self.last_nonce_time:
                self.nonce_stream = fingerprint.nonce_stream
                self.last_nonce_time = fingerprint.last_nonce_time
            if fingerprint.last_key_time > self.last_key_time:
                self.last_key_time = fingerprint.last_key_time
            if fingerprint.last_request_time > self.last_request_time:
                self.last_request_time = fingerprint.last_request_time

class ParanoidSessionMiddleware(object):
    """Middleware implementing paranoid session checking.

    This middleware ensures that each session contains a SessionFingerprint
    object, and asks that object to validate each incoming request.
    """

    def process_request(self,request):
        if settings.PSESSION_REQUEST_FILTER_FUNCTION(request):
            try:
                fingerprint = request.session[settings.PSESSION_SESSION_KEY]
            except KeyError:
                fingerprint = SessionFingerprint(request)
                request.session[settings.PSESSION_SESSION_KEY] = fingerprint
                request.session.save()
            else:
                if not fingerprint.check_request(request):
                    settings.PSESSION_CLEAR_SESSION_FUNCTION(request)
            request.session.paranoid = True
        else:
            request.session.paranoid = False

    def process_response(self,request,response):
        if request.session.paranoid:
            try:
                fingerprint = request.session[settings.PSESSION_SESSION_KEY]
            except KeyError:
                fingerprint = SessionFingerprint(request)
                request.session[settings.PSESSION_SESSION_KEY] = fingerprint
            fingerprint.process_response(request,response)
        return response

<|endoftext|>


<commit_before>#! /usr/bin/env python
from __future__ import print_function
from atlasseq.graph import ProbabilisticMultiColourDeBruijnGraph as Graph
import sys
import logging
import json
logger = logging.getLogger(__name__)
from atlasseq.utils import DEFAULT_LOGGING_LEVEL
logger.setLevel(DEFAULT_LOGGING_LEVEL)

import numpy as np
from bitarray import bitarray
from tempfile import mkdtemp

import os
import psutil
process = psutil.Process(os.getpid())
import time

def load_memmap(filename, size):
    return np.memmap(filename, dtype='bool_', mode='r', shape=tuple(size))

def merge(uncompressed_graphs, sizes, outfile):
    ncols = sum([j for i, j in sizes])
    nrows = sizes[0][0]
    _shape = (nrows, ncols)
    fp = np.memmap(outfile, dtype='bool_', mode='w+',
                   shape=_shape)
    ii = 0
    for f, size in zip(uncompressed_graphs, sizes):
        jj = ii+size[1]
        a = load_memmap(f, size)
        logger.info("%i MB" % int(process.memory_info().rss/1000000))
        fp[:, ii:jj] = a
        ii = jj
        fp.flush()
    fp.flush()
    return {"shape": _shape, "uncompressed_graph": outfile}

    # start = time.time()
    # ugs = []
    # for f, size in zip(uncompressed_graphs, sizes):
    #     a = load_memmap(f, size)
    #     ugs.append(a)
    # for i in range(nrows):
    #     a = np.append([], [ugs[j][i, ] for j in range(len(ugs))])
    #     if i % 1000 == 0:
    #         logger.info(i)
    #         logger.info(time.time()-start)
    #         logger.info("%i MB" % int(process.memory_info().rss/1000000))
<commit_msg>Don't require full graph in mem to merge<commit_after>#! /usr/bin/env python
from __future__ import print_function
from atlasseq.graph import ProbabilisticMultiColourDeBruijnGraph as Graph
import sys
import logging
import json
logger = logging.getLogger(__name__)
from atlasseq.utils import DEFAULT_LOGGING_LEVEL
logger.setLevel(DEFAULT_LOGGING_LEVEL)

import numpy as np
from bitarray import bitarray
from tempfile import mkdtemp

import os
import psutil
process = psutil.Process(os.getpid())
import time

def load_memmap(filename, size):
    return np.memmap(filename, dtype='bool_', mode='r', shape=tuple(size))

def merge(uncompressed_graphs, sizes, outfile):
    ncols = sum([j for i, j in sizes])
    nrows = sizes[0][0]
    _shape = (nrows, ncols)
    start = time.time()
    ugs = []
    for f, size in zip(uncompressed_graphs, sizes):
        a = load_memmap(f, size)
        ugs.append(a)
    with open(outfile, 'wb') as outf:
        for i in range(nrows):
            a = np.append([], [ugs[j][i, ] for j in range(len(ugs))])
            if i % 100000 == 0:
                logger.info(i)
                logger.info(time.time()-start)
                logger.info("%i MB" % int(process.memory_info().rss/1000000))
            ba_out = bitarray(a.tolist())
            outf.write(ba_out.tobytes())
    return {'graph': outfile, "shape": _shape}
<|endoftext|>


<commit_before># file: plotForceCoefficientsVsAoAFlatPlate3d.py
# author: Olivier Mesnard (mesnardo@gwu.edu)
# description: Plots the lift and drag coefficients for a 3d flat-plate
#              with different angles of attack

import os
import argparse
import collections

import numpy
from matplotlib import pyplot

from snake import miscellaneous
from snake.simulation import Simulation
from snake.force import Force

def parse_command_line():
  """Parses the command-line and returns the namespace."""
  print('[info] parsing the command-line ...'),
  # create parser
  parser = argparse.ArgumentParser(description='Plots the drag coefficient vs. Re',
                        formatter_class= argparse.ArgumentDefaultsHelpFormatter)
  # fill parser with arguments
  parser.add_argument('--software', dest='software',
                      type=str,
                      choices=['petibm'],
                      help='software used to generate solution')
  parser.add_argument('--directory', dest='directory',
                      type=str,
                      default=os.getcwd(),
                      help='main directory containing series directories')
  parser.add_argument('--series', dest='series',
                      type=str, nargs='+',
                      help='directory of each series of simulations')
  parser.add_argument('--descriptions', dest='descriptions',
                      type=str, nargs='+',
                      help='description for each series of simulations')
  parser.add_argument('--angles', dest='angles',
                      type=float, nargs='+',
                      help='list of angles of attack to consider')
  parser.add_argument('--coefficient', dest='coefficient',
                      type=float,
                      default=1.0,
                      help='coefficient to convert force into force coefficient')
  parser.add_argument('--validation-data', dest='validation_data',
                      type=str, nargs='+',
                      default=None,
                      help='path of files containing Taira (2008) data')
  parser.add_argument('--limits', dest='plot_limits',
                      type=float, nargs=4,
                      default=[None, None, None, None],
                      metavar=('x-start', 'x-end', 'y-start', 'y-end'),
                      help='limits of the plot')
  parser.add_argument('--no-show', dest='show',
                      action='store_false',
                      help='does not display the figures')
  parser.add_argument('--save-names', dest='save_names',
                      type=str, nargs='+',
                      default=None,
                      help='name of .png files to save')
  parser.add_argument('--save-directory', dest='save_directory',
                      type=str,
                      default=os.getcwd(),
                      help='directory where to save the .png files')
  parser.set_defaults(show=True)
  # parse given options file
  parser.add_argument('--options',
                      type=open, action=miscellaneous.ReadOptionsFromFile,
                      help='path of the file with options to parse')
  print('done')
  return parser.parse_args()

class TairaEtAl2007FlatPlate(object):
  """Contains experimental force coefficients for various angles-of-attack
  of a flat-plate with aspect-ratio 2.

  The experimental results are reported in Taira et al. (2007) and Taira (2008).
  """
  def __init__(self):
    pass

  def read_drag_coefficients(self, file_path):
    """Reads the angles-of-attack and the drag coefficients from file.

    Parameters
    ----------
    file_path: string
      Path of the file containing the angles and drag coefficients.
    """
    print('[info] reading drag coefficients from file {} ...'.format(file_path))
    with open(file_path, 'r') as infile:
      angles, values = numpy.loadtxt(infile, dtype=float, comments='#', unpack=True)
    self.cd = [angles, values]

  def read_lift_coefficients(self, file_path):
    """Reads the angles-of-attack and the lift coefficients from file.

    Parameters
    ----------
    file_path: string
      Path of the file containing the angles and lift coefficients.
    """
    print('[info] reading lift coefficients from file {} ...'.format(file_path))
    with open(file_path, 'r') as infile:
      angles, values = numpy.loadtxt(infile, dtype=float, comments='#', unpack=True)
    self.cl = [angles, values]

def plot_coefficients_vs_angles(simulations, validation,
                                coefficient=1.0,
                                save_directory=os.getcwd(),
                                save_names=[],
                                limits=[],
                                show=False):
  """Plots the force coefficients versus the angle-of-attack of the flat-plate.

  Parameters
  ----------
  simulations: dictionary of lists of Simulation objects
    Contains info about all simulations to consider.
  validation: TairaEtAl2007FlatPlate object
    Contains experimental results reported by Taira et al. (2007).
  coefficient: float, optional
    Scale value to convert a force into a force coefficient;
    default: 1.0.
  save_directory: string, optional
    Directory where to create the `images` folder;
    default: current directory.
  save_names: list of strings, optional
    Name of the files to save;
    default: [] (does not save figures)
  limits: list of floats, optional
    Limits of the plot (x-limits followed by y-limits);
    default: [].
  show: boolean, optional
    Set `True` to display the figures;
    default: False.
  """
  print('[info] plotting the coefficients versus the angle-of-attack ...'),
  images_directory = os.path.join(save_directory, 'images')
  if save_names and not os.path.isdir(images_directory):
    os.makedirs(images_directory)
  try:
    style_path = os.path.join(os.environ['SNAKE'], 'snake', 'styles',
                              'mesnardo.mplstyle')
    pyplot.style.use(style_path)
  except:
    pass
  software_labels = {'petibm': 'PetIBM'}
  fig, ax = pyplot.subplots(figsize=(8, 6))
  color_cycle = ax._get_lines.prop_cycler
  markers = iter(['o', '^', 'v'])
  ax.grid(True)
  ax.set_xlabel('angle of attack (deg)', fontsize=18)
  ax.set_ylabel('drag coefficient', fontsize=18)
  # experimental data
  ax.scatter(validation.cd[0], validation.cd[1],
             label='Taira et al. (2007)',
             marker='s', s=40,
             facecolors='none', edgecolors='#1B9E77',
             zorder=3)
  # numerical data
  for description, series in simulations.iteritems():
    ax.scatter([simulation.angle for simulation in series],
               [coefficient*simulation.force_x.values[-1] for simulation in series],
               label='{} - {}'.format(software_labels[series[0].software], description),
               marker=next(markers), s=80,
               facecolors='none',
               edgecolors=next(color_cycle)['color'],
               linewidth=1.5,
               zorder=4)
  ax.legend(loc='upper left', prop={'size': 18})
  ax.axis(limits)
  if save_names:
    pyplot.savefig(os.path.join(images_directory, save_names[0] + '.png'))
  if show:
    pyplot.show()
    pyplot.close()
  fig, ax = pyplot.subplots(figsize=(8, 6))
  color_cycle = ax._get_lines.prop_cycler
  markers = iter(['o', '^', 'v'])
  ax.grid(True)
  ax.set_xlabel('angle of attack (deg)', fontsize=18)
  ax.set_ylabel('lift coefficient', fontsize=18)
  # experimental data
  ax.scatter(validation.cl[0], validation.cl[1],
             label='Taira et al. (2007)',
             marker='s', s=40,
             facecolors='none', edgecolors='#1B9E77',
             zorder=3)
  # numerical data
  for description, series in simulations.iteritems():
    ax.scatter([simulation.angle for simulation in series],
               [coefficient*simulation.force_y.values[-1] for simulation in series],
               label='{} - {}'.format(software_labels[series[0].software], description),
               marker=next(markers), s=80,
               facecolors='none',
               edgecolors=next(color_cycle)['color'],
               linewidth=1.5,
               zorder=4)
  ax.legend(loc='upper left', prop={'size': 18})
  ax.axis(limits)
  if save_names:
    pyplot.savefig(os.path.join(images_directory, save_names[1] + '.png'))
  if show:
    pyplot.show()
    pyplot.close()
  print('done')

def main(args):
  """Plots the lift and drag coefficients versus the angle-of-attack of the
  flat-plate and compares to experimental results reported by
  Taira et al. (2007) and Taira (2008).
  """
  simulations = collections.OrderedDict()
  for description, series in zip(args.descriptions, args.series):
    series_directory = os.path.join(args.directory, series)
    subfolders = sorted([folder for folder in os.listdir(series_directory)
                         if os.path.isdir(os.path.join(series_directory, folder))])
    simulations[description] = []
    for i, subfolder in enumerate(subfolders):
      simulation_directory = os.path.join(series_directory, subfolder)
      simulations[description].append(Simulation(directory=simulation_directory,
                                                 software=args.software,
                                                 angle=args.angles[i]))
      simulations[description][-1].read_forces()

  experiment = TairaEtAl2007FlatPlate()
  experiment.read_drag_coefficients(args.validation_data[0])
  experiment.read_lift_coefficients(args.validation_data[1])

  plot_coefficients_vs_angles(simulations, experiment,
                              coefficient=args.coefficient,
                              save_directory=args.save_directory,
                              save_names=args.save_names,
                              limits=args.plot_limits,
                              show=args.show)

if __name__ == '__main__':
  print('\n[{}] START\n'.format(os.path.basename(__file__)))
  args = parse_command_line()
  main(args)
  print('\n[{}] END\n'.format(os.path.basename(__file__)))<commit_msg>Adapt script to library<commit_after># file: plotForceCoefficientsVsAoAFlatPlate3d.py
# author: Olivier Mesnard (mesnardo@gwu.edu)
# description: Plots the lift and drag coefficients for a 3d flat-plate
#              with different angles of attack

import os
import argparse
import collections

import numpy
from matplotlib import pyplot

from snake import miscellaneous
from snake.simulation import Simulation
from snake.force import Force

def parse_command_line():
  """Parses the command-line and returns the namespace."""
  print('[info] parsing the command-line ...'),
  # create parser
  parser = argparse.ArgumentParser(description='Plots the drag coefficient vs. Re',
                        formatter_class= argparse.ArgumentDefaultsHelpFormatter)
  # fill parser with arguments
  parser.add_argument('--software', dest='software',
                      type=str,
                      choices=['petibm'],
                      help='software used to generate solution')
  parser.add_argument('--directory', dest='directory',
                      type=str,
                      default=os.getcwd(),
                      help='main directory containing series directories')
  parser.add_argument('--series', dest='series',
                      type=str, nargs='+',
                      help='directory of each series of simulations')
  parser.add_argument('--descriptions', dest='descriptions',
                      type=str, nargs='+',
                      help='description for each series of simulations')
  parser.add_argument('--angles', dest='angles',
                      type=float, nargs='+',
                      help='list of angles of attack to consider')
  parser.add_argument('--coefficient', dest='coefficient',
                      type=float,
                      default=1.0,
                      help='coefficient to convert force into force coefficient')
  parser.add_argument('--validation-data', dest='validation_data',
                      type=str, nargs='+',
                      default=None,
                      help='path of files containing Taira (2008) data')
  parser.add_argument('--limits', dest='plot_limits',
                      type=float, nargs=4,
                      default=[None, None, None, None],
                      metavar=('x-start', 'x-end', 'y-start', 'y-end'),
                      help='limits of the plot')
  parser.add_argument('--no-show', dest='show',
                      action='store_false',
                      help='does not display the figures')
  parser.add_argument('--save-names', dest='save_names',
                      type=str, nargs='+',
                      default=None,
                      help='name of .png files to save')
  parser.add_argument('--save-directory', dest='save_directory',
                      type=str,
                      default=os.getcwd(),
                      help='directory where to save the .png files')
  parser.set_defaults(show=True)
  # parse given options file
  parser.add_argument('--options',
                      type=open, action=miscellaneous.ReadOptionsFromFile,
                      help='path of the file with options to parse')
  print('done')
  return parser.parse_args()

class TairaEtAl2007FlatPlate(object):
  """Contains experimental force coefficients for various angles-of-attack
  of a flat-plate with aspect-ratio 2.

  The experimental results are reported in Taira et al. (2007) and Taira (2008).
  """
  def __init__(self):
    pass

  def read_drag_coefficients(self, file_path):
    """Reads the angles-of-attack and the drag coefficients from file.

    Parameters
    ----------
    file_path: string
      Path of the file containing the angles and drag coefficients.
    """
    print('[info] reading drag coefficients from file {} ...'.format(file_path))
    with open(file_path, 'r') as infile:
      angles, values = numpy.loadtxt(infile, dtype=float, comments='#', unpack=True)
    self.cd = [angles, values]

  def read_lift_coefficients(self, file_path):
    """Reads the angles-of-attack and the lift coefficients from file.

    Parameters
    ----------
    file_path: string
      Path of the file containing the angles and lift coefficients.
    """
    print('[info] reading lift coefficients from file {} ...'.format(file_path))
    with open(file_path, 'r') as infile:
      angles, values = numpy.loadtxt(infile, dtype=float, comments='#', unpack=True)
    self.cl = [angles, values]

def plot_coefficients_vs_angles(simulations, validation,
                                coefficient=1.0,
                                save_directory=os.getcwd(),
                                save_names=[],
                                limits=[],
                                show=False):
  """Plots the force coefficients versus the angle-of-attack of the flat-plate.

  Parameters
  ----------
  simulations: dictionary of lists of Simulation objects
    Contains info about all simulations to consider.
  validation: TairaEtAl2007FlatPlate object
    Contains experimental results reported by Taira et al. (2007).
  coefficient: float, optional
    Scale value to convert a force into a force coefficient;
    default: 1.0.
  save_directory: string, optional
    Directory where to create the `images` folder;
    default: current directory.
  save_names: list of strings, optional
    Name of the files to save;
    default: [] (does not save figures)
  limits: list of floats, optional
    Limits of the plot (x-limits followed by y-limits);
    default: [].
  show: boolean, optional
    Set `True` to display the figures;
    default: False.
  """
  print('[info] plotting the coefficients versus the angle-of-attack ...'),
  images_directory = os.path.join(save_directory, 'images')
  if save_names and not os.path.isdir(images_directory):
    os.makedirs(images_directory)
  try:
    style_path = os.path.join(os.environ['SNAKE'], 'snake', 'styles',
                              'mesnardo.mplstyle')
    pyplot.style.use(style_path)
  except:
    pass
  software_labels = {'petibm': 'PetIBM'}
  fig, ax = pyplot.subplots(figsize=(8, 6))
  color_cycle = ax._get_lines.prop_cycler
  markers = iter(['o', '^', 'v'])
  ax.grid(True)
  ax.set_xlabel('angle of attack (deg)', fontsize=18)
  ax.set_ylabel('drag coefficient', fontsize=18)
  # experimental data
  ax.scatter(validation.cd[0], validation.cd[1],
             label='Taira et al. (2007)',
             marker='s', s=40,
             facecolors='none', edgecolors='#1B9E77',
             zorder=3)
  # numerical data
  for description, series in simulations.iteritems():
    ax.scatter([simulation.angle for simulation in series],
               [coefficient*simulation.forces[0].values[-1] for simulation in series],
               label='{} - {}'.format(software_labels[series[0].software], description),
               marker=next(markers), s=80,
               facecolors='none',
               edgecolors=next(color_cycle)['color'],
               linewidth=1.5,
               zorder=4)
  ax.legend(loc='upper left', prop={'size': 18})
  ax.axis(limits)
  if save_names:
    pyplot.savefig(os.path.join(images_directory, save_names[0] + '.png'))
  if show:
    pyplot.show()
    pyplot.close()
  fig, ax = pyplot.subplots(figsize=(8, 6))
  color_cycle = ax._get_lines.prop_cycler
  markers = iter(['o', '^', 'v'])
  ax.grid(True)
  ax.set_xlabel('angle of attack (deg)', fontsize=18)
  ax.set_ylabel('lift coefficient', fontsize=18)
  # experimental data
  ax.scatter(validation.cl[0], validation.cl[1],
             label='Taira et al. (2007)',
             marker='s', s=40,
             facecolors='none', edgecolors='#1B9E77',
             zorder=3)
  # numerical data
  for description, series in simulations.iteritems():
    ax.scatter([simulation.angle for simulation in series],
               [coefficient*simulation.forces[1].values[-1] for simulation in series],
               label='{} - {}'.format(software_labels[series[0].software], description),
               marker=next(markers), s=80,
               facecolors='none',
               edgecolors=next(color_cycle)['color'],
               linewidth=1.5,
               zorder=4)
  ax.legend(loc='upper left', prop={'size': 18})
  ax.axis(limits)
  if save_names:
    pyplot.savefig(os.path.join(images_directory, save_names[1] + '.png'))
  if show:
    pyplot.show()
    pyplot.close()
  print('done')

def main(args):
  """Plots the lift and drag coefficients versus the angle-of-attack of the
  flat-plate and compares to experimental results reported by
  Taira et al. (2007) and Taira (2008).
  """
  simulations = collections.OrderedDict()
  for description, series in zip(args.descriptions, args.series):
    series_directory = os.path.join(args.directory, series)
    subfolders = sorted([folder for folder in os.listdir(series_directory)
                         if os.path.isdir(os.path.join(series_directory, folder))])
    simulations[description] = []
    for i, subfolder in enumerate(subfolders):
      simulation_directory = os.path.join(series_directory, subfolder)
      simulations[description].append(Simulation(directory=simulation_directory,
                                                 software=args.software,
                                                 angle=args.angles[i]))
      simulations[description][-1].read_forces()

  experiment = TairaEtAl2007FlatPlate()
  experiment.read_drag_coefficients(args.validation_data[0])
  experiment.read_lift_coefficients(args.validation_data[1])

  plot_coefficients_vs_angles(simulations, experiment,
                              coefficient=args.coefficient,
                              save_directory=args.save_directory,
                              save_names=args.save_names,
                              limits=args.plot_limits,
                              show=args.show)

if __name__ == '__main__':
  print('\n[{}] START\n'.format(os.path.basename(__file__)))
  args = parse_command_line()
  main(args)
  print('\n[{}] END\n'.format(os.path.basename(__file__)))<|endoftext|>