<commit_before>import pytest
from mock import Mock
import json

from botocore.exceptions import WaiterError, ClientError

from formica.change_set import ChangeSet, CHANGE_SET_HEADER
from tests.unit.constants import (
    STACK, TEMPLATE, CHANGE_SET_TYPE, CHANGESETNAME, CHANGESETCHANGES,
    CHANGE_SET_PARAMETERS, ROLE_ARN, CHANGE_SET_STACK_TAGS,
    CHANGESETCHANGES_WITH_DUPLICATE_CHANGED_PARAMETER, REGION, UUID, RESOURCES
)

@pytest.fixture
def logger(mocker):
    return mocker.patch('formica.change_set.logger')

@pytest.fixture
def client(mocker):
    s3_boto = mocker.patch('formica.s3.boto3')
    boto = mocker.patch('formica.change_set.boto3')
    client = mocker.patch('formica.change_set.cf')
    return client

@pytest.fixture
def temp_bucket(mocker):
    t = mocker.patch('formica.change_set.temporary_bucket')
    tempbucket_mock = mocker.Mock()
    t.return_value.__enter__.return_value = tempbucket_mock
    return tempbucket_mock

# @pytest.fixture
# def uuid(mocker):
#     uuid = mocker.patch('formica.change_set.uuid')
#     uuid.uuid4.return_value = UUID
#     return uuid

def test_submits_changeset_and_waits(client):
    change_set = ChangeSet(STACK)

    change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE)

    client.create_change_set.assert_called_with(
        StackName=STACK, TemplateBody=TEMPLATE,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE)

    client.get_waiter.assert_called_with(
        'change_set_create_complete')
    client.get_waiter.return_value.wait.assert_called_with(
        StackName=STACK, ChangeSetName=CHANGESETNAME)

def test_creates_and_removes_bucket_for_s3_flag(client, temp_bucket):
    change_set = ChangeSet(STACK)
    bucket_name = 'formica-deploy-{}'.format("test")
    bucket_path = '{}-template.json'.format(STACK)
    template_url = 'https://{}.s3.amazonaws.com/{}'.format(bucket_name, bucket_path)
    temp_bucket.add.return_value = bucket_path
    temp_bucket.name = bucket_name

    change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE, s3=True)

    temp_bucket.add.assert_called_with(TEMPLATE)

    client.create_change_set.assert_called_with(
        StackName=STACK, TemplateURL=template_url,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE)

def test_submits_changeset_with_parameters(client):
    change_set = ChangeSet(STACK)

    change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE, parameters=CHANGE_SET_PARAMETERS)

    Parameters = [
        {'ParameterKey': 'A', 'ParameterValue': 'B', 'UsePreviousValue': False},
        {'ParameterKey': 'B', 'ParameterValue': '2', 'UsePreviousValue': False},
        {'ParameterKey': 'C', 'ParameterValue': 'True', 'UsePreviousValue': False},
    ]
    client.create_change_set.assert_called_with(
        StackName=STACK, TemplateBody=TEMPLATE,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE, Parameters=Parameters)

    client.get_waiter.assert_called_with(
        'change_set_create_complete')
    client.get_waiter.return_value.wait.assert_called_with(
        StackName=STACK, ChangeSetName=CHANGESETNAME)

def test_submits_changeset_with_stack_tags(client):
    change_set = ChangeSet(STACK)

    change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE, tags=CHANGE_SET_STACK_TAGS)

    Tags = [
        {'Key': 'T1', 'Value': 'TV1'},
        {'Key': 'T2', 'Value': 'TV2'}
    ]
    client.create_change_set.assert_called_with(
        StackName=STACK, TemplateBody=TEMPLATE,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE, Tags=Tags)

    client.get_waiter.assert_called_with(
        'change_set_create_complete')
    client.get_waiter.return_value.wait.assert_called_with(
        StackName=STACK, ChangeSetName=CHANGESETNAME)

def test_submits_changeset_with_role_arn(client):
    change_set = ChangeSet(STACK)

    change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE, role_arn=ROLE_ARN)

    client.create_change_set.assert_called_with(
        StackName=STACK, TemplateBody=TEMPLATE,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE, RoleARN=ROLE_ARN)

    client.get_waiter.assert_called_with(
        'change_set_create_complete')
    client.get_waiter.return_value.wait.assert_called_with(
        StackName=STACK, ChangeSetName=CHANGESETNAME)

def test_submits_changeset_with_capabilities(client):
    change_set = ChangeSet(STACK)

    change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE, parameters={},
                      tags={}, capabilities=['A', 'B'])

    client.create_change_set.assert_called_with(
        StackName=STACK, TemplateBody=TEMPLATE,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE, Capabilities=['A', 'B'])

    client.get_waiter.assert_called_with(
        'change_set_create_complete')
    client.get_waiter.return_value.wait.assert_called_with(
        StackName=STACK, ChangeSetName=CHANGESETNAME)

def test_prints_error_message_for_failed_submit_and_exits(capsys, logger, client):
    change_set = ChangeSet(STACK)

    error = WaiterError('name', 'reason', {'StatusReason': 'StatusReason'})
    client.get_waiter.return_value.wait.side_effect = error

    with pytest.raises(SystemExit) as pytest_wrapped_e:
        change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE)
    logger.info.assert_called_with('StatusReason')
    assert pytest_wrapped_e.value.code == 1

def test_prints_error_message_and_does_not_fail_without_StatusReason(capsys, logger, client):
    change_set = ChangeSet(STACK)

    error = WaiterError('name', 'reason', {})
    client.get_waiter.return_value.wait.side_effect = error

    with pytest.raises(SystemExit) as pytest_wrapped_e:
        change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE)
    logger.info.assert_called()
    assert pytest_wrapped_e.value.code == 1

def test_prints_error_message_but_exits_successfully_for_no_changes(capsys, logger, mocker, client):
    change_set = ChangeSet(STACK)
    status_reason = "The submitted information didn't contain changes. " \
                    "Submit different information to create a change set."

    error = WaiterError('name', 'reason', {'StatusReason': status_reason})
    client.get_waiter.return_value.wait.side_effect = error

    change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE)
    logger.info.assert_called_with(status_reason)

def test_remove_existing_changeset_for_update_type(mocker, capsys, client):
    mocker.patch.object(ChangeSet, 'describe')
    change_set = ChangeSet(STACK)
    change_set.create(template=TEMPLATE, change_set_type='UPDATE')
    client.describe_change_set.assert_called_with(StackName=STACK, ChangeSetName=CHANGESETNAME)
    client.delete_change_set.assert_called_with(StackName=STACK, ChangeSetName=CHANGESETNAME)

def test_do_not_remove_changeset_if_non_existent(client):
    change_set = ChangeSet(STACK)
    exception = ClientError(dict(Error=dict(Code='ChangeSetNotFound')), "DescribeChangeSet")
    client.describe_change_set.side_effect = exception
    change_set.remove_existing_changeset()
    client.delete_change_set.assert_not_called()

def test_reraises_exception_when_not_change_set_not_found(client):
    change_set = ChangeSet(STACK)
    exception = ClientError(dict(Error=dict(
        Code='ValidationError')), "DescribeChangeSet")
    client.describe_change_set.side_effect = exception
    with pytest.raises(ClientError):
        change_set.remove_existing_changeset()

def test_prints_changes(logger, client):
    client.describe_change_set.return_value = CHANGESETCHANGES
    change_set = ChangeSet(STACK)

    change_set.describe()

    change_set_output = '\n'.join([call[1][0] for call in logger.info.mock_calls])

    to_search = []
    to_search.extend(CHANGE_SET_HEADER)
    to_search.extend(['Remove', 'Modify', 'Add'])
    to_search.extend(['DeploymentBucket', 'DeploymentBucket2', 'DeploymentBucket3'])
    to_search.extend(['simpleteststack-deploymentbucket-1l7p61v6fxpry ',
                      'simpleteststack-deploymentbucket2-11ngaeftydtn7 '])
    to_search.extend(['AWS::S3::Bucket'])
    to_search.extend(['True'])
    to_search.extend(['BucketName, Tags'])
    # Parameters
    to_search.extend(['bucketname=formicatestbucketname, bucketname2=formicatestbucketname2'])
    # Capabilities
    to_search.extend(['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM'])
    # Stack Tags
    to_search.extend(['StackKey=StackValue, StackKey2=StackValue2'])
    for term in to_search:
        assert term in change_set_output

    assert 'None' not in change_set_output

def test_only_prints_unique_changed_parameters(logger, client):
    client.describe_change_set.return_value = CHANGESETCHANGES_WITH_DUPLICATE_CHANGED_PARAMETER
    change_set = ChangeSet(STACK)

    change_set.describe()

    change_set_output = logger.info.call_args[0][0]

    assert change_set_output.count('BucketName') == 1

def test_change_set_with_resource_types(client):
    resources = {
        'Resources': {resource: {'Type': resource} for resource in RESOURCES}
    }
    template = json.dumps(resources)
    change_set = ChangeSet(STACK)

    change_set.create(template=template, change_set_type=CHANGE_SET_TYPE, resource_types=True)

    client.create_change_set.assert_called_with(
        StackName=STACK, TemplateBody=template,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE, ResourceTypes=list(set(RESOURCES)))

def test_change_set_with_previous_template(client):
    change_set = ChangeSet(STACK)

    change_set.create(change_set_type=CHANGE_SET_TYPE, use_previous_template=True)

    client.create_change_set.assert_called_with(
        StackName=STACK,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE, UsePreviousTemplate=True)

def test_change_set_with_previous_parameters(client):
    Parameters = [
        {'ParameterKey': 'A', 'UsePreviousValue': True},
        {'ParameterKey': 'B', 'UsePreviousValue': True},
        {'ParameterKey': 'C', 'ParameterValue': '12345', 'UsePreviousValue': False},
        {'ParameterKey': 'D', 'ParameterValue': '7890', 'UsePreviousValue': False},
    ]

    client.describe_stacks.return_value = {
        'Stacks': [{'Parameters': [{'ParameterKey': 'A'}, {'ParameterKey': 'B'}]}]
    }
    change_set = ChangeSet(STACK)

    change_set.create(change_set_type=CHANGE_SET_TYPE, use_previous_template=True, use_previous_parameters=True,
                      parameters={'C': '12345', 'D': '7890'})
    client.describe_stacks.assert_called_once_with(StackName=STACK)

    client.create_change_set.assert_called_with(
        StackName=STACK,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE, Parameters=Parameters, UsePreviousTemplate=True)
<commit_msg>Test case for property changes without "Name" attribute<commit_after>import pytest
from mock import Mock
import json
import copy

from botocore.exceptions import WaiterError, ClientError

from formica.change_set import ChangeSet, CHANGE_SET_HEADER
from tests.unit.constants import (
    STACK, TEMPLATE, CHANGE_SET_TYPE, CHANGESETNAME, CHANGESETCHANGES,
    CHANGE_SET_PARAMETERS, ROLE_ARN, CHANGE_SET_STACK_TAGS,
    CHANGESETCHANGES_WITH_DUPLICATE_CHANGED_PARAMETER, REGION, UUID, RESOURCES
)

@pytest.fixture
def logger(mocker):
    return mocker.patch('formica.change_set.logger')

@pytest.fixture
def client(mocker):
    s3_boto = mocker.patch('formica.s3.boto3')
    boto = mocker.patch('formica.change_set.boto3')
    client = mocker.patch('formica.change_set.cf')
    return client

@pytest.fixture
def temp_bucket(mocker):
    t = mocker.patch('formica.change_set.temporary_bucket')
    tempbucket_mock = mocker.Mock()
    t.return_value.__enter__.return_value = tempbucket_mock
    return tempbucket_mock

# @pytest.fixture
# def uuid(mocker):
#     uuid = mocker.patch('formica.change_set.uuid')
#     uuid.uuid4.return_value = UUID
#     return uuid

def test_submits_changeset_and_waits(client):
    change_set = ChangeSet(STACK)

    change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE)

    client.create_change_set.assert_called_with(
        StackName=STACK, TemplateBody=TEMPLATE,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE)

    client.get_waiter.assert_called_with(
        'change_set_create_complete')
    client.get_waiter.return_value.wait.assert_called_with(
        StackName=STACK, ChangeSetName=CHANGESETNAME)

def test_creates_and_removes_bucket_for_s3_flag(client, temp_bucket):
    change_set = ChangeSet(STACK)
    bucket_name = 'formica-deploy-{}'.format("test")
    bucket_path = '{}-template.json'.format(STACK)
    template_url = 'https://{}.s3.amazonaws.com/{}'.format(bucket_name, bucket_path)
    temp_bucket.add.return_value = bucket_path
    temp_bucket.name = bucket_name

    change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE, s3=True)

    temp_bucket.add.assert_called_with(TEMPLATE)

    client.create_change_set.assert_called_with(
        StackName=STACK, TemplateURL=template_url,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE)

def test_submits_changeset_with_parameters(client):
    change_set = ChangeSet(STACK)

    change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE, parameters=CHANGE_SET_PARAMETERS)

    Parameters = [
        {'ParameterKey': 'A', 'ParameterValue': 'B', 'UsePreviousValue': False},
        {'ParameterKey': 'B', 'ParameterValue': '2', 'UsePreviousValue': False},
        {'ParameterKey': 'C', 'ParameterValue': 'True', 'UsePreviousValue': False},
    ]
    client.create_change_set.assert_called_with(
        StackName=STACK, TemplateBody=TEMPLATE,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE, Parameters=Parameters)

    client.get_waiter.assert_called_with(
        'change_set_create_complete')
    client.get_waiter.return_value.wait.assert_called_with(
        StackName=STACK, ChangeSetName=CHANGESETNAME)

def test_submits_changeset_with_stack_tags(client):
    change_set = ChangeSet(STACK)

    change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE, tags=CHANGE_SET_STACK_TAGS)

    Tags = [
        {'Key': 'T1', 'Value': 'TV1'},
        {'Key': 'T2', 'Value': 'TV2'}
    ]
    client.create_change_set.assert_called_with(
        StackName=STACK, TemplateBody=TEMPLATE,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE, Tags=Tags)

    client.get_waiter.assert_called_with(
        'change_set_create_complete')
    client.get_waiter.return_value.wait.assert_called_with(
        StackName=STACK, ChangeSetName=CHANGESETNAME)

def test_submits_changeset_with_role_arn(client):
    change_set = ChangeSet(STACK)

    change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE, role_arn=ROLE_ARN)

    client.create_change_set.assert_called_with(
        StackName=STACK, TemplateBody=TEMPLATE,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE, RoleARN=ROLE_ARN)

    client.get_waiter.assert_called_with(
        'change_set_create_complete')
    client.get_waiter.return_value.wait.assert_called_with(
        StackName=STACK, ChangeSetName=CHANGESETNAME)

def test_submits_changeset_with_capabilities(client):
    change_set = ChangeSet(STACK)

    change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE, parameters={},
                      tags={}, capabilities=['A', 'B'])

    client.create_change_set.assert_called_with(
        StackName=STACK, TemplateBody=TEMPLATE,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE, Capabilities=['A', 'B'])

    client.get_waiter.assert_called_with(
        'change_set_create_complete')
    client.get_waiter.return_value.wait.assert_called_with(
        StackName=STACK, ChangeSetName=CHANGESETNAME)

def test_prints_error_message_for_failed_submit_and_exits(capsys, logger, client):
    change_set = ChangeSet(STACK)

    error = WaiterError('name', 'reason', {'StatusReason': 'StatusReason'})
    client.get_waiter.return_value.wait.side_effect = error

    with pytest.raises(SystemExit) as pytest_wrapped_e:
        change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE)
    logger.info.assert_called_with('StatusReason')
    assert pytest_wrapped_e.value.code == 1

def test_prints_error_message_and_does_not_fail_without_StatusReason(capsys, logger, client):
    change_set = ChangeSet(STACK)

    error = WaiterError('name', 'reason', {})
    client.get_waiter.return_value.wait.side_effect = error

    with pytest.raises(SystemExit) as pytest_wrapped_e:
        change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE)
    logger.info.assert_called()
    assert pytest_wrapped_e.value.code == 1

def test_prints_error_message_but_exits_successfully_for_no_changes(capsys, logger, mocker, client):
    change_set = ChangeSet(STACK)
    status_reason = "The submitted information didn't contain changes. " \
                    "Submit different information to create a change set."

    error = WaiterError('name', 'reason', {'StatusReason': status_reason})
    client.get_waiter.return_value.wait.side_effect = error

    change_set.create(template=TEMPLATE, change_set_type=CHANGE_SET_TYPE)
    logger.info.assert_called_with(status_reason)

def test_remove_existing_changeset_for_update_type(mocker, capsys, client):
    mocker.patch.object(ChangeSet, 'describe')
    change_set = ChangeSet(STACK)
    change_set.create(template=TEMPLATE, change_set_type='UPDATE')
    client.describe_change_set.assert_called_with(StackName=STACK, ChangeSetName=CHANGESETNAME)
    client.delete_change_set.assert_called_with(StackName=STACK, ChangeSetName=CHANGESETNAME)

def test_do_not_remove_changeset_if_non_existent(client):
    change_set = ChangeSet(STACK)
    exception = ClientError(dict(Error=dict(Code='ChangeSetNotFound')), "DescribeChangeSet")
    client.describe_change_set.side_effect = exception
    change_set.remove_existing_changeset()
    client.delete_change_set.assert_not_called()

def test_reraises_exception_when_not_change_set_not_found(client):
    change_set = ChangeSet(STACK)
    exception = ClientError(dict(Error=dict(
        Code='ValidationError')), "DescribeChangeSet")
    client.describe_change_set.side_effect = exception
    with pytest.raises(ClientError):
        change_set.remove_existing_changeset()

def test_prints_changes(logger, client):
    client.describe_change_set.return_value = CHANGESETCHANGES
    change_set = ChangeSet(STACK)

    change_set.describe()

    change_set_output = '\n'.join([call[1][0] for call in logger.info.mock_calls])

    to_search = []
    to_search.extend(CHANGE_SET_HEADER)
    to_search.extend(['Remove', 'Modify', 'Add'])
    to_search.extend(['DeploymentBucket', 'DeploymentBucket2', 'DeploymentBucket3'])
    to_search.extend(['simpleteststack-deploymentbucket-1l7p61v6fxpry ',
                      'simpleteststack-deploymentbucket2-11ngaeftydtn7 '])
    to_search.extend(['AWS::S3::Bucket'])
    to_search.extend(['True'])
    to_search.extend(['BucketName, Tags'])
    # Parameters
    to_search.extend(['bucketname=formicatestbucketname, bucketname2=formicatestbucketname2'])
    # Capabilities
    to_search.extend(['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM'])
    # Stack Tags
    to_search.extend(['StackKey=StackValue, StackKey2=StackValue2'])
    for term in to_search:
        assert term in change_set_output

    assert 'None' not in change_set_output

def test_only_prints_unique_changed_parameters(logger, client):
    client.describe_change_set.return_value = CHANGESETCHANGES_WITH_DUPLICATE_CHANGED_PARAMETER
    change_set = ChangeSet(STACK)

    change_set.describe()

    change_set_output = logger.info.call_args[0][0]

    assert change_set_output.count('BucketName') == 1

def test_change_set_with_resource_types(client):
    resources = {
        'Resources': {resource: {'Type': resource} for resource in RESOURCES}
    }
    template = json.dumps(resources)
    change_set = ChangeSet(STACK)

    change_set.create(template=template, change_set_type=CHANGE_SET_TYPE, resource_types=True)

    client.create_change_set.assert_called_with(
        StackName=STACK, TemplateBody=template,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE, ResourceTypes=list(set(RESOURCES)))

def test_change_set_with_previous_template(client):
    change_set = ChangeSet(STACK)

    change_set.create(change_set_type=CHANGE_SET_TYPE, use_previous_template=True)

    client.create_change_set.assert_called_with(
        StackName=STACK,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE, UsePreviousTemplate=True)

def test_change_set_with_previous_parameters(client):
    Parameters = [
        {'ParameterKey': 'A', 'UsePreviousValue': True},
        {'ParameterKey': 'B', 'UsePreviousValue': True},
        {'ParameterKey': 'C', 'ParameterValue': '12345', 'UsePreviousValue': False},
        {'ParameterKey': 'D', 'ParameterValue': '7890', 'UsePreviousValue': False},
    ]

    client.describe_stacks.return_value = {
        'Stacks': [{'Parameters': [{'ParameterKey': 'A'}, {'ParameterKey': 'B'}]}]
    }
    change_set = ChangeSet(STACK)

    change_set.create(change_set_type=CHANGE_SET_TYPE, use_previous_template=True, use_previous_parameters=True,
                      parameters={'C': '12345', 'D': '7890'})
    client.describe_stacks.assert_called_once_with(StackName=STACK)

    client.create_change_set.assert_called_with(
        StackName=STACK,
        ChangeSetName=CHANGESETNAME, ChangeSetType=CHANGE_SET_TYPE, Parameters=Parameters, UsePreviousTemplate=True)

def test_change_set_without_named_properties(client):
    CHANGESET = copy.deepcopy(CHANGESETCHANGES)

    # Find a property change in the changeset
    PROPERTY_CHANGE = CHANGESET['Changes'][1]['ResourceChange']['Details'][1]
    assert PROPERTY_CHANGE['Target']['Attribute'] == 'Properties'
    assert PROPERTY_CHANGE['Target']['Name'] == 'BucketName'

    # Remove the "Name" attribute
    del PROPERTY_CHANGE['Target']['Name']

    client.describe_change_set.return_value = CHANGESET

    change_set = ChangeSet(STACK)
    change_set.describe()
<|endoftext|>


<commit_before>import os
import datetime
import zipfile
import urllib
import sys
from colorama import *

# inits colorama
init()

# returns true for Windows as an OS
def isWindows():
    return os.name == "nt"

# Returns true if the directory did not exist before
def make_dir(d):
    if not os.path.exists(d):
        os.makedirs(d)
        return True
    return False

# Prints colored text surrounded by square brackets with a time stamp before it
def msg(text, color_in = "cyan", indent = 0):
    color_in = color_in.upper()
    color = Fore.RESET

    if color_in == "BLACK": color = Fore.BLACK
    if color_in == "RED": color = Fore.RED
    if color_in == "GREEN": color = Fore.GREEN
    if color_in == "YELLOW": color = Fore.YELLOW
    if color_in == "BLUE": color = Fore.BLUE
    if color_in == "MAGENTA": color = Fore.MAGENTA
    if color_in == "CYAN": color = Fore.CYAN
    if color_in == "WHITE": color = Fore.WHITE

    now = datetime.datetime.now()
    text = "[" + str(datetime.time(now.hour, now.minute, now.second)) + "] [" + text + "]"

    print(color + '\t' * indent + "%s" % str(text) + Fore.RESET + Back.RESET + Style.RESET_ALL)

# Used for printing messages if the env var is set
def trace(text):
    envVar = os.environ.get('TRACE_MSG')
    if envVar and int(envVar) == 1:
        msg("TRACE] [" + str(text), "RED")

# Extracts a zip file and prints a percentage of the progress
def extractZip(zip, outdir = "./"):
    z = zipfile.ZipFile(zip, "r")
    uncompress_size = sum((file.file_size for file in z.infolist()))
    extracted_size = 0
    for file in z.infolist():
        extracted_size += file.file_size
        print("%s %%\r" % (extracted_size * 100/uncompress_size)),
        z.extract(file, outdir)
    print("100")

# Downloads a file
def downloadFile(url, dest):
    msg("downloading " + dest)
    try:
        if sys.version_info < (3,0):
            file = urllib.FancyURLopener()
            file.retrieve(url, dest)
        else:
            import urllib.request
            file = urllib.request.FancyURLopener()
            file.retrieve(url, dest)
    except Exception as e:
        msg('could not download "' + dest + '" because of this: ' + str(e), "RED")
        sys.exit(1)

# using a .zip because python 2.7 cant extract from .7z
def downloadAndExtractZip(url, zip_name, extract_dir):
    downloadFile(url, zip_name)
    msg("extracting " + zip_name)
    extractZip(zip_name, extract_dir)
<commit_msg>the script now really works with python 2 and 3...<commit_after>import os
import datetime
import zipfile
import urllib
import sys
from colorama import *

# inits colorama
init()

# returns true for Windows as an OS
def isWindows():
    return os.name == "nt"

# Returns true if the directory did not exist before
def make_dir(d):
    if not os.path.exists(d):
        os.makedirs(d)
        return True
    return False

# Prints colored text surrounded by square brackets with a time stamp before it
def msg(text, color_in = "cyan", indent = 0):
    color_in = color_in.upper()
    color = Fore.RESET

    if color_in == "BLACK": color = Fore.BLACK
    if color_in == "RED": color = Fore.RED
    if color_in == "GREEN": color = Fore.GREEN
    if color_in == "YELLOW": color = Fore.YELLOW
    if color_in == "BLUE": color = Fore.BLUE
    if color_in == "MAGENTA": color = Fore.MAGENTA
    if color_in == "CYAN": color = Fore.CYAN
    if color_in == "WHITE": color = Fore.WHITE

    now = datetime.datetime.now()
    text = "[" + str(datetime.time(now.hour, now.minute, now.second)) + "] [" + text + "]"

    print(color + '\t' * indent + "%s" % str(text) + Fore.RESET + Back.RESET + Style.RESET_ALL)

# Used for printing messages if the env var is set
def trace(text):
    envVar = os.environ.get('TRACE_MSG')
    if envVar and int(envVar) == 1:
        msg("TRACE] [" + str(text), "RED")

# Extracts a zip file and prints a percentage of the progress
def extractZip(zip, outdir = "./"):
    z = zipfile.ZipFile(zip, "r")
    uncompress_size = sum((file.file_size for file in z.infolist()))
    extracted_size = 0
    for file in z.infolist():
        extracted_size += file.file_size
        print("%s %%\r" % (extracted_size * 100/uncompress_size)),
        z.extract(file, outdir)
    print("100")

# Downloads a file
def downloadFile(url, dest):
    msg("downloading " + dest)
    try:
        if sys.version_info < (3,0):
            file = urllib.FancyURLopener()
            file.retrieve(url, dest)
        else:
            from urllib import request
            file = request.FancyURLopener()
            file.retrieve(url, dest)
    except Exception as e:
        msg('could not download "' + dest + '" because of this: ' + str(e), "RED")
        sys.exit(1)

# using a .zip because python 2.7 cant extract from .7z
def downloadAndExtractZip(url, zip_name, extract_dir):
    downloadFile(url, zip_name)
    msg("extracting " + zip_name)
    extractZip(zip_name, extract_dir)
<|endoftext|>


<commit_before>"""
This is a slight adaptation (for __iter__) by Terry of
http://hg.python.org/cpython/file/7cfbebadb90b/Lib/unittest/mock.py#l2239
because mock.mock_open in Python 2.7 doesn't support using a file descriptor
from "with open() as fp" as an iterator.
"""

import six

try:
    from unittest.mock import MagicMock
except ImportError:
    from mock import MagicMock

file_spec = None

def _iterate_read_data(read_data):
    # Helper for mock_open:
    # Retrieve lines from read_data via a generator so that separate calls to
    # readline, read, and readlines are properly interleaved
    data_as_list = ['{}\n'.format(l) for l in read_data.split('\n')]

    if data_as_list[-1] == '\n':
        # If the last line ended in a newline, the list comprehension will
        # have an extra entry that's just a newline.  Remove this.
        data_as_list = data_as_list[:-1]
    else:
        # If there wasn't an extra newline by itself, then the file being
        # emulated doesn't have a newline to end the last line  remove the
        # newline that our naive format() added
        data_as_list[-1] = data_as_list[-1][:-1]

    for line in data_as_list:
        yield line

class NoStopIterationReadline(six.Iterator):
    """
    A class providing a readline method whose calling does not result
    in C{StopIteration} being raised.
    """
    def __init__(self, handle, data):
        self.handle = handle
        self.data = list(data)
        self.index = 0

    def readline(self):
        if self.handle.readline.return_value is not None:
            return self.handle.readline.return_value
        elif self.index < len(self.data):
            self.index += 1
            return self.data[self.index - 1]
        else:
            # Ran out of data.
            return ''

    def __next__(self):
        line = self.readline()
        if line:
            return line
        else:
            raise StopIteration

    def __iter__(self):
        return self

def mockOpen(mock=None, read_data=''):
    """
    A helper function to create a mock to replace the use of `open`. It works
    for `open` called directly or used as a context manager.

    The `mock` argument is the mock object to configure. If `None` (the
    default) then a `MagicMock` will be created for you, with the API limited
    to methods or attributes available on standard file handles.

    `read_data` is a string for the `read` methoddline`, and `readlines` of the
    file handle to return.  This is an empty string by default.
    """
    def _readlines_side_effect(*args, **kwargs):
        if handle.readlines.return_value is not None:
            return handle.readlines.return_value
        return list(_data)

    def _read_side_effect(*args, **kwargs):
        if handle.read.return_value is not None:
            return handle.read.return_value
        return ''.join(_data)

    def _readline_side_effect():
        if handle.readline.return_value is not None:
            while True:
                yield handle.readline.return_value
        for line in _data:
            yield line

    global file_spec
    if file_spec is None:
        import _io
        file_spec = list(set(dir(_io.TextIOWrapper)).union(
            set(dir(_io.BytesIO))))

    if mock is None:
        mock = MagicMock(name='open', spec=open)

    handle = MagicMock(spec=file_spec)
    handle.__enter__.return_value = handle

    _data = _iterate_read_data(read_data)

    noStopIterationReadline = NoStopIterationReadline(
        handle, _iterate_read_data(read_data))

    handle.write.return_value = None
    handle.read.return_value = None
    handle.readline.return_value = None
    handle.readlines.return_value = None
    handle.__iter__.return_value = noStopIterationReadline

    handle.read.side_effect = _read_side_effect
    handle.readline.side_effect = noStopIterationReadline.readline
    handle.readlines.side_effect = _readlines_side_effect

    mock.return_value = handle
    return mock

class File(object):
    """
    A file mock.
    """
    def __init__(self, data):
        self._data = data
        self._index = 0

    def close(self):
        pass

    def readline(self):
        self._index += 1
        try:
            return self._data[self._index - 1]
        except IndexError:
            return ''

    def __iter__(self):
        index = self._index
        self._index = len(self._data)
        return iter(self._data[index:])

    def __exit__(self, *args):
        pass

    def __enter__(self):
        return self
<commit_msg>Fixed docstring<commit_after>"""
This is a slight adaptation (for __iter__) by Terry of
http://hg.python.org/cpython/file/7cfbebadb90b/Lib/unittest/mock.py#l2239
because mock.mock_open in Python 2.7 doesn't support using a file descriptor
from "with open() as fp" as an iterator.
"""

import six

try:
    from unittest.mock import MagicMock
except ImportError:
    from mock import MagicMock

file_spec = None

def _iterate_read_data(read_data):
    # Helper for mock_open:
    # Retrieve lines from read_data via a generator so that separate calls to
    # readline, read, and readlines are properly interleaved
    data_as_list = ['{}\n'.format(l) for l in read_data.split('\n')]

    if data_as_list[-1] == '\n':
        # If the last line ended in a newline, the list comprehension will
        # have an extra entry that's just a newline.  Remove this.
        data_as_list = data_as_list[:-1]
    else:
        # If there wasn't an extra newline by itself, then the file being
        # emulated doesn't have a newline to end the last line  remove the
        # newline that our naive format() added
        data_as_list[-1] = data_as_list[-1][:-1]

    for line in data_as_list:
        yield line

class NoStopIterationReadline(six.Iterator):
    """
    A class providing a readline method whose calling does not result
    in C{StopIteration} being raised.
    """
    def __init__(self, handle, data):
        self.handle = handle
        self.data = list(data)
        self.index = 0

    def readline(self):
        if self.handle.readline.return_value is not None:
            return self.handle.readline.return_value
        elif self.index < len(self.data):
            self.index += 1
            return self.data[self.index - 1]
        else:
            # Ran out of data.
            return ''

    def __next__(self):
        line = self.readline()
        if line:
            return line
        else:
            raise StopIteration

    def __iter__(self):
        return self

def mockOpen(mock=None, read_data=''):
    """
    A helper function to create a mock to replace the use of `open`. It works
    for `open` called directly or used as a context manager.

    The `mock` argument is the mock object to configure. If `None` (the
    default) then a `MagicMock` will be created for you, with the API limited
    to methods or attributes available on standard file handles.

    `read_data` is a string for the `read` methoddline`, and `readlines` of the
    file handle to return.  This is an empty string by default.
    """
    def _readlines_side_effect(*args, **kwargs):
        if handle.readlines.return_value is not None:
            return handle.readlines.return_value
        return list(_data)

    def _read_side_effect(*args, **kwargs):
        if handle.read.return_value is not None:
            return handle.read.return_value
        return ''.join(_data)

    def _readline_side_effect():
        if handle.readline.return_value is not None:
            while True:
                yield handle.readline.return_value
        for line in _data:
            yield line

    global file_spec
    if file_spec is None:
        import _io
        file_spec = list(set(dir(_io.TextIOWrapper)).union(
            set(dir(_io.BytesIO))))

    if mock is None:
        mock = MagicMock(name='open', spec=open)

    handle = MagicMock(spec=file_spec)
    handle.__enter__.return_value = handle

    _data = _iterate_read_data(read_data)

    noStopIterationReadline = NoStopIterationReadline(
        handle, _iterate_read_data(read_data))

    handle.write.return_value = None
    handle.read.return_value = None
    handle.readline.return_value = None
    handle.readlines.return_value = None
    handle.__iter__.return_value = noStopIterationReadline

    handle.read.side_effect = _read_side_effect
    handle.readline.side_effect = noStopIterationReadline.readline
    handle.readlines.side_effect = _readlines_side_effect

    mock.return_value = handle
    return mock

class File(object):
    """
    A file mock.

    @param data: A C{list} of C{str} lines the file should contain.
    """
    def __init__(self, data):
        self._data = data
        self._index = 0

    def close(self):
        pass

    def readline(self):
        self._index += 1
        try:
            return self._data[self._index - 1]
        except IndexError:
            return ''

    def __iter__(self):
        index = self._index
        self._index = len(self._data)
        return iter(self._data[index:])

    def __exit__(self, *args):
        pass

    def __enter__(self):
        return self
<|endoftext|>


<commit_before>"""
Generate posterior samples over orbital parameters for a given APOGEE ID.

How to use
==========

TODO: pass in prior params via P_min, P_max, jitter

"""

# Standard library
from os.path import join
import os
import sys
import time

# Third-party
import astropy.units as u
import h5py
import numpy as np
from schwimmbad import choose_pool
from thejoker.log import log as joker_logger
from thejoker import TheJoker, JokerParams

# Project
from twoface.log import log as logger
from twoface.db import db_connect
from twoface.db import JokerRun, AllStar, StarResult, Status
from twoface.config import TWOFACE_CACHE_PATH
from twoface.sample_prior import make_prior_cache

def main(db_file, pool, seed, overwrite=False):

    db_path = join(TWOFACE_CACHE_PATH, db_file)
    if not os.path.exists(db_path):
        raise IOError("sqlite database not found at '{0}'\n Did you run "
                      "scripts/initdb.py yet for that database?"
                      .format(db_path))

    logger.debug("Connecting to sqlite database at '{0}'".format(db_path))
    Session, engine = db_connect(database_path=db_path,
                                 ensure_db_exists=False)
    session = Session()

    # TODO: all hard-set, these should be options
    params = JokerParams(P_min=10 * u.day,
                         P_max=1000 * u.day,
                         jitter=150 * u.m/u.s)
    n_prior_samples = 2**26
    run_name = 'apogee-jitter'
    apogee_id = '2M01231070+1801407'

    results_filename = join(TWOFACE_CACHE_PATH, '{0}.hdf5'.format(apogee_id))
    prior_samples_file = join(TWOFACE_CACHE_PATH,
                              '{0}-prior.hdf5'.format(apogee_id))

    # Create TheJoker sampler instance with the specified random seed and pool
    rnd = np.random.RandomState(seed=seed)
    logger.debug("Creating TheJoker instance with {0}, {1}".format(rnd, pool))
    joker = TheJoker(params, random_state=rnd, pool=pool)

    # Create a cache of prior samples (if it doesn't exist) and store the
    # filename in the database.
    if not os.path.exists(prior_samples_file) or overwrite:
        logger.debug("Prior samples file not found - generating {0} samples..."
                     .format(n_prior_samples))
        make_prior_cache(prior_samples_file, joker,
                         N=n_prior_samples,
                         max_batch_size=2**22) # MAGIC NUMBER
        logger.debug("...done")

    # Query to get all stars associated with this run that need processing:
    # they should have a status id = 0 (needs processing)
    star_query = session.query(AllStar).join(StarResult, JokerRun, Status)\
                                       .filter(JokerRun.name == run_name)\
                                       .filter(AllStar.apogee_id == apogee_id)
    star = star_query.limit(1).one()

    logger.log(1, "Starting star {0}".format(star.apogee_id))
    t0 = time.time()

    data = star.apogeervdata()
    logger.log(1, "\t visits loaded ({:.2f} seconds)"
               .format(time.time()-t0))
    try:
        samples = joker.rejection_sample(
            data=data, prior_cache_file=prior_samples_file,
            return_logprobs=False)

    except Exception as e:
        logger.warning("\t Failed sampling for star {0} \n Error: {1}"
                       .format(star.apogee_id, str(e)))
        pool.close()
        sys.exit(1)

    logger.debug("\t done sampling ({:.2f} seconds)".format(time.time()-t0))

    # Write the samples that pass to the results file
    with h5py.File(results_filename, 'w') as f:
        samples.to_hdf5(f)

    logger.debug("\t saved samples ({:.2f} seconds)".format(time.time()-t0))
    logger.debug("...done with star {} ({:.2f} seconds)"
                 .format(star.apogee_id, time.time()-t0))

    pool.close()

if __name__ == "__main__":
    from argparse import ArgumentParser
    import logging

    # Define parser object
    parser = ArgumentParser(description="")

    vq_group = parser.add_mutually_exclusive_group()
    vq_group.add_argument('-v', '--verbose', action='count', default=0,
                          dest='verbosity')
    vq_group.add_argument('-q', '--quiet', action='count', default=0,
                          dest='quietness')

    parser.add_argument("-s", "--seed", dest="seed", default=None, type=int,
                        help="Random number seed")

    parser.add_argument("-o", "--overwrite", dest="overwrite", default=False, action='store_true')

    group = parser.add_mutually_exclusive_group()
    group.add_argument("--procs", dest="n_procs", default=1,
                       type=int, help="Number of processes.")
    group.add_argument("--mpi", dest="mpi", default=False,
                       action="store_true", help="Run with MPI.")

    parser.add_argument("--db", dest="db_file", required=True,
                        type=str, help="Path to the database file")

    args = parser.parse_args()

    loggers = [joker_logger, logger]

    # Set logger level based on verbose flags
    if args.verbosity != 0:
        if args.verbosity == 1:
            logger.setLevel(logging.DEBUG)
        else: # anything >= 2
            logger.setLevel(1)
            joker_logger.setLevel(1)

    elif args.quietness != 0:
        if args.quietness == 1:
            logger.setLevel(logging.WARNING)
            joker_logger.setLevel(logging.WARNING)
        else: # anything >= 2
            logger.setLevel(logging.ERROR)
            joker_logger.setLevel(logging.ERROR)

    else: # default
        logger.setLevel(logging.INFO)
        joker_logger.setLevel(logging.INFO)

    pool_kwargs = dict(mpi=args.mpi, processes=args.n_procs)
    pool = choose_pool(**pool_kwargs)

    main(db_file=args.db_file, pool=pool, seed=args.seed, overwrite=args.overwrite)
<commit_msg>sample over jitter<commit_after>"""
Generate posterior samples over orbital parameters for a given APOGEE ID.

How to use
==========

TODO: pass in prior params via P_min, P_max, jitter

"""

# Standard library
from os.path import join
import os
import sys
import time

# Third-party
import astropy.units as u
import h5py
import numpy as np
from schwimmbad import choose_pool
from thejoker.log import log as joker_logger
from thejoker import TheJoker, JokerParams

# Project
from twoface.log import log as logger
from twoface.db import db_connect
from twoface.db import JokerRun, AllStar, StarResult, Status
from twoface.config import TWOFACE_CACHE_PATH
from twoface.sample_prior import make_prior_cache

def main(db_file, pool, seed, overwrite=False):

    db_path = join(TWOFACE_CACHE_PATH, db_file)
    if not os.path.exists(db_path):
        raise IOError("sqlite database not found at '{0}'\n Did you run "
                      "scripts/initdb.py yet for that database?"
                      .format(db_path))

    logger.debug("Connecting to sqlite database at '{0}'".format(db_path))
    Session, engine = db_connect(database_path=db_path,
                                 ensure_db_exists=False)
    session = Session()

    # TODO: all hard-set, these should be options
    params = JokerParams(P_min=10 * u.day,
                         P_max=1000 * u.day,
                         jitter=(9.5, 1.64),
                         jitter_unit=u.m/u.s)
    n_prior_samples = 2**26
    run_name = 'apogee-jitter'
    apogee_id = '2M01231070+1801407'

    results_filename = join(TWOFACE_CACHE_PATH, '{0}.hdf5'.format(apogee_id))
    prior_samples_file = join(TWOFACE_CACHE_PATH,
                              '{0}-prior.hdf5'.format(apogee_id))

    # Create TheJoker sampler instance with the specified random seed and pool
    rnd = np.random.RandomState(seed=seed)
    logger.debug("Creating TheJoker instance with {0}, {1}".format(rnd, pool))
    joker = TheJoker(params, random_state=rnd, pool=pool)

    # Create a cache of prior samples (if it doesn't exist) and store the
    # filename in the database.
    if not os.path.exists(prior_samples_file) or overwrite:
        logger.debug("Prior samples file not found - generating {0} samples..."
                     .format(n_prior_samples))
        make_prior_cache(prior_samples_file, joker,
                         N=n_prior_samples,
                         max_batch_size=2**22) # MAGIC NUMBER
        logger.debug("...done")

    # Query to get all stars associated with this run that need processing:
    # they should have a status id = 0 (needs processing)
    star_query = session.query(AllStar).join(StarResult, JokerRun, Status)\
                                       .filter(JokerRun.name == run_name)\
                                       .filter(AllStar.apogee_id == apogee_id)
    star = star_query.limit(1).one()

    logger.log(1, "Starting star {0}".format(star.apogee_id))
    t0 = time.time()

    data = star.apogeervdata()
    logger.log(1, "\t visits loaded ({:.2f} seconds)"
               .format(time.time()-t0))
    try:
        samples = joker.rejection_sample(
            data=data, prior_cache_file=prior_samples_file,
            return_logprobs=False)

    except Exception as e:
        logger.warning("\t Failed sampling for star {0} \n Error: {1}"
                       .format(star.apogee_id, str(e)))
        pool.close()
        sys.exit(1)

    logger.debug("\t done sampling ({:.2f} seconds)".format(time.time()-t0))

    # Write the samples that pass to the results file
    with h5py.File(results_filename, 'w') as f:
        samples.to_hdf5(f)

    logger.debug("\t saved samples ({:.2f} seconds)".format(time.time()-t0))
    logger.debug("...done with star {} ({:.2f} seconds)"
                 .format(star.apogee_id, time.time()-t0))

    pool.close()

if __name__ == "__main__":
    from argparse import ArgumentParser
    import logging

    # Define parser object
    parser = ArgumentParser(description="")

    vq_group = parser.add_mutually_exclusive_group()
    vq_group.add_argument('-v', '--verbose', action='count', default=0,
                          dest='verbosity')
    vq_group.add_argument('-q', '--quiet', action='count', default=0,
                          dest='quietness')

    parser.add_argument("-s", "--seed", dest="seed", default=None, type=int,
                        help="Random number seed")

    parser.add_argument("-o", "--overwrite", dest="overwrite", default=False, action='store_true')

    group = parser.add_mutually_exclusive_group()
    group.add_argument("--procs", dest="n_procs", default=1,
                       type=int, help="Number of processes.")
    group.add_argument("--mpi", dest="mpi", default=False,
                       action="store_true", help="Run with MPI.")

    parser.add_argument("--db", dest="db_file", required=True,
                        type=str, help="Path to the database file")

    args = parser.parse_args()

    loggers = [joker_logger, logger]

    # Set logger level based on verbose flags
    if args.verbosity != 0:
        if args.verbosity == 1:
            logger.setLevel(logging.DEBUG)
        else: # anything >= 2
            logger.setLevel(1)
            joker_logger.setLevel(1)

    elif args.quietness != 0:
        if args.quietness == 1:
            logger.setLevel(logging.WARNING)
            joker_logger.setLevel(logging.WARNING)
        else: # anything >= 2
            logger.setLevel(logging.ERROR)
            joker_logger.setLevel(logging.ERROR)

    else: # default
        logger.setLevel(logging.INFO)
        joker_logger.setLevel(logging.INFO)

    pool_kwargs = dict(mpi=args.mpi, processes=args.n_procs)
    pool = choose_pool(**pool_kwargs)

    main(db_file=args.db_file, pool=pool, seed=args.seed, overwrite=args.overwrite)
<|endoftext|>


<commit_before>import unittest, sys, os, re
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../.."))
import framework.util as util

class UtilTests (unittest.TestCase):
    def test_try_f(self):
        self.assertEqual(util.try_f(int, "10", -1), 10, "Did not convert string to int.")
        self.assertEqual(util.try_f(int, "foo", -1), -1, "Did not get given default value as expected.")
        self.assertEqual(util.try_f(int, "foo"), None, "Did not get None as expected.")
        self.assertEqual(util.try_f(str, 10, "ten"), "10", "Did not convert int to string.")

    def test_dictsort(self):
        #NOTE: dictsort needs to be fixed.
        pass
        dl = [{"id":"c"}, {"id":"a"}, {"id":"b"}]
        #sdl = util.dictsort(dl, "id")

    def test_safeuni(self):
        class HasUnicode(object):
            def __unicode__(self):
                return u"unicode here"

        uni = u"unicode"
        base = "basestring"
        hasUni = HasUnicode()
        noUni = 5

        self.assertEqual(type(util.safeuni(uni)), unicode, "Did not return a unicode string from uni.")
        self.assertEqual(type(util.safeuni(base)), unicode, "Did not return a unicode string from base.")
        self.assertEqual(type(util.safeuni(hasUni)), unicode, "Did not return a unicode string from hasUni.")
        self.assertEqual(util.safeuni(hasUni), "unicode here", "Did not return the expected string from hasUni.")
        self.assertEqual(type(util.safeuni(noUni)), unicode, "Did not return a unicode string from noUni.")

    def test_safestr(self):
        class HasUnicode(object):
            def __unicode__(self):
                return u"unicode here"

        uni = u"unicode"
        base = "basestring"
        hasUni = HasUnicode()
        noUni = 5
        uniList = [u"z", u"y"]
        strList = ["a", "b", "c"]
        numList = [1, 2, 3]

        self.assertEqual(type(util.safestr(uni)), str)
        self.assertEqual(type(util.safestr(base)), str)
        self.assertEqual(type(util.safestr(hasUni)), str)
        self.assertEqual(type(util.safestr(noUni)), str)
        self.assertEqual(type(util.safestr(iter(strList)).next()), str)
        self.assertNotEqual(type(uniList[0]), str)
        self.assertEqual(type(util.safestr(iter(uniList)).next()), str)
        self.assertNotEqual(type(numList[0]), str)
        self.assertEqual(type(util.safestr(iter(numList)).next()), str)
        self.assertEqual(list(util.safestr(iter(numList))), ["1", "2", "3"])
        self.assertEqual(util.safestr(numList), "[1, 2, 3]")

    def test_validate_email(self):
        self.assertTrue(util.validate_email("i@u.nu"))
        self.assertTrue(util.validate_email("jake@lp.com"))
        self.assertTrue(util.validate_email("jake@lplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplplp.com"))
        self.assertFalse(util.validate_email("i@o.u"))
        self.assertFalse(util.validate_email("jake@lp."))
        self.assertFalse(util.validate_email("i@.in"))
        self.assertFalse(util.validate_email("@lp.com"))
        self.assertFalse(util.validate_email("@."))
        self.assertFalse(util.validate_email("."))
        self.assertFalse(util.validate_email("@"))

    def test_validateUSPhone(self):
        self.assertTrue(util.validateUSPhone("2134567890"))
        self.assertFalse(util.validateUSPhone("hello"))
        self.assertFalse(util.validateUSPhone("1234567890"))
        self.assertFalse(util.validateUSPhone("0213456789"))
        self.assertFalse(util.validateUSPhone("21345678900000000"))
        self.assertFalse(util.validateUSPhone("(213) 456-7890"))
        self.assertFalse(util.validateUSPhone("213-456-7890"))

    def test_cleanUSPhone(self):
        self.assertEqual(util.cleanUSPhone("2134567890"), "2134567890")
        self.assertEqual(util.cleanUSPhone("hel2134567890lo"), "2134567890")
        self.assertEqual(util.cleanUSPhone("(213) 456-7890"), "2134567890")
        self.assertEqual(util.cleanUSPhone("213-456-7890"), "2134567890")
        self.assertEqual(util.cleanUSPhone("123-456-7890"), None)

    def test_parse_tags(self):
        tag_str = '"foo, bar","abc",123'
        self.assertEqual(util.parse_tags(tag_str), ['"foo, bar"', '"abc"', '123'])

    def test_list_to_str(self):
        self.assertEqual(util.list_to_str(["a", "b", "c"]), "a b c")
        self.assertEqual(util.list_to_str([1, 2, 3]), "1 2 3")

    def test_wordcount(self):
        self.assertEqual(util.wordcount("one 2 3 four"), 4)

    def test_filesizeformat(self):
        self.assertEqual(util.filesizeformat(0), "0 bytes")
        self.assertEqual(util.filesizeformat(1), "1 byte")
        self.assertEqual(util.filesizeformat(1024), "1.0 KB")
        self.assertEqual(util.filesizeformat(1048576), "1.0 MB")

    def test_strip_html(self):
        html = "I'm not


<commit_before>import unittest
from importlib import import_module
from discretize import TensorMesh
mesh = TensorMesh([2,2,2])

deprecated_modules = [
    'SimPEG.utils.codeutils',
    'SimPEG.utils.coordutils',
    'SimPEG.utils.CounterUtils',
    'SimPEG.utils.curvutils',
    'SimPEG.utils.matutils',
    'SimPEG.utils.meshutils',
    'SimPEG.utils.ModelBuilder',
    'SimPEG.utils.PlotUtils',
    'SimPEG.utils.SolverUtils',
    'SimPEG.electromagnetics.utils.EMUtils',
    'SimPEG.electromagnetics.utils.AnalyticUtils',
    'SimPEG.electromagnetics.utils.CurrentUtils',
    'SimPEG.electromagnetics.utils.testingUtils',
    'SimPEG.electromagnetics.static.utils.StaticUtils',
]

deprecated_problems = [
    ['SimPEG.electromagnetics.frequency_domain',
        ('Problem3D_e', 'Problem3D_b', 'Problem3D_h', 'Problem3D_j')],
    ['SimPEG.electromagnetics.time_domain',
        ('Problem3D_e', 'Problem3D_b', 'Problem3D_h', 'Problem3D_j')],
    ['SimPEG.electromagnetics.natural_source',
        ('Problem3D_ePrimSec', 'Problem1D_ePrimSec')],
    ['SimPEG.electromagnetics.static.induced_polarization',
        ('Problem3D_CC', 'Problem3D_N', 'Problem2D_CC', 'Problem2D_N')],
    ['SimPEG.electromagnetics.static.resistivity',
        ('Problem3D_CC', 'Problem3D_N', 'Problem2D_CC', 'Problem2D_N')],
    ['SimPEG.electromagnetics.static.spectral_induced_polarization',
        ('Problem3D_CC', 'Problem3D_N', 'Problem2D_CC', 'Problem2D_N')],
    ['SimPEG.electromagnetics.viscous_remanent_magnetization',
        ('Problem_Linear', 'Problem_LogUnifrom')],
]

deprecated_fields = [
    ['SimPEG.electromagnetics.frequency_domain',
        ('Fields3D_e', 'Fields3D_b', 'Fields3D_h', 'Fields3D_j')],
    ['SimPEG.electromagnetics.time_domain',
        ('Fields3D_e', 'Fields3D_b', 'Fields3D_h', 'Fields3D_j')],
    ['SimPEG.electromagnetics.natural_source',
        ('Fields1D_ePrimSec', 'Fields3D_ePrimSec')],
    ['SimPEG.electromagnetics.static.resistivity',
        ('Fields_CC', 'Fields_N', 'Fields_ky', 'Fields_ky_CC', 'Fields_ky_N')],
]

class DeprecateTest(unittest.TestCase):

    def test_module_deprecations(self):
        for module in deprecated_modules:
            print(module, end='...')
            with self.assertWarns(DeprecationWarning):
                import_module(module)
            print('ok')

    def test_problem_deprecations(self):
        for module in deprecated_problems:
            mod = import_module(module[0])
            for Problem in module[1]:
                Prob = getattr(mod, Problem)
                print(f'{module[0]}.{Problem}...', end='')
                with self.assertWarns(DeprecationWarning):
                    Prob(mesh=mesh)
                print('ok')

    """
    def test_field_deprecations(self):
        for module in deprecated_fields:
            mod = import_module(module[0])
            for Field in module[1]:
                field = getattr(mod, Field)
                print(f'{module[0]}.{Field}...', end='')
                with self.assertWarns(DeprecationWarning):
                    field(mesh)
                print('ok')
    """

if __name__ == '__main__':
    unittest.main()
<commit_msg>test fields<commit_after>import unittest
from importlib import import_module
from discretize import TensorMesh
mesh = TensorMesh([2,2,2])

deprecated_modules = [
    'SimPEG.utils.codeutils',
    'SimPEG.utils.coordutils',
    'SimPEG.utils.CounterUtils',
    'SimPEG.utils.curvutils',
    'SimPEG.utils.matutils',
    'SimPEG.utils.meshutils',
    'SimPEG.utils.ModelBuilder',
    'SimPEG.utils.PlotUtils',
    'SimPEG.utils.SolverUtils',
    'SimPEG.electromagnetics.utils.EMUtils',
    'SimPEG.electromagnetics.utils.AnalyticUtils',
    'SimPEG.electromagnetics.utils.CurrentUtils',
    'SimPEG.electromagnetics.utils.testingUtils',
    'SimPEG.electromagnetics.static.utils.StaticUtils',
]

deprecated_problems = [
    ['SimPEG.electromagnetics.frequency_domain',
        ('Problem3D_e', 'Problem3D_b', 'Problem3D_h', 'Problem3D_j')],
    ['SimPEG.electromagnetics.time_domain',
        ('Problem3D_e', 'Problem3D_b', 'Problem3D_h', 'Problem3D_j')],
    ['SimPEG.electromagnetics.natural_source',
        ('Problem3D_ePrimSec', 'Problem1D_ePrimSec')],
    ['SimPEG.electromagnetics.static.induced_polarization',
        ('Problem3D_CC', 'Problem3D_N', 'Problem2D_CC', 'Problem2D_N')],
    ['SimPEG.electromagnetics.static.resistivity',
        ('Problem3D_CC', 'Problem3D_N', 'Problem2D_CC', 'Problem2D_N')],
    ['SimPEG.electromagnetics.static.spectral_induced_polarization',
        ('Problem3D_CC', 'Problem3D_N', 'Problem2D_CC', 'Problem2D_N')],
    ['SimPEG.electromagnetics.viscous_remanent_magnetization',
        ('Problem_Linear', 'Problem_LogUnifrom')],
]

deprecated_fields = [
    ['SimPEG.electromagnetics.frequency_domain',
        ('Fields3D_e', 'Fields3D_b', 'Fields3D_h', 'Fields3D_j')],
    ['SimPEG.electromagnetics.time_domain',
        ('Fields3D_e', 'Fields3D_b', 'Fields3D_h', 'Fields3D_j')],
    ['SimPEG.electromagnetics.natural_source',
        ('Fields1D_ePrimSec', 'Fields3D_ePrimSec')],
    ['SimPEG.electromagnetics.static.resistivity',
        ('Fields_CC', 'Fields_N', 'Fields_ky', 'Fields_ky_CC', 'Fields_ky_N')],
]

class DeprecateTest(unittest.TestCase):

    def test_module_deprecations(self):
        for module in deprecated_modules:
            print(module, end='...')
            with self.assertWarns(DeprecationWarning):
                import_module(module)
            print('ok')

    def test_problem_deprecations(self):
        for module in deprecated_problems:
            mod = import_module(module[0])
            for Problem in module[1]:
                Prob = getattr(mod, Problem)
                print(f'{module[0]}.{Problem}...', end='')
                with self.assertWarns(DeprecationWarning):
                    Prob(mesh=mesh)
                print('ok')

    def test_field_deprecations(self):
        for module in deprecated_fields:
            mod = import_module(module[0])
            for Field in module[1]:
                field = getattr(mod, Field)
                # Only testing for a deprecation warning so removing startup of Fields
                field.startup = lambda self: None
                print(f'{module[0]}.{Field}...', end='')
                with self.assertWarns(DeprecationWarning):
                    field(mesh)
                print('ok')

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import os
import unittest
from pprint import pprint

from illiad.account import parsers

#Directory where test data is stored.
DATA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                         'data')

class ParserTest(unittest.TestCase):

    def test_login(self):
        path = os.path.join(DATA_PATH, 'login.html')
        content = open(path).read()
        login = parsers.main_menu(content)
        self.assertTrue(login['authenticated'])
        self.assertEqual(login['session_id'],
                         'Q102112146D')

    def test_needs_registration(self):
        path = os.path.join(DATA_PATH, 'not_registered.html')
        content = open(path).read()
        login = parsers.main_menu(content)
        self.assertFalse(login['registered'])
        #self.assertEqual(login['session_id'],
        #                 'Q102112146D')

    def test_article_request_key(self):
        """
        Test a few values for an article request key.
        """
        def _key_check(key_list, rdict):
            for k in key_list:
                self.assertTrue(rdict.has_key(k),
                                msg="Request key missing key -- %s." % k)

        path = os.path.join(DATA_PATH, 'article_request.html')
        content = open(path).read()
        submit_details = parsers.request_form(content)
        request_key = submit_details
        _key_check(['PhotoJournalInclusivePages',
                    'PhotoJournalYear',
                    'PhotoJournalTitle'],
                   request_key)
        jtitle = request_key.get('PhotoJournalTitle')
        self.assertEqual(jtitle,
                         'Current pharmaceutical design')
        atitle = request_key.get('PhotoArticleTitle')
        self.assertEqual(atitle,
                         u'Targeting \xce\u01057 Nicotinic Acetylcholine Receptors in the Treatment of Schizophrenia.')
        issn = request_key.get('ISSN')
        self.assertEqual(issn,
                         u'1381-6128')

    def test_blocked_patron(self):
        path = os.path.join(DATA_PATH, 'blocked.html')
        content = open(path).read()
        request_key = parsers.request_form(content)
        self.assertTrue(request_key['blocked'])
        blocked_found = request_key['errors'].rfind('blocked')
        self.assertNotEqual(blocked_found, -1)

    def test_completed_request_submission(self):
        path = os.path.join(DATA_PATH, 'completed_request.html')
        content = open(path).read()
        submit = parsers.request_submission(content)
        self.assertFalse(submit['error'])
        self.assertEqual(submit['transaction_number'],
                        '476993')
        self.assertTrue(submit['submitted'])

    def test_failed_request_submission(self):
        path = os.path.join(DATA_PATH, 'failed_request_missing_field.html')
        content = open(path).read()
        submit = parsers.request_submission(content)
        self.assertTrue(submit['error'])
        self.assertEqual(submit['message'],
                         'Photo Journal Year is a required field.')

def suite():
    suite = unittest.makeSuite(ParserTest, 'test')
    return suite

if __name__ == '__main__':
    unittest.main()<commit_msg>Only looking for text in a string to account for variation.<commit_after>import os
import unittest
from pprint import pprint

from illiad.account import parsers

#Directory where test data is stored.
DATA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                         'data')

class ParserTest(unittest.TestCase):

    def test_login(self):
        path = os.path.join(DATA_PATH, 'login.html')
        content = open(path).read()
        login = parsers.main_menu(content)
        self.assertTrue(login['authenticated'])
        self.assertEqual(login['session_id'],
                         'Q102112146D')

    def test_needs_registration(self):
        path = os.path.join(DATA_PATH, 'not_registered.html')
        content = open(path).read()
        login = parsers.main_menu(content)
        self.assertFalse(login['registered'])
        #self.assertEqual(login['session_id'],
        #                 'Q102112146D')

    def test_article_request_key(self):
        """
        Test a few values for an article request key.
        """
        def _key_check(key_list, rdict):
            for k in key_list:
                self.assertTrue(rdict.has_key(k),
                                msg="Request key missing key -- %s." % k)

        path = os.path.join(DATA_PATH, 'article_request.html')
        content = open(path).read()
        submit_details = parsers.request_form(content)
        request_key = submit_details
        _key_check(['PhotoJournalInclusivePages',
                    'PhotoJournalYear',
                    'PhotoJournalTitle'],
                   request_key)
        jtitle = request_key.get('PhotoJournalTitle')
        self.assertEqual(jtitle,
                         'Current pharmaceutical design')
        atitle = request_key.get('PhotoArticleTitle')
        self.assertTrue(u'Nicotinic Acetylcholine Receptors' in atitle)
        issn = request_key.get('ISSN')
        self.assertEqual(issn,
                         u'1381-6128')

    def test_blocked_patron(self):
        path = os.path.join(DATA_PATH, 'blocked.html')
        content = open(path).read()
        request_key = parsers.request_form(content)
        self.assertTrue(request_key['blocked'])
        blocked_found = request_key['errors'].rfind('blocked')
        self.assertNotEqual(blocked_found, -1)

    def test_completed_request_submission(self):
        path = os.path.join(DATA_PATH, 'completed_request.html')
        content = open(path).read()
        submit = parsers.request_submission(content)
        self.assertFalse(submit['error'])
        self.assertEqual(submit['transaction_number'],
                        '476993')
        self.assertTrue(submit['submitted'])

    def test_failed_request_submission(self):
        path = os.path.join(DATA_PATH, 'failed_request_missing_field.html')
        content = open(path).read()
        submit = parsers.request_submission(content)
        self.assertTrue(submit['error'])
        self.assertEqual(submit['message'],
                         'Photo Journal Year is a required field.')

def suite():
    suite = unittest.makeSuite(ParserTest, 'test')
    return suite

if __name__ == '__main__':
    unittest.main()<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf8 -*-

"""
An implementation of stow in Python

This program assumes that the dotfiles folder is located at
$HOME/dotfiles
"""

__author__ = "curious"

import argparse
import pathlib
import sys

# Globals
ignore = [".git"]

parser = argparse.ArgumentParser(
        description="A symlinking script for handling dotfiles, similar to stow.")
parser.add_argument("-s", "--skip", dest="skip", action="store_true", default=False,
                    help="skip any conflicts")
parser.add_argument("-N", "--NO", dest="no", action="store_true", default=False,
                    help="don't perform any symlinks, just print the results.")
parser.add_argument("-Y", "--YES", dest="yes", action="store_true", default=False,
                    help="say yes to all prompts")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", default=False,
                    help="verbose mode")
parser.add_argument("-r", "--remove", dest="remove", action="store_true", default=False,
                    help="remove all symlinks")
args = parser.parse_args()

if args.no or args.yes:
    args.verbose = True

def sub_path(path, target):
    for sub in path.iterdir():
        if str(sub).split("/")[-1] == ".config":
            for sub_sub in sub.iterdir():
                target = str(target) + "/.config"
                symlink(sub_sub, pathlib.Path(target))
        else:
            symlink(sub, target)

def symlink(origin, target):
    if len(str(target).split("/")) != 2:
        target = pathlib.Path(str(target) + "/" + str(origin).split("/")[-1])

    if args.verbose:
        print_ln(origin, target)

    if not args.skip:
        confirm = None
        if not args.no and not args.yes:
            while confirm is None:
                try:
                    confirm = prompt(origin, target)
                except SyntaxError:
                    continue

        if not args.remove and (args.yes or confirm):
            if not target.is_symlink:
                origin.symlink_to(target, target.is_dir())

        if args.remove and (args.yes or confirm):
            if target.is_symlink():
                target.unlink()

def prompt(origin, target):
    if not args.remove:
        text = """Do you wish to symlink '%s' to '%s'?
{y(es) / n(o); YES (to all) / NO (to all)}: """ % (str(origin), str(target))
    else:
        text = """Do you wish to remove '%s'?
{y(es) / n(o); YES (to all) / NO (to all)}: """ % (str(target))

    inp = input(text)

    if inp.startswith("y"):
        return True
    elif inp.startswith("n"):
        return False
    elif inp == "YES":
        args.yes = True
        args.verbose = True
        return True
    elif inp == "NO":
        args.no = True
        args.verbose = True
        return False
    else:
        raise(SyntaxError)

def print_ln(origin, target):
    if not args.remove:
        print("ln -s %s %s" % (str(origin), str(target)))
    else:
        print("unlink %s" % str(target))

if __name__ == "__main__":
    home = pathlib.Path.home()
    dotfiles_dir = pathlib.Path(str(home) + "/dotfiles")

    if not dotfiles_dir.exists():
        sys.exit('%s is not a directory' % (str(home) + "/dotfiles"))

    for path in dotfiles_dir.iterdir():
        if path.is_dir():
            if str(path).split("/")[-1] in ignore:
                continue
            elif str(path).endswith("/etc"):
                symlink(path, pathlib.Path("/etc"))
            else:
                sub_path(path, home)

<commit_msg>stow.py: Added replacing<commit_after>#!/usr/bin/env python3
# -*- coding: utf8 -*-

"""
An implementation of stow in Python

This program assumes that the dotfiles folder is located at
$HOME/dotfiles
"""

__author__ = "curious"

import argparse
import pathlib
import textwrap
import shutil

from sys import exit

# Globals
ignore = [".git"]

parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=textwrap.dedent("""\
                A symlinking script for handling dotfiles in a similar manner to stow.

                WARNING: This script may crush your dreams (and files) if you are
                not careful."""))
parser.add_argument("-s", "--skip", dest="skip", action="store_true", default=False,
                    help="skip any conflicts")
parser.add_argument("-N", "--NO", dest="no", action="store_true", default=False,
                    help="don't perform any symlinks, just print the results.")
parser.add_argument("-Y", "--YES", dest="yes", action="store_true", default=False,
                    help="say yes to all prompts")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", default=False,
                    help="verbose mode")
parser.add_argument("-r", "--remove", dest="remove", action="store_true", default=False,
                    help="remove all symlinks")
parser.add_argument("-R", "--replace", dest="replace", action="store_true", default=False,
                    help="replace all existing files (no prompt)")
args = parser.parse_args()

if args.no or args.yes:
    args.verbose = True

def sub_path(path, target):
    for sub in path.iterdir():
        if str(sub).split("/")[-1] == ".config":
            for sub_sub in sub.iterdir():
                target = str(target) + "/.config"
                symlink(sub_sub, pathlib.Path(target))
        else:
            symlink(sub, target)

def symlink(origin, target):
    if len(str(target).split("/")) != 2:
        target = pathlib.Path(str(target) + "/" + str(origin).split("/")[-1])

    if args.no or args.verbose:
        print_ln(origin, target)
        return

    if not args.remove and (args.yes or prompt(origin, target)):
        if not target.is_symlink():
            if not target.is_file() and not target.is_dir():
                target.symlink_to(origin, origin.is_dir())
            else:
                if args.skip or args.no:
                    return

                if str(target) == "/etc":
                    print("Skipping... /etc will never be replaced by this script.")
                    return

                print("%s will be replaced!" % str(target))
                if args.replace or prompt(origin, target, "replace"):
                    if target.is_dir():
                        shutil.rmtree(target)
                    elif target.is_file():
                        target.unlink()

                    target.symlink_to(origin, origin.is_dir())

    elif args.remove and (args.yes or prompt(origin, target, "remove")):
        if str(target) == "/etc":
            print("Skipping... /etc will never be replaced by this script.")
            return

        if target.is_symlink():
            target.unlink()

def prompt(origin, target, action=None):
    if action == "remove":
        text = """Do you wish to remove '%s'?
{y(es) / n(o); YES (to all) / NO (to all)}: """ % (str(target))
    elif action == "replace":
        text = """Do you wish to replace '%s' with '%s'?
{y(es) / n(o); YES (to all) / NO (to all)}: """ % (str(origin), str(target))
    else:
        text = """Do you wish to symlink '%s' to '%s'?
{y(es) / n(o); YES (to all) / NO (to all)}: """ % (str(origin), str(target))

    if target.is_dir() and (action == "replace" or action == "remove"):
        text += "\nWARNING: This will delete all contents in the directory: "

    while True:
        inp = input(text)
        if len(inp) == 0:
            return True
        elif inp.startswith("y"):
            return True
        elif inp.startswith("n"):
            return False
        elif inp == "YES":
            args.yes = True
            args.verbose = True
            if action == "replace":
                args.replace = True
            return True
        elif inp == "NO":
            args.no = True
            args.verbose = True
            if action == "replace":
                args.skip = True
            return False

def print_ln(origin, target):
    if not args.remove:
        print("ln -s %s %s" % (str(origin), str(target)))
    else:
        print("unlink %s" % str(target))

if __name__ == "__main__":
    home = pathlib.Path.home()
    dotfiles_dir = pathlib.Path(str(home) + "/dotfiles")

    if not dotfiles_dir.exists():
        exit('%s is not a directory' % (str(home) + "/dotfiles"))

    for path in dotfiles_dir.iterdir():
        if path.is_dir():
            if str(path).split("/")[-1] in ignore:
                continue
            elif str(path).endswith("/etc"):
                symlink(path, pathlib.Path("/etc"))
            else:
                sub_path(path, home)

<|endoftext|>


<commit_before>import os
import requests
import yaml
from datetime import datetime
from pytest import raises, skip

from teuthology.config import config
from teuthology import suite

class TestSuiteOffline(object):
    def test_name_timestamp_passed(self):
        stamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
        name = suite.make_run_name('suite', 'ceph', 'kernel', 'flavor',
                                   'mtype', timestamp=stamp)
        assert str(stamp) in name

    def test_name_timestamp_not_passed(self):
        stamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
        name = suite.make_run_name('suite', 'ceph', 'kernel', 'flavor',
                                   'mtype')
        assert str(stamp) in name

    def test_name_user(self):
        name = suite.make_run_name('suite', 'ceph', 'kernel', 'flavor',
                                   'mtype', user='USER')
        assert name.startswith('USER-')

    def test_distro_defaults_saya(self):
        assert suite.get_distro_defaults('ubuntu', 'saya') == ('armv7l',
                                                               'saucy', 'deb')

    def test_distro_defaults_plana(self):
        assert suite.get_distro_defaults('ubuntu', 'plana') == ('x86_64',
                                                                'precise',
                                                                'deb')

    def test_gitbuilder_url(self):
        ref_url = "http://gitbuilder.ceph.com/ceph-deb-squeeze-x86_64-basic/"
        assert suite.get_gitbuilder_url('ceph', 'squeeze', 'deb', 'x86_64',
                                        'basic') == ref_url

class TestSuiteOnline(object):
    def setup(self):
        if 'TEST_ONLINE' not in os.environ:
            skip("To run these sets, set the environment variable TEST_ONLINE")

    def test_ceph_hash(self):
        resp = requests.get(
            'https://api.github.com/repos/ceph/ceph/git/refs/heads/master')
        ref_hash = resp.json()['object']['sha']
        assert suite.get_hash('ceph') == ref_hash

    def test_config_bogus_kernel_branch(self):
        # Don't attempt to send email
        config.results_email = None
        with raises(suite.ScheduleFailError):
            suite.create_initial_config('s', 'c', 't', 'bogus_kernel_branch',
                                        'f', 'd', 'm')

    def test_config_bogus_kernel_flavor(self):
        # Don't attempt to send email
        config.results_email = None
        with raises(suite.ScheduleFailError):
            suite.create_initial_config('s', 'c', 't', 'k',
                                        'bogus_kernel_flavor', 'd', 'm')

    def test_config_bogus_ceph_branch(self):
        # Don't attempt to send email
        config.results_email = None
        with raises(suite.ScheduleFailError):
            suite.create_initial_config('s', 'bogus_ceph_branch', 't', 'k',
                                        'f', 'd', 'm')

    def test_config_substitution(self):
        # Don't attempt to send email
        config.results_email = None
        job_config_str = suite.create_initial_config('MY_SUITE', 'master',
                                                     'master', 'testing',
                                                     'basic', 'centos',
                                                     'plana')
        job_config = yaml.safe_load(job_config_str)
        assert job_config['suite'] == 'MY_SUITE'

# maybe use notario for the above?
<commit_msg>Update existing unit test, and add another.<commit_after>import os
import requests
from datetime import datetime
from pytest import raises, skip

from teuthology.config import config
from teuthology import suite

class TestSuiteOffline(object):
    def test_name_timestamp_passed(self):
        stamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
        name = suite.make_run_name('suite', 'ceph', 'kernel', 'flavor',
                                   'mtype', timestamp=stamp)
        assert str(stamp) in name

    def test_name_timestamp_not_passed(self):
        stamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
        name = suite.make_run_name('suite', 'ceph', 'kernel', 'flavor',
                                   'mtype')
        assert str(stamp) in name

    def test_name_user(self):
        name = suite.make_run_name('suite', 'ceph', 'kernel', 'flavor',
                                   'mtype', user='USER')
        assert name.startswith('USER-')

    def test_distro_defaults_saya(self):
        assert suite.get_distro_defaults('ubuntu', 'saya') == ('armv7l',
                                                               'saucy', 'deb')

    def test_distro_defaults_plana(self):
        assert suite.get_distro_defaults('ubuntu', 'plana') == ('x86_64',
                                                                'precise',
                                                                'deb')

    def test_gitbuilder_url(self):
        ref_url = "http://gitbuilder.ceph.com/ceph-deb-squeeze-x86_64-basic/"
        assert suite.get_gitbuilder_url('ceph', 'squeeze', 'deb', 'x86_64',
                                        'basic') == ref_url

class TestSuiteOnline(object):
    def setup(self):
        if 'TEST_ONLINE' not in os.environ:
            skip("To run these sets, set the environment variable TEST_ONLINE")

    def test_ceph_hash(self):
        resp = requests.get(
            'https://api.github.com/repos/ceph/ceph/git/refs/heads/master')
        ref_hash = resp.json()['object']['sha']
        assert suite.get_hash('ceph') == ref_hash

    def test_config_bogus_kernel_branch(self):
        # Don't attempt to send email
        config.results_email = None
        with raises(suite.ScheduleFailError):
            suite.create_initial_config('s', 'c', 't', 'bogus_kernel_branch',
                                        'f', 'd', 'm')

    def test_config_bogus_kernel_flavor(self):
        # Don't attempt to send email
        config.results_email = None
        with raises(suite.ScheduleFailError):
            suite.create_initial_config('s', 'c', 't', 'k',
                                        'bogus_kernel_flavor', 'd', 'm')

    def test_config_bogus_ceph_branch(self):
        # Don't attempt to send email
        config.results_email = None
        with raises(suite.ScheduleFailError):
            suite.create_initial_config('s', 'bogus_ceph_branch', 't', 'k',
                                        'f', 'd', 'm')

    def test_config_substitution(self):
        # Don't attempt to send email
        config.results_email = None
        job_config = suite.create_initial_config('MY_SUITE', 'master',
                                                 'master', 'testing', 'basic',
                                                 'centos', 'plana')
        assert job_config['suite'] == 'MY_SUITE'

    def test_config_kernel_section(self):
        # Don't attempt to send email
        config.results_email = None
        job_config = suite.create_initial_config('MY_SUITE', 'master',
                                                 'master', 'testing', 'basic',
                                                 'centos', 'plana')
        assert job_config['kernel']['kdb'] is True

# maybe use notario for the above?
<|endoftext|>


<commit_before>import sys
import csv
import click
from textkit.utils import read_tokens

@click.command('count')
@click.argument('tokens', type=click.File('r'))
@click.option('--sep', default=',',
              help='Separator between token and count in output.',
              show_default=True)
def count_tokens(sep, tokens):
    '''Count unique tokens in a list of tokens.
    Tokens are sorted by top counts.'''
    content = read_tokens(tokens)
    counts = sort_counts(get_counts(content))

    # using csv writer to ensure proper encoding of the seperator.
    writer = csv.writer(sys.stdout, delimiter=sep)
    [writer.writerow(map(str, vals)) for vals in counts]

def get_counts(tokens):
    '''Count unique tokens in a list'''
    counts = {}
    for token in tokens:
        if token in counts:
            counts[token] += 1
        else:
            counts[token] = 1
    return counts

def sort_counts(counts):
    '''Sorts dict of counts by count and returns array of counts'''
    return sorted(counts.items(), key=lambda count: count[1], reverse=True)
<commit_msg>Use [get_text_stream](http://click.pocoo.org/5/api/#click.get_text_stream)  to better support python 2/3 differences. <commit_after>import sys
import csv
import click
from textkit.utils import read_tokens

@click.command('count')
@click.argument('tokens', type=click.File('r'))
@click.option('--sep', default=',',
              help='Separator between token and count in output.',
              show_default=True)
def count_tokens(sep, tokens):
    '''Count unique tokens in a list of tokens.
    Tokens are sorted by top counts.'''
    content = read_tokens(tokens)
    counts = sort_counts(get_counts(content))

    # using csv writer to ensure proper encoding of the seperator.
    writer = csv.writer(click.get_text_stream('stdout'), delimiter=sep)
    [writer.writerow(map(str, vals)) for vals in counts]

def get_counts(tokens):
    '''Count unique tokens in a list'''
    counts = {}
    for token in tokens:
        if token in counts:
            counts[token] += 1
        else:
            counts[token] = 1
    return counts

def sort_counts(counts):
    '''Sorts dict of counts by count and returns array of counts'''
    return sorted(counts.items(), key=lambda count: count[1], reverse=True)
<|endoftext|>


<commit_before># Copyright (c) 2009 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

{
  'includes': [
    '../../build/common.gypi',
  ],
  'target_defaults': {
    'conditions': [
      ['OS!="linux"', {'sources/': [['exclude', '/linux/']]}],
      ['OS!="mac"', {'sources/': [['exclude', '/mac/']]}],
      ['OS!="win"', {'sources/': [['exclude', '/win/']]}],
    ],
  },
  'targets': [
    {
      'variables': {
        'generate_stubs_script': 'generate_stubs.py',
        'sig_files': [
          'avcodec-52.sigs',
          'avformat-52.sigs',
          'avutil-50.sigs',
        ],
        'extra_header': 'ffmpeg_stub_headers.fragment',
      },
      'target_name': 'ffmpeg',
      'msvs_guid': 'D7A94F58-576A-45D9-A45F-EB87C63ABBB0',
      'sources': [
        'include/libavcodec/avcodec.h',
        'include/libavcodec/opt.h',
        'include/libavcodec/vdpau.h',
        'include/libavcodec/xvmc.h',
        'include/libavformat/avformat.h',
        'include/libavformat/avio.h',
        'include/libavutil/adler32.h',
        'include/libavutil/avstring.h',
        'include/libavutil/avutil.h',
        'include/libavutil/base64.h',
        'include/libavutil/common.h',
        'include/libavutil/crc.h',
        'include/libavutil/fifo.h',
        'include/libavutil/intfloat_readwrite.h',
        'include/libavutil/log.h',
        'include/libavutil/lzo.h',
        'include/libavutil/mathematics.h',
        'include/libavutil/md5.h',
        'include/libavutil/mem.h',
        'include/libavutil/pixfmt.h',
        'include/libavutil/rational.h',
        'include/libavutil/sha1.h',
        'include/win/inttypes.h',
        'include/win/stdint.h',
        '<@(sig_files)',
        '<(extra_header)'
      ],
      'hard_dependency': 1,
      'direct_dependent_settings': {
        'include_dirs': [
          'include',
        ],
      },
      'conditions': [
        ['OS=="win"',
          {
            'variables': {
              'outfile_type': 'windows_lib',
              'output_dir': '<(SHARED_INTERMEDIATE_DIR)'
            },
            'type': 'none',
            'dependencies': [
              'ffmpeg_binaries',
            ],
            'sources!': [
              '<(extra_header)',
            ],
            'direct_dependent_settings': {
              'link_settings': {
                'libraries': [
                  '<(output_dir)/avcodec-52.lib',
                  '<(output_dir)/avformat-52.lib',
                  '<(output_dir)/avutil-50.lib',
                ],
              },
            },
            'rules': [
              {
                'rule_name': 'generate_libs',
                'extension': 'sigs',
                'inputs': [
                  '<(generate_stubs_script)',
                  '<@(sig_files)',
                ],
                'outputs': [
                  '<(output_dir)/<(RULE_INPUT_ROOT).lib',
                ],
                'action': ['python', '<(generate_stubs_script)',
                           '-o', '<(output_dir)',
                           '-t', '<(outfile_type)',
                           '<@(RULE_INPUT_PATH)',
                ],
                'message': 'Generating FFmpeg import libraries.',
              },
            ],
          }, {  # else OS!="win"
            'variables': {
              'outfile_type': 'posix_stubs',
              'stubs_filename_root': 'ffmpeg_stubs',
              'project_path': 'third_party/ffmpeg',
              'output_root': '<(SHARED_INTERMEDIATE_DIR)/ffmpeg',
            },
            'type': '<(library)',
            'include_dirs': [
              'include',
              '<(output_root)',
              '../..',  # The chromium 'src' directory.
            ],
            'direct_dependent_settings': {
              'include_dirs': [
                '<(output_root)',
                '../..',  # The chromium 'src' directory.
              ],
            },
            'actions': [
              {
                'action_name': 'generate_stubs',
                'inputs': [
                  '<(generate_stubs_script)',
                  '<(extra_header)',
                  '<@(sig_files)',
                ],
                'outputs': [
                  '<(output_root)/<(project_path)/<(stubs_filename_root).cc',
                  '<(output_root)/<(project_path)/<(stubs_filename_root).h',
                ],
                'action': ['python',
                           '<(generate_stubs_script)',
                           '-o', '<(output_root)/<(project_path)',
                           '-t', '<(outfile_type)',
                           '-e', '<(extra_header)',
                           '-s', '<(stubs_filename_root)',
                           '-p', '<(project_path)',
                           '<@(_inputs)',
                ],
                'message': 'Generating FFmpeg stubs for dynamic loading.',
                'process_outputs_as_sources': 1,
              },
            ],
          }
        ],
      ],
    },
    {
      'target_name': 'ffmpeg_binaries',
      'type': 'none',
      'msvs_guid': '4E4070E1-EFD9-4EF1-8634-3960956F6F10',
      'conditions': [
        ['OS=="win"', {
          'sources': [
            'binaries/avcodec-52.dll',
            'binaries/avformat-52.dll',
            'binaries/avutil-50.dll',
            'binaries/pthreadGC2.dll',
          ],
          'dependencies': ['../../build/win/system.gyp:cygwin'],
          'rules': [
            {
              'rule_name': 'copy_binaries',
              'extension': 'dll',
              'inputs': [
                'copy_binaries.sh',
              ],
              'outputs': [
                '<(PRODUCT_DIR)/<(RULE_INPUT_NAME)',
              ],
              'action': ['./copy_binaries.sh', '"<@(RULE_INPUT_PATH)"', '"<@(PRODUCT_DIR)/<@(RULE_INPUT_NAME)"'],
              'message': 'Copying binaries...',
            },
          ],
        }],
      ],
    },
  ],
}
<commit_msg>Fixing include path for windows.<commit_after># Copyright (c) 2009 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

{
  'includes': [
    '../../build/common.gypi',
  ],
  'target_defaults': {
    'conditions': [
      ['OS!="linux"', {'sources/': [['exclude', '/linux/']]}],
      ['OS!="mac"', {'sources/': [['exclude', '/mac/']]}],
      ['OS!="win"', {'sources/': [['exclude', '/win/']]}],
    ],
  },
  'targets': [
    {
      'variables': {
        'generate_stubs_script': 'generate_stubs.py',
        'sig_files': [
          'avcodec-52.sigs',
          'avformat-52.sigs',
          'avutil-50.sigs',
        ],
        'extra_header': 'ffmpeg_stub_headers.fragment',
      },
      'target_name': 'ffmpeg',
      'msvs_guid': 'D7A94F58-576A-45D9-A45F-EB87C63ABBB0',
      'sources': [
        'include/libavcodec/avcodec.h',
        'include/libavcodec/opt.h',
        'include/libavcodec/vdpau.h',
        'include/libavcodec/xvmc.h',
        'include/libavformat/avformat.h',
        'include/libavformat/avio.h',
        'include/libavutil/adler32.h',
        'include/libavutil/avstring.h',
        'include/libavutil/avutil.h',
        'include/libavutil/base64.h',
        'include/libavutil/common.h',
        'include/libavutil/crc.h',
        'include/libavutil/fifo.h',
        'include/libavutil/intfloat_readwrite.h',
        'include/libavutil/log.h',
        'include/libavutil/lzo.h',
        'include/libavutil/mathematics.h',
        'include/libavutil/md5.h',
        'include/libavutil/mem.h',
        'include/libavutil/pixfmt.h',
        'include/libavutil/rational.h',
        'include/libavutil/sha1.h',
        'include/win/inttypes.h',
        'include/win/stdint.h',
        '<@(sig_files)',
        '<(extra_header)'
      ],
      'hard_dependency': 1,
      'direct_dependent_settings': {
        'include_dirs': [
          'include',
        ],
      },
      'conditions': [
        ['OS=="win"',
          {
            'variables': {
              'outfile_type': 'windows_lib',
              'output_dir': '<(SHARED_INTERMEDIATE_DIR)'
            },
            'type': 'none',
            'dependencies': [
              'ffmpeg_binaries',
            ],
            'sources!': [
              '<(extra_header)',
            ],
            'direct_dependent_settings': {
              'include_dirs': [
                'include/win',
              ],
              'link_settings': {
                'libraries': [
                  '<(output_dir)/avcodec-52.lib',
                  '<(output_dir)/avformat-52.lib',
                  '<(output_dir)/avutil-50.lib',
                ],
              },
            },
            'rules': [
              {
                'rule_name': 'generate_libs',
                'extension': 'sigs',
                'inputs': [
                  '<(generate_stubs_script)',
                  '<@(sig_files)',
                ],
                'outputs': [
                  '<(output_dir)/<(RULE_INPUT_ROOT).lib',
                ],
                'action': ['python', '<(generate_stubs_script)',
                           '-o', '<(output_dir)',
                           '-t', '<(outfile_type)',
                           '<@(RULE_INPUT_PATH)',
                ],
                'message': 'Generating FFmpeg import libraries.',
              },
            ],
          }, {  # else OS!="win"
            'variables': {
              'outfile_type': 'posix_stubs',
              'stubs_filename_root': 'ffmpeg_stubs',
              'project_path': 'third_party/ffmpeg',
              'output_root': '<(SHARED_INTERMEDIATE_DIR)/ffmpeg',
            },
            'type': '<(library)',
            'include_dirs': [
              'include',
              '<(output_root)',
              '../..',  # The chromium 'src' directory.
            ],
            'direct_dependent_settings': {
              'include_dirs': [
                '<(output_root)',
                '../..',  # The chromium 'src' directory.
              ],
            },
            'actions': [
              {
                'action_name': 'generate_stubs',
                'inputs': [
                  '<(generate_stubs_script)',
                  '<(extra_header)',
                  '<@(sig_files)',
                ],
                'outputs': [
                  '<(output_root)/<(project_path)/<(stubs_filename_root).cc',
                  '<(output_root)/<(project_path)/<(stubs_filename_root).h',
                ],
                'action': ['python',
                           '<(generate_stubs_script)',
                           '-o', '<(output_root)/<(project_path)',
                           '-t', '<(outfile_type)',
                           '-e', '<(extra_header)',
                           '-s', '<(stubs_filename_root)',
                           '-p', '<(project_path)',
                           '<@(_inputs)',
                ],
                'message': 'Generating FFmpeg stubs for dynamic loading.',
                'process_outputs_as_sources': 1,
              },
            ],
          }
        ],
      ],
    },
    {
      'target_name': 'ffmpeg_binaries',
      'type': 'none',
      'msvs_guid': '4E4070E1-EFD9-4EF1-8634-3960956F6F10',
      'conditions': [
        ['OS=="win"', {
          'sources': [
            'binaries/avcodec-52.dll',
            'binaries/avformat-52.dll',
            'binaries/avutil-50.dll',
            'binaries/pthreadGC2.dll',
          ],
          'dependencies': ['../../build/win/system.gyp:cygwin'],
          'rules': [
            {
              'rule_name': 'copy_binaries',
              'extension': 'dll',
              'inputs': [
                'copy_binaries.sh',
              ],
              'outputs': [
                '<(PRODUCT_DIR)/<(RULE_INPUT_NAME)',
              ],
              'action': ['./copy_binaries.sh', '"<@(RULE_INPUT_PATH)"', '"<@(PRODUCT_DIR)/<@(RULE_INPUT_NAME)"'],
              'message': 'Copying binaries...',
            },
          ],
        }],
      ],
    },
  ],
}
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (C) 2011-2012 Lukas Lalinsky
# Distributed under the MIT license, see the LICENSE file for details.

from acoustid.script import run_script
from acoustid.data.fingerprint import update_fingerprint_index

def main(script, opts, args):
    with closing(script.engine.connect()) as db:
        update_fingerprint_index(db, script.index)

run_script(main)

<commit_msg>Missing import<commit_after>#!/usr/bin/env python

# Copyright (C) 2011-2012 Lukas Lalinsky
# Distributed under the MIT license, see the LICENSE file for details.

from contextlib import closing
from acoustid.script import run_script
from acoustid.data.fingerprint import update_fingerprint_index

def main(script, opts, args):
    with closing(script.engine.connect()) as db:
        update_fingerprint_index(db, script.index)

run_script(main)

<|endoftext|>