<commit_before>#!/usr/bin/env python
from setuptools import find_namespace_packages
from setuptools import setup
import os

def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

package_name = "dbt-core"
package_version = "0.15.0b2"
description = """dbt (data build tool) is a command line tool that helps \
analysts and engineers transform data in their warehouse more effectively"""

setup(
    name=package_name,
    version=package_version,
    description=description,
    long_description=description,
    author="Fishtown Analytics",
    author_email="info@fishtownanalytics.com",
    url="https://github.com/fishtown-analytics/dbt",
    packages=find_namespace_packages(include=['dbt', 'dbt.*']),
    package_data={
        'dbt': [
            'include/index.html',
            'include/global_project/dbt_project.yml',
            'include/global_project/docs/*.md',
            'include/global_project/macros/*.sql',
            'include/global_project/macros/**/*.sql',
            'include/global_project/macros/**/**/*.sql',
            'py.typed',
        ]
    },
    test_suite='test',
    entry_points={
        'console_scripts': [
            'dbt = dbt.main:main',
        ],
    },
    scripts=[
        'scripts/dbt',
    ],
    install_requires=[
        'Jinja2>=2.10',
        'PyYAML>=3.11',
        'sqlparse>=0.2.3,<0.4',
        'networkx>=2.3,<3',
        'minimal-snowplow-tracker==0.0.2',
        'requests>=2.18.0,<3',
        'colorama>=0.3.9,<0.5',
        'agate>=1.6,<2',
        'jsonschema>=3.0.1,<4',
        'json-rpc>=1.12,<2',
        'werkzeug>=0.15,<0.17',
        'dataclasses==0.6;python_version<"3.7"',
        'hologram==0.0.4',
        'logbook>=1.5,<1.6',
        'pytest-logbook>=1.2.0,<1.3',
        'typing-extensions>=3.7.4,<3.8',
    ],
    zip_safe=False,
)
<commit_msg>Updated hologram to handle parallel json_schema<commit_after>#!/usr/bin/env python
from setuptools import find_namespace_packages
from setuptools import setup
import os

def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

package_name = "dbt-core"
package_version = "0.15.0b2"
description = """dbt (data build tool) is a command line tool that helps \
analysts and engineers transform data in their warehouse more effectively"""

setup(
    name=package_name,
    version=package_version,
    description=description,
    long_description=description,
    author="Fishtown Analytics",
    author_email="info@fishtownanalytics.com",
    url="https://github.com/fishtown-analytics/dbt",
    packages=find_namespace_packages(include=['dbt', 'dbt.*']),
    package_data={
        'dbt': [
            'include/index.html',
            'include/global_project/dbt_project.yml',
            'include/global_project/docs/*.md',
            'include/global_project/macros/*.sql',
            'include/global_project/macros/**/*.sql',
            'include/global_project/macros/**/**/*.sql',
            'py.typed',
        ]
    },
    test_suite='test',
    entry_points={
        'console_scripts': [
            'dbt = dbt.main:main',
        ],
    },
    scripts=[
        'scripts/dbt',
    ],
    install_requires=[
        'Jinja2>=2.10',
        'PyYAML>=3.11',
        'sqlparse>=0.2.3,<0.4',
        'networkx>=2.3,<3',
        'minimal-snowplow-tracker==0.0.2',
        'requests>=2.18.0,<3',
        'colorama>=0.3.9,<0.5',
        'agate>=1.6,<2',
        'jsonschema>=3.0.1,<4',
        'json-rpc>=1.12,<2',
        'werkzeug>=0.15,<0.17',
        'dataclasses==0.6;python_version<"3.7"',
        'hologram==0.0.5',
        'logbook>=1.5,<1.6',
        'pytest-logbook>=1.2.0,<1.3',
        'typing-extensions>=3.7.4,<3.8',
    ],
    zip_safe=False,
)
<|endoftext|>


<commit_before>import numpy as np
import tensorflow as tf
import h5py
import time
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import sys

FAST_FLAG = 0
# Functions for initializing neural nets parameters
def init_weight_variable(shape):
	initial = tf.truncated_normal(shape, stddev=0.1, dtype=tf.float32)
	return tf.Variable(initial)

def init_bias_variable(shape):
	initial = tf.constant(0.1, shape=shape, dtype=tf.float32)
	return tf.Variable(initial)

def batch_nm(x, eps=1e-5):
	# batch normalization to have zero mean and unit variance
	mu, var = tf.nn.moments(x, [0])
	return tf.nn.batch_normalization(x, mu, var, None, None, eps)

# ==============================================
# ==============================================
# 					main driver
# ==============================================
# ==============================================
print('==> Experiment 4h: Batch normalization (Post-activated)')
filepath = '/pylon2/ci560sp/haunter/exp3_taylorswift_d15_1s_C1C8.mat'
if FAST_FLAG:
	filepath = '/pylon2/ci560sp/haunter/exp3_small.mat'
print('==> Loading data from {}...'.format(filepath))
# benchmark
t_start = time.time()

# ==============================================
# 				reading data
# ==============================================
f = h5py.File(filepath)
X_train = np.array(f.get('trainingFeatures'))
y_train = np.array(f.get('trainingLabels'))
X_val = np.array(f.get('validationFeatures'))
y_val = np.array(f.get('validationLabels'))

t_end = time.time()
print('--Time elapsed for loading data: {t:.2f} \
		seconds'.format(t = t_end - t_start))
del f
print('-- Number of training samples: {}'.format(X_train.shape[0]))
print('-- Number of validation samples: {}'.format(X_val.shape[0]))

# ==============================================
# Neural-network model set-up
# ==============================================

num_training_vec, total_features = X_train.shape
num_freq = 169
num_frames = int(total_features / num_freq)
num_classes = int(max(y_train.max(), y_val.max()) + 1)

num_layers = 2
fac = 0.5

try:
	fac = float(sys.argv[1])
	num_layers = int(sys.argv[2])
	assert(num_layers >= 2 and num_layers <= 4)
except Exception, e:
	print('-- {}'.format(e))
print('-- Decreasing factor = {}'.format(fac))
print('-- Number of layers = {}'.format(num_layers))

size_list = []
for num in range(num_layers + 1):
	size_list.append(int(total_features * np.power(fac, num)))
print('-- Layer sizes = {}'.format(size_list))

batch_size = 1000

num_epochs = 500
print_freq = 10
if FAST_FLAG:
	num_epochs = 5
	print_freq = 1

# Transform labels into on-hot encoding form
y_train_OHEnc = tf.one_hot(y_train.copy(), num_classes)
y_val_OHEnc = tf.one_hot(y_val.copy(), num_classes)

# reset placeholders
x = tf.placeholder(tf.float32, [None, total_features])
y_ = tf.placeholder(tf.float32, [None, num_classes])

W_ae_list = [init_weight_variable([size_list[i], size_list[i + 1]]) \
	for i in range(num_layers)]
b_ae_list = [init_bias_variable([size_list[i + 1]])\
	for i in range(num_layers)]
a_list = [batch_nm(tf.nn.relu(tf.matmul(x, W_ae_list[0]) + b_ae_list[0]))]
for i in range(num_layers - 1):
	# batch normalization for post-activated values
	a_i = batch_nm(tf.nn.relu(tf.matmul(a_list[-1], W_ae_list[i + 1]) + b_ae_list[i + 1]))
	a_list.append(a_i)

# dropout
keep_prob = tf.placeholder(tf.float32)
a_drop = tf.nn.dropout(a_list[-1], keep_prob)
W_sm = init_weight_variable([size_list[-1], num_classes])
b_sm = init_bias_variable([num_classes])
y_sm = tf.matmul(a_drop, W_sm) + b_sm

cross_entropy = tf.reduce_mean(
		tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_sm))
train_step = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_sm, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
y_train = sess.run(y_train_OHEnc)[:, 0, :]
y_val = sess.run(y_val_OHEnc)[:, 0, :]

# evaluation metrics
train_acc_list = []
val_acc_list = []
train_err_list = []
val_err_list = []

print('==> Training the full network...')
t_start = time.time()
for epoch in range(num_epochs):
	for i in range(0, num_training_vec, batch_size):
		batch_end_point = min(i + batch_size, num_training_vec)
		train_batch_data = X_train[i : batch_end_point]
		train_batch_label = y_train[i : batch_end_point]
		train_step.run(feed_dict={x: train_batch_data, y_: train_batch_label, keep_prob: 0.5})
	if (epoch + 1) % print_freq == 0:
		train_acc = accuracy.eval(feed_dict={x:X_train, y_: y_train, keep_prob: 1.0})
		train_acc_list.append(train_acc)
		val_acc = accuracy.eval(feed_dict={x: X_val, y_: y_val, keep_prob: 1.0})
		val_acc_list.append(val_acc)
		train_err = cross_entropy.eval(feed_dict={x: X_train, y_: y_train, keep_prob: 1.0})
		train_err_list.append(train_err)
		val_err = cross_entropy.eval(feed_dict={x: X_val, y_: y_val, keep_prob: 1.0})
		val_err_list.append(val_err)
		print("-- epoch: %d, training error %g"%(epoch + 1, train_err))

t_end = time.time()
print('--Time elapsed for training: {t:.2f} \
		seconds'.format(t = t_end - t_start))

# ==============================================
# Reports
print('-- Training accuracy: {:.4f}'.format(train_acc_list[-1]))
print('-- Validation accuracy: {:.4f}'.format(val_acc_list[-1]))
print('-- Training error: {:.4E}'.format(train_err_list[-1]))
print('-- Validation error: {:.4E}'.format(val_err_list[-1]))

print('==> Generating error plot...')
x_list = range(0, print_freq * len(train_acc_list), print_freq)
train_err_plot = plt.plot(x_list, train_err_list, 'b-', label='training')
val_err_plot = plt.plot(x_list, val_err_list, '-', color='orange', label='validation')
plt.xlabel('Number of epochs')
plt.ylabel('Cross-Entropy Error')
plt.title('Error vs Number of Epochs with {} Layers and Decreasing Factor {}'.format(num_layers, fac))
plt.legend(loc='best')
plt.savefig('exp4h_L{}F{}.png'.format(num_layers, fac), format='png')
plt.close()

print('==> Finished!')
<commit_msg>flag change<commit_after>import numpy as np
import tensorflow as tf
import h5py
import time
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import sys

FAST_FLAG = 1
# Functions for initializing neural nets parameters
def init_weight_variable(shape):
	initial = tf.truncated_normal(shape, stddev=0.1, dtype=tf.float32)
	return tf.Variable(initial)

def init_bias_variable(shape):
	initial = tf.constant(0.1, shape=shape, dtype=tf.float32)
	return tf.Variable(initial)

def batch_nm(x, eps=1e-5):
	# batch normalization to have zero mean and unit variance
	mu, var = tf.nn.moments(x, [0])
	return tf.nn.batch_normalization(x, mu, var, None, None, eps)

# ==============================================
# ==============================================
# 					main driver
# ==============================================
# ==============================================
print('==> Experiment 4h: Batch normalization (Post-activated)')
filepath = '/pylon2/ci560sp/haunter/exp3_taylorswift_d15_1s_C1C8.mat'
if FAST_FLAG:
	filepath = '/pylon2/ci560sp/haunter/exp3_small.mat'
print('==> Loading data from {}...'.format(filepath))
# benchmark
t_start = time.time()

# ==============================================
# 				reading data
# ==============================================
f = h5py.File(filepath)
X_train = np.array(f.get('trainingFeatures'))
y_train = np.array(f.get('trainingLabels'))
X_val = np.array(f.get('validationFeatures'))
y_val = np.array(f.get('validationLabels'))

t_end = time.time()
print('--Time elapsed for loading data: {t:.2f} \
		seconds'.format(t = t_end - t_start))
del f
print('-- Number of training samples: {}'.format(X_train.shape[0]))
print('-- Number of validation samples: {}'.format(X_val.shape[0]))

# ==============================================
# Neural-network model set-up
# ==============================================

num_training_vec, total_features = X_train.shape
num_freq = 169
num_frames = int(total_features / num_freq)
num_classes = int(max(y_train.max(), y_val.max()) + 1)

num_layers = 2
fac = 0.5

try:
	fac = float(sys.argv[1])
	num_layers = int(sys.argv[2])
	assert(num_layers >= 2 and num_layers <= 4)
except Exception, e:
	print('-- {}'.format(e))
print('-- Decreasing factor = {}'.format(fac))
print('-- Number of layers = {}'.format(num_layers))

size_list = []
for num in range(num_layers + 1):
	size_list.append(int(total_features * np.power(fac, num)))
print('-- Layer sizes = {}'.format(size_list))

batch_size = 1000

num_epochs = 500
print_freq = 10
if FAST_FLAG:
	num_epochs = 5
	print_freq = 1

# Transform labels into on-hot encoding form
y_train_OHEnc = tf.one_hot(y_train.copy(), num_classes)
y_val_OHEnc = tf.one_hot(y_val.copy(), num_classes)

# reset placeholders
x = tf.placeholder(tf.float32, [None, total_features])
y_ = tf.placeholder(tf.float32, [None, num_classes])

W_ae_list = [init_weight_variable([size_list[i], size_list[i + 1]]) \
	for i in range(num_layers)]
b_ae_list = [init_bias_variable([size_list[i + 1]])\
	for i in range(num_layers)]
a_list = [batch_nm(tf.nn.relu(tf.matmul(x, W_ae_list[0]) + b_ae_list[0]))]
for i in range(num_layers - 1):
	# batch normalization for post-activated values
	a_i = batch_nm(tf.nn.relu(tf.matmul(a_list[-1], W_ae_list[i + 1]) + b_ae_list[i + 1]))
	a_list.append(a_i)

# dropout
keep_prob = tf.placeholder(tf.float32)
a_drop = tf.nn.dropout(a_list[-1], keep_prob)
W_sm = init_weight_variable([size_list[-1], num_classes])
b_sm = init_bias_variable([num_classes])
y_sm = tf.matmul(a_drop, W_sm) + b_sm

cross_entropy = tf.reduce_mean(
		tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_sm))
train_step = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_sm, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
y_train = sess.run(y_train_OHEnc)[:, 0, :]
y_val = sess.run(y_val_OHEnc)[:, 0, :]

# evaluation metrics
train_acc_list = []
val_acc_list = []
train_err_list = []
val_err_list = []

print('==> Training the full network...')
t_start = time.time()
for epoch in range(num_epochs):
	for i in range(0, num_training_vec, batch_size):
		batch_end_point = min(i + batch_size, num_training_vec)
		train_batch_data = X_train[i : batch_end_point]
		train_batch_label = y_train[i : batch_end_point]
		train_step.run(feed_dict={x: train_batch_data, y_: train_batch_label, keep_prob: 0.5})
	if (epoch + 1) % print_freq == 0:
		train_acc = accuracy.eval(feed_dict={x:X_train, y_: y_train, keep_prob: 1.0})
		train_acc_list.append(train_acc)
		val_acc = accuracy.eval(feed_dict={x: X_val, y_: y_val, keep_prob: 1.0})
		val_acc_list.append(val_acc)
		train_err = cross_entropy.eval(feed_dict={x: X_train, y_: y_train, keep_prob: 1.0})
		train_err_list.append(train_err)
		val_err = cross_entropy.eval(feed_dict={x: X_val, y_: y_val, keep_prob: 1.0})
		val_err_list.append(val_err)
		print("-- epoch: %d, training error %g"%(epoch + 1, train_err))

t_end = time.time()
print('--Time elapsed for training: {t:.2f} \
		seconds'.format(t = t_end - t_start))

# ==============================================
# Reports
print('-- Training accuracy: {:.4f}'.format(train_acc_list[-1]))
print('-- Validation accuracy: {:.4f}'.format(val_acc_list[-1]))
print('-- Training error: {:.4E}'.format(train_err_list[-1]))
print('-- Validation error: {:.4E}'.format(val_err_list[-1]))

print('==> Generating error plot...')
x_list = range(0, print_freq * len(train_acc_list), print_freq)
train_err_plot = plt.plot(x_list, train_err_list, 'b-', label='training')
val_err_plot = plt.plot(x_list, val_err_list, '-', color='orange', label='validation')
plt.xlabel('Number of epochs')
plt.ylabel('Cross-Entropy Error')
plt.title('Error vs Number of Epochs with {} Layers and Decreasing Factor {}'.format(num_layers, fac))
plt.legend(loc='best')
plt.savefig('exp4h_L{}F{}.png'.format(num_layers, fac), format='png')
plt.close()

print('==> Finished!')
<|endoftext|>


<commit_before>from datetime import timedelta
from logging import getLogger
from socket import error, inet_pton, AF_INET6

from django.core.cache import caches
from django.http import HttpResponse, HttpResponseRedirect, JsonResponse
from django.shortcuts import render

import ipware.ip2

from axes.conf import settings
from axes.models import AccessAttempt

logger = getLogger(__name__)

def get_axes_cache():
    return caches[getattr(settings, 'AXES_CACHE', 'default')]

def query2str(dictionary, max_length=1024):
    """
    Turns a dictionary into an easy-to-read list of key-value pairs.

    If there is a field called password it will be excluded from the output.

    The length of the output is limited to max_length to avoid a DoS attack via excessively large payloads.
    """

    return '\n'.join([
        '%s=%s' % (k, v) for k, v in dictionary.items()
        if k != settings.AXES_PASSWORD_FORM_FIELD
    ][:int(max_length / 2)])[:max_length]

def get_client_str(username, ip_address, user_agent=None, path_info=None):
    if settings.AXES_VERBOSE:
        if path_info and isinstance(path_info, tuple):
            path_info = path_info[0]

        details = '{{user: "{0}", ip: "{1}", user-agent: "{2}", path: "{3}"}}'
        return details.format(username, ip_address, user_agent, path_info)

    if settings.AXES_ONLY_USER_FAILURES:
        client = username
    elif settings.AXES_LOCK_OUT_BY_COMBINATION_USER_AND_IP:
        client = '{0} from {1}'.format(username, ip_address)
    else:
        client = ip_address

    if settings.AXES_USE_USER_AGENT:
        client += '(user-agent={0})'.format(user_agent)

    return client

def get_client_ip(request):
    client_ip_attribute = 'axes_client_ip'

    if not hasattr(request, client_ip_attribute):
        client_ip, _ = ipware.ip2.get_client_ip(
            request,
            proxy_order=settings.AXES_PROXY_ORDER,
            proxy_count=settings.AXES_PROXY_COUNT,
            proxy_trusted_ips=settings.AXES_PROXY_TRUSTED_IPS,
            request_header_order=settings.AXES_META_PRECEDENCE_ORDER,
        )
        setattr(request, client_ip_attribute, client_ip)
    return getattr(request, client_ip_attribute)

def get_client_username(request, credentials=None):
    """
    Resolve client username from the given request or credentials if supplied.

    The order of preference for fetching the username is as follows:

    1. If configured, use ``AXES_USERNAME_CALLABLE``, and supply ``request, credentials`` as arguments
    2. If given, use ``credentials`` and fetch username from ``AXES_USERNAME_FORM_FIELD`` (defaults to ``username``)
    3. Use request.POST and fetch username from ``AXES_USERNAME_FORM_FIELD`` (defaults to ``username``)

    :param request: incoming Django ``HttpRequest`` or similar object from authentication backend or other source
    :param credentials: incoming credentials ``dict`` or similar object from authentication backend or other source
    """

    if settings.AXES_USERNAME_CALLABLE:
        logger.debug('Using AXES_USERNAME_CALLABLE to get username')
        return settings.AXES_USERNAME_CALLABLE(request, credentials)

    if credentials:
        logger.debug('Using `credentials` to get username with key AXES_USERNAME_FORM_FIELD')
        return credentials.get(settings.AXES_USERNAME_FORM_FIELD, None)

    logger.debug('Using `request.POST` to get username with key AXES_USERNAME_FORM_FIELD')
    return request.POST.get(settings.AXES_USERNAME_FORM_FIELD, None)

def get_credentials(username=None, **kwargs):
    credentials = {settings.AXES_USERNAME_FORM_FIELD: username}
    credentials.update(kwargs)
    return credentials

def is_ipv6(ip):
    try:
        inet_pton(AF_INET6, ip)
    except (OSError, error):
        return False
    return True

def reset(ip=None, username=None):
    """
    Reset records that match IP or username, and return the count of removed attempts.
    """

    attempts = AccessAttempt.objects.all()
    if ip:
        attempts = attempts.filter(ip_address=ip)
    if username:
        attempts = attempts.filter(username=username)

    count, _ = attempts.delete()

    return count

def iso8601(timestamp):
    """
    Return datetime.timedelta translated to ISO 8601 formatted duration.
    """

    seconds = timestamp.total_seconds()
    minutes, seconds = divmod(seconds, 60)
    hours, minutes = divmod(minutes, 60)
    days, hours = divmod(hours, 24)

    date = '{:.0f}D'.format(days) if days else ''

    time_values = hours, minutes, seconds
    time_designators = 'H', 'M', 'S'

    time = ''.join([
        ('{:.0f}'.format(value) + designator)
        for value, designator in zip(time_values, time_designators)
        if value]
    )
    return 'P' + date + ('T' + time if time else '')

def get_lockout_message():
    if settings.AXES_COOLOFF_TIME:
        return settings.AXES_COOLOFF_MESSAGE
    return settings.AXES_PERMALOCK_MESSAGE

def get_lockout_response(request):
    context = {
        'failure_limit': settings.AXES_FAILURE_LIMIT,
        'username': get_client_username(request) or ''
    }

    cool_off = settings.AXES_COOLOFF_TIME
    if cool_off:
        if isinstance(cool_off, (int, float)):
            cool_off = timedelta(hours=cool_off)

        context.update({
            'cooloff_time': iso8601(cool_off)
        })

    status = 403

    if request.is_ajax():
        return JsonResponse(
            context,
            status=status,
        )

    if settings.AXES_LOCKOUT_TEMPLATE:
        return render(
            request,
            settings.AXES_LOCKOUT_TEMPLATE,
            context,
            status=status,
        )

    if settings.AXES_LOCKOUT_URL:
        return HttpResponseRedirect(settings.AXES_LOCKOUT_URL)

    return HttpResponse(get_lockout_message(), status=status)
<commit_msg>Add type annotations to utils module<commit_after>from datetime import timedelta
from logging import getLogger
from socket import error, inet_pton, AF_INET6

from django.core.cache import caches, BaseCache
from django.http import HttpResponse, HttpResponseRedirect, HttpRequest, JsonResponse
from django.shortcuts import render

import ipware.ip2

from axes.conf import settings
from axes.models import AccessAttempt

logger = getLogger(__name__)

def get_axes_cache() -> BaseCache:
    return caches[getattr(settings, 'AXES_CACHE', 'default')]

def query2str(dictionary: dict, max_length: int = 1024) -> str:
    """
    Turns a dictionary into an easy-to-read list of key-value pairs.

    If there is a field called password it will be excluded from the output.

    The length of the output is limited to max_length to avoid a DoS attack via excessively large payloads.
    """

    return '\n'.join([
        '%s=%s' % (k, v) for k, v in dictionary.items()
        if k != settings.AXES_PASSWORD_FORM_FIELD
    ][:int(max_length / 2)])[:max_length]

def get_client_str(username, ip_address, user_agent=None, path_info=None):
    if settings.AXES_VERBOSE:
        if path_info and isinstance(path_info, tuple):
            path_info = path_info[0]

        details = '{{user: "{0}", ip: "{1}", user-agent: "{2}", path: "{3}"}}'
        return details.format(username, ip_address, user_agent, path_info)

    if settings.AXES_ONLY_USER_FAILURES:
        client = username
    elif settings.AXES_LOCK_OUT_BY_COMBINATION_USER_AND_IP:
        client = '{0} from {1}'.format(username, ip_address)
    else:
        client = ip_address

    if settings.AXES_USE_USER_AGENT:
        client += '(user-agent={0})'.format(user_agent)

    return client

def get_client_ip(request: HttpRequest) -> str:
    client_ip_attribute = 'axes_client_ip'

    if not hasattr(request, client_ip_attribute):
        client_ip, _ = ipware.ip2.get_client_ip(
            request,
            proxy_order=settings.AXES_PROXY_ORDER,
            proxy_count=settings.AXES_PROXY_COUNT,
            proxy_trusted_ips=settings.AXES_PROXY_TRUSTED_IPS,
            request_header_order=settings.AXES_META_PRECEDENCE_ORDER,
        )
        setattr(request, client_ip_attribute, client_ip)
    return getattr(request, client_ip_attribute)

def get_client_username(request: HttpRequest, credentials: dict = None):
    """
    Resolve client username from the given request or credentials if supplied.

    The order of preference for fetching the username is as follows:

    1. If configured, use ``AXES_USERNAME_CALLABLE``, and supply ``request, credentials`` as arguments
    2. If given, use ``credentials`` and fetch username from ``AXES_USERNAME_FORM_FIELD`` (defaults to ``username``)
    3. Use request.POST and fetch username from ``AXES_USERNAME_FORM_FIELD`` (defaults to ``username``)

    :param request: incoming Django ``HttpRequest`` or similar object from authentication backend or other source
    :param credentials: incoming credentials ``dict`` or similar object from authentication backend or other source
    """

    if settings.AXES_USERNAME_CALLABLE:
        logger.debug('Using AXES_USERNAME_CALLABLE to get username')
        return settings.AXES_USERNAME_CALLABLE(request, credentials)

    if credentials:
        logger.debug('Using `credentials` to get username with key AXES_USERNAME_FORM_FIELD')
        return credentials.get(settings.AXES_USERNAME_FORM_FIELD, None)

    logger.debug('Using `request.POST` to get username with key AXES_USERNAME_FORM_FIELD')
    return request.POST.get(settings.AXES_USERNAME_FORM_FIELD, None)

def get_credentials(username: str = None, **kwargs):
    credentials = {settings.AXES_USERNAME_FORM_FIELD: username}
    credentials.update(kwargs)
    return credentials

def is_ipv6(ip: str):
    try:
        inet_pton(AF_INET6, ip)
    except (OSError, error):
        return False
    return True

def reset(ip: str = None, username: str = None):
    """
    Reset records that match IP or username, and return the count of removed attempts.
    """

    attempts = AccessAttempt.objects.all()
    if ip:
        attempts = attempts.filter(ip_address=ip)
    if username:
        attempts = attempts.filter(username=username)

    count, _ = attempts.delete()

    return count

def iso8601(delta: timedelta) -> str:
    """
    Return datetime.timedelta translated to ISO 8601 formatted duration.
    """

    seconds = delta.total_seconds()
    minutes, seconds = divmod(seconds, 60)
    hours, minutes = divmod(minutes, 60)
    days, hours = divmod(hours, 24)

    date = '{:.0f}D'.format(days) if days else ''

    time_values = hours, minutes, seconds
    time_designators = 'H', 'M', 'S'

    time = ''.join([
        ('{:.0f}'.format(value) + designator)
        for value, designator in zip(time_values, time_designators)
        if value]
    )
    return 'P' + date + ('T' + time if time else '')

def get_lockout_message() -> str:
    if settings.AXES_COOLOFF_TIME:
        return settings.AXES_COOLOFF_MESSAGE
    return settings.AXES_PERMALOCK_MESSAGE

def get_lockout_response(request: HttpRequest) -> HttpResponse:
    context = {
        'failure_limit': settings.AXES_FAILURE_LIMIT,
        'username': get_client_username(request) or ''
    }

    cool_off = settings.AXES_COOLOFF_TIME
    if cool_off:
        if isinstance(cool_off, (int, float)):
            cool_off = timedelta(hours=cool_off)

        context.update({
            'cooloff_time': iso8601(cool_off)
        })

    status = 403

    if request.is_ajax():
        return JsonResponse(
            context,
            status=status,
        )

    if settings.AXES_LOCKOUT_TEMPLATE:
        return render(
            request,
            settings.AXES_LOCKOUT_TEMPLATE,
            context,
            status=status,
        )

    if settings.AXES_LOCKOUT_URL:
        return HttpResponseRedirect(settings.AXES_LOCKOUT_URL)

    return HttpResponse(get_lockout_message(), status=status)
<|endoftext|>


<commit_before>from __future__ import unicode_literals
import ast
import functools

import six

from babelvueextractor.lexer import (
    Lexer,
    Token,
    TOKEN_VAR,
    TOKEN_CONST,
    TOKEN_RAW_HTML,
    TOKEN_DOUBLE_WAY_BINDING,
    TOKEN_DIRECTIVE
)

TOKENS = [TOKEN_VAR, TOKEN_CONST, TOKEN_RAW_HTML, TOKEN_DOUBLE_WAY_BINDING, TOKEN_DIRECTIVE]

def extract_vue(fileobj, keywords, comment_tags, options):
    """Extract messages from Vue template files.

    :param fileobj: the file-like the messages should be extracted from
    :param keywords: a list of keywords (i.e. function names) that should be recognize as translation functions
    :param comment_tags: a list of translator tags to search for and include in the results
    :param options: a dictionary of additional options (optional)
    :return: an iterator over ``(lineno, funcname, message, comments)``
    :rtype: ``iterator``
    """
    contents = fileobj.read()
    u = six.text_type
    if not isinstance(contents, u):
        contents = u(contents, encoding=options.get('encoding', 'utf-8'))
    lexer = Lexer(contents, None)
    u = functools.partial(u, encoding='utf-8')
    for t in lexer.tokenize():  # type: Token
        if t.token_type in TOKENS:
            try:
                tree = ast.parse(t.contents).body[0]
            except (SyntaxError, IndexError, AttributeError):
                pass
            else:
                for node in ast.walk(tree):  # type: ast.Call
                    if (isinstance(node, ast.Call)
                            and isinstance(node.func, ast.Name)):
                        func_name = node.func.id
                        args = node.args
                        messages = None
                        if func_name == "gettext":
                            if len(args) != 1:
                                raise TypeError("Error at line %s: Function gettext()"
                                                " requires exactly one argument" % t.lineno)
                            if isinstance(args[0], ast.Str):  # ignore other argument types such as variables
                                messages = u(args[0].s)
                        elif func_name == "ngettext":
                            if len(node.args) != 3:
                                raise TypeError("Error at line %s: Function ngettext()"
                                                " requires exactly 3 arguments" % t.lineno)
                            if isinstance(args[0], ast.Str) and isinstance(args[1], ast.Str):
                                messages = u(args[0].s), u(args[1].s)

                        if messages:
                            yield t.lineno, func_name, messages, []
<commit_msg>fix PY3<commit_after>from __future__ import unicode_literals
import ast
import functools

import six

from babelvueextractor.lexer import (
    Lexer,
    Token,
    TOKEN_VAR,
    TOKEN_CONST,
    TOKEN_RAW_HTML,
    TOKEN_DOUBLE_WAY_BINDING,
    TOKEN_DIRECTIVE
)

TOKENS = [TOKEN_VAR, TOKEN_CONST, TOKEN_RAW_HTML, TOKEN_DOUBLE_WAY_BINDING, TOKEN_DIRECTIVE]

def extract_vue(fileobj, keywords, comment_tags, options):
    """Extract messages from Vue template files.

    :param fileobj: the file-like the messages should be extracted from
    :param keywords: a list of keywords (i.e. function names) that should be recognize as translation functions
    :param comment_tags: a list of translator tags to search for and include in the results
    :param options: a dictionary of additional options (optional)
    :return: an iterator over ``(lineno, funcname, message, comments)``
    :rtype: ``iterator``
    """
    contents = fileobj.read()
    u = lambda s: s if isinstance(s, six.text_type) else s.decode(encoding=options.get('encoding', 'utf-8'))
    contents = u(contents)
    lexer = Lexer(contents, None)
    for t in lexer.tokenize():  # type: Token
        if t.token_type in TOKENS:
            try:
                tree = ast.parse(t.contents).body[0]
            except (SyntaxError, IndexError, AttributeError):
                pass
            else:
                for node in ast.walk(tree):  # type: ast.Call
                    if (isinstance(node, ast.Call)
                            and isinstance(node.func, ast.Name)):
                        func_name = node.func.id
                        args = node.args
                        messages = None
                        if func_name == "gettext":
                            if len(args) != 1:
                                raise TypeError("Error at line %s: Function gettext()"
                                                " requires exactly one argument" % t.lineno)
                            if isinstance(args[0], ast.Str):  # ignore other argument types such as variables
                                messages = u(args[0].s)
                        elif func_name == "ngettext":
                            if len(node.args) != 3:
                                raise TypeError("Error at line %s: Function ngettext()"
                                                " requires exactly 3 arguments" % t.lineno)
                            if isinstance(args[0], ast.Str) and isinstance(args[1], ast.Str):
                                messages = u(args[0].s), u(args[1].s)

                        if messages:
                            yield t.lineno, func_name, messages, []
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import json
import hashlib
from flask import current_app
import uuid
from share.kit import kit

def pack_data(json_data):
    """
    :param json_data:
    :return:
    """

    if json_data is None:
        return ""

    data = json.dumps(json_data)
    sign = hashlib.md5('|'.join([current_app.config['SECRET_KEY'], data])).hexdigest()

    return json.dumps(dict(
        data=data,
        sign=sign,
    ))

def unpack_data(body):
    if not body:
        return None

    json_body = json.loads(body)

    data = json_body.get("data")
    sign = json_body.get("sign")

    calc_sign = hashlib.md5('|'.join([current_app.config['SECRET_KEY'], data])).hexdigest()

    if sign != calc_sign:
        return None

    return json.loads(data)

def alloc_autoid(name):
    """
    获取一个自增的id
    :return:
    """

    autoid_table = kit.mongo_client.get_default_database()[current_app.config['MONGO_TB_AUTOID']]

    autoid_info = autoid_table.find_and_modify(
        query=dict(
            name=name,
        ),
        update={
            "$inc": {
                "value": 1,
                }
        },
        upsert=True,
        new=True,  # 返回修改过的
    )

    # print autoid_info
    return autoid_info["value"]

def get_appinfo_by_appkey(appkey):
    """
    获取appinfo
    """

    appinfo_table = kit.mongo_client.get_default_database()[current_app.config['MONGO_TB_APPINFO']]

    return appinfo_table.find_one(dict(
        appkey=appkey
    ))

def get_or_create_user(user_info):
    """
    返回或者创建user
    """

    user_table = kit.mongo_client.get_default_database()[current_app.config['MONGO_TB_USER']]

    user = user_table.find_one(dict(
        appid=user_info['appid'],
        device_id=user_info['device_id'],
    ))

    if user is None:
        new_user_info = dict(
            uid=alloc_autoid("user_appid:%d" % user_info["appid"]),
            key=uuid.uuid4().hex,
        )
        new_user_info.update(user_info)

        # 没有这个用户
        user_table.insert(new_user_info)
    else:
        # 更新字段, 要用 $set 才会只更新指定的字段
        user_table.update({
            '_id': user['_id'],
            }, {
            "$set": user_info,
            })

    return user_table.find_one(dict(
        appid=user_info['appid'],
        device_id=user_info['device_id'],
    ))

<commit_msg>突然想起来gateway将uid作为唯一标识，不能重复<commit_after># -*- coding: utf-8 -*-

import json
import hashlib
from flask import current_app
import uuid
from share.kit import kit

def pack_data(json_data):
    """
    :param json_data:
    :return:
    """

    if json_data is None:
        return ""

    data = json.dumps(json_data)
    sign = hashlib.md5('|'.join([current_app.config['SECRET_KEY'], data])).hexdigest()

    return json.dumps(dict(
        data=data,
        sign=sign,
    ))

def unpack_data(body):
    if not body:
        return None

    json_body = json.loads(body)

    data = json_body.get("data")
    sign = json_body.get("sign")

    calc_sign = hashlib.md5('|'.join([current_app.config['SECRET_KEY'], data])).hexdigest()

    if sign != calc_sign:
        return None

    return json.loads(data)

def alloc_autoid(name):
    """
    获取一个自增的id
    :return:
    """

    autoid_table = kit.mongo_client.get_default_database()[current_app.config['MONGO_TB_AUTOID']]

    autoid_info = autoid_table.find_and_modify(
        query=dict(
            name=name,
        ),
        update={
            "$inc": {
                "value": 1,
                }
        },
        upsert=True,
        new=True,  # 返回修改过的
    )

    # print autoid_info
    return autoid_info["value"]

def get_appinfo_by_appkey(appkey):
    """
    获取appinfo
    """

    appinfo_table = kit.mongo_client.get_default_database()[current_app.config['MONGO_TB_APPINFO']]

    return appinfo_table.find_one(dict(
        appkey=appkey
    ))

def get_or_create_user(user_info):
    """
    返回或者创建user
    """

    user_table = kit.mongo_client.get_default_database()[current_app.config['MONGO_TB_USER']]

    user = user_table.find_one(dict(
        appid=user_info['appid'],
        device_id=user_info['device_id'],
    ))

    if user is None:
        new_user_info = dict(
            uid=alloc_autoid("user"),
            key=uuid.uuid4().hex,
        )
        new_user_info.update(user_info)

        # 没有这个用户
        user_table.insert(new_user_info)
    else:
        # 更新字段, 要用 $set 才会只更新指定的字段
        user_table.update({
            '_id': user['_id'],
            }, {
            "$set": user_info,
            })

    return user_table.find_one(dict(
        appid=user_info['appid'],
        device_id=user_info['device_id'],
    ))

<|endoftext|>


<commit_before>import matplotlib
from matplotlib import pyplot as plt

from ..utils.decorators import wraps

MPL_VERSION = matplotlib.__version__

ROOT = "http://{server}/testing/astropy/2018-02-01T23:31:45.013149/{mpl_version}/"

IMAGE_REFERENCE_DIR = ROOT.format(server='astropy.github.io/astropy-data', mpl_version=MPL_VERSION[:3] + '.x')

def ignore_matplotlibrc(func):
    # This is a decorator for tests that use matplotlib but not pytest-mpl
    # (which already handles rcParams)
    @wraps(func)
    def wrapper(*args, **kwargs):
        with plt.style.context({}, after_reset=True):
            return func(*args, **kwargs)
    return wrapper
<commit_msg>Fix reference URL for images<commit_after>import matplotlib
from matplotlib import pyplot as plt

from ..utils.decorators import wraps

MPL_VERSION = matplotlib.__version__

ROOT = "http://{server}/testing/astropy/2018-02-01T23:31:45.013149/{mpl_version}/"

IMAGE_REFERENCE_DIR = ROOT.format(server='data.astropy.org', mpl_version=MPL_VERSION[:3] + '.x')

def ignore_matplotlibrc(func):
    # This is a decorator for tests that use matplotlib but not pytest-mpl
    # (which already handles rcParams)
    @wraps(func)
    def wrapper(*args, **kwargs):
        with plt.style.context({}, after_reset=True):
            return func(*args, **kwargs)
    return wrapper
<|endoftext|>


<commit_before>"""
Convenient access to spreadsheet-style game data files. CSV is the only
included implementation, but writing new readers is trivial.

Here's a simple example::

  def f_char(val):
    "Ensure *val* is a one-character string"
    if isinstance(val, str) and len(val) == 1:
      return val
    else:
      raise ValueError("Bad character value: {!r}".format(val))

  # Imagine you have a directory of CSVs with monster definitions.
  # One of them, gnomes.csv, might look like this:
  #  ID,            Strength,   Color,    Char
  #  gnome_easy,    1,          #ff0000,  g
  #  gnome_medium,  2,          #ffff00,  g
  #  gnome_hard,    3,          #ff00ff,  g
  #  gnome_huge,    4,          #880000,  G

  # To read them, create a :py:class:`DataStore`, passing a class name for
  # items and an ordered mapping of field name to value:
  monster_types = DataStore('MonsterType', (
    ('id', str),
    ('strength', int), # int('5') = 5
    ('color', str),
    ('char', f_char),  # see helper at top of example
  ))

  # You can load all the CSVs at once, like this:
  monster_types.add_sources_with_glob('data/monsters/*.csv', CSVReader)

  # Now you can access any monster type easily:
  print(monster_types.gnome_easy)
  # > MonsterType(id='gnome_easy', strength=1, color='#ff0000', char='g')
  print(monster_types['gnome_easy'])
  # > same thing

  # And if you edit your data files, you can reload them at runtime to
  # immediately pick up changes:

  monster_types.reload()

Limitations and details:

* The first field in the mapping must be a unique identifier, unique across all
  readers.
* Identifiers may overlap built-in Python method names or start with numbers.
  To access these values by attribute, prefix them with ``r_``. So a value with
  id ``5`` could be accessed by ``data_store.r_5``. Or you could just use
  dict-style lookup, like ``data_store['5']``. To avoid these issues, use
  Python-identifier-legal uppercase string identifiers.
* You may pass default values to the :py:class:`DataStore`. These only matter
  if your data may be missing keys (i.e. if your reader emits ``dict``s), so
  you can just pass ``None`` if you want.
"""

import csv
import glob
from collections import namedtuple
from uuid import uuid4

class Reader:
  """
  Abstract base class for a reader. You can subclass this for your own readers
  if you want, but duck typing is probably less verbose.

  .. py:attribute:: identifier

    String identifier for this reader. Only used to provide helpful
    exception messages.
  """

  def read(self):
    """
    Iterator of sequences or dicts from the file or data structure. Like
    ``('a', 'b', 'c')`` or ``{'f1': 'a', 'f2': 'b', 'f3': 'c'}``.
    """

class CSVReader:
  """
  Returns the lines from a single CSV file. If your first line is just column
  labels, you can skip it (this is the default behavior).

  .. py:attribute:: path

    Path to the file

  .. py:attribute:: identifier

    See :py:attr:`Reader.identifier`

  .. py:attribute:: skip_first_line

    If ``True`` (default), always skip the first line of the file.
  """
  def __init__(self, path, skip_first_line=True):
    """
    """
    self.path = path
    self.skip_first_line = skip_first_line
    self.identifier = path

  def read(self):
    """Iterator of CSV values. Values are lists."""
    with open(self.path) as f:
      reader = csv.reader(f)
      skip_next_line = self.skip_first_line
      for line in reader:
        if skip_next_line:
          skip_next_line = False
          continue
        yield line

def _read_row(row_class, fields, defaults, values):
  if isinstance(values, dict):
    kwargs = {}
    for k, v in fields:
      if k in values:
        kwargs[k] = v(values[k])
      else:
        kwargs[k] = defaults[k]
    return row_class(**kwargs)
  else:
    return row_class(**{
      k: fn(v) for (k, fn), v in zip(fields, values)})

class Source:
  """
  Stores and indexes values from a reader. You shouldn't need to worry about
  this class much; it's created automatically for you, and sources are accessed
  in aggregate via a :py:class:`DataStore`.

  .. py:attribute:: reader

    Some :py:class:`Reader` subclass or duck-typed class.

  .. py:attribute:: row_class

    Class to instantiate for each row

  .. py:attribute:: fields

    Sequence of pairs mapping field name to value factory

  .. py:attribute::  defaults

    Default values for the fields, or ``None`` if you're quite sure all values
    will be specified

  .. py:attribute:: items

    List of items read from the source
  """
  def __init__(self, reader, row_class, fields, defaults=None):
    self.reader = reader
    self.row_class = row_class
    self.fields = fields
    self.defaults = defaults
    self._items_by_key = {}
    self.items = []

  def reload(self):
    """Run the reader again; replace all stored items"""
    self._items_by_key = {}
    self.items = []
    for values in self.reader.read():
      item = _read_row(self.row_class, self.fields, self.defaults, values)
      self.items.append(item)
      if item[0] in self._items_by_key:
        raise ValueError(
          "Duplicate key {!r} in {}".format(item[0], self.reader.identifier))
      self._items_by_key[item[0]] = item

  def keys(self):
    """Iterator of all keys in this source"""
    # strong assumption: first field is unique id
    return [i[0] for i in self.items]

  def __contains__(self, k):
    return k in self._items_by_key

  def __getitem__(self, k):
    return self._items_by_key[k]

class DataStore:
  """
  Collection of data sources for convenient access. See example in module docs
  for a basic guide.
  """
  def __init__(self, type_name, fields, defaults=None):
    """
    :param str type_name: Name of the class to create for values
    :param [(name, factory), ...] fields: Sequence mapping field name to a
                                          function that converts a string to
                                          the field's value
    :param dict|None defaults: Default values for fields, or ``None`` if
                               irrelevant.

    The class creation stuff merits some explanation. The idea is that you'll
    have one data store per type, and having to separately specify the class
    would add unnecessary boilerplate.

    So what you do instead is to pass a class name and a field mapping. This
    is turned into a ``namedtuple``, and each field is run through the mapping
    function when the data is parsed from a file.
    """
    self.row_class = namedtuple(type_name, [f[0] for f in fields])
    self.fields = fields
    self.defaults = defaults
    self.sources = []

  def _add_source(self, reader):
    source = Source(reader, self.row_class, self.fields, self.defaults)
    self.sources.append(source)
    source.reload()

  def add_source(self, reader):
    """
    :param Reader reader:

    Add a source that reads from the given reader.
    """
    self._add_source(reader)
    self.validate()

  def add_sources_with_glob(self, pattern, reader_factory):
    """
    :param str pattern: Glob pattern (`docs <https://docs.python.org/3/library/glob.html>`_)
    :param function reader_factory: ``reader_factory(path) -> Reader``

    Add multiple sources for paths that match the given pattern (glob).
    For example, if you wanted to read all CSV files in your "monsters"
    directory, you'd do this::

      data_store.add_sources_with_glob('data/monsters/*.csv', CSVReader)
    """
    for path in glob.glob(pattern, recursive=True):
      self._add_source(reader_factory(path))
    self.validate()

  def validate(self):
    for key in self.keys():
      sources = [source for source in self.sources if key in source]
      if len(sources) > 1:
          raise ValueError("Duplicate key {!r} in ".format(
            key, [s.reader.identifier for s in sources]))

  def reload(self):
    """
    Reload all sources. Does not pick up new files if you used a glob. To
    handle that case, call :py:meth:`unload` and then re-add all your sources.
    """
    for source in self.sources:
      source.reload()

  def unload(self):
    """
    Remove all sources.
    """
    self.sources = []

  def keys(self):
    """Iterator of all item identifiers"""
    for source in self.sources:
      yield from source.keys()

  @property
  def items(self):
    """Iterator of all items"""
    for source in self.sources:
      yield from source.items

  def __contains__(self, k):
    return any(k in source for source in self.sources)

  def __getitem__(self, k):
    for source in self.sources:
      if k in source:
        return source[k]
    raise KeyError(k)

  def __getattr__(self, k):
    try:
      object.__getattr__(self, k)
    except AttributeError:
      if k.startswith('r_'):
        return self[k[2:]]
      else:
        return self[k]
<commit_msg>Doc fix<commit_after>"""
Convenient access to spreadsheet-style game data files. CSV is the only
included implementation, but writing new readers is trivial.

Here's a simple example::

  def f_char(val):
    "Ensure *val* is a one-character string"
    if isinstance(val, str) and len(val) == 1:
      return val
    else:
      raise ValueError("Bad character value: {!r}".format(val))

  # Imagine you have a directory of CSVs with monster definitions.
  # One of them, gnomes.csv, might look like this:
  #  ID,            Strength,   Color,    Char
  #  gnome_easy,    1,          #ff0000,  g
  #  gnome_medium,  2,          #ffff00,  g
  #  gnome_hard,    3,          #ff00ff,  g
  #  gnome_huge,    4,          #880000,  G

  # To read them, create a :py:class:`DataStore`, passing a class name
  # for items and an ordered mapping of field name to value:
  monster_types = DataStore('MonsterType', (
    ('id', str),
    ('strength', int), # int('5') = 5
    ('color', str),
    ('char', f_char),  # see helper at top of example
  ))

  # You can load all the CSVs at once, like this:
  monster_types.add_sources_with_glob('data/monsters/*.csv', CSVReader)

  # Now you can access any monster type easily:
  print(monster_types.gnome_easy)
  # > MonsterType(id='gnome_easy', strength=1, color='#ff0000', char='g')
  print(monster_types['gnome_easy'])
  # > same thing

  # And if you edit your data files, you can reload them at runtime to
  # immediately pick up changes:

  monster_types.reload()

Limitations and details:

* The first field in the mapping must be a unique identifier, unique across all
  readers.
* Identifiers may overlap built-in Python method names or start with numbers.
  To access these values by attribute, prefix them with ``r_``. So a value with
  id ``5`` could be accessed by ``data_store.r_5``. Or you could just use
  dict-style lookup, like ``data_store['5']``. To avoid these issues, use
  Python-identifier-legal uppercase string identifiers.
* You may pass default values to the :py:class:`DataStore`. These only matter
  if your data may be missing keys (i.e. if your reader emits ``dict``s), so
  you can just pass ``None`` if you want.
"""

import csv
import glob
from collections import namedtuple
from uuid import uuid4

class Reader:
  """
  Abstract base class for a reader. You can subclass this for your own readers
  if you want, but duck typing is probably less verbose.

  .. py:attribute:: identifier

    String identifier for this reader. Only used to provide helpful
    exception messages.
  """

  def read(self):
    """
    Iterator of sequences or dicts from the file or data structure. Like
    ``('a', 'b', 'c')`` or ``{'f1': 'a', 'f2': 'b', 'f3': 'c'}``.
    """

class CSVReader:
  """
  Returns the lines from a single CSV file. If your first line is just column
  labels, you can skip it (this is the default behavior).

  .. py:attribute:: path

    Path to the file

  .. py:attribute:: identifier

    See :py:attr:`Reader.identifier`

  .. py:attribute:: skip_first_line

    If ``True`` (default), always skip the first line of the file.
  """
  def __init__(self, path, skip_first_line=True):
    """
    """
    self.path = path
    self.skip_first_line = skip_first_line
    self.identifier = path

  def read(self):
    """Iterator of CSV values. Values are lists."""
    with open(self.path) as f:
      reader = csv.reader(f)
      skip_next_line = self.skip_first_line
      for line in reader:
        if skip_next_line:
          skip_next_line = False
          continue
        yield line

def _read_row(row_class, fields, defaults, values):
  if isinstance(values, dict):
    kwargs = {}
    for k, v in fields:
      if k in values:
        kwargs[k] = v(values[k])
      else:
        kwargs[k] = defaults[k]
    return row_class(**kwargs)
  else:
    return row_class(**{
      k: fn(v) for (k, fn), v in zip(fields, values)})

class Source:
  """
  Stores and indexes values from a reader. You shouldn't need to worry about
  this class much; it's created automatically for you, and sources are accessed
  in aggregate via a :py:class:`DataStore`.

  .. py:attribute:: reader

    Some :py:class:`Reader` subclass or duck-typed class.

  .. py:attribute:: row_class

    Class to instantiate for each row

  .. py:attribute:: fields

    Sequence of pairs mapping field name to value factory

  .. py:attribute::  defaults

    Default values for the fields, or ``None`` if you're quite sure all values
    will be specified

  .. py:attribute:: items

    List of items read from the source
  """
  def __init__(self, reader, row_class, fields, defaults=None):
    self.reader = reader
    self.row_class = row_class
    self.fields = fields
    self.defaults = defaults
    self._items_by_key = {}
    self.items = []

  def reload(self):
    """Run the reader again; replace all stored items"""
    self._items_by_key = {}
    self.items = []
    for values in self.reader.read():
      item = _read_row(self.row_class, self.fields, self.defaults, values)
      self.items.append(item)
      if item[0] in self._items_by_key:
        raise ValueError(
          "Duplicate key {!r} in {}".format(item[0], self.reader.identifier))
      self._items_by_key[item[0]] = item

  def keys(self):
    """Iterator of all keys in this source"""
    # strong assumption: first field is unique id
    return [i[0] for i in self.items]

  def __contains__(self, k):
    return k in self._items_by_key

  def __getitem__(self, k):
    return self._items_by_key[k]

class DataStore:
  """
  Collection of data sources for convenient access. See example in module docs
  for a basic guide.
  """
  def __init__(self, type_name, fields, defaults=None):
    """
    :param str type_name: Name of the class to create for values
    :param [(name, factory), ...] fields: Sequence mapping field name to a
                                          function that converts a string to
                                          the field's value
    :param dict|None defaults: Default values for fields, or ``None`` if
                               irrelevant.

    The class creation stuff merits some explanation. The idea is that you'll
    have one data store per type, and having to separately specify the class
    would add unnecessary boilerplate.

    So what you do instead is to pass a class name and a field mapping. This
    is turned into a ``namedtuple``, and each field is run through the mapping
    function when the data is parsed from a file.
    """
    self.row_class = namedtuple(type_name, [f[0] for f in fields])
    self.fields = fields
    self.defaults = defaults
    self.sources = []

  def _add_source(self, reader):
    source = Source(reader, self.row_class, self.fields, self.defaults)
    self.sources.append(source)
    source.reload()

  def add_source(self, reader):
    """
    :param Reader reader:

    Add a source that reads from the given reader.
    """
    self._add_source(reader)
    self.validate()

  def add_sources_with_glob(self, pattern, reader_factory):
    """
    :param str pattern: Glob pattern (`docs <https://docs.python.org/3/library/glob.html>`_)
    :param function reader_factory: ``reader_factory(path) -> Reader``

    Add multiple sources for paths that match the given pattern (glob).
    For example, if you wanted to read all CSV files in your "monsters"
    directory, you'd do this::

      data_store.add_sources_with_glob('data/monsters/*.csv', CSVReader)
    """
    for path in glob.glob(pattern, recursive=True):
      self._add_source(reader_factory(path))
    self.validate()

  def validate(self):
    for key in self.keys():
      sources = [source for source in self.sources if key in source]
      if len(sources) > 1:
          raise ValueError("Duplicate key {!r} in ".format(
            key, [s.reader.identifier for s in sources]))

  def reload(self):
    """
    Reload all sources. Does not pick up new files if you used a glob. To
    handle that case, call :py:meth:`unload` and then re-add all your sources.
    """
    for source in self.sources:
      source.reload()

  def unload(self):
    """
    Remove all sources.
    """
    self.sources = []

  def keys(self):
    """Iterator of all item identifiers"""
    for source in self.sources:
      yield from source.keys()

  @property
  def items(self):
    """Iterator of all items"""
    for source in self.sources:
      yield from source.items

  def __contains__(self, k):
    return any(k in source for source in self.sources)

  def __getitem__(self, k):
    for source in self.sources:
      if k in source:
        return source[k]
    raise KeyError(k)

  def __getattr__(self, k):
    try:
      object.__getattr__(self, k)
    except AttributeError:
      if k.startswith('r_'):
        return self[k[2:]]
      else:
        return self[k]
<|endoftext|>


<commit_before>#/usr/bin/env python

import frontend, backend, common
import argparse, os, shutil

def move(src, dst):
  if os.path.exists(src):
    shutil.move(src, dst)

def copytree(src, dst):
  if os.path.exists(src):
    shutil.copytree(src, dst)

def collect_stray_output(project_list, out_dir):
  dljc_out_dir = os.path.join(out_dir, common.DLJC_OUTPUT_DIR)
  common.mkdir(dljc_out_dir)

  jaif_out_dir = os.path.join(out_dir, "jaif")
  common.mkdir(jaif_out_dir)

  move(os.path.join(common.WORKING_DIR, 'class_info.json'),
       os.path.join(out_dir, 'class_info.json'))

  for project in project_list:
    collect_jars(project, out_dir)
    dljc_in_dir = common.get_dljc_dir(project)
    copytree(dljc_in_dir, os.path.join(dljc_out_dir, project))

    move(os.path.join(common.get_project_dir(project), 'default.jaif'),
         os.path.join(jaif_out_dir, "{}.jaif".format(project)))

  move(os.path.join(common.CORPUS_DIR, 'corpus.jaif'),
       os.path.join(jaif_out_dir, 'corpus.jaif'))

def collect_jars(project_name, out_dir):
  jars_dir = os.path.join(out_dir, "jars", project_name)
  common.mkdir(jars_dir)
  project_dir = common.get_project_dir(project_name)
  for path, _, files in os.walk(project_dir):
    if ".mvn" in path:
      continue
    for file in files:
      if file.endswith('.jar'):
        move(os.path.join(path, file),
             os.path.join(jars_dir))

def rotate_log_dir(d):
  i = 1
  dirformat = "{}.{}"

  while os.path.exists(dirformat.format(d, i)):
    i += 1

  move(d, dirformat.format(d, i))

def main():
  project_list = common.get_project_list()

  parser = argparse.ArgumentParser()

  parser.add_argument("-rc", "--recompute_clusters", action="store_true", help="recompute clustering for selected projects")
  parser.add_argument("-c", "--cluster", type=str, help="path to the json file that contains clustering information")
  parser.add_argument("-g", "--graph", action="store_true", help="set to regenerate graphs from the programs")
  parser.add_argument("-d", "--dir", type=str, required=True, help="The output directory")
  parser.add_argument("-p", "--projectset", type=str, help="A project set name to work on")
  args = parser.parse_args()

  if args.projectset:
    project_list = common.get_corpus_set(args.projectset)

  args.dir = os.path.abspath(os.path.join(common.WORKING_DIR, 'results', args.dir))

  if os.path.exists(args.dir):
    rotate_log_dir(args.dir)

  common.mkdir(args.dir)
  common.set_output_dir(args.dir)
  kernel_dir = os.path.join(args.dir, "kernel_directory")
  common.mkdir(kernel_dir)

  backend.run(project_list, args, kernel_dir)
  print("\n********* END OF BACKEND **********\n")
  frontend.run(project_list, args, kernel_dir)

  collect_stray_output(project_list, args.dir)
  check_run(project_list, args.dir)

if __name__ == '__main__':
  main()
<commit_msg>Add missing import to experiment.py<commit_after>#/usr/bin/env python

import argparse, os, shutil
import frontend, backend, common, sanity

def move(src, dst):
  if os.path.exists(src):
    shutil.move(src, dst)

def copytree(src, dst):
  if os.path.exists(src):
    shutil.copytree(src, dst)

def collect_stray_output(project_list, out_dir):
  dljc_out_dir = os.path.join(out_dir, common.DLJC_OUTPUT_DIR)
  common.mkdir(dljc_out_dir)

  jaif_out_dir = os.path.join(out_dir, "jaif")
  common.mkdir(jaif_out_dir)

  move(os.path.join(common.WORKING_DIR, 'class_info.json'),
       os.path.join(out_dir, 'class_info.json'))

  for project in project_list:
    collect_jars(project, out_dir)
    dljc_in_dir = common.get_dljc_dir(project)
    copytree(dljc_in_dir, os.path.join(dljc_out_dir, project))

    move(os.path.join(common.get_project_dir(project), 'default.jaif'),
         os.path.join(jaif_out_dir, "{}.jaif".format(project)))

  move(os.path.join(common.CORPUS_DIR, 'corpus.jaif'),
       os.path.join(jaif_out_dir, 'corpus.jaif'))

def collect_jars(project_name, out_dir):
  jars_dir = os.path.join(out_dir, "jars", project_name)
  common.mkdir(jars_dir)
  project_dir = common.get_project_dir(project_name)
  for path, _, files in os.walk(project_dir):
    if ".mvn" in path:
      continue
    for file in files:
      if file.endswith('.jar'):
        move(os.path.join(path, file),
             os.path.join(jars_dir))

def rotate_log_dir(d):
  i = 1
  dirformat = "{}.{}"

  while os.path.exists(dirformat.format(d, i)):
    i += 1

  move(d, dirformat.format(d, i))

def main():
  project_list = common.get_project_list()

  parser = argparse.ArgumentParser()

  parser.add_argument("-rc", "--recompute_clusters", action="store_true", help="recompute clustering for selected projects")
  parser.add_argument("-c", "--cluster", type=str, help="path to the json file that contains clustering information")
  parser.add_argument("-g", "--graph", action="store_true", help="set to regenerate graphs from the programs")
  parser.add_argument("-d", "--dir", type=str, required=True, help="The output directory")
  parser.add_argument("-p", "--projectset", type=str, help="A project set name to work on")
  args = parser.parse_args()

  if args.projectset:
    project_list = common.get_corpus_set(args.projectset)

  args.dir = os.path.abspath(os.path.join(common.WORKING_DIR, 'results', args.dir))

  if os.path.exists(args.dir):
    rotate_log_dir(args.dir)

  common.mkdir(args.dir)
  common.set_output_dir(args.dir)
  kernel_dir = os.path.join(args.dir, "kernel_directory")
  common.mkdir(kernel_dir)

  backend.run(project_list, args, kernel_dir)
  print("\n********* END OF BACKEND **********\n")
  frontend.run(project_list, args, kernel_dir)

  collect_stray_output(project_list, args.dir)
  sanity.check_run(project_list, args.dir)

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>"""Test functions in markdown utils"""

import unittest
from unittest import mock

from wqflask.markdown_routes import render_markdown

class MockRequests404:
    @property
    def status_code():
        return 404

class MockRequests200:
    @property
    def status_code():
        return 200

    @property
    def content():
        return """
        # Glossary

        This is some content

        ## Sub-heading
        This is another sub-heading
        """

class TestMarkdownRoutesFunctions(unittest.TestCase):
    """Test cases for functions in markdown_routes"""

    @mock.patch('wqflask.markdown_routes.requests.get')
    def test_render_markdown(self, requests_mock):
        requests_mock.return_value = MockRequests404
        markdown_content = render_markdown("glossary.md")
        requests_mock.assert_called_with(
            "https://raw.githubusercontent.com"
            "/genenetwork/genenetwork2/"
            "wqflask/wqflask/static/"
            "glossary.md")
        self.assertEqual("
Content
\n",
                         markdown_content)
<commit_msg>Fix false-positive tests<commit_after>"""Test functions in markdown utils"""

import unittest
from unittest import mock

from wqflask.markdown_routes import render_markdown

class MockRequests404:
    @property
    def status_code(self):
        return 404

class MockRequests200:
    @property
    def status_code(self):
        return 200

    @property
    def content(self):
        return b"""
# Glossary
This is some content

## Sub-heading
This is another sub-heading
        """

class TestMarkdownRoutesFunctions(unittest.TestCase):
    """Test cases for functions in markdown_routes"""

    @mock.patch('wqflask.markdown_routes.requests.get')
    def test_render_markdown_when_fetching_locally(self, requests_mock):
        requests_mock.return_value = MockRequests404()
        markdown_content = render_markdown("glossary.md")
        requests_mock.assert_called_with(
            "https://raw.githubusercontent.com"
            "/genenetwork/genenetwork2/"
            "wqflask/wqflask/static/"
            "glossary.md")
        self.assertEqual("
Content
\n",
                         markdown_content)

    @mock.patch('wqflask.markdown_routes.requests.get')
    def test_render_markdown_when_fetching_remotely(self, requests_mock):
        requests_mock.return_value = MockRequests200()
        markdown_content = render_markdown("glossary.md")
        requests_mock.assert_called_with(
            "https://raw.githubusercontent.com"
            "/genenetwork/genenetwork2/"
            "wqflask/wqflask/static/"
            "glossary.md")
        self.assertEqual("""
Glossary

This is some content

Sub-heading

This is another sub-heading

""",
                         markdown_content)
<|endoftext|>


<commit_before># Copyright (c) Charl P. Botha, TU Delft.
# All rights reserved.
# See COPYRIGHT for details.

import ConfigParser
from ConfigParser import NoOptionError
import config
import getopt
import os
import sys
import utils

def usage():
    message = """
Welcome to johannes, the ugliest
downloading/unpacking/configuring/building and installation system of
them all.  It could save you a lot of time though.  This instance of
johannes will get, build and install the following: python, numpy,
wxpython, matplotlib, cmake, dcmtk, vtk, vtktudoss, vtkdevide, itk,
itktudoss, itkvtkglue, devide

Please read the included README.txt file NOW.

Build method A (the default) is as follows: Before starting
johannes.py, first run bootstrap_stage1.sh and bootstrap_stage2.sh to
download and install python.  After that, run johannes as follows:

/you/new/python johannes.py -w working_directory

Options are as follows:
-w, --working-dir      : specify working directory [REQUIRED]
-h, --help             : show this help
-m, --mode             : working mode, 'everything' (default),
                         'clean_build', 'get_only' or 'configure_only'
-p, --install-packages : specify comma-separated list of packages to work on,
                         default all.  Example: -p "CMake,CableSwig"
                         Correct capitalisation IS important!
-d, --auto-deps        : Automatically build per install package dependencies.
                         The default is not to do this, i.e. you have to
                         specify all required packages on the command-line
                         or in the johannes.cfg project file.
--no-win-prereq        : do NOT do Windows prerequisites check.
-v, --versions         : display installed versions of all packages.

You can also specify project-specific options (for example specifying
install packages) in a johannes.cfg file placed at the top-level
of your working directory.  See example-johannes.cfg for more info.

All of this ugliness is copyright 2006-2011 Charl P. Botha http://cpbotha.net/
and is hereby put under a BSD license.
"""

    print message

def posix_prereq_check(working_dir):
    """Perform posix system check for prerequisite software.

    Largest part of this checking is done in the second bootstrap
    shell script (executed before this file).  Here we check for basic
    stuff like cvs, svn and patch.
    """

    v = utils.find_command_with_ver(
            'CVS', '%s -v' % (config.CVS,),
            '\(CVS\)\s+(.*)\s+')

    v = v and utils.find_command_with_ver(
            'Subversion (SVN)', '%s --version' % (config.SVN,),
            'version\s+(.*)$')

    v = v and utils.find_command_with_ver(
            'patch', '%s -v' % (config.PATCH,),
            '^patch\s+(.*)$')

    # now check that working_dir contains the required subdirs
    dv = True
    for wsub in ['archive', 'build', 'inst']:
        cdir = os.path.join(working_dir, wsub)
        if os.path.isdir(cdir):
            msg = '%s exists.' % (cdir,)
        else:
            msg = '%s does not exist.' % (cdir,)
            dv = False

        utils.output(msg)

    return v and dv

def windows_prereq_check(working_dir):
    """Perform Windows system check for prerequisite software and
    directory structure.
    """

    utils.output("Windows prerequisites check", 70, '#')

    v = utils.find_command_with_ver(
            'MS Visual Studio', '%s /?' % (config.DEVENV,),
            'Visual Studio Version (.*)\.$')

    #v = v and utils.find_command_with_ver(
    #        'CMake', '%s --version' % (config.CMAKE_BINPATH,),
    #        '^cmake version\s+(.*)$')

    v = v and utils.find_command_with_ver(
            'CVS', '%s -v' % (config.CVS,),
            '\((CVS|CVSNT)\)\s+(.*)\s+')

    v = v and utils.find_command_with_ver(
            'Subversion (SVN)', '%s --version' % (config.SVN,),
            'version\s+(.*)$')

    v = v and utils.find_command_with_ver(
            'patch', '%s -v' % (config.PATCH,),
            '^patch\s+(.*)$')

    # now check that setuptools is NOT installed (it screws up
    # everything on Windows)
    try:
        import setuptools
    except ImportError:
        # this is what we want
        utils.output(
                'setuptools not found. Good!')
        sut_v = True
    else:
        utils.output(
                """setuptools is installed.
                This will break the complete DeVIDE build.
                Please uninstall by doing:
                \Python25\Scripts\easy_install -m setuptools
                del \Python25\Lib\site-packages\setuptools*.*
                You can reinstall later by using ez_setup.py again.
                """)
        sut_v = False

    # now check that working_dir contains the required subdirs
    dv = True
    for wsub in ['archive', 'build', 'inst']:
        cdir = os.path.join(working_dir, wsub)
        if os.path.isdir(cdir):
            msg = '%s exists.' % (cdir,)
        else:
            msg = '%s does not exist.' % (cdir,)
            dv = False

        utils.output(msg)

    return v and sut_v and dv

def main():

    if len(sys.argv) < 2:
        usage()

    else:
        rpad = 60
        rpad_char = '+'

        # this is the default list of install packages
        #
        # you can override this by:
        # - specifying packages on the johannes command line
        # - specifying packages in the working dir johannes.py
        # (command line has preference over config file)
        #
        # capitalisation has to match the capitalisation of your
        # install package class, name of install package module is
        # exactly that, but all lower case, so e.g. MyModule will
        # become: install_packages.ip_mymodule.MyModule()
        #
        # johannes will:
        # - attempt to import the ip_name from install_packages
        # - instantiate ip_name.Name
        #
        ip_names = [
                'NumPy',
                'WXPython',
                'matplotlib',
                'CMake',
                'DCMTK',
                'VTK',
                'IPython',
                'VTKTUDOSS',
                'VTKDEVIDE',
                'ITK',
                'SWIG',
                'CableSwig',
                'WrapITK',
                'ItkVtkGlue',
                'itkPyBuffer',
                'ITKTUDOSS',
                'GDCM',
                'DeVIDE',
                'SetupEnvironment',
                ]

        try:
            optlist, args = getopt.getopt(
                sys.argv[1:], 'hm:p:dw:v',
                ['help', 'mode=', 'install-packages=',
                    'auto-deps',
                    'working-dir=',
                    'no-prereq-check', 'versions'])

        except getopt.GetoptError,e:
            usage()
            return

        mode = 'everything'
        #ip_names = None
        working_dir = None
        profile = 'default'
        no_prereq_check = False
        ip_names_cli = False
        auto_deps = False

        for o, a in optlist:
            if o in ('-h', '--help'):
                usage()
                return

            elif o in ('-m', '--mode'):
                if a in ('clean', 'clean_build'):
                    mode = 'clean_build'
                elif a in ['get_only', 'unpack_only',
                        'configure_only', 'rebuild', 'reinstall']:
                    mode = a

            elif o in ('--install-packages'):
                # list of package name to perform the action on
                ip_names = [i.strip() for i in a.split(',')]
                # remember that the user has specified ip_names on the command-line
                ip_names_cli = True

            elif o in ('-d', '--auto-deps'):
                auto_deps = True

            elif o in ('-w', '--working-dir'):
                working_dir = a

            elif o in ('--profile'):
                profile = a

            elif o in ('--no-prereq-check'):
                no_prereq_check = True

            elif o in ('-v', '--versions'):
                mode = 'show_versions'

        # we need at LEAST a working directory
        if not working_dir:
            usage()
            return

        # init config (DURR)
        config.init(working_dir, profile)

        # set some variables we'll need to check later depending on
        # the configuration
        ip_dirs = []
        # now try to read johannes config file from the working dir
        cp = ConfigParser.ConfigParser()
        # returns list of filenames successfully parsed
        cfgfns = cp.read(os.path.join(working_dir, 'johannes.cfg'))
        if cfgfns and cp.has_section('default'):
            if not ip_names_cli:
                # first packages that need to be installed
                # we only do this if the user has NOT specified install
                # packages on the command line.
                try:
                    ip_names = [i.strip()
                            for i in cp.get(
                                'default', 'packages').split(',')]
                except NoOptionError:
                    pass

            # also try to read extra install package paths
            # this is also a comma separated list
            try:
                ip_dirs = [i.strip()
                        for i in cp.get(
                            'default', 'ip_dirs').split(',')]
            except NoOptionError:
                pass

        # if user is asking for versions, we don't do the
        # prerequisites check as we're not going to build anything
        if mode == 'show_versions':
            no_prereq_check = True

        if os.name == 'nt' and not no_prereq_check:
            if not windows_prereq_check(working_dir):
                utils.output(
                     'Windows prerequisites do not check out.  '
                     'Fix and try again.', 70, '-')
                return
            else:
                utils.output(
                        'Windows prerequisites all good.', 70, '-')

        elif os.name == 'posix' and not no_prereq_check:
            if not posix_prereq_check(working_dir):
                utils.output(
                     'Posix prerequisites do not check out.  '
                     'Fix and try again.', 70, '-')
                return
            else:
                utils.output(
                        'Posix prerequisites all good.', 70, '-')

        # we're going to do some imports, so let's set the sys.path
        # correctly.
        # 1. first the default install packages dir config.ip_dir
        sys.path.insert(0, config.ip_dir)

        # 2. insert the extra specified paths BEFORE that, so they get
        # preference
        for uip_dir in ip_dirs:
            sys.path.insert(0, uip_dir)

        # now import only the specified packages
        ip_instance_list = []
        imported_names = []

        def import_ip(ip_name):
            # don't import more than once
            if ip_name in imported_names:
                return

            # turn Name into ip_name
            ip_name_l = 'ip_' + ip_name.lower()
            # import the module, but don't instantiate the ip class yet
            ip_m = __import__(ip_name_l)

            # import dependencies first if user has specified
            # auto-deps.
            if auto_deps:
                for dep in ip_m.dependencies:
                    import_ip(dep)

            # instantiate ip_name.Name
            ip = getattr(ip_m, ip_name)()
            ip_instance_list.append(ip)
            imported_names.append(ip_name)
            print "%s from %s." % \
                    (ip_name, ip_m.__file__)

        # import all ip_names, including dependencies
        for ip_name in ip_names:
            import_ip(ip_name)

        def get_stage(ip, n):
            utils.output("%s :: get()" % (n,), rpad, rpad_char)
            ip.get()

        def unpack_stage(ip, n):
            utils.output("%s :: unpack()" % (n,), rpad, rpad_char)
            ip.unpack()

        def configure_stage(ip, n):
            utils.output("%s :: configure()" % (n,), rpad, rpad_char)
            ip.configure()

        def build_stage(ip, n):
            utils.output("%s :: build()" % (n,), rpad, rpad_char)
            ip.build()

        def install_stage(ip, n):
            utils.output("%s :: install()" % (n,), rpad, rpad_char)
            ip.install()

        def all_stages(ip, n):
            get_stage(ip, n)

            unpack_stage(ip, n)

            configure_stage(ip, n)

            build_stage(ip, n)

            install_stage(ip, n)

        if mode == 'show_versions':
            utils.output('Extracting all install_package versions.')
            print "python: %d.%d.%d (%s)" % \
                    (sys.version_info[0:3] +
                            (config.PYTHON_EXECUTABLE,))

        for ip in ip_instance_list:
            n = ip.__class__.__name__

            if not n in ip_names:
                # n is a dependency, so do everything
                utils.output("%s (DEPENDENCY)" % (n,), 70, '#')
                all_stages(ip, n)

            elif mode == 'get_only':
                utils.output("%s GET_ONLY" % (n,), 70, '#')
                utils.output("%s" % (n,), 70, '#')
                get_stage(ip, n)

            elif mode == 'unpack_only':
                utils.output("%s UNPACK_ONLY" % (n,), 70, '#')
                utils.output("%s" % (n,), 70, '#')
                unpack_stage(ip, n)

            elif mode == 'configure_only':
                utils.output("%s CONFIGURE_ONLY" % (n,), 70, '#')
                utils.output("%s" % (n,), 70, '#')
                configure_stage(ip, n)

            elif mode == 'everything':
                utils.output("%s" % (n,), 70, '#')
                all_stages(ip, n)

            elif mode == 'clean_build':
                utils.output("%s CLEAN_BUILD" % (n,), 70, '#')
                ip.clean_build()

            elif mode == 'show_versions':
                print '%s: %s' % (n, ip.get_installed_version())

            elif mode == 'rebuild':
                utils.output("%s REBUILD" % (n,), 70, '#')
                # clean up
                ip.clean_build()
                # rebuild (executes all stages, as previous
                # stages are required and user will likely
                # need an install also)
                all_stages(ip, n)

            elif mode == 'reinstall':
                utils.output("%s REINSTALL" % (n,), 70, '#')
                # clean up
                ip.clean_install()
                # reinstall
                all_stages(ip, n)

        if mode != 'show_versions':
            utils.output("Now please read the RESULTS section of README.txt!")

if __name__ == '__main__':
    main()
<commit_msg>Implemented custom modes. Any method name in an install package can now be specified as the mode.<commit_after># Copyright (c) Charl P. Botha, TU Delft.
# All rights reserved.
# See COPYRIGHT for details.

import ConfigParser
from ConfigParser import NoOptionError
import config
import getopt
import os
import sys
import utils

def usage():
    message = """
Welcome to johannes, the ugliest
downloading/unpacking/configuring/building and installation system of
them all.  It could save you a lot of time though.  This instance of
johannes will get, build and install the following: python, numpy,
wxpython, matplotlib, cmake, dcmtk, vtk, vtktudoss, vtkdevide, itk,
itktudoss, itkvtkglue, devide

Please read the included README.txt file NOW.

Build method A (the default) is as follows: Before starting
johannes.py, first run bootstrap_stage1.sh and bootstrap_stage2.sh to
download and install python.  After that, run johannes as follows:

/you/new/python johannes.py -w working_directory

Options are as follows:
-w, --working-dir      : specify working directory [REQUIRED]
-h, --help             : show this help
-m, --mode             : working mode, 'everything' (default),
                         'clean_build', 'get_only' or 'configure_only'
-p, --install-packages : specify comma-separated list of packages to work on,
                         default all.  Example: -p "CMake,CableSwig"
                         Correct capitalisation IS important!
-d, --auto-deps        : Automatically build per install package dependencies.
                         The default is not to do this, i.e. you have to
                         specify all required packages on the command-line
                         or in the johannes.cfg project file.
--no-win-prereq        : do NOT do Windows prerequisites check.
-v, --versions         : display installed versions of all packages.

You can also specify project-specific options (for example specifying
install packages) in a johannes.cfg file placed at the top-level
of your working directory.  See example-johannes.cfg for more info.

All of this ugliness is copyright 2006-2011 Charl P. Botha http://cpbotha.net/
and is hereby put under a BSD license.
"""

    print message

def posix_prereq_check(working_dir):
    """Perform posix system check for prerequisite software.

    Largest part of this checking is done in the second bootstrap
    shell script (executed before this file).  Here we check for basic
    stuff like cvs, svn and patch.
    """

    v = utils.find_command_with_ver(
            'CVS', '%s -v' % (config.CVS,),
            '\(CVS\)\s+(.*)\s+')

    v = v and utils.find_command_with_ver(
            'Subversion (SVN)', '%s --version' % (config.SVN,),
            'version\s+(.*)$')

    v = v and utils.find_command_with_ver(
            'patch', '%s -v' % (config.PATCH,),
            '^patch\s+(.*)$')

    # now check that working_dir contains the required subdirs
    dv = True
    for wsub in ['archive', 'build', 'inst']:
        cdir = os.path.join(working_dir, wsub)
        if os.path.isdir(cdir):
            msg = '%s exists.' % (cdir,)
        else:
            msg = '%s does not exist.' % (cdir,)
            dv = False

        utils.output(msg)

    return v and dv

def windows_prereq_check(working_dir):
    """Perform Windows system check for prerequisite software and
    directory structure.
    """

    utils.output("Windows prerequisites check", 70, '#')

    v = utils.find_command_with_ver(
            'MS Visual Studio', '%s /?' % (config.DEVENV,),
            'Visual Studio Version (.*)\.$')

    #v = v and utils.find_command_with_ver(
    #        'CMake', '%s --version' % (config.CMAKE_BINPATH,),
    #        '^cmake version\s+(.*)$')

    v = v and utils.find_command_with_ver(
            'CVS', '%s -v' % (config.CVS,),
            '\((CVS|CVSNT)\)\s+(.*)\s+')

    v = v and utils.find_command_with_ver(
            'Subversion (SVN)', '%s --version' % (config.SVN,),
            'version\s+(.*)$')

    v = v and utils.find_command_with_ver(
            'patch', '%s -v' % (config.PATCH,),
            '^patch\s+(.*)$')

    # now check that setuptools is NOT installed (it screws up
    # everything on Windows)
    try:
        import setuptools
    except ImportError:
        # this is what we want
        utils.output(
                'setuptools not found. Good!')
        sut_v = True
    else:
        utils.output(
                """setuptools is installed.
                This will break the complete DeVIDE build.
                Please uninstall by doing:
                \Python25\Scripts\easy_install -m setuptools
                del \Python25\Lib\site-packages\setuptools*.*
                You can reinstall later by using ez_setup.py again.
                """)
        sut_v = False

    # now check that working_dir contains the required subdirs
    dv = True
    for wsub in ['archive', 'build', 'inst']:
        cdir = os.path.join(working_dir, wsub)
        if os.path.isdir(cdir):
            msg = '%s exists.' % (cdir,)
        else:
            msg = '%s does not exist.' % (cdir,)
            dv = False

        utils.output(msg)

    return v and sut_v and dv

def main():

    if len(sys.argv) < 2:
        usage()

    else:
        rpad = 60
        rpad_char = '+'

        # this is the default list of install packages
        #
        # you can override this by:
        # - specifying packages on the johannes command line
        # - specifying packages in the working dir johannes.py
        # (command line has preference over config file)
        #
        # capitalisation has to match the capitalisation of your
        # install package class, name of install package module is
        # exactly that, but all lower case, so e.g. MyModule will
        # become: install_packages.ip_mymodule.MyModule()
        #
        # johannes will:
        # - attempt to import the ip_name from install_packages
        # - instantiate ip_name.Name
        #
        ip_names = [
                'NumPy',
                'WXPython',
                'matplotlib',
                'CMake',
                'DCMTK',
                'VTK',
                'IPython',
                'VTKTUDOSS',
                'VTKDEVIDE',
                'ITK',
                'SWIG',
                'CableSwig',
                'WrapITK',
                'ItkVtkGlue',
                'itkPyBuffer',
                'ITKTUDOSS',
                'GDCM',
                'DeVIDE',
                'SetupEnvironment',
                ]

        try:
            optlist, args = getopt.getopt(
                sys.argv[1:], 'hm:p:dw:v',
                ['help', 'mode=', 'install-packages=',
                    'auto-deps',
                    'working-dir=',
                    'no-prereq-check', 'versions'])

        except getopt.GetoptError,e:
            usage()
            return

        mode = 'everything'
        #ip_names = None
        working_dir = None
        profile = 'default'
        no_prereq_check = False
        ip_names_cli = False
        auto_deps = False

        for o, a in optlist:
            if o in ('-h', '--help'):
                usage()
                return

            elif o in ('-m', '--mode'):
                if a in ('clean', 'clean_build'):
                    mode = 'clean_build'
                else:
                    mode = a

            elif o in ('--install-packages'):
                # list of package name to perform the action on
                ip_names = [i.strip() for i in a.split(',')]
                # remember that the user has specified ip_names on the command-line
                ip_names_cli = True

            elif o in ('-d', '--auto-deps'):
                auto_deps = True

            elif o in ('-w', '--working-dir'):
                working_dir = a

            elif o in ('--profile'):
                profile = a

            elif o in ('--no-prereq-check'):
                no_prereq_check = True

            elif o in ('-v', '--versions'):
                mode = 'show_versions'

        # we need at LEAST a working directory
        if not working_dir:
            usage()
            return

        # init config (DURR)
        config.init(working_dir, profile)

        # set some variables we'll need to check later depending on
        # the configuration
        ip_dirs = []
        # now try to read johannes config file from the working dir
        cp = ConfigParser.ConfigParser()
        # returns list of filenames successfully parsed
        cfgfns = cp.read(os.path.join(working_dir, 'johannes.cfg'))
        if cfgfns and cp.has_section('default'):
            if not ip_names_cli:
                # first packages that need to be installed
                # we only do this if the user has NOT specified install
                # packages on the command line.
                try:
                    ip_names = [i.strip()
                            for i in cp.get(
                                'default', 'packages').split(',')]
                except NoOptionError:
                    pass

            # also try to read extra install package paths
            # this is also a comma separated list
            try:
                ip_dirs = [i.strip()
                        for i in cp.get(
                            'default', 'ip_dirs').split(',')]
            except NoOptionError:
                pass

        # if user is asking for versions, we don't do the
        # prerequisites check as we're not going to build anything
        if mode == 'show_versions':
            no_prereq_check = True

        if os.name == 'nt' and not no_prereq_check:
            if not windows_prereq_check(working_dir):
                utils.output(
                     'Windows prerequisites do not check out.  '
                     'Fix and try again.', 70, '-')
                return
            else:
                utils.output(
                        'Windows prerequisites all good.', 70, '-')

        elif os.name == 'posix' and not no_prereq_check:
            if not posix_prereq_check(working_dir):
                utils.output(
                     'Posix prerequisites do not check out.  '
                     'Fix and try again.', 70, '-')
                return
            else:
                utils.output(
                        'Posix prerequisites all good.', 70, '-')

        # we're going to do some imports, so let's set the sys.path
        # correctly.
        # 1. first the default install packages dir config.ip_dir
        sys.path.insert(0, config.ip_dir)

        # 2. insert the extra specified paths BEFORE that, so they get
        # preference
        for uip_dir in ip_dirs:
            sys.path.insert(0, uip_dir)

        # now import only the specified packages
        ip_instance_list = []
        imported_names = []

        def import_ip(ip_name):
            # don't import more than once
            if ip_name in imported_names:
                return

            # turn Name into ip_name
            ip_name_l = 'ip_' + ip_name.lower()
            # import the module, but don't instantiate the ip class yet
            ip_m = __import__(ip_name_l)

            # import dependencies first if user has specified
            # auto-deps.
            if auto_deps:
                for dep in ip_m.dependencies:
                    import_ip(dep)

            # instantiate ip_name.Name
            ip = getattr(ip_m, ip_name)()
            ip_instance_list.append(ip)
            imported_names.append(ip_name)
            print "%s from %s." % \
                    (ip_name, ip_m.__file__)

        # import all ip_names, including dependencies
        for ip_name in ip_names:
            import_ip(ip_name)

        def get_stage(ip, n):
            utils.output("%s :: get()" % (n,), rpad, rpad_char)
            ip.get()

        def unpack_stage(ip, n):
            utils.output("%s :: unpack()" % (n,), rpad, rpad_char)
            ip.unpack()

        def configure_stage(ip, n):
            utils.output("%s :: configure()" % (n,), rpad, rpad_char)
            ip.configure()

        def build_stage(ip, n):
            utils.output("%s :: build()" % (n,), rpad, rpad_char)
            ip.build()

        def install_stage(ip, n):
            utils.output("%s :: install()" % (n,), rpad, rpad_char)
            ip.install()

        def all_stages(ip, n):
            get_stage(ip, n)

            unpack_stage(ip, n)

            configure_stage(ip, n)

            build_stage(ip, n)

            install_stage(ip, n)

        if mode == 'show_versions':
            utils.output('Extracting all install_package versions.')
            print "python: %d.%d.%d (%s)" % \
                    (sys.version_info[0:3] +
                            (config.PYTHON_EXECUTABLE,))

        for ip in ip_instance_list:
            n = ip.__class__.__name__

            if not n in ip_names:
                # n is a dependency, so do everything
                utils.output("%s (DEPENDENCY)" % (n,), 70, '#')
                all_stages(ip, n)

            elif mode == 'get_only':
                utils.output("%s GET_ONLY" % (n,), 70, '#')
                utils.output("%s" % (n,), 70, '#')
                get_stage(ip, n)

            elif mode == 'unpack_only':
                utils.output("%s UNPACK_ONLY" % (n,), 70, '#')
                utils.output("%s" % (n,), 70, '#')
                unpack_stage(ip, n)

            elif mode == 'configure_only':
                utils.output("%s CONFIGURE_ONLY" % (n,), 70, '#')
                utils.output("%s" % (n,), 70, '#')
                configure_stage(ip, n)

            elif mode == 'everything':
                utils.output("%s" % (n,), 70, '#')
                all_stages(ip, n)

            elif mode == 'clean_build':
                utils.output("%s CLEAN_BUILD" % (n,), 70, '#')
                ip.clean_build()

            elif mode == 'show_versions':
                print '%s: %s' % (n, ip.get_installed_version())

            elif mode == 'rebuild':
                utils.output("%s REBUILD" % (n,), 70, '#')
                # clean up
                ip.clean_build()
                # rebuild (executes all stages, as previous
                # stages are required and user will likely
                # need an install also)
                all_stages(ip, n)

            elif mode == 'reinstall':
                utils.output("%s REINSTALL" % (n,), 70, '#')
                # clean up
                ip.clean_install()
                # reinstall
                all_stages(ip, n)

            else:
                utils.output("%s CUSTOM MODE" % (n,), 70, '#')
                if hasattr(ip, mode):
                    utils.output("%s :: %s()" % (n, mode), rpad, rpad_char)
                    getattr(ip, mode)()
                else:
                    utils.error("Mode not found: %s" % (mode,))

        if mode != 'show_versions':
            utils.output("Now please read the RESULTS section of README.txt!")

if __name__ == '__main__':
    main()
<|endoftext|>