<commit_before>import json
from django.db.models import Count
from django.http import HttpResponse
import analysis
from django.core.cache import cache

def get_entities_categories_ranking(flush_cache=False):
    cache_name = 'entities_categories_ranking'

    entities = cache.get(cache_name)
    if entities is None or flush_cache:
        entities = analysis.get_ranking()
        entities = entities.annotate(contracts_number=Count('contracts_made__id'))
        entities = list(entities)
        cache.set(cache_name, entities, 60*60*48)

    return entities

def entities_category_ranking_json(request):
    data = []
    count = 0
    for entity in get_entities_categories_ranking():
        count += 1
        name = entity.name.split(' ')[2:]
        name = ' '.join(name)
        data.append({'name': name, 'rank': count, 'avg_depth': entity.avg_depth})

    return HttpResponse(json.dumps(data), content_type="application/json")

def entities_category_ranking_histogram_json(request):

    entities = get_entities_categories_ranking()

    min_value = entities[-1].avg_depth - 0.00000001  # avoid rounding, this caused a bug before.
    max_value = entities[0].avg_depth + 0.00000001   # avoid rounding, this caused a bug before.
    n_bins = 20

    # center between max and min: min_value + (max_value - min_value)*(count)/n_bins + (max_value - min_value)/n_bins/2

    # create the histogram
    data = [{'bin': x,
             'value': 0,
             'min_position': min_value + (max_value - min_value)*x/n_bins,
             'max_position': min_value + (max_value - min_value)*(x+1)/n_bins
            } for x in range(n_bins)]

    for entity in entities:
        for x in range(n_bins):
            if data[x]['min_position'] < entity.avg_depth <= data[x]['max_position']:
                data[x]['value'] += 1
                break

    return HttpResponse(json.dumps(data), content_type="application/json")

def get_contracts_price_distribution(flush_cache=False):
    cache_name = 'contracts_price_distribution'

    distribution = cache.get(cache_name)
    if distribution is None or flush_cache:
        distribution = analysis.get_price_histogram()
        cache.set(cache_name, distribution, 60*60*24)

    return distribution

def contracts_price_histogram_json(request):

    distribution = get_contracts_price_distribution()

    data = []
    for entry in distribution:
        if entry[1]:
            data.append({'min_position': entry[0],
                         'max_position': entry[0]*2,
                         'value': entry[1]})

    return HttpResponse(json.dumps(data), content_type="application/json")

def get_contracts_macro_statistics(flush_cache=False):
    cache_name = 'contract_prices'

    values = cache.get(cache_name)
    if values is None or flush_cache:
        values = analysis.get_contracts_macro_statistics()
        cache.set(cache_name, values, 60*60*48)

    return values

def contracts_macro_statistics_json(request):
    data = get_contracts_macro_statistics()
    return HttpResponse(json.dumps(data), content_type="application/json")
<commit_msg>Added viewers and caching for new analysis.<commit_after>import json
from django.db.models import Count
from django.http import HttpResponse
import analysis
from django.core.cache import cache

def get_entities_categories_ranking(flush_cache=False):
    cache_name = 'entities_categories_ranking'

    entities = cache.get(cache_name)
    if entities is None or flush_cache:
        entities = analysis.get_ranking()
        entities = entities.annotate(contracts_number=Count('contracts_made__id'))
        entities = list(entities)
        cache.set(cache_name, entities, 60*60*48)

    return entities

def entities_category_ranking_json(request):
    data = []
    count = 0
    for entity in get_entities_categories_ranking():
        count += 1
        name = entity.name.split(' ')[2:]
        name = ' '.join(name)
        data.append({'name': name, 'rank': count, 'avg_depth': entity.avg_depth})

    return HttpResponse(json.dumps(data), content_type="application/json")

def entities_category_ranking_histogram_json(request):

    entities = get_entities_categories_ranking()

    min_value = entities[-1].avg_depth - 0.00000001  # avoid rounding, this caused a bug before.
    max_value = entities[0].avg_depth + 0.00000001   # avoid rounding, this caused a bug before.
    n_bins = 20

    # center between max and min: min_value + (max_value - min_value)*(count)/n_bins + (max_value - min_value)/n_bins/2

    # create the histogram
    data = [{'bin': x,
             'value': 0,
             'min_position': min_value + (max_value - min_value)*x/n_bins,
             'max_position': min_value + (max_value - min_value)*(x+1)/n_bins
            } for x in range(n_bins)]

    for entity in entities:
        for x in range(n_bins):
            if data[x]['min_position'] < entity.avg_depth <= data[x]['max_position']:
                data[x]['value'] += 1
                break

    return HttpResponse(json.dumps(data), content_type="application/json")

def get_contracts_price_distribution(flush_cache=False):
    cache_name = 'contracts_price_distribution'

    distribution = cache.get(cache_name)
    if distribution is None or flush_cache:
        distribution = analysis.get_price_histogram()
        cache.set(cache_name, distribution, 60*60*24)

    return distribution

def contracts_price_histogram_json(request):

    distribution = get_contracts_price_distribution()

    data = []
    for entry in distribution:
        if entry[1]:
            data.append({'min_position': entry[0],
                         'max_position': entry[0]*2,
                         'value': entry[1]})

    return HttpResponse(json.dumps(data), content_type="application/json")

def get_contracts_macro_statistics(flush_cache=False):
    cache_name = 'contract_prices'

    values = cache.get(cache_name)
    if values is None or flush_cache:
        values = analysis.get_contracts_macro_statistics()
        cache.set(cache_name, values, 60*60*48)

    return values

def contracts_macro_statistics_json(request):
    data = get_contracts_macro_statistics()
    return HttpResponse(json.dumps(data), content_type="application/json")

def get_procedure_types_time_series(flush_cache=False):
    cache_name = 'procedure_type_time_series'

    values = cache.get(cache_name)
    if values is None or flush_cache:
        values = analysis.get_procedure_types_time_series()
        cache.set(cache_name, values, 60*60*24*30)  # one month

    return values

def get_procedure_types_time_series_json(request):
    data = get_procedure_types_time_series()

    for x in data:
        x['from'] = x['from'].strftime('%Y-%m-%d')
        x['to'] = x['to'].strftime('%Y-%m-%d')

    return HttpResponse(json.dumps(data), content_type="application/json")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import warnings
import theano
import pylearn2
import numpy as np
import cPickle as pk

from data import Data, RotationalDDM

from sklearn.datasets import fetch_mldata
from sklearn import neighbors
from sklearn.metrics import accuracy_score, classification_report

from pdb import set_trace as debug

from theano import tensor as T

from pylearn2.models import mlp, maxout
from pylearn2.training_algorithms import sgd, learning_rule
from pylearn2.training_algorithms.sgd import MonitorBasedLRAdjuster
from pylearn2.termination_criteria import EpochCounter
from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix
from pylearn2.space import Conv2DSpace, VectorSpace
from pylearn2.costs.cost import SumOfCosts, MethodCost
from pylearn2.costs.mlp import dropout, WeightDecay

warnings.filterwarnings("ignore")

def ac_score(y, pred):
    s = np.array(np.array(y) == np.array(pred))
    return float(np.sum(s)) / float(len(s))

def convert_one_hot(data):
    return np.array([[1 if y == c else 0 for c in xrange(
        len(np.unique(data)))] for y in data])

def convert_categorical(data):
    return np.argmax(data, axis=1)

def train(d=None):
    train_X = np.array(d.train_X)
    train_y = np.array(d.train_Y)
    valid_X = np.array(d.valid_X)
    valid_y = np.array(d.valid_Y)
    test_X = np.array(d.test_X)
    test_y = np.array(d.test_Y)
    nb_classes = len(np.unique(train_y))
    train_y = convert_one_hot(train_y)
    valid_y = convert_one_hot(valid_y)
    # train_set = RotationalDDM(X=train_X, y=train_y)
    train_set = DenseDesignMatrix(X=train_X, y=train_y)
    valid_set = DenseDesignMatrix(X=valid_X, y=valid_y)
    print 'Setting up'
    batch_size = 128
    c0 = mlp.ConvRectifiedLinear(
        layer_name='c0',
        output_channels=64,
        irange=.05,
        kernel_shape=[5, 5],
        pool_shape=[4, 4],
        pool_stride=[2, 2],
        # W_lr_scale=0.25,
        max_kernel_norm=1.9365
    )
    c1 = mlp.ConvRectifiedLinear(
        layer_name='c1',
        output_channels=64,
        irange=.05,
        kernel_shape=[5, 5],
        pool_shape=[4, 4],
        pool_stride=[2, 2],
        # W_lr_scale=0.25,
        max_kernel_norm=1.9365
    )
    c2 = mlp.ConvRectifiedLinear(
        layer_name='c2',
        output_channels=128,
        irange=.05,
        kernel_shape=[2, 2],
        pool_shape=[2, 2],
        pool_stride=[2, 2],
        W_lr_scale=0.25,
        # max_kernel_norm=1.9365
    )
    sp0 = mlp.SoftmaxPool(
        detector_layer_dim=16,
        layer_name='sp0',
        pool_size=4,
        sparse_init=512,
    )
    sp1 = mlp.SoftmaxPool(
        detector_layer_dim=16,
        layer_name='sp1',
        pool_size=4,
        sparse_init=512,
    )
    r0 = mlp.RectifiedLinear(
        layer_name='r0',
        dim=512,
        sparse_init=512,
    )
    r1 = mlp.RectifiedLinear(
        layer_name='r1',
        dim=512,
        sparse_init=512,
    )
    out = mlp.Softmax(
        n_classes=nb_classes,
        layer_name='output',
        # irange=.235,
        max_col_norm=1.9365,
        sparse_init=nb_classes,
    )
    epochs = EpochCounter(100)
    layers = [c0, c1, out]
    decay_coeffs = [.00005, .00005, .00005]
    in_space = Conv2DSpace(
        shape=[d.size, d.size],
        num_channels=1,
        # axes=['c', 0, 1, 'b'],
    )
    vec_space = VectorSpace(d.size ** 2)
    nn = mlp.MLP(layers=layers, input_space=in_space, batch_size=batch_size)
    trainer = sgd.SGD(
        learning_rate=.01,
        cost=SumOfCosts(costs=[
            # dropout.Dropout(),
            MethodCost(method='cost_from_X'),
            WeightDecay(decay_coeffs),
        ]),
        batch_size=batch_size,
        train_iteration_mode='even_shuffled_sequential',
        termination_criterion=epochs,
        learning_rule=learning_rule.Momentum(init_momentum=0.5),
    )
    momentum_adjustor = learning_rule.MomentumAdjustor(
        final_momentum=.99,
        start=1,
        saturate=10
    )
    trainer.setup(nn, train_set)
    print 'Learning'
    test_X = vec_space.np_format_as(test_X, nn.get_input_space())
    train_X = vec_space.np_format_as(train_X, nn.get_input_space())
    i = 0
    X = nn.get_input_space().make_theano_batch()
    Y = nn.fprop(X)
    predict = theano.function([X], Y)
    best = -40
    best_iter = -1
    while trainer.continue_learning(nn):
        print '--------------'
        print 'Training Epoch ' + str(i)
        trainer.train(dataset=train_set)
        print 'Evaluating...'
        predictions = convert_categorical(predict(train_X[:2000]))
        score = accuracy_score(
            convert_categorical(train_y[:2000]), predictions)
        print 'Score on train: ' + str(score)
        predictions = convert_categorical(predict(test_X))
        score = accuracy_score(test_y, predictions)
        print 'Score on test: ' + str(score)
        best, best_iter = (best, best_iter) if best > score else (score, i)
        print 'Current best: ' + str(best) + ' at iter ' + str(best_iter)
        print classification_report(test_y, predictions)
        print 'Adjusting parameters...'
        momentum_adjustor.on_monitor(nn, valid_set, trainer)
        i += 1
        print ' '

if __name__ == '__main__':
    mnist = fetch_mldata('MNIST original')
    d = Data(dataset=mnist, train_perc=0.7, valid_perc=0.15, test_perc=0.15,
             shuffle=False)
    train(d=d)
<commit_msg>New commits.<commit_after># -*- coding: utf-8 -*-

import warnings
import theano
import pylearn2
import numpy as np
import cPickle as pk

from data import Data, RotationalDDM

from sklearn.datasets import fetch_mldata
from sklearn import neighbors
from sklearn.metrics import accuracy_score, classification_report

from pdb import set_trace as debug

from theano import tensor as T

from pylearn2.models import mlp, maxout
from pylearn2.training_algorithms import sgd, learning_rule
from pylearn2.training_algorithms.sgd import MonitorBasedLRAdjuster
from pylearn2.termination_criteria import EpochCounter
from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix
from pylearn2.space import Conv2DSpace, VectorSpace
from pylearn2.costs.cost import SumOfCosts, MethodCost
from pylearn2.costs.mlp import dropout, WeightDecay

warnings.filterwarnings("ignore")

def ac_score(y, pred):
    s = np.array(np.array(y) == np.array(pred))
    return float(np.sum(s)) / float(len(s))

def convert_one_hot(data):
    return np.array([[1 if y == c else 0 for c in xrange(
        len(np.unique(data)))] for y in data])

def convert_categorical(data):
    return np.argmax(data, axis=1)

def train(d=None):
    train_X = np.array(d.train_X)
    train_y = np.array(d.train_Y)
    valid_X = np.array(d.valid_X)
    valid_y = np.array(d.valid_Y)
    test_X = np.array(d.test_X)
    test_y = np.array(d.test_Y)
    nb_classes = len(np.unique(train_y))
    train_y = convert_one_hot(train_y)
    valid_y = convert_one_hot(valid_y)
    # train_set = RotationalDDM(X=train_X, y=train_y)
    train_set = DenseDesignMatrix(X=train_X, y=train_y)
    valid_set = DenseDesignMatrix(X=valid_X, y=valid_y)
    print 'Setting up'
    batch_size = 128
    c0 = mlp.ConvRectifiedLinear(
        layer_name='c0',
        output_channels=64,
        irange=.05,
        kernel_shape=[5, 5],
        pool_shape=[4, 4],
        pool_stride=[2, 2],
        # W_lr_scale=0.25,
        max_kernel_norm=1.9365
    )
    c1 = mlp.ConvRectifiedLinear(
        layer_name='c1',
        output_channels=64,
        irange=.05,
        kernel_shape=[5, 5],
        pool_shape=[4, 4],
        pool_stride=[2, 2],
        # W_lr_scale=0.25,
        max_kernel_norm=1.9365
    )
    c2 = mlp.ConvRectifiedLinear(
        layer_name='c2',
        output_channels=128,
        irange=.05,
        kernel_shape=[2, 2],
        pool_shape=[2, 2],
        pool_stride=[2, 2],
        W_lr_scale=0.25,
        # max_kernel_norm=1.9365
    )
    sp0 = mlp.SoftmaxPool(
        detector_layer_dim=16,
        layer_name='sp0',
        pool_size=4,
        sparse_init=512,
    )
    sp1 = mlp.SoftmaxPool(
        detector_layer_dim=16,
        layer_name='sp1',
        pool_size=4,
        sparse_init=512,
    )
    r0 = mlp.RectifiedLinear(
        layer_name='r0',
        dim=512,
        sparse_init=512,
    )
    r1 = mlp.RectifiedLinear(
        layer_name='r1',
        dim=512,
        sparse_init=512,
    )
    out = mlp.Softmax(
        n_classes=nb_classes,
        layer_name='output',
        # irange=.235,
        max_col_norm=1.9365,
        sparse_init=nb_classes,
    )
    epochs = EpochCounter(100)
    layers = [c0, c1, out]
    decay_coeffs = [.00005, .00005, .00005]
    in_space = Conv2DSpace(
        shape=[d.size, d.size],
        num_channels=1,
        # axes=['c', 0, 1, 'b'],
    )
    vec_space = VectorSpace(d.size ** 2)
    nn = mlp.MLP(layers=layers, input_space=in_space, batch_size=batch_size)
    trainer = sgd.SGD(
        learning_rate=.01,
        cost=SumOfCosts(costs=[
            # dropout.Dropout(),
            MethodCost(method='cost_from_X'),
            WeightDecay(decay_coeffs),
        ]),
        batch_size=batch_size,
        train_iteration_mode='even_shuffled_sequential',
        termination_criterion=epochs,
        learning_rule=learning_rule.Momentum(init_momentum=0.5),
    )
    momentum_adjustor = learning_rule.MomentumAdjustor(
        final_momentum=.99,
        start=1,
        saturate=10
    )
    trainer.setup(nn, train_set)
    print 'Learning'
    test_X = vec_space.np_format_as(test_X, nn.get_input_space())
    train_X = vec_space.np_format_as(train_X, nn.get_input_space())
    i = 0
    X = nn.get_input_space().make_theano_batch()
    Y = nn.fprop(X)
    predict = theano.function([X], Y)
    best = -40
    best_iter = -1
    while trainer.continue_learning(nn):
        print '--------------'
        print 'Training Epoch ' + str(i)
        trainer.train(dataset=train_set)
        print 'Evaluating...'
        predictions = convert_categorical(predict(train_X[:2000]))
        score = accuracy_score(
            convert_categorical(train_y[:2000]), predictions)
        print 'Score on train: ' + str(score)
        predictions = convert_categorical(predict(test_X))
        score = accuracy_score(test_y, predictions)
        print 'Score on test: ' + str(score)
        best, best_iter = (best, best_iter) if best > score else (score, i)
        print 'Current best: ' + str(best) + ' at iter ' + str(best_iter)
        print classification_report(test_y, predictions)
        print 'Adjusting parameters...'
        momentum_adjustor.on_monitor(nn, valid_set, trainer)
        i += 1
        print ' '

if __name__ == '__main__':
    mnist = fetch_mldata('MNIST original')
    d = Data(dataset=mnist, train_perc=0.7, valid_perc=0.2, test_perc=0.1,
             shuffle=False)
    train(d=d)
<|endoftext|>


<commit_before>from django import template

from wafer.sponsors.models import Sponsor, SponsorshipPackage

register = template.Library()

@register.inclusion_tag('wafer.sponsors/sponsors_block.html')
def sponsors():
    return {
        'sponsors': Sponsor.objects.all().order_by('packages'),
        'packages': SponsorshipPackage.objects.all().prefetch_related('files'),
    }
<commit_msg>Add template tag to look up sponsor images<commit_after>from django import template

from wafer.sponsors.models import Sponsor, SponsorshipPackage

register = template.Library()

@register.inclusion_tag('wafer.sponsors/sponsors_block.html')
def sponsors():
    return {
        'sponsors': Sponsor.objects.all().order_by('packages'),
        'packages': SponsorshipPackage.objects.all().prefetch_related('files'),
    }

# We use assignment_tag for compatibility with Django 1.8
# Once we drop 1.8 support, we should change this to
# simple_tag
@register.assignment_tag(takes_context=True)
def sponsor_image_url(context, name):
    """Returns the corresponding url from the sponsors images"""
    sponsor = context['sponsor']
    if sponsor.files.filter(name=name).exists():
        # We avoid worrying about multiple matches by always
        # returning the first one.
        return sponsor.files.filter(name=name).first().item.url
    return ''
<|endoftext|>


<commit_before>import os, sys
import urllib2
import json
import datetime
from flask import jsonify

class PsiturkOrgServices:
    """
        PsiturkOrgServices
        this class provides an interface to the API provided
        by the psiturk_org website.  the two main features
        of this API are registering secure ads and
        interfacing with the experiment exchange
        see: https://github.com/NYUCCL/psiTurk_website
    """
    def __init__(self, ad_server_location, contact_email):
        self.adServer = ad_server_location # by default for now
        if self.adServer[-1]=='/':
            self.adServer = self.adServer[:-1]
        self.contactEmail = contact_email

    def connect(self, server):
        """
            connect:
            "connects to server"  since the is a fairly
            basic API, just allows overriding of which Ad server
            you are talking to
        """
        self.adServer = server

    def get_system_status(self):
        """
            get_system_status:
        """
        try:
            ad_server_status_link= self.adServer + '/status_msg'
            response=urllib2.urlopen(ad_server_status_link,timeout=1)
            status_msg = json.load(response)['status']
        except urllib2.URLError as err:
            status_msg = "Sorry, can't connect to psiturk.org, please check your internet connection.\n You will not be able to create new hits, but testing locally should work."
        return status_msg

    def get_my_ip(self):
        """
            get_my_ip:
            asks and external server what your ip appears to be
            (useful is running from behind a NAT/wifi router).
            Of course, incoming port to the router must be
            forwarded correctly.
        """
        return json.load(urllib2.urlopen('http://httpbin.org/ip'))['origin']

    def get_ad_url(self, adId):
        """
            get_ad_url:
            gets ad server thing
        """
        return self.adServer + '/ad/' + str(adId)

    def set_ad_hitid(self, adId, hitId):
        """
            get_ad_hitid:
            updates the ad with the corresponding hitid
        """
        ad_server_update_link = self.adServer + '/ad/' + str(adId) + '/link?hitid=' + hitId
        response = urllib2.urlopen(ad_server_update_link)
        if json.load(response)['status']=="we're good!":
            return True
        else:
            return False

    def delete_ad(self, hitId):
        """
            delete_ad:
        """
        ad_server_expire_link = self.adServer + '/ad/delete?hitid=' + hitId
        response = urllib2.urlopen(ad_server_expire_link)
        status_msg = json.load(response)['status']
        if status_msg=="we're good!" or status_msg=="add not found":
            return True
        else:
            return False

    def create_ad(self, ad_content):
        """
            create_ad:
        """
        ad_server_register_url = self.adServer + '/ad/register'
        #ad_server_register_url = 'http://0.0.0.0:5004/ad/register'
        req = urllib2.Request(ad_server_register_url)
        req.add_header('Content-Type', 'application/json')
        response = urllib2.urlopen(req, json.dumps(ad_content))

        # 2. get id in response
        data = json.load(response)
        if isinstance(data['id'], int):
            return data['id']
        else:
            if data['id'] == "correct parameters not provided":
                print "Error: registering Ad with server, you didn't provide all the required parameters (server, port, support_ie)"
                return False
            elif data['id'] == "localhost not allowed":
                print "Error: attempting to localhost or 127.0.0.1 as your server location to the Ad server, but this is not allowed.  Check the 'host' parameter in config.txt and make it a publically accessible hostname/ip."
                return False
            elif data['id'][:25] == "The maximum template size":
                print data['id']  # print out the message about template size
                return False
<commit_msg>get status via api now<commit_after>import os, sys
import urllib2
import json
import datetime
from flask import jsonify

class PsiturkOrgServices:
    """
        PsiturkOrgServices
        this class provides an interface to the API provided
        by the psiturk_org website.  the two main features
        of this API are registering secure ads and
        interfacing with the experiment exchange
        see: https://github.com/NYUCCL/psiTurk_website
    """
    def __init__(self, ad_server_location, contact_email):
        self.adServer = ad_server_location # by default for now
        if self.adServer[-1]=='/':
            self.adServer = self.adServer[:-1]
        self.contactEmail = contact_email

    def connect(self, server):
        """
            connect:
            "connects to server"  since the is a fairly
            basic API, just allows overriding of which Ad server
            you are talking to
        """
        self.adServer = server

    def get_system_status(self):
        """
            get_system_status:
        """
        try:
            ad_server_status_link= 'http://api.psiturk.org/get_status_msg'
            response=urllib2.urlopen(ad_server_status_link,timeout=1)
            status_msg = json.load(response)['status']
        except urllib2.URLError as err:
            status_msg = "Sorry, can't connect to psiturk.org, please check your internet connection.\n You will not be able to create new hits, but testing locally should work."
        return status_msg

    def get_my_ip(self):
        """
            get_my_ip:
            asks and external server what your ip appears to be
            (useful is running from behind a NAT/wifi router).
            Of course, incoming port to the router must be
            forwarded correctly.
        """
        return json.load(urllib2.urlopen('http://httpbin.org/ip'))['origin']

    def get_ad_url(self, adId):
        """
            get_ad_url:
            gets ad server thing
        """
        return self.adServer + '/ad/' + str(adId)

    def set_ad_hitid(self, adId, hitId):
        """
            get_ad_hitid:
            updates the ad with the corresponding hitid
        """
        ad_server_update_link = self.adServer + '/ad/' + str(adId) + '/link?hitid=' + hitId
        response = urllib2.urlopen(ad_server_update_link)
        if json.load(response)['status']=="we're good!":
            return True
        else:
            return False

    def delete_ad(self, hitId):
        """
            delete_ad:
        """
        ad_server_expire_link = self.adServer + '/ad/delete?hitid=' + hitId
        response = urllib2.urlopen(ad_server_expire_link)
        status_msg = json.load(response)['status']
        if status_msg=="we're good!" or status_msg=="add not found":
            return True
        else:
            return False

    def create_ad(self, ad_content):
        """
            create_ad:
        """
        ad_server_register_url = self.adServer + '/ad/register'
        #ad_server_register_url = 'http://0.0.0.0:5004/ad/register'
        req = urllib2.Request(ad_server_register_url)
        req.add_header('Content-Type', 'application/json')
        response = urllib2.urlopen(req, json.dumps(ad_content))

        # 2. get id in response
        data = json.load(response)
        if isinstance(data['id'], int):
            return data['id']
        else:
            if data['id'] == "correct parameters not provided":
                print "Error: registering Ad with server, you didn't provide all the required parameters (server, port, support_ie)"
                return False
            elif data['id'] == "localhost not allowed":
                print "Error: attempting to localhost or 127.0.0.1 as your server location to the Ad server, but this is not allowed.  Check the 'host' parameter in config.txt and make it a publically accessible hostname/ip."
                return False
            elif data['id'][:25] == "The maximum template size":
                print data['id']  # print out the message about template size
                return False
<|endoftext|>


<commit_before>import unittest
import sys
import os
import glob
import yaml

class VerifyBoshRelease(unittest.TestCase):

	def test_has_manifest(self):
		self.assertTrue(os.path.exists('release/release.MF'))

	def test_has_correct_number_of_jobs(self):
		self.assertEqual(len(glob.glob('release/jobs/*')), 25)

	def test_has_correct_number_of_app_jobs(self):
		self.assertEqual(len(glob.glob('release/jobs/deploy-app-*')), 6)
		self.assertEqual(len(glob.glob('release/jobs/delete-app-*')), 6)

	def test_has_correct_number_of_broker_jobs(self):
		self.assertEqual(len(glob.glob('release/jobs/register-broker-*')), 4)
		self.assertEqual(len(glob.glob('release/jobs/destroy-broker-*')), 4)

	def test_has_correct_number_of_buildpack_jobs(self):
		self.assertEqual(len(glob.glob('release/jobs/deploy-buildpack-*')), 2)
		self.assertEqual(len(glob.glob('release/jobs/delete-buildpack-*')), 2)

	def test_has_correct_number_of_docker_jobs(self):
		self.assertEqual(len(glob.glob('release/jobs/docker-bosh-*')), 1)

	def test_all_jobs_have_monit(self):
		self.assertEqual(len(glob.glob('release/jobs/*/monit')), len(glob.glob('release/jobs/*')))

	def test_errands_have_empty_monit(self):
		for monit in glob.glob('release/jobs/*/monit'):
			if not monit.startswith('release/jobs/docker-bosh-'):
				self.assertTrue(os.path.exists(monit), monit)
				self.assertEqual(os.path.getsize(monit), 0, monit)

	def test_non_errands_have_nonempty_monit(self):
		for monit in glob.glob('release/jobs/*'):
			if monit.startswith('release/jobs/docker-bosh-'):
				self.assertTrue(os.path.exists(monit), monit)
				self.assertNotEqual(os.path.getsize(monit), 0, monit)

	def test_all_jobs_have_manifest(self):
		self.assertEqual(len(glob.glob('release/jobs/*/job.MF')), len(glob.glob('release/jobs/*')))

	def test_cf_errand_manifest_has_cf_cli_package(self):
		for manifest in glob.glob('release/jobs/*/job.MF'):
			if not manifest.startswith('release/jobs/docker-bosh-'):
				self.assertTrue('cf_cli' in read_yaml(manifest).get('packages', []), manifest)

	def test_bosh_job_spec_has_no_cf_cli_package(self):
		for manifest in glob.glob('release/jobs/*/job.MF'):
			if manifest.startswith('release/jobs/docker-bosh-'):
				self.assertFalse('cf_cli' in read_yaml(manifest).get('packages', []), manifest)

def read_yaml(filename):
	with open(filename, 'rb') as file:
		return yaml.safe_load(file)

if __name__ == '__main__':
	unittest.main()<commit_msg>More tests<commit_after>import unittest
import sys
import os
import glob
import yaml

class VerifyBoshRelease(unittest.TestCase):

	def test_has_manifest(self):
		self.assertTrue(os.path.exists('release/release.MF'))

	def test_has_correct_number_of_jobs(self):
		self.assertEqual(len(glob.glob('release/jobs/*')), 25)

	def test_has_correct_number_of_app_jobs(self):
		self.assertEqual(len(glob.glob('release/jobs/deploy-app-*')), 6)
		self.assertEqual(len(glob.glob('release/jobs/delete-app-*')), 6)

	def test_has_correct_number_of_broker_jobs(self):
		self.assertEqual(len(glob.glob('release/jobs/register-broker-*')), 4)
		self.assertEqual(len(glob.glob('release/jobs/destroy-broker-*')), 4)

	def test_has_correct_number_of_buildpack_jobs(self):
		self.assertEqual(len(glob.glob('release/jobs/deploy-buildpack-*')), 2)
		self.assertEqual(len(glob.glob('release/jobs/delete-buildpack-*')), 2)

	def test_has_correct_number_of_docker_jobs(self):
		self.assertEqual(len(glob.glob('release/jobs/docker-bosh-*')), 1)

	def test_all_jobs_have_monit(self):
		self.assertEqual(len(glob.glob('release/jobs/*/monit')), len(glob.glob('release/jobs/*')))

	def test_errands_have_empty_monit(self):
		for monit in glob.glob('release/jobs/*/monit'):
			if not monit.startswith('release/jobs/docker-bosh-'):
				self.assertTrue(os.path.exists(monit), monit)
				self.assertEqual(os.path.getsize(monit), 0, monit)

	def test_non_errands_have_nonempty_monit(self):
		for monit in glob.glob('release/jobs/*'):
			if monit.startswith('release/jobs/docker-bosh-'):
				self.assertTrue(os.path.exists(monit), monit)
				self.assertNotEqual(os.path.getsize(monit), 0, monit)

	def test_all_jobs_have_manifest(self):
		self.assertEqual(len(glob.glob('release/jobs/*/job.MF')), len(glob.glob('release/jobs/*')))

	def test_cf_errand_manifest_has_cf_cli_package(self):
		for manifest in glob.glob('release/jobs/*/job.MF'):
			if not manifest.startswith('release/jobs/docker-bosh-'):
				self.assertTrue('cf_cli' in read_yaml(manifest).get('packages', []), manifest)

	def test_bosh_job_spec_has_no_cf_cli_package(self):
		for manifest in glob.glob('release/jobs/*/job.MF'):
			if manifest.startswith('release/jobs/docker-bosh-'):
				self.assertFalse('cf_cli' in read_yaml(manifest).get('packages', []), manifest)

	def test_all_jobs_have_template(self):
		self.assertEqual(len(glob.glob('release/jobs/*/templates/*.sh.erb')), len(glob.glob('release/jobs/*')))

	def test_has_correct_number_of_packages(self):
		self.assertEqual(len(glob.glob('release/packages/*')), 13)

	def test_has_complete_cf_cli_package(self):
		self.assertEqual(len(glob.glob('release/packages/cf_cli')), 1)
		self.assertEqual(len(glob.glob('release/packages/cf_cli/cf_cli/cf-linux-amd64.tgz')), 1)
		self.assertEqual(len(glob.glob('release/packages/cf_cli/packaging')), 1)
		self.assertTrue('cf-linux-amd64.tgz' in read_file('release/packages/cf_cli/packaging'))

	def test_has_complete_common_package(self):
		self.assertEqual(len(glob.glob('release/packages/common')), 1)
		self.assertEqual(len(glob.glob('release/packages/common/common/utils.sh')), 1)

def read_yaml(filename):
	with open(filename, 'rb') as file:
		return yaml.safe_load(file)

def read_file(filename):
	with open(filename, 'rb') as file:
		return file.read()

if __name__ == '__main__':
	unittest.main()<|endoftext|>


<commit_before>import os
import shutil
from pipenv.project import Project
from pipenv._compat import Path

from pipenv.utils import mkdir_p, temp_environ

import pytest

from flaky import flaky

@pytest.mark.extras
@pytest.mark.install
@pytest.mark.local
@pytest.mark.parametrize(
    "line, pipfile",
    [["-e .[dev]", {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}}]],
)
def test_local_extras_install(PipenvInstance, pypi, line, pipfile):
    """Ensure -e .[extras] installs.
    """
    with PipenvInstance(pypi=pypi, chdir=True) as p:
        project = Project()
        setup_py = os.path.join(p.path, "setup.py")
        with open(setup_py, "w") as fh:
            contents = """
from setuptools import setup, find_packages
setup(
name='testpipenv',
version='0.1',
description='Pipenv Test Package',
author='Pipenv Test',
author_email='test@pipenv.package',
license='MIT',
packages=find_packages(),
install_requires=[],
extras_require={'dev': ['six']},
zip_safe=False
)
            """.strip()
            fh.write(contents)
        project.write_toml({"packages": pipfile, "dev-packages": {}})
        c = p.pipenv("install")
        assert c.return_code == 0
        assert "testpipenv" in p.lockfile["default"]
        assert p.lockfile["default"]["testpipenv"]["extras"] == ["dev"]
        assert "six" in p.lockfile["default"]
        c = p.pipenv("--rm")
        assert c.return_code == 0
        project.write_toml({"packages": {}, "dev-packages": {}})
        c = p.pipenv("install {0}".format(line))
        assert c.return_code == 0
        assert "testpipenv" in p.pipfile["packages"]
        assert p.pipfile["packages"]["testpipenv"]["path"] == "."
        assert p.pipfile["packages"]["testpipenv"]["extras"] == ["dev"]
        assert "six" in p.lockfile["default"]

class TestDependencyLinks(object):
    """Ensure dependency_links are parsed and installed.

    This is needed for private repo dependencies.
    """

    @staticmethod
    def helper_dependency_links_install_make_setup(pipenv_instance, deplink):
        setup_py = os.path.join(pipenv_instance.path, "setup.py")
        with open(setup_py, "w") as fh:
            contents = """
from setuptools import setup

setup(
    name='testdeplinks',
    version='0.1',
    packages=[],
    install_requires=[
        'test-private-dependency'
    ],
    dependency_links=[
        '{0}'
    ]
)
            """.strip().format(deplink)
            fh.write(contents)

    @staticmethod
    def helper_dependency_links_install_test(pipenv_instance, deplink):
        TestDependencyLinks.helper_dependency_links_install_make_setup(pipenv_instance, deplink)
        c = pipenv_instance.pipenv("install -v -e .")
        assert c.return_code == 0
        assert "test-private-dependency" in pipenv_instance.lockfile["default"]
        assert "version" in pipenv_instance.lockfile["default"]["test-private-dependency"]
        assert "0.1" in pipenv_instance.lockfile["default"]["test-private-dependency"]["version"]

    @pytest.mark.install
    @pytest.mark.local
    @pytest.mark.needs_internet
    @flaky
    def test_https_dependency_links_install(self, PipenvInstance, pypi):
        """Ensure dependency_links are parsed and installed (needed for private repo dependencies).
        """
        with temp_environ(), PipenvInstance(pypi=pypi, chdir=True) as p:
            os.environ['PIP_PROCESS_DEPENDENCY_LINKS'] = '1'
            TestDependencyLinks.helper_dependency_links_install_test(
                p,
                'git+https://github.com/atzannes/test-private-dependency@v0.1#egg=test-private-dependency-v0.1'
            )

    @pytest.mark.install
    @pytest.mark.local
    @pytest.mark.needs_internet
    @pytest.mark.needs_github_ssh
    @flaky
    def test_ssh_dependency_links_install(self, PipenvInstance, pypi):
        with temp_environ(), PipenvInstance(pypi=pypi, chdir=True) as p:
            os.environ['PIP_PROCESS_DEPENDENCY_LINKS'] = '1'
            TestDependencyLinks.helper_dependency_links_install_test(
                p,
                'git+ssh://git@github.com/atzannes/test-private-dependency@v0.1#egg=test-private-dependency-v0.1'
            )

@pytest.mark.e
@pytest.mark.install
@pytest.mark.skip(reason="this doesn't work on windows")
def test_e_dot(PipenvInstance, pip_src_dir):
    with PipenvInstance() as p:
        path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
        c = p.pipenv("install -e '{0}' --dev".format(path))

        assert c.return_code == 0

        key = [k for k in p.pipfile["dev-packages"].keys()][0]
        assert "path" in p.pipfile["dev-packages"][key]
        assert "requests" in p.lockfile["develop"]

@pytest.mark.install
@flaky
def test_multiprocess_bug_and_install(PipenvInstance, pypi):
    with temp_environ():
        os.environ["PIPENV_MAX_SUBPROCESS"] = "2"

        with PipenvInstance(pypi=pypi, chdir=True) as p:
            with open(p.pipfile_path, "w") as f:
                contents = """
[packages]
pytz = "*"
six = "*"
urllib3 = "*"
                """.strip()
                f.write(contents)

            c = p.pipenv("install")
            assert c.return_code == 0

            assert "pytz" in p.lockfile["default"]
            assert "six" in p.lockfile["default"]
            assert "urllib3" in p.lockfile["default"]

            c = p.pipenv('run python -c "import six; import pytz; import urllib3;"')
            assert c.return_code == 0

@pytest.mark.sequential
@pytest.mark.install
@flaky
def test_sequential_mode(PipenvInstance, pypi):

    with PipenvInstance(pypi=pypi, chdir=True) as p:
        with open(p.pipfile_path, "w") as f:
            contents = """
[packages]
six = "*"
urllib3 = "*"
pytz = "*"
            """.strip()
            f.write(contents)

        c = p.pipenv("install --sequential")
        assert c.return_code == 0

        assert "six" in p.lockfile["default"]
        assert "pytz" in p.lockfile["default"]
        assert "urllib3" in p.lockfile["default"]

        c = p.pipenv('run python -c "import six; import urllib3; import pytz;"')
        assert c.return_code == 0

@pytest.mark.install
@pytest.mark.run
def test_normalize_name_install(PipenvInstance, pypi):
    with PipenvInstance(pypi=pypi) as p:
        with open(p.pipfile_path, "w") as f:
            contents = """
# Pre comment
[packages]
Requests = "==2.14.0"   # Inline comment
"""
            f.write(contents)

        c = p.pipenv("install")
        assert c.return_code == 0

        c = p.pipenv("install requests")
        assert c.return_code == 0
        assert "requests" not in p.pipfile["packages"]
        assert p.pipfile["packages"]["Requests"] == "==2.14.0"
        c = p.pipenv("install requests==2.18.4")
        assert c.return_code == 0
        assert p.pipfile["packages"]["Requests"] == "==2.18.4"
        c = p.pipenv("install python_DateUtil")
        assert c.return_code == 0
        assert "python-dateutil" in p.pipfile["packages"]
        contents = open(p.pipfile_path).read()
        assert "# Pre comment" in contents
        assert "# Inline comment" in contents

@pytest.mark.files
@pytest.mark.resolver
@pytest.mark.eggs
@flaky
def test_local_package(PipenvInstance, pip_src_dir, pypi, testsroot):
    """This test ensures that local packages (directories with a setup.py)
    installed in editable mode have their dependencies resolved as well"""
    file_name = "requests-2.19.1.tar.gz"
    package = "requests-2.19.1"
    # Not sure where travis/appveyor run tests from
    source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name))
    with PipenvInstance(chdir=True, pypi=pypi) as p:
        # This tests for a bug when installing a zipfile in the current dir
        copy_to = os.path.join(p.path, file_name)
        shutil.copy(source_path, copy_to)
        import tarfile

        with tarfile.open(copy_to, "r:gz") as tgz:
            tgz.extractall(path=p.path)
        c = p.pipenv("install -e {0}".format(package))
        assert c.return_code == 0
        assert all(
            pkg in p.lockfile["default"]
            for pkg in ["urllib3", "idna", "certifi", "chardet"]
        )

@pytest.mark.files
@flaky
def test_local_zipfiles(PipenvInstance, pypi, testsroot):
    file_name = "requests-2.19.1.tar.gz"
    # Not sure where travis/appveyor run tests from
    source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name))

    with PipenvInstance(chdir=True, pypi=pypi) as p:
        # This tests for a bug when installing a zipfile in the current dir
        shutil.copy(source_path, os.path.join(p.path, file_name))

        c = p.pipenv("install {}".format(file_name))
        assert c.return_code == 0
        key = [k for k in p.pipfile["packages"].keys()][0]
        dep = p.pipfile["packages"][key]

        assert "file" in dep or "path" in dep
        assert c.return_code == 0

        key = [k for k in p.lockfile["default"].keys()][0]
        dep = p.lockfile["default"][key]

        assert "file" in dep or "path" in dep

@pytest.mark.files
@flaky
def test_relative_paths(PipenvInstance, pypi, testsroot):
    file_name = "requests-2.19.1.tar.gz"
    source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name))

    with PipenvInstance(pypi=pypi) as p:
        artifact_dir = "artifacts"
        artifact_path = os.path.join(p.path, artifact_dir)
        mkdir_p(artifact_path)
        shutil.copy(source_path, os.path.join(artifact_path, file_name))
        # Test installing a relative path in a subdirectory
        c = p.pipenv("install {}/{}".format(artifact_dir, file_name))
        assert c.return_code == 0
        key = next(k for k in p.pipfile["packages"].keys())
        dep = p.pipfile["packages"][key]

        assert "path" in dep
        assert Path(".", artifact_dir, file_name) == Path(dep["path"])
        assert c.return_code == 0

@pytest.mark.install
@pytest.mark.local_file
@flaky
def test_install_local_file_collision(PipenvInstance, pypi):
    with PipenvInstance(pypi=pypi) as p:
        target_package = "alembic"
        fake_file = os.path.join(p.path, target_package)
        with open(fake_file, "w") as f:
            f.write("")
        c = p.pipenv("install {}".format(target_package))
        assert c.return_code == 0
        assert target_package in p.pipfile["packages"]
        assert p.pipfile["packages"][target_package] == "*"
        assert target_package in p.lockfile["default"]

@pytest.mark.url
@pytest.mark.install
def test_install_local_uri_special_character(PipenvInstance, testsroot):
    file_name = "six-1.11.0+mkl-py2.py3-none-any.whl"
    source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name))
    with PipenvInstance() as p:
        artifact_dir = "artifacts"
        artifact_path = os.path.join(p.path, artifact_dir)
        mkdir_p(artifact_path)
        shutil.copy(source_path, os.path.join(artifact_path, file_name))
        with open(p.pipfile_path, "w") as f:
            contents = """
# Pre comment
[packages]
six = {{path = "./artifacts/{}"}}
            """.format(
                file_name
            )
            f.write(contents.strip())
        c = p.pipenv("install")
        assert c.return_code == 0
        assert "six" in p.lockfile["default"]
<commit_msg>Factor-out decorators for class tests<commit_after>import os
import shutil
from pipenv.project import Project
from pipenv._compat import Path

from pipenv.utils import mkdir_p, temp_environ

import pytest

from flaky import flaky

@pytest.mark.extras
@pytest.mark.install
@pytest.mark.local
@pytest.mark.parametrize(
    "line, pipfile",
    [["-e .[dev]", {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}}]],
)
def test_local_extras_install(PipenvInstance, pypi, line, pipfile):
    """Ensure -e .[extras] installs.
    """
    with PipenvInstance(pypi=pypi, chdir=True) as p:
        project = Project()
        setup_py = os.path.join(p.path, "setup.py")
        with open(setup_py, "w") as fh:
            contents = """
from setuptools import setup, find_packages
setup(
name='testpipenv',
version='0.1',
description='Pipenv Test Package',
author='Pipenv Test',
author_email='test@pipenv.package',
license='MIT',
packages=find_packages(),
install_requires=[],
extras_require={'dev': ['six']},
zip_safe=False
)
            """.strip()
            fh.write(contents)
        project.write_toml({"packages": pipfile, "dev-packages": {}})
        c = p.pipenv("install")
        assert c.return_code == 0
        assert "testpipenv" in p.lockfile["default"]
        assert p.lockfile["default"]["testpipenv"]["extras"] == ["dev"]
        assert "six" in p.lockfile["default"]
        c = p.pipenv("--rm")
        assert c.return_code == 0
        project.write_toml({"packages": {}, "dev-packages": {}})
        c = p.pipenv("install {0}".format(line))
        assert c.return_code == 0
        assert "testpipenv" in p.pipfile["packages"]
        assert p.pipfile["packages"]["testpipenv"]["path"] == "."
        assert p.pipfile["packages"]["testpipenv"]["extras"] == ["dev"]
        assert "six" in p.lockfile["default"]

@pytest.mark.install
@pytest.mark.local
@pytest.mark.needs_internet
@flaky
class TestDependencyLinks(object):
    """Ensure dependency_links are parsed and installed.

    This is needed for private repo dependencies.
    """

    @staticmethod
    def helper_dependency_links_install_make_setup(pipenv_instance, deplink):
        setup_py = os.path.join(pipenv_instance.path, "setup.py")
        with open(setup_py, "w") as fh:
            contents = """
from setuptools import setup

setup(
    name='testdeplinks',
    version='0.1',
    packages=[],
    install_requires=[
        'test-private-dependency'
    ],
    dependency_links=[
        '{0}'
    ]
)
            """.strip().format(deplink)
            fh.write(contents)

    @staticmethod
    def helper_dependency_links_install_test(pipenv_instance, deplink):
        TestDependencyLinks.helper_dependency_links_install_make_setup(pipenv_instance, deplink)
        c = pipenv_instance.pipenv("install -v -e .")
        assert c.return_code == 0
        assert "test-private-dependency" in pipenv_instance.lockfile["default"]
        assert "version" in pipenv_instance.lockfile["default"]["test-private-dependency"]
        assert "0.1" in pipenv_instance.lockfile["default"]["test-private-dependency"]["version"]

    def test_https_dependency_links_install(self, PipenvInstance, pypi):
        """Ensure dependency_links are parsed and installed (needed for private repo dependencies).
        """
        with temp_environ(), PipenvInstance(pypi=pypi, chdir=True) as p:
            os.environ['PIP_PROCESS_DEPENDENCY_LINKS'] = '1'
            TestDependencyLinks.helper_dependency_links_install_test(
                p,
                'git+https://github.com/atzannes/test-private-dependency@v0.1#egg=test-private-dependency-v0.1'
            )

    @pytest.mark.needs_github_ssh
    def test_ssh_dependency_links_install(self, PipenvInstance, pypi):
        with temp_environ(), PipenvInstance(pypi=pypi, chdir=True) as p:
            os.environ['PIP_PROCESS_DEPENDENCY_LINKS'] = '1'
            TestDependencyLinks.helper_dependency_links_install_test(
                p,
                'git+ssh://git@github.com/atzannes/test-private-dependency@v0.1#egg=test-private-dependency-v0.1'
            )

@pytest.mark.e
@pytest.mark.install
@pytest.mark.skip(reason="this doesn't work on windows")
def test_e_dot(PipenvInstance, pip_src_dir):
    with PipenvInstance() as p:
        path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
        c = p.pipenv("install -e '{0}' --dev".format(path))

        assert c.return_code == 0

        key = [k for k in p.pipfile["dev-packages"].keys()][0]
        assert "path" in p.pipfile["dev-packages"][key]
        assert "requests" in p.lockfile["develop"]

@pytest.mark.install
@flaky
def test_multiprocess_bug_and_install(PipenvInstance, pypi):
    with temp_environ():
        os.environ["PIPENV_MAX_SUBPROCESS"] = "2"

        with PipenvInstance(pypi=pypi, chdir=True) as p:
            with open(p.pipfile_path, "w") as f:
                contents = """
[packages]
pytz = "*"
six = "*"
urllib3 = "*"
                """.strip()
                f.write(contents)

            c = p.pipenv("install")
            assert c.return_code == 0

            assert "pytz" in p.lockfile["default"]
            assert "six" in p.lockfile["default"]
            assert "urllib3" in p.lockfile["default"]

            c = p.pipenv('run python -c "import six; import pytz; import urllib3;"')
            assert c.return_code == 0

@pytest.mark.sequential
@pytest.mark.install
@flaky
def test_sequential_mode(PipenvInstance, pypi):

    with PipenvInstance(pypi=pypi, chdir=True) as p:
        with open(p.pipfile_path, "w") as f:
            contents = """
[packages]
six = "*"
urllib3 = "*"
pytz = "*"
            """.strip()
            f.write(contents)

        c = p.pipenv("install --sequential")
        assert c.return_code == 0

        assert "six" in p.lockfile["default"]
        assert "pytz" in p.lockfile["default"]
        assert "urllib3" in p.lockfile["default"]

        c = p.pipenv('run python -c "import six; import urllib3; import pytz;"')
        assert c.return_code == 0

@pytest.mark.install
@pytest.mark.run
def test_normalize_name_install(PipenvInstance, pypi):
    with PipenvInstance(pypi=pypi) as p:
        with open(p.pipfile_path, "w") as f:
            contents = """
# Pre comment
[packages]
Requests = "==2.14.0"   # Inline comment
"""
            f.write(contents)

        c = p.pipenv("install")
        assert c.return_code == 0

        c = p.pipenv("install requests")
        assert c.return_code == 0
        assert "requests" not in p.pipfile["packages"]
        assert p.pipfile["packages"]["Requests"] == "==2.14.0"
        c = p.pipenv("install requests==2.18.4")
        assert c.return_code == 0
        assert p.pipfile["packages"]["Requests"] == "==2.18.4"
        c = p.pipenv("install python_DateUtil")
        assert c.return_code == 0
        assert "python-dateutil" in p.pipfile["packages"]
        contents = open(p.pipfile_path).read()
        assert "# Pre comment" in contents
        assert "# Inline comment" in contents

@pytest.mark.files
@pytest.mark.resolver
@pytest.mark.eggs
@flaky
def test_local_package(PipenvInstance, pip_src_dir, pypi, testsroot):
    """This test ensures that local packages (directories with a setup.py)
    installed in editable mode have their dependencies resolved as well"""
    file_name = "requests-2.19.1.tar.gz"
    package = "requests-2.19.1"
    # Not sure where travis/appveyor run tests from
    source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name))
    with PipenvInstance(chdir=True, pypi=pypi) as p:
        # This tests for a bug when installing a zipfile in the current dir
        copy_to = os.path.join(p.path, file_name)
        shutil.copy(source_path, copy_to)
        import tarfile

        with tarfile.open(copy_to, "r:gz") as tgz:
            tgz.extractall(path=p.path)
        c = p.pipenv("install -e {0}".format(package))
        assert c.return_code == 0
        assert all(
            pkg in p.lockfile["default"]
            for pkg in ["urllib3", "idna", "certifi", "chardet"]
        )

@pytest.mark.files
@flaky
def test_local_zipfiles(PipenvInstance, pypi, testsroot):
    file_name = "requests-2.19.1.tar.gz"
    # Not sure where travis/appveyor run tests from
    source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name))

    with PipenvInstance(chdir=True, pypi=pypi) as p:
        # This tests for a bug when installing a zipfile in the current dir
        shutil.copy(source_path, os.path.join(p.path, file_name))

        c = p.pipenv("install {}".format(file_name))
        assert c.return_code == 0
        key = [k for k in p.pipfile["packages"].keys()][0]
        dep = p.pipfile["packages"][key]

        assert "file" in dep or "path" in dep
        assert c.return_code == 0

        key = [k for k in p.lockfile["default"].keys()][0]
        dep = p.lockfile["default"][key]

        assert "file" in dep or "path" in dep

@pytest.mark.files
@flaky
def test_relative_paths(PipenvInstance, pypi, testsroot):
    file_name = "requests-2.19.1.tar.gz"
    source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name))

    with PipenvInstance(pypi=pypi) as p:
        artifact_dir = "artifacts"
        artifact_path = os.path.join(p.path, artifact_dir)
        mkdir_p(artifact_path)
        shutil.copy(source_path, os.path.join(artifact_path, file_name))
        # Test installing a relative path in a subdirectory
        c = p.pipenv("install {}/{}".format(artifact_dir, file_name))
        assert c.return_code == 0
        key = next(k for k in p.pipfile["packages"].keys())
        dep = p.pipfile["packages"][key]

        assert "path" in dep
        assert Path(".", artifact_dir, file_name) == Path(dep["path"])
        assert c.return_code == 0

@pytest.mark.install
@pytest.mark.local_file
@flaky
def test_install_local_file_collision(PipenvInstance, pypi):
    with PipenvInstance(pypi=pypi) as p:
        target_package = "alembic"
        fake_file = os.path.join(p.path, target_package)
        with open(fake_file, "w") as f:
            f.write("")
        c = p.pipenv("install {}".format(target_package))
        assert c.return_code == 0
        assert target_package in p.pipfile["packages"]
        assert p.pipfile["packages"][target_package] == "*"
        assert target_package in p.lockfile["default"]

@pytest.mark.url
@pytest.mark.install
def test_install_local_uri_special_character(PipenvInstance, testsroot):
    file_name = "six-1.11.0+mkl-py2.py3-none-any.whl"
    source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name))
    with PipenvInstance() as p:
        artifact_dir = "artifacts"
        artifact_path = os.path.join(p.path, artifact_dir)
        mkdir_p(artifact_path)
        shutil.copy(source_path, os.path.join(artifact_path, file_name))
        with open(p.pipfile_path, "w") as f:
            contents = """
# Pre comment
[packages]
six = {{path = "./artifacts/{}"}}
            """.format(
                file_name
            )
            f.write(contents.strip())
        c = p.pipenv("install")
        assert c.return_code == 0
        assert "six" in p.lockfile["default"]
<|endoftext|>


<commit_before># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Base class for host-driven test cases.

This test case is intended to serve as the base class for any host-driven
test cases. It is similar to the Python unitttest module in that test cases
inherit from this class and add methods which will be run as tests.

When a HostDrivenTestCase object is instantiated, its purpose is to run only one
test method in the derived class. The test runner gives it the name of the test
method the instance will run. The test runner calls SetUp with the device ID
which the test method will run against. The test runner runs the test method
itself, collecting the result, and calls TearDown.

Tests can perform arbitrary Python commands and asserts in test methods. Tests
that run instrumentation tests can make use of the _RunJavaTestFilters helper
function to trigger Java tests and convert results into a single host-driven
test result.
"""

import logging
import os
import time

from pylib import android_commands
from pylib import constants
from pylib import forwarder
from pylib import valgrind_tools
from pylib.base import base_test_result
from pylib.instrumentation import test_package
from pylib.instrumentation import test_result
from pylib.instrumentation import test_runner

# aka the parent of com.google.android
BASE_ROOT = 'src' + os.sep

class HostDrivenTestCase(object):
  """Base class for host-driven test cases."""

  _HOST_DRIVEN_TAG = 'HostDriven'

  def __init__(self, test_name, instrumentation_options=None):
    """Create a test case initialized to run |test_name|.

    Args:
      test_name: The name of the method to run as the test.
      instrumentation_options: An InstrumentationOptions object.
    """
    self.test_name = test_name
    class_name = self.__class__.__name__
    self.qualified_name = '%s.%s' % (class_name, self.test_name)
    # Use tagged_name when creating results, so that we can identify host-driven
    # tests in the overall results.
    self.tagged_name = '%s_%s' % (self._HOST_DRIVEN_TAG, self.qualified_name)

    self.instrumentation_options = instrumentation_options
    self.ports_to_forward = []
    self.has_forwarded_ports = False

  # TODO(bulach): make ports_to_forward not optional and move the Forwarder
  # mapping here.
  def SetUp(self, device, shard_index, push_deps,
            cleanup_test_files, ports_to_forward=[]):
    self.device_id = device
    self.shard_index = shard_index
    self.adb = android_commands.AndroidCommands(self.device_id)
    self.push_deps = push_deps
    self.cleanup_test_files = cleanup_test_files
    if ports_to_forward:
      self.ports_to_forward = ports_to_forward

  def TearDown(self):
    if self.ports_to_forward:
      forwarder.Forwarder.UnmapAllDevicePorts(self.adb)

  # TODO(craigdh): Remove GetOutDir once references have been removed
  # downstream.
  def GetOutDir(self):
    return constants.GetOutDirectory()

  def Run(self):
    logging.info('Running host-driven test: %s', self.tagged_name)
    # Get the test method on the derived class and execute it
    return getattr(self, self.test_name)()

  def __GetHostForwarderLog(self):
    return ('-- Begin Full HostForwarder log\n'
            '%s\n'
            '--End Full HostForwarder log\n' % forwarder.Forwarder.GetHostLog())

  def __StartForwarder(self):
    # Unmap any left over from previous test.
    forwarder.Forwarder.UnmapAllDevicePorts(self.adb)
    logging.warning('Forwarding %s %s', self.ports_to_forward,
                    self.has_forwarded_ports)
    if self.ports_to_forward and not self.has_forwarded_ports:
      self.has_forwarded_ports = True
      tool = valgrind_tools.CreateTool(None, self.adb)
      forwarder.Forwarder.Map([(port, port) for port in self.ports_to_forward],
                              self.adb, tool)

  def __RunJavaTest(self, test, test_pkg, additional_flags=None):
    """Runs a single Java test in a Java TestRunner.

    Args:
      test: Fully qualified test name (ex. foo.bar.TestClass#testMethod)
      test_pkg: TestPackage object.
      additional_flags: A list of additional flags to add to the command line.

    Returns:
      TestRunResults object with a single test result.
    """
    # TODO(bulach): move this to SetUp() stage.
    self.__StartForwarder()

    java_test_runner = test_runner.TestRunner(self.instrumentation_options,
                                              self.device_id,
                                              self.shard_index, test_pkg,
                                              additional_flags=additional_flags)
    try:
      java_test_runner.SetUp()
      return java_test_runner.RunTest(test)[0]
    finally:
      java_test_runner.TearDown()

  def _RunJavaTestFilters(self, test_filters, additional_flags=None):
    """Calls a list of tests and stops at the first test failure.

    This method iterates until either it encounters a non-passing test or it
    exhausts the list of tests. Then it returns the appropriate overall result.

    Test cases may make use of this method internally to assist in running
    instrumentation tests. This function relies on instrumentation_options
    being defined.

    Args:
      test_filters: A list of Java test filters.
      additional_flags: A list of addition flags to add to the command line.

    Returns:
      A TestRunResults object containing an overall result for this set of Java
      tests. If any Java tests do not pass, this is a fail overall.
    """
    test_type = base_test_result.ResultType.PASS
    log = ''

    test_pkg = test_package.TestPackage(
        self.instrumentation_options.test_apk_path,
        self.instrumentation_options.test_apk_jar_path)

    start_ms = int(time.time()) * 1000
    done = False
    for test_filter in test_filters:
      tests = test_pkg._GetAllMatchingTests(None, None, test_filter)
      # Filters should always result in >= 1 test.
      if len(tests) == 0:
        raise Exception('Java test filter "%s" returned no tests.'
                        % test_filter)
      for test in tests:
        # We're only running one test at a time, so this TestRunResults object
        # will hold only one result.
        java_result = self.__RunJavaTest(test, test_pkg, additional_flags)
        assert len(java_result.GetAll()) == 1
        if not java_result.DidRunPass():
          result = java_result.GetNotPass().pop()
          log = result.GetLog()
          log += self.__GetHostForwarderLog()
          test_type = result.GetType()
          done = True
          break
      if done:
        break
    duration_ms = int(time.time()) * 1000 - start_ms

    overall_result = base_test_result.TestRunResults()
    overall_result.AddResult(
        test_result.InstrumentationTestResult(
            self.tagged_name, test_type, start_ms, duration_ms, log=log))
    return overall_result

  def __str__(self):
    return self.tagged_name

  def __repr__(self):
    return self.tagged_name
<commit_msg>Don't unmap ports in host_driven/test_case.py.<commit_after># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Base class for host-driven test cases.

This test case is intended to serve as the base class for any host-driven
test cases. It is similar to the Python unitttest module in that test cases
inherit from this class and add methods which will be run as tests.

When a HostDrivenTestCase object is instantiated, its purpose is to run only one
test method in the derived class. The test runner gives it the name of the test
method the instance will run. The test runner calls SetUp with the device ID
which the test method will run against. The test runner runs the test method
itself, collecting the result, and calls TearDown.

Tests can perform arbitrary Python commands and asserts in test methods. Tests
that run instrumentation tests can make use of the _RunJavaTestFilters helper
function to trigger Java tests and convert results into a single host-driven
test result.
"""

import logging
import os
import time

from pylib import android_commands
from pylib import constants
from pylib import forwarder
from pylib import valgrind_tools
from pylib.base import base_test_result
from pylib.instrumentation import test_package
from pylib.instrumentation import test_result
from pylib.instrumentation import test_runner

# aka the parent of com.google.android
BASE_ROOT = 'src' + os.sep

class HostDrivenTestCase(object):
  """Base class for host-driven test cases."""

  _HOST_DRIVEN_TAG = 'HostDriven'

  def __init__(self, test_name, instrumentation_options=None):
    """Create a test case initialized to run |test_name|.

    Args:
      test_name: The name of the method to run as the test.
      instrumentation_options: An InstrumentationOptions object.
    """
    self.test_name = test_name
    class_name = self.__class__.__name__
    self.qualified_name = '%s.%s' % (class_name, self.test_name)
    # Use tagged_name when creating results, so that we can identify host-driven
    # tests in the overall results.
    self.tagged_name = '%s_%s' % (self._HOST_DRIVEN_TAG, self.qualified_name)

    self.instrumentation_options = instrumentation_options
    self.ports_to_forward = []
    self.has_forwarded_ports = False

  # TODO(bulach): make ports_to_forward not optional and move the Forwarder
  # mapping here.
  def SetUp(self, device, shard_index, push_deps,
            cleanup_test_files, ports_to_forward=[]):
    self.device_id = device
    self.shard_index = shard_index
    self.adb = android_commands.AndroidCommands(self.device_id)
    self.push_deps = push_deps
    self.cleanup_test_files = cleanup_test_files
    if ports_to_forward:
      self.ports_to_forward = ports_to_forward

  def TearDown(self):
    pass

  # TODO(craigdh): Remove GetOutDir once references have been removed
  # downstream.
  def GetOutDir(self):
    return constants.GetOutDirectory()

  def Run(self):
    logging.info('Running host-driven test: %s', self.tagged_name)
    # Get the test method on the derived class and execute it
    return getattr(self, self.test_name)()

  def __GetHostForwarderLog(self):
    return ('-- Begin Full HostForwarder log\n'
            '%s\n'
            '--End Full HostForwarder log\n' % forwarder.Forwarder.GetHostLog())

  def __StartForwarder(self):
    logging.warning('Forwarding %s %s', self.ports_to_forward,
                    self.has_forwarded_ports)
    if self.ports_to_forward and not self.has_forwarded_ports:
      self.has_forwarded_ports = True
      tool = valgrind_tools.CreateTool(None, self.adb)
      forwarder.Forwarder.Map([(port, port) for port in self.ports_to_forward],
                              self.adb, tool)

  def __RunJavaTest(self, test, test_pkg, additional_flags=None):
    """Runs a single Java test in a Java TestRunner.

    Args:
      test: Fully qualified test name (ex. foo.bar.TestClass#testMethod)
      test_pkg: TestPackage object.
      additional_flags: A list of additional flags to add to the command line.

    Returns:
      TestRunResults object with a single test result.
    """
    # TODO(bulach): move this to SetUp() stage.
    self.__StartForwarder()

    java_test_runner = test_runner.TestRunner(self.instrumentation_options,
                                              self.device_id,
                                              self.shard_index, test_pkg,
                                              additional_flags=additional_flags)
    try:
      java_test_runner.SetUp()
      return java_test_runner.RunTest(test)[0]
    finally:
      java_test_runner.TearDown()

  def _RunJavaTestFilters(self, test_filters, additional_flags=None):
    """Calls a list of tests and stops at the first test failure.

    This method iterates until either it encounters a non-passing test or it
    exhausts the list of tests. Then it returns the appropriate overall result.

    Test cases may make use of this method internally to assist in running
    instrumentation tests. This function relies on instrumentation_options
    being defined.

    Args:
      test_filters: A list of Java test filters.
      additional_flags: A list of addition flags to add to the command line.

    Returns:
      A TestRunResults object containing an overall result for this set of Java
      tests. If any Java tests do not pass, this is a fail overall.
    """
    test_type = base_test_result.ResultType.PASS
    log = ''

    test_pkg = test_package.TestPackage(
        self.instrumentation_options.test_apk_path,
        self.instrumentation_options.test_apk_jar_path)

    start_ms = int(time.time()) * 1000
    done = False
    for test_filter in test_filters:
      tests = test_pkg._GetAllMatchingTests(None, None, test_filter)
      # Filters should always result in >= 1 test.
      if len(tests) == 0:
        raise Exception('Java test filter "%s" returned no tests.'
                        % test_filter)
      for test in tests:
        # We're only running one test at a time, so this TestRunResults object
        # will hold only one result.
        java_result = self.__RunJavaTest(test, test_pkg, additional_flags)
        assert len(java_result.GetAll()) == 1
        if not java_result.DidRunPass():
          result = java_result.GetNotPass().pop()
          log = result.GetLog()
          log += self.__GetHostForwarderLog()
          test_type = result.GetType()
          done = True
          break
      if done:
        break
    duration_ms = int(time.time()) * 1000 - start_ms

    overall_result = base_test_result.TestRunResults()
    overall_result.AddResult(
        test_result.InstrumentationTestResult(
            self.tagged_name, test_type, start_ms, duration_ms, log=log))
    return overall_result

  def __str__(self):
    return self.tagged_name

  def __repr__(self):
    return self.tagged_name
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Make a SQLite DB out of articles XML files
"""

import argparse
import datetime
import os
import random

from tqdm import tqdm
import sqlite3

from peewee import Model, CharField, ForeignKeyField, TextField, \
    DateTimeField, BooleanField, IntegerField, IntegrityError
from playhouse.sqlite_ext import SqliteExtDatabase

from .corpus import Corpus
from .transformations import filename_to_doi, convert_country
from . import starterdir
from .article import Article

journal_title_dict = {
    'PLOS ONE': 'PLOS ONE',
    'PLOS GENETICS': 'PLOS Genetics',
    'PLOS GENET': 'PLOS Genetics',
    'PLOS NEGLECTED TROPICAL DISEASES': 'PLOS Neglected Tropical Diseases',
    'PLOS NEGL TROP DIS': 'PLOS Neglected Tropical Diseases',
    'PLOS BIOLOGY': 'PLOS Biology',
    'PLOS BIOL': 'PLOS Biology',
    'PLOS CLINICAL TRIALS': 'PLOS Clinical Trials',
    'PLOS MEDICINE':'PLOS Medicine',
    'PLOS MEDICIN': 'PLOS Medicine',
    'PLOS MED': 'PLOS Medicine',
    'PLOS PATHOG': 'PLOS Pathogens',
    'PLOS PATHOGENS': 'PLOS Pathogens',
    'PLOS COMPUTATIONAL BIOLOGY': 'PLOS Computational Biology',
    'PLOS COMPUT BIOL': 'PLOS Computational Biology',
    'PLOS CLINICAL TRIALS': 'PLOS Clinical Trials',
}

parser = argparse.ArgumentParser()
parser.add_argument('--db', action='store', help=
                    'Name the db', default='ploscorpus.db')
parser.add_argument('--random', action='store', type=int, help=
                    'Number of articles in a random subset. '
                    'By default will use all articles')
parser.add_argument('--starterdb', action='store_true', help=
                    'Make the starter database', dest='starter')
args = parser.parse_args()

# TODO: Put a warning that the DB will be deleted
if os.path.isfile(args.db):
    os.remove(args.db)

if args.starter:
    if os.path.isfile('starter.db'):
        os.remove('starter.db')
    db = SqliteExtDatabase('starter.db')
else:
    db = SqliteExtDatabase(args.db)

class BaseModel(Model):
    class Meta:
        database = db

class Journal(BaseModel):
    journal = CharField(unique=True)

class ArticleType(BaseModel):
    article_type = CharField(unique=True)

class Country(BaseModel):
    country = CharField(unique=True)

class Affiliations(BaseModel):
    affiliations = CharField(unique=True)

class Subjects(BaseModel):
    subjects = CharField(unique=True)

class CorrespondingAuthor(BaseModel):
    corr_author_email = CharField(unique=True)
    tld = TextField(null=True)
    given_name = TextField(null=True)
    surname = TextField(null=True)
    group_name = TextField(null=True)
    affiliation = ForeignKeyField(Affiliations, related_name='aff')
    country = ForeignKeyField(Country, related_name='aff')

class JATSType(BaseModel):
    jats_type = CharField(unique=True)

class PLOSArticle(BaseModel):
    DOI = TextField(unique=True)
    abstract = TextField()
    title = TextField()
    plostype = ForeignKeyField(ArticleType, related_name='arttype')
    journal = ForeignKeyField(Journal, related_name='journals')
    created_date = DateTimeField(default=datetime.datetime.now)
    word_count = IntegerField()
    JATS_type = ForeignKeyField(JATSType, related_name='jats')

class SubjectsPLOSArticle(BaseModel):
    subject = ForeignKeyField(Subjects)
    article = ForeignKeyField(PLOSArticle)

class CoAuthorPLOSArticle(BaseModel):
    corr_author = ForeignKeyField(CorrespondingAuthor)
    article = ForeignKeyField(PLOSArticle)

db.connect()
db.create_tables([Journal, PLOSArticle, ArticleType, CoAuthorPLOSArticle,
                  CorrespondingAuthor, JATSType, Affiliations, Country,
                  SubjectsPLOSArticle, Subjects])

corpus_dir = starterdir if args.starter else None
allfiles = Corpus(corpus_dir).files
files = random.sample(allfiles, args.random) if args.random else allfiles

for file_ in tqdm(files):
    doi = filename_to_doi(file_)
    article = Article(doi)
    journal_name = journal_title_dict[article.journal.upper()]
    with db.atomic() as atomic:
        try:
            journal = Journal.create(journal = journal_name)
        except IntegrityError:
            db.rollback()
            journal = Journal.get(Journal.journal == journal_name)
    with db.atomic() as atomic:
        try:
            article_type = ArticleType.create(article_type = article.plostype)
        except IntegrityError:
            db.rollback()
            article_type = ArticleType.get(ArticleType.article_type == article.plostype)
    with db.atomic() as atomic:
        try:
            j_type = JATSType.create(jats_type = article.type_)
        except IntegrityError:
            db.rollback()
            j_type = JATSType.get(JATSType.jats_type == article.type_)
    p_art = PLOSArticle.create(
        DOI = doi,
        journal = journal,
        abstract=article.abstract.replace('\n', '').replace('\t', ''),
        title = article.title.replace('\n', '').replace('\t', ''),
        plostype = article_type,
        created_date = article.pubdate,
        word_count=article.word_count,
        JATS_type = j_type)
    # Get subject information
    taxonomy_set = set()
    taxonomy = article.taxonomy
    for values in taxonomy.values():
        for value in values:
            for taxon in value:
                taxonomy_set.add(taxon)
    for taxon in taxonomy_set:
        with db.atomic() as atomic:
            try:
                subject = Subjects.create(subjects = taxon)
            except (sqlite3.IntegrityError, IntegrityError):
                db.rollback()
                subject = Subjects.get(Subjects.subjects == taxon)
        SubjectsPLOSArticle.create(
                subject = subject,
                article = p_art
            )
    if article.authors:
        iterable_authors = article.authors
    else:
        iterable_authors = []
    for auths in iterable_authors:
        if auths['email']:
            with db.atomic() as atomic:
                if auths['affiliations']:
                    author_aff = auths['affiliations'][0]
                else:
                    author_aff = 'N/A'
                try:
                    aff = Affiliations.create(affiliations = author_aff)
                except (sqlite3.IntegrityError, IntegrityError):
                    db.rollback()
                    aff = Affiliations.get(Affiliations.affiliations ==
                                           author_aff)
            with db.atomic() as atomic:
                try:
                    if auths['affiliations'][0] == '':
                        country_from_aff = 'N/A'
                    else:
                        country_from_aff = auths['affiliations'][0].\
                                           split(',')[-1].strip()
                except IndexError:
                    country_from_aff = 'N/A'
                country_from_aff = convert_country(country_from_aff)
                try:
                    country = Country.create(country = country_from_aff)
                except IntegrityError:
                    db.rollback()
                    country = Country.get(Country.country == country_from_aff)

            try:
                co_author = CorrespondingAuthor.create(
                    corr_author_email = auths['email'][0],
                    tld = auths['email'][0].split('.')[-1],
                    given_name = auths['given_names'],
                    surname = auths['surname'],
                    group_name = auths['group_name'],
                    affiliation = aff,
                    country = country
                    )
            except IntegrityError:
                co_author = CorrespondingAuthor.\
                            get(CorrespondingAuthor.corr_author_email == auths['email'][0])
            coauthplosart = CoAuthorPLOSArticle.create(
                    corr_author = co_author,
                    article = p_art
                )
<commit_msg>Adding docstrings to tables/classes definition<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Make a SQLite DB out of articles XML files
"""

import argparse
import datetime
import os
import random

from tqdm import tqdm
import sqlite3

from peewee import Model, CharField, ForeignKeyField, TextField, \
    DateTimeField, BooleanField, IntegerField, IntegrityError
from playhouse.sqlite_ext import SqliteExtDatabase

from .corpus import Corpus
from .transformations import filename_to_doi, convert_country
from . import starterdir
from .article import Article

journal_title_dict = {
    'PLOS ONE': 'PLOS ONE',
    'PLOS GENETICS': 'PLOS Genetics',
    'PLOS GENET': 'PLOS Genetics',
    'PLOS NEGLECTED TROPICAL DISEASES': 'PLOS Neglected Tropical Diseases',
    'PLOS NEGL TROP DIS': 'PLOS Neglected Tropical Diseases',
    'PLOS BIOLOGY': 'PLOS Biology',
    'PLOS BIOL': 'PLOS Biology',
    'PLOS CLINICAL TRIALS': 'PLOS Clinical Trials',
    'PLOS MEDICINE':'PLOS Medicine',
    'PLOS MEDICIN': 'PLOS Medicine',
    'PLOS MED': 'PLOS Medicine',
    'PLOS PATHOG': 'PLOS Pathogens',
    'PLOS PATHOGENS': 'PLOS Pathogens',
    'PLOS COMPUTATIONAL BIOLOGY': 'PLOS Computational Biology',
    'PLOS COMPUT BIOL': 'PLOS Computational Biology',
    'PLOS CLINICAL TRIALS': 'PLOS Clinical Trials',
}

parser = argparse.ArgumentParser()
parser.add_argument('--db', action='store', help=
                    'Name the db', default='ploscorpus.db')
parser.add_argument('--random', action='store', type=int, help=
                    'Number of articles in a random subset. '
                    'By default will use all articles')
parser.add_argument('--starterdb', action='store_true', help=
                    'Make the starter database', dest='starter')
args = parser.parse_args()

# TODO: Put a warning that the DB will be deleted
if os.path.isfile(args.db):
    os.remove(args.db)

if args.starter:
    if os.path.isfile('starter.db'):
        os.remove('starter.db')
    db = SqliteExtDatabase('starter.db')
else:
    db = SqliteExtDatabase(args.db)

class BaseModel(Model):
    class Meta:
        database = db

class Journal(BaseModel):
    """
    Journal table stores journals values. It is used in the PLOSArticle table.
    Fields:
    journal: Journal name. Based on the values of journal_title_dict
    """
    journal = CharField(unique=True)

class ArticleType(BaseModel):
    """
    ArticleType table stores article types used in PLOSArticle table.
    Fields:
    article_type: Article type such as Research Article, Retraction,
      Essay, Perspective and others.
    """
    article_type = CharField(unique=True)

class Country(BaseModel):
    """
    Country table stores countries that are related with the corresponding
      author.
    Fields:
    country: Country.
    """
    country = CharField(unique=True)

class Affiliations(BaseModel):
    """
    Affiliations table stores affiliations that are related with the
      corresponding author.
    Fields:
    affiliations: Author affiliations such as "Department of Public Health,
      University of Helsinki, Finland".
    """
    affiliations = CharField(unique=True)

class Subjects(BaseModel):
    """
    Subjects table stores subjects that are related to PLOSArticle table by
      using a link table (since an article may have multiple subjects)
    Fields:
    subjects: Subjects like "Molecular Biology", "Cell processes" and so on.
    """
    subjects = CharField(unique=True)

class CorrespondingAuthor(BaseModel):
    """
    CorrespondingAuthor table stores the information of the corresponding
      author.
    Fields:
    corr_author_email: e-mail of the corresponding author
    tld: Top Level Domain, last part of the email, such as .ar, .es, .com.
    given_name: Given name of the author.
    surname: Surname of the author.
    group_name: Name of the group if a group is the author. If not, it is null.
    affiliation: Author affiliation, this is a linked field.
    country: Country of the author, this is a linked field.
    """
    corr_author_email = CharField(unique=True)
    tld = TextField(null=True)
    given_name = TextField(null=True)
    surname = TextField(null=True)
    group_name = TextField(null=True)
    affiliation = ForeignKeyField(Affiliations, related_name='aff')
    country = ForeignKeyField(Country, related_name='aff')

class JATSType(BaseModel):
    """
    JATSType table stores the article type using the JATS (Journal Article
      Tag Suite) standard (https://jats.nlm.nih.gov/).
    Fields:
    jats_type: JATS type, such as "research-article", "discussion", "editorial".
    """
    jats_type = CharField(unique=True)

class PLOSArticle(BaseModel):
    """
    PLOSArticle table is the main table of the database. It stores the articles.
    Fields:
    DOI: DOI for the article. For example: 10.1371/journal.pcbi.0030199
    abstract: Abstract for the article.
    title: Title for the article.
    plostype = Article type (internal PLOS classification), this is a linked
      field.
    journal = Journal name, this is a linked field.
    created_date = Article creation date. For example: 2006-04-11 00:00:00
    word_count = Amount of words in the article.
    JATS_type = Article type (JATS classification), this is a linked field.
    """
    DOI = TextField(unique=True)
    abstract = TextField()
    title = TextField()
    plostype = ForeignKeyField(ArticleType, related_name='arttype')
    journal = ForeignKeyField(Journal, related_name='journals')
    created_date = DateTimeField(default=datetime.datetime.now)
    word_count = IntegerField()
    JATS_type = ForeignKeyField(JATSType, related_name='jats')

class SubjectsPLOSArticle(BaseModel):
    """
    SubjectsPLOSArticle is a link table to relate subject with articles
    Fields:
    subject: This field is linked to Subjects table.
    article: This field is linked to PLOSArticle table.
    """
    subject = ForeignKeyField(Subjects)
    article = ForeignKeyField(PLOSArticle)

class CoAuthorPLOSArticle(BaseModel):
    """
    CoAuthorPLOSArticle is a link table to relate a co-author with a PLOS
      article.
    Fields:
    corr_author: This field is linked to CorrespondingAuthor table.
    article: This field is linked to PLOSArticle table.
    """
    corr_author = ForeignKeyField(CorrespondingAuthor)
    article = ForeignKeyField(PLOSArticle)

db.connect()
db.create_tables([Journal, PLOSArticle, ArticleType, CoAuthorPLOSArticle,
                  CorrespondingAuthor, JATSType, Affiliations, Country,
                  SubjectsPLOSArticle, Subjects])

corpus_dir = starterdir if args.starter else None
allfiles = Corpus(corpus_dir).files
files = random.sample(allfiles, args.random) if args.random else allfiles

for file_ in tqdm(files):
    doi = filename_to_doi(file_)
    article = Article(doi)
    journal_name = journal_title_dict[article.journal.upper()]
    with db.atomic() as atomic:
        try:
            journal = Journal.create(journal = journal_name)
        except IntegrityError:
            db.rollback()
            journal = Journal.get(Journal.journal == journal_name)
    with db.atomic() as atomic:
        try:
            article_type = ArticleType.create(article_type = article.plostype)
        except IntegrityError:
            db.rollback()
            article_type = ArticleType.get(ArticleType.article_type == article.plostype)
    with db.atomic() as atomic:
        try:
            j_type = JATSType.create(jats_type = article.type_)
        except IntegrityError:
            db.rollback()
            j_type = JATSType.get(JATSType.jats_type == article.type_)
    p_art = PLOSArticle.create(
        DOI = doi,
        journal = journal,
        abstract=article.abstract.replace('\n', '').replace('\t', ''),
        title = article.title.replace('\n', '').replace('\t', ''),
        plostype = article_type,
        created_date = article.pubdate,
        word_count=article.word_count,
        JATS_type = j_type)
    # Get subject information
    taxonomy_set = set()
    taxonomy = article.taxonomy
    for values in taxonomy.values():
        for value in values:
            for taxon in value:
                taxonomy_set.add(taxon)
    for taxon in taxonomy_set:
        with db.atomic() as atomic:
            try:
                subject = Subjects.create(subjects = taxon)
            except (sqlite3.IntegrityError, IntegrityError):
                db.rollback()
                subject = Subjects.get(Subjects.subjects == taxon)
        SubjectsPLOSArticle.create(
                subject = subject,
                article = p_art
            )
    if article.authors:
        iterable_authors = article.authors
    else:
        iterable_authors = []
    for auths in iterable_authors:
        if auths['email']:
            with db.atomic() as atomic:
                if auths['affiliations']:
                    author_aff = auths['affiliations'][0]
                else:
                    author_aff = 'N/A'
                try:
                    aff = Affiliations.create(affiliations = author_aff)
                except (sqlite3.IntegrityError, IntegrityError):
                    db.rollback()
                    aff = Affiliations.get(Affiliations.affiliations ==
                                           author_aff)
            with db.atomic() as atomic:
                try:
                    if auths['affiliations'][0] == '':
                        country_from_aff = 'N/A'
                    else:
                        country_from_aff = auths['affiliations'][0].\
                                           split(',')[-1].strip()
                except IndexError:
                    country_from_aff = 'N/A'
                country_from_aff = convert_country(country_from_aff)
                try:
                    country = Country.create(country = country_from_aff)
                except IntegrityError:
                    db.rollback()
                    country = Country.get(Country.country == country_from_aff)

            try:
                co_author = CorrespondingAuthor.create(
                    corr_author_email = auths['email'][0],
                    tld = auths['email'][0].split('.')[-1],
                    given_name = auths['given_names'],
                    surname = auths['surname'],
                    group_name = auths['group_name'],
                    affiliation = aff,
                    country = country
                    )
            except IntegrityError:
                co_author = CorrespondingAuthor.\
                            get(CorrespondingAuthor.corr_author_email == auths['email'][0])
            coauthplosart = CoAuthorPLOSArticle.create(
                    corr_author = co_author,
                    article = p_art
                )
<|endoftext|>