<commit_before># -*- coding: utf-8 -*-

from django.shortcuts import render_to_response, render, redirect
import util

def configure_actions(request):
    return render_to_response('apache/configure_actions.html')

def status(request):
    exception = None
    configuration = None

    try:
        configuration = util.get_apache_anomaly_detection_configuration()
    except IOError as e:
        exception = e.strerror

    return render_to_response("apache/status.html",
                              {'exception' : exception,
                               'configuration' : configuration,
                               })

def configure_anomaly_detection_select_agent_name(request):
    exception = None
    agent_names = None

    try:
        agent_names = util.get_apache_agent_names()
    except IOError as e:
        exception = e.strerror

    return render_to_response('apache/configure_anomaly_detection/select_agent_name.html',
                              {'exception' : exception,
                               'agent_names' : agent_names,
                               })

def configure_anomaly_detection_select_data_range(request):
    agent_name = request.GET.get('agent_name', '')
    virtualhost_name = request.GET.get('virtualhost_name', '')
    exception = None

    return render_to_response('apache/configure_anomaly_detection/select_data_range.html',
                              {'exception' : exception,
                               'agent_name' : agent_name,
                               'virtualhost_name' : virtualhost_name,
                               })

def configure_anomaly_detection_select_virtualhost(request):
    agent_name = request.GET.get('agent_name', '')
    exception = None
    virtualhosts_names = None

    try:
        virtualhosts_names = util.get_apache_virtualhost_names(agent_name)
    except IOError as e:
        exception = e.strerror

    return render_to_response('apache/configure_anomaly_detection/select_virtualhost.html',
                              {'exception' : exception,
                               'agent_name' : agent_name,
                               'virtualhosts_names' : virtualhosts_names,
                               })

def configure_anomaly_detection_correct_sessions_marks(request):
    exception = None
    sessions = []
    agent_name = request.GET.get('agent_name', '')
    virtualhost_name = request.GET.get('virtualhost_name', '')
    begin_date = request.GET.get('begin-date', '')
    end_date = request.GET.get('end-date', '')

    try:
        sessions = util.get_apache_sessions(agent_name, virtualhost_name, begin_date, end_date)
        util.set_apache_anomaly_detection_configuration(agent_name, virtualhost_name, begin_date, end_date)
    except IOError as e:
        exception = e.strerror

    return render(request,
                  'apache/configure_anomaly_detection/correct_sessions_marks.html',
                  {'exception' : exception,
                   'agent_name' : agent_name,
                   'virtualhost_name' : virtualhost_name,
                   'begin_date' : begin_date,
                   'end_date' : end_date,
                   'sessions' : sessions,
                   })

def configure_anomaly_detection_save_settings(request):
    exception = None
    agent_name = request.POST.get('agent_name', '')
    virtualhost_name = request.POST.get('virtualhost_name', '')
    begin_date = request.POST.get('begin-date', '')
    end_date = request.POST.get('end-date', '')
    all_rows_ids = request.POST.getlist('rows_ids')
    anomalies = request.POST.getlist('checks')

    try:
        util.set_apache_sessions_as_anomaly(agent_name,
                                            virttualhost_name,
                                            [int(i) for i in all_rows_ids],
                                            [int(i) for i in anomalies])
    except IOError as e:
        exception = e.strerror

    if exception:
        return redirect('apache:configure_anomaly_detection_correct_sessions_marks',
                        permanent=False,
                        args=(agent_name, virtualhost_name, begin_date, end_date, exception))
    else:
        return redirect('apache:status', permanent=False)
<commit_msg>Fix virtualhost variable name typo<commit_after># -*- coding: utf-8 -*-

from django.shortcuts import render_to_response, render, redirect
import util

def configure_actions(request):
    return render_to_response('apache/configure_actions.html')

def status(request):
    exception = None
    configuration = None

    try:
        configuration = util.get_apache_anomaly_detection_configuration()
    except IOError as e:
        exception = e.strerror

    return render_to_response("apache/status.html",
                              {'exception' : exception,
                               'configuration' : configuration,
                               })

def configure_anomaly_detection_select_agent_name(request):
    exception = None
    agent_names = None

    try:
        agent_names = util.get_apache_agent_names()
    except IOError as e:
        exception = e.strerror

    return render_to_response('apache/configure_anomaly_detection/select_agent_name.html',
                              {'exception' : exception,
                               'agent_names' : agent_names,
                               })

def configure_anomaly_detection_select_data_range(request):
    agent_name = request.GET.get('agent_name', '')
    virtualhost_name = request.GET.get('virtualhost_name', '')
    exception = None

    return render_to_response('apache/configure_anomaly_detection/select_data_range.html',
                              {'exception' : exception,
                               'agent_name' : agent_name,
                               'virtualhost_name' : virtualhost_name,
                               })

def configure_anomaly_detection_select_virtualhost(request):
    agent_name = request.GET.get('agent_name', '')
    exception = None
    virtualhosts_names = None

    try:
        virtualhosts_names = util.get_apache_virtualhost_names(agent_name)
    except IOError as e:
        exception = e.strerror

    return render_to_response('apache/configure_anomaly_detection/select_virtualhost.html',
                              {'exception' : exception,
                               'agent_name' : agent_name,
                               'virtualhosts_names' : virtualhosts_names,
                               })

def configure_anomaly_detection_correct_sessions_marks(request):
    exception = None
    sessions = []
    agent_name = request.GET.get('agent_name', '')
    virtualhost_name = request.GET.get('virtualhost_name', '')
    begin_date = request.GET.get('begin-date', '')
    end_date = request.GET.get('end-date', '')

    try:
        sessions = util.get_apache_sessions(agent_name, virtualhost_name, begin_date, end_date)
        util.set_apache_anomaly_detection_configuration(agent_name, virtualhost_name, begin_date, end_date)
    except IOError as e:
        exception = e.strerror

    return render(request,
                  'apache/configure_anomaly_detection/correct_sessions_marks.html',
                  {'exception' : exception,
                   'agent_name' : agent_name,
                   'virtualhost_name' : virtualhost_name,
                   'begin_date' : begin_date,
                   'end_date' : end_date,
                   'sessions' : sessions,
                   })

def configure_anomaly_detection_save_settings(request):
    exception = None
    agent_name = request.POST.get('agent_name', '')
    virtualhost_name = request.POST.get('virtualhost_name', '')
    begin_date = request.POST.get('begin-date', '')
    end_date = request.POST.get('end-date', '')
    all_rows_ids = request.POST.getlist('rows_ids')
    anomalies = request.POST.getlist('checks')

    try:
        util.set_apache_sessions_as_anomaly(agent_name,
                                            virtualhost_name,
                                            [int(i) for i in all_rows_ids],
                                            [int(i) for i in anomalies])
    except IOError as e:
        exception = e.strerror

    if exception:
        return redirect('apache:configure_anomaly_detection_correct_sessions_marks',
                        permanent=False,
                        args=(agent_name, virtualhost_name, begin_date, end_date, exception))
    else:
        return redirect('apache:status', permanent=False)
<|endoftext|>


<commit_before>from __future__ import print_function, division
import plac
import numpy
import time
from timeit import default_timer as timer
import dill as pickle

import spacy
from spacy.attrs import ORTH, LOWER, PREFIX, SUFFIX, SHAPE
from spacy.tokens.doc import Doc

from thinc.i2v import Embed, HashEmbed
from thinc.api import with_flatten, wrap
from thinc.t2t import BiLSTM
from thinc.v2v import Model, Maxout, Softmax

from thinc.api import layerize, chain, concatenate, clone, add
from thinc.api import with_getitem, flatten_add_lengths, with_square_sequences
from thinc.neural.util import flatten_sequences, remap_ids, to_categorical
from thinc.neural.optimizers import SGD
from thinc.neural.util import get_array_module

from thinc.extra.datasets import ancora_pos_tags

def FeatureExtracter(lang, attrs=[LOWER, SHAPE, PREFIX, SUFFIX], tokenized=True):
    nlp = spacy.blank(lang)
    nlp.vocab.lex_attr_getters[PREFIX] = lambda string: string[:3]
    nlp.vocab.lex_attr_getters[SUFFIX] = lambda string: string[-3:]
    def forward(texts, drop=0.):
        if tokenized:
            docs = [Doc(nlp.vocab, words) for words in texts]
        else:
            docs = [nlp(text) for text in texts]
        features = [doc.to_array(attrs) for doc in docs]
        def backward(d_features, sgd=None):
            return d_features
        return features, backward
    return layerize(forward)

epoch_train_acc = 0.
def track_progress(**context):
    model = context['model']
    dev_X = context['dev_X']
    dev_y = model.ops.flatten(context['dev_y'])
    n_train = context['n_train']
    trainer = context['trainer']
    n_dev = len(dev_y)
    epoch_times = [timer()]
    def each_epoch():
        global epoch_train_acc
        epoch_start = epoch_times[-1]
        epoch_end = timer()
        wps_train = n_train / (epoch_end-epoch_start)
        dev_start = timer()
        acc = model.evaluate(dev_X, dev_y)
        dev_end = timer()
        wps_run = n_dev / (dev_end-dev_start)
        stats = (acc, wps_train, wps_run)
        print("%.3f dev acc, %d wps train, %d wps run" % stats)
        epoch_train_acc = 0.
        epoch_times.append(timer())
    return each_epoch

def preprocess(ops, get_feats, data, nr_tag, npad=4):
    Xs, ys = zip(*data)
    Xs = [ops.asarray(x) for x in get_feats(Xs)]
    ys = [ops.asarray(to_categorical(y, nb_classes=nr_tag)) for y in ys]
    return Xs, ys

_i = 0
def debug(X, drop=0.):
    global _i
    if _i % 1000 == 0:
        print(X.mean(), X.var())
    _i += 1
    return X, lambda d, sgd: d

@plac.annotations(
    width=("Width of the hidden layers", "option", "w", int),
    vector_length=("Width of the word vectors", "option", "V", int),
    depth=("Depth of the hidden layers", "option", "d", int),
    min_batch_size=("Minimum minibatch size during training", "option", "b", int),
    max_batch_size=("Maximum minibatch size during training", "option", "B", int),
    learn_rate=("Learning rate", "option", "e", float),
    momentum=("Momentum", "option", "m", float),
    dropout=("Dropout rate", "option", "D", float),
    dropout_decay=("Dropout decay", "option", "C", float),
    nb_epoch=("Maximum passes over the training data", "option", "i", int),
    L2=("L2 regularization penalty", "option", "L", float),
)
def main(width=32, depth=1, vector_length=32,
         min_batch_size=16, max_batch_size=16, learn_rate=0.001,
         momentum=0.9, dropout=0.5, dropout_decay=1e-4,
         nb_epoch=20, L2=1e-6):
    cfg = dict(locals())
    print(cfg)
    train_data, check_data, nr_tag = ancora_pos_tags()

    extracter = FeatureExtracter('es', attrs=[LOWER, SHAPE, PREFIX, SUFFIX])
    with Model.define_operators({'**': clone, '>>': chain, '+': add,
                                 '|': concatenate}):
        lower_case = HashEmbed(width, 100, column=0)
        shape      = HashEmbed(width//2, 200, column=1)
        prefix     = HashEmbed(width//2, 100, column=2)
        suffix     = HashEmbed(width//2, 100, column=3)

        model = (
            with_flatten(
                (lower_case | shape | prefix | suffix)
                >> Maxout(width, pieces=3)
            )
            >> BiLSTM(width, width) ** depth
            >> with_flatten(Softmax(nr_tag))
        )

    train_X, train_y = preprocess(model.ops, extracter, train_data, nr_tag)
    dev_X, dev_y = preprocess(model.ops, extracter, check_data, nr_tag)

    n_train = float(sum(len(x) for x in train_X))
    global epoch_train_acc
    with model.begin_training(train_X[:10], train_y[:10], **cfg) as (trainer, optimizer):
        trainer.each_epoch.append(track_progress(**locals()))
        trainer.batch_size = min_batch_size
        batch_size = float(min_batch_size)
        for X, y in trainer.iterate(train_X, train_y):
            yh, backprop = model.begin_update(X, drop=trainer.dropout)

            gradient = [yh[i]-y[i] for i in range(len(yh))]

            backprop(gradient, optimizer)

            trainer.batch_size = min(int(batch_size), max_batch_size)
            batch_size *= 1.001
    print(model.evaluate(dev_X, model.ops.flatten(dev_y)))
    with open('/tmp/model.pickle', 'wb') as file_:
        pickle.dump(model, file_)

if __name__ == '__main__':
    if 1:
        plac.call(main)
    else:
        import cProfile
        import pstats
        cProfile.runctx("plac.call(main)", globals(), locals(), "Profile.prof")
        s = pstats.Stats("Profile.prof")
        s.strip_dirs().sort_stats("time").print_stats()
<commit_msg>Improve lstm_tagger example<commit_after>from __future__ import print_function, division
import plac
import numpy
import time
from timeit import default_timer as timer
import dill as pickle

import spacy
from spacy.attrs import ORTH, LOWER, PREFIX, SUFFIX, SHAPE
from spacy.tokens.doc import Doc

from thinc.i2v import Embed, HashEmbed
from thinc.api import with_flatten, wrap
from thinc.t2t import BiLSTM
from thinc.v2v import Model, Maxout, Softmax

from thinc.api import layerize, chain, concatenate, clone, add
from thinc.api import with_getitem, flatten_add_lengths, with_square_sequences
from thinc.neural.util import flatten_sequences, remap_ids, to_categorical
from thinc.neural.util import prefer_gpu
from thinc.neural.optimizers import SGD
from thinc.neural.util import get_array_module

from thinc.extra.datasets import ancora_pos_tags

def FeatureExtracter(lang, attrs=[LOWER, SHAPE, PREFIX, SUFFIX], tokenized=True):
    nlp = spacy.blank(lang)
    nlp.vocab.lex_attr_getters[PREFIX] = lambda string: string[:3]
    nlp.vocab.lex_attr_getters[SUFFIX] = lambda string: string[-3:]
    def forward(texts, drop=0.):
        if tokenized:
            docs = [Doc(nlp.vocab, words) for words in texts]
        else:
            docs = [nlp(text) for text in texts]
        features = [doc.to_array(attrs) for doc in docs]
        def backward(d_features, sgd=None):
            return d_features
        return features, backward
    return layerize(forward)

epoch_train_acc = 0.
def track_progress(**context):
    model = context['model']
    dev_X = context['dev_X']
    dev_y = model.ops.flatten(context['dev_y'])
    n_train = context['n_train']
    trainer = context['trainer']
    n_dev = len(dev_y)
    epoch_times = [timer()]
    def each_epoch():
        global epoch_train_acc
        epoch_start = epoch_times[-1]
        epoch_end = timer()
        wps_train = n_train / (epoch_end-epoch_start)
        dev_start = timer()
        acc = model.evaluate(dev_X, dev_y)
        dev_end = timer()
        wps_run = n_dev / (dev_end-dev_start)
        stats = (acc, wps_train, wps_run)
        print("%.3f dev acc, %d wps train, %d wps run" % stats)
        epoch_train_acc = 0.
        epoch_times.append(timer())
    return each_epoch

def preprocess(ops, get_feats, data, nr_tag, npad=4):
    Xs, ys = zip(*data)
    Xs = [ops.asarray(x) for x in get_feats(Xs)]
    ys = [ops.asarray(to_categorical(y, nb_classes=nr_tag)) for y in ys]
    return Xs, ys

_i = 0
def debug(X, drop=0.):
    global _i
    if _i % 1000 == 0:
        print(X.mean(), X.var())
    _i += 1
    return X, lambda d, sgd: d

@plac.annotations(
    width=("Width of the hidden layers", "option", "w", int),
    vector_length=("Width of the word vectors", "option", "V", int),
    depth=("Depth of the hidden layers", "option", "d", int),
    min_batch_size=("Minimum minibatch size during training", "option", "b", int),
    max_batch_size=("Maximum minibatch size during training", "option", "B", int),
    learn_rate=("Learning rate", "option", "e", float),
    momentum=("Momentum", "option", "m", float),
    dropout=("Dropout rate", "option", "D", float),
    dropout_decay=("Dropout decay", "option", "C", float),
    nb_epoch=("Maximum passes over the training data", "option", "i", int),
    L2=("L2 regularization penalty", "option", "L", float),
)
def main(width=128, depth=1, vector_length=128,
         min_batch_size=16, max_batch_size=16, learn_rate=0.001,
         momentum=0.9, dropout=0.5, dropout_decay=1e-4,
         nb_epoch=20, L2=1e-6):
    prefer_gpu()
    cfg = dict(locals())
    print(cfg)
    train_data, check_data, nr_tag = ancora_pos_tags()

    extracter = FeatureExtracter('es', attrs=[LOWER, SHAPE, PREFIX, SUFFIX])
    with Model.define_operators({'**': clone, '>>': chain, '+': add,
                                 '|': concatenate}):
        lower_case = HashEmbed(width, 100, column=0)
        shape      = HashEmbed(width//2, 200, column=1)
        prefix     = HashEmbed(width//2, 100, column=2)
        suffix     = HashEmbed(width//2, 100, column=3)

        model = (
            with_flatten(
                (lower_case | shape | prefix | suffix)
                >> Maxout(width, pieces=3)
            )
            >> BiLSTM(width, width) ** depth
            >> with_flatten(Softmax(nr_tag))
        )

    train_X, train_y = preprocess(model.ops, extracter, train_data, nr_tag)
    dev_X, dev_y = preprocess(model.ops, extracter, check_data, nr_tag)

    n_train = float(sum(len(x) for x in train_X))
    global epoch_train_acc
    with model.begin_training(train_X[:10], train_y[:10], **cfg) as (trainer, optimizer):
        trainer.each_epoch.append(track_progress(**locals()))
        trainer.batch_size = min_batch_size
        batch_size = float(min_batch_size)
        for X, y in trainer.iterate(train_X, train_y):
            yh, backprop = model.begin_update(X, drop=trainer.dropout)

            gradient = [yh[i]-y[i] for i in range(len(yh))]

            backprop(gradient, optimizer)

            trainer.batch_size = min(int(batch_size), max_batch_size)
            batch_size *= 1.001
    print(model.evaluate(dev_X, model.ops.flatten(dev_y)))
    with open('/tmp/model.pickle', 'wb') as file_:
        pickle.dump(model, file_)

if __name__ == '__main__':
    if 1:
        plac.call(main)
    else:
        import cProfile
        import pstats
        cProfile.runctx("plac.call(main)", globals(), locals(), "Profile.prof")
        s = pstats.Stats("Profile.prof")
        s.strip_dirs().sort_stats("time").print_stats()
<|endoftext|>


<commit_before>"""
Unit tests for padding functions
"""
import pytest
import numpy as np
import xarray as xr
import numpy.testing as npt

from ..padding import pad_coordinates

@pytest.fixture
def coords():
    """
    Define a dictionary with sample coordinates
    """
    x = np.linspace(-4, 5, 10)
    y = np.linspace(-1, 4, 11)
    coords = {
        "x": xr.DataArray(x, coords={"x": x}, dims=("x",)),
        "y": xr.DataArray(y, coords={"y": y}, dims=("y",)),
    }
    return coords

def test_pad_coordinates(coords):
    """
    Test pad_coordinates function
    """
    # Pad a single coordinate
    padded_coords = pad_coordinates(coords, {"x": 3})
    npt.assert_allclose(padded_coords["x"], np.linspace(-7, 8, 16))
    npt.assert_allclose(padded_coords["y"], coords["y"])
    # Pad two coordinates
    padded_coords = pad_coordinates(coords, {"x": 2, "y": 3})
    npt.assert_allclose(padded_coords["x"], np.linspace(-6, 7, 14))
    npt.assert_allclose(padded_coords["y"], np.linspace(-2.5, 5.5, 17))
    # Pad a single coordinate asymmetrically
    padded_coords = pad_coordinates(coords, {"x": (3, 2)})
    npt.assert_allclose(padded_coords["x"], np.linspace(-7, 7, 15))
    npt.assert_allclose(padded_coords["y"], coords["y"])
    # Pad two coordinates assymetrically
    padded_coords = pad_coordinates(coords, {"x": (2, 1), "y": (3, 4)})
    npt.assert_allclose(padded_coords["x"], np.linspace(-6, 6, 13))
    npt.assert_allclose(padded_coords["y"], np.linspace(-2.5, 6, 18))

def test_pad_coordinates_invalid(coords):
    """
    Test if pad_coordinates raises error after unevenly spaced coords
    """
    x = coords["x"]
    x[3] += 0.1
    with pytest.raises(ValueError):
        pad_coordinates(coords, pad_width={"x": 2})
<commit_msg>Add tests for pad function<commit_after>"""
Unit tests for padding functions
"""
import pytest
import numpy as np
import xarray as xr
import numpy.testing as npt

from ..padding import pad, pad_coordinates

@pytest.fixture
def coords():
    """
    Define a dictionary with sample coordinates
    """
    x = np.linspace(-4, 5, 10)
    y = np.linspace(-1, 4, 11)
    coords = {
        "x": xr.DataArray(x, coords={"x": x}, dims=("x",)),
        "y": xr.DataArray(y, coords={"y": y}, dims=("y",)),
    }
    return coords

@pytest.fixture
def sample_da_2d():
    """
    Defines a 2D sample xarray.DataArray
    """
    x = np.linspace(0, 10, 11)
    y = np.linspace(-4, 4, 17)
    z = np.arange(11 * 17, dtype=float).reshape(17, 11)
    return xr.DataArray(z, coords={"x": x, "y": y}, dims=("y", "x"))

def test_pad_coordinates(coords):
    """
    Test pad_coordinates function
    """
    # Pad a single coordinate
    padded_coords = pad_coordinates(coords, {"x": 3})
    npt.assert_allclose(padded_coords["x"], np.linspace(-7, 8, 16))
    npt.assert_allclose(padded_coords["y"], coords["y"])
    # Pad two coordinates
    padded_coords = pad_coordinates(coords, {"x": 2, "y": 3})
    npt.assert_allclose(padded_coords["x"], np.linspace(-6, 7, 14))
    npt.assert_allclose(padded_coords["y"], np.linspace(-2.5, 5.5, 17))
    # Pad a single coordinate asymmetrically
    padded_coords = pad_coordinates(coords, {"x": (3, 2)})
    npt.assert_allclose(padded_coords["x"], np.linspace(-7, 7, 15))
    npt.assert_allclose(padded_coords["y"], coords["y"])
    # Pad two coordinates assymetrically
    padded_coords = pad_coordinates(coords, {"x": (2, 1), "y": (3, 4)})
    npt.assert_allclose(padded_coords["x"], np.linspace(-6, 6, 13))
    npt.assert_allclose(padded_coords["y"], np.linspace(-2.5, 6, 18))

def test_pad_coordinates_invalid(coords):
    """
    Test if pad_coordinates raises error after unevenly spaced coords
    """
    x = coords["x"]
    x[3] += 0.1
    with pytest.raises(ValueError):
        pad_coordinates(coords, pad_width={"x": 2})

def test_pad_with_kwargs(sample_da_2d):
    """
    Test pad function by passing pad_width as kwargs
    """
    # Check if the array is padded with nans by default
    padded_da = pad(sample_da_2d, x=2, y=1)
    assert padded_da.shape == (19, 15)
    npt.assert_allclose(padded_da.values[:1, :], np.nan)
    npt.assert_allclose(padded_da.values[-1:, :], np.nan)
    npt.assert_allclose(padded_da.values[:, :2], np.nan)
    npt.assert_allclose(padded_da.values[:, -2:], np.nan)
    npt.assert_allclose(padded_da.values[1:-1, 2:-2], sample_da_2d)
    npt.assert_allclose(padded_da.x, np.linspace(-2, 12, 15))
    npt.assert_allclose(padded_da.y, np.linspace(-4.5, 4.5, 19))

def test_pad_with_pad_width(sample_da_2d):
    """
    Test pad function by passing pad_width as argument
    """
    # Check if the array is padded with nans by default
    pad_width = {"x": (2, 3), "y": (1, 3)}
    padded_da = pad(sample_da_2d, pad_width)
    assert padded_da.shape == (21, 16)
    npt.assert_allclose(padded_da.values[:1, :], np.nan)
    npt.assert_allclose(padded_da.values[-3:, :], np.nan)
    npt.assert_allclose(padded_da.values[:, :2], np.nan)
    npt.assert_allclose(padded_da.values[:, -3:], np.nan)
    npt.assert_allclose(padded_da.values[1:-3, 2:-3], sample_da_2d)
    npt.assert_allclose(padded_da.x, np.linspace(-2, 13, 16))
    npt.assert_allclose(padded_da.y, np.linspace(-4.5, 5.5, 21))
<|endoftext|>


<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This script is used to synthesize generated parts of this library."""

import synthtool as s
import synthtool.gcp as gcp

gapic = gcp.DiscoGAPICGenerator()

library = gapic.java_library(
    service='compute',
    version='v1',
    config_path='artman_compute.yaml',
    artman_output_name='')

s.copy(library / 'gapic-google-cloud-compute-v1/src', 'src')
<commit_msg>Add lint test and format generated code (#4114)<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This script is used to synthesize generated parts of this library."""

import synthtool as s
import synthtool.gcp as gcp
import synthtool.languages.java as java

gapic = gcp.DiscoGAPICGenerator()

service = 'compute'
versions = ['v1']
config_pattern = '/gapic/google/compute/artman_compute.yaml'

for version in versions:
  library = gapic.java_library(
      service=service,
      version=version,
      config_path=config_pattern.format(version=version),
      artman_output_name='')

  s.copy(library / f'gapic-google-cloud-{service}-{version}/src', 'src')

  java.format_code('./src')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# http://wsgi.tutorial.codepoint.net/

import os
import sys
import re
import inspect
from functools import wraps
from urlparse import parse_qs
from urllib import urlencode
from jinja2 import Environment, FileSystemLoader

"""
error page
debug report
"""

"""
support methods
HEAD
OPTIONS
"""

"""
Authentication fills:
AUTH_TYPE
REMOTE_USER

Authorization returns groups based on REMOTE_USER
SQL 'WITH RECURSIVE' query

Every controller lists authorized groups (maybe allow/deny ?) in decorator
and returns
'401 Unauthorized' if not logged in OR offer authentication
'403 Forbidden' if logged in and denied
"""

def default_error_handler(environ, start_response, status):
    output = 'E R R O R'
    output = output.encode('utf-8')
    headers = [('Content-Type', 'text/plain;charset=UTF-8'),
               ('Content-Length', str(len(output)))]
    start_response(status, headers, sys.exc_info())
    return [output]

def env(environ, start_response):
    output = []
    for k in environ.keys():
        output.append("=".join([k, str(environ[k])]))
    output = "\n".join(output).encode('utf-8')
    status = "200 OK"
    headers = [('Content-Type', 'text/plain;charset=UTF-8'),
               ('Content-Length', str(len(output)))]
    start_response(status, headers, sys.exc_info())
    return [output]

class Router(object):
    def __init__(self, *args):
        self.routes = []
        for route in args:
            self.append(*route)

    def __call__(self, environ, start_response):
        try:
            error_handler = environ['ERROR_HANDLER']
        except KeyError:
            error_handler = default_error_handler
        m = self.pattern.match(environ['PATH_INFO'])
        if m:
            index = m.end()
            if index == 0:
                environ['PATH_INFO'] += '/'
            else:
                if environ['PATH_INFO'][index - 1] == '/':
                    index -= 1
            environ['SCRIPT_NAME'] += environ['PATH_INFO'][:index]
            environ['PATH_INFO'] = environ['PATH_INFO'][index:]
            route = self.routes[m.lastindex - 1][1]
            if len(m.groupdict()) > 0:
                try:
                    d = parse_qs(environ['ARGUMENT_STRING'])
                except KeyError:
                    d = {}
                d.update((k, v) for k, v in m.groupdict().iteritems() if v is not None)
                environ['ARGUMENT_STRING'] = urlencode(d, True)
            try:
                controller = route[environ['REQUEST_METHOD']]
            except KeyError:
                return error_handler(environ, start_response, '405 Method Not Allowed')
            output = controller(environ, start_response)
            if output is not None:
                return output
            else:
                return error_handler(environ, start_response, '404 Not Found')
        else:
            return error_handler(environ, start_response, '404 Not Found')

    def append(self, app, url, methods=['GET']):
        route = next((i for i in self.routes if i[0] == url), None)
        if route is None:
            route = (url, {})
            patt = re.compile('({0})'.format(route[0]))
            for _ in xrange(patt.groups):
                self.routes.append(route)
        for method in methods:
            if method in route[1]:
                print >> sys.stderr, \
                    'Route url={} method={}, {} is overriden with {}'.format(
                        route[0], method, route[1][method].__name__, app.__name__)
            route[1][method] = app
        routes = []
        rl = None
        for r in self.routes:
            if r != rl:
                routes.append('({0})'.format(r[0]))
            rl = r
        self.pattern = re.compile('|'.join(routes))

    def route(self, url, method=None):
        assert isinstance(url, basestring), "route decorator needs url parameter"

        def decorate(function):
            if method is None:
                self.append(function, url)
            else:
                self.append(function, url, method)
            return function

        return decorate

    def __str__(self):
        ret = []
        for r in self.routes:
            for m, ro in r[1].iteritems():
                if isinstance(ro, Router):
                    for rr in ro.__str__().split("\n"):
                        ret.append(r[0] + rr)
                else:
                    ret.append(r[0] + " " + m)
        return "\n".join(ret)

class Static(object):
    block_size = 1024
    types = {'.ico': 'image/x-icon',
             '.gif': 'image/gif',
             '.jpg': 'image/jpeg',
             '.jpeg': 'image/jpeg',
             '.png': 'image/png',
             '.svg': 'image/svg+xml',
             '.js': 'application/javascript',
             '.otf': 'application/font-sfnt',
             '.eot': 'application/vnd.ms-fontobject',
             '.ttf': 'application/font-ttf',
             '.woff': 'application/font-woff',
             '.woff2': 'application/font-woff2',
             '.css': 'text/css;charset=UTF-8',
             '.html': 'text/html;charset=UTF-8',
             }

    def __init__(self, path):
        self.path = path

    def __call__(self, environ, start_response):
        path = environ['PATH_INFO']
        if path != '':
            filename = os.path.join(self.path, *environ['PATH_INFO'].split('/'))
        else:
            filename = self.path
        try:
            fin = open(filename, "rb")
            size = os.path.getsize(filename)
            status = '200 OK'
            extension = os.path.splitext(filename)[1]
            headers = [('Content-Type', self.types[extension]),
                       ('Content-Length', str(size))]
            start_response(status, headers)
            if 'wsgi.file_wrapper' in environ:
                return environ['wsgi.file_wrapper'](fin, self.block_size)
            else:
                return iter(lambda: fin.read(self.block_size), '')
        except IOError:
            return

def redirect(start_response, status, url, headers=None):
    if headers is None:
        headers = []
    output = 'R E D I R E C T'
    output = output.encode('utf-8')
    headers.append(('Location', url))
    start_response(status, headers)
    return [output]

class Redirect(object):
    def __init__(self, url):
        self.url = url

    def __call__(self, environ, start_response):
        output = 'R E D I R E C T'
        output = output.encode('utf-8')
        url = self.url
        if len(environ['QUERY_STRING']) > 0:
            url += "?" + environ['QUERY_STRING']
        start_response('301 Moved Permanently', [('Location', url),
                                                 ('Content-type', 'text/plain'),
                                                 ('Content-Length', str(len(output)))])
        return [output]

class Template(object):
    def __init__(self, templates):
        scriptname = inspect.getfile(sys._getframe(1))
        scriptpath = os.path.dirname(os.path.realpath(scriptname))
        searchpath = os.path.realpath(os.path.join(scriptpath, templates))

        loader = FileSystemLoader(searchpath=searchpath)
        self.environment = Environment(loader=loader,
                                       trim_blocks=True, lstrip_blocks=True)

    def render_and_respond(self, start_response, template_name, status='200 OK',
                           content_type='text/html;charset=UTF-8', **kwargs):
        template = self.environment.get_template(template_name)
        output = template.render(**kwargs)

        output = output.encode('utf-8')
        headers = [('Content-Type', content_type),
                   ('Content-Length', str(len(output)))]
        start_response(status, headers)
        return [output]

    def render(self, template_name, **kwargs):
        template = self.environment.get_template(template_name)
        return template.render(**kwargs).encode('utf-8')

def parse_get_data(environ):
    return parse_qs(environ['QUERY_STRING'])

def parse_post_data(environ):
    try:
        request_body_size = int(environ.get('CONTENT_LENGTH', 0))
    except ValueError:
        request_body_size = 0
    return parse_qs(environ['wsgi.input'].read(request_body_size))

def get_client_address(environ):
    """ Get HTTP request address """
    try:
        return environ['HTTP_X_FORWARDED_FOR'].split(',')[-1].strip()
    except KeyError:
        return environ['REMOTE_ADDR']

def controller(a=None, m=None):
    content_type = 'text/html'
    if isinstance(a, basestring):
        content_type = a

    def decorate(f):
        @wraps(f)
        def ctrl(environ, start_response):
            try:
                error_handler = environ['ERROR_HANDLER']
            except KeyError:
                error_handler = default_error_handler
            data_get = parse_qs(environ['QUERY_STRING'])
            if environ['REQUEST_METHOD'] == 'POST':
                data_post = parse_post_data()
            try:
                data_url = parse_qs(environ['ARGUMENT_STRING'])
            except KeyError:
                data_url = {}
            args = {}
            defaults = f.__code__.co_argcount
            if f.__defaults__ is not None:
                defaults -= len(f.__defaults__)
            for i, arg in enumerate(f.__code__.co_varnames):
                if m is not None and arg in m:
                    args[arg] = environ[m[arg]]
                elif arg in data_url:
                    args[arg] = data_url[arg][0]
                elif environ['REQUEST_METHOD'] == 'POST' and arg in data_post:
                    args[arg] = data_post[arg][0]
                elif arg in data_get:
                    args[arg] = data_get[arg][0]
                elif i < defaults:
                    # Missing argument which is not default
                    return error_handler(environ, start_response, '404 Not Found')
            output = f(**args)
            if output is not None:
                start_response('200 OK', [('Content-type', content_type),
                                          ('Content-Length', str(len(output)))])
                return [output]
            else:
                return error_handler(environ, start_response, '404 Not Found')

        return ctrl

    if callable(a):
        return decorate(a)
    else:
        return decorate

class DBSession(object):
    def __init__(self, controller, session_obj):
        self.controller = controller
        self.session_obj = session_obj

    def __call__(self, environ, start_response):
        session = self.session_obj()
        environ['DB_SESSION'] = session
        try:
            output = self.controller(environ, start_response)
            session.commit()
        except:
            session.rollback()
            raise
        finally:
            session.close()
        return output

def dbsession(session_obj):
    def decorate(f):
        @wraps(f)
        def controller(environ, start_response):
            session = session_obj()
            environ['DB_SESSION'] = session
            try:
                output = f(environ, start_response)
                session.commit()
            except:
                session.rollback()
                raise
            finally:
                session.close()
            return output

        return controller

    return decorate

class Link(object):
    def __init__(self, text, url):
        self.text = text
        self.url = url

class Paging(object):
    def __init__(self, rows, perpage, page, link):
        """ rows in db table, perpage rows on one page, page to show"""
        self.perpage = perpage
        try:
            self.page = int(page)
        except (TypeError, ValueError):
            self.page = 1
        self.pages = rows / perpage + ((rows % perpage) > 0)
        self.link = link

    def in_range(self):
        return self.page >= 1 and self.page <= self.pages

    def limit(self):
        return self.perpage * (self.page - 1), self.perpage

    def _linky(self, x):
        return Link(x, None) if x == self.page else Link(x, '%s/%d' % (self.link, x) if x > 1 else self.link)

    def links(self):
        paging = [self._linky(1)]
        if self.page > 4:
            paging.append(Link('...', None))
        if self.page == 4:
            paging.append(self._linky(self.page - 2))
        if self.page - 1 > 1:
            paging.append(self._linky(self.page - 1))
        if self.page > 1 and self.page < self.pages:
            paging.append(self._linky(self.page))
        if self.page + 1 < self.pages:
            paging.append(self._linky(self.page + 1))
        if self.page == self.pages - 3:
            paging.append(self._linky(self.page + 2))
        if self.page < self.pages - 3:
            paging.append(Link('...', None))
        paging.append(self._linky(self.pages))
        return paging
<commit_msg>Add docstring for controller decorator<commit_after># -*- coding: utf-8 -*-
# http://wsgi.tutorial.codepoint.net/

import os
import sys
import re
import inspect
from functools import wraps
from urlparse import parse_qs
from urllib import urlencode
from jinja2 import Environment, FileSystemLoader

"""
error page
debug report
"""

"""
support methods
HEAD
OPTIONS
"""

"""
Authentication fills:
AUTH_TYPE
REMOTE_USER

Authorization returns groups based on REMOTE_USER
SQL 'WITH RECURSIVE' query

Every controller lists authorized groups (maybe allow/deny ?) in decorator
and returns
'401 Unauthorized' if not logged in OR offer authentication
'403 Forbidden' if logged in and denied
"""

def default_error_handler(environ, start_response, status):
    output = 'E R R O R'
    output = output.encode('utf-8')
    headers = [('Content-Type', 'text/plain;charset=UTF-8'),
               ('Content-Length', str(len(output)))]
    start_response(status, headers, sys.exc_info())
    return [output]

def env(environ, start_response):
    output = []
    for k in environ.keys():
        output.append("=".join([k, str(environ[k])]))
    output = "\n".join(output).encode('utf-8')
    status = "200 OK"
    headers = [('Content-Type', 'text/plain;charset=UTF-8'),
               ('Content-Length', str(len(output)))]
    start_response(status, headers, sys.exc_info())
    return [output]

class Router(object):
    def __init__(self, *args):
        self.routes = []
        for route in args:
            self.append(*route)

    def __call__(self, environ, start_response):
        try:
            error_handler = environ['ERROR_HANDLER']
        except KeyError:
            error_handler = default_error_handler
        m = self.pattern.match(environ['PATH_INFO'])
        if m:
            index = m.end()
            if index == 0:
                environ['PATH_INFO'] += '/'
            else:
                if environ['PATH_INFO'][index - 1] == '/':
                    index -= 1
            environ['SCRIPT_NAME'] += environ['PATH_INFO'][:index]
            environ['PATH_INFO'] = environ['PATH_INFO'][index:]
            route = self.routes[m.lastindex - 1][1]
            if len(m.groupdict()) > 0:
                try:
                    d = parse_qs(environ['ARGUMENT_STRING'])
                except KeyError:
                    d = {}
                d.update((k, v) for k, v in m.groupdict().iteritems() if v is not None)
                environ['ARGUMENT_STRING'] = urlencode(d, True)
            try:
                controller = route[environ['REQUEST_METHOD']]
            except KeyError:
                return error_handler(environ, start_response, '405 Method Not Allowed')
            output = controller(environ, start_response)
            if output is not None:
                return output
            else:
                return error_handler(environ, start_response, '404 Not Found')
        else:
            return error_handler(environ, start_response, '404 Not Found')

    def append(self, app, url, methods=['GET']):
        route = next((i for i in self.routes if i[0] == url), None)
        if route is None:
            route = (url, {})
            patt = re.compile('({0})'.format(route[0]))
            for _ in xrange(patt.groups):
                self.routes.append(route)
        for method in methods:
            if method in route[1]:
                print >> sys.stderr, \
                    'Route url={} method={}, {} is overriden with {}'.format(
                        route[0], method, route[1][method].__name__, app.__name__)
            route[1][method] = app
        routes = []
        rl = None
        for r in self.routes:
            if r != rl:
                routes.append('({0})'.format(r[0]))
            rl = r
        self.pattern = re.compile('|'.join(routes))

    def route(self, url, method=None):
        assert isinstance(url, basestring), "route decorator needs url parameter"

        def decorate(function):
            if method is None:
                self.append(function, url)
            else:
                self.append(function, url, method)
            return function

        return decorate

    def __str__(self):
        ret = []
        for r in self.routes:
            for m, ro in r[1].iteritems():
                if isinstance(ro, Router):
                    for rr in ro.__str__().split("\n"):
                        ret.append(r[0] + rr)
                else:
                    ret.append(r[0] + " " + m)
        return "\n".join(ret)

class Static(object):
    block_size = 1024
    types = {'.ico': 'image/x-icon',
             '.gif': 'image/gif',
             '.jpg': 'image/jpeg',
             '.jpeg': 'image/jpeg',
             '.png': 'image/png',
             '.svg': 'image/svg+xml',
             '.js': 'application/javascript',
             '.otf': 'application/font-sfnt',
             '.eot': 'application/vnd.ms-fontobject',
             '.ttf': 'application/font-ttf',
             '.woff': 'application/font-woff',
             '.woff2': 'application/font-woff2',
             '.css': 'text/css;charset=UTF-8',
             '.html': 'text/html;charset=UTF-8',
             }

    def __init__(self, path):
        self.path = path

    def __call__(self, environ, start_response):
        path = environ['PATH_INFO']
        if path != '':
            filename = os.path.join(self.path, *environ['PATH_INFO'].split('/'))
        else:
            filename = self.path
        try:
            fin = open(filename, "rb")
            size = os.path.getsize(filename)
            status = '200 OK'
            extension = os.path.splitext(filename)[1]
            headers = [('Content-Type', self.types[extension]),
                       ('Content-Length', str(size))]
            start_response(status, headers)
            if 'wsgi.file_wrapper' in environ:
                return environ['wsgi.file_wrapper'](fin, self.block_size)
            else:
                return iter(lambda: fin.read(self.block_size), '')
        except IOError:
            return

def redirect(start_response, status, url, headers=None):
    if headers is None:
        headers = []
    output = 'R E D I R E C T'
    output = output.encode('utf-8')
    headers.append(('Location', url))
    start_response(status, headers)
    return [output]

class Redirect(object):
    def __init__(self, url):
        self.url = url

    def __call__(self, environ, start_response):
        output = 'R E D I R E C T'
        output = output.encode('utf-8')
        url = self.url
        if len(environ['QUERY_STRING']) > 0:
            url += "?" + environ['QUERY_STRING']
        start_response('301 Moved Permanently', [('Location', url),
                                                 ('Content-type', 'text/plain'),
                                                 ('Content-Length', str(len(output)))])
        return [output]

class Template(object):
    def __init__(self, templates):
        scriptname = inspect.getfile(sys._getframe(1))
        scriptpath = os.path.dirname(os.path.realpath(scriptname))
        searchpath = os.path.realpath(os.path.join(scriptpath, templates))

        loader = FileSystemLoader(searchpath=searchpath)
        self.environment = Environment(loader=loader,
                                       trim_blocks=True, lstrip_blocks=True)

    def render_and_respond(self, start_response, template_name, status='200 OK',
                           content_type='text/html;charset=UTF-8', **kwargs):
        template = self.environment.get_template(template_name)
        output = template.render(**kwargs)

        output = output.encode('utf-8')
        headers = [('Content-Type', content_type),
                   ('Content-Length', str(len(output)))]
        start_response(status, headers)
        return [output]

    def render(self, template_name, **kwargs):
        template = self.environment.get_template(template_name)
        return template.render(**kwargs).encode('utf-8')

def parse_get_data(environ):
    return parse_qs(environ['QUERY_STRING'])

def parse_post_data(environ):
    try:
        request_body_size = int(environ.get('CONTENT_LENGTH', 0))
    except ValueError:
        request_body_size = 0
    return parse_qs(environ['wsgi.input'].read(request_body_size))

def get_client_address(environ):
    """ Get HTTP request address """
    try:
        return environ['HTTP_X_FORWARDED_FOR'].split(',')[-1].strip()
    except KeyError:
        return environ['REMOTE_ADDR']

def controller(a=None, m=None):
    """ Serve GET, POST and RegEx data as function arguments

        without parentheses uses default values
        a, alternative content-type
        m, dictionary where keys are decorated function arguments and values are environment variables
    """
    content_type = 'text/html'
    if isinstance(a, basestring):
        content_type = a

    def decorate(f):
        @wraps(f)
        def ctrl(environ, start_response):
            try:
                error_handler = environ['ERROR_HANDLER']
            except KeyError:
                error_handler = default_error_handler
            data_get = parse_qs(environ['QUERY_STRING'])
            if environ['REQUEST_METHOD'] == 'POST':
                data_post = parse_post_data()
            try:
                data_url = parse_qs(environ['ARGUMENT_STRING'])
            except KeyError:
                data_url = {}
            args = {}
            defaults = f.__code__.co_argcount
            if f.__defaults__ is not None:
                defaults -= len(f.__defaults__)
            for i, arg in enumerate(f.__code__.co_varnames):
                if m is not None and arg in m:
                    args[arg] = environ[m[arg]]
                elif arg in data_url:
                    args[arg] = data_url[arg][0]
                elif environ['REQUEST_METHOD'] == 'POST' and arg in data_post:
                    args[arg] = data_post[arg][0]
                elif arg in data_get:
                    args[arg] = data_get[arg][0]
                elif i < defaults:
                    # Missing argument which is not default
                    return error_handler(environ, start_response, '404 Not Found')
            output = f(**args)
            if output is not None:
                start_response('200 OK', [('Content-type', content_type),
                                          ('Content-Length', str(len(output)))])
                return [output]
            else:
                return error_handler(environ, start_response, '404 Not Found')

        return ctrl

    if callable(a):
        return decorate(a)
    else:
        return decorate

class DBSession(object):
    def __init__(self, controller, session_obj):
        self.controller = controller
        self.session_obj = session_obj

    def __call__(self, environ, start_response):
        session = self.session_obj()
        environ['DB_SESSION'] = session
        try:
            output = self.controller(environ, start_response)
            session.commit()
        except:
            session.rollback()
            raise
        finally:
            session.close()
        return output

def dbsession(session_obj):
    def decorate(f):
        @wraps(f)
        def controller(environ, start_response):
            session = session_obj()
            environ['DB_SESSION'] = session
            try:
                output = f(environ, start_response)
                session.commit()
            except:
                session.rollback()
                raise
            finally:
                session.close()
            return output

        return controller

    return decorate

class Link(object):
    def __init__(self, text, url):
        self.text = text
        self.url = url

class Paging(object):
    def __init__(self, rows, perpage, page, link):
        """ rows in db table, perpage rows on one page, page to show"""
        self.perpage = perpage
        try:
            self.page = int(page)
        except (TypeError, ValueError):
            self.page = 1
        self.pages = rows / perpage + ((rows % perpage) > 0)
        self.link = link

    def in_range(self):
        return self.page >= 1 and self.page <= self.pages

    def limit(self):
        return self.perpage * (self.page - 1), self.perpage

    def _linky(self, x):
        return Link(x, None) if x == self.page else Link(x, '%s/%d' % (self.link, x) if x > 1 else self.link)

    def links(self):
        paging = [self._linky(1)]
        if self.page > 4:
            paging.append(Link('...', None))
        if self.page == 4:
            paging.append(self._linky(self.page - 2))
        if self.page - 1 > 1:
            paging.append(self._linky(self.page - 1))
        if self.page > 1 and self.page < self.pages:
            paging.append(self._linky(self.page))
        if self.page + 1 < self.pages:
            paging.append(self._linky(self.page + 1))
        if self.page == self.pages - 3:
            paging.append(self._linky(self.page + 2))
        if self.page < self.pages - 3:
            paging.append(Link('...', None))
        paging.append(self._linky(self.pages))
        return paging
<|endoftext|>


<commit_before>import pkg_resources
import os
import sys

class RequirementNotFoundError(Exception):
    pass

def collect_dependency_paths(package_name):
    """
    TODO docstrings
    """
    deps = []
    try:
        dist = pkg_resources.get_distribution(package_name)
    except (pkg_resources.DistributionNotFound, ValueError):
        message = "Distribution '{}' not found.".format(package_name)
        raise RequirementNotFoundError(message)

    if dist.has_metadata('top_level.txt'):
        for line in dist.get_metadata('top_level.txt').split():
            # do not consider subpackages (e.g. the form 'package/subpackage')
            if not os.path.split(line)[0]:
                pkg = os.path.join(dist.location, line)
                # handle single module packages
                if not os.path.isdir(pkg) and os.path.exists(pkg+'.py'):
                    pkg += '.py'
                deps.append(pkg)

    for req in dist.requires():
        deps.extend(collect_dependency_paths(req.project_name))

    return deps

def parse_requirements_file(req_file):
    """
    TODO docstrings
    """
    lines = []
    for line in req_file.readlines():
        line = line.strip()
        if not line or line.startswith('#'):
            continue
        lines.append(line)
    return lines

def make_simlinks(dest_dir, paths_list):
    """
    TODO docstrings
    """
    for path in paths_list:
        dest = os.path.join(dest_dir, os.path.split(path)[-1])
        if os.path.exists(dest):
            if os.path.islink(dest):
                os.remove(dest)
            else:
                sys.stderr.write('A file or dir named {} already exists, skipping...\n'.format(dest))
                continue
        os.symlink(path, dest)
<commit_msg>function returning code for appengine_config.py<commit_after>import pkg_resources
import os
import sys

class RequirementNotFoundError(Exception):
    pass

def collect_dependency_paths(package_name):
    """
    TODO docstrings
    """
    deps = []
    try:
        dist = pkg_resources.get_distribution(package_name)
    except (pkg_resources.DistributionNotFound, ValueError):
        message = "Distribution '{}' not found.".format(package_name)
        raise RequirementNotFoundError(message)

    if dist.has_metadata('top_level.txt'):
        for line in dist.get_metadata('top_level.txt').split():
            # do not consider subpackages (e.g. the form 'package/subpackage')
            if not os.path.split(line)[0]:
                pkg = os.path.join(dist.location, line)
                # handle single module packages
                if not os.path.isdir(pkg) and os.path.exists(pkg+'.py'):
                    pkg += '.py'
                deps.append(pkg)

    for req in dist.requires():
        deps.extend(collect_dependency_paths(req.project_name))

    return deps

def parse_requirements_file(req_file):
    """
    TODO docstrings
    """
    lines = []
    for line in req_file.readlines():
        line = line.strip()
        if not line or line.startswith('#'):
            continue
        lines.append(line)
    return lines

def make_simlinks(dest_dir, paths_list):
    """
    TODO docstrings
    """
    for path in paths_list:
        dest = os.path.join(dest_dir, os.path.split(path)[-1])
        if os.path.exists(dest):
            if os.path.islink(dest):
                os.remove(dest)
            else:
                sys.stderr.write('A file or dir named {} already exists, skipping...\n'.format(dest))
                continue
        os.symlink(path, dest)

def get_config_code(dependencies_root):
    """
    TODO docstrings
    """
    code = ("# code generated by Appengine Toolkit\n"
            "import sys\n"
            "import os\n"
            "sys.path.append(os.path.join(os.path.dirname(__file__), '{}'))\n"
            "# end generated code\n".format(dependencies_root))

    return code
<|endoftext|>


<commit_before>import os
import sys
import site
import glob
import tarfile
import subprocess

if sys.version_info >= (3, 0):
    import urllib.request as urllib
else:
    import urllib

def install(target_path, venv):
    """
    This function downloads the COMPSs installable from the specified repository and version
    and installs it. It is intended to be called from the setup.py script when we are installing.
    It also leaves a script on /etc/profile.d/compss.sh equivalent to the "compssenv" one on
    the supercomputers packages.

    Since this script calls another script which calls another script which calls... it may be
    possible that not all errors will be properly caught, leading to incomplete installations.
    :param target_path: <string> Destination folder
    :param venv: <boolean> True if installed within virtual environment. False on the contrary.
    :return: a list of strings (messages)
    """
    messages = []

    # download and decompress the COMPSs_version.tar.gz file
    base_url = open('url', 'r').read().rstrip()
    version_name = open('VERSION.txt', 'r').read().rstrip()
    tgz_name = 'COMPSs_{0}.tar.gz'.format(version_name)

    full_url = os.path.join(base_url, tgz_name)

    urllib.urlretrieve(full_url, tgz_name)

    tar = tarfile.open(tgz_name, 'r:gz')
    tar.extractall('.')

    os.remove(tgz_name)

    # ###################################################################### #
    # ######################## INSTALLATION ################################ #
    # ###################################################################### #

    # call the SuperComputers install script
    if subprocess.call(['./COMPSs/install', target_path]) != 0:
        raise Exception('COMPSs install script ended with some error. Please check stdout & stderr for more info.')

    pref = os.path.join(target_path, 'COMPSs')

    messages.append('*****************************************************')

    # ###################################################################### #
    # ################## SETUP ENVIRONMENT VARIABLES ####################### #
    # ###################################################################### #

    # create a script that defines some environment variables
    # if called as root and pip does not detect pycompss as an installed module
    # it will delete COMPSs
    try:
        site_packages = site.getsitepackages()[0]
    except AttributeError:
        # within a virtual environment
        from distutils.sysconfig import get_python_lib
        site_packages = get_python_lib()
    substitution_map = {
        '##VERSION##': str(sys.version_info[0])+'.'+str(sys.version_info[1]),
        '##SITE_PACKAGES##': site_packages,
        '##COMPSS_PATH##': pref,
        '##JAVA_HOME##': os.environ['JAVA_HOME']
    }
    s = open('pycompssenv', 'r').read()
    for (key, value) in list(substitution_map.items()):
        s = s.replace(key, value)
    try:
        # Try as sudo
        open('/etc/profile.d/compss.sh', 'w').write(s)
    except IOError:
        # Could not include system wide, then try to do it locally

        def update_export(sources_file, target_path):
            local_compss_sh = os.path.join(target_path, 'compss.sh')
            open(local_compss_sh, 'w').write(s)
            exports = 'source ' + str(local_compss_sh) + ' ##PyCOMPSs_EXPORTS##'
            messages.append('NOTE! ENVIRONMENT VARIABLES STORED IN %s' % local_compss_sh)
            if '##PyCOMPSs_EXPORTS##' in open(sources_file, 'r').read():
                # Update the existing line
                file_lines = open(sources_file, 'r').readlines()
                for i in range(len(file_lines)):
                    if '##PyCOMPSs_EXPORTS##' in file_lines[i]:
                        file_lines[i] = exports + '\n'
                open(sources_file, 'w').write(''.join(file_lines))
                messages.append('MESSAGE: Updated %s within %s' % (exports, sources_file))
            else:
                # Append the line:
                open(sources_file, 'a').write(exports)
                messages.append('MESSAGE: Added %s within %s' % (exports, sources_file))
                # TODO: INCLUDE ENVIRONMENT MANAGEMENT IF VENV WITHIN ACTIVATE
            messages.append('MESSAGE: Do not forget to source %s' % sources_file)

        if venv:
            # Add export to virtual environment activate
            update_export(os.path.join(os.environ['VIRTUAL_ENV'], 'bin', 'activate'), target_path)
        else:
            # Add export to .bashrc
            update_export(os.path.join(os.path.expanduser('~'), '.bashrc'), target_path)

    messages.append('*****************************************************')

    # ###################################################################### #
    # ###################### SETUP SYMBOLIK LINKS ########################## #
    # ###################################################################### #

    # create symbolic links
    def create_symlink(original_file, symbolic_place):
        try:
            os.remove(symbolic_place)
        except:
            pass
        os.symlink(original_file, symbolic_place)

    # create symbolic links to the python package contents
    original_compss_path = os.path.join(pref, 'Bindings', 'python', str(sys.version_info[0]), 'pycompss')
    for target_file in glob.iglob(os.path.join(original_compss_path, "*")):
        symbolic_place = os.path.join(target_path, os.path.split(target_file)[-1])
        create_symlink(target_file, symbolic_place)
        messages.append("SIMBOLIK LINK: From %s to %s" % (symbolic_place, target_file))

    # create symbolic links to the C extensions
    original_extensions_path = os.path.join(pref, 'Bindings', 'python', str(sys.version_info[0]))
    site_packages_path = os.path.split(target_path)[0]
    for target_file in glob.iglob(os.path.join(original_extensions_path, "*.so")):  # just .so files
        symbolic_place = os.path.join(site_packages_path, os.path.split(target_file)[-1])
        create_symlink(target_file, symbolic_place)
        messages.append("SIMBOLIK LINK: From %s to %s" % (symbolic_place, target_file))

    messages.append('*****************************************************')

    return messages<commit_msg>Unified .bashrc export label.<commit_after>import os
import sys
import site
import glob
import tarfile
import subprocess

if sys.version_info >= (3, 0):
    import urllib.request as urllib
else:
    import urllib

# Used for .bashrc export if necessary
EXPORT_LABEL = '##PyCOMPSs_EXPORTS___DO_NOT_REMOVE_THIS_LINE##'

def install(target_path, venv):
    """
    This function downloads the COMPSs installable from the specified repository and version
    and installs it. It is intended to be called from the setup.py script when we are installing.
    It also leaves a script on /etc/profile.d/compss.sh equivalent to the "compssenv" one on
    the supercomputers packages.

    Since this script calls another script which calls another script which calls... it may be
    possible that not all errors will be properly caught, leading to incomplete installations.
    :param target_path: <string> Destination folder
    :param venv: <boolean> True if installed within virtual environment. False on the contrary.
    :return: a list of strings (messages)
    """
    messages = []

    # download and decompress the COMPSs_version.tar.gz file
    base_url = open('url', 'r').read().rstrip()
    version_name = open('VERSION.txt', 'r').read().rstrip()
    tgz_name = 'COMPSs_{0}.tar.gz'.format(version_name)

    full_url = os.path.join(base_url, tgz_name)

    urllib.urlretrieve(full_url, tgz_name)

    tar = tarfile.open(tgz_name, 'r:gz')
    tar.extractall('.')

    os.remove(tgz_name)

    # ###################################################################### #
    # ######################## INSTALLATION ################################ #
    # ###################################################################### #

    # call the SuperComputers install script
    if subprocess.call(['./COMPSs/install', target_path]) != 0:
        raise Exception('COMPSs install script ended with some error. Please check stdout & stderr for more info.')

    pref = os.path.join(target_path, 'COMPSs')

    messages.append('*****************************************************')

    # ###################################################################### #
    # ################## SETUP ENVIRONMENT VARIABLES ####################### #
    # ###################################################################### #

    # create a script that defines some environment variables
    # if called as root and pip does not detect pycompss as an installed module
    # it will delete COMPSs
    try:
        site_packages = site.getsitepackages()[0]
    except AttributeError:
        # within a virtual environment
        from distutils.sysconfig import get_python_lib
        site_packages = get_python_lib()
    substitution_map = {
        '##VERSION##': str(sys.version_info[0])+'.'+str(sys.version_info[1]),
        '##SITE_PACKAGES##': site_packages,
        '##COMPSS_PATH##': pref,
        '##JAVA_HOME##': os.environ['JAVA_HOME']
    }
    s = open('pycompssenv', 'r').read()
    for (key, value) in list(substitution_map.items()):
        s = s.replace(key, value)
    try:
        # Try as sudo
        open('/etc/profile.d/compss.sh', 'w').write(s)
    except IOError:
        # Could not include system wide, then try to do it locally

        def update_export(sources_file, target_path):
            local_compss_sh = os.path.join(target_path, 'compss.sh')
            open(local_compss_sh, 'w').write(s)
            exports = 'source ' + str(local_compss_sh) + ' ' + EXPORT_LABEL
            messages.append('NOTE! ENVIRONMENT VARIABLES STORED IN %s' % local_compss_sh)
            if EXPORT_LABEL in open(sources_file, 'r').read():
                # Update the existing line
                file_lines = open(sources_file, 'r').readlines()
                for i in range(len(file_lines)):
                    if EXPORT_LABEL in file_lines[i]:
                        file_lines[i] = exports + '\n'
                open(sources_file, 'w').write(''.join(file_lines))
                messages.append('MESSAGE: Updated %s within %s' % (exports, sources_file))
            else:
                # Append the line:
                open(sources_file, 'a').write(exports)
                messages.append('MESSAGE: Added %s within %s' % (exports, sources_file))
                # TODO: INCLUDE ENVIRONMENT MANAGEMENT IF VENV WITHIN ACTIVATE
            messages.append('MESSAGE: Do not forget to source %s' % sources_file)

        if venv:
            # Add export to virtual environment activate
            update_export(os.path.join(os.environ['VIRTUAL_ENV'], 'bin', 'activate'), target_path)
        else:
            # Add export to .bashrc
            update_export(os.path.join(os.path.expanduser('~'), '.bashrc'), target_path)

    messages.append('*****************************************************')

    # ###################################################################### #
    # ###################### SETUP SYMBOLIK LINKS ########################## #
    # ###################################################################### #

    # create symbolic links
    def create_symlink(original_file, symbolic_place):
        try:
            os.remove(symbolic_place)
        except:
            pass
        os.symlink(original_file, symbolic_place)

    # create symbolic links to the python package contents
    original_compss_path = os.path.join(pref, 'Bindings', 'python', str(sys.version_info[0]), 'pycompss')
    for target_file in glob.iglob(os.path.join(original_compss_path, "*")):
        symbolic_place = os.path.join(target_path, os.path.split(target_file)[-1])
        create_symlink(target_file, symbolic_place)
        messages.append("SIMBOLIK LINK: From %s to %s" % (symbolic_place, target_file))

    # create symbolic links to the C extensions
    original_extensions_path = os.path.join(pref, 'Bindings', 'python', str(sys.version_info[0]))
    site_packages_path = os.path.split(target_path)[0]
    for target_file in glob.iglob(os.path.join(original_extensions_path, "*.so")):  # just .so files
        symbolic_place = os.path.join(site_packages_path, os.path.split(target_file)[-1])
        create_symlink(target_file, symbolic_place)
        messages.append("SIMBOLIK LINK: From %s to %s" % (symbolic_place, target_file))

    messages.append('*****************************************************')

    return messages<|endoftext|>


<commit_before>from Numeric import *
from pl import *

XPTS = 35
YPTS = 46
XSPA = 2./(XPTS-1)
YSPA = 2./(YPTS-1)
#polar plot data
PERIMETERPTS = 100
RPTS = 40
THETAPTS = 40

#potential plot data
PPERIMETERPTS = 100
PRPTS = 40
PTHETAPTS = 64

tr = array((XSPA, 0.0, -1.0, 0.0, YSPA, -1.0))

def mypltr(x, y):
    global tr
    result0 = tr[0] * x + tr[1] * y + tr[2]
    result1 = tr[3] * x + tr[4] * y + tr[5]
    return array((result0, result1))

def polar():
    #polar contour plot example.
    plenv(-1., 1., -1., 1., 0, -2,)
    plcol0(1)

    # Perimeter
    t = (2.*pi/(PERIMETERPTS-1))*arange(PERIMETERPTS)
    px = cos(t)
    py = sin(t)
    plline(px, py)

    # create data to be contoured.
    r = arange(RPTS)/float(RPTS-1)
    r.shape = (-1,1)
    theta = (2.*pi/float(THETAPTS-1))*arange(THETAPTS)
    xg = r*cos(theta)
    yg = r*sin(theta)
    zg = r*ones(THETAPTS)

    lev = 0.05 + 0.10*arange(10)

    plcol0(2)
    plcont( zg, lev, "pltr2", xg, yg, 2 )
    #                                 ^-- :-).  Means: "2nd coord is wrapped."
    plcol0(1)
    pllab("", "", "Polar Contour Plot")

def potential():
    #shielded potential contour plot example.

    # create data to be contoured.
    r = 0.5 + arange(PRPTS)
    r.shape = (-1,1)
    theta = (2.*pi/float(PTHETAPTS))*(0.5+arange(PTHETAPTS))
    xg = r*cos(theta)
    yg = r*sin(theta)

    rmax = max(r.flat)
    xmin = min(xg.flat)
    xmax = max(xg.flat)
    ymin = min(yg.flat)
    ymax = max(yg.flat)
    x0 = (xmin + xmax)/2.
    y0 = (ymin + ymax)/2.
    #perturbed (expanded) limits

    peps = 0.05
    xpmin = xmin - abs(xmin)*peps
    xpmax = xmax + abs(xmax)*peps
    ypmin = ymin - abs(ymin)*peps
    ypmax = ymax + abs(ymax)*peps

    # Potential inside a conducting cylinder (or sphere) by method of images.
    # Charge 1 is placed at (d1, d1), with image charge at (d2, d2).
    # Charge 2 is placed at (d1, -d1), with image charge at (d2, -d2).
    # Also put in smoothing term at small distances.

    eps = 2.

    q1 = 1.
    d1 = rmax/4.

    q1i = - q1*rmax/d1
    d1i = rmax**2/d1

    q2 = -1.
    d2 = rmax/4.

    q2i = - q2*rmax/d2
    d2i = rmax**2/d2

    div1 = sqrt((xg-d1)**2 + (yg-d1)**2 + eps**2)
    div1i = sqrt((xg-d1i)**2 + (yg-d1i)**2 + eps**2)

    div2 = sqrt((xg-d2)**2 + (yg+d2)**2 + eps**2)
    div2i = sqrt((xg-d2i)**2 + (yg+d2i)**2 + eps**2)

    zg = q1/div1 + q1i/div1i + q2/div2 + q2i/div2i

    zmin = min(zg.flat)
    zmax = max(zg.flat)
    print q1, d1, q1i, d1i, q2, d2, q2i, d2i
    print xmin, xmax, ymin, ymax, zmin, zmax

    # Positive and negative contour levels.
    nlevel = 20
    dz = (zmax-zmin)/float(nlevel)
    clevel = zmin + (arange(20)+0.5)*dz
    clevelpos = compress(clevel > 0., clevel)
    clevelneg = compress(clevel <= 0., clevel)

    #Colours!
    ncollin = 11
    ncolbox = 1
    ncollab = 2

    #Finally start plotting this page!
    pladv(0)
    plcol0(ncolbox)

    plvpas(0.1, 0.9, 0.1, 0.9, 1.0)
    plwind(xpmin, xpmax, ypmin, ypmax)
    plbox("", 0., 0, "", 0., 0)

    plcol0(ncollin)
    # Negative contours
    pllsty(2)
    plcont( zg, clevelneg, "pltr2", xg, yg, 2 )

    # Positive contours
    pllsty(1)
    plcont( zg, clevelpos, "pltr2", xg, yg, 2 )

    # Draw outer boundary
    t = (2.*pi/(PPERIMETERPTS-1))*arange(PPERIMETERPTS)
    px = x0 + rmax*cos(t)
    py = y0 + rmax*sin(t)

    plcol0(ncolbox)
    plline(px, py)

    plcol0(ncollab)
    pllab("", "", "Shielded potential of charges in a conducting sphere")

def main():

    mark = 1500
    space = 1500

    clevel = -1. + 0.2*arange(11)

    xx = (arrayrange(XPTS) - XPTS/2) / float((XPTS/2))
    yy = (arrayrange(YPTS) - YPTS/2) / float((YPTS/2)) - 1.
    xx.shape = (-1,1)
    z = (xx*xx)-(yy*yy)
    # 2.*outerproduct(xx,yy) for new versions of Numeric which have outerproduct.
    w = 2.*xx*yy

    # Set up grids.

    xg0 = zeros(XPTS,"double")
    yg0 = zeros(YPTS,"double")
    # Note *for the given* tr, mypltr(i,j)[0] is only a function of i
    # and mypltr(i,j)[1] is only function of j.
    for i in range(XPTS):
	xg0[i] = mypltr(i,0)[0]
    for j in range(YPTS):
	yg0[j] = mypltr(0,j)[1]

    distort = 0.4
    cos_x = cos((pi/2.)*xg0)
    cos_y = cos((pi/2.)*yg0)
    xg1 = xg0 + distort*cos_x
    yg1 = yg0 - distort*cos_y
    #Essential to copy (via slice) rather than assign by reference.
    xg0t = xg0[:]
    cos_x.shape = (-1,1)
    xg0t.shape = (-1,1)
    xg2 = xg0t + distort*cos_x*cos_y
    yg2 = yg0 - distort*cos_x*cos_y

    # Plot using mypltr (scaled identity) transformation used to create
    # xg0 and yg0
    pl_setcontlabelparam(0.006, 0.3, 0.1, 0)
    plenv(-1.0, 1.0, -1.0, 1.0, 0, 0)
    plcol0(2)
    plcont( z, clevel, "pltr1", xg0, yg0, 0 )
    plstyl( 1, mark, space )
    plcol0(3)
    plcont( w, clevel, "pltr1", xg0, yg0, 0 )
    plstyl( 0, mark, space )
    plcol0(1)
    pllab("X Coordinate", "Y Coordinate", "Streamlines of flow");

    pl_setcontlabelparam(0.006, 0.3, 0.1, 1)
    plenv(-1.0, 1.0, -1.0, 1.0, 0, 0)
    plcol0(2)
    plcont( z, clevel, "pltr1", xg0, yg0, 0 )
    plstyl( 1, mark, space )
    plcol0(3)
    plcont( w, clevel, "pltr1", xg0, yg0, 0 )
    plstyl( 0, mark, space )
    plcol0(1)
    pllab("X Coordinate", "Y Coordinate", "Streamlines of flow");

    # Plot using 1D coordinate transformation.
    pl_setcontlabelparam(0.006, 0.3, 0.1, 0)
    plenv(-1.0, 1.0, -1.0, 1.0, 0, 0)
    plcol0(2)
    plcont( z, clevel, "pltr1", xg1, yg1, 0 )
    plstyl( 1, mark, space )
    plcol0(3)
    plcont( w, clevel, "pltr1", xg1, yg1, 0 )
    plstyl( 0, mark, space )
    plcol0(1)
    pllab("X Coordinate", "Y Coordinate", "Streamlines of flow");

    pl_setcontlabelparam(0.006, 0.3, 0.1, 1)
    plenv(-1.0, 1.0, -1.0, 1.0, 0, 0)
    plcol0(2)
    plcont( z, clevel, "pltr1", xg1, yg1, 0 )
    plstyl( 1, mark, space )
    plcol0(3)
    plcont( w, clevel, "pltr1", xg1, yg1, 0 )
    plstyl( 0, mark, space )
    plcol0(1)
    pllab("X Coordinate", "Y Coordinate", "Streamlines of flow");

    # Plot using 2D coordinate transformation.
    pl_setcontlabelparam(0.006, 0.3, 0.1, 0)
    plenv(-1.0, 1.0, -1.0, 1.0, 0, 0)
    plcol0(2)
    plcont( z, clevel, "pltr2", xg2, yg2, 0 )
    plstyl( 1, mark, space )
    plcol0(3)
    plcont( w, clevel, "pltr2", xg2, yg2, 0 )
    plstyl( 0, mark, space )
    plcol0(1)
    pllab("X Coordinate", "Y Coordinate", "Streamlines of flow");

    pl_setcontlabelparam(0.006, 0.3, 0.1, 1)
    plenv(-1.0, 1.0, -1.0, 1.0, 0, 0)
    plcol0(2)
    plcont( z, clevel, "pltr2", xg2, yg2, 0 )
    plstyl( 1, mark, space )
    plcol0(3)
    plcont( w, clevel, "pltr2", xg2, yg2, 0 )
    plstyl( 0, mark, space )
    plcol0(1)
    pllab("X Coordinate", "Y Coordinate", "Streamlines of flow");

#   polar contour examples.
    pl_setcontlabelparam(0.006, 0.3, 0.1, 0)
    polar()
    pl_setcontlabelparam(0.006, 0.3, 0.1, 1)
    polar()

#   potential contour examples.
    pl_setcontlabelparam(0.006, 0.3, 0.1, 0)
    potential()
    pl_setcontlabelparam(0.006, 0.3, 0.1, 1)
    potential()

# Restore defaults
    plcol0(1)
    pl_setcontlabelparam(0.006, 0.3, 0.1, 0)

main()
<commit_msg>OOPS. comment out print statements<commit_after>from Numeric import *
from pl import *

XPTS = 35
YPTS = 46
XSPA = 2./(XPTS-1)
YSPA = 2./(YPTS-1)
#polar plot data
PERIMETERPTS = 100
RPTS = 40
THETAPTS = 40

#potential plot data
PPERIMETERPTS = 100
PRPTS = 40
PTHETAPTS = 64

tr = array((XSPA, 0.0, -1.0, 0.0, YSPA, -1.0))

def mypltr(x, y):
    global tr
    result0 = tr[0] * x + tr[1] * y + tr[2]
    result1 = tr[3] * x + tr[4] * y + tr[5]
    return array((result0, result1))

def polar():
    #polar contour plot example.
    plenv(-1., 1., -1., 1., 0, -2,)
    plcol0(1)

    # Perimeter
    t = (2.*pi/(PERIMETERPTS-1))*arange(PERIMETERPTS)
    px = cos(t)
    py = sin(t)
    plline(px, py)

    # create data to be contoured.
    r = arange(RPTS)/float(RPTS-1)
    r.shape = (-1,1)
    theta = (2.*pi/float(THETAPTS-1))*arange(THETAPTS)
    xg = r*cos(theta)
    yg = r*sin(theta)
    zg = r*ones(THETAPTS)

    lev = 0.05 + 0.10*arange(10)

    plcol0(2)
    plcont( zg, lev, "pltr2", xg, yg, 2 )
    #                                 ^-- :-).  Means: "2nd coord is wrapped."
    plcol0(1)
    pllab("", "", "Polar Contour Plot")

def potential():
    #shielded potential contour plot example.

    # create data to be contoured.
    r = 0.5 + arange(PRPTS)
    r.shape = (-1,1)
    theta = (2.*pi/float(PTHETAPTS))*(0.5+arange(PTHETAPTS))
    xg = r*cos(theta)
    yg = r*sin(theta)

    rmax = max(r.flat)
    xmin = min(xg.flat)
    xmax = max(xg.flat)
    ymin = min(yg.flat)
    ymax = max(yg.flat)
    x0 = (xmin + xmax)/2.
    y0 = (ymin + ymax)/2.
    #perturbed (expanded) limits

    peps = 0.05
    xpmin = xmin - abs(xmin)*peps
    xpmax = xmax + abs(xmax)*peps
    ypmin = ymin - abs(ymin)*peps
    ypmax = ymax + abs(ymax)*peps

    # Potential inside a conducting cylinder (or sphere) by method of images.
    # Charge 1 is placed at (d1, d1), with image charge at (d2, d2).
    # Charge 2 is placed at (d1, -d1), with image charge at (d2, -d2).
    # Also put in smoothing term at small distances.

    eps = 2.

    q1 = 1.
    d1 = rmax/4.

    q1i = - q1*rmax/d1
    d1i = rmax**2/d1

    q2 = -1.
    d2 = rmax/4.

    q2i = - q2*rmax/d2
    d2i = rmax**2/d2

    div1 = sqrt((xg-d1)**2 + (yg-d1)**2 + eps**2)
    div1i = sqrt((xg-d1i)**2 + (yg-d1i)**2 + eps**2)

    div2 = sqrt((xg-d2)**2 + (yg+d2)**2 + eps**2)
    div2i = sqrt((xg-d2i)**2 + (yg+d2i)**2 + eps**2)

    zg = q1/div1 + q1i/div1i + q2/div2 + q2i/div2i

    zmin = min(zg.flat)
    zmax = max(zg.flat)
    #print q1, d1, q1i, d1i, q2, d2, q2i, d2i
    #print xmin, xmax, ymin, ymax, zmin, zmax

    # Positive and negative contour levels.
    nlevel = 20
    dz = (zmax-zmin)/float(nlevel)
    clevel = zmin + (arange(20)+0.5)*dz
    clevelpos = compress(clevel > 0., clevel)
    clevelneg = compress(clevel <= 0., clevel)

    #Colours!
    ncollin = 11
    ncolbox = 1
    ncollab = 2

    #Finally start plotting this page!
    pladv(0)
    plcol0(ncolbox)

    plvpas(0.1, 0.9, 0.1, 0.9, 1.0)
    plwind(xpmin, xpmax, ypmin, ypmax)
    plbox("", 0., 0, "", 0., 0)

    plcol0(ncollin)
    # Negative contours
    pllsty(2)
    plcont( zg, clevelneg, "pltr2", xg, yg, 2 )

    # Positive contours
    pllsty(1)
    plcont( zg, clevelpos, "pltr2", xg, yg, 2 )

    # Draw outer boundary
    t = (2.*pi/(PPERIMETERPTS-1))*arange(PPERIMETERPTS)
    px = x0 + rmax*cos(t)
    py = y0 + rmax*sin(t)

    plcol0(ncolbox)
    plline(px, py)

    plcol0(ncollab)
    pllab("", "", "Shielded potential of charges in a conducting sphere")

def main():

    mark = 1500
    space = 1500

    clevel = -1. + 0.2*arange(11)

    xx = (arrayrange(XPTS) - XPTS/2) / float((XPTS/2))
    yy = (arrayrange(YPTS) - YPTS/2) / float((YPTS/2)) - 1.
    xx.shape = (-1,1)
    z = (xx*xx)-(yy*yy)
    # 2.*outerproduct(xx,yy) for new versions of Numeric which have outerproduct.
    w = 2.*xx*yy

    # Set up grids.

    xg0 = zeros(XPTS,"double")
    yg0 = zeros(YPTS,"double")
    # Note *for the given* tr, mypltr(i,j)[0] is only a function of i
    # and mypltr(i,j)[1] is only function of j.
    for i in range(XPTS):
	xg0[i] = mypltr(i,0)[0]
    for j in range(YPTS):
	yg0[j] = mypltr(0,j)[1]

    distort = 0.4
    cos_x = cos((pi/2.)*xg0)
    cos_y = cos((pi/2.)*yg0)
    xg1 = xg0 + distort*cos_x
    yg1 = yg0 - distort*cos_y
    #Essential to copy (via slice) rather than assign by reference.
    xg0t = xg0[:]
    cos_x.shape = (-1,1)
    xg0t.shape = (-1,1)
    xg2 = xg0t + distort*cos_x*cos_y
    yg2 = yg0 - distort*cos_x*cos_y

    # Plot using mypltr (scaled identity) transformation used to create
    # xg0 and yg0
    pl_setcontlabelparam(0.006, 0.3, 0.1, 0)
    plenv(-1.0, 1.0, -1.0, 1.0, 0, 0)
    plcol0(2)
    plcont( z, clevel, "pltr1", xg0, yg0, 0 )
    plstyl( 1, mark, space )
    plcol0(3)
    plcont( w, clevel, "pltr1", xg0, yg0, 0 )
    plstyl( 0, mark, space )
    plcol0(1)
    pllab("X Coordinate", "Y Coordinate", "Streamlines of flow");

    pl_setcontlabelparam(0.006, 0.3, 0.1, 1)
    plenv(-1.0, 1.0, -1.0, 1.0, 0, 0)
    plcol0(2)
    plcont( z, clevel, "pltr1", xg0, yg0, 0 )
    plstyl( 1, mark, space )
    plcol0(3)
    plcont( w, clevel, "pltr1", xg0, yg0, 0 )
    plstyl( 0, mark, space )
    plcol0(1)
    pllab("X Coordinate", "Y Coordinate", "Streamlines of flow");

    # Plot using 1D coordinate transformation.
    pl_setcontlabelparam(0.006, 0.3, 0.1, 0)
    plenv(-1.0, 1.0, -1.0, 1.0, 0, 0)
    plcol0(2)
    plcont( z, clevel, "pltr1", xg1, yg1, 0 )
    plstyl( 1, mark, space )
    plcol0(3)
    plcont( w, clevel, "pltr1", xg1, yg1, 0 )
    plstyl( 0, mark, space )
    plcol0(1)
    pllab("X Coordinate", "Y Coordinate", "Streamlines of flow");

    pl_setcontlabelparam(0.006, 0.3, 0.1, 1)
    plenv(-1.0, 1.0, -1.0, 1.0, 0, 0)
    plcol0(2)
    plcont( z, clevel, "pltr1", xg1, yg1, 0 )
    plstyl( 1, mark, space )
    plcol0(3)
    plcont( w, clevel, "pltr1", xg1, yg1, 0 )
    plstyl( 0, mark, space )
    plcol0(1)
    pllab("X Coordinate", "Y Coordinate", "Streamlines of flow");

    # Plot using 2D coordinate transformation.
    pl_setcontlabelparam(0.006, 0.3, 0.1, 0)
    plenv(-1.0, 1.0, -1.0, 1.0, 0, 0)
    plcol0(2)
    plcont( z, clevel, "pltr2", xg2, yg2, 0 )
    plstyl( 1, mark, space )
    plcol0(3)
    plcont( w, clevel, "pltr2", xg2, yg2, 0 )
    plstyl( 0, mark, space )
    plcol0(1)
    pllab("X Coordinate", "Y Coordinate", "Streamlines of flow");

    pl_setcontlabelparam(0.006, 0.3, 0.1, 1)
    plenv(-1.0, 1.0, -1.0, 1.0, 0, 0)
    plcol0(2)
    plcont( z, clevel, "pltr2", xg2, yg2, 0 )
    plstyl( 1, mark, space )
    plcol0(3)
    plcont( w, clevel, "pltr2", xg2, yg2, 0 )
    plstyl( 0, mark, space )
    plcol0(1)
    pllab("X Coordinate", "Y Coordinate", "Streamlines of flow");

#   polar contour examples.
    pl_setcontlabelparam(0.006, 0.3, 0.1, 0)
    polar()
    pl_setcontlabelparam(0.006, 0.3, 0.1, 1)
    polar()

#   potential contour examples.
    pl_setcontlabelparam(0.006, 0.3, 0.1, 0)
    potential()
    pl_setcontlabelparam(0.006, 0.3, 0.1, 1)
    potential()

# Restore defaults
    plcol0(1)
    pl_setcontlabelparam(0.006, 0.3, 0.1, 0)

main()
<|endoftext|>


<commit_before>#! python2
# xkcd.py - Downloads comics from xkcd.com.
#
# Takes 2 optional arguments when launched from the command line
# which in default are "download=latest" and
# "path=C:\XKCD"
#
# 'download' argument specifies whether to download the 'latest' or
# 'all' XKCD comics
# Using "latest" will download all the comics published since you
# the last comic you downloaded. If no prior comics have been downloaded,
# only the most recent comic will be downloaded.
#
#
# 'path' argument specifies the path where to save the comics

# Standard library modules
import os, sys

# Third-party modules
import requests, bs4

# Project-specific modules

def get_args():
	""" Function to read the system arguments and return a tuple of
	the 'path' and 'download' arguments respectively
	"""
	defaultpath = 'C:\\XKCD'
	path = defaultpath
	download = 'latest'

	# Checks whether the default path exists and creates it if
	# necessary.
	# The default path hosts a file named: 'pathfile' which stores
	# the path to the xkcd comics.
	if not os.path.lexists(path):
		os.mkdir(path)
		record_path(path, defaultpath)
	else:
		# Reading the path to the xkcd comics.
		os.chdir(path)
		with open('pathfile', 'rb') as f:
			path = f.readline()
		if not os.path.lexists(path):
			os.mkdir(path)

		os.chdir(path)

	# If the argument is only the file name, return the default
	# values.
	if len(sys.argv) == 1:
		return path, download

	else:
		for i in sys.argv[1:]:
			t = tuple(i.split('='))
			if t[0] == 'path':
				path = t[1]

				# Changes the path where xkcd comics are to be stored
				# as specified by the user comments.
				# Records the path to the file 'pathfile' for future
				#reference
				if not os.path.lexists(path):
					os.mkdir(path)
					record_path(defaultpath, path)
				else:
					record_path(defaultpath, path)
				continue

			elif t[0] == 'download':
				download = t[1]
				continue
		else:
			return path, download

def record_path(filepath, newpath):
	""" Function to record the path to the xkcd comics.

	filepath parameter specifies where the file:'pathfile' exists.
	This should be: 'C:\\XKCD' or the value in defaultpath

	newpath parameter specifies the path to where the xkcd comics are
	to be stored.
	"""

	os.chdir(filepath)
	with open('pathfile', 'wb') as f:
		f.write(newpath.encode('utf-8', 'replace'))

	os.chdir(newpath)

def main():

	path, download = get_args()

	# Opening the file that lists the xkcd comics
	# already downloaded.
	#
	# Format for writing statusfile:
	# Comic no.***Comic Title***status:
	# status include: 'Comic image not found', 'Error downloading'
	# and 'Success.'
	with open('xkcd', 'a+b') as statusfile:
		if download == 'latest':
			try:
				pass
			except Exception, e:
				print 'There was a problem: {0}'.format(str(e))

		elif download == 'all':
			try:
				download_comic(statusfile)
			except Exception, e:
				print 'There was a problem: {0}'.format(str(e))

def download_comic(statusfile, start='1', end='#'):
	""" Function to download the comics
	on the xkcd website
	"""

	status = ('Comic image not found', 'Error downloading', 'Success')
	url = 'http://xkcd.com/{0}'.format(start)
	while not url.endswith(end):

		# Getting the webpage
		res = requests.get(url)
		try:
			# Check whether the website is received successfully
			# and raises and exception if an error occurs
			res.raise_for_status()
		except Exception, e:
			print sys.exc_traceback.tb_lineno
			raise e

		soup = bs4.BeautifulSoup(res.text, "html.parser")

		# Getting the comic number and title
		comicno = os.path.split(url)[1]
		title = soup.select('#ctitle')[0].getText()

		# Getting the url for the image.
		comicElem = soup.select('#comic img')
		if comicElem == []:
			print 'Could not find comic image.'

			statusfile.write('{0}***{1}***{2} \n'\
				.format(comicno, title, status[0])\
				.encode('utf-8', 'replace'))

			# skip to the next link
			url = get_url(soup)
			continue
		else:
			try:
				comicUrl = 'http:{0}'.format(comicElem[0].get('src'))

				# Download the image.
				res = download_image(comicUrl)

			except requests.exceptions.MissingSchema:
				statusfile.write('{0}***{1}***{2} \n'\
					.format(comicno, title, status[1])\
					.encode('utf-8', 'replace'))

				# skip this comic
				url = get_url(soup)
				continue

		# Save the image to path
		save_image(res, statusfile,\
					 comicnumber=comicno,\
					 url=comicUrl,\
					 stat=status,\
					 comictitle=title)

		# Get the Prev button's url
		url = get_url(soup)

def get_url(soupobj, link='next'):
	""" This function returns a link to the previous or next comic

	It takes a BeautifulSoup object as an argument.

	Link argument takes either 'next' or 'prev'.
	"""

	nextLink = soupobj.select('a[rel="{0}"]'.format(link))[0]
	return 'http://xkcd.com{0}'.format(nextLink.get('href'))

def download_image(imgurl):
	""" This function downloads the image.

	It returns a requests object.
	"""

	print 'Downloading image {0}...'.format(os.path.basename(imgurl))
	res = requests.get(imgurl)
	try:
		res.raise_for_status()
	except Exception, e:
		print sys.exc_traceback.tb_lineno
		raise e

	return res

def save_image(req, statusfile, **stats):
	""" This function takes a requests object and a file object as
	parameters.

	The file object is used to keep a record of the image being saved.
	"""
	with open(os.path.basename(stats['url']), 'wb') as imageFile:
		for chunk in req.iter_content(100000):
				imageFile.write(chunk)

	statusfile.write('{0}***{1}***{2} \n'\
		.format(stats['comicnumber'], stats['comictitle'], stats['stat'][2])\
		.encode('utf-8', 'replace'))

if __name__ == '__main__':
	main()
<commit_msg>Interface Cleanup<commit_after>#! python2
# xkcd.py - Downloads comics from xkcd.com.
#
# Takes 2 optional arguments when launched from the command line
# which in default are "download=latest" and
# "path=C:\XKCD"
#
# 'download' argument specifies whether to download the 'latest' or
# 'all' XKCD comics
# Using "latest" will download all the comics published since you
# the last comic you downloaded. If no prior comics have been downloaded,
# only the most recent comic will be downloaded.
#
#
# 'path' argument specifies the path where to save the comics

# Standard library modules
import os, sys

# Third-party modules
import requests, bs4

# Project-specific modules

def get_args():
	""" Function to read the system arguments and return a tuple of
	the 'path' and 'download' arguments respectively
	"""
	defaultpath = 'C:\\XKCD'
	path = defaultpath
	download = 'latest'

	# Checks whether the default path exists and creates it if
	# necessary.
	# The default path hosts a file named: 'pathfile' which stores
	# the path to the xkcd comics.
	if not os.path.lexists(path):
		os.mkdir(path)
		record_path(path, defaultpath)
	else:
		# Reading the path to the xkcd comics.
		os.chdir(path)
		with open('pathfile', 'rb') as f:
			path = f.readline()
		if not os.path.lexists(path):
			os.mkdir(path)

		os.chdir(path)

	# If the argument is only the file name, return the default
	# values.
	if len(sys.argv) == 1:
		return path, download

	else:
		for i in sys.argv[1:]:
			t = tuple(i.split('='))
			if t[0] == 'path':
				path = t[1]

				# Changes the path where xkcd comics are to be stored
				# as specified by the user comments.
				# Records the path to the file 'pathfile' for future
				#reference
				if not os.path.lexists(path):
					os.mkdir(path)
					record_path(defaultpath, path)
				else:
					record_path(defaultpath, path)
				continue

			elif t[0] == 'download':
				download = t[1]
				continue
		else:
			return path, download

def record_path(filepath, newpath):
	""" Function to record the path to the xkcd comics.

	filepath parameter specifies where the file:'pathfile' exists.
	This should be: 'C:\\XKCD' or the value in defaultpath

	newpath parameter specifies the path to where the xkcd comics are
	to be stored.
	"""

	os.chdir(filepath)
	with open('pathfile', 'wb') as f:
		f.write(newpath.encode('utf-8', 'replace'))

	os.chdir(newpath)

def main():

	path, download = get_args()

	if download == 'latest':
		try:
			pass
		except Exception, e:
			print 'There was a problem: {0}'.format(str(e))

	elif download == 'all':
		try:
			download_comic()
		except Exception, e:
			print 'There was a problem: {0}'.format(str(e))

def download_comic(start='1', end='#'):
	""" Function to download the comics
	on the xkcd website
	"""

	status = ('Comic image not found', 'Error downloading', 'Success')
	url = 'http://xkcd.com/{0}'.format(start)
	while not url.endswith(end):

		# Getting the webpage
		res = requests.get(url)
		try:
			# Check whether the website is received successfully
			# and raises and exception if an error occurs
			res.raise_for_status()
		except Exception, e:
			print sys.exc_traceback.tb_lineno
			raise e

		soup = bs4.BeautifulSoup(res.text, "html.parser")

		# Getting the comic number and title
		comicno = os.path.split(url)[1]
		title = soup.select('#ctitle')[0].getText()

		# Getting the url for the image.
		comicElem = soup.select('#comic img')
		if comicElem == []:
			print 'Could not find comic image.'

			with open('xkcd', 'a+b') as fin:
				fin.write('{0}***{1}***{2} \n'\
				.format(comicno, title, status[0])\
				.encode('utf-8', 'replace'))

			# skip to the next link
			url = get_url(soup)
			continue
		else:
			try:
				comicUrl = 'http:{0}'.format(comicElem[0].get('src'))

				# Download the image.
				res = download_image(comicUrl)

			except requests.exceptions.MissingSchema:
				with open('xkcd', 'a+b') as fin:
					fin.write('{0}***{1}***{2} \n'\
					.format(comicno, title, status[1])\
					.encode('utf-8', 'replace'))

				# skip this comic
				url = get_url(soup)
				continue

		# Save the image to path
		save_image(res,\
					 comicnumber=comicno,\
					 url=comicUrl,\
					 stat=status,\
					 comictitle=title)

		# Get the Prev button's url
		url = get_url(soup)

def get_url(soupobj, link='next'):
	""" This function returns a link to the previous or next comic

	It takes a BeautifulSoup object as an argument.

	Link argument takes either 'next' or 'prev'.
	"""

	nextLink = soupobj.select('a[rel="{0}"]'.format(link))[0]
	return 'http://xkcd.com{0}'.format(nextLink.get('href'))

def download_image(imgurl):
	""" This function downloads the image.

	It returns a requests object.
	"""

	print 'Downloading image {0}...'.format(os.path.basename(imgurl))
	res = requests.get(imgurl)
	try:
		res.raise_for_status()
	except Exception, e:
		print sys.exc_traceback.tb_lineno
		raise e

	return res

def save_image(req, **stats):
	""" This function takes a requests object and a file object as
	parameters.

	The file object is used to keep a record of the image being saved.
	"""
	with open(os.path.basename(stats['url']), 'wb') as imageFile:
		for chunk in req.iter_content(100000):
				imageFile.write(chunk)

	with open('xkcd', 'a+b') as fin:
		fin.write('{0}***{1}***{2} \n'\
		.format(stats['comicnumber'], stats['comictitle'], stats['stat'][2])\
		.encode('utf-8', 'replace'))

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>#! /usr/bin/env python3
import os, random
os.system('curl -sL c.xkcd.com/random/comic/ > temp')
link = os.popen("grep 'Image URL' temp").read().split()[-1]

img = link.split('/')[-1]
os.system('wget -q ' + link + ' && ./imgcat ' + img)
os.system('rm temp ' + img)
<commit_msg>Described xkcd<commit_after>#! /usr/bin/env python3
# To display random comics from xkcd in iTerm2 and obviously you need iTerm2 for this
import os, random
os.system('curl -sL c.xkcd.com/random/comic/ > temp')
link = os.popen("grep 'Image URL' temp").read().split()[-1]

img = link.split('/')[-1]
os.system('wget -q ' + link + ' && ./imgcat ' + img)
os.system('rm temp ' + img)
<|endoftext|>


<commit_before>import argparse
import matplotlib.pyplot as plt

import chainer

from chainercv.datasets import camvid_label_colors
from chainercv.datasets import camvid_label_names
from chainercv.links import SegNetBasic
from chainercv import utils
from chainercv.visualizations import vis_image
from chainercv.visualizations import vis_semantic_segmentation

def main():
    chainer.config.train = False

    parser = argparse.ArgumentParser()
    parser.add_argument('--gpu', type=int, default=-1)
    parser.add_argument('--pretrained-model', default='camvid')
    parser.add_argument('image')
    args = parser.parse_args()

    model = SegNetBasic(
        n_class=len(camvid_label_names),
        pretrained_model=args.pretrained_model)

    if args.gpu >= 0:
        chainer.cuda.get_device_from_id(args.gpu).use()
        model.to_gpu()

    img = utils.read_image(args.image, color=True)
    labels = model.predict([img])
    label = labels[0]

    fig = plt.figure()
    ax1 = fig.add_subplot(1, 2, 1)
    vis_image(img, ax=ax1)
    ax2 = fig.add_subplot(1, 2, 2)
    # Do not overlay the label image on the color image
    vis_semantic_segmentation(
        None, label, camvid_label_names, camvid_label_colors, ax=ax2)
    plt.show()

if __name__ == '__main__':
    main()
<commit_msg>add --dataset arg in segnet/demo.py<commit_after>import argparse
import matplotlib.pyplot as plt

import chainer

from chainercv.datasets import camvid_label_colors
from chainercv.datasets import camvid_label_names
from chainercv.links import SegNetBasic
from chainercv import utils
from chainercv.visualizations import vis_image
from chainercv.visualizations import vis_semantic_segmentation

def main():
    chainer.config.train = False

    parser = argparse.ArgumentParser()
    parser.add_argument('--gpu', type=int, default=-1)
    parser.add_argument('--pretrained-model')
    parser.add_argument('--dataset', choices=('camvid'), default='camvid')
    parser.add_argument('image')
    args = parser.parse_args()

    if args.dataset == 'camvid':
        if args.pretrained_model is None:
            args.pretrained_model = 'camvid'
        label_names = camvid_label_names
        colors = camvid_label_colors

    model = SegNetBasic(
        n_class=len(label_names),
        pretrained_model=args.pretrained_model)

    if args.gpu >= 0:
        chainer.cuda.get_device_from_id(args.gpu).use()
        model.to_gpu()

    img = utils.read_image(args.image, color=True)
    labels = model.predict([img])
    label = labels[0]

    fig = plt.figure()
    ax1 = fig.add_subplot(1, 2, 1)
    vis_image(img, ax=ax1)
    ax2 = fig.add_subplot(1, 2, 2)
    # Do not overlay the label image on the color image
    vis_semantic_segmentation(None, label, label_names, colors, ax=ax2)
    plt.show()

if __name__ == '__main__':
    main()
<|endoftext|>