<commit_before>import re
import torch
import torch.nn as nn
import torch.nn.functional as F
from collections import Counter
from os import remove
from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.metrics import accuracy_score, make_scorer, roc_auc_score
from sklearn.model_selection import train_test_split as split
from sklearn.utils.class_weight import compute_sample_weight
from time import time
from torch.autograd import Variable
from torchtext.data import Dataset, Example, Field, Iterator, Pipeline

class CNNClassifier(BaseEstimator, ClassifierMixin):
    def __init__(self, lr=0.001, epochs=256, batch_size=64, test_interval=100,
                 early_stop=1000, save_best=True, dropout=0.5, max_norm=0.0,
                 embed_dim=128, kernel_num=100, kernel_sizes=(3, 4, 5),
                 static=False, device=-1, cuda=True, activation_func="relu",
                 scoring=make_scorer(accuracy_score), pos_label=None,
                 vectors=None, split_ratio=0.8, preprocessor=None,
                 class_weight=None, random_state=None, verbose=0):
        self.lr = lr
        self.epochs = epochs
        self.batch_size = batch_size
        self.test_interval = test_interval
        self.early_stop = early_stop
        self.save_best = save_best
        self.dropout = dropout
        self.max_norm = max_norm
        self.embed_dim = embed_dim
        self.kernel_num = kernel_num
        self.kernel_sizes = kernel_sizes
        self.static = static
        self.device = device
        self.cuda = cuda
        self.activation_func = activation_func
        self.scoring = scoring
        self.pos_label = pos_label
        self.vectors = vectors
        self.split_ratio = split_ratio
        self.preprocessor = preprocessor
        self.class_weight = class_weight
        self.random_state = random_state
        self.verbose = verbose

    def __clean_str(self, string):
        string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
        string = re.sub(r"\'s", " \'s", string)
        string = re.sub(r"\'ve", " \'ve", string)
        string = re.sub(r"n\'t", " n\'t", string)
        string = re.sub(r"\'re", " \'re", string)
        string = re.sub(r"\'d", " \'d", string)
        string = re.sub(r"\'ll", " \'ll", string)
        string = re.sub(r",", " , ", string)
        string = re.sub(r"!", " ! ", string)
        string = re.sub(r"\(", " ( ", string)
        string = re.sub(r"\)", " ) ", string)
        string = re.sub(r"\?", " ? ", string)
        string = re.sub(r"\s{2,}", " ", string)
        return string.strip()

    def __eval(self, data_iter):
        self.__model.eval()

        preds, targets = [], []
        softmax = nn.Softmax(dim=1) if self.scoring == "roc_auc" else None

        for batch in data_iter:
            feature, target = batch.text.data.t(), batch.label.data.sub(1)

            if self.cuda and torch.cuda.is_available():
                feature, target = feature.cuda(), target.cuda()

            logit = self.__model(feature)

            F.cross_entropy(logit, target, reduction="sum")

            if self.scoring == "roc_auc":
                pred = [[float(p) for p in dist] for dist in softmax(logit)]
            else:
                pred = torch.max(logit, 1)[1].view(target.size()).data.tolist()

            preds += pred
            targets += target.data.tolist()

        targets = [self.__label_field.vocab.itos[targ + 1] for targ in targets]

        if self.scoring == "roc_auc":
            pos_index = self.__label_field.vocab.stoi[self.pos_label] - 1
            return roc_auc_score(targets, [pred[pos_index] for pred in preds])

        preds = [self.__label_field.vocab.itos[pred + 1] for pred in preds]
        return self.scoring(_Eval(preds), None, targets)

    def fit(self, X, y, sample_weight=None):
        if self.random_state is not None:
            torch.manual_seed(self.random_state)

        torch.backends.cudnn.deterministic = self.random_state is not None
        torch.backends.cudnn.benchmark = self.random_state is None

        if self.verbose > 1:
            params = self.get_params().items()

            print("Fitting with the following parameters:")
            print("\n".join([": ".join([k, str(v)]) for k, v in params]))

        start = time() if self.verbose > 0 else None
        train_iter, dev_iter = self.__preprocess(X, y, sample_weight)
        embed_num = len(self.__text_field.vocab)
        class_num = len(self.__label_field.vocab) - 1
        self.__model = _CNNText(embed_num, self.embed_dim, class_num,
                                self.kernel_num, self.kernel_sizes,
                                self.dropout, self.static,
                                self.activation_func,
                                vectors=self.__text_field.vocab.vectors)

        if self.cuda and torch.cuda.is_available():
            torch.cuda.set_device(self.device)
            self.__model.cuda()

        optimizer = torch.optim.Adam(self.__model.parameters(), lr=self.lr,
                                     weight_decay=self.max_norm)
        steps, best_acc, last_step = 0, 0, 0
        active = True
        filename = "./{}.model".format(time())

        self.__model.train()

        for epoch in range(self.epochs):
            for batch in train_iter:
                feature, target = batch.text.data.t(), batch.label.data.sub(1)

                if self.cuda and torch.cuda.is_available():
                    feature, target = feature.cuda(), target.cuda()

                optimizer.zero_grad()
                F.cross_entropy(self.__model(feature), target).backward()
                optimizer.step()

                steps += 1

                if steps % self.test_interval == 0:
                    dev_acc = self.__eval(dev_iter)

                    if dev_acc > best_acc:
                        best_acc = dev_acc
                        last_step = steps

                        if self.save_best:
                            torch.save(self.__model.state_dict(), filename)
                    elif steps - last_step >= self.early_stop:
                        active = False
                        break

            if not active:
                break

        if self.save_best:
            self.__model.load_state_dict(torch.load(filename))
            remove(filename)

        self.classes_ = self.__label_field.vocab.itos[1:]

        if self.verbose > 0:
            self.__print_elapsed_time(time() - start)

        torch.cuda.empty_cache()
        return self

    def __predict(self, X):
        y_output = []
        max_kernel_size = max(self.kernel_sizes)

        self.__model.eval()

        for text in X:
            assert isinstance(text, str)

            text = self.__text_field.preprocess(text)
            text = self.__pad(text, max_kernel_size, True)
            text = [[self.__text_field.vocab.stoi[x] for x in text]]
            x = Variable(torch.tensor(text))
            x = x.cuda() if self.cuda and torch.cuda.is_available() else x

            y_output.append(self.__model(x))

        torch.cuda.empty_cache()
        return y_output

    def predict(self, X):
        y_pred = [torch.argmax(yi, 1) for yi in self.__predict(X)]
        return [self.__label_field.vocab.itos[yi.data[0] + 1] for yi in y_pred]

    def predict_proba(self, X):
        softmax = nn.Softmax(dim=1)
        y_prob = [softmax(yi) for yi in self.__predict(X)]
        return [[float(yij) for yij in yi[0]] for yi in y_prob]

    def __pad(self, x, max_kernel_size, preprocessed=False):
        tokens = x if preprocessed else self.__text_field.preprocess(x)
        difference = max_kernel_size - len(tokens)

        if difference > 0:
            padding = [self.__text_field.pad_token] * difference
            return x + padding if preprocessed else " ".join([x] + padding)

        return x

    def __preprocess(self, X, y, sample_weight):
        self.__text_field = Field(lower=True)
        self.__label_field = Field(sequential=False)
        self.__text_field.preprocessing = Pipeline(self.__preprocess_text)
        max_kernel_size = max(self.kernel_sizes)
        sample_weight = None if sample_weight is None else list(sample_weight)

        for i in range(len(X)):
            X[i] = self.__pad(X[i], max_kernel_size)

        sw = [1 for yi in y] if sample_weight is None else sample_weight
        s = y if Counter(y).most_common()[-1][1] > 1 else None
        X_t, X_d, y_t, y_d, w_t, _ = split(X, y, sw, shuffle=True, stratify=s,
                                           random_state=self.random_state,
                                           train_size=self.split_ratio)
        fields = [("text", self.__text_field), ("label", self.__label_field)]
        examples = [[X_t[i], y_t[i]] for i in range(len(X_t))]
        examples = [Example.fromlist(example, fields) for example in examples]
        weights = compute_sample_weight(self.class_weight, y_t)
        weights = [weights[i] * w_t[i] for i in range(len(y_t))]
        min_weight = min(weights)
        weights = [int(round(weight / min_weight)) for weight in weights]

        for i in range(len(X_t)):
            Xi = [X_t[i] for j in range(weights[i] - 1)]
            examples += [Example.fromlist([x, y_t[i]], fields) for x in Xi]

        train_data = Dataset(examples, fields)
        dev_data = [[X_d[i], y_d[i]] for i in range(len(X_d))]
        dev_data = [Example.fromlist(example, fields) for example in dev_data]
        dev_data = Dataset(dev_data, fields)

        self.__text_field.build_vocab(train_data, dev_data,
                                      vectors=self.vectors)
        self.__label_field.build_vocab(train_data, dev_data)

        batch_sizes = (self.batch_size, len(dev_data))
        return Iterator.splits((train_data, dev_data), batch_sizes=batch_sizes,
                               sort_key=lambda ex: len(ex.text), repeat=False)

    def __preprocess_text(self, text):
        if self.preprocessor is None:
            return self.__clean_str(text)

        return self.__clean_str(self.preprocessor(text))

    def __print_elapsed_time(self, seconds):
        sc = round(seconds)
        mn = int(sc / 60)
        sc = sc % 60
        hr = int(mn / 60)
        mn = mn % 60
        hr = "{} hour{}".format(hr, "s" if hr > 1 else "") if hr > 0 else ""
        mn = "{} minute{}".format(mn, "s" if mn > 1 else "") if mn > 0 else ""
        sc = "{} second{}".format(sc, "s" if sc > 1 else "") if sc > 0 else ""
        times = [t for t in [hr, mn, sc] if len(t) > 0]

        if len(times) == 3:
            times = " and ".join([", ".join([hr, mn]), sc])
        elif len(times) == 2:
            times = " and ".join(times)
        else:
            times = times[0] if len(times) > 0 else "less than 1 second"

        print("Completed training in {}.".format(times))

class _CNNText(nn.Module):
    def __init__(self, embed_num, embed_dim, class_num, kernel_num,
                 kernel_sizes, dropout, static, activation_func, vectors=None):
        super(_CNNText, self).__init__()

        if vectors is None:
            self.__embed = nn.Embedding(embed_num, embed_dim)
        else:
            self.__embed = nn.Embedding.from_pretrained(vectors)
            embed_dim = self.__embed.embedding_dim

        Ks = kernel_sizes
        module_list = [nn.Conv2d(1, kernel_num, (K, embed_dim)) for K in Ks]
        self.__convs1 = nn.ModuleList(module_list)
        self.__dropout = nn.Dropout(dropout)
        self.__fc1 = nn.Linear(len(Ks) * kernel_num, class_num)
        self.__static = static

        if activation_func == "relu":
            self.__f = F.relu
        elif activation_func == "tanh":
            self.__f = torch.tanh
        else:
            self.__f = lambda x: x

    def forward(self, x):
        x = Variable(self.__embed(x)) if self.__static else self.__embed(x)
        x = x.unsqueeze(1)
        x = [self.__f(conv(x), inplace=True).squeeze(3) for conv in self.__convs1]
        x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x]
        return self.__fc1(self.__dropout(torch.cat(x, 1)))

class _Eval():
    def __init__(self, preds):
        self.__preds = preds

    def predict(self, X):
        return self.__preds
<commit_msg>Compatability updates<commit_after>import re
import torch
import torch.nn as nn
import torch.nn.functional as F
from collections import Counter
from os import remove
from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.metrics import accuracy_score, make_scorer, roc_auc_score
from sklearn.model_selection import train_test_split as split
from sklearn.utils.class_weight import compute_sample_weight
from time import time
from torchtext.data import Dataset, Example, Field, Iterator, Pipeline

class CNNClassifier(BaseEstimator, ClassifierMixin):
    def __init__(self, lr=0.001, epochs=256, batch_size=64, test_interval=100,
                 early_stop=1000, save_best=True, dropout=0.5, max_norm=0.0,
                 embed_dim=128, kernel_num=100, kernel_sizes=(3, 4, 5),
                 static=False, device=-1, cuda=True, activation_func="relu",
                 scoring=make_scorer(accuracy_score), pos_label=None,
                 vectors=None, split_ratio=0.8, preprocessor=None,
                 class_weight=None, random_state=None, verbose=0):
        self.lr = lr
        self.epochs = epochs
        self.batch_size = batch_size
        self.test_interval = test_interval
        self.early_stop = early_stop
        self.save_best = save_best
        self.dropout = dropout
        self.max_norm = max_norm
        self.embed_dim = embed_dim
        self.kernel_num = kernel_num
        self.kernel_sizes = kernel_sizes
        self.static = static
        self.device = device
        self.cuda = cuda
        self.activation_func = activation_func
        self.scoring = scoring
        self.pos_label = pos_label
        self.vectors = vectors
        self.split_ratio = split_ratio
        self.preprocessor = preprocessor
        self.class_weight = class_weight
        self.random_state = random_state
        self.verbose = verbose

    def __clean_str(self, string):
        string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
        string = re.sub(r"\'s", " \'s", string)
        string = re.sub(r"\'ve", " \'ve", string)
        string = re.sub(r"n\'t", " n\'t", string)
        string = re.sub(r"\'re", " \'re", string)
        string = re.sub(r"\'d", " \'d", string)
        string = re.sub(r"\'ll", " \'ll", string)
        string = re.sub(r",", " , ", string)
        string = re.sub(r"!", " ! ", string)
        string = re.sub(r"\(", " ( ", string)
        string = re.sub(r"\)", " ) ", string)
        string = re.sub(r"\?", " ? ", string)
        string = re.sub(r"\s{2,}", " ", string)
        return string.strip()

    def __eval(self, data_iter):
        self.__model.eval()

        preds, targets = [], []
        softmax = nn.Softmax(dim=1) if self.scoring == "roc_auc" else None

        for batch in data_iter:
            feature, target = batch.text.t_(), batch.label.sub_(1)

            if self.cuda and torch.cuda.is_available():
                feature, target = feature.cuda(), target.cuda()

            logit = self.__model(feature)

            F.cross_entropy(logit, target, reduction="sum")

            if self.scoring == "roc_auc":
                pred = [[float(p) for p in dist] for dist in softmax(logit)]
            else:
                pred = torch.max(logit, 1)[1].view(target.size()).tolist()

            preds += pred
            targets += target.tolist()

        targets = [self.__label_field.vocab.itos[targ + 1] for targ in targets]

        if self.scoring == "roc_auc":
            pos_index = self.__label_field.vocab.stoi[self.pos_label] - 1
            return roc_auc_score(targets, [pred[pos_index] for pred in preds])

        preds = [self.__label_field.vocab.itos[pred + 1] for pred in preds]
        return self.scoring(_Eval(preds), None, targets)

    def fit(self, X, y, sample_weight=None):
        if self.random_state is not None:
            torch.manual_seed(self.random_state)

        torch.backends.cudnn.deterministic = self.random_state is not None
        torch.backends.cudnn.benchmark = self.random_state is None

        if self.verbose > 1:
            params = self.get_params().items()

            print("Fitting with the following parameters:")
            print("\n".join([": ".join([k, str(v)]) for k, v in params]))

        start = time() if self.verbose > 0 else None
        train_iter, dev_iter = self.__preprocess(X, y, sample_weight)
        embed_num = len(self.__text_field.vocab)
        class_num = len(self.__label_field.vocab) - 1
        self.__model = _CNNText(embed_num, self.embed_dim, class_num,
                                self.kernel_num, self.kernel_sizes,
                                self.dropout, self.static,
                                self.activation_func,
                                vectors=self.__text_field.vocab.vectors)

        if self.cuda and torch.cuda.is_available():
            torch.cuda.set_device(self.device)
            self.__model.cuda()

        optimizer = torch.optim.Adam(self.__model.parameters(), lr=self.lr,
                                     weight_decay=self.max_norm)
        steps, best_acc, last_step = 0, 0, 0
        active = True
        filename = "./{}.model".format(time())

        self.__model.train()

        for epoch in range(self.epochs):
            for batch in train_iter:
                feature, target = batch.text.t_(), batch.label.sub_(1)

                if self.cuda and torch.cuda.is_available():
                    feature, target = feature.cuda(), target.cuda()

                optimizer.zero_grad()
                F.cross_entropy(self.__model(feature), target).backward()
                optimizer.step()

                steps += 1

                if steps % self.test_interval == 0:
                    dev_acc = self.__eval(dev_iter)

                    if dev_acc > best_acc:
                        best_acc = dev_acc
                        last_step = steps

                        if self.save_best:
                            torch.save(self.__model.state_dict(), filename)
                    elif steps - last_step >= self.early_stop:
                        active = False
                        break

            if not active:
                break

        if self.save_best:
            self.__model.load_state_dict(torch.load(filename))
            remove(filename)

        self.classes_ = self.__label_field.vocab.itos[1:]

        if self.verbose > 0:
            self.__print_elapsed_time(time() - start)

        return self

    def __predict(self, X):
        y_output = []
        max_kernel_size = max(self.kernel_sizes)

        self.__model.eval()

        for text in X:
            assert isinstance(text, str)

            text = self.__text_field.preprocess(text)
            text = self.__pad(text, max_kernel_size, True)
            text = [[self.__text_field.vocab.stoi[x] for x in text]]
            x = torch.tensor(text)
            x = x.cuda() if self.cuda and torch.cuda.is_available() else x

            y_output.append(self.__model(x))

        return y_output

    def predict(self, X):
        y_pred = [torch.argmax(yi, 1) for yi in self.__predict(X)]
        return [self.__label_field.vocab.itos[yi.item() + 1] for yi in y_pred]

    def predict_proba(self, X):
        softmax = nn.Softmax(dim=1)
        y_prob = [softmax(yi) for yi in self.__predict(X)]
        return [[float(yij) for yij in yi[0]] for yi in y_prob]

    def __pad(self, x, max_kernel_size, preprocessed=False):
        tokens = x if preprocessed else self.__text_field.preprocess(x)
        difference = max_kernel_size - len(tokens)

        if difference > 0:
            padding = [self.__text_field.pad_token] * difference
            return x + padding if preprocessed else " ".join([x] + padding)

        return x

    def __preprocess(self, X, y, sample_weight):
        self.__text_field = Field(lower=True)
        self.__label_field = Field(sequential=False)
        self.__text_field.preprocessing = Pipeline(self.__preprocess_text)
        max_kernel_size = max(self.kernel_sizes)
        sample_weight = None if sample_weight is None else list(sample_weight)

        for i in range(len(X)):
            X[i] = self.__pad(X[i], max_kernel_size)

        sw = [1 for yi in y] if sample_weight is None else sample_weight
        s = y if Counter(y).most_common()[-1][1] > 1 else None
        X_t, X_d, y_t, y_d, w_t, _ = split(X, y, sw, shuffle=True, stratify=s,
                                           random_state=self.random_state,
                                           train_size=self.split_ratio)
        fields = [("text", self.__text_field), ("label", self.__label_field)]
        examples = [[X_t[i], y_t[i]] for i in range(len(X_t))]
        examples = [Example.fromlist(example, fields) for example in examples]
        weights = compute_sample_weight(self.class_weight, y_t)
        weights = [weights[i] * w_t[i] for i in range(len(y_t))]
        min_weight = min(weights)
        weights = [int(round(weight / min_weight)) for weight in weights]

        for i in range(len(X_t)):
            Xi = [X_t[i] for j in range(weights[i] - 1)]
            examples += [Example.fromlist([x, y_t[i]], fields) for x in Xi]

        train_data = Dataset(examples, fields)
        dev_data = [[X_d[i], y_d[i]] for i in range(len(X_d))]
        dev_data = [Example.fromlist(example, fields) for example in dev_data]
        dev_data = Dataset(dev_data, fields)

        self.__text_field.build_vocab(train_data, dev_data,
                                      vectors=self.vectors)
        self.__label_field.build_vocab(train_data, dev_data)

        batch_sizes = (self.batch_size, len(dev_data))
        return Iterator.splits((train_data, dev_data), batch_sizes=batch_sizes,
                               sort_key=lambda ex: len(ex.text), repeat=False)

    def __preprocess_text(self, text):
        if self.preprocessor is None:
            return self.__clean_str(text)

        return self.preprocessor(text)

    def __print_elapsed_time(self, seconds):
        sc = round(seconds)
        mn = int(sc / 60)
        sc = sc % 60
        hr = int(mn / 60)
        mn = mn % 60
        hr = "{} hour{}".format(hr, "s" if hr > 1 else "") if hr > 0 else ""
        mn = "{} minute{}".format(mn, "s" if mn > 1 else "") if mn > 0 else ""
        sc = "{} second{}".format(sc, "s" if sc > 1 else "") if sc > 0 else ""
        times = [t for t in [hr, mn, sc] if len(t) > 0]

        if len(times) == 3:
            times = " and ".join([", ".join([hr, mn]), sc])
        elif len(times) == 2:
            times = " and ".join(times)
        else:
            times = times[0] if len(times) > 0 else "less than 1 second"

        print("Completed training in {}.".format(times))

class _CNNText(nn.Module):
    def __init__(self, embed_num, embed_dim, class_num, kernel_num,
                 kernel_sizes, dropout, static, activation_func, vectors=None):
        super(_CNNText, self).__init__()

        if vectors is None:
            self.__embed = nn.Embedding(embed_num, embed_dim)
        else:
            self.__embed = nn.Embedding.from_pretrained(vectors)
            embed_dim = self.__embed.embedding_dim

        Ks = kernel_sizes
        module_list = [nn.Conv2d(1, kernel_num, (K, embed_dim)) for K in Ks]
        self.__convs = nn.ModuleList(module_list)
        self.__dropout = nn.Dropout(dropout)
        self.__fc = nn.Linear(len(Ks) * kernel_num, class_num)
        self.__embed.weight.requires_grad = not static

        if activation_func == "relu":
            self.__f = F.relu
        elif activation_func == "tanh":
            self.__f = torch.tanh
        else:
            self.__f = lambda x: x

    def forward(self, x):
        x = self.__embed(x).unsqueeze(1)
        x = [self.__f(cnv(x), inplace=True).squeeze(3) for cnv in self.__convs]
        x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x]
        return self.__fc(self.__dropout(torch.cat(x, 1)))

class _Eval():
    def __init__(self, preds):
        self.__preds = preds

    def predict(self, X):
        return self.__preds
<|endoftext|>


<commit_before>"""
CLI frontend for package API.
"""
import sys
import argparse
from . import _runner, __version__

def entrypoint(argv=sys.argv[1:]):
    """
    Parse arguments and perform appropriate action.
    Wrap it into `sys.exit()` call.

    :param argv: list of arguments
    :return: status code 0 if succeeded or exception object if failed
    """
    parser = _configure_parser()
    args = vars(parser.parse_args(argv))

    if 'action' not in args:
        args['action'] = 'up'
    try:
        if args['action'] == 'create':
            method_args = _transform_args(args, [
                'name',
                'migrations_dir',
                'template_file',
            ])
            _runner.create(**method_args)
        elif args['action'] == 'show':
            method_args = _transform_args(args, [
                'migrations_dir',
                'template_file',
            ])
            _runner.show(**method_args)
        else:
            method_args = _transform_args(args, [
                ('action', 'direction'),
                'target',
                'migrations_dir',
                'state_file',
            ])
            _runner.perform(**method_args)
        return 0
    except _runner.MigrationError as e:
        return e

def _configure_parser():
    parser = argparse.ArgumentParser(fromfile_prefix_chars='@')

    parser.add_argument(
        '-v', '--version',
        help='show version and exit',
        action='version',
        version='migrations v' + __version__,
    )

    parser.add_argument(
        '-d', '--migrations-dir',
        help='directory where migrations are stored'
    )
    parser.add_argument(
        '-s', '--state-file',
        help='location of file which stores database state'
    )
    parser.add_argument(
        '-t', '--template-file',
        help='location of template file for new migrations'
    )

    subparsers = parser.add_subparsers()

    action_create = subparsers.add_parser('create')
    action_create.add_argument(
        'name',
        help='name of new migration file'
    )
    action_create.set_defaults(action='create')

    action_show = subparsers.add_parser('show')
    action_show.set_defaults(action='show')

    action_up = subparsers.add_parser('up')
    action_up.add_argument(
        'target', nargs='?',
        help='name of the last migration or number of migrations '
             '(by default perform all available)'
    )
    action_up.set_defaults(action='up')

    action_down = subparsers.add_parser('down')
    action_down.add_argument(
        'target', nargs='?',
        help='name of the last migration or number of migrations '
             '(by default revert one)'
    )
    action_down.set_defaults(action='down')

    return parser

def _transform_args(args, required):
    result = {}
    for required_arg in required:
        if isinstance(required_arg, tuple):
            old_key, key = required_arg
        else:  # type(required_arg) == str
            old_key = key = required_arg
        try:
            value = args[old_key]
        except KeyError:
            value = None
        if value is not None:
            result[key] = value
    return result
<commit_msg>make cli module executable<commit_after>#!/usr/bin/env python3
"""
CLI frontend for package API.
"""
import sys
import argparse
from . import _runner, __version__

def entrypoint(argv=sys.argv[1:]):
    """
    Parse arguments and perform appropriate action.
    Wrap it into `sys.exit()` call.

    :param argv: list of arguments
    :return: status code 0 if succeeded or exception object if failed
    """
    parser = _configure_parser()
    args = vars(parser.parse_args(argv))

    if 'action' not in args:
        args['action'] = 'up'
    try:
        if args['action'] == 'create':
            method_args = _transform_args(args, [
                'name',
                'migrations_dir',
                'template_file',
            ])
            _runner.create(**method_args)
        elif args['action'] == 'show':
            method_args = _transform_args(args, [
                'migrations_dir',
                'template_file',
            ])
            _runner.show(**method_args)
        else:
            method_args = _transform_args(args, [
                ('action', 'direction'),
                'target',
                'migrations_dir',
                'state_file',
            ])
            _runner.perform(**method_args)
        return 0
    except _runner.MigrationError as e:
        return e

def _configure_parser():
    parser = argparse.ArgumentParser(fromfile_prefix_chars='@')

    parser.add_argument(
        '-v', '--version',
        help='show version and exit',
        action='version',
        version='migrations v' + __version__,
    )

    parser.add_argument(
        '-d', '--migrations-dir',
        help='directory where migrations are stored'
    )
    parser.add_argument(
        '-s', '--state-file',
        help='location of file which stores database state'
    )
    parser.add_argument(
        '-t', '--template-file',
        help='location of template file for new migrations'
    )

    subparsers = parser.add_subparsers()

    action_create = subparsers.add_parser('create')
    action_create.add_argument(
        'name',
        help='name of new migration file'
    )
    action_create.set_defaults(action='create')

    action_show = subparsers.add_parser('show')
    action_show.set_defaults(action='show')

    action_up = subparsers.add_parser('up')
    action_up.add_argument(
        'target', nargs='?',
        help='name of the last migration or number of migrations '
             '(by default perform all available)'
    )
    action_up.set_defaults(action='up')

    action_down = subparsers.add_parser('down')
    action_down.add_argument(
        'target', nargs='?',
        help='name of the last migration or number of migrations '
             '(by default revert one)'
    )
    action_down.set_defaults(action='down')

    return parser

def _transform_args(args, required):
    result = {}
    for required_arg in required:
        if isinstance(required_arg, tuple):
            old_key, key = required_arg
        else:  # type(required_arg) == str
            old_key = key = required_arg
        try:
            value = args[old_key]
        except KeyError:
            value = None
        if value is not None:
            result[key] = value
    return result

if __name__ == '__main__':
    sys.exit(entrypoint())
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 01-07-2017 23:20
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This module runs PoGo for a file, using the given GTF and FA reference files, it will produce the results in the same
folder where the input file is located
"""<commit_msg>PEP 8 compliance<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 01-07-2017 23:20
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This module runs PoGo for a file, using the given GTF and FA reference files, it will produce the results in the same
folder where the input file is located
"""
<|endoftext|>


<commit_before><commit_msg>Fix all_gather_list for arbitrary inputs (#1105)<commit_after><|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 29-06-2017 15:05
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This is a template pipeline for refactoring out things from final pipelines as I identify how they're gonna look like
"""

import config_manager
import toolbox
from exceptions import PipelineDirectorException

__configuration_file = None
__configuration_manager = None

def __get_configuration_manager():
    global __configuration_manager
    if __configuration_manager is None:
        __configuration_manager = DirectorConfigurationManager(toolbox.read_json(__configuration_file), __configuration_file)
    return __configuration_manager

class DirectorConfigurationManager(config_manager.ConfigurationManager):
    def __init__(self, configuration_object, configuration_file):
        super(DirectorConfigurationManager, self).__init__(configuration_object, configuration_file)

class Director:
    """
    This is the director of the pipeline
    """
    def __init__(self, config_file_name, runner_id = 0):
        self.__logger = config_manager.get_app_config_manager().get_logger_for(__name__)

    def _before(self):
        """
        This method implements some logic that is run before running the main pipeline director
        :return:
        """
        self._get_logger().debug("No behaviour has been defined for 'before' running the pipeline")
        return True

    def run(self):
        if not self._before():
            self._get_logger().error("The logic executed BEFORE running the pipeline has FAILED")
            return False
        if not self._run_pipeline():
            self._get_logger().error("The PIPELINE execution has FAILED")
            return False
        if not self._after():
            self._get_logger().error("The logic executed AFTER running the pipeline has FAILED")
            return False
        return True

    def _run_pipeline(self):
        raise NotImplementedError("Implement your main pipeline logic here")

    def _after(self):
        self._get_logger().debug("No behaviour has been defined for 'after' running the pipeline")
        return True

    def _get_logger(self):
        return self.__logger

    def _set_logger(self, new_logger):
        self.__logger = new_logger
        return self._get_logger()
<commit_msg>extra documentation for pipeline 'before method'<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 29-06-2017 15:05
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This is a template pipeline for refactoring out things from final pipelines as I identify how they're gonna look like
"""

import config_manager
import toolbox
from exceptions import PipelineDirectorException

__configuration_file = None
__configuration_manager = None

def __get_configuration_manager():
    global __configuration_manager
    if __configuration_manager is None:
        __configuration_manager = DirectorConfigurationManager(toolbox.read_json(__configuration_file), __configuration_file)
    return __configuration_manager

class DirectorConfigurationManager(config_manager.ConfigurationManager):
    def __init__(self, configuration_object, configuration_file):
        super(DirectorConfigurationManager, self).__init__(configuration_object, configuration_file)

class Director:
    """
    This is the director of the pipeline
    """
    def __init__(self, config_file_name, runner_id = 0):
        self.__logger = config_manager.get_app_config_manager().get_logger_for(__name__)

    def _before(self):
        """
        This method implements some logic that is run before running the main pipeline director
        :return: it returns True if there was no problem, False otherwise
        """
        self._get_logger().debug("No behaviour has been defined for 'before' running the pipeline")
        return True

    def run(self):
        if not self._before():
            self._get_logger().error("The logic executed BEFORE running the pipeline has FAILED")
            return False
        if not self._run_pipeline():
            self._get_logger().error("The PIPELINE execution has FAILED")
            return False
        if not self._after():
            self._get_logger().error("The logic executed AFTER running the pipeline has FAILED")
            return False
        return True

    def _run_pipeline(self):
        raise NotImplementedError("Implement your main pipeline logic here")

    def _after(self):
        self._get_logger().debug("No behaviour has been defined for 'after' running the pipeline")
        return True

    def _get_logger(self):
        return self.__logger

    def _set_logger(self, new_logger):
        self.__logger = new_logger
        return self._get_logger()
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 29-06-2017 15:05
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This is a template pipeline for refactoring out things from final pipelines as I identify how they're gonna look like
"""

import config_manager
from exceptions import PipelineDirectorException

class Director:
    """
    This is the director of the pipeline
    """
    def __init__(self, config_file_name, runner_id = 0):
        self.__logger = config_manager.get_app_config_manager().get_logger_for(__name__)

    def _before(self):
        self._get_logger().debug("No behaviour has been defined for 'before' running the pipeline")
        return True

    def run(self):
        if not self._before():
            self._get_logger().error("The logic executed BEFORE running the pipeline has FAILED")
            return False
        if not self._run_pipeline():
            self._get_logger().error("The PIPELINE execution has FAILED")
            return False
        self._after()

    def _run_pipeline(self):
        raise NotImplementedError("Implement your main pipeline logic here")

    def _after(self):
        self._get_logger().debug("No behaviour has been defined for 'after' running the pipeline")
        return True

    def _get_logger(self):
        return self.__logger

    def _set_logger(self, new_logger):
        self.__logger = new_logger
        return self._get_logger()<commit_msg>logic for dealing with failed post-pipeline execution<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 29-06-2017 15:05
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This is a template pipeline for refactoring out things from final pipelines as I identify how they're gonna look like
"""

import config_manager
from exceptions import PipelineDirectorException

class Director:
    """
    This is the director of the pipeline
    """
    def __init__(self, config_file_name, runner_id = 0):
        self.__logger = config_manager.get_app_config_manager().get_logger_for(__name__)

    def _before(self):
        self._get_logger().debug("No behaviour has been defined for 'before' running the pipeline")
        return True

    def run(self):
        if not self._before():
            self._get_logger().error("The logic executed BEFORE running the pipeline has FAILED")
            return False
        if not self._run_pipeline():
            self._get_logger().error("The PIPELINE execution has FAILED")
            return False
        if not self._after():
            self._get_logger().error("The logic executed AFTER running the pipeline has FAILED")
            return False

    def _run_pipeline(self):
        raise NotImplementedError("Implement your main pipeline logic here")

    def _after(self):
        self._get_logger().debug("No behaviour has been defined for 'after' running the pipeline")
        return True

    def _get_logger(self):
        return self.__logger

    def _set_logger(self, new_logger):
        self.__logger = new_logger
        return self._get_logger()<|endoftext|>


<commit_before>"""RyuApp shim between Ryu and Valve."""

# Copyright (C) 2013 Nippon Telegraph and Telephone Corporation.
# Copyright (C) 2015 Brad Cowie, Christopher Lorier and Joe Stringer.
# Copyright (C) 2015 Research and Education Advanced Network New Zealand Ltd.
# Copyright (C) 2015--2018 The Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import time

from functools import partial

from ryu.controller.handler import CONFIG_DISPATCHER
from ryu.controller.handler import MAIN_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.controller import dpset
from ryu.controller import event
from ryu.controller import ofp_event
from ryu.lib import hub

from faucet.config_parser import get_config_for_api
from faucet.valve_ryuapp import EventReconfigure, RyuAppBase
from faucet.valve_util import dpid_log, kill_on_exception
from faucet import faucet_experimental_api
from faucet import faucet_experimental_event
from faucet import faucet_bgp
from faucet import faucet_dot1x
from faucet import valves_manager
from faucet import faucet_metrics
from faucet import valve_of

class EventFaucetExperimentalAPIRegistered(event.EventBase):
    """Event used to notify that the API is registered with Faucet."""
    pass

class EventFaucetResolveGateways(event.EventBase):
    """Event used to trigger gateway re/resolution."""
    pass

class EventFaucetStateExpire(event.EventBase):
    """Event used to trigger expiration of state in controller."""
    pass

class EventFaucetMetricUpdate(event.EventBase):
    """Event used to trigger update of metrics."""
    pass

class EventFaucetAdvertise(event.EventBase):
    """Event used to trigger periodic network advertisements (eg IPv6 RAs)."""
    pass

class EventFaucetLLDPAdvertise(event.EventBase):
    """Event used to trigger periodic LLDP beacons."""
    pass

class EventFaucetStackLinkStates(event.EventBase):
    """Event used to update link stack states."""
    pass

class Faucet(RyuAppBase):
    """A RyuApp that implements an L2/L3 learning VLAN switch.

    Valve provides the switch implementation; this is a shim for the Ryu
    event handling framework to interface with Valve.
    """
    _CONTEXTS = {
        'dpset': dpset.DPSet,
        'faucet_experimental_api': faucet_experimental_api.FaucetExperimentalAPI,
        }
    _EVENTS = [EventFaucetExperimentalAPIRegistered]
    _VALVE_SERVICES = {
        EventFaucetMetricUpdate: (None, 5),
        EventFaucetResolveGateways: ('resolve_gateways', 2),
        EventFaucetStateExpire: ('state_expire', 5),
        EventFaucetAdvertise: ('advertise', 5),
        EventFaucetLLDPAdvertise: ('send_lldp_beacons', 5),
        EventFaucetStackLinkStates: ('update_stack_link_states', 2),
    }
    logname = 'faucet'
    exc_logname = logname + '.exception'
    bgp = None
    metrics = None
    notifier = None
    valves_manager = None

    def __init__(self, *args, **kwargs):
        super(Faucet, self).__init__(*args, **kwargs)
        self.api = kwargs['faucet_experimental_api']
        self.metrics = faucet_metrics.FaucetMetrics(reg=self._reg)
        self.bgp = faucet_bgp.FaucetBgp(self.logger, self.metrics, self._send_flow_msgs)
        self.dot1x = faucet_dot1x.FaucetDot1x(
            self.logger, self.metrics, self._send_flow_msgs)
        self.notifier = faucet_experimental_event.FaucetExperimentalEventNotifier(
            self.get_setting('EVENT_SOCK'), self.metrics, self.logger)
        self.valves_manager = valves_manager.ValvesManager(
            self.logname, self.logger, self.metrics, self.notifier, self.bgp,
            self.dot1x, self._send_flow_msgs)

    @kill_on_exception(exc_logname)
    def start(self):
        super(Faucet, self).start()

        # Start Prometheus
        prom_port = int(self.get_setting('PROMETHEUS_PORT'))
        prom_addr = self.get_setting('PROMETHEUS_ADDR')
        self.metrics.start(prom_port, prom_addr)

        # Start event notifier
        notifier_thread = self.notifier.start()
        if notifier_thread is not None:
            self.threads.append(notifier_thread)

        for service_event, service_pair in list(self._VALVE_SERVICES.items()):
            _, interval = service_pair
            self.threads.append(hub.spawn(
                partial(self._thread_reschedule, service_event(), interval)))

        # Register to API
        self.api._register(self)
        self.send_event_to_observers(EventFaucetExperimentalAPIRegistered())

    def _delete_deconfigured_dp(self, deleted_dpid):
        self.logger.info(
            'Deleting de-configured %s', dpid_log(deleted_dpid))
        ryu_dp = self.dpset.get(deleted_dpid)
        if ryu_dp is not None:
            ryu_dp.close()

    @set_ev_cls(EventReconfigure, MAIN_DISPATCHER)
    @kill_on_exception(exc_logname)
    def reload_config(self, ryu_event):
        """Handle a request to reload configuration."""
        super(Faucet, self).reload_config(ryu_event)
        self.valves_manager.request_reload_configs(
            time.time(), self.config_file, delete_dp=self._delete_deconfigured_dp)

    @kill_on_exception(exc_logname)
    def _send_flow_msgs(self, valve, flow_msgs, ryu_dp=None):
        """Send OpenFlow messages to a connected datapath.

        Args:
            Valve instance or None.
            flow_msgs (list): OpenFlow messages to send.
            ryu_dp: Override datapath from DPSet.
        """
        if ryu_dp is None:
            ryu_dp = self.dpset.get(valve.dp.dp_id)
        if not ryu_dp:
            valve.logger.error('send_flow_msgs: DP not up')
            return
        valve.send_flows(ryu_dp, flow_msgs)

    def _get_valve(self, ryu_event, require_running=False):
        """Get Valve instance to response to an event.

        Args:
            ryu_event (ryu.controller.event.Event): event
            require_running (bool): require DP to be running.
        Returns:
            valve, ryu_dp, msg: tuple of Nones, or datapath object, Ryu datapath, and Ryu msg (if any)
        """
        valve, ryu_dp, msg = self._get_datapath_obj(
            self.valves_manager.valves, ryu_event)
        if valve:
            if msg:
                valve.ofchannel_log([msg])
            if require_running and not valve.dp.running:
                valve = None
        return (valve, ryu_dp, msg)

    def _config_files_changed(self):
        return self.valves_manager.config_watcher.files_changed()

    @set_ev_cls(EventFaucetMetricUpdate, MAIN_DISPATCHER)
    @kill_on_exception(exc_logname)
    def metric_update(self, _):
        """Handle a request to update metrics in the controller."""
        self.valves_manager.update_metrics(time.time())

    @set_ev_cls(EventFaucetResolveGateways, MAIN_DISPATCHER)
    @set_ev_cls(EventFaucetStateExpire, MAIN_DISPATCHER)
    @set_ev_cls(EventFaucetAdvertise, MAIN_DISPATCHER)
    @set_ev_cls(EventFaucetLLDPAdvertise, MAIN_DISPATCHER)
    @set_ev_cls(EventFaucetStackLinkStates, MAIN_DISPATCHER)
    @kill_on_exception(exc_logname)
    def _valve_flow_services(self, ryu_event):
        """Call a method on all Valves and send any resulting flows."""
        self.valves_manager.valve_flow_services(
            time.time(),
            self._VALVE_SERVICES[type(ryu_event)][0])

    def get_config(self):
        """FAUCET experimental API: return config for all Valves."""
        return get_config_for_api(self.valves_manager.valves)

    def get_tables(self, dp_id):
        """FAUCET experimental API: return config tables for one Valve."""
        if dp_id in self.valves_manager.valves:
            return self.valves_manager.valves[dp_id].dp.get_tables()
        return {}

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER) # pylint: disable=no-member
    @kill_on_exception(exc_logname)
    def packet_in_handler(self, ryu_event):
        """Handle a packet in event from the dataplane.

        Args:
            ryu_event (ryu.controller.event.EventReplyBase): packet in message.
        """
        valve, _, msg = self._get_valve(ryu_event, require_running=True)
        if valve is None:
            return
        self.valves_manager.valve_packet_in(ryu_event.timestamp, valve, msg)

    @set_ev_cls(ofp_event.EventOFPErrorMsg, MAIN_DISPATCHER) # pylint: disable=no-member
    @kill_on_exception(exc_logname)
    def error_handler(self, ryu_event):
        """Handle an OFPError from a datapath.

        Args:
            ryu_event (ryu.controller.ofp_event.EventOFPErrorMsg): trigger
        """
        valve, _, msg = self._get_valve(ryu_event)
        if valve is None:
            return
        valve.oferror(msg)

    @set_ev_cls(ofp_event.EventOFPSwitchFeatures, CONFIG_DISPATCHER) # pylint: disable=no-member
    @kill_on_exception(exc_logname)
    def features_handler(self, ryu_event):
        """Handle receiving a switch features message from a datapath.

        Args:
            ryu_event (ryu.controller.ofp_event.EventOFPStateChange): trigger.
        """
        valve, ryu_dp, msg = self._get_valve(ryu_event)
        if valve is None:
            return
        self._send_flow_msgs(valve, valve.switch_features(msg), ryu_dp=ryu_dp)

    @kill_on_exception(exc_logname)
    def _datapath_connect(self, ryu_event):
        """Handle any/all re/connection of a datapath.

        Args:
            ryu_event (ryu.controller.ofp_event.Event)
        """
        now = time.time()
        valve, ryu_dp, _ = self._get_valve(ryu_event)
        if valve is None:
            return
        discovered_up_ports = [
            port.port_no for port in list(ryu_dp.ports.values())
            if valve_of.port_status_from_state(port.state) and not valve_of.ignore_port(port.port_no)]
        self._send_flow_msgs(valve, valve.datapath_connect(now, discovered_up_ports))
        self.valves_manager.stack_topo_change(now, valve)

    @kill_on_exception(exc_logname)
    def _datapath_disconnect(self, ryu_event):
        """Handle any/all disconnection of a datapath.

        Args:
            ryu_event (ryu.controller.ofp_event.Event)
        """
        valve, _, _ = self._get_valve(ryu_event)
        if valve is None:
            return
        valve.datapath_disconnect()
        self.valves_manager.stack_topo_change(time.time(), valve)

    @set_ev_cls(ofp_event.EventOFPDescStatsReply, MAIN_DISPATCHER) # pylint: disable=no-member
    @kill_on_exception(exc_logname)
    def desc_stats_reply_handler(self, ryu_event):
        """Handle OFPDescStatsReply from datapath.

        Args:
            ryu_event (ryu.controller.ofp_event.EventOFPDescStatsReply): trigger.
        """
        valve, _, msg = self._get_valve(ryu_event)
        if valve is None:
            return
        valve.ofdescstats_handler(msg.body)

    @set_ev_cls(ofp_event.EventOFPPortStatus, MAIN_DISPATCHER) # pylint: disable=no-member
    @kill_on_exception(exc_logname)
    def port_status_handler(self, ryu_event):
        """Handle a port status change event.

        Args:
            ryu_event (ryu.controller.ofp_event.EventOFPPortStatus): trigger.
        """
        valve, _, msg = self._get_valve(ryu_event, require_running=True)
        if valve is None:
            return
        self._send_flow_msgs(valve, valve.port_status_handler(
            msg.desc.port_no, msg.reason, msg.desc.state))

    @set_ev_cls(ofp_event.EventOFPFlowRemoved, MAIN_DISPATCHER) # pylint: disable=no-member
    @kill_on_exception(exc_logname)
    def flowremoved_handler(self, ryu_event):
        """Handle a flow removed event.

        Args:
            ryu_event (ryu.controller.ofp_event.EventOFPFlowRemoved): trigger.
        """
        valve, ryu_dp, msg = self._get_valve(ryu_event, require_running=True)
        if valve is None:
            return
        if msg.reason == ryu_dp.ofproto.OFPRR_IDLE_TIMEOUT:
            self._send_flow_msgs(valve, valve.flow_timeout(time.time(), msg.table_id, msg.match))
<commit_msg>Missed reference to dyn_running.<commit_after>"""RyuApp shim between Ryu and Valve."""

# Copyright (C) 2013 Nippon Telegraph and Telephone Corporation.
# Copyright (C) 2015 Brad Cowie, Christopher Lorier and Joe Stringer.
# Copyright (C) 2015 Research and Education Advanced Network New Zealand Ltd.
# Copyright (C) 2015--2018 The Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import time

from functools import partial

from ryu.controller.handler import CONFIG_DISPATCHER
from ryu.controller.handler import MAIN_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.controller import dpset
from ryu.controller import event
from ryu.controller import ofp_event
from ryu.lib import hub

from faucet.config_parser import get_config_for_api
from faucet.valve_ryuapp import EventReconfigure, RyuAppBase
from faucet.valve_util import dpid_log, kill_on_exception
from faucet import faucet_experimental_api
from faucet import faucet_experimental_event
from faucet import faucet_bgp
from faucet import faucet_dot1x
from faucet import valves_manager
from faucet import faucet_metrics
from faucet import valve_of

class EventFaucetExperimentalAPIRegistered(event.EventBase):
    """Event used to notify that the API is registered with Faucet."""
    pass

class EventFaucetResolveGateways(event.EventBase):
    """Event used to trigger gateway re/resolution."""
    pass

class EventFaucetStateExpire(event.EventBase):
    """Event used to trigger expiration of state in controller."""
    pass

class EventFaucetMetricUpdate(event.EventBase):
    """Event used to trigger update of metrics."""
    pass

class EventFaucetAdvertise(event.EventBase):
    """Event used to trigger periodic network advertisements (eg IPv6 RAs)."""
    pass

class EventFaucetLLDPAdvertise(event.EventBase):
    """Event used to trigger periodic LLDP beacons."""
    pass

class EventFaucetStackLinkStates(event.EventBase):
    """Event used to update link stack states."""
    pass

class Faucet(RyuAppBase):
    """A RyuApp that implements an L2/L3 learning VLAN switch.

    Valve provides the switch implementation; this is a shim for the Ryu
    event handling framework to interface with Valve.
    """
    _CONTEXTS = {
        'dpset': dpset.DPSet,
        'faucet_experimental_api': faucet_experimental_api.FaucetExperimentalAPI,
        }
    _EVENTS = [EventFaucetExperimentalAPIRegistered]
    _VALVE_SERVICES = {
        EventFaucetMetricUpdate: (None, 5),
        EventFaucetResolveGateways: ('resolve_gateways', 2),
        EventFaucetStateExpire: ('state_expire', 5),
        EventFaucetAdvertise: ('advertise', 5),
        EventFaucetLLDPAdvertise: ('send_lldp_beacons', 5),
        EventFaucetStackLinkStates: ('update_stack_link_states', 2),
    }
    logname = 'faucet'
    exc_logname = logname + '.exception'
    bgp = None
    metrics = None
    notifier = None
    valves_manager = None

    def __init__(self, *args, **kwargs):
        super(Faucet, self).__init__(*args, **kwargs)
        self.api = kwargs['faucet_experimental_api']
        self.metrics = faucet_metrics.FaucetMetrics(reg=self._reg)
        self.bgp = faucet_bgp.FaucetBgp(self.logger, self.metrics, self._send_flow_msgs)
        self.dot1x = faucet_dot1x.FaucetDot1x(
            self.logger, self.metrics, self._send_flow_msgs)
        self.notifier = faucet_experimental_event.FaucetExperimentalEventNotifier(
            self.get_setting('EVENT_SOCK'), self.metrics, self.logger)
        self.valves_manager = valves_manager.ValvesManager(
            self.logname, self.logger, self.metrics, self.notifier, self.bgp,
            self.dot1x, self._send_flow_msgs)

    @kill_on_exception(exc_logname)
    def start(self):
        super(Faucet, self).start()

        # Start Prometheus
        prom_port = int(self.get_setting('PROMETHEUS_PORT'))
        prom_addr = self.get_setting('PROMETHEUS_ADDR')
        self.metrics.start(prom_port, prom_addr)

        # Start event notifier
        notifier_thread = self.notifier.start()
        if notifier_thread is not None:
            self.threads.append(notifier_thread)

        for service_event, service_pair in list(self._VALVE_SERVICES.items()):
            _, interval = service_pair
            self.threads.append(hub.spawn(
                partial(self._thread_reschedule, service_event(), interval)))

        # Register to API
        self.api._register(self)
        self.send_event_to_observers(EventFaucetExperimentalAPIRegistered())

    def _delete_deconfigured_dp(self, deleted_dpid):
        self.logger.info(
            'Deleting de-configured %s', dpid_log(deleted_dpid))
        ryu_dp = self.dpset.get(deleted_dpid)
        if ryu_dp is not None:
            ryu_dp.close()

    @set_ev_cls(EventReconfigure, MAIN_DISPATCHER)
    @kill_on_exception(exc_logname)
    def reload_config(self, ryu_event):
        """Handle a request to reload configuration."""
        super(Faucet, self).reload_config(ryu_event)
        self.valves_manager.request_reload_configs(
            time.time(), self.config_file, delete_dp=self._delete_deconfigured_dp)

    @kill_on_exception(exc_logname)
    def _send_flow_msgs(self, valve, flow_msgs, ryu_dp=None):
        """Send OpenFlow messages to a connected datapath.

        Args:
            Valve instance or None.
            flow_msgs (list): OpenFlow messages to send.
            ryu_dp: Override datapath from DPSet.
        """
        if ryu_dp is None:
            ryu_dp = self.dpset.get(valve.dp.dp_id)
        if not ryu_dp:
            valve.logger.error('send_flow_msgs: DP not up')
            return
        valve.send_flows(ryu_dp, flow_msgs)

    def _get_valve(self, ryu_event, require_running=False):
        """Get Valve instance to response to an event.

        Args:
            ryu_event (ryu.controller.event.Event): event
            require_running (bool): require DP to be running.
        Returns:
            valve, ryu_dp, msg: tuple of Nones, or datapath object, Ryu datapath, and Ryu msg (if any)
        """
        valve, ryu_dp, msg = self._get_datapath_obj(
            self.valves_manager.valves, ryu_event)
        if valve:
            if msg:
                valve.ofchannel_log([msg])
            if require_running and not valve.dp.dyn_running:
                valve = None
        return (valve, ryu_dp, msg)

    def _config_files_changed(self):
        return self.valves_manager.config_watcher.files_changed()

    @set_ev_cls(EventFaucetMetricUpdate, MAIN_DISPATCHER)
    @kill_on_exception(exc_logname)
    def metric_update(self, _):
        """Handle a request to update metrics in the controller."""
        self.valves_manager.update_metrics(time.time())

    @set_ev_cls(EventFaucetResolveGateways, MAIN_DISPATCHER)
    @set_ev_cls(EventFaucetStateExpire, MAIN_DISPATCHER)
    @set_ev_cls(EventFaucetAdvertise, MAIN_DISPATCHER)
    @set_ev_cls(EventFaucetLLDPAdvertise, MAIN_DISPATCHER)
    @set_ev_cls(EventFaucetStackLinkStates, MAIN_DISPATCHER)
    @kill_on_exception(exc_logname)
    def _valve_flow_services(self, ryu_event):
        """Call a method on all Valves and send any resulting flows."""
        self.valves_manager.valve_flow_services(
            time.time(),
            self._VALVE_SERVICES[type(ryu_event)][0])

    def get_config(self):
        """FAUCET experimental API: return config for all Valves."""
        return get_config_for_api(self.valves_manager.valves)

    def get_tables(self, dp_id):
        """FAUCET experimental API: return config tables for one Valve."""
        if dp_id in self.valves_manager.valves:
            return self.valves_manager.valves[dp_id].dp.get_tables()
        return {}

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER) # pylint: disable=no-member
    @kill_on_exception(exc_logname)
    def packet_in_handler(self, ryu_event):
        """Handle a packet in event from the dataplane.

        Args:
            ryu_event (ryu.controller.event.EventReplyBase): packet in message.
        """
        valve, _, msg = self._get_valve(ryu_event, require_running=True)
        if valve is None:
            return
        self.valves_manager.valve_packet_in(ryu_event.timestamp, valve, msg)

    @set_ev_cls(ofp_event.EventOFPErrorMsg, MAIN_DISPATCHER) # pylint: disable=no-member
    @kill_on_exception(exc_logname)
    def error_handler(self, ryu_event):
        """Handle an OFPError from a datapath.

        Args:
            ryu_event (ryu.controller.ofp_event.EventOFPErrorMsg): trigger
        """
        valve, _, msg = self._get_valve(ryu_event)
        if valve is None:
            return
        valve.oferror(msg)

    @set_ev_cls(ofp_event.EventOFPSwitchFeatures, CONFIG_DISPATCHER) # pylint: disable=no-member
    @kill_on_exception(exc_logname)
    def features_handler(self, ryu_event):
        """Handle receiving a switch features message from a datapath.

        Args:
            ryu_event (ryu.controller.ofp_event.EventOFPStateChange): trigger.
        """
        valve, ryu_dp, msg = self._get_valve(ryu_event)
        if valve is None:
            return
        self._send_flow_msgs(valve, valve.switch_features(msg), ryu_dp=ryu_dp)

    @kill_on_exception(exc_logname)
    def _datapath_connect(self, ryu_event):
        """Handle any/all re/connection of a datapath.

        Args:
            ryu_event (ryu.controller.ofp_event.Event)
        """
        now = time.time()
        valve, ryu_dp, _ = self._get_valve(ryu_event)
        if valve is None:
            return
        discovered_up_ports = [
            port.port_no for port in list(ryu_dp.ports.values())
            if valve_of.port_status_from_state(port.state) and not valve_of.ignore_port(port.port_no)]
        self._send_flow_msgs(valve, valve.datapath_connect(now, discovered_up_ports))
        self.valves_manager.stack_topo_change(now, valve)

    @kill_on_exception(exc_logname)
    def _datapath_disconnect(self, ryu_event):
        """Handle any/all disconnection of a datapath.

        Args:
            ryu_event (ryu.controller.ofp_event.Event)
        """
        valve, _, _ = self._get_valve(ryu_event)
        if valve is None:
            return
        valve.datapath_disconnect()
        self.valves_manager.stack_topo_change(time.time(), valve)

    @set_ev_cls(ofp_event.EventOFPDescStatsReply, MAIN_DISPATCHER) # pylint: disable=no-member
    @kill_on_exception(exc_logname)
    def desc_stats_reply_handler(self, ryu_event):
        """Handle OFPDescStatsReply from datapath.

        Args:
            ryu_event (ryu.controller.ofp_event.EventOFPDescStatsReply): trigger.
        """
        valve, _, msg = self._get_valve(ryu_event)
        if valve is None:
            return
        valve.ofdescstats_handler(msg.body)

    @set_ev_cls(ofp_event.EventOFPPortStatus, MAIN_DISPATCHER) # pylint: disable=no-member
    @kill_on_exception(exc_logname)
    def port_status_handler(self, ryu_event):
        """Handle a port status change event.

        Args:
            ryu_event (ryu.controller.ofp_event.EventOFPPortStatus): trigger.
        """
        valve, _, msg = self._get_valve(ryu_event, require_running=True)
        if valve is None:
            return
        self._send_flow_msgs(valve, valve.port_status_handler(
            msg.desc.port_no, msg.reason, msg.desc.state))

    @set_ev_cls(ofp_event.EventOFPFlowRemoved, MAIN_DISPATCHER) # pylint: disable=no-member
    @kill_on_exception(exc_logname)
    def flowremoved_handler(self, ryu_event):
        """Handle a flow removed event.

        Args:
            ryu_event (ryu.controller.ofp_event.EventOFPFlowRemoved): trigger.
        """
        valve, ryu_dp, msg = self._get_valve(ryu_event, require_running=True)
        if valve is None:
            return
        if msg.reason == ryu_dp.ofproto.OFPRR_IDLE_TIMEOUT:
            self._send_flow_msgs(valve, valve.flow_timeout(time.time(), msg.table_id, msg.match))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from ctypes import c_int, c_uint, c_longlong, c_ushort, c_char, \
                   c_void_p, c_char_p, POINTER, pointer, cast, \
                   sizeof, Structure, CDLL, get_errno

from exceptions.exceptions import *
from collections import defaultdict

libc = CDLL('libc.so.6', use_errno=True)

libc.open.restype = c_int
libc.socket.restype = c_int
libc.accept.restype = c_int
libc.strerror.argtype = c_int
libc.strerror.restype = c_char_p

# set command protocols for communication to server-client
#
# to start have the send/request of a fd - basic -
# an additional cmd might be to request server status information
# on a multiplex server the cmds will natural become more involved
#
# commands for client-server communica

LOAD = 0x0
LOAD_RDONLY = 0x0
LOAD_WRONLY = 0x1
LOAD_RDWR   = 0x2

RECV = 0x1

PASS = 0x10
PEER_DUMP = 0x10
PEER_RECV = 0x20
PASS_FD   = 0x40

CLOSE = 0x100
CLS_FD  = 0x100
CLS_ALL = 0x200

REFERENCE = 0x200
RET_FD    = 0x100
REFCNT_FD = 0x200

# linux values
SOCK_ADDRDATA_SZ = 14
UNIX_PATH_MAX    = 108

FDBUS_IOVLEN = 0x1

O_RDONLY = 0x0
O_WRONLY = 0x1
O_RDWR   = 0x2

SCM_RIGHTS = c_int(0x01)
SOL_SOCKET = c_int(0x01)

SEEK_SET = 0
SEEK_CUR = 1
SEEK_END = 2

PROTO_DEFAULT = 0
SOCK_STREAM   = 1
AF_UNIX       = 1

DEFAULT_CLIENTS = 3

MSG_CMSG_CLOEXEC = 2 << 29
MSG_SERV = c_int(0)

# conversion of linux data types set to their ctypes size
socklen_t = c_uint
size_t = c_uint
off_t = c_longlong

def get_error_msg():
    errno = get_errno()
    return libc.strerror(c_int(errno))

class peermsg(Structure):

    _fields_ = [('length', c_int), ('peers', c_void_p)]

    def __init__(self, peers=None):
        if peers is None:
            self.length = c_int(0)
            self.peers = c_void_p()
        else:
            peer_array = ARRAY(c_int, len(peers))
            self.length = c_int(len(peers))
            self.peers = cast(c_void_p, peer_array(peers))

class fdmsg(Structure):

    _fields_ = [('protocol', c_int), ('command', c_int), ('name', c_char_p),
                ('path', c_char_p), ('created', c_char_p), ('mode', c_int),
                ('client', c_int)]

    def __init__(self, proto, cmd=None, fdobj=None, client=None):
        self.protocol = proto
        self.command = c_int(cmd) if cmd else c_int(0)
        self.client = c_int(-1) if client is None else c_int(client)
        if fdobj is None:
            self.name = self.path = self.created = c_char_p(None)
            self.mode = c_int(-1)
        else:
            self.name = c_char_p(fdobj.name)
            self.path = c_char_p(fdobj.path)
            self.created = c_char_p(fdobj.created)
            self.mode = c_int(fdobj.mode)

class sockaddr(Structure):

    _fields_ = [('sa_family', c_ushort),
                ('sa_data', c_char * SOCK_ADDRDATA_SZ)]

class sockaddr_un(Structure):

    _fields_ = [('sun_family', c_ushort),
                ('sun_path', c_char * UNIX_PATH_MAX)]

    def __init__(self, family, path):
        self.sun_family = c_ushort(family)
        self.sun_path = path

class iovec(Structure):

    _fields_ = [('iov_base', c_void_p), ('iov_len', size_t)]

    def __init__(self, io_data):
        self.iov_base = cast(io_data, c_void_p)
        self.iov_len = size_t(sizeof(fdmsg) * FDBUS_IOVLEN)

class msghdr(Structure):

    _fields_ = [('msg_name', c_void_p), ('msg_namelen', socklen_t),
                ('msg_iov', POINTER(iovec)), ('msg_iovlen', size_t),
                ('msg_control', c_void_p), ('msg_controllen', size_t),
                ('msg_flags', c_int)]

    def __init__(self, proto, cmd=None, fdobj=None, client=None):
        # If no 'fd' parameter is passed and no 'cmd' parameter is passed upon
        # initialization, this is a header for a "receiver" call.  Otherwise
        # this will initialized for a "sender" call, which will need a slightly
        # different arrangement of the structure's fields.
        #
        # Of The "sender" and "receiver" 'msghdr's, the "sender" 'iovec' struct
        # will be set with a command protocol number inside of 'iovec' struct
        # base field array for the transmission over `sendmsg` and the
        # "receiver" 'iovec' will be initialized with a empty int array large
        # enough to contain that message passed through  `recvmsg`.
        #
        # With the cmsghdr its the similiar idea, where the "sender" will
        # have the struct initialized with its data set and the "receiver"
        # will have an empty array assigned to its data field.
        if proto == RECV or cmd is not None and (PEER_DUMP | PEER_RECV) & cmd:
            ctrl_msg = CTRL_MSG_RECV()
        elif cmd is not None and not (PEER_DUMP | PEER_RECV) & cmd:
            ctrl_msg = pointer(cmsghdr(fdobj.fd))
        else:
            raise MsghdrError('InvalidArg fd needs cmd')
        if cmd is not None and PEER_RECV & cmd:
            iov_base = fdobj
        else:
            iov_base = pointer(fdmsg(proto, cmd, fdobj, client))
        self.msg_iov = pointer(iovec(iov_base))
        self.msg_iovlen = size_t(FDBUS_IOVLEN)
        self.msg_control = cast(ctrl_msg, c_void_p)
        self.msg_controllen = CTRL_MSG_LEN

class cmsghdr(Structure):

    _fields_ = [('cmsg_len', size_t), ('cmsg_level', c_int),
                ('cmsg_type', c_int), ('cmsg_data', c_int)]

    def __init__(self, fd):
        self.cmsg_len = CMSG_LEN(sizeof(c_int))
        self.cmsg_level = SOL_SOCKET
        self.cmsg_type = SCM_RIGHTS
        self.cmsg_data = fd

class cmsghdr_flex(Structure):

    # This is a custom cmsghdr to use with the CMSG macros.  The only difference
    # between this struct and the cmsghdr struct above is that the 'cmsg_data'
    # field isn't in the former.  This is to account for the 'cmsg_data' field
    # being a flex array and the macros calculating the size.

    _fields_ = [('cmsg_len', c_int),
                ('cmsg_level', c_int),
                ('cmsg_type', c_int)]

# cmsg macros
CMSG_SPACE = lambda container: (CMSG_ALIGN(sizeof(cmsghdr_flex)) +
                                CMSG_ALIGN(container))

CMSG_LEN = lambda data_type: CMSG_ALIGN(sizeof(cmsghdr_flex)) + data_type

CMSG_ALIGN = lambda data_len: ((data_len + sizeof(c_int) - 1) &
                               ~(sizeof(c_int) - 1))

CMSG_DATA = lambda msg: cast(msg, POINTER(cmsghdr)).contents.cmsg_data

CTRL_MSG_LEN = CMSG_SPACE(sizeof(c_int))
CTRL_MSG_RECV = c_char * CTRL_MSG_LEN
<commit_msg>reworking protocol<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from ctypes import c_int, c_uint, c_longlong, c_ushort, c_char, \
                   c_void_p, c_char_p, POINTER, pointer, cast, \
                   sizeof, Structure, CDLL, get_errno

from exceptions.exceptions import *
from collections import defaultdict

libc = CDLL('libc.so.6', use_errno=True)

libc.open.restype = c_int
libc.socket.restype = c_int
libc.accept.restype = c_int
libc.strerror.argtype = c_int
libc.strerror.restype = c_char_p

# set command protocols for communication to server-client
#
# to start have the send/request of a fd - basic -
# an additional cmd might be to request server status information
# on a multiplex server the cmds will natural become more involved
#
# commands for client-server communica

LOAD = 0x0
LOAD_RDONLY = 0x0
LOAD_WRONLY = 0x1
LOAD_RDWR   = 0x2

RECV = 0x1
RECV_FD   = 0x1
RECV_PEER = 0x2
RECV_CMD  = 0x4

PASS = 0x10
PASS_FD   = 0x10
PASS_PEER = 0x20

CLOSE = 0x100
CLS_FD  = 0x100
CLS_ALL = 0x200

REFERENCE = 0x200
RET_FD    = 0x100
REFCNT_FD = 0x200

# linux values
SOCK_ADDRDATA_SZ = 14
UNIX_PATH_MAX    = 108

FDBUS_IOVLEN = 0x1

O_RDONLY = 0x0
O_WRONLY = 0x1
O_RDWR   = 0x2

SCM_RIGHTS = c_int(0x01)
SOL_SOCKET = c_int(0x01)

SEEK_SET = 0
SEEK_CUR = 1
SEEK_END = 2

PROTO_DEFAULT = 0
SOCK_STREAM   = 1
AF_UNIX       = 1

DEFAULT_CLIENTS = 3

MSG_CMSG_CLOEXEC = 2 << 29
MSG_SERV = c_int(0)

# conversion of linux data types set to their ctypes size
socklen_t = c_uint
size_t = c_uint
off_t = c_longlong

def get_error_msg():
    errno = get_errno()
    return libc.strerror(c_int(errno))

class peermsg(Structure):

    _fields_ = [('length', c_int), ('peers', c_void_p)]

    def __init__(self, peers=None):
        if peers is None:
            self.length = c_int(0)
            self.peers = c_void_p()
        else:
            peer_length = len(peers)
            peer_array = ARRAY(c_int, peer_length)
            self.length = c_int(peer_length)
            self.peers = cast(c_void_p, peer_array(peers))

class fdmsg(Structure):

    _fields_ = [('protocol', c_int), ('command', c_int), ('name', c_char_p),
                ('path', c_char_p), ('created', c_char_p), ('mode', c_int),
                ('client', c_int)]

    def __init__(self, proto, cmd=None, fdobj=None, client=None):
        self.protocol = proto
        self.command = c_int(cmd) if cmd else c_int(0)
        self.client = c_int(-1) if client is None else c_int(client)
        if fdobj is None:
            self.name = self.path = self.created = c_char_p(None)
            self.mode = c_int(-1)
        else:
            self.name = c_char_p(fdobj.name)
            self.path = c_char_p(fdobj.path)
            self.created = c_char_p(fdobj.created)
            self.mode = c_int(fdobj.mode)

class sockaddr(Structure):

    _fields_ = [('sa_family', c_ushort),
                ('sa_data', c_char * SOCK_ADDRDATA_SZ)]

class sockaddr_un(Structure):

    _fields_ = [('sun_family', c_ushort),
                ('sun_path', c_char * UNIX_PATH_MAX)]

    def __init__(self, family, path):
        self.sun_family = c_ushort(family)
        self.sun_path = path

class iovec(Structure):

    _fields_ = [('iov_base', c_void_p), ('iov_len', size_t)]

    def __init__(self, io_data):
        self.iov_base = cast(io_data, c_void_p)
        self.iov_len = size_t(sizeof(fdmsg) * FDBUS_IOVLEN)

class msghdr(Structure):

    _fields_ = [('msg_name', c_void_p), ('msg_namelen', socklen_t),
                ('msg_iov', POINTER(iovec)), ('msg_iovlen', size_t),
                ('msg_control', c_void_p), ('msg_controllen', size_t),
                ('msg_flags', c_int)]

    def __init__(self, proto, cmd, fdobj=None, client=None):
        # If no 'fd' parameter is passed and no 'cmd' parameter is passed upon
        # initialization, this is a header for a "receiver" call.  Otherwise
        # this will initialized for a "sender" call, which will need a slightly
        # different arrangement of the structure's fields.
        #
        # Of The "sender" and "receiver" 'msghdr's, the "sender" 'iovec' struct
        # will be set with a command protocol number inside of 'iovec' struct
        # base field array for the transmission over `sendmsg` and the
        # "receiver" 'iovec' will be initialized with a empty int array large
        # enough to contain that message passed through  `recvmsg`.
        #
        # With the cmsghdr its the similiar idea, where the "sender" will
        # have the struct initialized with its data set and the "receiver"
        # will have an empty array assigned to its data field.
        if proto == RECV:
            ctrl_msg = CTRL_MSG_RECV()
        else:
            ctrl_msg = pointer(cmsghdr(fdobj.fd))
        if cmd == PASS_PEER:
            iov_base = fdobj
        else:
            iov_base = pointer(fdmsg(proto, cmd, fdobj, client))
        self.msg_iov = pointer(iovec(iov_base))
        self.msg_iovlen = size_t(FDBUS_IOVLEN)
        self.msg_control = cast(ctrl_msg, c_void_p)
        self.msg_controllen = CTRL_MSG_LEN

class cmsghdr(Structure):

    _fields_ = [('cmsg_len', size_t), ('cmsg_level', c_int),
                ('cmsg_type', c_int), ('cmsg_data', c_int)]

    def __init__(self, fd):
        self.cmsg_len = CMSG_LEN(sizeof(c_int))
        self.cmsg_level = SOL_SOCKET
        self.cmsg_type = SCM_RIGHTS
        self.cmsg_data = fd

class cmsghdr_flex(Structure):

    # This is a custom cmsghdr to use with the CMSG macros.  The only difference
    # between this struct and the cmsghdr struct above is that the 'cmsg_data'
    # field isn't in the former.  This is to account for the 'cmsg_data' field
    # being a flex array and the macros calculating the size.

    _fields_ = [('cmsg_len', c_int),
                ('cmsg_level', c_int),
                ('cmsg_type', c_int)]

# cmsg macros
CMSG_SPACE = lambda container: (CMSG_ALIGN(sizeof(cmsghdr_flex)) +
                                CMSG_ALIGN(container))

CMSG_LEN = lambda data_type: CMSG_ALIGN(sizeof(cmsghdr_flex)) + data_type

CMSG_ALIGN = lambda data_len: ((data_len + sizeof(c_int) - 1) &
                               ~(sizeof(c_int) - 1))

CMSG_DATA = lambda msg: cast(msg, POINTER(cmsghdr)).contents.cmsg_data

CTRL_MSG_LEN = CMSG_SPACE(sizeof(c_int))
CTRL_MSG_RECV = c_char * CTRL_MSG_LEN
<|endoftext|>


<commit_before>import yaml

def configReader(self):
    stream = open('spudscale.config', 'r')
    return yaml.load_all(stream)
<commit_msg>fixed small error<commit_after>import yaml

def configReader():
    stream = open('spudscale.config', 'r')
    return yaml.load_all(stream)
<|endoftext|>


<commit_before><commit_msg>Support setting a title for the no-app choice<commit_after><|endoftext|>