<commit_before>import sys
import json
import numpy as np

from scipy.stats import sem

from sklearn.cross_validation import KFold
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.grid_search import RandomizedSearchCV, GridSearchCV
from sklearn.utils import shuffle
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import StandardScaler
from sklearn.feature_selection import SelectKBest, chi2

from ambra.classifiers import IntervalLogisticRegression
from ambra.interval_scoring import semeval_interval_scorer

class MyPipeline(Pipeline):
    def predict(self, X, Y_possible):
            """Applies transforms to the data, and the predict method of the
            final estimator. Valid only if the final estimator implements
            predict."""
            Xt = X
            for name, transform in self.steps[:-1]:
                Xt = transform.transform(Xt)
            return self.steps[-1][-1].predict(Xt, Y_possible)

class Proj(BaseEstimator, TransformerMixin):
    """Projection metaestimator for list-of-dict kind of data"""
    def __init__(self, transf, key):
        self.transf = transf
        self.key = key

    def fit(self, X, y=None):
        return self.transf.fit([x[self.key] for x in X], y)

    def transform(self, X):
        return self.transf.transform([x[self.key] for x in X])

    def fit_transform(self, X, y=None):
        return self.transf.fit_transform([x[self.key] for x in X], y)

    def get_feature_names(self):
        return self.transf.get_feature_names()

class LengthFeatures(BaseEstimator, TransformerMixin):
    def fit(self, X, y=None):
        return self

    def _doc_features(self, doc):
        n_sents = len(doc)
        all_toks = [tok for sent in doc for tok in sent]
        n_tokens = len(all_toks)
        n_types = len(set(all_toks))
        type_token_ratio = n_tokens / float(n_types)
        return np.array([n_sents, n_tokens, n_types, type_token_ratio],
                        dtype=np.float)

    def transform(self, X, y=None):
        #x in X is a list of sents
        return np.row_stack([self._doc_features(doc) for doc in X])

    def get_feature_names(self):
        return ['n_sents', 'n_tokens', 'n_types', 'type_token_ratio']

class NgramLolAnalyzer(BaseEstimator):
    def _word_ngrams(self, tokens, stop_words=None):
        """Turn tokens into a sequence of n-grams after stop words filtering"""
        # handle stop words
        if stop_words is not None:
            tokens = [w for w in tokens if w not in stop_words]
        if self.lower:
            tokens = [w.lower() for w in tokens]
        # handle token n-grams
        min_n, max_n = self.ngram_range
        if max_n != 1:
            original_tokens = tokens
            tokens = []
            n_original_tokens = len(original_tokens)
            for n in xrange(min_n,
                            min(max_n + 1, n_original_tokens + 1)):
                for i in xrange(n_original_tokens - n + 1):
                    tokens.append(" ".join(original_tokens[i: i + n]))
        return tokens

    def __init__(self, ngram_range=(1,1), lower=False):
        self.ngram_range=ngram_range
        self.lower = lower

    def __call__(self, doc):
        return [feature for sentence in doc
                for feature in self._word_ngrams(sentence)]

class MySelectKBest(SelectKBest):
    def fit(self, X, y):
	y_flat = [get_coarse_interval(np.mean(yelem)) for yelem in y]
        return super(MySelectKBest, self).fit(X, y_flat)

def get_coarse_interval(year):
    year = int(year)
    lower = year / 100 * 100 + 50 * (year % 100 >= 50)
    return str(lower) + "-" + str(lower + 50)

with open(sys.argv[1]) as f:
    entries = json.load(f)

# some buggy docs are empty
entries = [entry for entry in entries if len(entry['lemmas'])]

X = np.array(entries)
Y = np.array([doc['interval'] for doc in entries])
Y_possible = np.array([doc['all_fine_intervals'] for doc in entries])

X, Y, Y_possible = shuffle(X, Y, Y_possible, random_state=0)

# make it run fast
limit = 500
X = X[:limit]
Y = Y[:limit]
Y_possible = Y_possible[:limit]

pipe = MyPipeline([('vect', Proj(LengthFeatures(), key='lemmas')),
                   ('scale', StandardScaler(with_mean=False, with_std=True)),
                   ('clf', IntervalLogisticRegression(n_neighbors=10,
                                                      limit_pairs=0.1,
                                                      random_state=0))])

grid = GridSearchCV(pipe,
                    dict(clf__C=np.logspace(-3, 3, 3)),
                    verbose=True, cv=KFold(len(X), n_folds=5),
                    scoring=semeval_interval_scorer,
                    scorer_params=dict(Y_possible=Y_possible),
                    n_jobs=1)

grid.fit(X, Y)
grid_scores = [k.mean_validation_score for k in grid.grid_scores_]
print "{:.3f} +/- {:.4f}".format(grid.best_score_, sem(grid_scores))
print grid.best_estimator_.steps[-1][-1].n_pairs_, " total pairs."
print grid.best_estimator_.steps[-1][-1].coef_.ravel()

vectorizer = TfidfVectorizer(use_idf=False, norm='l1', analyzer=NgramLolAnalyzer(lower=False))

union = FeatureUnion([('lenghts', Proj(LengthFeatures(), key='lemmas')),
                      ('pos', Proj(vectorizer, key='pos')),
                      ('tokens', Proj(vectorizer, key='tokens'))])

pipe = MyPipeline([('union', union),
                   ('scale', StandardScaler(with_mean=False, with_std=True)),
                   ('fs', MySelectKBest(chi2)),
                   ('clf', IntervalLogisticRegression(n_neighbors=10,
                                                      limit_pairs=0.1,
                                                      random_state=0))])

grid_params = {
	'clf__C' : np.logspace(-3, 3, 3),
	'union__pos__transf__min_df' : [10, 5, 1],
	'union__pos__transf__max_df' : [1.0, 0.99, 0.9, 0.8],
    'union__pos__transf__analyzer__ngram_range' : [(2, 3), (2, 2)],
	'union__tokens__transf__min_df' : [10, 5, 1],
	'union__tokens__transf__max_df' : [1.0, 0.99, 0.9, 0.8],
    'union__tokens__transf__analyzer__ngram_range' : [(1, 3), (1, 2), (1, 1)],
    'fs__k' : [50, 200, 400]
}

grid = RandomizedSearchCV(pipe,
                          grid_params,
                          verbose=True, cv=KFold(len(X), n_folds=5),
                          scoring=semeval_interval_scorer,
                          scorer_params=dict(Y_possible=Y_possible),
                          n_iter=10,
                          n_jobs=4)

grid.fit(X, Y)
grid_scores = [k.mean_validation_score for k in grid.grid_scores_]
print "{:.3f} +/- {:.4f}".format(grid.best_score_, sem(grid_scores))
print grid.best_params_

feature_names = grid.best_estimator_.steps[0][1].get_feature_names()
coef = grid.best_estimator_.steps[-1][-1].coef_.ravel()

for idx in np.argsort(np.abs(coef))[:10]:
    print("{:.2f}\t{}".format(coef[idx], feature_names[idx]))

<commit_msg>Update semeval script to use style features<commit_after>from __future__ import print_function
import sys
import json
import numpy as np

from scipy.stats import sem

from sklearn.cross_validation import KFold
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.grid_search import RandomizedSearchCV, GridSearchCV
from sklearn.utils import shuffle
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import StandardScaler
from sklearn.feature_selection import SelectKBest, chi2

from ambra.classifiers import IntervalLogisticRegression
from ambra.interval_scoring import semeval_interval_scorer

class MyPipeline(Pipeline):
    def predict(self, X, Y_possible):
            """Applies transforms to the data, and the predict method of the
            final estimator. Valid only if the final estimator implements
            predict."""
            Xt = X
            for name, transform in self.steps[:-1]:
                Xt = transform.transform(Xt)
            return self.steps[-1][-1].predict(Xt, Y_possible)

class Proj(BaseEstimator, TransformerMixin):
    """Projection metaestimator for list-of-dict kind of data"""
    def __init__(self, transf, key):
        self.transf = transf
        self.key = key

    def fit(self, X, y=None):
        return self.transf.fit([x[self.key] for x in X], y)

    def transform(self, X):
        return self.transf.transform([x[self.key] for x in X])

    def fit_transform(self, X, y=None):
        return self.transf.fit_transform([x[self.key] for x in X], y)

    def get_feature_names(self):
        return self.transf.get_feature_names()

class LengthFeatures(BaseEstimator, TransformerMixin):
    def fit(self, X, y=None):
        return self

    def _doc_features(self, doc):
        n_sents = len(doc)
        all_toks = [tok for sent in doc for tok in sent]
        n_tokens = len(all_toks)
        n_types = len(set(all_toks))
        #type_token_ratio = n_tokens / float(n_types)
        return np.array([n_sents, n_tokens, n_types,
                         #type_token_ratio
                         ],
                        dtype=np.float)

    def transform(self, X, y=None):
        #x in X is a list of sents
        return np.row_stack([self._doc_features(doc) for doc in X])

    def get_feature_names(self):
        return ['n_sents', 'n_tokens', 'n_types',
        #'type_token_ratio'
        ]

class StylisticFeatures(BaseEstimator, TransformerMixin):
    def __init__(self, lower=True):
        self.lower = lower

    def fit(self, X, y=None):
        return self

    def _doc_features(self, doc):
        # doc is a dict
        tokens = doc['tokens']
        lemmas = doc['lemmas']
        all_tokens = [w.lower() if self.lower else w
                      for sent in tokens for w in sent]
        all_lemmas = [w.lower() if self.lower else w
                      for sent in lemmas for w in sent]
        avg_sent_len = np.mean([len(sent) for sent in tokens])
        avg_word_len = np.mean([len(w) for w in tokens])
        lex_density = len(set(all_tokens)) / len(all_tokens)
        lex_richness = len(set(all_lemmas)) / len(all_lemmas)
        return np.array([avg_sent_len, avg_word_len, lex_density, lex_richness],
                        dtype=np.float)

    def transform(self, X, y=None):
        #x in X is a list of sents
        return np.row_stack([self._doc_features(doc) for doc in X])

    def get_feature_names(self):
        return ['ASL', 'AWL', 'LD', 'LR']

class NgramLolAnalyzer(BaseEstimator):
    def _word_ngrams(self, tokens, stop_words=None):
        """Turn tokens into a sequence of n-grams after stop words filtering"""
        # handle stop words
        if stop_words is not None:
            tokens = [w for w in tokens if w not in stop_words]
        if self.lower:
            tokens = [w.lower() for w in tokens]
        # handle token n-grams
        min_n, max_n = self.ngram_range
        if max_n != 1:
            original_tokens = tokens
            tokens = []
            n_original_tokens = len(original_tokens)
            for n in xrange(min_n,
                            min(max_n + 1, n_original_tokens + 1)):
                for i in xrange(n_original_tokens - n + 1):
                    tokens.append(" ".join(original_tokens[i: i + n]))
        return tokens

    def __init__(self, ngram_range=(1,1), lower=False):
        self.ngram_range=ngram_range
        self.lower = lower

    def __call__(self, doc):
        return [feature for sentence in doc
                for feature in self._word_ngrams(sentence)]

class MySelectKBest(SelectKBest):
    def fit(self, X, y):
	y_flat = [get_coarse_interval(np.mean(yelem)) for yelem in y]
        return super(MySelectKBest, self).fit(X, y_flat)

def get_coarse_interval(year):
    year = int(year)
    lower = year / 100 * 100 + 50 * (year % 100 >= 50)
    return str(lower) + "-" + str(lower + 50)

with open(sys.argv[1]) as f:
    entries = json.load(f)

# some buggy docs are empty
entries = [entry for entry in entries if len(entry['lemmas'])]

X = np.array(entries)
Y = np.array([doc['interval'] for doc in entries])
Y_possible = np.array([doc['all_fine_intervals'] for doc in entries])

X, Y, Y_possible = shuffle(X, Y, Y_possible, random_state=0)

# make it run fast
limit = None  # 500
limit_pairs = 0.01
if limit:
    X = X[:limit]
    Y = Y[:limit]
    Y_possible = Y_possible[:limit]

print("Length features")
print("===============")
pipe = MyPipeline([('vect', Proj(LengthFeatures(), key='lemmas')),
                   ('scale', StandardScaler(with_mean=False, with_std=True)),
                   ('clf', IntervalLogisticRegression(n_neighbors=10,
                                                      limit_pairs=limit_pairs,
                                                      random_state=0))])

grid = GridSearchCV(pipe,
                    dict(clf__C=np.logspace(-3, 3, 7)),
                    verbose=False, cv=KFold(len(X), n_folds=5),
                    scoring=semeval_interval_scorer,
                    scorer_params=dict(Y_possible=Y_possible),
                    n_jobs=1)

grid.fit(X, Y)
grid_scores = [k.mean_validation_score for k in grid.grid_scores_]
print("{:.3f} +/- {:.4f}".format(grid.best_score_, sem(grid_scores)))
print(grid.best_estimator_.steps[-1][-1].n_pairs_, " total pairs.")
print(grid.best_estimator_.steps[-1][-1].coef_.ravel())

print()
print("Stylistic features")
print("==================")

union = FeatureUnion([('lenghts', Proj(LengthFeatures(), key='lemmas')),
                      ('style', StylisticFeatures())])

pipe = MyPipeline([('vect', union),
                   ('scale', StandardScaler(with_mean=False, with_std=True)),
                   ('clf', IntervalLogisticRegression(n_neighbors=10,
                                                      limit_pairs=limit_pairs,
                                                      random_state=0))])

grid = GridSearchCV(pipe,
                    dict(clf__C=np.logspace(-3, 3, 7)),
                    verbose=False, cv=KFold(len(X), n_folds=5),
                    scoring=semeval_interval_scorer,
                    scorer_params=dict(Y_possible=Y_possible),
                    n_jobs=1)

grid.fit(X, Y)
grid_scores = [k.mean_validation_score for k in grid.grid_scores_]
print("{:.3f} +/- {:.4f}".format(grid.best_score_, sem(grid_scores)))
print(grid.best_estimator_.steps[-1][-1].n_pairs_, " total pairs.")
print(grid.best_estimator_.steps[-1][-1].coef_.ravel())

print()
print("Lex and POS features")
print("====================")

vectorizer = TfidfVectorizer(use_idf=False, norm='l1', analyzer=NgramLolAnalyzer(lower=False))

union = FeatureUnion([('lenghts', Proj(LengthFeatures(), key='lemmas')),
                      ('style', StylisticFeatures()),
                      ('pos', Proj(vectorizer, key='pos')),
                      ('tokens', Proj(vectorizer, key='tokens'))])

pipe = MyPipeline([('union', union),
                   ('scale', StandardScaler(with_mean=False, with_std=True)),
                   ('fs', MySelectKBest(chi2)),
                   ('clf', IntervalLogisticRegression(n_neighbors=10,
                                                      limit_pairs=limit_pairs,
                                                      random_state=0))])

grid_params = {
	'clf__C' : np.logspace(-3, 3, 7),
	'union__pos__transf__min_df' : [10, 5, 1],
	'union__pos__transf__max_df' : [1.0, 0.99, 0.9, 0.8],
    'union__pos__transf__analyzer__ngram_range' : [(2, 3), (2, 2)],
	'union__tokens__transf__min_df' : [10, 5, 1],
	'union__tokens__transf__max_df' : [1.0, 0.99, 0.9, 0.8],
    'union__tokens__transf__analyzer__ngram_range' : [(1, 3), (1, 2), (1, 1)],
    'fs__k' : [50, 200, 400]
}

grid = RandomizedSearchCV(pipe,
                          grid_params,
                          verbose=True, cv=KFold(len(X), n_folds=5),
                          scoring=semeval_interval_scorer,
                          scorer_params=dict(Y_possible=Y_possible),
                          n_iter=4,
                          n_jobs=1,
                          random_state=0)

grid.fit(X, Y)
grid_scores = [k.mean_validation_score for k in grid.grid_scores_]
print("{:.3f} +/- {:.4f}".format(grid.best_score_, sem(grid_scores)))
print(grid.best_params_)

feature_names = grid.best_estimator_.steps[0][1].get_feature_names()
coef = grid.best_estimator_.steps[-1][-1].coef_.ravel()

for idx in np.argsort(-np.abs(coef))[:50]:
    print("{:.2f}\t{}".format(coef[idx], feature_names[idx]))

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from holmes.validators import BASE_VALIDATORS  # NOQA

__version__ = '0.9.0'
<commit_msg>Releasing 0.9.1<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from holmes.validators import BASE_VALIDATORS  # NOQA

__version__ = '0.9.1'
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Tests for cli.py."""
import os
from unittest import mock

from click.testing import CliRunner
import pytest

from ubittool import cli, cmds

@pytest.fixture
def check_no_board_connected():
    """Check that there is no mbed board that PyOCD can connect to."""
    try:
        cmds._read_continuous_memory(address=0x00, count=16)
    except Exception:
        # Good: Exception raised if no board is found
        pass
    else:
        raise Exception("Found an Mbed device connected, please unplug.")

@mock.patch("ubittool.cli.os.path.exists", autospec=True)
@mock.patch("ubittool.cli.click.echo", autospec=True)
@mock.patch("ubittool.cli.sys.exit", autospec=True)
def test_file_checker(mock_exit, mock_echo, mock_exists):
    """Test the file checker perform the required checks and prints info."""
    mock_exists.return_value = False

    cli._file_checker("subject", "file/path.py")

    mock_exists.assert_called_once_with("file/path.py")
    assert mock_echo.call_count == 1
    assert "subject will be written to: file/path.py" in mock_echo.call_args[0]
    assert mock_exit.call_count == 0

@mock.patch("ubittool.cli.os.path.exists", autospec=True)
@mock.patch("ubittool.cli.click.echo", autospec=True)
@mock.patch("ubittool.cli.sys.exit", autospec=True)
def test_file_checker_existing_path(mock_exit, mock_echo, mock_exists):
    """Test file checker exits with error if the file exists."""
    mock_exists.return_value = True

    cli._file_checker("subject", "file/path.py")

    mock_exists.assert_called_once_with("file/path.py")
    assert mock_echo.call_count == 1
    assert (
        "Abort: The file/path.py file already exists."
        in mock_echo.call_args[0][0]
    )
    mock_exit.assert_called_once_with(1)

@mock.patch("ubittool.cli.click.echo", autospec=True)
@mock.patch("ubittool.cli.sys.exit", autospec=True)
def test_file_checker_no_path(mock_exit, mock_echo):
    """Test the file check informs about console output if no file is given."""
    cli._file_checker("subject", None)

    assert mock_echo.call_count == 1
    assert "subject will be output to console." in mock_echo.call_args[0]
    assert mock_exit.call_count == 0

@mock.patch("ubittool.cli.read_python_code", autospec=True)
def test_read_code(mock_read_python_code, check_no_board_connected):
    """Test the read-code command without a file option."""
    python_code = "Python code here"
    mock_read_python_code.return_value = python_code
    runner = CliRunner()

    result = runner.invoke(cli.read_code)

    assert "MicroPython code will be output to console." in result.output
    assert "Printing the MicroPython code" in result.output
    assert python_code in result.output
    assert "Finished successfully" in result.output
    assert result.exit_code == 0

def test_read_code_no_board(check_no_board_connected):
    """Test the read-code command when no board is connected."""
    runner = CliRunner()

    result = runner.invoke(cli.read_code)

    assert result.exit_code != 0
    assert "MicroPython code will be output to console." in result.output
    assert "Did not find any connected boards." in result.output

@mock.patch("ubittool.cli.read_python_code", autospec=True)
def test_read_code_path(mock_read_python_code, check_no_board_connected):
    """Test the read-code command with a file option."""
    mock_read_python_code.return_value = "Python code here"
    runner = CliRunner()

    with mock.patch("ubittool.cli.open", mock.mock_open()) as m_open:
        result = runner.invoke(cli.read_code, ["--file_path", "thisfile.py"])

    m_open.assert_called_once_with("thisfile.py", "w")
    m_open().write.assert_called_once_with("Python code here")
    assert "MicroPython code will be written to: thisfile.py" in result.output
    assert "Saving the MicroPython code..." in result.output
    assert "Finished successfully" in result.output
    assert result.exit_code == 0

def test_read_code_path_no_board(check_no_board_connected):
    """Test read-code command with a file option and no board connected."""
    file_name = "thisfile.py"
    runner = CliRunner()

    results = [
        runner.invoke(cli.read_code, ["--file_path", file_name]),
        runner.invoke(cli.read_code, ["-f", file_name]),
    ]

    for result in results:
        assert result.exit_code != 0, "Exit code non-zero"
        assert (
            "MicroPython code will be written to: {}".format(file_name)
            in result.output
        ), "Message written to file"
        assert (
            "Did not find any connected boards." in result.output
        ), "Message error, board not found"
    # File not mocked, so checking command hasn't created it
    assert not os.path.isfile(file_name), "File does not exist"

@mock.patch("ubittool.cli.read_flash_hex", autospec=True)
def test_read_flash(mock_read_flash_hex, check_no_board_connected):
    """Test the read-flash command without a file option."""
    flash_hex_content = "Intel Hex lines here"
    mock_read_flash_hex.return_value = flash_hex_content
    runner = CliRunner()

    result = runner.invoke(cli.read_flash)

    assert "micro:bit flash hex will be output to console." in result.output
    assert "Printing the flash contents" in result.output
    assert flash_hex_content in result.output
    assert "Finished successfully" in result.output
    assert result.exit_code == 0

def test_read_flash_no_board(check_no_board_connected):
    """Test the read-flash command when no board is connected."""
    runner = CliRunner()

    result = runner.invoke(cli.read_flash)

    assert result.exit_code != 0
    assert "micro:bit flash hex will be output to console." in result.output
    assert "Did not find any connected boards." in result.output

@mock.patch("ubittool.cli.read_flash_hex", autospec=True)
def test_read_flash_path(mock_read_flash_hex, check_no_board_connected):
    """Test the read-code command with a file option."""
    flash_hex_content = "Intel Hex lines here"
    mock_read_flash_hex.return_value = flash_hex_content
    file_name = "thisfile.py"
    runner = CliRunner()

    with mock.patch("ubittool.cli.open", mock.mock_open()) as m_open:
        results = [runner.invoke(cli.read_flash, ["--file_path", file_name])]
    with mock.patch("ubittool.cli.open", mock.mock_open()) as m_open2:
        results.append(runner.invoke(cli.read_flash, ["-f", file_name]))

    m_open.assert_called_once_with(file_name, "w")
    m_open2.assert_called_once_with(file_name, "w")
    m_open().write.assert_called_once_with(flash_hex_content)
    m_open2().write.assert_called_once_with(flash_hex_content)
    for result in results:
        assert (
            "micro:bit flash hex will be written to: {}".format(file_name)
            in result.output
        )
        assert "Saving the flash contents..." in result.output
        assert "Finished successfully" in result.output
        assert result.exit_code == 0

def test_read_flash_path_no_board(check_no_board_connected):
    """Test read-flash command with a file option and no board connected."""
    file_name = "thisfile.py"
    runner = CliRunner()

    results = [
        runner.invoke(cli.read_flash, ["--file_path", file_name]),
        runner.invoke(cli.read_flash, ["-f", file_name]),
    ]

    for result in results:
        assert result.exit_code != 0, "Exit code non-zero"
        assert (
            "micro:bit flash hex will be written to: {}".format(file_name)
            in result.output
        ), "Message written to file"
        assert (
            "Did not find any connected boards." in result.output
        ), "Message error, board not found"
    # File not mocked, so checking command hasn't created it
    assert not os.path.isfile(file_name), "File does not exist"

@mock.patch("ubittool.cli.os.path.isfile", autospec=True)
@mock.patch("ubittool.cli.compare_full_flash_hex", autospec=True)
def test_compare_flash(mock_compare, mock_isfile, check_no_board_connected):
    """Test the compare-flash command."""
    file_name = "random_file_name.hex"
    mock_isfile.return_value = True
    runner = CliRunner()

    results = [
        runner.invoke(cli.compare_flash, ["-f", file_name]),
        runner.invoke(cli.compare_flash, ["--file_path", file_name]),
    ]

    assert mock_compare.call_count == len(results)
    for result in results:
        assert "Diff output loaded in default browser." in result.output
        assert "Finished successfully!" in result.output
        assert result.exit_code == 0, "Exit code 0"

@mock.patch("ubittool.cli.os.path.isfile", autospec=True)
def test_compare_flash_no_board(mock_isfile, check_no_board_connected):
    """Test the compare-flash command when no board is connected."""
    file_name = "random_file_name.hex"
    file_content = "Intel Hex lines here"
    mock_isfile.return_value = True
    runner = CliRunner()

    with mock.patch(
        "ubittool.cmds.open", mock.mock_open(read_data=file_content)
    ) as m_open:
        results = [
            runner.invoke(cli.compare_flash, ["-f", file_name]),
            runner.invoke(cli.compare_flash, ["--file_path", file_name]),
        ]

    assert m_open.call_count == len(results)
    for result in results:
        assert result.exit_code != 0, "Exit code non-zero"
        assert "Did not find any connected boards." in result.output

def test_compare_flash_invalid_file():
    """Check error is thrown when compare-flash file does not exist."""
    file_name = "random_file_does_not_exist.hex"
    runner = CliRunner()

    results = [
        runner.invoke(cli.compare_flash, ["--file_path", file_name]),
        runner.invoke(cli.compare_flash, ["-f", file_name]),
    ]

    for result in results:
        assert result.exit_code != 0, "Exit code non-zero"
        assert "Abort: File does not exists" in result.output

def test_compare_flash_no_file():
    """Test there is an error when compare-flash doesn't have a file arg."""
    runner = CliRunner()

    result = runner.invoke(cli.compare_flash)

    assert result.exit_code != 0, "Exit code non-zero"
    assert 'Error: Missing option "-f" / "--file_path".' in result.output

@mock.patch("ubittool.gui.open_gui", autospec=True)
def test_gui(mock_open_gui, check_no_board_connected):
    """Test the gui command."""
    runner = CliRunner()

    result = runner.invoke(cli.gui)

    assert result.exit_code == 0, "Exit code 0"
    assert mock_open_gui.call_count == 1, "open_gui() function called"
<commit_msg>Tests: Update tested Click output text.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Tests for cli.py."""
import os
from unittest import mock

from click.testing import CliRunner
import pytest

from ubittool import cli, cmds

@pytest.fixture
def check_no_board_connected():
    """Check that there is no mbed board that PyOCD can connect to."""
    try:
        cmds._read_continuous_memory(address=0x00, count=16)
    except Exception:
        # Good: Exception raised if no board is found
        pass
    else:
        raise Exception("Found an Mbed device connected, please unplug.")

@mock.patch("ubittool.cli.os.path.exists", autospec=True)
@mock.patch("ubittool.cli.click.echo", autospec=True)
@mock.patch("ubittool.cli.sys.exit", autospec=True)
def test_file_checker(mock_exit, mock_echo, mock_exists):
    """Test the file checker perform the required checks and prints info."""
    mock_exists.return_value = False

    cli._file_checker("subject", "file/path.py")

    mock_exists.assert_called_once_with("file/path.py")
    assert mock_echo.call_count == 1
    assert "subject will be written to: file/path.py" in mock_echo.call_args[0]
    assert mock_exit.call_count == 0

@mock.patch("ubittool.cli.os.path.exists", autospec=True)
@mock.patch("ubittool.cli.click.echo", autospec=True)
@mock.patch("ubittool.cli.sys.exit", autospec=True)
def test_file_checker_existing_path(mock_exit, mock_echo, mock_exists):
    """Test file checker exits with error if the file exists."""
    mock_exists.return_value = True

    cli._file_checker("subject", "file/path.py")

    mock_exists.assert_called_once_with("file/path.py")
    assert mock_echo.call_count == 1
    assert (
        "Abort: The file/path.py file already exists."
        in mock_echo.call_args[0][0]
    )
    mock_exit.assert_called_once_with(1)

@mock.patch("ubittool.cli.click.echo", autospec=True)
@mock.patch("ubittool.cli.sys.exit", autospec=True)
def test_file_checker_no_path(mock_exit, mock_echo):
    """Test the file check informs about console output if no file is given."""
    cli._file_checker("subject", None)

    assert mock_echo.call_count == 1
    assert "subject will be output to console." in mock_echo.call_args[0]
    assert mock_exit.call_count == 0

@mock.patch("ubittool.cli.read_python_code", autospec=True)
def test_read_code(mock_read_python_code, check_no_board_connected):
    """Test the read-code command without a file option."""
    python_code = "Python code here"
    mock_read_python_code.return_value = python_code
    runner = CliRunner()

    result = runner.invoke(cli.read_code)

    assert "MicroPython code will be output to console." in result.output
    assert "Printing the MicroPython code" in result.output
    assert python_code in result.output
    assert "Finished successfully" in result.output
    assert result.exit_code == 0

def test_read_code_no_board(check_no_board_connected):
    """Test the read-code command when no board is connected."""
    runner = CliRunner()

    result = runner.invoke(cli.read_code)

    assert result.exit_code != 0
    assert "MicroPython code will be output to console." in result.output
    assert "Did not find any connected boards." in result.output

@mock.patch("ubittool.cli.read_python_code", autospec=True)
def test_read_code_path(mock_read_python_code, check_no_board_connected):
    """Test the read-code command with a file option."""
    mock_read_python_code.return_value = "Python code here"
    runner = CliRunner()

    with mock.patch("ubittool.cli.open", mock.mock_open()) as m_open:
        result = runner.invoke(cli.read_code, ["--file_path", "thisfile.py"])

    m_open.assert_called_once_with("thisfile.py", "w")
    m_open().write.assert_called_once_with("Python code here")
    assert "MicroPython code will be written to: thisfile.py" in result.output
    assert "Saving the MicroPython code..." in result.output
    assert "Finished successfully" in result.output
    assert result.exit_code == 0

def test_read_code_path_no_board(check_no_board_connected):
    """Test read-code command with a file option and no board connected."""
    file_name = "thisfile.py"
    runner = CliRunner()

    results = [
        runner.invoke(cli.read_code, ["--file_path", file_name]),
        runner.invoke(cli.read_code, ["-f", file_name]),
    ]

    for result in results:
        assert result.exit_code != 0, "Exit code non-zero"
        assert (
            "MicroPython code will be written to: {}".format(file_name)
            in result.output
        ), "Message written to file"
        assert (
            "Did not find any connected boards." in result.output
        ), "Message error, board not found"
    # File not mocked, so checking command hasn't created it
    assert not os.path.isfile(file_name), "File does not exist"

@mock.patch("ubittool.cli.read_flash_hex", autospec=True)
def test_read_flash(mock_read_flash_hex, check_no_board_connected):
    """Test the read-flash command without a file option."""
    flash_hex_content = "Intel Hex lines here"
    mock_read_flash_hex.return_value = flash_hex_content
    runner = CliRunner()

    result = runner.invoke(cli.read_flash)

    assert "micro:bit flash hex will be output to console." in result.output
    assert "Printing the flash contents" in result.output
    assert flash_hex_content in result.output
    assert "Finished successfully" in result.output
    assert result.exit_code == 0

def test_read_flash_no_board(check_no_board_connected):
    """Test the read-flash command when no board is connected."""
    runner = CliRunner()

    result = runner.invoke(cli.read_flash)

    assert result.exit_code != 0
    assert "micro:bit flash hex will be output to console." in result.output
    assert "Did not find any connected boards." in result.output

@mock.patch("ubittool.cli.read_flash_hex", autospec=True)
def test_read_flash_path(mock_read_flash_hex, check_no_board_connected):
    """Test the read-code command with a file option."""
    flash_hex_content = "Intel Hex lines here"
    mock_read_flash_hex.return_value = flash_hex_content
    file_name = "thisfile.py"
    runner = CliRunner()

    with mock.patch("ubittool.cli.open", mock.mock_open()) as m_open:
        results = [runner.invoke(cli.read_flash, ["--file_path", file_name])]
    with mock.patch("ubittool.cli.open", mock.mock_open()) as m_open2:
        results.append(runner.invoke(cli.read_flash, ["-f", file_name]))

    m_open.assert_called_once_with(file_name, "w")
    m_open2.assert_called_once_with(file_name, "w")
    m_open().write.assert_called_once_with(flash_hex_content)
    m_open2().write.assert_called_once_with(flash_hex_content)
    for result in results:
        assert (
            "micro:bit flash hex will be written to: {}".format(file_name)
            in result.output
        )
        assert "Saving the flash contents..." in result.output
        assert "Finished successfully" in result.output
        assert result.exit_code == 0

def test_read_flash_path_no_board(check_no_board_connected):
    """Test read-flash command with a file option and no board connected."""
    file_name = "thisfile.py"
    runner = CliRunner()

    results = [
        runner.invoke(cli.read_flash, ["--file_path", file_name]),
        runner.invoke(cli.read_flash, ["-f", file_name]),
    ]

    for result in results:
        assert result.exit_code != 0, "Exit code non-zero"
        assert (
            "micro:bit flash hex will be written to: {}".format(file_name)
            in result.output
        ), "Message written to file"
        assert (
            "Did not find any connected boards." in result.output
        ), "Message error, board not found"
    # File not mocked, so checking command hasn't created it
    assert not os.path.isfile(file_name), "File does not exist"

@mock.patch("ubittool.cli.os.path.isfile", autospec=True)
@mock.patch("ubittool.cli.compare_full_flash_hex", autospec=True)
def test_compare_flash(mock_compare, mock_isfile, check_no_board_connected):
    """Test the compare-flash command."""
    file_name = "random_file_name.hex"
    mock_isfile.return_value = True
    runner = CliRunner()

    results = [
        runner.invoke(cli.compare_flash, ["-f", file_name]),
        runner.invoke(cli.compare_flash, ["--file_path", file_name]),
    ]

    assert mock_compare.call_count == len(results)
    for result in results:
        assert "Diff output loaded in default browser." in result.output
        assert "Finished successfully!" in result.output
        assert result.exit_code == 0, "Exit code 0"

@mock.patch("ubittool.cli.os.path.isfile", autospec=True)
def test_compare_flash_no_board(mock_isfile, check_no_board_connected):
    """Test the compare-flash command when no board is connected."""
    file_name = "random_file_name.hex"
    file_content = "Intel Hex lines here"
    mock_isfile.return_value = True
    runner = CliRunner()

    with mock.patch(
        "ubittool.cmds.open", mock.mock_open(read_data=file_content)
    ) as m_open:
        results = [
            runner.invoke(cli.compare_flash, ["-f", file_name]),
            runner.invoke(cli.compare_flash, ["--file_path", file_name]),
        ]

    assert m_open.call_count == len(results)
    for result in results:
        assert result.exit_code != 0, "Exit code non-zero"
        assert "Did not find any connected boards." in result.output

def test_compare_flash_invalid_file():
    """Check error is thrown when compare-flash file does not exist."""
    file_name = "random_file_does_not_exist.hex"
    runner = CliRunner()

    results = [
        runner.invoke(cli.compare_flash, ["--file_path", file_name]),
        runner.invoke(cli.compare_flash, ["-f", file_name]),
    ]

    for result in results:
        assert result.exit_code != 0, "Exit code non-zero"
        assert "Abort: File does not exists" in result.output

def test_compare_flash_no_file():
    """Test there is an error when compare-flash doesn't have a file arg."""
    runner = CliRunner()

    result = runner.invoke(cli.compare_flash)

    assert result.exit_code != 0, "Exit code non-zero"
    assert "Error: Missing option '-f' / '--file_path'." in result.output

@mock.patch("ubittool.gui.open_gui", autospec=True)
def test_gui(mock_open_gui, check_no_board_connected):
    """Test the gui command."""
    runner = CliRunner()

    result = runner.invoke(cli.gui)

    assert result.exit_code == 0, "Exit code 0"
    assert mock_open_gui.call_count == 1, "open_gui() function called"
<|endoftext|>


<commit_before>from flask import Flask, render_template, redirect, url_for
from models.models import *
from src import repositories
from functions.matchingfunction import get_recommended_books
app = Flask(__name__)

books_repo = repositories.BooksRepository()
users_repo = repositories.UsersRepository()

@app.route('/ping')
def hello_world_endpoint():
    return render_template('main.html', title="Main", content="Pong?")

@app.route('/user/<username>', methods=["GET", "POST"])
def user_page(username):
    user = users_repo.get_user(username)

    potentialbooks = sorted(sum(get_recommended_books(user), []), key=lambda x: x.score, reverse=True)
    output = []
    for item in potentialbooks:
        if item not in output and item not in user.books_read:
            output.append(item)
    if len(output) == 0:
        potentialbooks = books_repo.books
    else:
        potentialbooks = output

    if user:
        return render_template('user.html',
                               user=user,
                               allusers=users_repo.users,
                               potentialbooks=potentialbooks)
    else:
        return render_template('error.html')

@app.route("/user/<username>/follow/<anotheruser>")
def user_follows(username, anotheruser):
    first_user = users_repo.get_user(username)
    second_user = users_repo.get_user(anotheruser)
    first_user.follows(second_user)
    first_user.increment_score(1)
    return redirect(url_for('user_page', username=username))

@app.route("/user/<username>/read/<bookisbn>")
def user_reads(username, bookisbn):
    first_user = users_repo.get_user(username)
    book = books_repo.query_book_isbn(bookisbn)
    first_user.reads(book)
    first_user.increment_score(2)
    return redirect(url_for('user_page', username=username))

@app.route("/book/<isbn>")
def book_page(isbn):
    book = books_repo.query_book_isbn(isbn)
    return render_template('book.html', book=book)

@app.route("/user/<username>/rate/<isbn>/<rating>")
def rate_a_book(username, isbn, rating):
    user = users_repo.get_user(username)
    book = user.get_book_by_isbn(isbn)
    book.assign_rating(rating)
    user.increment_score(5)
    return redirect(url_for('user_page', username=user.username))

if __name__ == '__main__':
    app.debug = True

    # BEGIN MOCK USERS
    users = [User(name) for name in ["Tom", "John", "Mike"]]
    books = [Book(isbn="9788866543256", title="The Catcher in the Rye", category="Fiction"),
             Book(isbn="9788866546567", title="Cinderella", category="Children"),
             Book(isbn="9788866546568", title="Le Petit Prince", category="Children"),
             Book(isbn="9788866546786", title="Cancer Tropic", description="Lorem ipsum...", category="Fiction"),
             Book(isbn="97888663456467", title="Scorpion Tropic", description="Lorem ipsum...", category="Crime"),
             Book(isbn="9788866546566", title="The Most Beautiful Book in The World", description="Lorem ipsum...",
                  category="Romance")]

    for book in books:
        books_repo.books.append(book)

    for user in users:
        users_repo.add_user(user)

    # MIKE READS TWO BOOKS
    mike = users_repo.get_user("Mike")

    app.run()
<commit_msg>User score<commit_after>from flask import Flask, render_template, redirect, url_for
from models.models import *
from src import repositories
from functions.matchingfunction import get_recommended_books
app = Flask(__name__)

books_repo = repositories.BooksRepository()
users_repo = repositories.UsersRepository()

@app.route('/ping')
def hello_world_endpoint():
    return render_template('main.html', title="Main", content="Pong?")

@app.route('/user/<username>', methods=["GET", "POST"])
def user_page(username):
    user = users_repo.get_user(username)

    potentialbooks = sorted(sum(get_recommended_books(user), []), key=lambda x: x.score, reverse=True)
    output = []
    for item in potentialbooks:
        if item not in output and item not in user.books_read:
            output.append(item)
    if len(output) == 0:
        potentialbooks = books_repo.books
    else:
        potentialbooks = output

    if user:
        return render_template('user.html',
                               user=user,
                               allusers=users_repo.users,
                               potentialbooks=potentialbooks)
    else:
        return render_template('error.html')

@app.route("/user/<username>/follow/<anotheruser>")
def user_follows(username, anotheruser):
    first_user = users_repo.get_user(username)
    second_user = users_repo.get_user(anotheruser)
    first_user.follows(second_user)
    first_user.increment_score(1)
    return redirect(url_for('user_page', username=username))

@app.route("/user/<username>/read/<bookisbn>")
def user_reads(username, bookisbn):
    first_user = users_repo.get_user(username)
    book = books_repo.query_book_isbn(bookisbn)
    first_user.reads(book)
    first_user.increment_score(2)
    return redirect(url_for('user_page', username=username))

@app.route("/book/<isbn>")
def book_page(isbn):
    book = books_repo.query_book_isbn(isbn)
    return render_template('book.html', book=book)

@app.route("/user/<username>/rate/<isbn>/<rating>")
def rate_a_book(username, isbn, rating):
    user = users_repo.get_user(username)
    book = user.get_book_by_isbn(isbn)
    book.assign_rating(rating)
    user.increment_score(5)
    return redirect(url_for('user_page', username=user.username))

if __name__ == '__main__':
    app.debug = True

    # BEGIN MOCK USERS
    users = [User(name) for name in ["Tom", "John", "Mike"]]
    books = [Book(isbn="9788866543256", title="The Catcher in the Rye", category="Fiction"),
             Book(isbn="9788866546567", title="Cinderella", category="Children"),
             Book(isbn="9788866546568", title="Le Petit Prince", category="Children"),
             Book(isbn="9788866546786", title="Cancer Tropic", description="Lorem ipsum...", category="Fiction"),
             Book(isbn="97888663456467", title="Scorpion Tropic", description="Lorem ipsum...", category="Crime"),
             Book(isbn="9788866546566", title="The Most Beautiful Book in The World", description="Lorem ipsum...",
                  category="Romance")]

    for book in books:
        books_repo.books.append(book)

    for user in users:
        users_repo.add_user(user)

    # MIKE READS TWO BOOKS
    mike = users_repo.get_user("Mike")

    app.run(port=6667)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
@author: Federico Cerchiari <federicocerchiari@gmail.com>
"""
import os
import unittest
from copy import copy
from collections import Counter

from tempy import Css
from tempy.exceptions import WrongContentError

class TestTag(unittest.TestCase):

    def setUp(self):
        self.css_dict = {
            'html': {
                'background-color': 'lightblue'
                },
            'h1': {
                'color': 'white',
                'text-align': 'center'
                }
            }

    def test_init(self):
        # Test init from dict
        css = Css(self.css_dict)
        self.assertTrue(all(item in css.attrs['css_attrs'].items() for item in self.css_dict.items()))

        # Test init unpacked dict
        css = Css(**self.css_dict)
        self.assertTrue(all(item in css.attrs['css_attrs'].items() for item in self.css_dict.items()))

        # Test init list of dicts
        css_dict2 = {'div': {'color': 'blue'}}
        compare_dict = copy(self.css_dict)
        compare_dict.update(css_dict2)
        css = Css([self.css_dict, css_dict2])
        self.assertTrue(all(item in css.attrs['css_attrs'].items() for item in compare_dict.items()))

        # Test init generator of dicts
        css_dict2 = {'div': {'color': 'blue'}}
        compare_dict = copy(self.css_dict)
        compare_dict.update(css_dict2)
        css = Css((self.css_dict, css_dict2))
        self.assertTrue(all(item in css.attrs['css_attrs'].items() for item in compare_dict.items()))

        with self.assertRaises(WrongContentError):
            Css('test', 'wrong')

        with self.assertRaises(WrongContentError):
            Css(['test', 'wrong'])

    def test_function_content(self):
        css = Css({'h1': {'color': lambda: 'TEST'}})
        self.assertTrue('TEST' in css.render())

    def test_render(self):
        expected = '<style>html { background-color: lightblue; } h1 { color: white; text-align: center; } </style>'
        css = Css(self.css_dict)
        rendered_css = css.render()
        # We count chars occurrence 'cause in python < 3.6 kwargs is not an OrderedDict'
        self.assertEqual(Counter(rendered_css), Counter(expected))

    def test_dump(self):
        css = Css({'div': {'color': 'blue'}})
        expected = 'div { color: blue; } '
        filename = 'temp.css'
        css.dump(filename)
        with open(filename, 'r') as f:
            self.assertEqual(f.read(), expected)
        os.remove(filename)
<commit_msg>Add tests for Css.update<commit_after># -*- coding: utf-8 -*-
"""
@author: Federico Cerchiari <federicocerchiari@gmail.com>
"""
import os
import unittest
from copy import copy
from collections import Counter

from tempy import Css
from tempy.exceptions import WrongContentError

class TestTag(unittest.TestCase):

    def setUp(self):
        self.css_dict = {
            'html': {
                'background-color': 'lightblue'
                },
            'h1': {
                'color': 'white',
                'text-align': 'center'
                }
            }

    def test_init(self):
        # Test init from dict
        css = Css(self.css_dict)
        self.assertTrue(all(item in css.attrs['css_attrs'].items() for item in self.css_dict.items()))

        # Test init unpacked dict
        css = Css(**self.css_dict)
        self.assertTrue(all(item in css.attrs['css_attrs'].items() for item in self.css_dict.items()))

        # Test init list of dicts
        css_dict2 = {'div': {'color': 'blue'}}
        compare_dict = copy(self.css_dict)
        compare_dict.update(css_dict2)
        css = Css([self.css_dict, css_dict2])
        self.assertTrue(all(item in css.attrs['css_attrs'].items() for item in compare_dict.items()))

        # Test init generator of dicts
        css_dict2 = {'div': {'color': 'blue'}}
        compare_dict = copy(self.css_dict)
        compare_dict.update(css_dict2)
        css = Css((self.css_dict, css_dict2))
        self.assertTrue(all(item in css.attrs['css_attrs'].items() for item in compare_dict.items()))

        with self.assertRaises(WrongContentError):
            Css('test', 'wrong')

        with self.assertRaises(WrongContentError):
            Css(['test', 'wrong'])

    def test_update(self):
        add_dict = {'div': {'color': 'blue'}}
        modify_dict = {'html': {'background-color': 'black'}}

        add_compare_dict = copy(self.css_dict)
        add_compare_dict.update(add_dict)

        modify_compare_dict = copy(self.css_dict)
        modify_compare_dict.update(compare_dict)

        # Test add from dict
        css = Css(self.css_dict)
        css.update(add_dict)
        self.assertTrue(all(item in css.attrs['css_attrs'].items() for item in add_compare_dict.items()))

        # Test modify from dict
        css = Css(self.css_dict)
        css.update(modify_dict)
        self.assertTrue(all(item in css.attrs['css_attrs'].items() for item in add_modify_dict.items()))

        # Test add from unpacked dict
        css = Css(self.css_dict)
        css.update(**add_dict)
        self.assertTrue(all(item in css.attrs['css_attrs'].items() for item in add_compare_dict.items()))

        # Test modify from unpacked dict
        css = Css(self.css_dict)
        css.update(**modify_dict)
        self.assertTrue(all(item in css.attrs['css_attrs'].items() for item in modify_compare_dict.items()))

        with self.assertRaises(WrongContentError):
            css = Css(self.css_dict)
            css.update('test', 'wrong')

        with self.assertRaises(WrongContentError):
            css = Css(self.css_dict)
            css.update(['test', 'wrong'])

    def test_function_content(self):
        css = Css({'h1': {'color': lambda: 'TEST'}})
        self.assertTrue('TEST' in css.render())

    def test_render(self):
        expected = '<style>html { background-color: lightblue; } h1 { color: white; text-align: center; } </style>'
        css = Css(self.css_dict)
        rendered_css = css.render()
        # We count chars occurrence 'cause in python < 3.6 kwargs is not an OrderedDict'
        self.assertEqual(Counter(rendered_css), Counter(expected))

    def test_dump(self):
        css = Css({'div': {'color': 'blue'}})
        expected = 'div { color: blue; } '
        filename = 'temp.css'
        css.dump(filename)
        with open(filename, 'r') as f:
            self.assertEqual(f.read(), expected)
        os.remove(filename)
<|endoftext|>


<commit_before>import os
import sys
import copy

import numpy
import pandas

def __extractNodes(matrix):
    nodes = set()
    for key1 in matrix:
        nodes.add(key1)
        for key2 in matrix:
            nodes.add(key2)
    return nodes

def __makeSquare(matrix, keys, default=0.0):
    matrix = copy.deepcopy(matrix)
    for key1 in keys:
        for key2 in keys:
            if not key1 in matrix:
                matrix[key1] = {}
            if not key2 in matrix[key1]:
                matrix[key1][key2] = 0.0
    return matrix

def __startState(nodes):
    if len(nodes) == 0: raise ValueError("There must be at least one node.")
    startProb = 1.0 / float(len(nodes))
    return pandas.Series({node : startProb for node in nodes})

def __calculateAlpha(numNodes, rsp):
    return 1.0 / float(numNodes) * rsp

def __integrateRandomSurfer(nodes, transitionProbs, rsp):
    alpha = __calculateAlpha(len(nodes), rsp)
    new = transitionProbs.copy()
    return new.multiply(1.0 - float(len(nodes)) * alpha) + alpha

def __exponentiateMatrix(matrix, exponent):
    for _ in range(exponent - 1):
        matrix = matrix.dot(matrix)
    return matrix

def __normalizeRows(matrix):
    return matrix.div(matrix.sum(axis=1), axis=0)

def powerIteration(edgeWeights, rsp=0.15, maxIterations=100):
    # Clerical work:
    nodes = __extractNodes(edgeWeights)
    edgeWeights = __makeSquare(edgeWeights, nodes)
    edgeWeights = pandas.DataFrame(edgeWeights).T.fillna(0.0)

    # Setup:
    state = __startState(nodes)
    transitionProbs = __normalizeRows(edgeWeights)
    transitionProbs = __integrateRandomSurfer(nodes, transitionProbs, rsp)

    # Power iteration:
    for iteration in range(maxIterations):
        oldState = state.copy()
        state = state.dot(transitionProbs)
        if (state == oldState).all(): break

    return state
<commit_msg>minor clean up<commit_after>import os
import sys
import copy

import numpy
import pandas

def __extractNodes(matrix):
    nodes = set()
    for key1 in matrix:
        nodes.add(key1)
        for key2 in matrix[key1]:
            nodes.add(key2)
    return nodes

def __makeSquare(matrix, keys, default=0.0):
    matrix = copy.deepcopy(matrix)
    for key1 in keys:
        for key2 in keys:
            if not key1 in matrix:
                matrix[key1] = {}
            if not key2 in matrix[key1]:
                matrix[key1][key2] = 0.0
    return matrix

def __startState(nodes):
    if len(nodes) == 0: raise ValueError("There must be at least one node.")
    startProb = 1.0 / float(len(nodes))
    return pandas.Series({node : startProb for node in nodes})

def __calculateAlpha(numNodes, rsp):
    return 1.0 / float(numNodes) * rsp

def __integrateRandomSurfer(nodes, transitionProbs, rsp):
    alpha = __calculateAlpha(len(nodes), rsp)
    new = transitionProbs.copy()
    return new.multiply(1.0 - float(len(nodes)) * alpha) + alpha

def __exponentiateMatrix(matrix, exponent):
    for _ in range(exponent - 1):
        matrix = matrix.dot(matrix)
    return matrix

def __normalizeRows(matrix):
    return matrix.div(matrix.sum(axis=1), axis=0)

def powerIteration(edgeWeights, rsp=0.15, maxIterations=100):
    # Clerical work:
    nodes = __extractNodes(edgeWeights)
    edgeWeights = __makeSquare(edgeWeights, nodes)
    edgeWeights = pandas.DataFrame(edgeWeights).T.fillna(0.0)

    # Setup:
    state = __startState(nodes)
    transitionProbs = __normalizeRows(edgeWeights)
    transitionProbs = __integrateRandomSurfer(nodes, transitionProbs, rsp)

    # Power iteration:
    for iteration in range(maxIterations):
        oldState = state.copy()
        state = state.dot(transitionProbs)
        if (state == oldState).all(): break

    return state
<|endoftext|>


<commit_before>from django.db import models

class Subject(models.Model):
    name = models.CharField(max_length=100)
    code = models.IntegerField()
    credits = models.IntegerField()

class Teacher(models.Model):
    name = models.CharField(max_length=100)

class Group(models.Model):
    schedule = models.CommaSeparatedIntegerField(max_length=10)
    code = models.IntegerField()
    teacher = models.ForeignKey(Teacher)
    subject = models.ForeignKey(Subject)
<commit_msg>Postgres is more strict<commit_after>from django.db import models

class Subject(models.Model):
    name = models.CharField(max_length=100)
    code = models.IntegerField()
    credits = models.IntegerField()

class Teacher(models.Model):
    name = models.CharField(max_length=100)

class Group(models.Model):
    schedule = models.CommaSeparatedIntegerField(max_length=100)
    code = models.IntegerField()
    teacher = models.ForeignKey(Teacher)
    subject = models.ForeignKey(Subject)
<|endoftext|>


<commit_before># coding:utf-8

from os import environ, path

from webob import Response
from webob.dec import wsgify
from webob.static import DirectoryApp

@wsgify
def api(request):
    """WSGI-приложение"""
    return Response('Hello!')

# Приложение, раздающее статику
static_serving_app = DirectoryApp(
    path.join(environ['BUP_PATH'], 'static', 'barsup'),
    index_page='index.html',
    hide_index_with_redirect=True
)

@wsgify.middleware
def serve_static(request, app):
    # url = request.path
    # if url == '/' or (
    #     url.startswith('/index') or
    #     url.startswith('/static/')
    # ):
    #     return static_serving_app
    # else:
    #     return app
    res = request.call_application(static_serving_app)
    if res[0] != '200 OK':
        return app
    else:
        return static_serving_app

application = serve_static(api)
<commit_msg>Временная возможность загрузки страниц через url - http://localhost:8000/barsup/index.html<commit_after># coding:utf-8

from os import environ, path

from webob import Response
from webob.dec import wsgify
from webob.static import DirectoryApp

@wsgify
def api(request):
    """WSGI-приложение"""
    return Response('Hello!')

# Приложение, раздающее статику
static_serving_app = DirectoryApp(
    path.join(environ['BUP_PATH'], 'static'),
    index_page='barsup/',
    hide_index_with_redirect=False
)

@wsgify.middleware
def serve_static(request, app):
    url = request.path
    if url == '/' or (
        url.startswith('/barsup')
    ):
        return static_serving_app
    else:
        return app
    # res = request.call_application(static_serving_app)
    # if res[0] != '200 OK':
    #     return app
    # else:
    #     return static_serving_app

application = serve_static(api)
<|endoftext|>


<commit_before>import testlib
from testlib import randstr
import urlfetch

import unittest
import json
import os
import socket
import tempfile

class GetTest(unittest.TestCase):

    def test_fetch(self):
        r = urlfetch.fetch(testlib.url())
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(o['method'], 'GET')

    def test_fetch_data(self,):
        r = urlfetch.fetch(testlib.test_server_host, data='foo=bar')
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(o['method'], 'POST')

    def test_get(self):
        r = urlfetch.get(testlib.url())
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(o['method'], 'GET')

    def test_get_params(self):
        p1 = (randstr(), randstr())
        p2 = (randstr(), randstr())
        params = dict((p1, p2))
        r = urlfetch.get(testlib.test_server_host, params=params)
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(o['method'], 'GET')
        self.assertTrue(('%s=%s' % p1) in r.url)
        self.assertTrue(('%s=%s' % p2) in r.url)

    def test_fragment(self):
        r = urlfetch.get(testlib.url('#urlfetch'))
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(o['method'], 'GET')

    def test_query_string(self):
        qs = testlib.randdict(5)
        query_string = urlfetch.urlencode(qs)

        r = urlfetch.get(testlib.url('?'+ query_string))
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(o['method'], 'GET')
        self.assertEqual(o['query_string'], query_string)
        self.assertEqual(o['get'], qs)

    def test_fragment_query_string(self):
        qs = testlib.randdict(5)
        query_string = urlfetch.urlencode(qs)

        r = urlfetch.get(testlib.url('?'+ query_string + '#urlfetch'))
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(o['method'], 'GET')
        self.assertEqual(o['query_string'], query_string)
        self.assertEqual(o['get'], qs)

    def test_basic_auth(self):
        r = urlfetch.get(testlib.url('basic_auth'), auth=('urlfetch', 'fetchurl'))
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(o['method'], 'GET')

    def test_fragment_basic_auth(self):
        r = urlfetch.get(testlib.url('basic_auth#urlfetch'), auth=('urlfetch', 'fetchurl'))
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(o['method'], 'GET')

    def test_basic_auth_query_string(self):
        qs = testlib.randdict(5)
        query_string = urlfetch.urlencode(qs)

        r = urlfetch.get(testlib.url('basic_auth?'+ query_string), auth=('urlfetch', 'fetchurl'))
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(o['method'], 'GET')
        self.assertEqual(o['query_string'], query_string)
        self.assertEqual(o['get'], qs)

    def test_fragment_basic_auth_query_string(self):
        qs = testlib.randdict(5)
        query_string = urlfetch.urlencode(qs)

        r = urlfetch.get(testlib.url('basic_auth?'+ query_string + '#urlfetch'), auth=('urlfetch', 'fetchurl'))
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(o['method'], 'GET')
        self.assertEqual(o['query_string'], query_string)
        self.assertEqual(o['get'], qs)

    def test_timeout(self):
        self.assertRaises(socket.timeout, lambda: urlfetch.get(testlib.url('sleep/1'), timeout=0.5))

    def test_length_limit(self):
        self.assertRaises(urlfetch.UrlfetchException, lambda: urlfetch.get(testlib.url(), length_limit=1))

    def test_streaming(self):
        with tempfile.TemporaryFile() as f:
            with urlfetch.get(testlib.url('utf8.txt')) as r:
                for chunk in r:
                    f.write(chunk)
            f.seek(0)
            self.assertEqual(f.read(), open(os.path.join(os.path.dirname(__file__), 'test.file'), 'rb').read())

        with tempfile.TemporaryFile() as f:
            with urlfetch.get(testlib.url('/gbk.txt')) as r:
                for chunk in r:
                    f.write(chunk)
            f.seek(0)
            self.assertEqual(f.read(), open(os.path.join(os.path.dirname(__file__), 'test.file.gbk'), 'rb').read())

    def test_cookie(self):
        cookie = (randstr(), randstr())
        r = urlfetch.get(testlib.url('setcookie/%s/%s' % cookie))
        self.assertEqual(r.cookies[cookie[0]], cookie[1])
        self.assertTrue(('%s=%s' % cookie) in r.cookiestring)

    def test_redirect(self):
        r = urlfetch.get(testlib.url('/redirect/3/0'))
        self.assertTrue(r.status in (301, 302, 303, 307))
        self.assertTrue('location' in r.headers)

        self.assertRaises(urlfetch.UrlfetchException, lambda: urlfetch.get(testlib.url('/redirect/3/0'), max_redirects=1))

if __name__ == '__main__':
    unittest.main()
<commit_msg>tests for r.links<commit_after>import testlib
from testlib import randstr
import urlfetch

import unittest
import json
import os
import socket
import tempfile

class GetTest(unittest.TestCase):

    def test_fetch(self):
        r = urlfetch.fetch(testlib.url())
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(r.links, [])
        self.assertEqual(o['method'], 'GET')

    def test_fetch_data(self,):
        r = urlfetch.fetch(testlib.test_server_host, data='foo=bar')
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(r.links, [])
        self.assertEqual(o['method'], 'POST')

    def test_get(self):
        r = urlfetch.get(testlib.url())
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(r.links, [])
        self.assertEqual(o['method'], 'GET')

    def test_get_params(self):
        p1 = (randstr(), randstr())
        p2 = (randstr(), randstr())
        params = dict((p1, p2))
        r = urlfetch.get(testlib.test_server_host, params=params)
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(r.links, [])
        self.assertEqual(o['method'], 'GET')
        self.assertTrue(('%s=%s' % p1) in r.url)
        self.assertTrue(('%s=%s' % p2) in r.url)

    def test_fragment(self):
        r = urlfetch.get(testlib.url('#urlfetch'))
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(r.links, [])
        self.assertEqual(o['method'], 'GET')

    def test_query_string(self):
        qs = testlib.randdict(5)
        query_string = urlfetch.urlencode(qs)

        r = urlfetch.get(testlib.url('?'+ query_string))
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(r.links, [])
        self.assertEqual(o['method'], 'GET')
        self.assertEqual(o['query_string'], query_string)
        self.assertEqual(o['get'], qs)

    def test_fragment_query_string(self):
        qs = testlib.randdict(5)
        query_string = urlfetch.urlencode(qs)

        r = urlfetch.get(testlib.url('?'+ query_string + '#urlfetch'))
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(r.links, [])
        self.assertEqual(o['method'], 'GET')
        self.assertEqual(o['query_string'], query_string)
        self.assertEqual(o['get'], qs)

    def test_basic_auth(self):
        r = urlfetch.get(testlib.url('basic_auth'), auth=('urlfetch', 'fetchurl'))
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(r.links, [])
        self.assertEqual(o['method'], 'GET')

    def test_fragment_basic_auth(self):
        r = urlfetch.get(testlib.url('basic_auth#urlfetch'), auth=('urlfetch', 'fetchurl'))
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(r.links, [])
        self.assertEqual(o['method'], 'GET')

    def test_basic_auth_query_string(self):
        qs = testlib.randdict(5)
        query_string = urlfetch.urlencode(qs)

        r = urlfetch.get(testlib.url('basic_auth?'+ query_string), auth=('urlfetch', 'fetchurl'))
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(r.links, [])
        self.assertEqual(o['method'], 'GET')
        self.assertEqual(o['query_string'], query_string)
        self.assertEqual(o['get'], qs)

    def test_fragment_basic_auth_query_string(self):
        qs = testlib.randdict(5)
        query_string = urlfetch.urlencode(qs)

        r = urlfetch.get(testlib.url('basic_auth?'+ query_string + '#urlfetch'), auth=('urlfetch', 'fetchurl'))
        o = json.loads(r.text)

        self.assertEqual(r.status, 200)
        self.assertTrue(isinstance(r.json, dict))
        self.assertTrue(isinstance(r.text, urlfetch.unicode))
        self.assertEqual(r.links, [])
        self.assertEqual(o['method'], 'GET')
        self.assertEqual(o['query_string'], query_string)
        self.assertEqual(o['get'], qs)

    def test_timeout(self):
        self.assertRaises(socket.timeout, lambda: urlfetch.get(testlib.url('sleep/1'), timeout=0.5))

    def test_length_limit(self):
        self.assertRaises(urlfetch.UrlfetchException, lambda: urlfetch.get(testlib.url(), length_limit=1))

    def test_streaming(self):
        with tempfile.TemporaryFile() as f:
            with urlfetch.get(testlib.url('utf8.txt')) as r:
                for chunk in r:
                    f.write(chunk)
            f.seek(0)
            self.assertEqual(f.read(), open(os.path.join(os.path.dirname(__file__), 'test.file'), 'rb').read())

        with tempfile.TemporaryFile() as f:
            with urlfetch.get(testlib.url('/gbk.txt')) as r:
                for chunk in r:
                    f.write(chunk)
            f.seek(0)
            self.assertEqual(f.read(), open(os.path.join(os.path.dirname(__file__), 'test.file.gbk'), 'rb').read())

    def test_cookie(self):
        cookie = (randstr(), randstr())
        r = urlfetch.get(testlib.url('setcookie/%s/%s' % cookie))
        self.assertEqual(r.links, [])
        self.assertEqual(r.cookies[cookie[0]], cookie[1])
        self.assertTrue(('%s=%s' % cookie) in r.cookiestring)

    def test_redirect(self):
        r = urlfetch.get(testlib.url('/redirect/3/0'))
        self.assertTrue(r.status in (301, 302, 303, 307))
        self.assertEqual(r.links, [])
        self.assertTrue('location' in r.headers)

        self.assertRaises(urlfetch.UrlfetchException, lambda: urlfetch.get(testlib.url('/redirect/3/0'), max_redirects=1))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from __future__ import (absolute_import, print_function, unicode_literals)
from acli.output.ec2 import (output_ec2_list, output_ec2_info)
from acli.services.ec2 import (ec2_list, ec2_info)
from acli.config import Config
from moto import mock_ec2
import pytest
from boto3.session import Session
session = Session(region_name="eu-west-1")

@pytest.yield_fixture(scope='function')
def ec2_instances():
    """EC2 mock service"""
    mock = mock_ec2()
    mock.start()
    client = session.client('ec2')
    client.create_security_group(GroupName='group1', Description='my first sec group')
    reservations = client.run_instances(ImageId='ami-12345', MinCount=2, MaxCount=2, SecurityGroups=['group1'])
    for i, s in enumerate(reservations.get('Instances')):
        client.create_tags(
            Resources=[s.get('InstanceId')],
            Tags=[{'Key': 'Name', 'Value': 'Bob'}])
    ec2_resource = session.resource('ec2')
    all_instances = ec2_resource.instances.all()
    print(list(all_instances)[0].id)
    yield all_instances
    mock.stop()

config = Config(cli_args={'--region': 'eu-west-1',
                          '--access_key_id': 'AKIAIOSFODNN7EXAMPLE',
                          '--secret_access_key': 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'})

@mock_ec2
def test_ec2_list_service():
    with pytest.raises(SystemExit):
        assert ec2_list(aws_config=config)

def test_ec2_info_service(ec2_instances):
    config = Config(cli_args={'--region': 'eu-west-1',
                              '--access_key_id': 'AKIAIOSFODNN7EXAMPLE',
                              '--secret_access_key': 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'})
    with pytest.raises(SystemExit):
        print(list(ec2_instances))
        assert ec2_info(aws_config=config, instance_id=list(ec2_instances)[0].id)

def test_ec2_list_output(ec2_instances):
    with pytest.raises(SystemExit):
        assert output_ec2_list(output_media='console', instances=ec2_instances)

def test_ec2_output(ec2_instances):
    with pytest.raises(SystemExit):
        instance = list(ec2_instances)[0]
        print(dir(instance))
        assert output_ec2_info(output_media='console', instance=instance)
<commit_msg>add test for ec2 service info.<commit_after>from __future__ import (absolute_import, print_function, unicode_literals)
from acli.output.ec2 import (output_ec2_list, output_ec2_info)
from acli.services.ec2 import (ec2_list, ec2_info)
from acli.config import Config
from moto import mock_ec2
import pytest
from boto3.session import Session
session = Session(region_name="eu-west-1")

@pytest.yield_fixture(scope='function')
def ec2_instances():
    """EC2 mock service"""
    mock = mock_ec2()
    mock.start()
    client = session.client('ec2')
    client.create_security_group(GroupName='group1', Description='my first sec group')
    reservations = client.run_instances(ImageId='ami-12345', MinCount=2, MaxCount=2, SecurityGroups=['group1'])
    for i, s in enumerate(reservations.get('Instances')):
        client.create_tags(
            Resources=[s.get('InstanceId')],
            Tags=[{'Key': 'Name', 'Value': 'Bob'}])
    ec2_resource = session.resource('ec2')
    all_instances = ec2_resource.instances.all()
    print(list(all_instances)[0].id)
    yield all_instances
    mock.stop()

config = Config(cli_args={'--region': 'eu-west-1',
                          '--access_key_id': 'AKIAIOSFODNN7EXAMPLE',
                          '--secret_access_key': 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'})

@mock_ec2
def test_ec2_list_service():
    with pytest.raises(SystemExit):
        assert ec2_list(aws_config=config)

def test_ec2_info_service(ec2_instances):
    with pytest.raises(SystemExit):
        print(list(ec2_instances))
        assert ec2_info(aws_config=config, instance_id=list(ec2_instances)[0].id)

def test_ec2_list_output(ec2_instances):
    with pytest.raises(SystemExit):
        assert output_ec2_list(output_media='console', instances=ec2_instances)

def test_ec2_output(ec2_instances):
    with pytest.raises(SystemExit):
        instance = list(ec2_instances)[0]
        print(dir(instance))
        assert output_ec2_info(output_media='console', instance=instance)
<|endoftext|>


<commit_before>import base64
import io
import os
import subprocess

class Keyring():
    KEK_SCHEMA = "org.eu.nullroute.Accdb.MasterKey"

    def _make_attrs(self, uuid):
        raise NotImplementedError()

    def _store_kek(self, label, secret, uuid):
        return self.store(label, secret, self._make_attrs(uuid))

    def _lookup_kek(self, uuid):
        return self.lookup(self._make_attrs(uuid))

    def _clear_kek(self, uuid):
        return self.clear(self._make_attrs(uuid))

    def store_kek(self, uuid, kek):
        label = "accdb master key for {%s}" % uuid
        secret = base64.b64encode(kek).decode()
        if not self._store_kek(label, secret, str(uuid)):
            raise Exception("failed to store master key in keyring")

    def lookup_kek(self, uuid):
        secret = self._lookup_kek(str(uuid))
        return base64.b64decode(secret) if secret else None

    def cache_kek(self, uuid, kek):
        pass

    def clear_kek(self, uuid):
        return self._clear_kek(str(uuid))

class GitKeyring(Keyring):
    def __init__(self, helper="cache"):
        self.helper = helper

    def _talk(self, action, attrs):
        for a in ["host", "username"]:
            if a not in attrs:
                raise ValueError("attribute %r is required" % a)

        if action == "get":
            stdout = subprocess.PIPE
        else:
            stdout = subprocess.DEVNULL

        with subprocess.Popen(["git", "credential-%s" % self.helper, action],
                              stdin=subprocess.PIPE,
                              stdout=stdout) as proc:
            buf = "".join(["%s=%s\n" % (k, v) for k, v in attrs.items()])
            out, err = proc.communicate(buf.encode())
            if out is None:
                return proc.wait() == 0
            else:
                ret = [l.split("=", 1) for l in out.decode().splitlines()]
                ret = {i[0]: i[1] for i in ret}
                return ret or None

    def store(self, label, secret, attrs):
        attrs["label"] = label
        attrs["password"] = secret
        return self._talk("store", attrs)

    def search(self, attrs):
        return self._talk("get", attrs)

    def lookup(self, attrs):
        ret = self.search(attrs)
        return ret["password"] if ret else None

    def clear(self, attrs):
        return self._talk("erase", attrs)

    def _make_attrs(self, uuid):
        return {
            "host": "accdb://%s" % uuid,
            "protocol": Keyring.KEK_SCHEMA,
            "username": uuid,
        }

class XdgKeyring(Keyring):
    @property
    def available(self):
        return subprocess.call(["secret-tool", "search", "", ""],
                               stdout=subprocess.DEVNULL,
                               stderr=subprocess.DEVNULL) == 0

    def store(self, label, secret, attrs):
        if not self.available:
            return False
        with subprocess.Popen(["secret-tool", "store", "--label", label] + attrs,
                               stdin=subprocess.PIPE) as proc:
            proc.communicate(secret.encode())
            return proc.wait() == 0

    def lookup(self, attrs):
        if not self.available:
            return None
        with subprocess.Popen(["secret-tool", "lookup"] + attrs,
                               stdout=subprocess.PIPE) as proc:
            out, err = proc.communicate()
            return out.decode().rstrip("\n")

    def clear(self, attrs):
        if not self.available:
            return False
        return subprocess.call(["secret-tool", "clear"] + attrs) == 0

    def _make_attrs(self, uuid):
        return [
            "xdg:schema", Keyring.KEK_SCHEMA,
            "uuid", uuid,
        ]

class Prompter():
    def get_password(self, desc, **kwargs):
        raise NotImplementedError()

class BasicPrompter(Prompter):
    def get_password(self, desc, **kwargs):
        import getpass
        return getpass.getpass("\033[1;33m%s\033[m " % desc)

class ShimKeyring(Keyring, BasicPrompter):
    def __init__(self):
        self.store = XdgKeyring()
        self.cache = GitKeyring("cache")

    def store_kek(self, uuid, kek):
        return self.store.store_kek(uuid, kek)

    def cache_kek(self, uuid, kek):
        return self.cache.store_kek(uuid, kek)

    def lookup_kek(self, uuid):
        return self.cache.lookup_kek(uuid) or self.store.lookup_kek(uuid)

    def clear_kek(self, uuid):
        sr = self.store.clear_kek(uuid)
        cr = self.cache.clear_kek(uuid)
        return sr

def default_keyring():
    return ShimKeyring()
<commit_msg>Revert "keyring: just completely remove pinentry usage"<commit_after>import base64
import io
import os
import subprocess

class Keyring():
    KEK_SCHEMA = "org.eu.nullroute.Accdb.MasterKey"

    def _make_attrs(self, uuid):
        raise NotImplementedError()

    def _store_kek(self, label, secret, uuid):
        return self.store(label, secret, self._make_attrs(uuid))

    def _lookup_kek(self, uuid):
        return self.lookup(self._make_attrs(uuid))

    def _clear_kek(self, uuid):
        return self.clear(self._make_attrs(uuid))

    def store_kek(self, uuid, kek):
        label = "accdb master key for {%s}" % uuid
        secret = base64.b64encode(kek).decode()
        if not self._store_kek(label, secret, str(uuid)):
            raise Exception("failed to store master key in keyring")

    def lookup_kek(self, uuid):
        secret = self._lookup_kek(str(uuid))
        return base64.b64decode(secret) if secret else None

    def cache_kek(self, uuid, kek):
        pass

    def clear_kek(self, uuid):
        return self._clear_kek(str(uuid))

class GitKeyring(Keyring):
    def __init__(self, helper="cache"):
        self.helper = helper

    def _talk(self, action, attrs):
        for a in ["host", "username"]:
            if a not in attrs:
                raise ValueError("attribute %r is required" % a)

        if action == "get":
            stdout = subprocess.PIPE
        else:
            stdout = subprocess.DEVNULL

        with subprocess.Popen(["git", "credential-%s" % self.helper, action],
                              stdin=subprocess.PIPE,
                              stdout=stdout) as proc:
            buf = "".join(["%s=%s\n" % (k, v) for k, v in attrs.items()])
            out, err = proc.communicate(buf.encode())
            if out is None:
                return proc.wait() == 0
            else:
                ret = [l.split("=", 1) for l in out.decode().splitlines()]
                ret = {i[0]: i[1] for i in ret}
                return ret or None

    def store(self, label, secret, attrs):
        attrs["label"] = label
        attrs["password"] = secret
        return self._talk("store", attrs)

    def search(self, attrs):
        return self._talk("get", attrs)

    def lookup(self, attrs):
        ret = self.search(attrs)
        return ret["password"] if ret else None

    def clear(self, attrs):
        return self._talk("erase", attrs)

    def _make_attrs(self, uuid):
        return {
            "host": "accdb://%s" % uuid,
            "protocol": Keyring.KEK_SCHEMA,
            "username": uuid,
        }

class XdgKeyring(Keyring):
    @property
    def available(self):
        return subprocess.call(["secret-tool", "search", "", ""],
                               stdout=subprocess.DEVNULL,
                               stderr=subprocess.DEVNULL) == 0

    def store(self, label, secret, attrs):
        if not self.available:
            return False
        with subprocess.Popen(["secret-tool", "store", "--label", label] + attrs,
                               stdin=subprocess.PIPE) as proc:
            proc.communicate(secret.encode())
            return proc.wait() == 0

    def lookup(self, attrs):
        if not self.available:
            return None
        with subprocess.Popen(["secret-tool", "lookup"] + attrs,
                               stdout=subprocess.PIPE) as proc:
            out, err = proc.communicate()
            return out.decode().rstrip("\n")

    def clear(self, attrs):
        if not self.available:
            return False
        return subprocess.call(["secret-tool", "clear"] + attrs) == 0

    def _make_attrs(self, uuid):
        return [
            "xdg:schema", Keyring.KEK_SCHEMA,
            "uuid", uuid,
        ]

class Prompter():
    def get_password(self, desc, **kwargs):
        raise NotImplementedError()

class BasicPrompter(Prompter):
    def get_password(self, desc, **kwargs):
        import getpass
        return getpass.getpass("\033[1;33m%s\033[m " % desc)

class PinentryPrompter(Prompter):
    def get_password(self, desc, **kwargs):
        with subprocess.Popen(["askpin",
                               "-t", "accdb",
                               "-d", desc,
                               "-p", "Password:"],
                              stdout=subprocess.PIPE) as proc:
            (out, err) = proc.communicate()
            if proc.wait() == 0:
                return out.decode().rstrip("\n")
            else:
                return None

class ShimPrompter(Prompter):
    def get_password(self, desc, **kwargs):
        if os.environ.get("DISPLAY"):
            p = PinentryPrompter()
        else:
            p = BasicPrompter()
        return p.get_password(desc, **kwargs)

class ShimKeyring(Keyring, ShimPrompter):
    def __init__(self):
        self.store = XdgKeyring()
        self.cache = GitKeyring("cache")

    def store_kek(self, uuid, kek):
        return self.store.store_kek(uuid, kek)

    def cache_kek(self, uuid, kek):
        return self.cache.store_kek(uuid, kek)

    def lookup_kek(self, uuid):
        return self.cache.lookup_kek(uuid) or self.store.lookup_kek(uuid)

    def clear_kek(self, uuid):
        sr = self.store.clear_kek(uuid)
        cr = self.cache.clear_kek(uuid)
        return sr

def default_keyring():
    return ShimKeyring()
<|endoftext|>


<commit_before>from matador import git
from pathlib import Path
from time import strftime, gmtime
from dulwich.objects import format_timezone

def test_stage_file(repo):
    repo_folder = Path(repo.path)
    test_file = Path(repo_folder, 'test_file')
    test_file.touch()
    git.stage_file(repo, test_file)

    index = repo.open_index()
    changes = [f.decode('UTF-8') for f in index]

    assert str(test_file.relative_to(repo_folder)) in changes

def test_commit(repo):
    repo_folder = Path(repo.path)
    test_file = Path(repo_folder, 'test_file')
    test_file.touch()
    message = 'Test commit'
    repo.stage([str(test_file.relative_to(repo_folder))])

    git.commit(repo, message)

    last_commit = repo.get_object(repo.head())
    commit_message = last_commit.message
    assert commit_message == bytes(message, encoding='UTF-8')

def test_full_ref_commit(project_repo):
    head = project_repo.head().decode(encoding='ascii')
    ref = git.full_ref(project_repo, head)
    assert ref == head

def test_full_ref_branch(project_repo):
    ref = git.full_ref(project_repo, 'master')
    print(project_repo.refs.keys())
    assert ref == 'refs/heads/master'

def test_full_ref_tag(project_repo):
    ref = git.full_ref(project_repo, 'test-tag')
    assert ref == 'refs/tags/test-tag'

def test_substitute_keywords(project_repo):
    test_text = """\
        First line
        version:
        date:
        author:
        Last line"""

    commit_ref = project_repo.head()
    short_ref = commit_ref[:7].decode(encoding='ascii')
    commit = project_repo.get_object(commit_ref)
    commit_time = strftime('%Y-%m-%d %H:%M:%S', gmtime(commit.commit_time))
    timezone = format_timezone(commit.commit_timezone).decode(encoding='ascii')
    commit_timestamp = commit_time + ' ' + timezone
    author = commit.author.decode(encoding='ascii')

    result = git.substitute_keywords(test_text, project_repo, 'master')
    expected_result = """\
        First line
        version: %s
        date: %s
        author: %s
        Last line""" % (short_ref, commit_timestamp, author)
    assert result == expected_result

    result = git.substitute_keywords(test_text, project_repo, 'test-tag')
    expected_result = """\
        First line
        version: %s
        date: %s
        author: %s
        Last line""" % (short_ref, commit_timestamp, author)
    assert result == expected_result

    result = git.substitute_keywords(test_text, project_repo, 'test-garbage')
    expected_result = ''
    assert result == expected_result
<commit_msg>Add test for git.fetch_all<commit_after>from matador import git
from pathlib import Path
from time import strftime, gmtime
from dulwich.objects import format_timezone
from dulwich.repo import Repo

def test_stage_file(repo):
    repo_folder = Path(repo.path)
    test_file = Path(repo_folder, 'test_file')
    test_file.touch()
    git.stage_file(repo, test_file)

    index = repo.open_index()
    changes = [f.decode('UTF-8') for f in index]

    assert str(test_file.relative_to(repo_folder)) in changes

def test_commit(repo):
    repo_folder = Path(repo.path)
    test_file = Path(repo_folder, 'test_file')
    test_file.touch()
    message = 'Test commit'
    repo.stage([str(test_file.relative_to(repo_folder))])

    git.commit(repo, message)

    last_commit = repo.get_object(repo.head())
    commit_message = last_commit.message
    assert commit_message == bytes(message, encoding='UTF-8')

def test_fetch_all(tmpdir, project_repo):
    target_repo_folder = Path(str(tmpdir), 'test_remote')
    target_repo = Repo.init(str(target_repo_folder), mkdir=True)
    ref = project_repo.head()

    git.fetch_all(project_repo, target_repo)

    assert b'refs/remotes/origin/master' in target_repo.refs
    assert b'refs/heads/master' in target_repo.refs
    assert target_repo.refs[b'refs/remotes/origin/master'] == ref
    assert target_repo.refs[b'refs/heads/master'] == ref

def test_full_ref_commit(project_repo):
    head = project_repo.head().decode(encoding='ascii')
    ref = git.full_ref(project_repo, head)
    assert ref == head

def test_full_ref_branch(project_repo):
    ref = git.full_ref(project_repo, 'master')
    print(project_repo.refs.keys())
    assert ref == 'refs/heads/master'

def test_full_ref_tag(project_repo):
    ref = git.full_ref(project_repo, 'test-tag')
    assert ref == 'refs/tags/test-tag'

def test_substitute_keywords(project_repo):
    test_text = """\
        First line
        version:
        date:
        author:
        Last line"""

    commit_ref = project_repo.head()
    short_ref = commit_ref[:7].decode(encoding='ascii')
    commit = project_repo.get_object(commit_ref)
    commit_time = strftime('%Y-%m-%d %H:%M:%S', gmtime(commit.commit_time))
    timezone = format_timezone(commit.commit_timezone).decode(encoding='ascii')
    commit_timestamp = commit_time + ' ' + timezone
    author = commit.author.decode(encoding='ascii')

    result = git.substitute_keywords(test_text, project_repo, 'master')
    expected_result = """\
        First line
        version: %s
        date: %s
        author: %s
        Last line""" % (short_ref, commit_timestamp, author)
    assert result == expected_result

    result = git.substitute_keywords(test_text, project_repo, 'test-tag')
    expected_result = """\
        First line
        version: %s
        date: %s
        author: %s
        Last line""" % (short_ref, commit_timestamp, author)
    assert result == expected_result

    result = git.substitute_keywords(test_text, project_repo, 'test-garbage')
    expected_result = ''
    assert result == expected_result
<|endoftext|>


<commit_before>from __future__ import (absolute_import, print_function)
from acli.output.ec2 import (output_ec2_info)
from moto import mock_ec2
import pytest
from boto.ec2.tag import Tag
import boto

@pytest.yield_fixture(scope='function')
def ec2():
    """EC2 mock service"""
    mock = mock_ec2()
    mock.start()
    conn = boto.ec2.connect_to_region("eu-west-1")

    reservations = conn.run_instances(image_id='ami-12345', min_count=2, max_count=2)
    for i, s in enumerate(reservations.instances):
        conn.create_tags(
            resource_ids=[s.id],
            tags={'Name': 'BOB'})
    yield{'servers': reservations}
    mock.stop()

def test_ec2_list(ec2):
    assert output_ec2_info(output_media='console', instance=ec2.get('servers').instances[0])
<commit_msg>fix test.<commit_after>from __future__ import (absolute_import, print_function)
from acli.output.ec2 import (output_ec2_info)
from moto import mock_ec2
import pytest
from boto.ec2.tag import Tag
import boto

@pytest.yield_fixture(scope='function')
def ec2():
    """EC2 mock service"""
    mock = mock_ec2()
    mock.start()
    conn = boto.ec2.connect_to_region("eu-west-1")

    reservations = conn.run_instances(image_id='ami-12345', min_count=2, max_count=2)
    for i, s in enumerate(reservations.instances):
        conn.create_tags(
            resource_ids=[s.id],
            tags={'Name': 'BOB'})
    yield{'servers': reservations}
    mock.stop()

def test_ec2_list(ec2):
    with pytest.raises(SystemExit):
        assert output_ec2_info(output_media='console', instance=ec2.get('servers').instances[0])

<|endoftext|>


<commit_before>import sys
import periphery
from .asserts import AssertRaises

if sys.version_info[0] == 3:
    raw_input = input

i2c_devpath1 = None
i2c_devpath2 = None
i2c_eeprom_addr = None

def test_arguments():
    print("Starting arguments test...")

    # Open with invalid type
    with AssertRaises(TypeError):
        periphery.I2C(123)

    print("Arguments test passed.")

def test_open_close():
    print("Starting open/close test...")

    # Open non-existent device
    with AssertRaises(periphery.I2CError):
        periphery.I2C("/foo/bar")

    # Open legitimate device
    i2c = periphery.I2C(i2c_devpath1)
    assert i2c.fd > 0

    # Close I2C
    i2c.close()

    print("Open/close test passed.")

def test_loopback():
    print("Starting loopback test...")

    # "Loopback" plan
    # 1. Read EEPROM via sysfs
    # 2. Read EEPROM via I2C
    # 3. Compare data

    # Read EEPROM via sysfs
    sysfs_path = "/sys/bus/i2c/devices/%s-00%02x/eeprom" % (i2c_devpath1[-1], i2c_eeprom_addr)
    with open(sysfs_path, "rb") as f:
        eeprom_data = f.read(256)

    # Read EEPROM via I2C
    data = bytearray()
    i2c = periphery.I2C(i2c_devpath1)
    for addr in range(0, 256):
        msgs = [periphery.I2C.Message([0x00, addr]), periphery.I2C.Message([0x00], read=True)]
        i2c.transfer(i2c_eeprom_addr, msgs)
        data.append(msgs[1].data[0])

    # Compare data
    assert eeprom_data == data

    # Try bytes read
    msgs = [periphery.I2C.Message(b"\x00\x00"), periphery.I2C.Message(b"\x00", read=True)]
    i2c.transfer(i2c_eeprom_addr, msgs)
    assert isinstance(msgs[1].data, bytes)
    assert bytearray(msgs[1].data)[0] == data[0]

    # Try bytearray read
    msgs = [periphery.I2C.Message(bytearray([0x00, 0x00])), periphery.I2C.Message(bytearray([0x00]), read=True)]
    i2c.transfer(i2c_eeprom_addr, msgs)
    assert isinstance(msgs[1].data, bytearray)
    assert msgs[1].data[0] == data[0]

    i2c.close()

    print("Loopback test passed.")

def test_interactive():
    print("Starting interactive test...")

    # Open device 2
    i2c = periphery.I2C(i2c_devpath2)

    print("")
    print("Starting interactive test. Get out your logic analyzer, buddy!")
    raw_input("Press enter to continue...")

    # There isn't much we can do without assuming a device on the other end,
    # because I2C needs an acknowledgement bit on each transferred byte.
    #
    # But we can send a transaction and expect it to time out.

    # S [ 0x7a W ] [0xaa] [0xbb] [0xcc] [0xdd] NA
    messages = [periphery.I2C.Message([0xaa, 0xbb, 0xcc, 0xdd])]

    raw_input("Press enter to start transfer...")

    # Transfer to non-existent device
    with AssertRaises(periphery.I2CError):
        i2c.transfer(0x7a, messages)

    i2c.close()

    success = raw_input("I2C transfer occurred? y/n ")
    assert success == "y"

    print("Interactive test passed.")

if __name__ == "__main__":
    if len(sys.argv) < 4:
        print("Usage: python -m tests.test_i2c <i2c device 1> <i2c device 2> <i2c eeprom address>")
        print("")
        print(" i2c device 1        i2c device path with EEPROM on bus")
        print(" i2c device 2        i2c device path for observation with logic analyzer / oscilloscope")
        print(" i2c eeprom address  address of EEPROM on i2c device 1 bus")
        print("")
        print("Hint: for BeagleBone Black, use onboard EEPROM on /dev/i2c-0 and")
        print("I2C1 (SCL=P9.24, SDA=P9.26) on /dev/i2c-2, and run this test:")
        print("    python -m tests.test_i2c /dev/i2c-0 /dev/i2c-2 0x50")
        sys.exit(1)

    i2c_devpath1 = sys.argv[1]
    i2c_devpath2 = sys.argv[2]
    i2c_eeprom_addr = int(sys.argv[3], 0)

    print("Starting I2C tests...")

    test_arguments()
    test_open_close()
    test_loopback()
    test_interactive()

    print("All I2C tests passed.")
<commit_msg>tests/i2c: remove beaglebone black specific loopback test<commit_after>import sys
import periphery
from .asserts import AssertRaises

if sys.version_info[0] == 3:
    raw_input = input

i2c_devpath = None

def test_arguments():
    print("Starting arguments test...")

    # Open with invalid type
    with AssertRaises(TypeError):
        periphery.I2C(123)

    print("Arguments test passed.")

def test_open_close():
    print("Starting open/close test...")

    # Open non-existent device
    with AssertRaises(periphery.I2CError):
        periphery.I2C("/foo/bar")

    # Open legitimate device
    i2c = periphery.I2C(i2c_devpath)
    assert i2c.fd > 0

    # Close I2C
    i2c.close()

    print("Open/close test passed.")

def test_loopback():
    print("Starting loopback test...")

    print("No general way to do a loopback test for I2C without a real component, skipping...")

    print("Loopback test passed.")

def test_interactive():
    print("Starting interactive test...")

    # Open device 2
    i2c = periphery.I2C(i2c_devpath)

    print("")
    print("Starting interactive test. Get out your logic analyzer, buddy!")
    raw_input("Press enter to continue...")

    # There isn't much we can do without assuming a device on the other end,
    # because I2C needs an acknowledgement bit on each transferred byte.
    #
    # But we can send a transaction and expect it to time out.

    # S [ 0x7a W ] [0xaa] [0xbb] [0xcc] [0xdd] NA
    messages = [periphery.I2C.Message([0xaa, 0xbb, 0xcc, 0xdd])]

    raw_input("Press enter to start transfer...")

    # Transfer to non-existent device
    with AssertRaises(periphery.I2CError):
        i2c.transfer(0x7a, messages)

    i2c.close()

    success = raw_input("I2C transfer occurred? y/n ")
    assert success == "y"

    print("Interactive test passed.")

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: python -m tests.test_i2c <i2c device>")
        print("")
        print(" i2c device          i2c device path for observation with an oscilloscope or logic analyzer")
        print("")
        print("Hint: for BeagleBone Black, use")
        print("I2C1 (SCL=P9.24, SDA=P9.26) on /dev/i2c-2, and run this test:")
        print("    python -m tests.test_i2c /dev/i2c-2")
        sys.exit(1)

    i2c_devpath = sys.argv[1]

    print("Starting I2C tests...")

    test_arguments()
    test_open_close()
    test_loopback()
    test_interactive()

    print("All I2C tests passed.")
<|endoftext|>