<commit_before>import numpy as np

from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import SkipTest
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_raises

from sklearn.decomposition import DictionaryLearning
from sklearn.decomposition import MiniBatchDictionaryLearning
from sklearn.decomposition import SparseCoder
from sklearn.decomposition import dict_learning_online
from sklearn.decomposition import sparse_encode

rng = np.random.RandomState(0)
n_samples, n_features = 10, 8
X = rng.randn(n_samples, n_features)

def test_dict_learning_shapes():
    n_components = 5
    dico = DictionaryLearning(n_components).fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_overcomplete():
    n_components = 12
    X = rng.randn(n_samples, n_features)
    dico = DictionaryLearning(n_components).fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_reconstruction():
    n_components = 12
    dico = DictionaryLearning(n_components, transform_algorithm='omp',
                              transform_alpha=0.001, random_state=0)
    code = dico.fit(X).transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X)

    dico.set_params(transform_algorithm='lasso_lars')
    code = dico.transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X, decimal=2)

    # used to test lars here too, but there's no guarantee the number of
    # nonzero atoms is right.

def test_dict_learning_nonzero_coefs():
    n_components = 4
    dico = DictionaryLearning(n_components, transform_algorithm='lars',
                              transform_n_nonzero_coefs=3, random_state=0)
    code = dico.fit(X).transform(X[1])
    assert_true(len(np.flatnonzero(code)) == 3)

    dico.set_params(transform_algorithm='omp')
    code = dico.transform(X[1])
    assert_equal(len(np.flatnonzero(code)), 3)

def test_dict_learning_unknown_fit_algorithm():
    n_components = 5
    dico = DictionaryLearning(n_components, fit_algorithm='<unknown>')
    assert_raises(ValueError, dico.fit, X)

def test_dict_learning_split():
    n_components = 5
    dico = DictionaryLearning(n_components, transform_algorithm='threshold')
    code = dico.fit(X).transform(X)
    dico.split_sign = True
    split_code = dico.transform(X)

    assert_array_equal(split_code[:, :n_components] -
                       split_code[:, n_components:], code)

def test_dict_learning_online_shapes():
#    rng = np.random.RandomState(0)
#    X = rng.randn(12, 10)
    n_components = 8
    code, dictionary = dict_learning_online(X, n_components=n_components,
                                            alpha=1, random_state=rng)
    assert_equal(code.shape, (n_samples, n_components))
    assert_equal(dictionary.shape, (n_components, n_features))
    assert_equal(np.dot(code, dictionary).shape, X.shape)

def test_dict_learning_online_verbosity():
    n_components = 5
    # test verbosity
    from cStringIO import StringIO
    import sys
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    dico = MiniBatchDictionaryLearning(n_components, n_iter=20, verbose=1)
    dico.fit(X)
    dico = MiniBatchDictionaryLearning(n_components, n_iter=20, verbose=2)
    dico.fit(X)
    dict_learning_online(X, n_components=n_components, alpha=1, verbose=1)
    dict_learning_online(X, n_components=n_components, alpha=1, verbose=2)
    sys.stdout = old_stdout
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_online_estimator_shapes():
    n_components = 5
    dico = MiniBatchDictionaryLearning(n_components, n_iter=20)
    dico.fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_online_overcomplete():
    n_components = 12
    dico = MiniBatchDictionaryLearning(n_components, n_iter=20).fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_online_initialization():
    n_components = 12
    V = rng.randn(n_components, n_features)
    dico = MiniBatchDictionaryLearning(n_components, n_iter=0,
                                       dict_init=V).fit(X)
    assert_array_equal(dico.components_, V)

def test_dict_learning_online_partial_fit():
    # this test was not actually passing before!
    raise SkipTest
    n_components = 12
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    rng1 = np.random.RandomState(0)
    rng2 = np.random.RandomState(0)
    dico1 = MiniBatchDictionaryLearning(n_components, n_iter=10, batch_size=1,
                                        shuffle=False, dict_init=V,
                                        random_state=rng1).fit(X)
    dico2 = MiniBatchDictionaryLearning(n_components, n_iter=1, dict_init=V,
                                        random_state=rng2)
    for ii, sample in enumerate(X):
        dico2.partial_fit(sample, iter_offset=ii * dico2.n_iter)
        # if ii == 1: break
    assert_true(not np.all(sparse_encode(X, dico1.components_, alpha=100) ==
                           0))
    assert_array_equal(dico1.components_, dico2.components_)

def test_sparse_encode_shapes():
    n_components = 12
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    for algo in ('lasso_lars', 'lasso_cd', 'lars', 'omp', 'threshold'):
        code = sparse_encode(X, V, algorithm=algo)
        assert_equal(code.shape, (n_samples, n_components))

def test_sparse_encode_error():
    n_components = 12
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    code = sparse_encode(X, V, alpha=0.001)
    assert_true(not np.all(code == 0))
    assert_less(np.sqrt(np.sum((np.dot(code, V) - X) ** 2)), 0.1)

def test_unknown_method():
    n_components = 12
    V = rng.randn(n_components, n_features)  # random init
    assert_raises(ValueError, sparse_encode, X, V, algorithm="<unknown>")

def test_sparse_coder_estimator():
    n_components = 12
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    code = SparseCoder(dictionary=V, transform_algorithm='lasso_lars',
                       transform_alpha=0.001).transform(X)
    assert_true(not np.all(code == 0))
    assert_less(np.sqrt(np.sum((np.dot(code, V) - X) ** 2)), 0.1)
<commit_msg>TST fix random states in all dict learning tests, make test independent of test sequence.<commit_after>import numpy as np

from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import SkipTest
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_less
from sklearn.utils.testing import assert_raises

from sklearn.decomposition import DictionaryLearning
from sklearn.decomposition import MiniBatchDictionaryLearning
from sklearn.decomposition import SparseCoder
from sklearn.decomposition import dict_learning_online
from sklearn.decomposition import sparse_encode

rng_global = np.random.RandomState(0)
n_samples, n_features = 10, 8
X = rng_global.randn(n_samples, n_features)

def test_dict_learning_shapes():
    n_components = 5
    dico = DictionaryLearning(n_components, random_state=0).fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_overcomplete():
    n_components = 12
    dico = DictionaryLearning(n_components, random_state=0).fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_reconstruction():
    n_components = 12
    dico = DictionaryLearning(n_components, transform_algorithm='omp',
                              transform_alpha=0.001, random_state=0)
    code = dico.fit(X).transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X)

    dico.set_params(transform_algorithm='lasso_lars')
    code = dico.transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X, decimal=2)

    # used to test lars here too, but there's no guarantee the number of
    # nonzero atoms is right.

def test_dict_learning_nonzero_coefs():
    n_components = 4
    dico = DictionaryLearning(n_components, transform_algorithm='lars',
                              transform_n_nonzero_coefs=3, random_state=0)
    code = dico.fit(X).transform(X[1])
    assert_true(len(np.flatnonzero(code)) == 3)

    dico.set_params(transform_algorithm='omp')
    code = dico.transform(X[1])
    assert_equal(len(np.flatnonzero(code)), 3)

def test_dict_learning_unknown_fit_algorithm():
    n_components = 5
    dico = DictionaryLearning(n_components, fit_algorithm='<unknown>')
    assert_raises(ValueError, dico.fit, X)

def test_dict_learning_split():
    n_components = 5
    dico = DictionaryLearning(n_components, transform_algorithm='threshold',
                              random_state=0)
    code = dico.fit(X).transform(X)
    dico.split_sign = True
    split_code = dico.transform(X)

    assert_array_equal(split_code[:, :n_components] -
                       split_code[:, n_components:], code)

def test_dict_learning_online_shapes():
    rng = np.random.RandomState(0)
    n_components = 8
    code, dictionary = dict_learning_online(X, n_components=n_components,
                                            alpha=1, random_state=rng)
    assert_equal(code.shape, (n_samples, n_components))
    assert_equal(dictionary.shape, (n_components, n_features))
    assert_equal(np.dot(code, dictionary).shape, X.shape)

def test_dict_learning_online_verbosity():
    n_components = 5
    # test verbosity
    from cStringIO import StringIO
    import sys
    old_stdout = sys.stdout
    sys.stdout = StringIO()
    dico = MiniBatchDictionaryLearning(n_components, n_iter=20, verbose=1,
                                       random_state=0)
    dico.fit(X)
    dico = MiniBatchDictionaryLearning(n_components, n_iter=20, verbose=2,
                                       random_state=0)
    dico.fit(X)
    dict_learning_online(X, n_components=n_components, alpha=1, verbose=1,
                         random_state=0)
    dict_learning_online(X, n_components=n_components, alpha=1, verbose=2,
                         random_state=0)
    sys.stdout = old_stdout
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_online_estimator_shapes():
    n_components = 5
    dico = MiniBatchDictionaryLearning(n_components, n_iter=20, random_state=0)
    dico.fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_online_overcomplete():
    n_components = 12
    dico = MiniBatchDictionaryLearning(n_components, n_iter=20,
                                       random_state=0).fit(X)
    assert_true(dico.components_.shape == (n_components, n_features))

def test_dict_learning_online_initialization():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)
    dico = MiniBatchDictionaryLearning(n_components, n_iter=0,
                                       dict_init=V, random_state=0).fit(X)
    assert_array_equal(dico.components_, V)

def test_dict_learning_online_partial_fit():
    # this test was not actually passing before!
    raise SkipTest
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    dico1 = MiniBatchDictionaryLearning(n_components, n_iter=10, batch_size=1,
                                        shuffle=False, dict_init=V,
                                        random_state=0).fit(X)
    dico2 = MiniBatchDictionaryLearning(n_components, n_iter=1, dict_init=V,
                                        random_state=0)
    for ii, sample in enumerate(X):
        dico2.partial_fit(sample, iter_offset=ii * dico2.n_iter)
        # if ii == 1: break
    assert_true(not np.all(sparse_encode(X, dico1.components_, alpha=100) ==
                           0))
    assert_array_equal(dico1.components_, dico2.components_)

def test_sparse_encode_shapes():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    for algo in ('lasso_lars', 'lasso_cd', 'lars', 'omp', 'threshold'):
        code = sparse_encode(X, V, algorithm=algo)
        assert_equal(code.shape, (n_samples, n_components))

def test_sparse_encode_error():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    code = sparse_encode(X, V, alpha=0.001)
    assert_true(not np.all(code == 0))
    assert_less(np.sqrt(np.sum((np.dot(code, V) - X) ** 2)), 0.1)

def test_unknown_method():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    assert_raises(ValueError, sparse_encode, X, V, algorithm="<unknown>")

def test_sparse_coder_estimator():
    n_components = 12
    rng = np.random.RandomState(0)
    V = rng.randn(n_components, n_features)  # random init
    V /= np.sum(V ** 2, axis=1)[:, np.newaxis]
    code = SparseCoder(dictionary=V, transform_algorithm='lasso_lars',
                       transform_alpha=0.001).transform(X)
    assert_true(not np.all(code == 0))
    assert_less(np.sqrt(np.sum((np.dot(code, V) - X) ** 2)), 0.1)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (C) 2020-2021 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import re
import unittest

from base_interfaces_test import BasicInterfaceTest
from vyos.configsession import ConfigSessionError
from vyos.ifconfig import Section
from vyos.util import cmd
from vyos.util import process_named_running
from vyos.util import read_file

ca_cert  = '/config/auth/eapol_test_ca.pem'
ssl_cert = '/config/auth/eapol_test_server.pem'
ssl_key  = '/config/auth/eapol_test_server.key'

def get_wpa_supplicant_value(interface, key):
    tmp = read_file(f'/run/wpa_supplicant/{interface}.conf')
    tmp = re.findall(r'\n?{}=(.*)'.format(key), tmp)
    return tmp[0]

class EthernetInterfaceTest(BasicInterfaceTest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls._test_ip = True
        cls._test_ipv6 = True
        cls._test_ipv6_pd = True
        cls._test_ipv6_dhcpc6 = True
        cls._test_mtu = True
        cls._test_vlan = True
        cls._test_qinq = True
        cls._base_path = ['interfaces', 'ethernet']
        cls._mirror_interfaces = ['dum21354']

        # we need to filter out VLAN interfaces identified by a dot (.)
        # in their name - just in case!
        if 'TEST_ETH' in os.environ:
            tmp = os.environ['TEST_ETH'].split()
            cls._interfaces = tmp
        else:
            for tmp in Section.interfaces('ethernet'):
                if not '.' in tmp:
                    cls._interfaces.append(tmp)

        cls._macs = {}
        for interface in cls._interfaces:
            cls._macs[interface] = read_file(f'/sys/class/net/{interface}/address')

        # call base-classes classmethod
        super(cls, cls).setUpClass()

    def tearDown(self):
        for interface in self._interfaces:
            # when using a dedicated interface to test via TEST_ETH environment
            # variable only this one will be cleared in the end - usable to test
            # ethernet interfaces via SSH
            self.cli_delete(self._base_path + [interface])
            self.cli_set(self._base_path + [interface, 'duplex', 'auto'])
            self.cli_set(self._base_path + [interface, 'speed', 'auto'])
            self.cli_set(self._base_path + [interface, 'hw-id', self._macs[interface]])

        # Tear down mirror interfaces for SPAN (Switch Port Analyzer)
        for span in self._mirror_interfaces:
            section = Section.section(span)
            self.cli_delete(['interfaces', section, span])

        self.cli_commit()

    def test_dhcp_disable_interface(self):
        # When interface is configured as admin down, it must be admin down
        # even when dhcpc starts on the given interface
        for interface in self._interfaces:
            self.cli_set(self._base_path + [interface, 'disable'])

            # Also enable DHCP (ISC DHCP always places interface in admin up
            # state so we check that we do not start DHCP client.
            # https://phabricator.vyos.net/T2767
            self.cli_set(self._base_path + [interface, 'address', 'dhcp'])

        self.cli_commit()

        # Validate interface state
        for interface in self._interfaces:
            flags = read_file(f'/sys/class/net/{interface}/flags')
            self.assertEqual(int(flags, 16) & 1, 0)

    def test_offloading_rps(self):
        # enable RPS on all available CPUs, RPS works woth a CPU bitmask,
        # where each bit represents a CPU (core/thread). The formula below
        # expands to rps_cpus = 255 for a 8 core system
        rps_cpus = (1 << os.cpu_count()) -1

        # XXX: we should probably reserve one core when the system is under
        # high preasure so we can still have a core left for housekeeping.
        # This is done by masking out the lowst bit so CPU0 is spared from
        # receive packet steering.
        rps_cpus &= ~1

        for interface in self._interfaces:
            self.cli_set(self._base_path + [interface, 'offload', 'rps'])

        self.cli_commit()

        for interface in self._interfaces:
            cpus = read_file('/sys/class/net/eth1/queues/rx-0/rps_cpus')
            # remove the nasty ',' separation on larger strings
            cpus = cpus.replace(',','')
            cpus = int(cpus, 16)

            self.assertEqual(f'{cpus:x}', f'{rps_cpus:x}')

    def test_non_existing_interface(self):
        unknonw_interface = self._base_path + ['eth667']
        self.cli_set(unknonw_interface)

        # check validate() - interface does not exist
        with self.assertRaises(ConfigSessionError):
            self.cli_commit()

        # we need to remove this wrong interface from the configuration
        # manually, else tearDown() will have problem in commit()
        self.cli_delete(unknonw_interface)

    def test_speed_duplex_verify(self):
        for interface in self._interfaces:
            self.cli_set(self._base_path + [interface, 'speed', '1000'])

            # check validate() - if either speed or duplex is not auto, the
            # other one must be manually configured, too
            with self.assertRaises(ConfigSessionError):
                self.cli_commit()
            self.cli_set(self._base_path + [interface, 'speed', 'auto'])
            self.cli_commit()

    def test_eapol_support(self):
        for interface in self._interfaces:
            # Enable EAPoL
            self.cli_set(self._base_path + [interface, 'eapol', 'ca-cert-file', ca_cert])
            self.cli_set(self._base_path + [interface, 'eapol', 'cert-file', ssl_cert])
            self.cli_set(self._base_path + [interface, 'eapol', 'key-file', ssl_key])

        self.cli_commit()

        # Check for running process
        self.assertTrue(process_named_running('wpa_supplicant'))

        # Validate interface config
        for interface in self._interfaces:
            tmp = get_wpa_supplicant_value(interface, 'key_mgmt')
            self.assertEqual('IEEE8021X', tmp)

            tmp = get_wpa_supplicant_value(interface, 'eap')
            self.assertEqual('TLS', tmp)

            tmp = get_wpa_supplicant_value(interface, 'eapol_flags')
            self.assertEqual('0', tmp)

            tmp = get_wpa_supplicant_value(interface, 'ca_cert')
            self.assertEqual(f'"{ca_cert}"', tmp)

            tmp = get_wpa_supplicant_value(interface, 'client_cert')
            self.assertEqual(f'"{ssl_cert}"', tmp)

            tmp = get_wpa_supplicant_value(interface, 'private_key')
            self.assertEqual(f'"{ssl_key}"', tmp)

            mac = read_file(f'/sys/class/net/{interface}/address')
            tmp = get_wpa_supplicant_value(interface, 'identity')
            self.assertEqual(f'"{mac}"', tmp)

if __name__ == '__main__':
    # Our SSL certificates need a subject ...
    subject = '/C=DE/ST=BY/O=VyOS/localityName=Cloud/commonName=vyos/' \
              'organizationalUnitName=VyOS/emailAddress=maintainers@vyos.io/'

    if not (os.path.isfile(ssl_key) and os.path.isfile(ssl_cert)):
        # Generate mandatory SSL certificate
        tmp = f'openssl req -newkey rsa:4096 -new -nodes -x509 -days 3650 '\
              f'-keyout {ssl_key} -out {ssl_cert} -subj {subject}'
        cmd(tmp)

    if not os.path.isfile(ca_cert):
        # Generate "CA"
        tmp = f'openssl req -new -x509 -key {ssl_key} -out {ca_cert} -subj {subject}'
        cmd(tmp)

    for file in [ca_cert, ssl_cert, ssl_key]:
        cmd(f'sudo chown radius_priv_user:vyattacfg {file}')

    unittest.main(verbosity=2)
<commit_msg>smoketest: ethernet: bugfix RPS CPU validation<commit_after>#!/usr/bin/env python3
#
# Copyright (C) 2020-2021 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import re
import unittest

from base_interfaces_test import BasicInterfaceTest
from vyos.configsession import ConfigSessionError
from vyos.ifconfig import Section
from vyos.util import cmd
from vyos.util import process_named_running
from vyos.util import read_file

ca_cert  = '/config/auth/eapol_test_ca.pem'
ssl_cert = '/config/auth/eapol_test_server.pem'
ssl_key  = '/config/auth/eapol_test_server.key'

def get_wpa_supplicant_value(interface, key):
    tmp = read_file(f'/run/wpa_supplicant/{interface}.conf')
    tmp = re.findall(r'\n?{}=(.*)'.format(key), tmp)
    return tmp[0]

class EthernetInterfaceTest(BasicInterfaceTest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls._test_ip = True
        cls._test_ipv6 = True
        cls._test_ipv6_pd = True
        cls._test_ipv6_dhcpc6 = True
        cls._test_mtu = True
        cls._test_vlan = True
        cls._test_qinq = True
        cls._base_path = ['interfaces', 'ethernet']
        cls._mirror_interfaces = ['dum21354']

        # we need to filter out VLAN interfaces identified by a dot (.)
        # in their name - just in case!
        if 'TEST_ETH' in os.environ:
            tmp = os.environ['TEST_ETH'].split()
            cls._interfaces = tmp
        else:
            for tmp in Section.interfaces('ethernet'):
                if not '.' in tmp:
                    cls._interfaces.append(tmp)

        cls._macs = {}
        for interface in cls._interfaces:
            cls._macs[interface] = read_file(f'/sys/class/net/{interface}/address')

        # call base-classes classmethod
        super(cls, cls).setUpClass()

    def tearDown(self):
        for interface in self._interfaces:
            # when using a dedicated interface to test via TEST_ETH environment
            # variable only this one will be cleared in the end - usable to test
            # ethernet interfaces via SSH
            self.cli_delete(self._base_path + [interface])
            self.cli_set(self._base_path + [interface, 'duplex', 'auto'])
            self.cli_set(self._base_path + [interface, 'speed', 'auto'])
            self.cli_set(self._base_path + [interface, 'hw-id', self._macs[interface]])

        # Tear down mirror interfaces for SPAN (Switch Port Analyzer)
        for span in self._mirror_interfaces:
            section = Section.section(span)
            self.cli_delete(['interfaces', section, span])

        self.cli_commit()

    def test_dhcp_disable_interface(self):
        # When interface is configured as admin down, it must be admin down
        # even when dhcpc starts on the given interface
        for interface in self._interfaces:
            self.cli_set(self._base_path + [interface, 'disable'])

            # Also enable DHCP (ISC DHCP always places interface in admin up
            # state so we check that we do not start DHCP client.
            # https://phabricator.vyos.net/T2767
            self.cli_set(self._base_path + [interface, 'address', 'dhcp'])

        self.cli_commit()

        # Validate interface state
        for interface in self._interfaces:
            flags = read_file(f'/sys/class/net/{interface}/flags')
            self.assertEqual(int(flags, 16) & 1, 0)

    def test_offloading_rps(self):
        # enable RPS on all available CPUs, RPS works woth a CPU bitmask,
        # where each bit represents a CPU (core/thread). The formula below
        # expands to rps_cpus = 255 for a 8 core system
        rps_cpus = (1 << os.cpu_count()) -1

        # XXX: we should probably reserve one core when the system is under
        # high preasure so we can still have a core left for housekeeping.
        # This is done by masking out the lowst bit so CPU0 is spared from
        # receive packet steering.
        rps_cpus &= ~1

        for interface in self._interfaces:
            self.cli_set(self._base_path + [interface, 'offload', 'rps'])

        self.cli_commit()

        for interface in self._interfaces:
            cpus = read_file(f'/sys/class/net/{interface}/queues/rx-0/rps_cpus')
            # remove the nasty ',' separation on larger strings
            cpus = cpus.replace(',','')
            cpus = int(cpus, 16)

            self.assertEqual(f'{cpus:x}', f'{rps_cpus:x}')

    def test_non_existing_interface(self):
        unknonw_interface = self._base_path + ['eth667']
        self.cli_set(unknonw_interface)

        # check validate() - interface does not exist
        with self.assertRaises(ConfigSessionError):
            self.cli_commit()

        # we need to remove this wrong interface from the configuration
        # manually, else tearDown() will have problem in commit()
        self.cli_delete(unknonw_interface)

    def test_speed_duplex_verify(self):
        for interface in self._interfaces:
            self.cli_set(self._base_path + [interface, 'speed', '1000'])

            # check validate() - if either speed or duplex is not auto, the
            # other one must be manually configured, too
            with self.assertRaises(ConfigSessionError):
                self.cli_commit()
            self.cli_set(self._base_path + [interface, 'speed', 'auto'])
            self.cli_commit()

    def test_eapol_support(self):
        for interface in self._interfaces:
            # Enable EAPoL
            self.cli_set(self._base_path + [interface, 'eapol', 'ca-cert-file', ca_cert])
            self.cli_set(self._base_path + [interface, 'eapol', 'cert-file', ssl_cert])
            self.cli_set(self._base_path + [interface, 'eapol', 'key-file', ssl_key])

        self.cli_commit()

        # Check for running process
        self.assertTrue(process_named_running('wpa_supplicant'))

        # Validate interface config
        for interface in self._interfaces:
            tmp = get_wpa_supplicant_value(interface, 'key_mgmt')
            self.assertEqual('IEEE8021X', tmp)

            tmp = get_wpa_supplicant_value(interface, 'eap')
            self.assertEqual('TLS', tmp)

            tmp = get_wpa_supplicant_value(interface, 'eapol_flags')
            self.assertEqual('0', tmp)

            tmp = get_wpa_supplicant_value(interface, 'ca_cert')
            self.assertEqual(f'"{ca_cert}"', tmp)

            tmp = get_wpa_supplicant_value(interface, 'client_cert')
            self.assertEqual(f'"{ssl_cert}"', tmp)

            tmp = get_wpa_supplicant_value(interface, 'private_key')
            self.assertEqual(f'"{ssl_key}"', tmp)

            mac = read_file(f'/sys/class/net/{interface}/address')
            tmp = get_wpa_supplicant_value(interface, 'identity')
            self.assertEqual(f'"{mac}"', tmp)

if __name__ == '__main__':
    # Our SSL certificates need a subject ...
    subject = '/C=DE/ST=BY/O=VyOS/localityName=Cloud/commonName=vyos/' \
              'organizationalUnitName=VyOS/emailAddress=maintainers@vyos.io/'

    if not (os.path.isfile(ssl_key) and os.path.isfile(ssl_cert)):
        # Generate mandatory SSL certificate
        tmp = f'openssl req -newkey rsa:4096 -new -nodes -x509 -days 3650 '\
              f'-keyout {ssl_key} -out {ssl_cert} -subj {subject}'
        cmd(tmp)

    if not os.path.isfile(ca_cert):
        # Generate "CA"
        tmp = f'openssl req -new -x509 -key {ssl_key} -out {ca_cert} -subj {subject}'
        cmd(tmp)

    for file in [ca_cert, ssl_cert, ssl_key]:
        cmd(f'sudo chown radius_priv_user:vyattacfg {file}')

    unittest.main(verbosity=2)
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

from __future__ import division, unicode_literals

from collections import Iterable, OrderedDict

from django.core.exceptions import ImproperlyConfigured
from django.http import HttpResponse
from django.utils.encoding import force_text, python_2_unicode_compatible
from django.utils.html import escape
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext as _

from yepes.types import Undefined
from yepes.utils.http import urlquote_plus
from yepes.utils.properties import cached_property
from yepes.utils.structures import OrderedDictWhichIteratesOverValues
from yepes.views.list import (
    Ordering, Page, PageSize,
    AvailablePages, VisiblePages,
    ListQuery, ListView,
)

@python_2_unicode_compatible
class Facet(object):

    name = Undefined
    is_multiple = False
    verbose_name = Undefined

    def __init__(self, query):
        assert self.name is not Undefined
        self._query = query
        if self.verbose_name is Undefined:
            self.verbose_name = force_text(self.name).capitalize()

    def __eq__(self, other):
        return (isinstance(other, Facet) and self.name == other.name)

    def __hash__(self):
        return hash((self.__class__.__name__, self.name))

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        return str('<{0}: {1}>'.format(self.__class__.__name__, self.name))

    def __str__(self):
        return self.name

    def filter_objects(self, object_list, constraints=Undefined):
        raise NotImplementedError('Subclasses of Facet must override filter_objects() method')

    def get_constraints(self):
        raise NotImplementedError('Subclasses of Facet must override get_constraints() method')

    def get_selected_constraints(self):
        raise NotImplementedError('Subclasses of Facet must override get_selected_constraints() method')

    @cached_property
    def _constraints(self):
        return self._query._facets.get(self.name) or ()

    @cached_property
    def available_constraints(self):
        return OrderedDictWhichIteratesOverValues([
            (constraint.name, constraint)
            for constraint
            in self.get_constraints()
        ])

    @cached_property
    def constraints(self):
        constraints = OrderedDictWhichIteratesOverValues()
        for constraint in self.get_selected_constraints():
            constraints[constraint.name] = constraint
            if not self.is_multiple:
                break

        return constraints

    @cached_property
    def remove_url(self):
        facets = [
            f
            for f
            in self._query.facets
            if f != self
        ]
        return self._query.get_absolute_url(facets=facets, page=None)

@python_2_unicode_compatible
class Constraint(object):

    def __init__(self, facet, value, name=None, verbose_name=None, count=None):
        self._query = facet._query
        self.facet = facet
        self.value = value
        if name is not None:
            self.name = force_text(name)
        else:
            self.name = force_text(value)

        if verbose_name is not None:
            self.verbose_name = force_text(verbose_name)
        else:
            self.verbose_name = self.name.capitalize()

        self.count = count

    def __eq__(self, other):
        return (isinstance(other, Constraint)
                and self.facet == other.facet
                and self.name == other.name)

    def __hash__(self):
        return hash((self.__class__.__name__, self.facet, self.name))

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        return str('<{0}: {1}>'.format(self.__class__.__name__, self.name))

    def __str__(self):
        return self.name

    def get_link(self):
        if self.count is None:
            if self.is_selected:
                template = '{verbose_name}'
                kwargs = {
                    'remove_url': escape(self.remove_url),
                    'verbose_name': escape(self.verbose_name),
                }
            else:
                template = '{verbose_name}'
                kwargs = {
                    'url': escape(self.url),
                    'verbose_name': escape(self.verbose_name),
                }
        else:
            if self.is_selected:
                template = '{verbose_name} ({count})'
                kwargs = {
                    'count': self.count,
                    'remove_url': escape(self.remove_url),
                    'verbose_name': escape(self.verbose_name),
                }
            else:
                template = '{verbose_name} ({count})'
                kwargs = {
                    'count': self.count,
                    'url': escape(self.url),
                    'verbose_name': escape(self.verbose_name),
                }
        return mark_safe(template.format(**kwargs))

    @cached_property
    def is_selected(self):
        return (self.name in self.facet.constraints)

    @cached_property
    def remove_url(self):
        facets = [
            f
            for f
            in self._query.facets
            if f != self.facet
        ]
        if self.facet.is_multiple:
            constraints = [
                constraint
                for constraint
                in self.facet.constraints
                if constraint != self
            ]
            if constraints:
                facets.append((self.facet, constraints))

        return self._query.get_absolute_url(facets=facets, page=None)

    @cached_property
    def url(self):
        facets = [
            f
            for f
            in self._query.facets
            if f != self.facet
        ]
        if self.facet.is_multiple:
            constraints = [
                constraint
                for constraint
                in self.facet.constraints
                if constraint != self
            ]
            constraints.append(self)
            facets.append((self.facet, constraints))
        else:
            facets.append((self.facet, [self]))

        return self._query.get_absolute_url(facets=facets, page=None)

class SearchQuery(ListQuery):

    def _filter_objects(self, object_list, user_query):
        user_query = self._prepare_user_query(user_query)
        if user_query:
            object_list = object_list.search(user_query)
        return object_list

    def _prepare_user_query(self, user_query):
        if user_query is Undefined:
            user_query = self.user_query
        return user_query

    def get_object_list(self, user_query=Undefined, ordering=Undefined,
                              page=Undefined, page_size=Undefined):
        object_list = self.space
        object_list = self._filter_objects(object_list, user_query)
        object_list = self._sort_objects(object_list, ordering)
        object_list = self._limit_objects(object_list, page, page_size)
        return object_list

    def get_query_string(self, user_query=Undefined, *args, **kwargs):
        params = []

        user_query = self._prepare_user_query(user_query)
        if user_query:
            params.append('{0}={1}'.format(
                self._user_query_kwarg,
                urlquote_plus(user_query, safe=''),
            ))

        extra = super(SearchQuery, self).get_query_string(*args, **kwargs)
        if extra:
            params.append(extra)

        return '&'.join(params)

    @cached_property
    def _user_query_kwarg(self):
        return self._view.user_query_kwarg

    @cached_property
    def available_orderings(self):
        orderings = OrderedDictWhichIteratesOverValues()
        for ordering_class in self._view.get_orderings():
            ordering = ordering_class(self)
            orderings[ordering.name] = ordering

        if self.user_query:
            if all((len(ord.fields) != 1 or ord.fields[0] != 'search_score')
                   for ord
                   in orderings):
                attrs = {
                    'name': 'relevance',
                    'fields': ['search_score'],
                    'verbose_name': _('Relevance'),
                }
                ordering_class = type(str('Ordering'), (Ordering, ), attrs)
                orderings['relevance'] = ordering_class(self)

        return orderings

    @cached_property
    def default_ordering(self):
        ordering = self.available_orderings.get(self._default_ordering)
        if self.user_query:
            for ord in self.available_orderings:
                if 'search_score' in ord.fields:
                    ordering = ord
                    break

        if not ordering:
            return next(iter(self.available_orderings), None)
        else:
            return ordering

    @cached_property
    def user_query(self):
        query = self._view.get_user_query()
        if query is None:
            return query
        else:
            return ' '.join(query.split())

class SearchView(ListView):

    query_class = SearchQuery
    search_signal = None
    user_query_kwarg = 'query'

    def dispatch(self, *args, **kwargs):
        response = super(SearchView, self).dispatch(*args, **kwargs)
        if (isinstance(response, HttpResponse)
                and response.status_code == 200
                and self.query.user_query):

            try:
                is_staff = self.request.user.is_staff
            except AttributeError:
                is_staff = False

            if not is_staff:
                self.send_search_signal(self.query.user_query, self.request)

        return response

    def get_template_names(self):
        names = super(SearchView, self).get_template_names()
        if not names or names[-1].endswith('_list.html'):
            model = self.get_model()
            if model is not None:
                names.insert(-1, '{0}/{1}_search.html'.format(
                    model._meta.app_label,
                    model._meta.model_name,
                ))
        return names

    def get_user_query(self):
        """
        Returns the query entered by the user.
        """
        return self.request.GET.get(self.user_query_kwarg)

    def send_search_signal(self, user_query, request):
        """
        Sends ``SearchView.search_signal`` if defined.
        """
        if self.search_signal is not None:
            self.search_signal.send(
                sender=self,
                query=user_query,
                request=request)

class FacetedSearchQuery(SearchQuery):

    def _filter_objects(self, object_list, user_query, facets):
        object_list = super(FacetedSearchQuery, self)._filter_objects(object_list, user_query)
        facets = self._prepare_facets(facets)
        if facets:
            for f in facets:
                if isinstance(f, Facet):
                    object_list = f.filter_objects(object_list, f.constraints)
                else:
                    facet, constraints = f
                    object_list = facet.filter_objects(object_list, constraints)
        return object_list

    def _prepare_facets(self, facets):
        if facets is Undefined:
            facets = self.facets
        return facets

    def get_object_list(self, user_query=Undefined, facets=Undefined,
                          ordering=Undefined, page=Undefined,
                          page_size=Undefined):
        object_list = self.space
        object_list = self._filter_objects(object_list, user_query, facets)
        object_list = self._sort_objects(object_list, ordering)
        object_list = self._limit_objects(object_list, page, page_size)
        return object_list

    def get_query_string(self, facets=Undefined, *args, **kwargs):
        params = []

        facets = self._prepare_facets(facets)
        if facets:
            facet_params = []
            for f in facets:
                if isinstance(f, Facet):
                    cons = f.constraints
                else:
                    f, cons = f

                facet_params.append('{0}:{1}'.format(f, ':'.join(
                    con.name
                    for con
                    in cons
                )))

            params.append('{0}={1}'.format(self._facets_kwarg, ','.join(facet_params)))

        extra = super(FacetedSearchQuery, self).get_query_string(*args, **kwargs)
        if extra:
            params.append(extra)

        return '&'.join(params)

    @cached_property
    def _facets(self):
        return self._view.get_selected_facets()

    @cached_property
    def _facets_kwarg(self):
        return self._view.facets_kwarg

    @cached_property
    def available_facets(self):
        facets = OrderedDictWhichIteratesOverValues()
        for facet_class in self._view.get_facets():
            facet = facet_class(self)
            facets[facet.name] = facet

        return facets

    @cached_property
    def facets(self):
        selected_facets = OrderedDictWhichIteratesOverValues()
        for name in self._facets:
            facet = self.available_facets.get(name)
            if facet is not None:
                selected_facets[facet.name] = facet

        return selected_facets

class FacetedSearchView(SearchView):

    facets = None
    _facets = Undefined
    facets_kwarg = 'filters'
    query_class = FacetedSearchQuery

    def get_facets(self):
        """
        Returns a list containing all available facets.
        """
        if self._facets is Undefined:
            self._facets = self.normalize_facets(self.facets)
        return self._facets

    def get_selected_facets(self):
        """
        Returns a list of facets which the user has selected.
        """
        facets = OrderedDict()
        raw_facets = self.request.GET.get(self.facets_kwarg)
        if raw_facets:
            for raw_facet in raw_facets.split(','):
                facet = raw_facet.split(':', 1)
                if len(facet) == 2:
                    name, value = facet
                    facets[name] = value.split(':')

        return facets

    def normalize_facets(self, facets):
        """
        Normalizes ``facets`` to a list of subclasses of ``Facet``.
        """
        if not facets:
            return []

        facets = list(facets)
        for facet in facets:
            if not isinstance(facet, type):
                msg = '{cls}.facets should contain subclasses of Facet.'
                raise ImproperlyConfigured(msg.format(cls=self.__class__.__name__))

        return facets

<commit_msg>Move the addition of 'relevance' ordering from SearchQuery to SearchView<commit_after># -*- coding:utf-8 -*-

from __future__ import division, unicode_literals

from collections import Iterable, OrderedDict

from django.core.exceptions import ImproperlyConfigured
from django.http import HttpResponse
from django.utils.encoding import force_text, python_2_unicode_compatible
from django.utils.html import escape
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext as _

from yepes.types import Undefined
from yepes.utils.http import urlquote_plus
from yepes.utils.properties import cached_property
from yepes.utils.structures import OrderedDictWhichIteratesOverValues
from yepes.views.list import (
    Ordering, Page, PageSize,
    AvailablePages, VisiblePages,
    ListQuery, ListView,
)

@python_2_unicode_compatible
class Facet(object):

    name = Undefined
    is_multiple = False
    verbose_name = Undefined

    def __init__(self, query):
        assert self.name is not Undefined
        self._query = query
        if self.verbose_name is Undefined:
            self.verbose_name = force_text(self.name).capitalize()

    def __eq__(self, other):
        return (isinstance(other, Facet) and self.name == other.name)

    def __hash__(self):
        return hash((self.__class__.__name__, self.name))

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        return str('<{0}: {1}>'.format(self.__class__.__name__, self.name))

    def __str__(self):
        return self.name

    def filter_objects(self, object_list, constraints=Undefined):
        raise NotImplementedError('Subclasses of Facet must override filter_objects() method')

    def get_constraints(self):
        raise NotImplementedError('Subclasses of Facet must override get_constraints() method')

    def get_selected_constraints(self):
        raise NotImplementedError('Subclasses of Facet must override get_selected_constraints() method')

    @cached_property
    def _constraints(self):
        return self._query._facets.get(self.name) or ()

    @cached_property
    def available_constraints(self):
        return OrderedDictWhichIteratesOverValues([
            (constraint.name, constraint)
            for constraint
            in self.get_constraints()
        ])

    @cached_property
    def constraints(self):
        constraints = OrderedDictWhichIteratesOverValues()
        for constraint in self.get_selected_constraints():
            constraints[constraint.name] = constraint
            if not self.is_multiple:
                break

        return constraints

    @cached_property
    def remove_url(self):
        facets = [
            f
            for f
            in self._query.facets
            if f != self
        ]
        return self._query.get_absolute_url(facets=facets, page=None)

@python_2_unicode_compatible
class Constraint(object):

    def __init__(self, facet, value, name=None, verbose_name=None, count=None):
        self._query = facet._query
        self.facet = facet
        self.value = value
        if name is not None:
            self.name = force_text(name)
        else:
            self.name = force_text(value)

        if verbose_name is not None:
            self.verbose_name = force_text(verbose_name)
        else:
            self.verbose_name = self.name.capitalize()

        self.count = count

    def __eq__(self, other):
        return (isinstance(other, Constraint)
                and self.facet == other.facet
                and self.name == other.name)

    def __hash__(self):
        return hash((self.__class__.__name__, self.facet, self.name))

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        return str('<{0}: {1}>'.format(self.__class__.__name__, self.name))

    def __str__(self):
        return self.name

    def get_link(self):
        if self.count is None:
            if self.is_selected:
                template = '{verbose_name}'
                kwargs = {
                    'remove_url': escape(self.remove_url),
                    'verbose_name': escape(self.verbose_name),
                }
            else:
                template = '{verbose_name}'
                kwargs = {
                    'url': escape(self.url),
                    'verbose_name': escape(self.verbose_name),
                }
        else:
            if self.is_selected:
                template = '{verbose_name} ({count})'
                kwargs = {
                    'count': self.count,
                    'remove_url': escape(self.remove_url),
                    'verbose_name': escape(self.verbose_name),
                }
            else:
                template = '{verbose_name} ({count})'
                kwargs = {
                    'count': self.count,
                    'url': escape(self.url),
                    'verbose_name': escape(self.verbose_name),
                }
        return mark_safe(template.format(**kwargs))

    @cached_property
    def is_selected(self):
        return (self.name in self.facet.constraints)

    @cached_property
    def remove_url(self):
        facets = [
            f
            for f
            in self._query.facets
            if f != self.facet
        ]
        if self.facet.is_multiple:
            constraints = [
                constraint
                for constraint
                in self.facet.constraints
                if constraint != self
            ]
            if constraints:
                facets.append((self.facet, constraints))

        return self._query.get_absolute_url(facets=facets, page=None)

    @cached_property
    def url(self):
        facets = [
            f
            for f
            in self._query.facets
            if f != self.facet
        ]
        if self.facet.is_multiple:
            constraints = [
                constraint
                for constraint
                in self.facet.constraints
                if constraint != self
            ]
            constraints.append(self)
            facets.append((self.facet, constraints))
        else:
            facets.append((self.facet, [self]))

        return self._query.get_absolute_url(facets=facets, page=None)

class SearchQuery(ListQuery):

    def _filter_objects(self, object_list, user_query):
        user_query = self._prepare_user_query(user_query)
        if user_query:
            object_list = object_list.search(user_query)
        return object_list

    def _prepare_user_query(self, user_query):
        if user_query is Undefined:
            user_query = self.user_query
        return user_query

    def get_object_list(self, user_query=Undefined, ordering=Undefined,
                              page=Undefined, page_size=Undefined):
        object_list = self.space
        object_list = self._filter_objects(object_list, user_query)
        object_list = self._sort_objects(object_list, ordering)
        object_list = self._limit_objects(object_list, page, page_size)
        return object_list

    def get_query_string(self, user_query=Undefined, *args, **kwargs):
        params = []

        user_query = self._prepare_user_query(user_query)
        if user_query:
            params.append('{0}={1}'.format(
                self._user_query_kwarg,
                urlquote_plus(user_query, safe=''),
            ))

        extra = super(SearchQuery, self).get_query_string(*args, **kwargs)
        if extra:
            params.append(extra)

        return '&'.join(params)

    @cached_property
    def _user_query_kwarg(self):
        return self._view.user_query_kwarg

    @cached_property
    def user_query(self):
        query = self._view.get_user_query()
        if query is None:
            return query
        else:
            return ' '.join(query.split())

class SearchView(ListView):

    query_class = SearchQuery
    search_signal = None
    user_query_kwarg = 'query'

    def dispatch(self, *args, **kwargs):
        response = super(SearchView, self).dispatch(*args, **kwargs)
        if (isinstance(response, HttpResponse)
                and response.status_code == 200
                and self.query.user_query):

            try:
                is_staff = self.request.user.is_staff
            except AttributeError:
                is_staff = False

            if not is_staff:
                self.send_search_signal(self.query.user_query, self.request)

        return response

    def get_default_ordering(self):
        if self.query.user_query:
            for ord in self.query.available_orderings:
                if 'search_score' in ord.fields:
                    return ord

        return super(SearchView, self).get_default_ordering()

    def get_orderings(self):
        if self._orderings is Undefined:
            orderings = super(SearchView, self).get_orderings()
            if (self.query.user_query
                    and all('search_score' not in ord.fields
                            for ord
                            in orderings)):
                attrs = {
                    'name': 'relevance',
                    'fields': ['search_score'],
                    'verbose_name': _('Relevance'),
                }
                orderings.insert(0, type(str('Ordering'), (Ordering, ), attrs))
                self._orderings = orderings

        return self._orderings

    def get_template_names(self):
        names = super(SearchView, self).get_template_names()
        if not names or names[-1].endswith('_list.html'):
            model = self.get_model()
            if model is not None:
                names.insert(-1, '{0}/{1}_search.html'.format(
                    model._meta.app_label,
                    model._meta.model_name,
                ))
        return names

    def get_user_query(self):
        """
        Returns the query entered by the user.
        """
        return self.request.GET.get(self.user_query_kwarg)

    def send_search_signal(self, user_query, request):
        """
        Sends ``SearchView.search_signal`` if defined.
        """
        if self.search_signal is not None:
            self.search_signal.send(
                sender=self,
                query=user_query,
                request=request)

class FacetedSearchQuery(SearchQuery):

    def _filter_objects(self, object_list, user_query, facets):
        object_list = super(FacetedSearchQuery, self)._filter_objects(object_list, user_query)
        facets = self._prepare_facets(facets)
        if facets:
            for f in facets:
                if isinstance(f, Facet):
                    object_list = f.filter_objects(object_list, f.constraints)
                else:
                    facet, constraints = f
                    object_list = facet.filter_objects(object_list, constraints)
        return object_list

    def _prepare_facets(self, facets):
        if facets is Undefined:
            facets = self.facets
        return facets

    def get_object_list(self, user_query=Undefined, facets=Undefined,
                          ordering=Undefined, page=Undefined,
                          page_size=Undefined):
        object_list = self.space
        object_list = self._filter_objects(object_list, user_query, facets)
        object_list = self._sort_objects(object_list, ordering)
        object_list = self._limit_objects(object_list, page, page_size)
        return object_list

    def get_query_string(self, facets=Undefined, *args, **kwargs):
        params = []

        facets = self._prepare_facets(facets)
        if facets:
            facet_params = []
            for f in facets:
                if isinstance(f, Facet):
                    cons = f.constraints
                else:
                    f, cons = f

                facet_params.append('{0}:{1}'.format(f, ':'.join(
                    con.name
                    for con
                    in cons
                )))

            params.append('{0}={1}'.format(self._facets_kwarg, ','.join(facet_params)))

        extra = super(FacetedSearchQuery, self).get_query_string(*args, **kwargs)
        if extra:
            params.append(extra)

        return '&'.join(params)

    @cached_property
    def _facets(self):
        return self._view.get_selected_facets()

    @cached_property
    def _facets_kwarg(self):
        return self._view.facets_kwarg

    @cached_property
    def available_facets(self):
        facets = OrderedDictWhichIteratesOverValues()
        for facet_class in self._view.get_facets():
            facet = facet_class(self)
            facets[facet.name] = facet

        return facets

    @cached_property
    def facets(self):
        selected_facets = OrderedDictWhichIteratesOverValues()
        for name in self._facets:
            facet = self.available_facets.get(name)
            if facet is not None:
                selected_facets[facet.name] = facet

        return selected_facets

class FacetedSearchView(SearchView):

    facets = None
    _facets = Undefined
    facets_kwarg = 'filters'
    query_class = FacetedSearchQuery

    def get_facets(self):
        """
        Returns a list containing all available facets.
        """
        if self._facets is Undefined:
            self._facets = self.normalize_facets(self.facets)
        return self._facets

    def get_selected_facets(self):
        """
        Returns a list of facets which the user has selected.
        """
        facets = OrderedDict()
        raw_facets = self.request.GET.get(self.facets_kwarg)
        if raw_facets:
            for raw_facet in raw_facets.split(','):
                facet = raw_facet.split(':', 1)
                if len(facet) == 2:
                    name, value = facet
                    facets[name] = value.split(':')

        return facets

    def normalize_facets(self, facets):
        """
        Normalizes ``facets`` to a list of subclasses of ``Facet``.
        """
        if not facets:
            return []

        facets = list(facets)
        for facet in facets:
            if not isinstance(facet, type):
                msg = '{cls}.facets should contain subclasses of Facet.'
                raise ImproperlyConfigured(msg.format(cls=self.__class__.__name__))

        return facets

<|endoftext|>


<commit_before># Copyright (c) 2015 Yubico AB
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#    1. Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#    2. Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

from __future__ import absolute_import, print_function

import ykman.logging_setup
from ykman import __version__
from ..util import TRANSPORT, Cve201715361VulnerableError, YUBIKEY
from ..native.pyusb import get_usb_backend_version
from ..driver_otp import libversion as ykpers_version
from ..driver_ccid import open_devices as open_ccid, list_readers
from ..device import YubiKey
from ..descriptor import (get_descriptors, list_devices, open_device,
                          FailedOpeningDeviceException, Descriptor)
from .util import UpperCaseChoice, YkmanContextObject
from .info import info
from .mode import mode
from .otp import otp
from .opgp import openpgp
from .oath import oath
from .piv import piv
from .fido import fido
from .config import config
import usb.core
import click
import logging
import sys

logger = logging.getLogger(__name__)

CLICK_CONTEXT_SETTINGS = dict(
    help_option_names=['-h', '--help'],
    max_content_width=999
)

def print_version(ctx, param, value):
    if not value or ctx.resilient_parsing:
        return
    click.echo('YubiKey Manager (ykman) version: {}'.format(__version__))
    libs = []
    libs.append('libykpers ' + ('.'.join('%d' % d for d in ykpers_version)
                                if ykpers_version is not None
                                else 'not found!'))
    usb_lib = get_usb_backend_version()
    libs.append(usb_lib or '<pyusb backend missing>')
    click.echo('Libraries:')
    for lib in libs:
        click.echo('    {}'.format(lib))
    ctx.exit()

def _disabled_transport(ctx, transports, cmd_name):
    req = ', '.join((t.name for t in TRANSPORT if t & transports))
    click.echo("Command '{}' requires one of the following USB interfaces "
               "to be enabled: '{}'.".format(cmd_name, req))
    ctx.fail("Use 'ykman mode' to set the enabled USB interfaces.")

def _run_cmd_for_serial(ctx, cmd, transports, serial):
    try:
        return open_device(transports, serial=serial)
    except FailedOpeningDeviceException:
        try:  # Retry, any transport
            dev = open_device(serial=serial)
            if not dev.mode.transports & transports:
                if dev.config.usb_supported & transports:
                    _disabled_transport(ctx, transports, cmd)
                else:
                    ctx.fail("Command '{}' is not supported by this device."
                             .format(cmd))
        except FailedOpeningDeviceException:
            ctx.fail('Failed connecting to a YubiKey with serial: {}'
                     .format(serial))

def _run_cmd_for_single(ctx, cmd, transports, reader=None):
    if reader:
        if TRANSPORT.has(transports, TRANSPORT.CCID):
            readers = list(open_ccid(reader))
            if len(readers) == 1:
                return YubiKey(Descriptor.from_driver(readers[0]), readers[0])
            elif len(readers) > 1:
                ctx.fail('Multiple YubiKeys on external readers detected.')
            else:
                ctx.fail('No YubiKey found on external reader.')
        else:
            ctx.fail('Not a CCID command.')
    try:
        descriptors = get_descriptors()
    except usb.core.NoBackendError:
        ctx.fail('No PyUSB backend detected!')
    n_keys = len(descriptors)
    if n_keys == 0:
        ctx.fail('No YubiKey detected!')
    if n_keys > 1:
        ctx.fail('Multiple YubiKeys detected. Use --device SERIAL to specify '
                 'which one to use.')
    descriptor = descriptors[0]
    if descriptor.mode.transports & transports:
        try:
            return descriptor.open_device(transports)
        except FailedOpeningDeviceException:
            ctx.fail('Failed connecting to the YubiKey.')
    else:
        _disabled_transport(ctx, transports, cmd)

@click.group(context_settings=CLICK_CONTEXT_SETTINGS)
@click.option('-v', '--version', is_flag=True, callback=print_version,
              expose_value=False, is_eager=True)
@click.option('-d', '--device', type=int, metavar='SERIAL')
@click.option('-l', '--log-level', default=None,
              type=UpperCaseChoice(ykman.logging_setup.LOG_LEVEL_NAMES),
              help='Enable logging at given verbosity level.',
              )
@click.option('--log-file', default=None,
              type=str, metavar='FILE',
              help='Write logs to the given FILE instead of standard error; '
                   'ignored unless --log-level is also set.',
              )
@click.option(
        '-r', '--reader',
        help='Use an external smart card reader. Conflicts with --device and '
             'list.',
        metavar='NAME', default=None)
@click.pass_context
def cli(ctx, device, log_level, log_file, reader):
    """
    Configure your YubiKey via the command line.

    Examples:

    \b
      List connected YubiKeys, only output serial number:
      $ ykman list --serials

    \b
      Show information about YubiKey with serial number 0123456:
      $ ykman --device 0123456 info
    """
    ctx.obj = YkmanContextObject()

    if log_level:
        ykman.logging_setup.setup(log_level, log_file=log_file)

    if reader and device:
        ctx.fail('--reader and --device options can\'t be combined.')

    subcmd = next(c for c in COMMANDS if c.name == ctx.invoked_subcommand)
    if subcmd == list_keys:
        if reader:
            ctx.fail('--reader and list command can\'t be combined.')
        return

    transports = getattr(subcmd, 'transports', TRANSPORT.usb_transports())
    if transports:
        def resolve_device():
            if device is not None:
                dev = _run_cmd_for_serial(ctx, subcmd.name, transports, device)
            else:
                dev = _run_cmd_for_single(ctx, subcmd.name, transports, reader)
            ctx.call_on_close(dev.close)
            return dev
        ctx.obj.add_resolver('dev', resolve_device)

@cli.command('list')
@click.option('-s', '--serials', is_flag=True, help='Output only serial '
              'numbers, one per line (devices without serial will be omitted).')
@click.option(
    '-r', '--readers', is_flag=True, help='List available smart card readers.')
@click.pass_context
def list_keys(ctx, serials, readers):
    """
    List connected YubiKeys.
    """
    if readers:
        for reader in list_readers():
            click.echo(reader.name)
        ctx.exit()

    all_descriptors = get_descriptors()
    descriptors = [d for d in all_descriptors if d.key_type != YUBIKEY.SKY]
    skys = len(all_descriptors) - len(descriptors)
    handled_serials = set()
    for dev in list_devices():
        handled = False
        if skys > 0 and dev.key_type == YUBIKEY.SKY:
            skys -= 1
            serial = None
            handled = True
        else:
            serial = dev.serial
            if serial not in handled_serials:
                handled_serials.add(serial)
                matches = [d for d in descriptors if (d.key_type, d.mode)
                           == (dev.driver.key_type, dev.driver.mode)]
                if len(matches) > 0:
                    d = matches[0]
                    descriptors.remove(d)
                    handled = True
        if handled:
            if serials:
                if serial:
                    click.echo(serial)
            else:
                click.echo('{} [{}]{}'.format(
                    dev.device_name,
                    dev.mode,
                    ' Serial: {}'.format(serial) if serial else '')
                )
        dev.close()
        if not descriptors and not skys:
            break

COMMANDS = (list_keys, info, mode, otp, openpgp, oath, piv, fido, config)

for cmd in COMMANDS:
    cli.add_command(cmd)

def main():
    try:
        cli(obj={})
    except ValueError as e:
        logger.error('Error', exc_info=e)
        click.echo('Error: ' + str(e))
        return 1

    except Cve201715361VulnerableError as err:
        logger.error('Error', exc_info=err)
        click.echo('Error: ' + str(err))
        return 2

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>ykman list: list based on descriptors<commit_after># Copyright (c) 2015 Yubico AB
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#    1. Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#    2. Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

from __future__ import absolute_import, print_function

import ykman.logging_setup
from ykman import __version__
from ..util import TRANSPORT, Cve201715361VulnerableError, YUBIKEY
from ..native.pyusb import get_usb_backend_version
from ..driver_otp import libversion as ykpers_version
from ..driver_ccid import open_devices as open_ccid, list_readers
from ..device import YubiKey
from ..descriptor import (get_descriptors, list_devices, open_device,
                          FailedOpeningDeviceException, Descriptor)
from .util import UpperCaseChoice, YkmanContextObject
from .info import info
from .mode import mode
from .otp import otp
from .opgp import openpgp
from .oath import oath
from .piv import piv
from .fido import fido
from .config import config
import usb.core
import click
import logging
import sys

logger = logging.getLogger(__name__)

CLICK_CONTEXT_SETTINGS = dict(
    help_option_names=['-h', '--help'],
    max_content_width=999
)

def print_version(ctx, param, value):
    if not value or ctx.resilient_parsing:
        return
    click.echo('YubiKey Manager (ykman) version: {}'.format(__version__))
    libs = []
    libs.append('libykpers ' + ('.'.join('%d' % d for d in ykpers_version)
                                if ykpers_version is not None
                                else 'not found!'))
    usb_lib = get_usb_backend_version()
    libs.append(usb_lib or '<pyusb backend missing>')
    click.echo('Libraries:')
    for lib in libs:
        click.echo('    {}'.format(lib))
    ctx.exit()

def _disabled_transport(ctx, transports, cmd_name):
    req = ', '.join((t.name for t in TRANSPORT if t & transports))
    click.echo("Command '{}' requires one of the following USB interfaces "
               "to be enabled: '{}'.".format(cmd_name, req))
    ctx.fail("Use 'ykman mode' to set the enabled USB interfaces.")

def _run_cmd_for_serial(ctx, cmd, transports, serial):
    try:
        return open_device(transports, serial=serial)
    except FailedOpeningDeviceException:
        try:  # Retry, any transport
            dev = open_device(serial=serial)
            if not dev.mode.transports & transports:
                if dev.config.usb_supported & transports:
                    _disabled_transport(ctx, transports, cmd)
                else:
                    ctx.fail("Command '{}' is not supported by this device."
                             .format(cmd))
        except FailedOpeningDeviceException:
            ctx.fail('Failed connecting to a YubiKey with serial: {}'
                     .format(serial))

def _run_cmd_for_single(ctx, cmd, transports, reader=None):
    if reader:
        if TRANSPORT.has(transports, TRANSPORT.CCID):
            readers = list(open_ccid(reader))
            if len(readers) == 1:
                return YubiKey(Descriptor.from_driver(readers[0]), readers[0])
            elif len(readers) > 1:
                ctx.fail('Multiple YubiKeys on external readers detected.')
            else:
                ctx.fail('No YubiKey found on external reader.')
        else:
            ctx.fail('Not a CCID command.')
    try:
        descriptors = get_descriptors()
    except usb.core.NoBackendError:
        ctx.fail('No PyUSB backend detected!')
    n_keys = len(descriptors)
    if n_keys == 0:
        ctx.fail('No YubiKey detected!')
    if n_keys > 1:
        ctx.fail('Multiple YubiKeys detected. Use --device SERIAL to specify '
                 'which one to use.')
    descriptor = descriptors[0]
    if descriptor.mode.transports & transports:
        try:
            return descriptor.open_device(transports)
        except FailedOpeningDeviceException:
            ctx.fail('Failed connecting to the YubiKey.')
    else:
        _disabled_transport(ctx, transports, cmd)

@click.group(context_settings=CLICK_CONTEXT_SETTINGS)
@click.option('-v', '--version', is_flag=True, callback=print_version,
              expose_value=False, is_eager=True)
@click.option('-d', '--device', type=int, metavar='SERIAL')
@click.option('-l', '--log-level', default=None,
              type=UpperCaseChoice(ykman.logging_setup.LOG_LEVEL_NAMES),
              help='Enable logging at given verbosity level.',
              )
@click.option('--log-file', default=None,
              type=str, metavar='FILE',
              help='Write logs to the given FILE instead of standard error; '
                   'ignored unless --log-level is also set.',
              )
@click.option(
        '-r', '--reader',
        help='Use an external smart card reader. Conflicts with --device and '
             'list.',
        metavar='NAME', default=None)
@click.pass_context
def cli(ctx, device, log_level, log_file, reader):
    """
    Configure your YubiKey via the command line.

    Examples:

    \b
      List connected YubiKeys, only output serial number:
      $ ykman list --serials

    \b
      Show information about YubiKey with serial number 0123456:
      $ ykman --device 0123456 info
    """
    ctx.obj = YkmanContextObject()

    if log_level:
        ykman.logging_setup.setup(log_level, log_file=log_file)

    if reader and device:
        ctx.fail('--reader and --device options can\'t be combined.')

    subcmd = next(c for c in COMMANDS if c.name == ctx.invoked_subcommand)
    if subcmd == list_keys:
        if reader:
            ctx.fail('--reader and list command can\'t be combined.')
        return

    transports = getattr(subcmd, 'transports', TRANSPORT.usb_transports())
    if transports:
        def resolve_device():
            if device is not None:
                dev = _run_cmd_for_serial(ctx, subcmd.name, transports, device)
            else:
                dev = _run_cmd_for_single(ctx, subcmd.name, transports, reader)
            ctx.call_on_close(dev.close)
            return dev
        ctx.obj.add_resolver('dev', resolve_device)

@cli.command('list')
@click.option('-s', '--serials', is_flag=True, help='Output only serial '
              'numbers, one per line (devices without serial will be omitted).')
@click.option(
    '-r', '--readers', is_flag=True, help='List available smart card readers.')
@click.pass_context
def list_keys(ctx, serials, readers):
    """
    List connected YubiKeys.
    """
    if readers:
        for reader in list_readers():
            click.echo(reader.name)
        ctx.exit()

    descriptors = get_descriptors()
    handled_serials = set()
    for dev in list_devices():
        handled = False
        if dev.key_type == YUBIKEY.SKY:
            serial = None
            # We have nothing to match on, so just drop a SKY descriptor
            d = next(x for x in descriptors if x.key_type == YUBIKEY.SKY)
            descriptors.remove(d)
            handled = True
        else:
            serial = dev.serial
            if serial not in handled_serials:
                # Drop a descriptor with a matching serial and mode
                handled_serials.add(serial)
                matches = [d for d in descriptors if (d.key_type, d.mode)
                           == (dev.driver.key_type, dev.driver.mode)]
                if len(matches) > 0:
                    d = matches[0]
                    descriptors.remove(d)
                    handled = True
        if handled:
            if serials:
                if serial:
                    click.echo(serial)
            else:
                click.echo('{} [{}]{}'.format(
                    dev.device_name,
                    dev.mode,
                    ' Serial: {}'.format(serial) if serial else '')
                )
        dev.close()
        if not descriptors:
            break

    # List descriptors that failed to open.
    logger.debug('Failed to open all devices, listing based on descriptors')
    for desc in descriptors:
        if desc.key_type == YUBIKEY.SKY:
            if desc.version >= (5, 1, 0):
                name = 'Security Key NFC'
            elif desc.version <= (5, 0, 0):
                name = 'FIDO U2F Security Key'
            else:
                name = desc.key_type.value
        elif desc.key_type == YUBIKEY.YK4 and desc.version >= (5, 0, 0):
            name = 'YubiKey 5'
        else:
            name = desc.key_type.value
        click.echo('{} [{}]'.format(name, desc.mode))

COMMANDS = (list_keys, info, mode, otp, openpgp, oath, piv, fido, config)

for cmd in COMMANDS:
    cli.add_command(cmd)

def main():
    try:
        cli(obj={})
    except ValueError as e:
        logger.error('Error', exc_info=e)
        click.echo('Error: ' + str(e))
        return 1

    except Cve201715361VulnerableError as err:
        logger.error('Error', exc_info=err)
        click.echo('Error: ' + str(err))
        return 2

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>import sys

from PySide import QtGui, QtCore

from youtube_syncer import YTSyncer

class MainWindow(QtGui.QMainWindow):
    """
    Dictionary containing checkbuttons for playlists and videos
    """
    playlists_checkboxes = dict()
    """
    YTSyncer object containing all the information of the videos
    and playlists that is needed
    """
    yt_syncer = None

    def __init__(self):
        self.yt_syncer = YTSyncer()

        super(MainWindow, self).__init__()
        self.setWindowTitle("YouTube Syncer")
        self.setGeometry(350, 350, 500, 500)

        self.master_layout = QtGui.QVBoxLayout()
        self.central_widget = QtGui.QWidget(self)
        self.central_widget.setLayout(self.master_layout)
        self.setCentralWidget(self.central_widget)

        self.master_layout.addLayout(self._panel_url_header())
        self.master_layout.addWidget(self._panel_checkboxes())
        self.setLayout(self.master_layout)
        self.show()

    def _panel_url_header(self):
        url_header = QtGui.QHBoxLayout()

        url_label = QtGui.QLabel("URL")
        url_header.addWidget(url_label)

        self.url_text_box = QtGui.QLineEdit()
        url_header.addWidget(self.url_text_box)

        load_videos_btn = QtGui.QPushButton("Load Videos")
        load_videos_btn.clicked.connect(self._load_videos)
        url_header.addWidget(load_videos_btn)

        return url_header

    def _panel_checkboxes(self):
        self.checkboxes_layout = QtGui.QFormLayout()

        checkboxes_container = QtGui.QWidget()
        checkboxes_container.setLayout(self.checkboxes_layout)

        scroll_area = QtGui.QScrollArea()
        scroll_area.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOn)
        scroll_area.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOn)
        scroll_area.setWidgetResizable(True)
        scroll_area.setWidget(checkboxes_container)

        return scroll_area

    def _load_videos(self):
        url = self.url_text_box.text()
        self.yt_syncer = YTSyncer(url)
        self._add_checkboxes(self.yt_syncer.playlists)

    def _add_checkboxes(self, playlists):
        self.playlists_checkboxes = dict()
        for playlist, videos in playlists.items():
            playlist_info = dict(checkbutton=None, videos=list())

            playlist_info['checkbutton'] = QtGui.QCheckBox(playlist)
            self.checkboxes_layout.addRow(playlist_info['checkbutton'])
            for video in videos:
                video_cb = QtGui.QCheckBox(video['title'])
                self.checkboxes_layout.addRow(video_cb)
                playlist_info['videos'].append(video_cb)

            self.playlists_checkboxes[playlist] = playlist_info

def main():
    app = QtGui.QApplication.instance()
    if not app:
        app = QtGui.QApplication(sys.argv)

    main_window = MainWindow()
    sys.exit(app.exec_())

if __name__ == "__main__":
    main()
<commit_msg>Add left margin to videos checkboxes<commit_after>import sys

from PySide import QtGui, QtCore

from youtube_syncer import YTSyncer

class MainWindow(QtGui.QMainWindow):
    """
    Dictionary containing checkbuttons for playlists and videos
    """
    playlists_checkboxes = dict()
    """
    YTSyncer object containing all the information of the videos
    and playlists that is needed
    """
    yt_syncer = None

    def __init__(self):
        self.yt_syncer = YTSyncer()

        super(MainWindow, self).__init__()
        self.setWindowTitle("YouTube Syncer")
        self.setGeometry(350, 350, 500, 500)

        self.master_layout = QtGui.QVBoxLayout()
        self.central_widget = QtGui.QWidget(self)
        self.central_widget.setLayout(self.master_layout)
        self.setCentralWidget(self.central_widget)

        self.master_layout.addLayout(self._panel_url_header())
        self.master_layout.addWidget(self._panel_checkboxes())
        self.setLayout(self.master_layout)

    def _panel_url_header(self):
        url_header = QtGui.QHBoxLayout()

        url_label = QtGui.QLabel("URL")
        url_header.addWidget(url_label)

        self.url_text_box = QtGui.QLineEdit()
        url_header.addWidget(self.url_text_box)

        load_videos_btn = QtGui.QPushButton("Load Videos")
        load_videos_btn.clicked.connect(self._load_videos)
        url_header.addWidget(load_videos_btn)

        return url_header

    def _panel_checkboxes(self):
        self.checkboxes_layout = QtGui.QFormLayout()
        checkboxes_container = QtGui.QWidget()
        checkboxes_container.setLayout(self.checkboxes_layout)

        scroll_area = QtGui.QScrollArea()
        scroll_area.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOn)
        scroll_area.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOn)
        scroll_area.setWidgetResizable(True)
        scroll_area.setWidget(checkboxes_container)

        return scroll_area

    def _load_videos(self):
        url = self.url_text_box.text()
        self.yt_syncer = YTSyncer(url)
        self._add_checkboxes(self.yt_syncer.playlists)

    def _add_checkboxes(self, playlists):
        self.playlists_checkboxes = dict()
        for playlist, videos in playlists.items():
            playlist_info = dict(checkbutton=None, videos=list())

            playlist_info['checkbutton'] = QtGui.QCheckBox(playlist)
            self.checkboxes_layout.addRow(playlist_info['checkbutton'])
            videos_checkboxes_container = QtGui.QVBoxLayout()
            videos_checkboxes_container.setContentsMargins(15, 0, 0, 0)
            for video in videos:
                video_cb = QtGui.QCheckBox(video['title'])
                # self.checkboxes_layout.addRow(video_cb)
                videos_checkboxes_container.addWidget(video_cb)
                playlist_info['videos'].append(video_cb)

            self.checkboxes_layout.addRow(videos_checkboxes_container)
            self.playlists_checkboxes[playlist] = playlist_info

def main():
    app = QtGui.QApplication.instance()
    if not app:
        app = QtGui.QApplication(sys.argv)

    main_window = MainWindow()
    main_window.show()
    sys.exit(app.exec_())

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before><commit_msg>[fix nupic issue 2340] fixes compute() function in CLACLassifierRegion.py in the case of one-step prediction. Warning: fix doesn't work for multi-step prediction<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

import os
from contextlib import contextmanager
import tempfile
import subprocess
import json
from collections import deque
from uuid import uuid4
from StringIO import StringIO
from path import path
import flask
from werkzeug.serving import run_simple
from werkzeug.wsgi import get_path_info, wrap_file
from pywps.app import (Process, Service, WPSResponse, LiteralInput,
                       ComplexInput, Format, FileReference)

recent_data_files = deque(maxlen=20)

@contextmanager
def temp_dir():
    tmp = path(tempfile.mkdtemp())
    try:
        yield tmp
    finally:
        tmp.rmtree()

def say_hello(request):
    return WPSResponse({'message': "Hello %s!" % request.inputs['name']})

def feature_count(request):
    import lxml.etree
    from pywps.app import xpath_ns
    doc = lxml.etree.parse(request.inputs['layer'])
    feature_elements = xpath_ns(doc, '//gml:featureMember')
    return WPSResponse({'count': str(len(feature_elements))})

def centroids(request):
    from shapely.geometry import shape, mapping
    with temp_dir() as tmp:
        input_gml = tmp / 'input.gml'
        input_gml.write_bytes(request.inputs['layer'].read().encode('utf-8'))
        input_geojson = tmp / 'input.geojson'
        subprocess.check_call(['ogr2ogr', '-f', 'geojson',
                               input_geojson, input_gml])
        data = json.loads(input_geojson.text(encoding='utf-8'))
        for feature in data['features']:
            geom = shape(feature['geometry'])
            feature['geometry'] = mapping(geom.centroid)
        out_bytes = json.dumps(data, indent=2)
        data_file = {
            'uuid': str(uuid4()),
            'bytes': out_bytes,
            'mime-type': 'application/json',
        }
        recent_data_files.append(data_file)
        url = flask.url_for('datafile', uuid=data_file['uuid'], _external=True)
        reference = FileReference(url, data_file['mime-type'])
        return WPSResponse({'centroids_layer': reference})

def create_app():
    service = Service(processes=[
        Process(say_hello, inputs=[LiteralInput('name', 'string')]),
        Process(feature_count,
                inputs=[ComplexInput('layer', [Format('text/xml')])]),
        Process(centroids,
                inputs=[ComplexInput('layer', [Format('text/xml')])]),
    ])

    app = flask.Flask(__name__)

    @app.route('/')
    def home():
        url = flask.url_for('wps', _external=True)
        return DESCRIPTION_HTML.format(url=url)

    @app.route('/wps', methods=['GET', 'POST'])
    def wps():
        return service

    @app.route('/datafile/<uuid>')
    def datafile(uuid):
        for data_file in recent_data_files:
            if data_file['uuid'] == uuid:
                return flask.Response(data_file['bytes'])
        else:
            flask.abort(404)

    return app

DESCRIPTION_HTML = """\
<!doctype html>

PyWPS-4 demo app

This is a demo WPS server. It exposes a few processes: say_hello,
feature_count, and centroids. Connect to <tt>{url}</tt>.

Source code and issue tracking
at github.com/jachym/pywps-4.

"""

if __name__ == '__main__':
    app = create_app()
    port = int(os.environ.get('PORT', '5000'))
    debug = os.environ.get('DEBUG') == 'on'
    run_simple('0.0.0.0', port, app, use_reloader=debug)
<commit_msg>some styling<commit_after>#!/usr/bin/env python

import os
from contextlib import contextmanager
import tempfile
import subprocess
import json
from collections import deque
from uuid import uuid4
from StringIO import StringIO
from path import path
import flask
from werkzeug.serving import run_simple
from werkzeug.wsgi import get_path_info, wrap_file
from pywps.app import (Process, Service, WPSResponse, LiteralInput,
                       ComplexInput, Format, FileReference)

recent_data_files = deque(maxlen=20)

@contextmanager
def temp_dir():
    tmp = path(tempfile.mkdtemp())
    try:
        yield tmp
    finally:
        tmp.rmtree()

def say_hello(request):
    return WPSResponse({'message': "Hello %s!" % request.inputs['name']})

def feature_count(request):
    import lxml.etree
    from pywps.app import xpath_ns
    doc = lxml.etree.parse(request.inputs['layer'])
    feature_elements = xpath_ns(doc, '//gml:featureMember')
    return WPSResponse({'count': str(len(feature_elements))})

def centroids(request):
    from shapely.geometry import shape, mapping
    with temp_dir() as tmp:
        input_gml = tmp / 'input.gml'
        input_gml.write_bytes(request.inputs['layer'].read().encode('utf-8'))
        input_geojson = tmp / 'input.geojson'
        subprocess.check_call(['ogr2ogr', '-f', 'geojson',
                               input_geojson, input_gml])
        data = json.loads(input_geojson.text(encoding='utf-8'))
        for feature in data['features']:
            geom = shape(feature['geometry'])
            feature['geometry'] = mapping(geom.centroid)
        out_bytes = json.dumps(data, indent=2)
        data_file = {
            'uuid': str(uuid4()),
            'bytes': out_bytes,
            'mime-type': 'application/json',
        }
        recent_data_files.append(data_file)
        url = flask.url_for('datafile', uuid=data_file['uuid'], _external=True)
        reference = FileReference(url, data_file['mime-type'])
        return WPSResponse({'centroids_layer': reference})

def create_app():
    service = Service(processes=[
        Process(say_hello, inputs=[LiteralInput('name', 'string')]),
        Process(feature_count,
                inputs=[ComplexInput('layer', [Format('text/xml')])]),
        Process(centroids,
                inputs=[ComplexInput('layer', [Format('text/xml')])]),
    ])

    app = flask.Flask(__name__)

    @app.route('/')
    def home():
        url = flask.url_for('wps', _external=True)
        return DESCRIPTION_HTML.format(url=url)

    @app.route('/wps', methods=['GET', 'POST'])
    def wps():
        return service

    @app.route('/datafile/<uuid>')
    def datafile(uuid):
        for data_file in recent_data_files:
            if data_file['uuid'] == uuid:
                return flask.Response(data_file['bytes'])
        else:
            flask.abort(404)

    return app

DESCRIPTION_HTML = """\
<!doctype html>
<link rel="stylesheet" href="//netdna.bootstrapcdn.com/bootstrap\
/3.0.0-rc1/css/bootstrap.min.css">

PyWPS-4 demo app

This is a demo WPS server. It exposes a few processes: say_hello,
feature_count, and centroids. Connect to <tt>{url}</tt>.

Source code and issue tracking
at github.com/jachym/pywps-4.

"""

if __name__ == '__main__':
    app = create_app()
    port = int(os.environ.get('PORT', '5000'))
    debug = os.environ.get('DEBUG') == 'on'
    run_simple('0.0.0.0', port, app, use_reloader=debug)
<|endoftext|>