<commit_before>#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
import copy
import io
import os
import tempfile
import zipfile

import numpy as np
import tensorflow as tf
from tensorflow.python.keras.backend import set_session

from arch.api.utils import log_utils
from federatedml.framework.weights import OrderDictWeights, Weights
from federatedml.nn.homo_nn.nn_model import NNModel, DataConverter

Logger = log_utils.getLogger()

def _zip_dir_as_bytes(path):
    with io.BytesIO() as io_bytes:
        with zipfile.ZipFile(io_bytes, 'w', zipfile.ZIP_DEFLATED) as zip:
            for root, dirs, files in os.walk(path, topdown=False):
                for name in files:
                    full_path = os.path.join(root, name)
                    relative_path = os.path.relpath(full_path, path)
                    zip.write(filename=full_path, arcname=relative_path)
                for name in dirs:
                    full_path = os.path.join(root, name)
                    relative_path = os.path.relpath(full_path, path)
                    zip.write(filename=full_path, arcname=relative_path)
        zip_bytes = io_bytes.getvalue()
    return zip_bytes

def _compile_model(model, loss, optimizer, metrics):
    optimizer_instance = getattr(tf.keras.optimizers, optimizer.optimizer)(**optimizer.kwargs)
    losses = getattr(tf.keras.losses, loss)
    model.compile(loss=losses,
                  optimizer=optimizer_instance,
                  metrics=metrics)
    return model

def _init_session():
    sess = tf.Session()
    tf.get_default_graph()
    set_session(sess)
    return sess

def _load_model(nn_struct_json):
    return tf.keras.models.model_from_json(nn_struct_json, custom_objects={})

def build_keras(nn_define, loss, optimizer, metrics):
    import json
    nn_define_json = json.dumps(nn_define)

    sess = _init_session()
    model = _load_model(nn_struct_json=nn_define_json)
    model = _compile_model(model=model,
                           loss=loss,
                           optimizer=optimizer,
                           metrics=metrics)
    return KerasNNModel(sess, model), KerasSequenceDataConverter()

def from_keras_sequential_model(model, loss, optimizer, metrics):
    sess = _init_session()
    model = _compile_model(model=model,
                           loss=loss,
                           optimizer=optimizer,
                           metrics=metrics)
    return KerasNNModel(sess, model)

def restore_keras_nn_model(model_bytes):
    return KerasNNModel.restore_model(model_bytes)

class KerasNNModel(NNModel):
    def __init__(self, sess, model):
        self._sess: tf.Session = sess
        self._model: tf.keras.Sequential = model
        self._trainable_weights = {self._trim_device_str(v.name): v for v in self._model.trainable_weights}

    @staticmethod
    def _trim_device_str(name):
        return name.split("/")[0]

    def get_model_weights(self) -> OrderDictWeights:
        return OrderDictWeights(self._sess.run(self._trainable_weights))

    def set_model_weights(self, weights: Weights):
        unboxed = weights.unboxed
        self._sess.run([tf.assign(v, unboxed[name]) for name, v in self._trainable_weights.items()])

    def train(self, data: tf.keras.utils.Sequence, **kwargs):
        epochs = 1
        left_kwargs = copy.deepcopy(kwargs)
        if "aggregate_every_n_epoch" in kwargs:
            epochs = kwargs["aggregate_every_n_epoch"]
            del left_kwargs["aggregate_every_n_epoch"]
        self._model.fit(x=data, epochs=epochs, verbose=1, shuffle=True, **left_kwargs)
        return epochs * len(data)

    def evaluate(self, data: tf.keras.utils.Sequence, **kwargs):
        names = self._model.metrics_names
        values = self._model.evaluate(x=data, verbose=1)
        if not isinstance(values, list):
            values = [values]
        return dict(zip(names, values))

    def predict(self, data: tf.keras.utils.Sequence, **kwargs):
        return self._model.predict(data)

    def export_model(self):
        with tempfile.TemporaryDirectory() as tmp_path:
            try:
                tf.keras.models.save_model(self._model, filepath=tmp_path, save_format="tf")
            except NotImplementedError:
                import warnings
                warnings.warn('Saving the model as SavedModel is still in experimental stages. '
                              'trying tf.keras.experimental.export_saved_model...')
                tf.keras.experimental.export_saved_model(self._model, saved_model_path=tmp_path)

            model_bytes = _zip_dir_as_bytes(tmp_path)

        return model_bytes

    @staticmethod
    def restore_model(model_bytes):  # todo: restore optimizer to support incremental learning
        sess = _init_session()
        with tempfile.TemporaryDirectory() as tmp_path:
            with io.BytesIO(model_bytes) as bytes_io:
                with zipfile.ZipFile(bytes_io, 'r', zipfile.ZIP_DEFLATED) as f:
                    f.extractall(tmp_path)

            try:
                model = tf.keras.models.load_model(filepath=tmp_path)
            except IOError:
                import warnings
                warnings.warn('loading the model as SavedModel is still in experimental stages. '
                              'trying tf.keras.experimental.load_from_saved_model...')
                model = tf.keras.experimental.load_from_saved_model(saved_model_path=tmp_path)
        return KerasNNModel(sess, model)

    def export_optimizer_config(self):
        return self._model.optimizer.get_config()

class KerasSequenceData(tf.keras.utils.Sequence):

    def get_shape(self):
        return self.x_shape, self.y_shape

    def __init__(self, data_instances, batch_size):
        self.size = data_instances.count()
        if self.size <= 0:
            raise ValueError("empty data")

        _, one_data = data_instances.first()
        self.x_shape = one_data.features.shape
        self.y_shape = getattr(one_data.label, "shape", (1,))
        self.x = np.zeros((self.size, *self.x_shape))
        self.y = np.zeros((self.size, *self.y_shape))
        index = 0
        self._keys = []
        for k, inst in data_instances.collect():
            self._keys.append(k)
            self.x[index] = inst.features
            self.y[index] = inst.label
            index += 1

        self.batch_size = batch_size if batch_size > 0 else self.size

    def __getitem__(self, index):
        """Gets batch at position `index`.

        # Arguments
            index: position of the batch in the Sequence.

        # Returns
            A batch
        """
        start = self.batch_size * index
        end = self.batch_size * (index + 1)
        return self.x[start: end], self.y[start: end]

    def __len__(self):
        """Number of batch in the Sequence.

        # Returns
            The number of batches in the Sequence.
        """
        return int(np.ceil(self.size / float(self.batch_size)))

    def get_keys(self):
        return self._keys

class KerasSequenceDataConverter(DataConverter):
    def convert(self, data, *args, **kwargs):
        return KerasSequenceData(data, *args, **kwargs)
<commit_msg>improve data converter for nn<commit_after>#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
import copy
import io
import os
import tempfile
import zipfile

import numpy as np
import tensorflow as tf
from tensorflow.python.keras.backend import set_session

from arch.api.utils import log_utils
from federatedml.framework.weights import OrderDictWeights, Weights
from federatedml.nn.homo_nn.nn_model import NNModel, DataConverter

Logger = log_utils.getLogger()

def _zip_dir_as_bytes(path):
    with io.BytesIO() as io_bytes:
        with zipfile.ZipFile(io_bytes, 'w', zipfile.ZIP_DEFLATED) as zipper:
            for root, dirs, files in os.walk(path, topdown=False):
                for name in files:
                    full_path = os.path.join(root, name)
                    relative_path = os.path.relpath(full_path, path)
                    zipper.write(filename=full_path, arcname=relative_path)
                for name in dirs:
                    full_path = os.path.join(root, name)
                    relative_path = os.path.relpath(full_path, path)
                    zipper.write(filename=full_path, arcname=relative_path)
        zip_bytes = io_bytes.getvalue()
    return zip_bytes

def _compile_model(model, loss, optimizer, metrics):
    optimizer_instance = getattr(tf.keras.optimizers, optimizer.optimizer)(**optimizer.kwargs)
    losses = getattr(tf.keras.losses, loss)
    model.compile(loss=losses,
                  optimizer=optimizer_instance,
                  metrics=metrics)
    return model

def _init_session():
    sess = tf.Session()
    tf.get_default_graph()
    set_session(sess)
    return sess

def _load_model(nn_struct_json):
    return tf.keras.models.model_from_json(nn_struct_json, custom_objects={})

def build_keras(nn_define, loss, optimizer, metrics):
    import json
    nn_define_json = json.dumps(nn_define)

    sess = _init_session()
    model = _load_model(nn_struct_json=nn_define_json)
    model = _compile_model(model=model,
                           loss=loss,
                           optimizer=optimizer,
                           metrics=metrics)
    return KerasNNModel(sess, model), KerasSequenceDataConverter()

def from_keras_sequential_model(model, loss, optimizer, metrics):
    sess = _init_session()
    model = _compile_model(model=model,
                           loss=loss,
                           optimizer=optimizer,
                           metrics=metrics)
    return KerasNNModel(sess, model)

def restore_keras_nn_model(model_bytes):
    return KerasNNModel.restore_model(model_bytes)

class KerasNNModel(NNModel):
    def __init__(self, sess, model):
        self._sess: tf.Session = sess
        self._model: tf.keras.Sequential = model
        self._trainable_weights = {self._trim_device_str(v.name): v for v in self._model.trainable_weights}

    @staticmethod
    def _trim_device_str(name):
        return name.split("/")[0]

    def get_model_weights(self) -> OrderDictWeights:
        return OrderDictWeights(self._sess.run(self._trainable_weights))

    def set_model_weights(self, weights: Weights):
        unboxed = weights.unboxed
        self._sess.run([tf.assign(v, unboxed[name]) for name, v in self._trainable_weights.items()])

    def train(self, data: tf.keras.utils.Sequence, **kwargs):
        epochs = 1
        left_kwargs = copy.deepcopy(kwargs)
        if "aggregate_every_n_epoch" in kwargs:
            epochs = kwargs["aggregate_every_n_epoch"]
            del left_kwargs["aggregate_every_n_epoch"]
        self._model.fit(x=data, epochs=epochs, verbose=1, shuffle=True, **left_kwargs)
        return epochs * len(data)

    def evaluate(self, data: tf.keras.utils.Sequence, **kwargs):
        names = self._model.metrics_names
        values = self._model.evaluate(x=data, verbose=1)
        if not isinstance(values, list):
            values = [values]
        return dict(zip(names, values))

    def predict(self, data: tf.keras.utils.Sequence, **kwargs):
        return self._model.predict(data)

    def export_model(self):
        with tempfile.TemporaryDirectory() as tmp_path:
            try:
                tf.keras.models.save_model(self._model, filepath=tmp_path, save_format="tf")
            except NotImplementedError:
                import warnings
                warnings.warn('Saving the model as SavedModel is still in experimental stages. '
                              'trying tf.keras.experimental.export_saved_model...')
                tf.keras.experimental.export_saved_model(self._model, saved_model_path=tmp_path)

            model_bytes = _zip_dir_as_bytes(tmp_path)

        return model_bytes

    @staticmethod
    def restore_model(model_bytes):  # todo: restore optimizer to support incremental learning
        sess = _init_session()
        with tempfile.TemporaryDirectory() as tmp_path:
            with io.BytesIO(model_bytes) as bytes_io:
                with zipfile.ZipFile(bytes_io, 'r', zipfile.ZIP_DEFLATED) as f:
                    f.extractall(tmp_path)

            try:
                model = tf.keras.models.load_model(filepath=tmp_path)
            except IOError:
                import warnings
                warnings.warn('loading the model as SavedModel is still in experimental stages. '
                              'trying tf.keras.experimental.load_from_saved_model...')
                model = tf.keras.experimental.load_from_saved_model(saved_model_path=tmp_path)
        return KerasNNModel(sess, model)

    def export_optimizer_config(self):
        return self._model.optimizer.get_config()

class KerasSequenceData(tf.keras.utils.Sequence):

    def get_shape(self):
        return self.x_shape, self.y_shape

    def __init__(self, data_instances, batch_size):
        self.size = data_instances.count()
        if self.size <= 0:
            raise ValueError("empty data")

        _, one_data = data_instances.first()
        self.x_shape = one_data.features.shape

        num_label = len(data_instances.map(lambda x, y: [x, {y.label}]).reduce(lambda x, y: x | y))
        if num_label == 2:
            self.y_shape = (1,)
            self.x = np.zeros((self.size, *self.x_shape))
            self.y = np.zeros((self.size, *self.y_shape))
            index = 0
            self._keys = []
            for k, inst in data_instances.collect():
                self._keys.append(k)
                self.x[index] = inst.features
                self.y[index] = inst.label
                index += 1

        # encoding label in one-hot
        elif num_label > 2:
            self.y_shape = (num_label,)
            self.x = np.zeros((self.size, *self.x_shape))
            self.y = np.zeros((self.size, *self.y_shape))
            index = 0
            self._keys = []
            for k, inst in data_instances.collect():
                self._keys.append(k)
                self.x[index] = inst.features
                self.y[index][inst.label] = 1
                index += 1
        else:
            raise ValueError(f"num_label is {num_label}")

        self.batch_size = batch_size if batch_size > 0 else self.size

    def __getitem__(self, index):
        """Gets batch at position `index`.

        # Arguments
            index: position of the batch in the Sequence.

        # Returns
            A batch
        """
        start = self.batch_size * index
        end = self.batch_size * (index + 1)
        return self.x[start: end], self.y[start: end]

    def __len__(self):
        """Number of batch in the Sequence.

        # Returns
            The number of batches in the Sequence.
        """
        return int(np.ceil(self.size / float(self.batch_size)))

    def get_keys(self):
        return self._keys

class KerasSequenceDataConverter(DataConverter):
    def convert(self, data, *args, **kwargs):
        return KerasSequenceData(data, *args, **kwargs)
<|endoftext|>


<commit_before>import os
import shutil

from conans import DEFAULT_REVISION_V1
from conans.model.manifest import FileTreeManifest
from conans.model.package_metadata import PackageMetadata

from conans.model.ref import ConanFileReference, PackageReference

from conans.paths import PACKAGE_METADATA
from conans.client.cache.cache import CONAN_CONF, PROFILES_FOLDER
from conans.client.tools import replace_in_file
from conans.errors import ConanException
from conans.migrations import Migrator
from conans.model.version import Version
from conans.paths import EXPORT_SOURCES_DIR_OLD
from conans.paths.package_layouts.package_cache_layout import PackageCacheLayout
from conans.util.files import list_folder_subdirs, load, save

class ClientMigrator(Migrator):

    def __init__(self, cache, current_version, out):
        self.cache = cache
        super(ClientMigrator, self).__init__(cache.conan_folder, cache.store,
                                             current_version, out)

    def _update_settings_yml(self, old_settings):
        from conans.client.conf import default_settings_yml
        settings_path = self.cache.settings_path
        if not os.path.exists(settings_path):
            self.out.warn("Migration: This conan installation doesn't have settings yet")
            self.out.warn("Nothing to migrate here, settings will be generated automatically")
            return

        current_settings = load(self.cache.settings_path)
        if current_settings != default_settings_yml:
            self.out.warn("Migration: Updating settings.yml")
            if current_settings != old_settings:
                new_path = self.cache.settings_path + ".new"
                save(new_path, default_settings_yml)
                self.out.warn("*" * 40)
                self.out.warn("settings.yml is locally modified, can't be updated")
                self.out.warn("The new settings.yml has been stored in: %s" % new_path)
                self.out.warn("*" * 40)
            else:
                save(self.cache.settings_path, default_settings_yml)

    def _make_migrations(self, old_version):
        # ############### FILL THIS METHOD WITH THE REQUIRED ACTIONS ##############
        # VERSION 0.1
        if old_version is None:
            return
        if old_version < Version("1.13.0"):
            old_settings = """
# Only for cross building, 'os_build/arch_build' is the system that runs Conan
os_build: [Windows, WindowsStore, Linux, Macos, FreeBSD, SunOS]
arch_build: [x86, x86_64, ppc32, ppc64le, ppc64, armv6, armv7, armv7hf, armv7s, armv7k, armv8, armv8_32, armv8.3, sparc, sparcv9, mips, mips64, avr]

# Only for building cross compilation tools, 'os_target/arch_target' is the system for
# which the tools generate code
os_target: [Windows, Linux, Macos, Android, iOS, watchOS, tvOS, FreeBSD, SunOS, Arduino]
arch_target: [x86, x86_64, ppc32, ppc64le, ppc64, armv6, armv7, armv7hf, armv7s, armv7k, armv8, armv8_32, armv8.3, sparc, sparcv9, mips, mips64, avr]

# Rest of the settings are "host" settings:
# - For native building/cross building: Where the library/program will run.
# - For building cross compilation tools: Where the cross compiler will run.
os:
    Windows:
        subsystem: [None, cygwin, msys, msys2, wsl]
    WindowsStore:
        version: ["8.1", "10.0"]
    Linux:
    Macos:
        version: [None, "10.6", "10.7", "10.8", "10.9", "10.10", "10.11", "10.12", "10.13", "10.14"]
    Android:
        api_level: ANY
    iOS:
        version: ["7.0", "7.1", "8.0", "8.1", "8.2", "8.3", "9.0", "9.1", "9.2", "9.3", "10.0", "10.1", "10.2", "10.3", "11.0", "11.1", "11.2", "11.3", "11.4", "12.0", "12.1"]
    watchOS:
        version: ["4.0", "4.1", "4.2", "4.3", "5.0", "5.1"]
    tvOS:
        version: ["11.0", "11.1", "11.2", "11.3", "11.4", "12.0", "12.1"]
    FreeBSD:
    SunOS:
    Arduino:
        board: ANY
arch: [x86, x86_64, ppc32, ppc64le, ppc64, armv6, armv7, armv7hf, armv7s, armv7k, armv8, armv8_32, armv8.3, sparc, sparcv9, mips, mips64, avr]
compiler:
    sun-cc:
        version: ["5.10", "5.11", "5.12", "5.13", "5.14"]
        threads: [None, posix]
        libcxx: [libCstd, libstdcxx, libstlport, libstdc++]
    gcc:
        version: ["4.1", "4.4", "4.5", "4.6", "4.7", "4.8", "4.9",
                  "5", "5.1", "5.2", "5.3", "5.4", "5.5",
                  "6", "6.1", "6.2", "6.3", "6.4",
                  "7", "7.1", "7.2", "7.3",
                  "8", "8.1", "8.2"]
        libcxx: [libstdc++, libstdc++11]
        threads: [None, posix, win32] #  Windows MinGW
        exception: [None, dwarf2, sjlj, seh] # Windows MinGW
    Visual Studio:
        runtime: [MD, MT, MTd, MDd]
        version: ["8", "9", "10", "11", "12", "14", "15"]
        toolset: [None, v90, v100, v110, v110_xp, v120, v120_xp,
                  v140, v140_xp, v140_clang_c2, LLVM-vs2012, LLVM-vs2012_xp,
                  LLVM-vs2013, LLVM-vs2013_xp, LLVM-vs2014, LLVM-vs2014_xp,
                  LLVM-vs2017, LLVM-vs2017_xp, v141, v141_xp, v141_clang_c2]
    clang:
        version: ["3.3", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "4.0",
                  "5.0", "6.0", "7.0",
                  "8"]
        libcxx: [libstdc++, libstdc++11, libc++]
    apple-clang:
        version: ["5.0", "5.1", "6.0", "6.1", "7.0", "7.3", "8.0", "8.1", "9.0", "9.1", "10.0"]
        libcxx: [libstdc++, libc++]

build_type: [None, Debug, Release, RelWithDebInfo, MinSizeRel]
cppstd: [None, 98, gnu98, 11, gnu11, 14, gnu14, 17, gnu17, 20, gnu20]
"""
            self._update_settings_yml(old_settings)

            # MIGRATE LOCAL CACHE TO GENERATE MISSING METADATA.json
            _migrate_create_metadata(self.cache, self.out)

        if old_version < Version("1.12.0"):
            migrate_plugins_to_hooks(self.cache)

        if old_version < Version("1.0"):
            _migrate_lock_files(self.cache, self.out)

        if old_version < Version("0.25"):
            from conans.paths import DEFAULT_PROFILE_NAME
            default_profile_path = os.path.join(self.cache.conan_folder, PROFILES_FOLDER,
                                                DEFAULT_PROFILE_NAME)
            if not os.path.exists(default_profile_path):
                self.out.warn("Migration: Moving default settings from %s file to %s"
                              % (CONAN_CONF, DEFAULT_PROFILE_NAME))
                conf_path = os.path.join(self.cache.conan_folder, CONAN_CONF)

                migrate_to_default_profile(conf_path, default_profile_path, output=self.out)

                self.out.warn("Migration: export_source cache new layout")
                migrate_c_src_export_source(self.cache, self.out)

def _get_refs(cache):
    folders = list_folder_subdirs(cache.store, 4)
    return [ConanFileReference(*s.split("/")) for s in folders]

def _get_prefs(layout):
    packages_folder = layout.packages()
    folders = list_folder_subdirs(packages_folder, 1)
    return [PackageReference(layout.ref, s) for s in folders]

def _migrate_create_metadata(cache, out):
    out.warn("Migration: Generating missing metadata files")
    refs = _get_refs(cache)

    for ref in refs:
        try:
            base_folder = os.path.normpath(os.path.join(cache.store, ref.dir_repr()))
            # Force using a package cache layout for everything, we want to alter the cache,
            # not the editables
            layout = PackageCacheLayout(base_folder=base_folder, ref=ref, short_paths=False,
                                        no_lock=True)
            folder = layout.export()
            try:
                manifest = FileTreeManifest.load(folder)
                rrev = manifest.summary_hash
            except:
                rrev = DEFAULT_REVISION_V1
            metadata_path = os.path.join(layout.conan(), PACKAGE_METADATA)
            if not os.path.exists(metadata_path):
                out.info("Creating {} for {}".format(PACKAGE_METADATA, ref))
                prefs = _get_prefs(layout)
                metadata = PackageMetadata()
                metadata.recipe.revision = rrev
                for pref in prefs:
                    try:
                        pmanifest = FileTreeManifest.load(layout.package(pref))
                        prev = pmanifest.summary_hash
                    except:
                        prev = DEFAULT_REVISION_V1
                    metadata.packages[pref.id].revision = prev
                    metadata.packages[pref.id].recipe_revision = metadata.recipe.revision
                save(metadata_path, metadata.dumps())
        except Exception as e:
            raise ConanException("Something went wrong while generating the metadata.json files "
                                 "in the cache, please try to fix the issue or wipe the cache: {}"
                                 ":{}".format(ref, e))
    out.success("Migration: Generating missing metadata files finished OK!\n")

def _migrate_lock_files(cache, out):
    out.warn("Migration: Removing old lock files")
    base_dir = cache.store
    pkgs = list_folder_subdirs(base_dir, 4)
    for pkg in pkgs:
        out.info("Removing locks for %s" % pkg)
        try:
            count = os.path.join(base_dir, pkg, "rw.count")
            if os.path.exists(count):
                os.remove(count)
            count = os.path.join(base_dir, pkg, "rw.count.lock")
            if os.path.exists(count):
                os.remove(count)
            locks = os.path.join(base_dir, pkg, "locks")
            if os.path.exists(locks):
                shutil.rmtree(locks)
        except Exception as e:
            raise ConanException("Something went wrong while removing %s locks\n"
                                 "Error: %s\n"
                                 "Please clean your local conan cache manually"
                                 % (pkg, str(e)))
    out.warn("Migration: Removing old lock files finished\n")

def migrate_to_default_profile(conf_path, default_profile_path):
    tag = "[settings_defaults]"
    old_conf = load(conf_path)
    if tag not in old_conf:
        return
    tmp = old_conf.find(tag)
    new_conf = old_conf[0:tmp]
    rest = old_conf[tmp + len(tag):]
    if tmp:
        if "]" in rest:  # More sections after the settings_defaults
            new_conf += rest[rest.find("["):]
            save(conf_path, new_conf)
            settings = rest[:rest.find("[")].strip()
        else:
            save(conf_path, new_conf)
            settings = rest.strip()
        # Now generate the default profile from the read settings_defaults
        new_profile = "[settings]\n%s" % settings
        save(default_profile_path, new_profile)

def migrate_c_src_export_source(cache, out):
    package_folders = list_folder_subdirs(cache.store, 4)
    for package in package_folders:
        package_folder = os.path.join(cache.store, package)
        c_src = os.path.join(package_folder, "export/%s" % EXPORT_SOURCES_DIR_OLD)
        if os.path.exists(c_src):
            out.warn("Migration: Removing package with old export_sources layout: %s" % package)
            try:
                shutil.rmtree(package_folder)
            except Exception:
                out.warn("Migration: Can't remove the '%s' directory, "
                         "remove it manually" % package_folder)

def migrate_plugins_to_hooks(cache, output=None):
    plugins_path = os.path.join(cache.conan_folder, "plugins")
    if os.path.exists(plugins_path) and not os.path.exists(cache.hooks_path):
        os.rename(plugins_path, cache.hooks_path)
    conf_path = cache.conan_conf_path
    replace_in_file(conf_path, "[plugins]", "[hooks]", strict=False, output=output)
<commit_msg>fix migration (#4665)<commit_after>import os
import shutil

from conans import DEFAULT_REVISION_V1
from conans.client.cache.cache import CONAN_CONF, PROFILES_FOLDER
from conans.client.tools import replace_in_file
from conans.errors import ConanException
from conans.migrations import Migrator
from conans.model.manifest import FileTreeManifest
from conans.model.package_metadata import PackageMetadata
from conans.model.ref import ConanFileReference, PackageReference
from conans.model.version import Version
from conans.paths import EXPORT_SOURCES_DIR_OLD
from conans.paths import PACKAGE_METADATA
from conans.paths.package_layouts.package_cache_layout import PackageCacheLayout
from conans.util.files import list_folder_subdirs, load, save

class ClientMigrator(Migrator):

    def __init__(self, cache, current_version, out):
        self.cache = cache
        super(ClientMigrator, self).__init__(cache.conan_folder, cache.store,
                                             current_version, out)

    def _update_settings_yml(self, old_settings):
        from conans.client.conf import default_settings_yml
        settings_path = self.cache.settings_path
        if not os.path.exists(settings_path):
            self.out.warn("Migration: This conan installation doesn't have settings yet")
            self.out.warn("Nothing to migrate here, settings will be generated automatically")
            return

        current_settings = load(self.cache.settings_path)
        if current_settings != default_settings_yml:
            self.out.warn("Migration: Updating settings.yml")
            if current_settings != old_settings:
                new_path = self.cache.settings_path + ".new"
                save(new_path, default_settings_yml)
                self.out.warn("*" * 40)
                self.out.warn("settings.yml is locally modified, can't be updated")
                self.out.warn("The new settings.yml has been stored in: %s" % new_path)
                self.out.warn("*" * 40)
            else:
                save(self.cache.settings_path, default_settings_yml)

    def _make_migrations(self, old_version):
        # ############### FILL THIS METHOD WITH THE REQUIRED ACTIONS ##############
        # VERSION 0.1
        if old_version is None:
            return
        if old_version < Version("1.13.0"):
            old_settings = """
# Only for cross building, 'os_build/arch_build' is the system that runs Conan
os_build: [Windows, WindowsStore, Linux, Macos, FreeBSD, SunOS]
arch_build: [x86, x86_64, ppc32, ppc64le, ppc64, armv6, armv7, armv7hf, armv7s, armv7k, armv8, armv8_32, armv8.3, sparc, sparcv9, mips, mips64, avr]

# Only for building cross compilation tools, 'os_target/arch_target' is the system for
# which the tools generate code
os_target: [Windows, Linux, Macos, Android, iOS, watchOS, tvOS, FreeBSD, SunOS, Arduino]
arch_target: [x86, x86_64, ppc32, ppc64le, ppc64, armv6, armv7, armv7hf, armv7s, armv7k, armv8, armv8_32, armv8.3, sparc, sparcv9, mips, mips64, avr]

# Rest of the settings are "host" settings:
# - For native building/cross building: Where the library/program will run.
# - For building cross compilation tools: Where the cross compiler will run.
os:
    Windows:
        subsystem: [None, cygwin, msys, msys2, wsl]
    WindowsStore:
        version: ["8.1", "10.0"]
    Linux:
    Macos:
        version: [None, "10.6", "10.7", "10.8", "10.9", "10.10", "10.11", "10.12", "10.13", "10.14"]
    Android:
        api_level: ANY
    iOS:
        version: ["7.0", "7.1", "8.0", "8.1", "8.2", "8.3", "9.0", "9.1", "9.2", "9.3", "10.0", "10.1", "10.2", "10.3", "11.0", "11.1", "11.2", "11.3", "11.4", "12.0", "12.1"]
    watchOS:
        version: ["4.0", "4.1", "4.2", "4.3", "5.0", "5.1"]
    tvOS:
        version: ["11.0", "11.1", "11.2", "11.3", "11.4", "12.0", "12.1"]
    FreeBSD:
    SunOS:
    Arduino:
        board: ANY
arch: [x86, x86_64, ppc32, ppc64le, ppc64, armv6, armv7, armv7hf, armv7s, armv7k, armv8, armv8_32, armv8.3, sparc, sparcv9, mips, mips64, avr]
compiler:
    sun-cc:
        version: ["5.10", "5.11", "5.12", "5.13", "5.14"]
        threads: [None, posix]
        libcxx: [libCstd, libstdcxx, libstlport, libstdc++]
    gcc:
        version: ["4.1", "4.4", "4.5", "4.6", "4.7", "4.8", "4.9",
                  "5", "5.1", "5.2", "5.3", "5.4", "5.5",
                  "6", "6.1", "6.2", "6.3", "6.4",
                  "7", "7.1", "7.2", "7.3",
                  "8", "8.1", "8.2"]
        libcxx: [libstdc++, libstdc++11]
        threads: [None, posix, win32] #  Windows MinGW
        exception: [None, dwarf2, sjlj, seh] # Windows MinGW
    Visual Studio:
        runtime: [MD, MT, MTd, MDd]
        version: ["8", "9", "10", "11", "12", "14", "15"]
        toolset: [None, v90, v100, v110, v110_xp, v120, v120_xp,
                  v140, v140_xp, v140_clang_c2, LLVM-vs2012, LLVM-vs2012_xp,
                  LLVM-vs2013, LLVM-vs2013_xp, LLVM-vs2014, LLVM-vs2014_xp,
                  LLVM-vs2017, LLVM-vs2017_xp, v141, v141_xp, v141_clang_c2]
    clang:
        version: ["3.3", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "4.0",
                  "5.0", "6.0", "7.0",
                  "8"]
        libcxx: [libstdc++, libstdc++11, libc++]
    apple-clang:
        version: ["5.0", "5.1", "6.0", "6.1", "7.0", "7.3", "8.0", "8.1", "9.0", "9.1", "10.0"]
        libcxx: [libstdc++, libc++]

build_type: [None, Debug, Release, RelWithDebInfo, MinSizeRel]
cppstd: [None, 98, gnu98, 11, gnu11, 14, gnu14, 17, gnu17, 20, gnu20]
"""
            self._update_settings_yml(old_settings)

            # MIGRATE LOCAL CACHE TO GENERATE MISSING METADATA.json
            _migrate_create_metadata(self.cache, self.out)

        if old_version < Version("1.12.0"):
            migrate_plugins_to_hooks(self.cache)

        if old_version < Version("1.0"):
            _migrate_lock_files(self.cache, self.out)

        if old_version < Version("0.25"):
            from conans.paths import DEFAULT_PROFILE_NAME
            default_profile_path = os.path.join(self.cache.conan_folder, PROFILES_FOLDER,
                                                DEFAULT_PROFILE_NAME)
            if not os.path.exists(default_profile_path):
                self.out.warn("Migration: Moving default settings from %s file to %s"
                              % (CONAN_CONF, DEFAULT_PROFILE_NAME))
                conf_path = os.path.join(self.cache.conan_folder, CONAN_CONF)

                migrate_to_default_profile(conf_path, default_profile_path)

                self.out.warn("Migration: export_source cache new layout")
                migrate_c_src_export_source(self.cache, self.out)

def _get_refs(cache):
    folders = list_folder_subdirs(cache.store, 4)
    return [ConanFileReference(*s.split("/")) for s in folders]

def _get_prefs(layout):
    packages_folder = layout.packages()
    folders = list_folder_subdirs(packages_folder, 1)
    return [PackageReference(layout.ref, s) for s in folders]

def _migrate_create_metadata(cache, out):
    out.warn("Migration: Generating missing metadata files")
    refs = _get_refs(cache)

    for ref in refs:
        try:
            base_folder = os.path.normpath(os.path.join(cache.store, ref.dir_repr()))
            # Force using a package cache layout for everything, we want to alter the cache,
            # not the editables
            layout = PackageCacheLayout(base_folder=base_folder, ref=ref, short_paths=False,
                                        no_lock=True)
            folder = layout.export()
            try:
                manifest = FileTreeManifest.load(folder)
                rrev = manifest.summary_hash
            except:
                rrev = DEFAULT_REVISION_V1
            metadata_path = os.path.join(layout.conan(), PACKAGE_METADATA)
            if not os.path.exists(metadata_path):
                out.info("Creating {} for {}".format(PACKAGE_METADATA, ref))
                prefs = _get_prefs(layout)
                metadata = PackageMetadata()
                metadata.recipe.revision = rrev
                for pref in prefs:
                    try:
                        pmanifest = FileTreeManifest.load(layout.package(pref))
                        prev = pmanifest.summary_hash
                    except:
                        prev = DEFAULT_REVISION_V1
                    metadata.packages[pref.id].revision = prev
                    metadata.packages[pref.id].recipe_revision = metadata.recipe.revision
                save(metadata_path, metadata.dumps())
        except Exception as e:
            raise ConanException("Something went wrong while generating the metadata.json files "
                                 "in the cache, please try to fix the issue or wipe the cache: {}"
                                 ":{}".format(ref, e))
    out.success("Migration: Generating missing metadata files finished OK!\n")

def _migrate_lock_files(cache, out):
    out.warn("Migration: Removing old lock files")
    base_dir = cache.store
    pkgs = list_folder_subdirs(base_dir, 4)
    for pkg in pkgs:
        out.info("Removing locks for %s" % pkg)
        try:
            count = os.path.join(base_dir, pkg, "rw.count")
            if os.path.exists(count):
                os.remove(count)
            count = os.path.join(base_dir, pkg, "rw.count.lock")
            if os.path.exists(count):
                os.remove(count)
            locks = os.path.join(base_dir, pkg, "locks")
            if os.path.exists(locks):
                shutil.rmtree(locks)
        except Exception as e:
            raise ConanException("Something went wrong while removing %s locks\n"
                                 "Error: %s\n"
                                 "Please clean your local conan cache manually"
                                 % (pkg, str(e)))
    out.warn("Migration: Removing old lock files finished\n")

def migrate_to_default_profile(conf_path, default_profile_path):
    tag = "[settings_defaults]"
    old_conf = load(conf_path)
    if tag not in old_conf:
        return
    tmp = old_conf.find(tag)
    new_conf = old_conf[0:tmp]
    rest = old_conf[tmp + len(tag):]
    if tmp:
        if "]" in rest:  # More sections after the settings_defaults
            new_conf += rest[rest.find("["):]
            save(conf_path, new_conf)
            settings = rest[:rest.find("[")].strip()
        else:
            save(conf_path, new_conf)
            settings = rest.strip()
        # Now generate the default profile from the read settings_defaults
        new_profile = "[settings]\n%s" % settings
        save(default_profile_path, new_profile)

def migrate_c_src_export_source(cache, out):
    package_folders = list_folder_subdirs(cache.store, 4)
    for package in package_folders:
        package_folder = os.path.join(cache.store, package)
        c_src = os.path.join(package_folder, "export/%s" % EXPORT_SOURCES_DIR_OLD)
        if os.path.exists(c_src):
            out.warn("Migration: Removing package with old export_sources layout: %s" % package)
            try:
                shutil.rmtree(package_folder)
            except Exception:
                out.warn("Migration: Can't remove the '%s' directory, "
                         "remove it manually" % package_folder)

def migrate_plugins_to_hooks(cache, output=None):
    plugins_path = os.path.join(cache.conan_folder, "plugins")
    if os.path.exists(plugins_path) and not os.path.exists(cache.hooks_path):
        os.rename(plugins_path, cache.hooks_path)
    conf_path = cache.conan_conf_path
    replace_in_file(conf_path, "[plugins]", "[hooks]", strict=False, output=output)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2016-2018 Akretion France
# @author: Alexis de Lattre <alexis.delattre@akretion.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

from odoo.tests.common import TransactionCase
import time

class TestDonationRecurring(TransactionCase):

    at_install = False
    post_install = True

    def setUp(self):
        super(TestDonationRecurring, self).setUp()
        self.test_company = self.env['res.company'].create({
            'name': 'TestCharity',
            })
        self.env.user.write({
            'company_id': self.test_company.id,
            'company_ids': [(4, self.test_company.id)],
            })

        generic_coa = self.env.ref(
            'l10n_generic_coa.configurable_chart_template')
        generic_coa.try_loading_for_current_company()
        # Warning: loading coa chart of accounts writes
        # USD as currency of company (even if company was created with
        # EUR as currency. So let's work on USD for the tests

        self.currency = self.test_company.currency_id
        self.bank_journal = self.env['account.journal'].search([
            ('company_id', '=', self.test_company.id),
            ('type', '=', 'bank')], limit=1)
        self.product = self.env.ref(
            'donation_base.product_product_donation')
        self.inkind_product = self.env.ref(
            'donation_base.product_product_inkind_donation')
        self.ddo = self.env['donation.donation']
        self.don_rec1 = self.ddo.create({
            'company_id': self.test_company.id,
            'currency_id': self.currency.id,
            'check_total': 30,
            'partner_id': self.env.ref('donation_recurring.donor_rec1').id,
            'donation_date': time.strftime('%Y-01-01'),
            'journal_id': self.bank_journal.id,
            'tax_receipt_option': 'annual',
            'recurring_template': 'active',
            'line_ids': [(0, 0, {
                'product_id': self.product.id,
                'quantity': 1,
                'unit_price': 30,
                })],
            })
        self.don_rec2 = self.ddo.create({
            'company_id': self.test_company.id,
            'currency_id': self.currency.id,
            'check_total': 25,
            'partner_id': self.env.ref('donation_recurring.donor_rec2').id,
            'donation_date': time.strftime('%Y-01-01'),
            'journal_id': self.bank_journal.id,
            'tax_receipt_option': 'annual',
            'recurring_template': 'active',
            'line_ids': [(0, 0, {
                'product_id': self.product.id,
                'quantity': 1,
                'unit_price': 25,
                })],
            })
        self.don_rec3 = self.ddo.create({
            'company_id': self.test_company.id,
            'currency_id': self.currency.id,
            'check_total': 35,
            'partner_id': self.env.ref('donation_recurring.donor_rec3').id,
            'donation_date': time.strftime('%Y-01-01'),
            'journal_id': self.bank_journal.id,
            'tax_receipt_option': 'annual',
            'recurring_template': 'suspended',
            'line_ids': [(0, 0, {
                'product_id': self.product.id,
                'quantity': 1,
                'unit_price': 35,
                })],
            })

    def test_donation_recurring(self):
        wizard = self.env['donation.recurring.generate'].create(
            {'payment_ref': 'Don Abbaye Sainte Madeleine'})
        action = wizard.generate()
        active_don_recs = self.don_rec1 + self.don_rec2
        for don_rec in active_don_recs:
            self.assertTrue(don_rec.recurring_donation_ids)
            don = don_rec.recurring_donation_ids[0]
            self.assertEquals(don.state, 'draft')
            self.assertEquals(don.amount_total, don_rec.amount_total)
            self.assertEquals(
                don.payment_ref,
                'Don Abbaye Sainte Madeleine')
            self.assertEquals(don.campaign_id, don_rec.campaign_id)
        self.assertFalse(self.don_rec3.recurring_donation_ids)
        active_ids = action['domain'][0][2]
        wizard_val = self.env['donation.validate'].with_context(
            active_ids=active_ids, active_model='donation.donation').\
            create({})
        wizard_val.run()
        for don_rec in active_don_recs:
            don = don_rec.recurring_donation_ids[0]
            self.assertEquals(don.state, 'done')
            self.assertEquals(don.move_id.journal_id, don_rec.journal_id)
            self.assertEquals(don.move_id.state, 'posted')
            self.assertEquals(don.move_id.ref, 'Don Abbaye Sainte Madeleine')
<commit_msg>New implementation for tests using account_minimal_test.xml<commit_after># -*- coding: utf-8 -*-
# Copyright 2016-2018 Akretion France
# @author: Alexis de Lattre <alexis.delattre@akretion.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

from odoo.tests.common import TransactionCase
from odoo import tools
from odoo.modules.module import get_resource_path
import time

class TestDonationRecurring(TransactionCase):

    at_install = False
    post_install = True

    def _load(self, module, *args):
        tools.convert_file(
            self.cr, module, get_resource_path(module, *args),
            {}, 'init', False, 'test', self.registry._assertion_report)

    def setUp(self):
        super(TestDonationRecurring, self).setUp()
        self._load('account', 'test', 'account_minimal_test.xml')

        self.bank_journal = self.env.ref('account.bank_journal')
        self.product = self.env.ref(
            'donation_base.product_product_donation')
        self.inkind_product = self.env.ref(
            'donation_base.product_product_inkind_donation')
        self.ddo = self.env['donation.donation']
        self.don_rec1 = self.ddo.create({
            'check_total': 30,
            'partner_id': self.env.ref('donation_recurring.donor_rec1').id,
            'donation_date': time.strftime('%Y-01-01'),
            'journal_id': self.bank_journal.id,
            'tax_receipt_option': 'annual',
            'recurring_template': 'active',
            'line_ids': [(0, 0, {
                'product_id': self.product.id,
                'quantity': 1,
                'unit_price': 30,
                })],
            })
        self.don_rec2 = self.ddo.create({
            'check_total': 25,
            'partner_id': self.env.ref('donation_recurring.donor_rec2').id,
            'donation_date': time.strftime('%Y-01-01'),
            'journal_id': self.bank_journal.id,
            'tax_receipt_option': 'annual',
            'recurring_template': 'active',
            'line_ids': [(0, 0, {
                'product_id': self.product.id,
                'quantity': 1,
                'unit_price': 25,
                })],
            })
        self.don_rec3 = self.ddo.create({
            'check_total': 35,
            'partner_id': self.env.ref('donation_recurring.donor_rec3').id,
            'donation_date': time.strftime('%Y-01-01'),
            'journal_id': self.bank_journal.id,
            'tax_receipt_option': 'annual',
            'recurring_template': 'suspended',
            'line_ids': [(0, 0, {
                'product_id': self.product.id,
                'quantity': 1,
                'unit_price': 35,
                })],
            })

    def test_donation_recurring(self):
        wizard = self.env['donation.recurring.generate'].create(
            {'payment_ref': 'Don Abbaye Sainte Madeleine'})
        action = wizard.generate()
        active_don_recs = self.don_rec1 + self.don_rec2
        for don_rec in active_don_recs:
            self.assertTrue(don_rec.recurring_donation_ids)
            don = don_rec.recurring_donation_ids[0]
            self.assertEquals(don.state, 'draft')
            self.assertEquals(don.amount_total, don_rec.amount_total)
            self.assertEquals(
                don.payment_ref,
                'Don Abbaye Sainte Madeleine')
            self.assertEquals(don.campaign_id, don_rec.campaign_id)
        self.assertFalse(self.don_rec3.recurring_donation_ids)
        active_ids = action['domain'][0][2]
        wizard_val = self.env['donation.validate'].with_context(
            active_ids=active_ids, active_model='donation.donation').\
            create({})
        wizard_val.run()
        for don_rec in active_don_recs:
            don = don_rec.recurring_donation_ids[0]
            self.assertEquals(don.state, 'done')
            self.assertEquals(don.move_id.journal_id, don_rec.journal_id)
            self.assertEquals(don.move_id.state, 'posted')
            self.assertEquals(don.move_id.ref, 'Don Abbaye Sainte Madeleine')
<|endoftext|>


<commit_before>import asyncio

import discord
from discord.ext import commands

class Fun:
    """
    Fun and useful stuff
    """

    def __init__(self, bot):
        self.bot = bot

    @commands.command(pass_context=True)
    async def marco(self, ctx):
        """
        Says "polo"
        """
        await self.bot.say(self.bot.msg_prefix + "polo")

    @commands.command(pass_context=True)
    async def soon(self, ctx, *, message: str = ""):
        """
        Makes a soon tm
        """
        await self.bot.delete_message(ctx.message)
        await self.bot.say("soon\u2122" + message)

    @commands.command(pass_context=True)
    async def give(self, ctx, *, message: str = ""):
        """
        Gives stuff
        """
        await self.bot.delete_message(ctx.message)
        await self.bot.say("༼ つ ◕\\_◕ ༽つ " + message + " ༼ つ ◕\\_◕ ༽つ")

    @commands.command(pass_context=True)
    async def shrug(self, ctx, *, message: str = ""):
        """
        Makes a shrug
        """
        await self.bot.delete_message(ctx.message)
        await self.bot.say("¯\_(ツ)_/¯ " + message)

    @commands.command(pass_context=True)
    async def lenny(self, ctx, *, message: str = ""):
        """
        Makes a lenny face
        """
        await self.bot.delete_message(ctx.message)
        await self.bot.say("( ͡° ͜ʖ ͡°) " + message)

    @commands.command(pass_context=True, aliases=["d"])
    async def justdeleteme(self, ctx, count: int):
        """
        Deletes 'count' number of message you have sent in the channel

        But only if they are in the first 1000 messages
        """
        count += 1
        iterator = self.bot.logs_from(ctx.message.channel, limit=1000)
        async for m in iterator:
            if isinstance(m, discord.Message):
                if m.author == self.bot.user:
                    await self.bot.delete_message(m)
                    count -= 1
                    if count <= 0:
                        return

    async def ask(self, question: str):
        to_del = await self.bot.say(self.bot.msg_prefix + question)
        await asyncio.sleep(0.5)
        response = await self.bot.wait_for_message(author=self.bot.user)
        string = response.content
        await self.bot.delete_message(to_del)
        await self.bot.delete_message(response)
        return string

    @commands.command(pass_context=True)
    async def nukemeplease(self, ctx):
        """
        Deletes all your messages from this channel
        """
        string = await self.ask("This will delete all your messages from this channel. Are you sure you want to continue? Say ´yes´ to continue...")
        if not string.lower().startswith("y"):
            to_del = self.bot.say(self.bot.msg_prefix + "Oki... Aborting...")
            await asyncio.sleep(5)
            await self.bot.delete_message(to_del)
            return

        date = None
        iterator = self.bot.logs_from(ctx.message.channel, limit=5000)
        while True:
            count = 0
            async for m in iterator:
                count += 1
                if isinstance(m, discord.Message):
                    date = m.timestamp
                    if m.author == self.bot.user:
                        await self.bot.delete_message(m)
            if count == 0:
                break
            iterator = self.bot.logs_from(ctx.message.channel, limit=5000, before=date)

    @commands.command(pass_context=True, hidden=True)
    async def whois(self, ctx, *, ingnore: str = ""):
        """
        Let's just ingore that
        """
        to_del = await self.bot.say(self.bot.msg_prefix + "Use debug...")
        await asyncio.sleep(5)
        await self.bot.delete_message(to_del)

    def _calculate_mutual_servers(self, member: discord.Member):
        # Calculates mutual servers.
        serverlist = []
        for server in self.bot.servers:
            assert isinstance(server, discord.Server)
            if server.get_member(member.id):
                serverlist += [server.name]
        return serverlist

    def _safe_roles(self, roles: list):
        names = []
        for role in roles:
            if role.name == "@everyone":
                names.append("@\u200beveryone")  # u200b is invisible space
            else:
                names.append(role.name)

        return names

def setup(bot):
    bot.add_cog(Fun(bot))
<commit_msg>Await this mf too<commit_after>import asyncio

import discord
from discord.ext import commands

class Fun:
    """
    Fun and useful stuff
    """

    def __init__(self, bot):
        self.bot = bot

    @commands.command(pass_context=True)
    async def marco(self, ctx):
        """
        Says "polo"
        """
        await self.bot.say(self.bot.msg_prefix + "polo")

    @commands.command(pass_context=True)
    async def soon(self, ctx, *, message: str = ""):
        """
        Makes a soon tm
        """
        await self.bot.delete_message(ctx.message)
        await self.bot.say("soon\u2122" + message)

    @commands.command(pass_context=True)
    async def give(self, ctx, *, message: str = ""):
        """
        Gives stuff
        """
        await self.bot.delete_message(ctx.message)
        await self.bot.say("༼ つ ◕\\_◕ ༽つ " + message + " ༼ つ ◕\\_◕ ༽つ")

    @commands.command(pass_context=True)
    async def shrug(self, ctx, *, message: str = ""):
        """
        Makes a shrug
        """
        await self.bot.delete_message(ctx.message)
        await self.bot.say("¯\_(ツ)_/¯ " + message)

    @commands.command(pass_context=True)
    async def lenny(self, ctx, *, message: str = ""):
        """
        Makes a lenny face
        """
        await self.bot.delete_message(ctx.message)
        await self.bot.say("( ͡° ͜ʖ ͡°) " + message)

    @commands.command(pass_context=True, aliases=["d"])
    async def justdeleteme(self, ctx, count: int):
        """
        Deletes 'count' number of message you have sent in the channel

        But only if they are in the first 1000 messages
        """
        count += 1
        iterator = self.bot.logs_from(ctx.message.channel, limit=1000)
        async for m in iterator:
            if isinstance(m, discord.Message):
                if m.author == self.bot.user:
                    await self.bot.delete_message(m)
                    count -= 1
                    if count <= 0:
                        return

    async def ask(self, question: str):
        to_del = await self.bot.say(self.bot.msg_prefix + question)
        await asyncio.sleep(0.5)
        response = await self.bot.wait_for_message(author=self.bot.user)
        string = response.content
        await self.bot.delete_message(to_del)
        await self.bot.delete_message(response)
        return string

    @commands.command(pass_context=True)
    async def nukemeplease(self, ctx):
        """
        Deletes all your messages from this channel
        """
        string = await self.ask("This will delete all your messages from this channel. Are you sure you want to continue? Say ´´yes´´ to continue...")
        if not string.lower().startswith("y"):
            to_del = await self.bot.say(self.bot.msg_prefix + "Oki... Aborting...")
            await asyncio.sleep(5)
            await self.bot.delete_message(to_del)
            return

        date = None
        iterator = self.bot.logs_from(ctx.message.channel, limit=5000)
        while True:
            count = 0
            async for m in iterator:
                count += 1
                if isinstance(m, discord.Message):
                    date = m.timestamp
                    if m.author == self.bot.user:
                        await self.bot.delete_message(m)
                        await asyncio.sleep(0.5)
            if count == 0:
                break
            iterator = self.bot.logs_from(ctx.message.channel, limit=5000, before=date)

    @commands.command(pass_context=True, hidden=True)
    async def whois(self, ctx, *, ingnore: str = ""):
        """
        Let's just ingore that
        """
        to_del = await self.bot.say(self.bot.msg_prefix + "Use debug...")
        await asyncio.sleep(5)
        await self.bot.delete_message(to_del)

    def _calculate_mutual_servers(self, member: discord.Member):
        # Calculates mutual servers.
        serverlist = []
        for server in self.bot.servers:
            assert isinstance(server, discord.Server)
            if server.get_member(member.id):
                serverlist += [server.name]
        return serverlist

    def _safe_roles(self, roles: list):
        names = []
        for role in roles:
            if role.name == "@everyone":
                names.append("@\u200beveryone")  # u200b is invisible space
            else:
                names.append(role.name)

        return names

def setup(bot):
    bot.add_cog(Fun(bot))
<|endoftext|>


<commit_before>from lxml import etree

from xmodule.x_module import XModule
from xmodule.raw_module import RawDescriptor

import comment_client

import json

class DiscussionModule(XModule):
    def get_html(self):
        context = {
            'discussion_id': self.discussion_id,
        }
        return self.system.render_template('discussion/_discussion_module.html', context)

    def __init__(self, system, location, definition, descriptor, instance_state=None, shared_state=None, **kwargs):
        XModule.__init__(self, system, location, definition, descriptor, instance_state, shared_state, **kwargs)

        if isinstance(instance_state, str):
            instance_state = json.loads(instance_state)
        xml_data = etree.fromstring(definition['data'])
        self.discussion_id = xml_data.attrib['id']
        self.title = xml_data.attrib['for']
        self.discussion_category = xml_data.attrib['discussion_category']

class DiscussionDescriptor(RawDescriptor):
    module_class = DiscussionModule
<commit_msg>removed unnecessary dependency<commit_after>from lxml import etree

from xmodule.x_module import XModule
from xmodule.raw_module import RawDescriptor

import json

class DiscussionModule(XModule):
    def get_html(self):
        context = {
            'discussion_id': self.discussion_id,
        }
        return self.system.render_template('discussion/_discussion_module.html', context)

    def __init__(self, system, location, definition, descriptor, instance_state=None, shared_state=None, **kwargs):
        XModule.__init__(self, system, location, definition, descriptor, instance_state, shared_state, **kwargs)

        if isinstance(instance_state, str):
            instance_state = json.loads(instance_state)
        xml_data = etree.fromstring(definition['data'])
        self.discussion_id = xml_data.attrib['id']
        self.title = xml_data.attrib['for']
        self.discussion_category = xml_data.attrib['discussion_category']

class DiscussionDescriptor(RawDescriptor):
    module_class = DiscussionModule
<|endoftext|>


<commit_before>import asyncio
import logging
from typing import Union

import discord
from discord.ext import commands

from cogs.utils import converter
from nabbot import NabBot
from .utils import checks, config, safe_delete_message
from .utils.context import NabCtx
from .utils.database import get_server_property
from .utils.errors import CannotPaginate
from .utils.pages import Pages

log = logging.getLogger("nabbot")

class LazyEntry:
    __slots__ = ('entity_id', 'guild', '_cache')

    def __init__(self, guild, entity_id):
        self.entity_id = entity_id
        self.guild = guild
        self._cache = None

    def __str__(self):
        if self._cache:
            return self._cache

        e = self.entity_id
        g = self.guild
        resolved = g.get_channel(e) or g.get_member(e)
        if resolved is None:
            self._cache = f'<Not Found: {e}>'
        else:
            self._cache = resolved.mention
        return self._cache

class Mod:
    """Moderating related commands."""
    def __init__(self, bot: NabBot):
        self.bot = bot

    async def __global_check_once(self, ctx: NabCtx):
        if ctx.guild is None:
            return True
        if await checks.is_owner(ctx):
            return True

        return await self.is_ignored(ctx.pool, ctx)

    # region Commands
    @commands.guild_only()
    @checks.channel_mod_only()
    @commands.command()
    async def cleanup(self, ctx: NabCtx, limit: int = 50):
        """Cleans the channel from bot commands.

        If the bot has `Manage Messages` permission, it will also delete command invocation messages."""
        count = 0
        prefixes = await get_server_property(ctx.pool, ctx.guild.id, "prefixes", default=config.command_prefix)
        # Also skip death and level up messages from cleanup
        announce_prefix = (config.levelup_emoji, config.death_emoji, config.pvpdeath_emoji)
        if ctx.bot_permissions.manage_messages:
            def check(m: discord.Message):
                return (m.author == ctx.me and not m.content.startswith(announce_prefix)) or \
                       m.content.startswith(tuple(prefixes))

            deleted = await ctx.channel.purge(limit=limit, check=check)
            count = len(deleted)
        else:
            with ctx.typing():
                async for msg in ctx.channel.history(limit=limit):
                    if msg.author == ctx.me:
                        await msg.delete()
                        count += 1
        if not count:
            return await ctx.send("There are no messages to clean.", delete_after=10)

        await ctx.success(f"Deleted {count:,} messages.", delete_after=20)

    @checks.server_mod_only()
    @commands.group(invoke_without_command=True, case_insensitive=True)
    async def ignore(self, ctx: NabCtx, *entries: converter.ChannelOrMember):
        """Makes the bot ignore a channel or user.

        Commands cannot be used in ignored channels or by ignored users.

        The command accepts a list of names, ids or mentions of users or channels.
        If the command is used with no parameters, it ignores the current channel.

        Ignores are bypassed by users with the `Manage Server` permission."""
        if len(entries) == 0:
            entries = [ctx.channel]
        if len(entries) == 1:
            entry: Union[discord.Member, discord.TextChannel] = entries[0]
            query = "INSERT INTO ignored_entry(server_id, entry_id) VALUES($1, $2) ON CONFLICT DO NOTHING RETURNING 1"
            ret = await ctx.pool.fetchval(query, ctx.guild.id, entry.id)
            rep = entry.mention if isinstance(entry, discord.TextChannel) else entry.display_name
            if ret:
                return await ctx.success(f"I'm now ignoring **{rep}**")
            return await ctx.error(f"{rep} is already ignored.")

        inserted = await self.bulk_ignore(ctx, entries)
        if inserted:
            if inserted != len(entries):
                await ctx.success(f"{inserted} entries are now ignored. The rest was already ignored.")
            else:
                await ctx.success(f"All {inserted} entries are now ignored.")
        else:
            await ctx.error("No entries were ignored. They were all already ignored.")

    @checks.server_mod_only()
    @ignore.command(name="list")
    async def ignore_list(self, ctx: NabCtx):
        """Shows a list of ignored channels and users."""
        query = "SELECT entry_id FROM ignored_entry WHERE server_id = $1"
        rows = await ctx.pool.fetch(query, ctx.guild.id)

        entries = [LazyEntry(ctx.guild, e[0]) for e in rows]
        if not entries:
            await ctx.send("There are no ignored entries in this server.")
            return
        pages = Pages(ctx, entries=entries)
        pages.embed.title = "Ignored entries"
        try:
            await pages.paginate()
        except CannotPaginate as e:
            await ctx.send(e)

    @commands.command()
    @checks.channel_mod_somewhere()
    async def makesay(self, ctx: NabCtx, *, message: str):
        """Makes the bot say a message.

        If it's used directly on a text channel, the bot will delete the command's message and repeat it itself.
        Note that deleting the message requires `Manage Messages` permissions in the channel.

        If it's used on a private message, the bot will ask on which channel he should say the message.
        Each channel in the list is numerated, by choosing a number, the message will be sent in the chosen channel.
        You can only send messages on channels where you have `Manage Channel` permissions.
        """
        if ctx.is_private:
            description_list = []
            channel_list = []
            prev_server = None
            num = 1
            for server in self.bot.guilds:
                author = server.get_member(ctx.author.id)
                bot_member = self.bot.get_member(self.bot.user.id, server)
                # Skip servers where the command user is not in
                if author is None:
                    continue
                # Check for every channel
                for channel in server.text_channels:
                    author_permissions = author.permissions_in(channel)
                    bot_permissions = bot_member.permissions_in(channel)
                    # Check if both the author and the bot have permissions to send messages and add channel to list
                    if author_permissions.send_messages and bot_permissions.send_messages \
                            and author_permissions.manage_channels:
                        separator = ""
                        if prev_server is not server:
                            separator = "---------------\n\t"
                        description_list.append("{2}{3}: **#{0}** in **{1}**".format(channel.name, server.name,
                                                                                separator, num))
                        channel_list.append(channel)
                        prev_server = server
                        num += 1
            if len(description_list) < 1:
                await ctx.send("We don't have any channels where we both can send messages and that you manage.")
                return
            await ctx.send("Choose a channel for me to send your message (number only): \n\t0: *Cancel*\n\t" +
                           "\n\t".join(["{0}".format(i) for i in description_list]))

            def check(m):
                return m.author == ctx.author and m.channel == ctx.channel
            try:
                answer = await self.bot.wait_for("message", timeout=60.0, check=check)
                answer = int(answer.content)
                if answer == 0:
                    await ctx.send("Changed your mind? Typical human.")
                    return
                await channel_list[answer-1].send(message)
                await ctx.send("Message sent on {0} ({1})".format(channel_list[answer-1].mention,
                                                                  channel_list[answer-1].guild))
            except IndexError:
                await ctx.send("That wasn't in the choices, you ruined it. Start from the beginning.")
            except ValueError:
                await ctx.send("That's not a valid answer!")
            except asyncio.TimeoutError:
                await ctx.send("... are you there? Fine, nevermind!")
        else:
            if not ctx.bot_permissions.manage_messages:
                return await ctx.send(f"{ctx.tick(False)} I need `Manage Messages` permission to use this command.")
            if not ctx.author_permissions.manage_channels:
                return await ctx.send(f"{ctx.tick(False)} You need `Manage Channel` permission to use this command.")
            await safe_delete_message(ctx.message)
            await ctx.message.delete()
            await ctx.channel.send(message)

    @commands.guild_only()
    @checks.channel_mod_only()
    @commands.command()
    async def unignore(self, ctx: NabCtx, *entries: converter.ChannelOrMember):
        """Removes a channel or user from the ignored list.

        If no parameter is provided, the current channel will be unignored.

        If the command is used with no parameters, it unignores the current channel."""
        if len(entries) == 0:
            query = "DELETE FROM ignored_entry WHERE server_id=$1 AND entry_id=$2 RETURNING true"
            res = await ctx.pool.fetchval(query, ctx.guild.id, ctx.channel.id)
            if res:
                return await ctx.success(f"{ctx.channel.mention} is no longer ignored.")
            return await ctx.error(f"{ctx.channel.mention} wasn't ignored.")
        query = "DELETE FROM ignored_entry WHERE server_id=$1 AND entry_id = ANY($2::bigint[]) RETURNING entry_id"
        # noinspection PyUnresolvedReferences
        entries = [c.id for c in entries]
        rows = await ctx.pool.fetch(query, ctx.guild.id, entries)
        if rows:
            return await ctx.success(f"{len(rows)} are now unignored.")
        await ctx.error("No entries were unignored.")

    @checks.channel_mod_only()
    @checks.tracking_world_only()
    @commands.command()
    async def unregistered(self, ctx: NabCtx):
        """Shows a list of users with no registered characters."""
        entries = []
        if ctx.world is None:
            await ctx.send("This server is not tracking any worlds.")
            return

        results = await ctx.pool.fetch('SELECT user_id FROM "character" WHERE world = $1 GROUP BY user_id', ctx.world)
        # Flatten list
        users = [i["user_id"] for i in results]
        for member in ctx.guild.members:  # type: discord.Member
            # Skip bots
            if member.bot:
                continue
            if member.id not in users:
                entries.append(f"@**{member.display_name}** \u2014 Joined: **{member.joined_at.date()}**")
        if len(entries) == 0:
            await ctx.send("There are no unregistered users.")
            return

        pages = Pages(ctx, entries=entries, per_page=10)
        pages.embed.title = "Unregistered members"
        try:
            await pages.paginate()
        except CannotPaginate as e:
            await ctx.send(e)
    # endregion

    @classmethod
    async def is_ignored(cls, conn, ctx: NabCtx):
        """Checks if the current context is ignored.

        A context could be ignored because either the channel or the user are in the ignored list."""
        query = "SELECT True FROM ingored_entry WHERE guild_id=$1 AND (entry_id=$2 OR entry_id=$3);"
        return await conn.fetchrow(query, ctx.guild.id, ctx.channel.id, ctx.author.id)

    @classmethod
    async def bulk_ignore(cls, ctx: NabCtx, entries):
        async with ctx.pool.acquire() as conn:
            async with conn.transaction():
                query = "SELECT entry_id FROM ignored_entry WHERE server_id=$1;"
                records = await conn.fetch(query, ctx.guild.id)

                # Removing duplicates
                current_entries = {r[0] for r in records}
                records = [(ctx.guild.id, e.id) for e in entries if e.id not in current_entries]

                # do a bulk COPY
                await conn.copy_records_to_table('ignored_entry', columns=['server_id', 'entry_id'], records=records)
                return len(records)

def setup(bot):
    bot.add_cog(Mod(bot))
<commit_msg>Fixed issues with ignore check<commit_after>import asyncio
import logging
from typing import Union

import discord
from discord.ext import commands

from cogs.utils import converter
from nabbot import NabBot
from .utils import checks, config, safe_delete_message
from .utils.context import NabCtx
from .utils.database import get_server_property
from .utils.errors import CannotPaginate
from .utils.pages import Pages

log = logging.getLogger("nabbot")

class LazyEntry:
    __slots__ = ('entity_id', 'guild', '_cache')

    def __init__(self, guild, entity_id):
        self.entity_id = entity_id
        self.guild = guild
        self._cache = None

    def __str__(self):
        if self._cache:
            return self._cache

        e = self.entity_id
        g = self.guild
        resolved = g.get_channel(e) or g.get_member(e)
        if resolved is None:
            self._cache = f'<Not Found: {e}>'
        else:
            self._cache = resolved.mention
        return self._cache

class Mod:
    """Moderating related commands."""
    def __init__(self, bot: NabBot):
        self.bot = bot

    async def __global_check_once(self, ctx: NabCtx):
        """Checks if the current channel or user is ignored.

        Bot owners and guild managers can bypass this.
        """
        if ctx.guild is None:
            return True
        if await checks.is_owner(ctx):
            return True
        if ctx.author_permissions.manage_guild:
            return True

        return not await self.is_ignored(ctx.pool, ctx)

    # region Commands
    @commands.guild_only()
    @checks.channel_mod_only()
    @commands.command()
    async def cleanup(self, ctx: NabCtx, limit: int = 50):
        """Cleans the channel from bot commands.

        If the bot has `Manage Messages` permission, it will also delete command invocation messages."""
        count = 0
        prefixes = await get_server_property(ctx.pool, ctx.guild.id, "prefixes", default=config.command_prefix)
        # Also skip death and level up messages from cleanup
        announce_prefix = (config.levelup_emoji, config.death_emoji, config.pvpdeath_emoji)
        if ctx.bot_permissions.manage_messages:
            def check(m: discord.Message):
                return (m.author == ctx.me and not m.content.startswith(announce_prefix)) or \
                       m.content.startswith(tuple(prefixes))

            deleted = await ctx.channel.purge(limit=limit, check=check)
            count = len(deleted)
        else:
            with ctx.typing():
                async for msg in ctx.channel.history(limit=limit):
                    if msg.author == ctx.me:
                        await msg.delete()
                        count += 1
        if not count:
            return await ctx.send("There are no messages to clean.", delete_after=10)

        await ctx.success(f"Deleted {count:,} messages.", delete_after=20)

    @checks.server_mod_only()
    @commands.group(invoke_without_command=True, case_insensitive=True)
    async def ignore(self, ctx: NabCtx, *entries: converter.ChannelOrMember):
        """Makes the bot ignore a channel or user.

        Commands cannot be used in ignored channels or by ignored users.

        The command accepts a list of names, ids or mentions of users or channels.
        If the command is used with no parameters, it ignores the current channel.

        Ignores are bypassed by users with the `Manage Server` permission."""
        if len(entries) == 0:
            entries = [ctx.channel]
        if len(entries) == 1:
            entry: Union[discord.Member, discord.TextChannel] = entries[0]
            query = "INSERT INTO ignored_entry(server_id, entry_id) VALUES($1, $2) ON CONFLICT DO NOTHING RETURNING 1"
            ret = await ctx.pool.fetchval(query, ctx.guild.id, entry.id)
            rep = entry.mention if isinstance(entry, discord.TextChannel) else entry.display_name
            if ret:
                return await ctx.success(f"I'm now ignoring **{rep}**")
            return await ctx.error(f"{rep} is already ignored.")

        inserted = await self.bulk_ignore(ctx, entries)
        if inserted:
            if inserted != len(entries):
                await ctx.success(f"{inserted} entries are now ignored. The rest was already ignored.")
            else:
                await ctx.success(f"All {inserted} entries are now ignored.")
        else:
            await ctx.error("No entries were ignored. They were all already ignored.")

    @checks.server_mod_only()
    @ignore.command(name="list")
    async def ignore_list(self, ctx: NabCtx):
        """Shows a list of ignored channels and users."""
        query = "SELECT entry_id FROM ignored_entry WHERE server_id = $1"
        rows = await ctx.pool.fetch(query, ctx.guild.id)

        entries = [LazyEntry(ctx.guild, e[0]) for e in rows]
        if not entries:
            await ctx.send("There are no ignored entries in this server.")
            return
        pages = Pages(ctx, entries=entries)
        pages.embed.title = "Ignored entries"
        try:
            await pages.paginate()
        except CannotPaginate as e:
            await ctx.send(e)

    @commands.command()
    @checks.channel_mod_somewhere()
    async def makesay(self, ctx: NabCtx, *, message: str):
        """Makes the bot say a message.

        If it's used directly on a text channel, the bot will delete the command's message and repeat it itself.
        Note that deleting the message requires `Manage Messages` permissions in the channel.

        If it's used on a private message, the bot will ask on which channel he should say the message.
        Each channel in the list is numerated, by choosing a number, the message will be sent in the chosen channel.
        You can only send messages on channels where you have `Manage Channel` permissions.
        """
        if ctx.is_private:
            description_list = []
            channel_list = []
            prev_server = None
            num = 1
            for server in self.bot.guilds:
                author = server.get_member(ctx.author.id)
                bot_member = self.bot.get_member(self.bot.user.id, server)
                # Skip servers where the command user is not in
                if author is None:
                    continue
                # Check for every channel
                for channel in server.text_channels:
                    author_permissions = author.permissions_in(channel)
                    bot_permissions = bot_member.permissions_in(channel)
                    # Check if both the author and the bot have permissions to send messages and add channel to list
                    if author_permissions.send_messages and bot_permissions.send_messages \
                            and author_permissions.manage_channels:
                        separator = ""
                        if prev_server is not server:
                            separator = "---------------\n\t"
                        description_list.append("{2}{3}: **#{0}** in **{1}**".format(channel.name, server.name,
                                                                                separator, num))
                        channel_list.append(channel)
                        prev_server = server
                        num += 1
            if len(description_list) < 1:
                await ctx.send("We don't have any channels where we both can send messages and that you manage.")
                return
            await ctx.send("Choose a channel for me to send your message (number only): \n\t0: *Cancel*\n\t" +
                           "\n\t".join(["{0}".format(i) for i in description_list]))

            def check(m):
                return m.author == ctx.author and m.channel == ctx.channel
            try:
                answer = await self.bot.wait_for("message", timeout=60.0, check=check)
                answer = int(answer.content)
                if answer == 0:
                    await ctx.send("Changed your mind? Typical human.")
                    return
                await channel_list[answer-1].send(message)
                await ctx.send("Message sent on {0} ({1})".format(channel_list[answer-1].mention,
                                                                  channel_list[answer-1].guild))
            except IndexError:
                await ctx.send("That wasn't in the choices, you ruined it. Start from the beginning.")
            except ValueError:
                await ctx.send("That's not a valid answer!")
            except asyncio.TimeoutError:
                await ctx.send("... are you there? Fine, nevermind!")
        else:
            if not ctx.bot_permissions.manage_messages:
                return await ctx.send(f"{ctx.tick(False)} I need `Manage Messages` permission to use this command.")
            if not ctx.author_permissions.manage_channels:
                return await ctx.send(f"{ctx.tick(False)} You need `Manage Channel` permission to use this command.")
            await safe_delete_message(ctx.message)
            await ctx.message.delete()
            await ctx.channel.send(message)

    @commands.guild_only()
    @checks.channel_mod_only()
    @commands.command()
    async def unignore(self, ctx: NabCtx, *entries: converter.ChannelOrMember):
        """Removes a channel or user from the ignored list.

        If no parameter is provided, the current channel will be unignored.

        If the command is used with no parameters, it unignores the current channel."""
        if len(entries) == 0:
            query = "DELETE FROM ignored_entry WHERE server_id=$1 AND entry_id=$2 RETURNING true"
            res = await ctx.pool.fetchval(query, ctx.guild.id, ctx.channel.id)
            if res:
                return await ctx.success(f"{ctx.channel.mention} is no longer ignored.")
            return await ctx.error(f"{ctx.channel.mention} wasn't ignored.")
        query = "DELETE FROM ignored_entry WHERE server_id=$1 AND entry_id = ANY($2::bigint[]) RETURNING entry_id"
        # noinspection PyUnresolvedReferences
        entries = [c.id for c in entries]
        rows = await ctx.pool.fetch(query, ctx.guild.id, entries)
        if rows:
            return await ctx.success(f"{len(rows)} are now unignored.")
        await ctx.error("No entries were unignored.")

    @checks.channel_mod_only()
    @checks.tracking_world_only()
    @commands.command()
    async def unregistered(self, ctx: NabCtx):
        """Shows a list of users with no registered characters."""
        entries = []
        if ctx.world is None:
            await ctx.send("This server is not tracking any worlds.")
            return

        results = await ctx.pool.fetch('SELECT user_id FROM "character" WHERE world = $1 GROUP BY user_id', ctx.world)
        # Flatten list
        users = [i["user_id"] for i in results]
        for member in ctx.guild.members:  # type: discord.Member
            # Skip bots
            if member.bot:
                continue
            if member.id not in users:
                entries.append(f"@**{member.display_name}** \u2014 Joined: **{member.joined_at.date()}**")
        if len(entries) == 0:
            await ctx.send("There are no unregistered users.")
            return

        pages = Pages(ctx, entries=entries, per_page=10)
        pages.embed.title = "Unregistered members"
        try:
            await pages.paginate()
        except CannotPaginate as e:
            await ctx.send(e)
    # endregion

    @classmethod
    async def is_ignored(cls, conn, ctx: NabCtx):
        """Checks if the current context is ignored.

        A context could be ignored because either the channel or the user are in the ignored list."""
        query = "SELECT True FROM ignored_entry WHERE server_id=$1 AND (entry_id=$2 OR entry_id=$3);"
        return await conn.fetchrow(query, ctx.guild.id, ctx.channel.id, ctx.author.id)

    @classmethod
    async def bulk_ignore(cls, ctx: NabCtx, entries):
        async with ctx.pool.acquire() as conn:
            async with conn.transaction():
                query = "SELECT entry_id FROM ignored_entry WHERE server_id=$1;"
                records = await conn.fetch(query, ctx.guild.id)

                # Removing duplicates
                current_entries = {r[0] for r in records}
                records = [(ctx.guild.id, e.id) for e in entries if e.id not in current_entries]

                # do a bulk COPY
                await conn.copy_records_to_table('ignored_entry', columns=['server_id', 'entry_id'], records=records)
                return len(records)

def setup(bot):
    bot.add_cog(Mod(bot))
<|endoftext|>


<commit_before>from lxml import etree
from pkg_resources import resource_string, resource_listdir

from xmodule.x_module import XModule
from xmodule.raw_module import RawDescriptor
from xblock.core import String, Scope

class DiscussionModule(XModule):
    js = {'coffee':
            [resource_string(__name__, 'js/src/time.coffee'),
            resource_string(__name__, 'js/src/discussion/display.coffee')]
        }
    js_module_name = "InlineDiscussion"

    discussion_id = String(scope=Scope.settings)
    discussion_category = String(scope=Scope.settings)
    discussion_target = String(scope=Scope.settings)
    sort_key = String(scope=Scope.settings)

    data = String(help="XML definition of inline discussion", scope=Scope.content)

    def get_html(self):
        context = {
            'discussion_id': self.discussion_id,
        }
        return self.system.render_template('discussion/_discussion_module.html', context)

    def __init__(self, *args, **kwargs):
        XModule.__init__(self, *args, **kwargs)

        xml_data = etree.fromstring(self.data)
        self.discussion_id = xml_data.attrib['id']
        self.title = xml_data.attrib['for']
        self.discussion_category = xml_data.attrib['discussion_category']

class DiscussionDescriptor(RawDescriptor):
    module_class = DiscussionModule
    template_dir_name = "discussion"

    # The discussion XML format uses `id` and `for` attributes,
    # but these would overload other module attributes, so we prefix them
    # for actual use in the code
    metadata_translations = dict(RawDescriptor.metadata_translations)
    metadata_translations['id'] = 'discussion_id'
    metadata_translations['for'] = 'discussion_target'

    discussion_id = String(scope=Scope.settings)
    discussion_category = String(scope=Scope.settings)
    discussion_target = String(scope=Scope.settings)
    sort_key = String(scope=Scope.settings)
<commit_msg>Don't update settings based on data from content for discussion modules during init<commit_after>from lxml import etree
from pkg_resources import resource_string, resource_listdir

from xmodule.x_module import XModule
from xmodule.raw_module import RawDescriptor
from xblock.core import String, Scope

class DiscussionModule(XModule):
    js = {'coffee':
            [resource_string(__name__, 'js/src/time.coffee'),
            resource_string(__name__, 'js/src/discussion/display.coffee')]
        }
    js_module_name = "InlineDiscussion"

    discussion_id = String(scope=Scope.settings)
    discussion_category = String(scope=Scope.settings)
    discussion_target = String(scope=Scope.settings)
    sort_key = String(scope=Scope.settings)

    def get_html(self):
        context = {
            'discussion_id': self.discussion_id,
        }
        return self.system.render_template('discussion/_discussion_module.html', context)

class DiscussionDescriptor(RawDescriptor):
    module_class = DiscussionModule
    template_dir_name = "discussion"

    # The discussion XML format uses `id` and `for` attributes,
    # but these would overload other module attributes, so we prefix them
    # for actual use in the code
    metadata_translations = dict(RawDescriptor.metadata_translations)
    metadata_translations['id'] = 'discussion_id'
    metadata_translations['for'] = 'discussion_target'

    discussion_id = String(scope=Scope.settings)
    discussion_category = String(scope=Scope.settings)
    discussion_target = String(scope=Scope.settings)
    sort_key = String(scope=Scope.settings)
<|endoftext|>


<commit_before>import discord
from discord.ext import commands
from osuapi import OsuApi, AHConnector
import osuapi.enums
import utils.errors

class Osu:
    def __init__(self, bot):
        self.bot = bot
        if bot.config['osu']:
            self.api = OsuApi(bot.config['osu'], connector=AHConnector())
        else:
            self.api = None

    def osu_mode_converter(self, mode=None):
        if mode is 'standard' or 'osu!standard' or None:
            return osuapi.enums.OsuMode.osu
        elif mode is 'ctb' or 'catchthebeat' or 'osu!catch' or 'catch':
            return osuapi.enums.OsuMode.catch
        elif mode is 'taiko' or 'osu!taiko':
            return osuapi.enums.OsuMode.taiko
        elif mode is 'mania' or 'osu!mania':
            return osuapi.enums.OsuMode.mania
        else:
            return 'Unknown'

    @commands.group()
    async def osu(self, ctx):
        """Commands for osu!"""
        if ctx.invoked_subcommand is None:
            help_em = discord.Embed(title='Commands for osu!', colour=0x690E8)
            help_em.add_field(name='user', value='Gets info on osu! players. `^osu user *user*`')
            await ctx.send(embed=help_em)

    @osu.command()
    async def user(self, ctx, u: str, mode: str):
        """Returns information on a osu! player.
        If the player name you are searching has spaces, use quotation marks.
        e.g. ^osu user "player name with spaces"
        Special thanks to khazhyk for the library this command uses.

        By default this command defaults to osu!standard.
        """
        if self.api:
            mode = self.osu_mode_converter(mode=mode)
            if mode == 'Unknown':
                raise utils.errors.ServiceError('Unknown mode')
            user = await self.api.get_user(u, mode=mode)
            user = user[0]
        else:
            raise utils.errors.ServiceError('osu! api key not configured')
        osu_embed = discord.Embed(title=f'osu! stats for {u}', colour=0x690E8)
        osu_embed.add_field(name='Country', value=user.country)
        osu_embed.add_field(name='User ID', value=user.user_id)
        osu_embed.add_field(name='Hits (300 score)', value=user.count300)
        osu_embed.add_field(name='Hits (100 score)', value=user.count100)
        osu_embed.add_field(name='Hits (50 score)', value=user.count50)
        osu_embed.add_field(name='Play count', value=user.playcount)
        osu_embed.add_field(name='Ranked score', value=user.ranked_score)
        osu_embed.add_field(name='Total score', value=user.total_score)
        osu_embed.add_field(name='Global rank', value=f'#{user.pp_rank}')
        osu_embed.add_field(name='Country rank', value=f'#{user.pp_country_rank}')
        osu_embed.add_field(name='Level', value=user.level)
        osu_embed.add_field(name='Total PP', value=user.pp_raw)
        osu_embed.add_field(name='Accuracy', value=f'{user.accuracy:.1f}%')
        osu_embed.add_field(name='Total SS plays', value=user.count_rank_ss)
        osu_embed.add_field(name='Total S plays', value=user.count_rank_s)
        osu_embed.add_field(name='Total A plays', value=user.count_rank_a)
        await ctx.send(embed=osu_embed)

def setup(bot):
    bot.add_cog(Osu(bot))
<commit_msg>cogs[osu]: more docs<commit_after>import discord
from discord.ext import commands
from osuapi import OsuApi, AHConnector
import osuapi.enums
import utils.errors

class Osu:
    def __init__(self, bot):
        self.bot = bot
        if bot.config['osu']:
            self.api = OsuApi(bot.config['osu'], connector=AHConnector())
        else:
            self.api = None

    def osu_mode_converter(self, mode=None):
        if mode is 0 or 'standard' or 'osu!standard' or 'osu!' or None:
            return osuapi.enums.OsuMode.osu
        elif mode is 1 or 'ctb' or 'catchthebeat' or 'osu!catch' or 'catch':
            return osuapi.enums.OsuMode.catch
        elif mode is 2 or 'taiko' or 'osu!taiko':
            return osuapi.enums.OsuMode.taiko
        elif mode is 3 or 'mania' or 'osu!mania':
            return osuapi.enums.OsuMode.mania
        else:
            return 'Unknown'

    @commands.group()
    async def osu(self, ctx):
        """Commands for osu!"""
        if ctx.invoked_subcommand is None:
            help_em = discord.Embed(title='Commands for osu!', colour=0x690E8)
            help_em.add_field(name='user', value='Gets info on osu! players. `^osu user *user*`')
            await ctx.send(embed=help_em)

    @osu.command()
    async def user(self, ctx, u: str, mode=None):
        """Returns information on a osu! player.
        If the player name you are searching has spaces, use quotation marks.
        e.g. ^osu user "player name with spaces"
        Special thanks to khazhyk for the library this command uses.

        By default this command defaults to osu!standard.
        All modes are supported.
        To use osu!standard, leave mode blank, or use 'standard', 'osu!standard', 'osu!' or 0.
        To use osu!catch, use 'catch', 'osu!catch', or 1.
        To use osu!taiko, use 'taiko', 'osu!taiko', or 2.
        To use osu!mania, use 'mania', 'osu!mania', or 3.
        Any other modes will return 'Unknown' error. (Service error)
        """
        if self.api:
            mode = self.osu_mode_converter(mode=mode)
            if mode == 'Unknown':
                raise utils.errors.ServiceError('Unknown mode')
            user = await self.api.get_user(u, mode=mode)
            user = user[0]
        else:
            raise utils.errors.ServiceError('osu! api key not configured')
        osu_embed = discord.Embed(title=f'osu! stats for {u}', colour=0x690E8)
        osu_embed.add_field(name='Country', value=user.country)
        osu_embed.add_field(name='User ID', value=user.user_id)
        osu_embed.add_field(name='Hits (300 score)', value=user.count300)
        osu_embed.add_field(name='Hits (100 score)', value=user.count100)
        osu_embed.add_field(name='Hits (50 score)', value=user.count50)
        osu_embed.add_field(name='Play count', value=user.playcount)
        osu_embed.add_field(name='Ranked score', value=user.ranked_score)
        osu_embed.add_field(name='Total score', value=user.total_score)
        osu_embed.add_field(name='Global rank', value=f'#{user.pp_rank}')
        osu_embed.add_field(name='Country rank', value=f'#{user.pp_country_rank}')
        osu_embed.add_field(name='Level', value=user.level)
        osu_embed.add_field(name='Total PP', value=user.pp_raw)
        osu_embed.add_field(name='Accuracy', value=f'{user.accuracy:.1f}%')
        osu_embed.add_field(name='Total SS plays', value=user.count_rank_ss)
        osu_embed.add_field(name='Total S plays', value=user.count_rank_s)
        osu_embed.add_field(name='Total A plays', value=user.count_rank_a)
        await ctx.send(embed=osu_embed)

def setup(bot):
    bot.add_cog(Osu(bot))
<|endoftext|>


<commit_before>import xbmcgui

class SeriesSelector(xbmcgui.WindowXMLDialog):
    def __init__(self, *args, **kwargs):
        super(SeriesSelector, self).__init__(args, kwargs)
        self.serieslist = kwargs.get('serieslist')
        self.selected = kwargs.get('selected', [])
        self.guilist = None

    def prompt(self):
        self.doModal()
        return self.selected

    def onInit(self):
        self.getControl(1).setLabel('$LOCALIZE[32400]')
        self.getControl(3).setVisible(False)
        self.getControl(5).setVisible(True)
        self.getControl(5).setLabel('$LOCALIZE[186]')
        self.guilist = self.getControl(6)
        self.guilist.setVisible(True)

        for series in self.serieslist:
            listitem = xbmcgui.ListItem(series['label'])
            listitem.setProperty('Addon.Summary', str(series['year']) + ' - ' + series['plot'])
            listitem.setProperty('imdbnumber', series['imdbnumber'])
            art = series['art']
            art['thumb'] = series['art']['poster']
            listitem.setArt(art)
            if series['imdbnumber'] in self.selected:
                listitem.select(True)
            self.guilist.addItem(listitem)
        self.setFocus(self.guilist)

    def onClick(self, controlid):
        if controlid == 6:
            item = self.guilist.getSelectedItem()
            if item.isSelected():
                item.select(False)
                self.selected.remove(item.getProperty('imdbnumber'))
            else:
                item.select(True)
                self.selected.append(item.getProperty('imdbnumber'))
        elif controlid == 5:
            self.close()
        elif controlid == 7:
            self.selected = []
            self.close()
<commit_msg>Better handle cancel and back in series selection<commit_after>import xbmcgui

class SeriesSelector(xbmcgui.WindowXMLDialog):
    def __init__(self, *args, **kwargs):
        super(SeriesSelector, self).__init__(args, kwargs)
        self.serieslist = kwargs.get('serieslist')
        self.originalselected = list(kwargs.get('selected', []))
        self.selected = []
        self.guilist = None

    def prompt(self):
        self.doModal()
        return self.selected

    def onInit(self):
        self.getControl(1).setLabel('$LOCALIZE[32400]')
        self.getControl(3).setVisible(False)
        self.getControl(5).setVisible(True)
        self.getControl(5).setLabel('$LOCALIZE[186]')
        self.guilist = self.getControl(6)
        self.guilist.setVisible(True)

        for series in self.serieslist:
            listitem = xbmcgui.ListItem(series['label'])
            listitem.setProperty('Addon.Summary', str(series['year']) + ' - ' + series['plot'])
            listitem.setProperty('imdbnumber', series['imdbnumber'])
            art = series['art']
            art['thumb'] = series['art']['poster']
            listitem.setArt(art)
            if series['imdbnumber'] in self.originalselected:
                listitem.select(True)
                self.selected.append(series['imdbnumber'])
            self.guilist.addItem(listitem)
        self.setFocus(self.guilist)

    def onClick(self, controlid):
        if controlid == 6:
            item = self.guilist.getSelectedItem()
            if item.isSelected():
                item.select(False)
                self.selected.remove(item.getProperty('imdbnumber'))
            else:
                item.select(True)
                self.selected.append(item.getProperty('imdbnumber'))
        elif controlid == 5:
            self.close()
        elif controlid == 7:
            self.selected = self.originalselected
            self.close()

    def onAction(self, action):
        if action.getId() in (xbmcgui.ACTION_NAV_BACK, xbmcgui.ACTION_PREVIOUS_MENU):
            self.selected = self.originalselected
            self.close()
<|endoftext|>