<commit_before>import glob
from os.path import join
import os;
from seizures.data.EEGData import EEGData
from seizures.preprocessing import preprocessing
import numpy as np
import random
from seizures.features.FFTFeatures import FFTFeatures
from seizures.features.FeatureExtractBase import FeatureExtractBase

# Wittawat: Many load_* methods do not actually use the patient_name argument

class DataLoader(object):
    """
    Class to load data (all segments) for a patient
    Loading from individual files (not the stitched data)
    @author Vincent (from Shaun original loader)
    """

    def __init__(self, base_dir, feature_extractor):
        """
        base_dir: path to the directory containing patient folders i.e., directory
        containing Dog_1/, Dog_2, ..., Patient_1, Patient_2, ....
        """
        if not os.path.isdir(base_dir):
            raise ValueError('%s is not a directory.' % base_dir)
        if not isinstance(feature_extractor, FeatureExtractBase):
            raise ValueError('feature_extractor must be an instance of FeatureExtractBase')

        self.base_dir = base_dir
        self.feature_extractor = feature_extractor
        # patient_name = e.g., Dog_1
        self.patient_name = None
        # type_labels = a list of {0, 1}. Indicators of a seizure (1 for seizure).
        self.type_labels = None
        # early_labels = a list of {0, 1}. Indicators of an early seizure
        # (1 for an early seizure).
        self.early_labels = None
        # a list of numpy arrays. Each element is from feature_extractor.extract()
        # The length of the list is the number of time steps in the data segment
        # specified by patient_name.
        self.features_train = None
        # a list of numpy arrays. Each element is from feature_extractor.extract()
        self.features_test = None
        # list of file names in base_dir directory
        self.files = None
        self.files_nopath = None

    def load_data(self, patient_name, type='training', max_segments=-1):
        """
        Loads data for a patient and a type of data into class variables
        No output
        :param patient_name: e.g., Dog_1
        :param type: training or test
        :param max_segments: maximum segments to load. -1 to use the number of
        total segments available. Otherwise, all segments (ictal and interictal)
        will be randomly subsampled without replacement.
        """
        self.patient_name = patient_name
        self._reset_lists()
        # For type='training', this will get all interictal and ictal file names
        files = self._get_files_for_patient(type)

        # reorder files so as to mix a bit interictal and ictal (to fasten
        # debug I crop the files to its early entries)
        print 'Load with extractor = %s'% (str(self.feature_extractor))
        random.seed(0)
        if type == 'training':
            files_interictal = [f for f in files if f.find("_interictal_") >= 0 ]
            random.shuffle(files_interictal)
            print '%d interictal segments for %s'%(len(files_interictal), patient_name)
            files_ictal = [f for f in files if f.find("_ictal_") >= 0]
            random.shuffle(files_ictal)
            print '%d ictal segments for %s'%(len(files_ictal), patient_name)

            files = []
            # The following loop just interleaves ictal and interictal segments
            # so that we have
            #[ictal_segment1, interictal_segment1, ictal_segment2, ...]
            for i in range(max(len(files_ictal), len(files_interictal))):
                if i < len(files_ictal):
                    files.append(files_ictal[i])
                if i < len(files_interictal):
                    files.append(files_interictal[i])

        #np.random.seed(0)
        #I = np.random.permutation(len(files))
        #I = range(len(files))

        if type == 'training':
            total_segments = len(files_interictal) + len(files_ictal)
        else:
            total_segments = len(files)

        subsegments = min(max_segments, total_segments)
        print 'subsampling from %d segments to %d'% (total_segments, subsegments)
        #self.files = [files[i] for i in I[0:subsegments]]
        self.files = files[0:subsegments]

        #self.files = [files[i] for i in I[0:200]]

        if type == 'test':
            self.files = files

        for i, filename in enumerate(self.files):
            print float(i)/float(len(self.files))*100.," percent complete         \r",
            # Each call of _load_data_from_file appends data to features_train
            # features_test lists depending on the (type) variable.
            # It also appends data to type_labels and early_labels.
            self._load_data_from_file(patient_name, filename)
        print "\ndone"

    def training_data(self, patient_name, max_segments=-1):
        """
        returns features as a matrix of 1D np.ndarray
        returns classification vectors as 1D np.ndarrays
        :param patient_name:
        :param max_segments: maximum segments to load. -1 to use the number of
        total segments available. Otherwise, all segments (ictal and interictal)
        will be randomly subsampled without replacement.
        :return: feature matrix and labels
        """
        print "\nLoading train data for " + patient_name
        self.load_data(patient_name, type='training', max_segments=max_segments)
        X = self._merge_vectors_into_matrix(self.features_train)
        return X, np.array(self.type_labels), np.array(self.early_labels)

    def blocks_for_Xvalidation(self, patient_name,n_fold =3, max_segments=-1):
        """
        Stratified partitions (partition such that class proportion remains same
        in each data fold) of data for cross validation. The sum of instances
        in all partitions may be less than the original total.

        :param max_segments: maximum segments to load. -1 to use the number of
        total segments available. Otherwise, all segments (ictal and interictal)
        will be randomly subsampled without replacement.

        returns
        - a list of 2D ndarrays of features
        - a list of 1D ndarrays of type_labels
        - a list of 1D ndarrays of early_labels
        All the lists have length = n_fold.
        These outputs can be used with XValidation.

        :param patient_name:
        :return: feature matrix and labels
        """
        # y1 = type_labels,  y2 = early_labels
        X,y1,y2 = self.training_data(patient_name, max_segments=max_segments)
        n = len(y1)

        assert(np.sum(y2)>n_fold)
        # do blocks for labels=1 and labels=0 separetaly and merge at the end
        # the aim is to always have both labels in each train/test sets
        Iy2_1 = np.where(y2==1)[0].tolist()
        Iy2_0 = np.where(y2==0)[0].tolist()
        assert(np.sum(Iy2_1)>n_fold)

        def chunks(l, n_block):
            """ Yield n_block blocks.
            """
            m = len(l)/n_block
            r = len(l)%n_block
            for i in xrange(0, n_block):
                if i == n_block-1:
                    yield l[i*m:-1]
                else:
                    yield l[i*m:(i+1)*m]

        Iy2_1_list = list(chunks(Iy2_1, n_fold))
        Iy2_0_list = list(chunks(Iy2_0, n_fold))
        #print len(Iy2_1_list)
        assert(len(Iy2_0_list)==n_fold)
        assert(len(Iy2_1_list)==n_fold)
        Iy2 = [Iy2_1_list[i]+Iy2_0_list[i] for i in range(n_fold)]

        y1p = [ y1[Iy2[i]] for i in range(n_fold)]
        y2p = [ y2[Iy2[i]] for i in range(n_fold)]
        Xp = [ X[Iy2[i],:] for i in range(n_fold)]

        return Xp,y1p,y2p

    def test_data(self, patient_name, max_segments=-1):
        """
        returns features as a matrix of 1D np.ndarray
        :rtype : numpy 2D ndarray
        :param name: str
        :param max_segments: maximum segments to load. -1 to use the number of
        total segments available. Otherwise, all segments (ictal and interictal)
        will be randomly subsampled without replacement.
        """
        print "\nLoading test data for " + patient_name
        self.load_data(patient_name, type='test', max_segments=max_segments)
        return self._merge_vectors_into_matrix(self.features_test)

    def _reset_lists(self):
        self.episode_matrices = []
        self.type_labels = []
        self.early_labels = []

    def _get_files_for_patient(self, type="training"):
        assert (type in ["test", "training"])
        if type == "training":
            self.features_train = []
            files = glob.glob(join(self.base_dir, self.patient_name + '/*ictal*'))

        elif type == "test":
            self.features_test = []
            self.early_labels = []
            self.type_labels = []
            files = glob.glob(join(self.base_dir, self.patient_name + '/*test*'))
        # files_nopath = [os.path.basename(x) for x in files]
        #self.files_nopath = files_nopath
        return files

    def _load_data_from_file(self, patient, filename):
        if filename.find('test') != -1:
            # if filename is a test segment
            self._load_test_data_from_file(patient, filename)
        else:
            self._load_training_data_from_file(patient, filename)

    def _load_training_data_from_file(self, patient, filename):
        """
        Loading single file training data
        :param patient:
        :param filename:
        :return:
        """
        #print "\nLoading train data for " + patient + filename
        eeg_data_tmp = EEGData(filename)
        # a list of Instance's
        eeg_data = eeg_data_tmp.get_instances()
        assert len(eeg_data) == 1
        # eeg_data is now an Instance

        eeg_data = eeg_data[0]
        if filename.find('interictal') > -1:
            y_seizure=0
            y_early=0
        elif eeg_data.latency < 15:
            y_seizure=1
            y_early=1
        else:
            y_seizure=1
            y_early=0

        fs = eeg_data.sample_rate

        # preprocessing
        data = eeg_data.eeg_data
        params = {'fs':fs,
          'anti_alias_cutoff': 100.,
          'anti_alias_width': 30.,
          'anti_alias_attenuation' : 40,
          'elec_noise_width' :3.,
          'elec_noise_attenuation' : 60.0,
          'elec_noise_cutoff' : [49.,51.]}

        eeg_data.eeg_data = preprocessing.preprocess_multichannel_data(data,params)
        x = self.feature_extractor.extract(eeg_data)
        self.features_train.append(np.hstack(x))
        self.type_labels.append(y_seizure)
        self.early_labels.append(y_early)

    def _load_test_data_from_file(self, patient, filename):
        """
        Loading single file test data
        :param patient:
        :param filename:
        :return:
        """
        assert ( filename.find('test'))
        print "\nLoading test data for " + patient + filename
        eeg_data_tmp = EEGData(filename)
        eeg_data = eeg_data_tmp.get_instances()
        assert len(eeg_data) == 1
        eeg_data = eeg_data[0]

        fs = eeg_data.sample_rate
        # preprocessing
        data = eeg_data.eeg_data
        params = {'fs':fs,
          'anti_alias_cutoff': 150.,
          'anti_alias_width': 30.,
          'anti_alias_attenuation' : 30,
          'elec_noise_width' :3.,
          'elec_noise_attenuation' : 60.0,
          'elec_noise_cutoff' : [59.,61.]}

        eeg_data.eeg_data = preprocessing.preprocess_multichannel_data(data,params)
        x = self.feature_extractor.extract(eeg_data)
        self.features_test.append(np.hstack(x))

    def _get_feature_vector_from_instance(self, instance):
        return self.feature_extractor.extract(instance)

    def _merge_vectors_into_matrix(self, feature_vectors):
        n = len(feature_vectors)
        d = len(feature_vectors[0])
        matrix = np.zeros((n, d))
        for i, _ in enumerate(feature_vectors):
            matrix[i, :] = feature_vectors[i].T
        return matrix

    def labels(self, patient):
        # Wittawat: Why do we need patient argument ?
        assert (self.patient_name == patient)
        return self.type_labels, self.early_labels

<commit_msg>moved params to init, extra param for preprocessing ON/OFF<commit_after>import glob
from os.path import join
import os;
from seizures.data.EEGData import EEGData
from seizures.preprocessing import preprocessing
import numpy as np
import random
from seizures.features.FFTFeatures import FFTFeatures
from seizures.features.FeatureExtractBase import FeatureExtractBase

# Wittawat: Many load_* methods do not actually use the patient_name argument

class DataLoader(object):
    """
    Class to load data (all segments) for a patient
    Loading from individual files (not the stitched data)
    @author Vincent (from Shaun original loader)
    """

    def __init__(self, base_dir, feature_extractor):
        """
        base_dir: path to the directory containing patient folders i.e., directory
        containing Dog_1/, Dog_2, ..., Patient_1, Patient_2, ....
        """
        if not os.path.isdir(base_dir):
            raise ValueError('%s is not a directory.' % base_dir)
        if not isinstance(feature_extractor, FeatureExtractBase):
            raise ValueError('feature_extractor must be an instance of FeatureExtractBase')

        self.base_dir = base_dir
        self.feature_extractor = feature_extractor
        # patient_name = e.g., Dog_1
        self.patient_name = None
        # type_labels = a list of {0, 1}. Indicators of a seizure (1 for seizure).
        self.type_labels = None
        # early_labels = a list of {0, 1}. Indicators of an early seizure
        # (1 for an early seizure).
        self.early_labels = None
        # a list of numpy arrays. Each element is from feature_extractor.extract()
        # The length of the list is the number of time steps in the data segment
        # specified by patient_name.
        self.features_train = None
        # a list of numpy arrays. Each element is from feature_extractor.extract()
        self.features_test = None
        # list of file names in base_dir directory
        self.files = None
        self.files_nopath = None
        self.params = { 'anti_alias_cutoff': 500.,
          'anti_alias_width': 30.,
          'anti_alias_attenuation' : 40,
          'elec_noise_width' :3.,
          'elec_noise_attenuation' : 60.0,
          'elec_noise_cutoff' : [59.,61.],
          'targetrate':1000}

    def load_data(self, patient_name, type='training', max_segments=-1,preprocess=True):
        """
        Loads data for a patient and a type of data into class variables
        No output
        :param patient_name: e.g., Dog_1
        :param type: training or test
        :param max_segments: maximum segments to load. -1 to use the number of
        total segments available. Otherwise, all segments (ictal and interictal)
        will be randomly subsampled without replacement.
        """
        self.patient_name = patient_name
        self._reset_lists()
        # For type='training', this will get all interictal and ictal file names
        files = self._get_files_for_patient(type)

        # reorder files so as to mix a bit interictal and ictal (to fasten
        # debug I crop the files to its early entries)
        print 'Load with extractor = %s'% (str(self.feature_extractor))
        print 'Load with params = %s'% (str(self.params))

        random.seed(0)
        if type == 'training':
            files_interictal = [f for f in files if f.find("_interictal_") >= 0 ]
            random.shuffle(files_interictal)
            print '%d interictal segments for %s'%(len(files_interictal), patient_name)
            files_ictal = [f for f in files if f.find("_ictal_") >= 0]
            random.shuffle(files_ictal)
            print '%d ictal segments for %s'%(len(files_ictal), patient_name)

            files = []
            # The following loop just interleaves ictal and interictal segments
            # so that we have
            #[ictal_segment1, interictal_segment1, ictal_segment2, ...]
            for i in range(max(len(files_ictal), len(files_interictal))):
                if i < len(files_ictal):
                    files.append(files_ictal[i])
                if i < len(files_interictal):
                    files.append(files_interictal[i])

        #np.random.seed(0)
        #I = np.random.permutation(len(files))
        #I = range(len(files))

        if type == 'training':
            total_segments = len(files_interictal) + len(files_ictal)
        else:
            total_segments = len(files)

        subsegments = min(max_segments, total_segments)
        print 'subsampling from %d segments to %d'% (total_segments, subsegments)
        #self.files = [files[i] for i in I[0:subsegments]]
        self.files = files[0:subsegments]

        #self.files = [files[i] for i in I[0:200]]

        if type == 'test':
            self.files = files

        for i, filename in enumerate(self.files):
            print float(i)/float(len(self.files))*100.," percent complete         \r",
            # Each call of _load_data_from_file appends data to features_train
            # features_test lists depending on the (type) variable.
            # It also appends data to type_labels and early_labels.
            self._load_data_from_file(patient_name, filename,preprocess=preprocess)
        print "\ndone"

    def training_data(self, patient_name, max_segments=-1,preprocess=True):
        """
        returns features as a matrix of 1D np.ndarray
        returns classification vectors as 1D np.ndarrays
        :param patient_name:
        :param max_segments: maximum segments to load. -1 to use the number of
        total segments available. Otherwise, all segments (ictal and interictal)
        will be randomly subsampled without replacement.
        :return: feature matrix and labels
        """
        print "\nLoading train data for " + patient_name
        self.load_data(patient_name, type='training', max_segments=max_segments,preprocess=preprocess)
        X = self._merge_vectors_into_matrix(self.features_train)
        return X, np.array(self.type_labels), np.array(self.early_labels)

    def blocks_for_Xvalidation(self, patient_name,n_fold =3, max_segments=-1,preprocess=True):
        """
        Stratified partitions (partition such that class proportion remains same
        in each data fold) of data for cross validation. The sum of instances
        in all partitions may be less than the original total.

        :param max_segments: maximum segments to load. -1 to use the number of
        total segments available. Otherwise, all segments (ictal and interictal)
        will be randomly subsampled without replacement.

        returns
        - a list of 2D ndarrays of features
        - a list of 1D ndarrays of type_labels
        - a list of 1D ndarrays of early_labels
        All the lists have length = n_fold.
        These outputs can be used with XValidation.

        :param patient_name:
        :return: feature matrix and labels
        """
        # y1 = type_labels,  y2 = early_labels
        X,y1,y2 = self.training_data(patient_name, max_segments=max_segments,preprocess=preprocess)
        n = len(y1)

        assert(np.sum(y2)>n_fold)
        # do blocks for labels=1 and labels=0 separetaly and merge at the end
        # the aim is to always have both labels in each train/test sets
        Iy2_1 = np.where(y2==1)[0].tolist()
        Iy2_0 = np.where(y2==0)[0].tolist()
        assert(np.sum(Iy2_1)>n_fold)

        def chunks(l, n_block):
            """ Yield n_block blocks.
            """
            m = len(l)/n_block
            r = len(l)%n_block
            for i in xrange(0, n_block):
                if i == n_block-1:
                    yield l[i*m:-1]
                else:
                    yield l[i*m:(i+1)*m]

        Iy2_1_list = list(chunks(Iy2_1, n_fold))
        Iy2_0_list = list(chunks(Iy2_0, n_fold))
        #print len(Iy2_1_list)
        assert(len(Iy2_0_list)==n_fold)
        assert(len(Iy2_1_list)==n_fold)
        Iy2 = [Iy2_1_list[i]+Iy2_0_list[i] for i in range(n_fold)]

        y1p = [ y1[Iy2[i]] for i in range(n_fold)]
        y2p = [ y2[Iy2[i]] for i in range(n_fold)]
        Xp = [ X[Iy2[i],:] for i in range(n_fold)]

        return Xp,y1p,y2p

    def test_data(self, patient_name, max_segments=-1):
        """
        returns features as a matrix of 1D np.ndarray
        :rtype : numpy 2D ndarray
        :param name: str
        :param max_segments: maximum segments to load. -1 to use the number of
        total segments available. Otherwise, all segments (ictal and interictal)
        will be randomly subsampled without replacement.
        """
        print "\nLoading test data for " + patient_name
        self.load_data(patient_name, type='test', max_segments=max_segments)
        return self._merge_vectors_into_matrix(self.features_test)

    def _reset_lists(self):
        self.episode_matrices = []
        self.type_labels = []
        self.early_labels = []

    def _get_files_for_patient(self, type="training"):
        assert (type in ["test", "training"])
        if type == "training":
            self.features_train = []
            files = glob.glob(join(self.base_dir, self.patient_name + '/*ictal*'))

        elif type == "test":
            self.features_test = []
            self.early_labels = []
            self.type_labels = []
            files = glob.glob(join(self.base_dir, self.patient_name + '/*test*'))
        # files_nopath = [os.path.basename(x) for x in files]
        #self.files_nopath = files_nopath
        return files

    def _load_data_from_file(self, patient, filename, preprocess=True):
        if filename.find('test') != -1:
            # if filename is a test segment
            self._load_test_data_from_file(patient, filename,preprocess=preprocess)
        else:
            self._load_training_data_from_file(patient, filename,preprocess=preprocess)

    def _load_training_data_from_file(self, patient, filename, preprocess=True):
        """
        Loading single file training data
        :param patient:
        :param filename:
        :return:
        """
        #print "\nLoading train data for " + patient + filename
        eeg_data_tmp = EEGData(filename)
        # a list of Instance's
        eeg_data = eeg_data_tmp.get_instances()
        assert len(eeg_data) == 1
        # eeg_data is now an Instance

        eeg_data = eeg_data[0]
        if filename.find('interictal') > -1:
            y_seizure=0
            y_early=0
        elif eeg_data.latency < 15:
            y_seizure=1
            y_early=1
        else:
            y_seizure=1
            y_early=0

        fs = eeg_data.sample_rate

        # preprocessing
        data = eeg_data.eeg_data

        params = self.params
        params['fs']=fs
        ### comment if no preprocessing
        if preprocess==True:
            eeg_data.eeg_data = preprocessing.preprocess_multichannel_data(data,params)
        ###
        x = self.feature_extractor.extract(eeg_data)
        self.features_train.append(np.hstack(x))
        self.type_labels.append(y_seizure)
        self.early_labels.append(y_early)

    def _load_test_data_from_file(self, patient, filename, preprocess=True):
        """
        Loading single file test data
        :param patient:
        :param filename:
        :return:
        """
        assert ( filename.find('test'))
        print "\nLoading test data for " + patient + filename
        eeg_data_tmp = EEGData(filename)
        eeg_data = eeg_data_tmp.get_instances()
        assert len(eeg_data) == 1
        eeg_data = eeg_data[0]

        fs = eeg_data.sample_rate
        # preprocessing
        data = eeg_data.eeg_data

        params = self.params
        params['fs']=fs

        ### comment if no preprocessing
        if preprocess==True:
            eeg_data.eeg_data = preprocessing.preprocess_multichannel_data(data,params)
        x = self.feature_extractor.extract(eeg_data)
        self.features_test.append(np.hstack(x))

    def _get_feature_vector_from_instance(self, instance):
        return self.feature_extractor.extract(instance)

    def _merge_vectors_into_matrix(self, feature_vectors):
        n = len(feature_vectors)
        d = len(feature_vectors[0])
        matrix = np.zeros((n, d))
        for i, _ in enumerate(feature_vectors):
            matrix[i, :] = feature_vectors[i].T
        return matrix

    def labels(self, patient):
        # Wittawat: Why do we need patient argument ?
        assert (self.patient_name == patient)
        return self.type_labels, self.early_labels

<|endoftext|>


<commit_before>from django.db import models
from django.db.models.signals import post_save, pre_save
from django.dispatch import receiver

from allauth.account.models import Profile

from easymoney import MoneyField

def create_client(sender, instance, created, **kwargs):
    if created and hasattr(instance.user, 'groups') and \
            instance.user.groups.filter(name='Client').exists():
        Client.objects.create(profile=instance)
post_save.connect(create_client, sender=Profile)

class ClientProfileManager(models.Manager):
    def get_queryset(self):
        return super(ClientProfileManager, self).get_queryset().filter(client_profile__isnull=False)

class ClientProfile(Profile):
    class Meta:
        proxy = True

    objects = ClientProfileManager()

class Client(models.Model):
    profile = models.OneToOneField(Profile,
                                unique=True,
                                verbose_name= 'profile',
                                related_name='client_profile'
                                )
    outstanding_balance = MoneyField(default=0)
    bill_day = models.IntegerField(default=1)

    def __unicode__(self):
        return unicode(self.profile)

    def clean(self):
        if self.bill_day > 28 or self.bill_day < 1:
            raise ValidationError("Please enter a day between 1 and 28.")

    @staticmethod
    def autocomplete_search_fields():
        return ("profile__first_name__icontains", "profile__last_name__icontains")

def create_client(sender, instance, created, **kwargs):
    if created and not hasattr(instance, 'client'):
        Client.objects.create(profile=instance)

post_save.connect(create_client, sender=ClientProfile)<commit_msg>client<commit_after>from django.db import models
from django.db.models.signals import post_save, pre_save
from django.dispatch import receiver

from allauth.account.models import Profile

from easymoney import MoneyField

def create_client(sender, instance, created, **kwargs):
    if created and hasattr(instance.user, 'groups') and \
            instance.user.groups.filter(name='Client').exists():
        Client.objects.create(profile=instance)
post_save.connect(create_client, sender=Profile)

class ClientProfileManager(models.Manager):
    def get_queryset(self):
        return super(ClientProfileManager, self).get_queryset().filter(client_profile__isnull=False)

class ClientProfile(Profile):
    class Meta:
        proxy = True

    objects = ClientProfileManager()

class Client(models.Model):
    profile = models.OneToOneField(Profile,
                                unique=True,
                                verbose_name= 'profile',
                                related_name='client_profile'
                                )
    outstanding_balance = MoneyField(default=0)
    bill_day = models.IntegerField(default=1)
    billing_name = models.CharField(max_length = 127, blank=True, null=True)

    def __unicode__(self):
        if self.billing_name:
            return unicode(self.billing_name)
        return unicode(self.profile)

    def clean(self):
        if self.bill_day > 28 or self.bill_day < 1:
            raise ValidationError("Please enter a day between 1 and 28.")<|endoftext|>


<commit_before>import asyncio, discord, re, random
from   operator import itemgetter
from   discord.ext import commands
from   Cogs import Utils, DisplayName, Message

def setup(bot):
	# Add the bot and deps
	settings = bot.get_cog("Settings")
	bot.add_cog(BotAdmin(bot, settings))

class BotAdmin(commands.Cog):

	# Init with the bot reference, and a reference to the settings var
	def __init__(self, bot, settings):
		self.bot = bot
		self.settings = settings
		self.dregex =  re.compile(r"(?i)(discord(\.gg|app\.com)\/)(?!attachments)([^\s]+)")
		self.mention_re = re.compile(r"[0-9]{17,21}")
		global Utils, DisplayName
		Utils = self.bot.get_cog("Utils")
		DisplayName = self.bot.get_cog("DisplayName")

	async def message(self, message):
		# Check for discord invite links and remove them if found - per server settings
		if not self.dregex.search(message.content): return None # No invite in the passed message - nothing to do
		# Got an invite - let's see if we care
		if not self.settings.getServerStat(message.guild,"RemoveInviteLinks",False): return None # We don't care
		# We *do* care, let's see if the author is admin/bot-admin as they'd have power to post invites
		ctx = await self.bot.get_context(message)
		if Utils.is_bot_admin(ctx): return None # We are immune!
		# At this point - we need to delete the message
		return { 'Ignore' : True, 'Delete' : True}

	@commands.command(pass_context=True)
	async def removeinvitelinks(self, ctx, *, yes_no = None):
		"""Enables/Disables auto-deleting discord invite links in chat (bot-admin only)."""
		if not await Utils.is_bot_admin_reply(ctx): return
		await ctx.send(Utils.yes_no_setting(ctx,"Remove discord invite links","RemoveInviteLinks",yes_no))

	@commands.command(pass_context=True)
	async def setuserparts(self, ctx, member : discord.Member = None, *, parts : str = None):
		"""Set another user's parts list (owner only)."""
		# Only allow owner
		isOwner = self.settings.isOwner(ctx.author)
		if isOwner == None:
			msg = 'I have not been claimed, *yet*.'
			return await ctx.send(msg)
		elif isOwner == False:
			msg = 'You are not the *true* owner of me.  Only the rightful owner can use this command.'
			return await ctx.send(msg)

		if member == None:
			msg = 'Usage: `{}setuserparts [member] "[parts text]"`'.format(ctx.prefix)
			return await ctx.send(msg)

		if type(member) is str:
			try:
				member = discord.utils.get(ctx.guild.members, name=member)
			except:
				return await ctx.send("That member does not exist")

		if not parts:
			parts = ""

		self.settings.setGlobalUserStat(member, "Parts", parts)
		msg = '*{}\'s* parts have been set to:\n{}'.format(DisplayName.name(member), parts)
		await ctx.send(Utils.suppressed(ctx,msg))

	@setuserparts.error
	async def setuserparts_error(self, error, ctx):
		# do stuff
		msg = 'setuserparts Error: {}'.format(error)
		await ctx.send(msg)

	@commands.command(pass_context=True)
	async def ignore(self, ctx, *, member = None):
		"""Adds a member to the bot's "ignore" list (bot-admin only)."""
		if not await Utils.is_bot_admin_reply(ctx): return

		if member == None:
			msg = 'Usage: `{}ignore [member]`'.format(ctx.prefix)
			return await ctx.send(msg)

		if type(member) is str:
			memberName = member
			member = DisplayName.memberForName(memberName, ctx.guild)
			if not member:
				msg = 'I couldn\'t find *{}*...'.format(memberName)
				return await ctx.send(Utils.suppressed(ctx,msg))

		ignoreList = self.settings.getServerStat(ctx.guild, "IgnoredUsers")

		for user in ignoreList:
			if str(member.id) == str(user["ID"]):
				# Found our user - already ignored
				return await ctx.send('*{}* is already being ignored.'.format(DisplayName.name(member)))
		# Let's ignore someone
		ignoreList.append({ "Name" : member.name, "ID" : member.id })
		self.settings.setServerStat(ctx.guild, "IgnoredUsers", ignoreList)

		await ctx.send('*{}* is now being ignored.'.format(DisplayName.name(member)))

	@ignore.error
	async def ignore_error(self, error, ctx):
		# do stuff
		msg = 'ignore Error: {}'.format(error)
		await ctx.send(msg)

	@commands.command(pass_context=True)
	async def listen(self, ctx, *, member = None):
		"""Removes a member from the bot's "ignore" list (bot-admin only)."""
		if not await Utils.is_bot_admin_reply(ctx): return

		if member == None:
			return await ctx.send('Usage: `{}listen [member]`'.format(ctx.prefix))

		if type(member) is str:
			memberName = member
			member = DisplayName.memberForName(memberName, ctx.guild)
			if not member:
				msg = 'I couldn\'t find *{}*...'.format(memberName)
				return await ctx.send(Utils.suppressed(ctx,msg))

		ignoreList = self.settings.getServerStat(ctx.guild, "IgnoredUsers")

		for user in ignoreList:
			if str(member.id) == str(user["ID"]):
				# Found our user - already ignored
				ignoreList.remove(user)
				self.settings.setServerStat(ctx.guild, "IgnoredUsers", ignoreList)
				return await ctx.send("*{}* is no longer being ignored.".format(DisplayName.name(member)))

		await ctx.send('*{}* wasn\'t being ignored...'.format(DisplayName.name(member)))

	@listen.error
	async def listen_error(self, error, ctx):
		# do stuff
		msg = 'listen Error: {}'.format(error)
		await ctx.send(msg)

	@commands.command(pass_context=True)
	async def ignored(self, ctx):
		"""Lists the users currently being ignored."""
		ignoreArray = self.settings.getServerStat(ctx.guild, "IgnoredUsers")
		promoSorted = sorted(ignoreArray, key=itemgetter('Name'))
		if not len(promoSorted):
			return await ctx.send("I'm not currently ignoring anyone.")
		ignored = ["*{}*".format(DisplayName.name(ctx.guild.get_member(int(x["ID"])))) for x in promoSorted if ctx.guild.get_member(int(x["ID"]))]
		await ctx.send("Currently Ignored Users:\n{}".format("\n".join(ignored)))

	async def kick_ban(self, ctx, members_and_reason = None, command_name = "kick"):
		# Helper method to handle the lifting for kick and ban
		if not await Utils.is_bot_admin_reply(ctx): return
		if not members_and_reason:
			return await ctx.send('Usage: `{}{} [space delimited member mention/id] [reason]`'.format(ctx.prefix, command_name))
		# Force a mention - we don't want any ambiguity
		args = members_and_reason.split()
		# Get our list of targets
		targets = []
		missed = []
		unable = []
		reason = ""
		for index,item in enumerate(args):
			if self.mention_re.search(item): # Check if it's a mention
				# Resolve the member
				mem_id = int(re.sub(r'\W+', '', item))
				member = ctx.guild.get_member(mem_id)
				# If we have an invalid mention, save it to report later
				if member is None:
					missed.append(str(mem_id))
					continue
				# We should have a valid member - let's make sure it's not:
				# 1. The bot, 2. The command caller, 3. Another bot-admin/admin
				if member.id == self.bot.user.id or member.id == ctx.author.id or Utils.is_bot_admin(ctx,member):
					unable.append(member.mention)
					continue
				if not member in targets: targets.append(member) # Only add them if we don't already have them
			else:
				# Not a mention - must be the reason, dump the rest of the items into a string
				# separated by a space
				reason = " ".join(args[index:])
				break
		reason = reason if len(reason) else "No reason provided."
		if not len(targets):
			msg = "**With reason:**\n\n{}".format(reason)
			if len(unable): msg = "**Unable to {}:**\n\n{}\n\n".format(command_name,"\n".join(unable)) + msg
			if len(missed): msg = "**Unmatched ID{}:**\n\n{}\n\n".format("" if len(missed) == 1 else "s","\n".join(missed)) + msg
			return await Message.EmbedText(title="No valid members passed!",description=msg,color=ctx.author).send(ctx)
		# We should have a list of targets, and the reason - let's list them for confirmation
		# then generate a 4-digit confirmation code that the original requestor needs to confirm
		# in order to follow through
		confirmation_code = "".join([str(random.randint(0,9)) for x in range(4)])
		msg = "**To {} the following member{}:**\n\n{}\n\n**With reason:**\n\n\"{}\"\n\n**Please type:**\n\n`{}`{}{}".format(
			command_name,
			"" if len(targets) == 1 else "s",
			"\n".join([x.name+"#"+x.discriminator for x in targets]),
			reason if len(reason) else "None",
			confirmation_code,
			"" if not len(missed) else "\n\n**Unmatched ID{}:**\n\n{}".format("" if len(missed) == 1 else "s", "\n".join(missed)),
			"" if not len(unable) else "\n\n**Unable to {}:**\n\n{}".format(command_name,"\n".join(unable))
			)
		confirmation_message = await Message.EmbedText(title="{} Confirmation".format(command_name.capitalize()),description=msg,color=ctx.author).send(ctx)
		def check_confirmation(message):
			return message.channel == ctx.channel and ctx.author == message.author # Just making sure it's the same user/channel
		try: confirmation_user = await self.bot.wait_for('message', timeout=60, check=check_confirmation)
		except: confirmation_user = ""
		# Delete the confirmation message
		await confirmation_message.delete()
		# Verify the confirmation
		if not confirmation_user.content == confirmation_code: return await ctx.send("{} cancelled!".format(command_name.capitalize()))
		# We got the authorization!
		message = await Message.EmbedText(title="{}ing...".format("Bann" if command_name == "ban" else "Kick"),color=ctx.author).send(ctx)
		canned = []
		cant = []
		command = ctx.guild.ban if command_name == "ban" else ctx.guild.kick
		for target in targets:
			try:
				await command(target,reason="{}#{}: {}".format(ctx.author.name,ctx.author.discriminator,reason))
				canned.append(target)
			except: cant.append(target)
		msg = ""
		if len(canned):
			msg += "**I was ABLE to {}:**\n\n{}\n\n".format(command_name,"\n".join([x.name+"#"+x.discriminator for x in canned]))
		if len(cant):
			msg += "**I was UNABLE to {}:**\n\n{}\n\n".format(command_name,"\n".join([x.name+"#"+x.discriminator for x in cant]))
		await Message.EmbedText(title="{} Results".format(command_name.capitalize()),description=msg).edit(ctx,message)

	@commands.command(pass_context=True)
	async def kick(self, ctx, *, members = None, reason = None):
		"""Kicks the passed members for the specified reason.
		All kick targets must be mentions to avoid ambiguity.
		You can kick up to 5 members at once.
		The reason is required (bot-admin only).

		eg:  $kick @user1#1234 @user2#5678 @user3#9012 for spamming"""
		await self.kick_ban(ctx,members, "kick")

	@commands.command(pass_context=True)
	async def ban(self, ctx, *, members = None, reason = None):
		"""Bans the passed members for the specified reason.
		All ban targets must be mentions to avoid ambiguity.
		You can ban up to 5 members at once.
		The reason is required (bot-admin only).

		eg:  $ban @user1#1234 @user2#5678 @user3#9012 for spamming"""
		await self.kick_ban(ctx,members, "ban")
<commit_msg>Add $banned for querying ban reasons<commit_after>import asyncio, discord, re, random
from   operator import itemgetter
from   discord.ext import commands
from   Cogs import Utils, DisplayName, Message, PickList

def setup(bot):
	# Add the bot and deps
	settings = bot.get_cog("Settings")
	bot.add_cog(BotAdmin(bot, settings))

class BotAdmin(commands.Cog):

	# Init with the bot reference, and a reference to the settings var
	def __init__(self, bot, settings):
		self.bot = bot
		self.settings = settings
		self.dregex =  re.compile(r"(?i)(discord(\.gg|app\.com)\/)(?!attachments)([^\s]+)")
		self.mention_re = re.compile(r"[0-9]{17,21}")
		global Utils, DisplayName
		Utils = self.bot.get_cog("Utils")
		DisplayName = self.bot.get_cog("DisplayName")

	async def message(self, message):
		# Check for discord invite links and remove them if found - per server settings
		if not self.dregex.search(message.content): return None # No invite in the passed message - nothing to do
		# Got an invite - let's see if we care
		if not self.settings.getServerStat(message.guild,"RemoveInviteLinks",False): return None # We don't care
		# We *do* care, let's see if the author is admin/bot-admin as they'd have power to post invites
		ctx = await self.bot.get_context(message)
		if Utils.is_bot_admin(ctx): return None # We are immune!
		# At this point - we need to delete the message
		return { 'Ignore' : True, 'Delete' : True}

	@commands.command(pass_context=True)
	async def removeinvitelinks(self, ctx, *, yes_no = None):
		"""Enables/Disables auto-deleting discord invite links in chat (bot-admin only)."""
		if not await Utils.is_bot_admin_reply(ctx): return
		await ctx.send(Utils.yes_no_setting(ctx,"Remove discord invite links","RemoveInviteLinks",yes_no))

	@commands.command(pass_context=True)
	async def setuserparts(self, ctx, member : discord.Member = None, *, parts : str = None):
		"""Set another user's parts list (owner only)."""
		# Only allow owner
		isOwner = self.settings.isOwner(ctx.author)
		if isOwner == None:
			msg = 'I have not been claimed, *yet*.'
			return await ctx.send(msg)
		elif isOwner == False:
			msg = 'You are not the *true* owner of me.  Only the rightful owner can use this command.'
			return await ctx.send(msg)

		if member == None:
			msg = 'Usage: `{}setuserparts [member] "[parts text]"`'.format(ctx.prefix)
			return await ctx.send(msg)

		if type(member) is str:
			try:
				member = discord.utils.get(ctx.guild.members, name=member)
			except:
				return await ctx.send("That member does not exist")

		if not parts:
			parts = ""

		self.settings.setGlobalUserStat(member, "Parts", parts)
		msg = '*{}\'s* parts have been set to:\n{}'.format(DisplayName.name(member), parts)
		await ctx.send(Utils.suppressed(ctx,msg))

	@setuserparts.error
	async def setuserparts_error(self, error, ctx):
		# do stuff
		msg = 'setuserparts Error: {}'.format(error)
		await ctx.send(msg)

	@commands.command(pass_context=True)
	async def ignore(self, ctx, *, member = None):
		"""Adds a member to the bot's "ignore" list (bot-admin only)."""
		if not await Utils.is_bot_admin_reply(ctx): return

		if member == None:
			msg = 'Usage: `{}ignore [member]`'.format(ctx.prefix)
			return await ctx.send(msg)

		if type(member) is str:
			memberName = member
			member = DisplayName.memberForName(memberName, ctx.guild)
			if not member:
				msg = 'I couldn\'t find *{}*...'.format(memberName)
				return await ctx.send(Utils.suppressed(ctx,msg))

		ignoreList = self.settings.getServerStat(ctx.guild, "IgnoredUsers")

		for user in ignoreList:
			if str(member.id) == str(user["ID"]):
				# Found our user - already ignored
				return await ctx.send('*{}* is already being ignored.'.format(DisplayName.name(member)))
		# Let's ignore someone
		ignoreList.append({ "Name" : member.name, "ID" : member.id })
		self.settings.setServerStat(ctx.guild, "IgnoredUsers", ignoreList)

		await ctx.send('*{}* is now being ignored.'.format(DisplayName.name(member)))

	@ignore.error
	async def ignore_error(self, error, ctx):
		# do stuff
		msg = 'ignore Error: {}'.format(error)
		await ctx.send(msg)

	@commands.command(pass_context=True)
	async def listen(self, ctx, *, member = None):
		"""Removes a member from the bot's "ignore" list (bot-admin only)."""
		if not await Utils.is_bot_admin_reply(ctx): return

		if member == None:
			return await ctx.send('Usage: `{}listen [member]`'.format(ctx.prefix))

		if type(member) is str:
			memberName = member
			member = DisplayName.memberForName(memberName, ctx.guild)
			if not member:
				msg = 'I couldn\'t find *{}*...'.format(memberName)
				return await ctx.send(Utils.suppressed(ctx,msg))

		ignoreList = self.settings.getServerStat(ctx.guild, "IgnoredUsers")

		for user in ignoreList:
			if str(member.id) == str(user["ID"]):
				# Found our user - already ignored
				ignoreList.remove(user)
				self.settings.setServerStat(ctx.guild, "IgnoredUsers", ignoreList)
				return await ctx.send("*{}* is no longer being ignored.".format(DisplayName.name(member)))

		await ctx.send('*{}* wasn\'t being ignored...'.format(DisplayName.name(member)))

	@listen.error
	async def listen_error(self, error, ctx):
		# do stuff
		msg = 'listen Error: {}'.format(error)
		await ctx.send(msg)

	@commands.command(pass_context=True)
	async def ignored(self, ctx):
		"""Lists the users currently being ignored."""
		ignoreArray = self.settings.getServerStat(ctx.guild, "IgnoredUsers")
		promoSorted = sorted(ignoreArray, key=itemgetter('Name'))
		if not len(promoSorted):
			return await ctx.send("I'm not currently ignoring anyone.")
		ignored = ["*{}*".format(DisplayName.name(ctx.guild.get_member(int(x["ID"])))) for x in promoSorted if ctx.guild.get_member(int(x["ID"]))]
		await ctx.send("Currently Ignored Users:\n{}".format("\n".join(ignored)))

	async def kick_ban(self, ctx, members_and_reason = None, command_name = "kick"):
		# Helper method to handle the lifting for kick and ban
		if not await Utils.is_bot_admin_reply(ctx): return
		if not members_and_reason:
			return await ctx.send('Usage: `{}{} [space delimited member mention/id] [reason]`'.format(ctx.prefix, command_name))
		# Force a mention - we don't want any ambiguity
		args = members_and_reason.split()
		# Get our list of targets
		targets = []
		missed = []
		unable = []
		reason = ""
		for index,item in enumerate(args):
			if self.mention_re.search(item): # Check if it's a mention
				# Resolve the member
				mem_id = int(re.sub(r'\W+', '', item))
				member = ctx.guild.get_member(mem_id)
				# If we have an invalid mention, save it to report later
				if member is None:
					missed.append(str(mem_id))
					continue
				# We should have a valid member - let's make sure it's not:
				# 1. The bot, 2. The command caller, 3. Another bot-admin/admin
				if member.id == self.bot.user.id or member.id == ctx.author.id or Utils.is_bot_admin(ctx,member):
					unable.append(member.mention)
					continue
				if not member in targets: targets.append(member) # Only add them if we don't already have them
			else:
				# Not a mention - must be the reason, dump the rest of the items into a string
				# separated by a space
				reason = " ".join(args[index:])
				break
		reason = reason if len(reason) else "No reason provided."
		if not len(targets):
			msg = "**With reason:**\n\n{}".format(reason)
			if len(unable): msg = "**Unable to {}:**\n\n{}\n\n".format(command_name,"\n".join(unable)) + msg
			if len(missed): msg = "**Unmatched ID{}:**\n\n{}\n\n".format("" if len(missed) == 1 else "s","\n".join(missed)) + msg
			return await Message.EmbedText(title="No valid members passed!",description=msg,color=ctx.author).send(ctx)
		# We should have a list of targets, and the reason - let's list them for confirmation
		# then generate a 4-digit confirmation code that the original requestor needs to confirm
		# in order to follow through
		confirmation_code = "".join([str(random.randint(0,9)) for x in range(4)])
		msg = "**To {} the following member{}:**\n\n{}\n\n**With reason:**\n\n\"{}\"\n\n**Please type:**\n\n`{}`{}{}".format(
			command_name,
			"" if len(targets) == 1 else "s",
			"\n".join([x.name+"#"+x.discriminator for x in targets]),
			reason if len(reason) else "None",
			confirmation_code,
			"" if not len(missed) else "\n\n**Unmatched ID{}:**\n\n{}".format("" if len(missed) == 1 else "s", "\n".join(missed)),
			"" if not len(unable) else "\n\n**Unable to {}:**\n\n{}".format(command_name,"\n".join(unable))
			)
		confirmation_message = await Message.EmbedText(title="{} Confirmation".format(command_name.capitalize()),description=msg,color=ctx.author).send(ctx)
		def check_confirmation(message):
			return message.channel == ctx.channel and ctx.author == message.author # Just making sure it's the same user/channel
		try: confirmation_user = await self.bot.wait_for('message', timeout=60, check=check_confirmation)
		except: confirmation_user = ""
		# Delete the confirmation message
		await confirmation_message.delete()
		# Verify the confirmation
		if not confirmation_user.content == confirmation_code: return await ctx.send("{} cancelled!".format(command_name.capitalize()))
		# We got the authorization!
		message = await Message.EmbedText(title="{}ing...".format("Bann" if command_name == "ban" else "Kick"),color=ctx.author).send(ctx)
		canned = []
		cant = []
		command = ctx.guild.ban if command_name == "ban" else ctx.guild.kick
		for target in targets:
			try:
				await command(target,reason="{}#{}: {}".format(ctx.author.name,ctx.author.discriminator,reason))
				canned.append(target)
			except: cant.append(target)
		msg = ""
		if len(canned):
			msg += "**I was ABLE to {}:**\n\n{}\n\n".format(command_name,"\n".join([x.name+"#"+x.discriminator for x in canned]))
		if len(cant):
			msg += "**I was UNABLE to {}:**\n\n{}\n\n".format(command_name,"\n".join([x.name+"#"+x.discriminator for x in cant]))
		await Message.EmbedText(title="{} Results".format(command_name.capitalize()),description=msg).edit(ctx,message)

	@commands.command(pass_context=True)
	async def kick(self, ctx, *, members = None, reason = None):
		"""Kicks the passed members for the specified reason.
		All kick targets must be mentions to avoid ambiguity.
		You can kick up to 5 members at once.
		The reason is required (bot-admin only).

		eg:  $kick @user1#1234 @user2#5678 @user3#9012 for spamming"""
		await self.kick_ban(ctx,members, "kick")

	@commands.command(pass_context=True)
	async def ban(self, ctx, *, members = None, reason = None):
		"""Bans the passed members for the specified reason.
		All ban targets must be mentions to avoid ambiguity.
		You can ban up to 5 members at once.
		The reason is required (bot-admin only).

		eg:  $ban @user1#1234 @user2#5678 @user3#9012 for spamming"""
		await self.kick_ban(ctx,members, "ban")

	@commands.command()
	async def banned(self, ctx, *, user_id = None):
		"""Queries the guild's ban list for the passed user id and responds with whether they've been banned and the reason (bot-admin only)."""
		if not await Utils.is_bot_admin_reply(ctx): return

		try: all_bans = await ctx.guild.bans()
		except: return await ctx.send("I couldn't get the ban list :(")

		if not len(all_bans): return await Message.EmbedText(title="Ban List",description="No bans found",color=ctx.author).send(ctx)

		orig_user = user_id
		try: user_id = int(user_id) if user_id != None else None
		except: user_id = -1 # Use -1 to indicate unresolved

		entries = []
		for ban in all_bans:
			entries.append({"name":"{}#{} ({})".format(ban.user.name,ban.user.discriminator,ban.user.id),"value":ban.reason if ban.reason else "No reason provided"})
			if user_id != None and user_id == ban.user.id:
				# Got a match - display it
				return await Message.Embed(
					title="Ban Found For {}".format(user_id),
					fields=[entries[-1]], # Send the last found entry
					color=ctx.author
				).send(ctx)
		return await PickList.PagePicker(title="Ban List ({:,} total)".format(len(entries)),description=None if user_id == None else "No match found for '{}'.".format(orig_user),list=entries,ctx=ctx).pick()
<|endoftext|>


<commit_before>import logging
import re

from multiprocessing.pool import ThreadPool
from data_extraction import doc_handler, link_handler

LINKS_WHITELIST_PATTERNS = (
    re.compile(r'.+developer\.mozilla\.org/[^/]+/.+'),
    re.compile(r'.+letsencrypt\.org/[^/]+/.+'),
    re.compile(r'.+hacks\.mozilla\.org/[^/]+/.+'),
    re.compile(r'.+github\.com/[^/]+/.+'),
    re.compile(r'(?:blog|journal|medium)[./].+'),
)

LINKS_BLACKLIST_TIPS = (
    'MDN_Product_Advisory_Board',
)

log = logging.getLogger('app')

def extract_contained_documents(docs):
    email_docs = filter(lambda doc: doc.get('from_mail') and doc.get('text'), docs)

    all_links = set()
    fake_docs = []

    for doc in email_docs:
        links = set(link_handler.extract_all_links(doc['text']))

        for link in links:
            link_ok = link not in all_links and \
                      any(p.match(link) for p in LINKS_WHITELIST_PATTERNS) and \
                      all(t not in link for t in LINKS_BLACKLIST_TIPS)

            all_links.add(link)

            if link_ok:
                doc_tags = filter(lambda t: t not in {'from_mail', 'composite'}, doc['tags'].split(','))
                doc_tags.append('extracted')
                doc_tags.sort()

                new_tags = ','.join(doc_tags)

                fake_doc = doc.copy()
                fake_doc.update({
                    'link': link,
                    'orig_link': link,
                    'extracted_from': doc['link'],
                    'tags': new_tags,
                    'title': '',
                    'source_type': 'extracted_from_email',
                    'description': '',
                    'text': None,
                    'from_mail': False,
                    'picture': None,
                })

                fake_docs.append(fake_doc)

    pool = ThreadPool()
    dressed_docs = pool.map(doc_handler.dress_document_with_metadata, fake_docs)
    pool.close()

    extracted_docs = filter(lambda doc: doc.get('title') and doc.get('description'), dressed_docs)

    log.info('Extracted %d docs from emails', len(extracted_docs))
    return extracted_docs
<commit_msg>Fix email tags list<commit_after>import logging
import re

from multiprocessing.pool import ThreadPool
from data_extraction import doc_handler, link_handler

LINKS_WHITELIST_PATTERNS = (
    re.compile(r'.+developer\.mozilla\.org/[^/]+/.+'),
    re.compile(r'.+letsencrypt\.org/[^/]+/.+'),
    re.compile(r'.+hacks\.mozilla\.org/[^/]+/.+'),
    re.compile(r'.+github\.com/[^/]+/.+'),
    re.compile(r'(?:blog|journal|medium)[./].+'),
)

LINKS_BLACKLIST_TIPS = (
    'MDN_Product_Advisory_Board',
)

log = logging.getLogger('app')

def extract_contained_documents(docs):
    email_docs = filter(lambda doc: doc.get('from_mail') and doc.get('text'), docs)

    all_links = set()
    fake_docs = []

    for doc in email_docs:
        links = set(link_handler.extract_all_links(doc['text']))

        for link in links:
            link_ok = link not in all_links and \
                      any(p.match(link) for p in LINKS_WHITELIST_PATTERNS) and \
                      all(t not in link for t in LINKS_BLACKLIST_TIPS)

            all_links.add(link)

            if link_ok:
                doc_tags = filter(lambda t: t not in {'from_mail', 'composite'}, doc['tags'])
                doc_tags.append('extracted')
                doc_tags.sort()

                new_tags = ','.join(doc_tags)

                fake_doc = doc.copy()
                fake_doc.update({
                    'link': link,
                    'orig_link': link,
                    'extracted_from': doc['link'],
                    'tags': new_tags,
                    'title': '',
                    'source_type': 'extracted_from_email',
                    'description': '',
                    'text': None,
                    'from_mail': False,
                    'picture': None,
                })

                fake_docs.append(fake_doc)

    pool = ThreadPool()
    dressed_docs = pool.map(doc_handler.dress_document_with_metadata, fake_docs)
    pool.close()

    extracted_docs = filter(lambda doc: doc.get('title') and doc.get('description'), dressed_docs)

    log.info('Extracted %d docs from emails', len(extracted_docs))
    return extracted_docs
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Script to upload images to wikipedia.

Arguments:

  -keep         Keep the filename as is
  -filename     Target filename
  -noverify     Do not ask for verification of the upload description if one
                is given

If any other arguments are given, the first is the URL or filename to upload,
and the rest is a proposed description to go with the upload. If none of these
are given, the user is asked for the file or URL to upload. The bot will then
upload the image to the wiki.

The script will ask for the location of an image, if not given as a parameter,
and for a description.
"""
#
# (C) Rob W.W. Hooft, Andre Engels 2003-2004
# (C) Pywikibot team, 2003-2014
#
# Distributed under the terms of the MIT license.
#
__version__ = '$Id$'
#

import os
import time
import urllib
import urlparse
import tempfile
import pywikibot
from pywikibot import config

class UploadRobot:
    def __init__(self, url, urlEncoding=None, description=u'',
                 useFilename=None, keepFilename=False,
                 verifyDescription=True, ignoreWarning=False,
                 targetSite=None, uploadByUrl=False):
        """
        @param ignoreWarning: Set this to True if you want to upload even if
            another file would be overwritten or another mistake would be
            risked.

        """
        self.url = url
        self.urlEncoding = urlEncoding
        self.description = description
        self.useFilename = useFilename
        self.keepFilename = keepFilename
        self.verifyDescription = verifyDescription
        self.ignoreWarning = ignoreWarning
        if config.upload_to_commons:
            self.targetSite = targetSite or pywikibot.Site('commons',
                                                           'commons')
        else:
            self.targetSite = targetSite or pywikibot.Site()
        self.targetSite.forceLogin()
        self.uploadByUrl = uploadByUrl

    def urlOK(self):
        """Return true if self.url looks like an URL or an existing local file.

        """
        return "://" in self.url or os.path.exists(self.url)

    def read_file_content(self):
        """Return name of temp file in which remote file is saved."""
        pywikibot.output(u'Reading file %s' % self.url)
        resume = False
        rlen = 0
        _contents = None
        dt = 15
        uo = urllib.URLopener()
        retrieved = False

        while not retrieved:
            if resume:
                pywikibot.output(u"Resume download...")
                uo.addheader('Range', 'bytes=%s-' % rlen)

            infile = uo.open(self.url)

            if 'text/html' in infile.info().getheader('Content-Type'):
                pywikibot.output(u"Couldn't download the image: the requested URL was not found on server.")
                return

            content_len = infile.info().getheader('Content-Length')
            accept_ranges = infile.info().getheader('Accept-Ranges') == 'bytes'

            if resume:
                _contents += infile.read()
            else:
                _contents = infile.read()

            infile.close()
            retrieved = True

            if content_len:
                rlen = len(_contents)
                content_len = int(content_len)
                if rlen < content_len:
                    retrieved = False
                    pywikibot.output(
                        u"Connection closed at byte %s (%s left)"
                        % (rlen, content_len))
                    if accept_ranges and rlen > 0:
                        resume = True
                    pywikibot.output(u"Sleeping for %d seconds..." % dt)
                    time.sleep(dt)
                    if dt <= 60:
                        dt += 15
                    elif dt < 360:
                        dt += 60
            else:
                pywikibot.log(
                    u"WARNING: No check length to retrieved data is possible.")
        handle, tempname = tempfile.mkstemp()
        t = os.fdopen(handle, "wb")
        t.write(_contents)
        t.close()
        return tempname

    def process_filename(self):
        """Return base filename portion of self.url."""
        # Isolate the pure name
        filename = self.url
        # Filename may be either a local file path or a URL
        if "://" in filename:
            # extract the path portion of the URL
            filename = urlparse.urlparse(filename).path
        filename = os.path.basename(filename)

        if self.useFilename:
            filename = self.useFilename
        if not self.keepFilename:
            pywikibot.output(
                u"The filename on the target wiki will default to: %s"
                % filename)
            # FIXME: these 2 belong somewhere else, presumably in family
            forbidden = '/'  # to be extended
            allowed_formats = (u'gif', u'jpg', u'jpeg', u'mid', u'midi',
                               u'ogg', u'png', u'svg', u'xcf', u'djvu',
                               u'ogv', u'oga', u'tif', u'tiff')
            # ask until it's valid
            while True:
                newfn = pywikibot.input(
                    u'Enter a better name, or press enter to accept:')
                if newfn == "":
                    newfn = filename
                    break
                ext = os.path.splitext(newfn)[1].lower().strip('.')
                # are any chars in forbidden also in newfn?
                invalid = set(forbidden) & set(newfn)
                if invalid:
                    c = "".join(invalid)
                    pywikibot.output("Invalid character(s): %s. Please try again" % c)
                    continue
                if ext not in allowed_formats:
                    choice = pywikibot.inputChoice(
                        u"File format is not one of [%s], but %s. Continue?"
                        % (u' '.join(allowed_formats), ext),
                        ['yes', 'no'], ['y', 'N'], 'N')
                    if choice == 'n':
                        continue
                break
            if newfn != '':
                filename = newfn
        # A proper description for the submission.
        pywikibot.output(u"The suggested description is:")
        pywikibot.output(self.description)
        if self.verifyDescription:
            newDescription = u''
            choice = pywikibot.inputChoice(
                u'Do you want to change this description?',
                ['Yes', 'No'], ['y', 'N'], 'n')
            if choice == 'y':
                from pywikibot import editor as editarticle
                editor = editarticle.TextEditor()
                newDescription = editor.edit(self.description)
            # if user saved / didn't press Cancel
            if newDescription:
                self.description = newDescription
        return filename

    def upload_image(self, debug=False):
        """Upload the image at self.url to the target wiki.

        Return the filename that was used to upload the image.
        If the upload fails, ask the user whether to try again or not.
        If the user chooses not to retry, return null.

        """
        filename = self.process_filename()

        site = self.targetSite
        imagepage = pywikibot.FilePage(site, filename)  # normalizes filename
        imagepage.text = self.description

        pywikibot.output(u'Uploading file to %s via API....' % site)

        try:
            if self.uploadByUrl:
                site.upload(imagepage, source_url=self.url,
                            ignore_warnings=self.ignoreWarning)
            else:
                if "://" in self.url:
                    temp = self.read_file_content()
                else:
                    temp = self.url
                site.upload(imagepage, source_filename=temp,
                            ignore_warnings=self.ignoreWarning)

        except pywikibot.UploadWarning as warn:
            pywikibot.output(u"We got a warning message: ", newline=False)
            pywikibot.output(str(warn))
            answer = pywikibot.inputChoice(u"Do you want to ignore?",
                                           ['Yes', 'No'], ['y', 'N'], 'N')
            if answer == "y":
                self.ignoreWarning = 1
                self.keepFilename = True
                return self.upload_image(debug)
            else:
                pywikibot.output(u"Upload aborted.")
                return

        except Exception:
            pywikibot.error("Upload error: ", exc_info=True)

        else:
            #No warning, upload complete.
            pywikibot.output(u"Upload successful.")
            return filename  # data['filename']

    def run(self):
        while not self.urlOK():
            if not self.url:
                pywikibot.output(u'No input filename given')
            else:
                pywikibot.output(u'Invalid input filename given. Try again.')
            self.url = pywikibot.input(u'File or URL where image is now:')
        return self.upload_image()

def main(*args):
    url = u''
    description = []
    keepFilename = False
    useFilename = None
    verifyDescription = True

    # process all global bot args
    # returns a list of non-global args, i.e. args for upload.py
    for arg in pywikibot.handleArgs(*args):
        if arg:
            if arg.startswith('-keep'):
                keepFilename = True
            elif arg.startswith('-filename:'):
                useFilename = arg[10:]
            elif arg.startswith('-noverify'):
                verifyDescription = False
            elif url == u'':
                url = arg
            else:
                description.append(arg)
    description = u' '.join(description)
    bot = UploadRobot(url, description=description, useFilename=useFilename,
                      keepFilename=keepFilename,
                      verifyDescription=verifyDescription)
    bot.run()

if __name__ == "__main__":
    main()
<commit_msg>upload.py: Add option to automatically abort upload on warning<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Script to upload images to wikipedia.

Arguments:

  -keep         Keep the filename as is
  -filename     Target filename
  -noverify     Do not ask for verification of the upload description if one
                is given
  -abortonwarn  Abort upload on warning

If any other arguments are given, the first is the URL or filename to upload,
and the rest is a proposed description to go with the upload. If none of these
are given, the user is asked for the file or URL to upload. The bot will then
upload the image to the wiki.

The script will ask for the location of an image, if not given as a parameter,
and for a description.
"""
#
# (C) Rob W.W. Hooft, Andre Engels 2003-2004
# (C) Pywikibot team, 2003-2014
#
# Distributed under the terms of the MIT license.
#
__version__ = '$Id$'
#

import os
import time
import urllib
import urlparse
import tempfile
import pywikibot
from pywikibot import config

class UploadRobot:
    def __init__(self, url, urlEncoding=None, description=u'',
                 useFilename=None, keepFilename=False,
                 verifyDescription=True, ignoreWarning=False,
                 targetSite=None, uploadByUrl=False, abortOnWarn=False):
        """
        @param ignoreWarning: Set this to True if you want to upload even if
            another file would be overwritten or another mistake would be
            risked.

        """
        self.url = url
        self.urlEncoding = urlEncoding
        self.description = description
        self.useFilename = useFilename
        self.keepFilename = keepFilename
        self.verifyDescription = verifyDescription
        self.ignoreWarning = ignoreWarning
        self.abortOnWarn = abortOnWarn
        if config.upload_to_commons:
            self.targetSite = targetSite or pywikibot.Site('commons',
                                                           'commons')
        else:
            self.targetSite = targetSite or pywikibot.Site()
        self.targetSite.forceLogin()
        self.uploadByUrl = uploadByUrl

    def urlOK(self):
        """Return true if self.url looks like an URL or an existing local file.

        """
        return "://" in self.url or os.path.exists(self.url)

    def read_file_content(self):
        """Return name of temp file in which remote file is saved."""
        pywikibot.output(u'Reading file %s' % self.url)
        resume = False
        rlen = 0
        _contents = None
        dt = 15
        uo = urllib.URLopener()
        retrieved = False

        while not retrieved:
            if resume:
                pywikibot.output(u"Resume download...")
                uo.addheader('Range', 'bytes=%s-' % rlen)

            infile = uo.open(self.url)

            if 'text/html' in infile.info().getheader('Content-Type'):
                pywikibot.output(u"Couldn't download the image: the requested URL was not found on server.")
                return

            content_len = infile.info().getheader('Content-Length')
            accept_ranges = infile.info().getheader('Accept-Ranges') == 'bytes'

            if resume:
                _contents += infile.read()
            else:
                _contents = infile.read()

            infile.close()
            retrieved = True

            if content_len:
                rlen = len(_contents)
                content_len = int(content_len)
                if rlen < content_len:
                    retrieved = False
                    pywikibot.output(
                        u"Connection closed at byte %s (%s left)"
                        % (rlen, content_len))
                    if accept_ranges and rlen > 0:
                        resume = True
                    pywikibot.output(u"Sleeping for %d seconds..." % dt)
                    time.sleep(dt)
                    if dt <= 60:
                        dt += 15
                    elif dt < 360:
                        dt += 60
            else:
                pywikibot.log(
                    u"WARNING: No check length to retrieved data is possible.")
        handle, tempname = tempfile.mkstemp()
        t = os.fdopen(handle, "wb")
        t.write(_contents)
        t.close()
        return tempname

    def process_filename(self):
        """Return base filename portion of self.url."""
        # Isolate the pure name
        filename = self.url
        # Filename may be either a local file path or a URL
        if "://" in filename:
            # extract the path portion of the URL
            filename = urlparse.urlparse(filename).path
        filename = os.path.basename(filename)

        if self.useFilename:
            filename = self.useFilename
        if not self.keepFilename:
            pywikibot.output(
                u"The filename on the target wiki will default to: %s"
                % filename)
            # FIXME: these 2 belong somewhere else, presumably in family
            forbidden = '/'  # to be extended
            allowed_formats = (u'gif', u'jpg', u'jpeg', u'mid', u'midi',
                               u'ogg', u'png', u'svg', u'xcf', u'djvu',
                               u'ogv', u'oga', u'tif', u'tiff')
            # ask until it's valid
            while True:
                newfn = pywikibot.input(
                    u'Enter a better name, or press enter to accept:')
                if newfn == "":
                    newfn = filename
                    break
                ext = os.path.splitext(newfn)[1].lower().strip('.')
                # are any chars in forbidden also in newfn?
                invalid = set(forbidden) & set(newfn)
                if invalid:
                    c = "".join(invalid)
                    pywikibot.output("Invalid character(s): %s. Please try again" % c)
                    continue
                if ext not in allowed_formats:
                    choice = pywikibot.inputChoice(
                        u"File format is not one of [%s], but %s. Continue?"
                        % (u' '.join(allowed_formats), ext),
                        ['yes', 'no'], ['y', 'N'], 'N')
                    if choice == 'n':
                        continue
                break
            if newfn != '':
                filename = newfn
        # A proper description for the submission.
        pywikibot.output(u"The suggested description is:")
        pywikibot.output(self.description)
        if self.verifyDescription:
            newDescription = u''
            choice = pywikibot.inputChoice(
                u'Do you want to change this description?',
                ['Yes', 'No'], ['y', 'N'], 'n')
            if choice == 'y':
                from pywikibot import editor as editarticle
                editor = editarticle.TextEditor()
                newDescription = editor.edit(self.description)
            # if user saved / didn't press Cancel
            if newDescription:
                self.description = newDescription
        return filename

    def upload_image(self, debug=False):
        """Upload the image at self.url to the target wiki.

        Return the filename that was used to upload the image.
        If the upload fails, ask the user whether to try again or not.
        If the user chooses not to retry, return null.

        """
        filename = self.process_filename()

        site = self.targetSite
        imagepage = pywikibot.FilePage(site, filename)  # normalizes filename
        imagepage.text = self.description

        pywikibot.output(u'Uploading file to %s via API....' % site)

        try:
            if self.uploadByUrl:
                site.upload(imagepage, source_url=self.url,
                            ignore_warnings=self.ignoreWarning)
            else:
                if "://" in self.url:
                    temp = self.read_file_content()
                else:
                    temp = self.url
                site.upload(imagepage, source_filename=temp,
                            ignore_warnings=self.ignoreWarning)

        except pywikibot.UploadWarning as warn:
            pywikibot.output(u"We got a warning message: ", newline=False)
            pywikibot.output(str(warn))
            if self.abortOnWarn:
                answer = "N"
            else:
                answer = pywikibot.inputChoice(u"Do you want to ignore?",
                                               ['Yes', 'No'], ['y', 'N'], 'N')
            if answer == "y":
                self.ignoreWarning = 1
                self.keepFilename = True
                return self.upload_image(debug)
            else:
                pywikibot.output(u"Upload aborted.")
                return

        except Exception:
            pywikibot.error("Upload error: ", exc_info=True)

        else:
            #No warning, upload complete.
            pywikibot.output(u"Upload successful.")
            return filename  # data['filename']

    def run(self):
        while not self.urlOK():
            if not self.url:
                pywikibot.output(u'No input filename given')
            else:
                pywikibot.output(u'Invalid input filename given. Try again.')
            self.url = pywikibot.input(u'File or URL where image is now:')
        return self.upload_image()

def main(*args):
    url = u''
    description = []
    keepFilename = False
    useFilename = None
    verifyDescription = True
    abortOnWarn = False

    # process all global bot args
    # returns a list of non-global args, i.e. args for upload.py
    for arg in pywikibot.handleArgs(*args):
        if arg:
            if arg.startswith('-keep'):
                keepFilename = True
            elif arg.startswith('-filename:'):
                useFilename = arg[10:]
            elif arg.startswith('-noverify'):
                verifyDescription = False
            elif arg.startswith('-abortonwarn'):
                abortOnWarn = True
            elif url == u'':
                url = arg
            else:
                description.append(arg)
    description = u' '.join(description)
    bot = UploadRobot(url, description=description, useFilename=useFilename,
                      keepFilename=keepFilename,
                      verifyDescription=verifyDescription,
                      abortOnWarn=abortOnWarn)
    bot.run()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import asyncio
import discord
import time
from   operator import itemgetter
from   discord.ext import commands
from   Cogs import ReadableTime
from   Cogs import DisplayName

# This is the Uptime module. It keeps track of how long the bot's been up

class Fliptime:

	# Init with the bot reference, and a reference to the settings var
	def __init__(self, bot, settings):
		self.bot = bot
		self.settings = settings

	async def message(self, message):
		# Check the message and see if we should allow it - always yes.
		# This module doesn't need to cancel messages.

		# Check for admin status
		isAdmin = message.author.permissions_in(message.channel).administrator
		if not isAdmin:
			checkAdmin = self.settings.getServerStat(message.server, "AdminArray")
			for role in message.author.roles:
				for aRole in checkAdmin:
					# Get the role that corresponds to the id
					if aRole['ID'] == role.id:
						isAdmin = True

		# Check if the message contains the flip chars

		# if message.content.startswith('(') and message.content.endswith('┻'):
		conts = message.content
		face = table = False
		if '(' in conts:
			if ')' in conts or '）' in conts:
				face = True
		if '┻' in conts or '┻' or '╙' in conts or '╨' in conts or '╜' in conts:
			table = True
		if face and table:
			# Contains all characters
			# Table flip - add time
			currentTime = int(time.time())
			cooldownFinal = currentTime+60
			alreadyMuted = self.settings.getUserStat(message.author, message.server, "Muted")
			if not isAdmin:
				# Check if we're muted already
				previousCooldown = self.settings.getUserStat(message.author, message.server, "Cooldown")
				if not previousCooldown:
					if alreadyMuted.lower() == "yes":
						# We're perma-muted - ignore
						return { 'Ignore' : False, 'Delete' : False}

					previousCooldown = 0
				if int(previousCooldown) > currentTime:
					# Already cooling down - add to it.
					cooldownFinal = previousCooldown+60
					coolText = ReadableTime.getReadableTimeBetween(currentTime, cooldownFinal)
					res = '┬─┬ ノ( ゜-゜ノ)  *{}*, I understand that you\'re frustrated, but we still don\'t flip tables here.  Why don\'t you cool down for *{}* instead.'.format(DisplayName.name(message.author), coolText)
				else:
					# Not cooling down - start it
					coolText = ReadableTime.getReadableTimeBetween(currentTime, cooldownFinal)
					res = '┬─┬ ノ( ゜-゜ノ)  *{}*, we don\'t flip tables here.  You should cool down for *{}*'.format(DisplayName.name(message.author), coolText)
				self.settings.setUserStat(message.author, message.server, "Cooldown", cooldownFinal)
				self.settings.setUserStat(message.author, message.server, "Muted", "Yes")
				await self.bot.send_message(message.channel, res)
				return { 'Ignore' : True, 'Delete' : True }

		return { 'Ignore' : False, 'Delete' : False}
<commit_msg>Update Fliptime.py<commit_after>import asyncio
import discord
import time
from   operator import itemgetter
from   discord.ext import commands
from   Cogs import ReadableTime
from   Cogs import DisplayName

# This is the Uptime module. It keeps track of how long the bot's been up

class Fliptime:

	# Init with the bot reference, and a reference to the settings var
	def __init__(self, bot, settings):
		self.bot = bot
		self.settings = settings

	async def message(self, message):
		# Check the message and see if we should allow it - always yes.
		# This module doesn't need to cancel messages.

		# Check for admin status
		isAdmin = message.author.permissions_in(message.channel).administrator
		if not isAdmin:
			checkAdmin = self.settings.getServerStat(message.server, "AdminArray")
			for role in message.author.roles:
				for aRole in checkAdmin:
					# Get the role that corresponds to the id
					if aRole['ID'] == role.id:
						isAdmin = True

		# Check if the message contains the flip chars

		# if message.content.startswith('(') and message.content.endswith('┻'):
		conts = message.content
		face = table = False
		if '(' in conts:
			if ')' in conts or '）' in conts:
				face = True
		if '┻' in conts or '┻' in conts or '╙' in conts or '╨' in conts or '╜' in conts:
			table = True
		if face and table:
			# Contains all characters
			# Table flip - add time
			currentTime = int(time.time())
			cooldownFinal = currentTime+60
			alreadyMuted = self.settings.getUserStat(message.author, message.server, "Muted")
			if not isAdmin:
				# Check if we're muted already
				previousCooldown = self.settings.getUserStat(message.author, message.server, "Cooldown")
				if not previousCooldown:
					if alreadyMuted.lower() == "yes":
						# We're perma-muted - ignore
						return { 'Ignore' : False, 'Delete' : False}

					previousCooldown = 0
				if int(previousCooldown) > currentTime:
					# Already cooling down - add to it.
					cooldownFinal = previousCooldown+60
					coolText = ReadableTime.getReadableTimeBetween(currentTime, cooldownFinal)
					res = '┬─┬ ノ( ゜-゜ノ)  *{}*, I understand that you\'re frustrated, but we still don\'t flip tables here.  Why don\'t you cool down for *{}* instead.'.format(DisplayName.name(message.author), coolText)
				else:
					# Not cooling down - start it
					coolText = ReadableTime.getReadableTimeBetween(currentTime, cooldownFinal)
					res = '┬─┬ ノ( ゜-゜ノ)  *{}*, we don\'t flip tables here.  You should cool down for *{}*'.format(DisplayName.name(message.author), coolText)
				self.settings.setUserStat(message.author, message.server, "Cooldown", cooldownFinal)
				self.settings.setUserStat(message.author, message.server, "Muted", "Yes")
				await self.bot.send_message(message.channel, res)
				return { 'Ignore' : True, 'Delete' : True }

		return { 'Ignore' : False, 'Delete' : False}
<|endoftext|>


<commit_before>
import os

from boto.s3.connection import S3Connection
from boto.s3.key import Key
import click

AWS_BUCKET = os.environ.get('AWS_BUCKET')

def get_or_create_bucket(conn, bucket_name):
    """Get or create an S3 bucket.

    :param conn: boto.s3.connection.S3Connection
    :param bucket_name: string
    :returns: boto.s3.bucket.Bucket
    """
    try:
        bucket = conn.get_bucket(bucket_name)
    except ValueError:
        bucket = conn.create_bucket(bucket_name)

    return bucket

def sizeof_fmt(num):
    """Human readable file size.

    Modified from http://stackoverflow.com/a/1094933/1377021

    :param num: float
    :returns: string
    """
    for unit in ['', 'k', 'm', 'g', 't', 'p', 'e', 'z']:
        if abs(num) < 1024.0:
            return "%.0f%s%s" % (num, unit, 'b')
        num /= 1024.0

    return "%.f%s%s" % (num, 'y', 'b')

@click.command()
@click.argument('directory', type=click.Path(exists=True))
def upload(directory):
    """Upload a directory to S3.

    DIRECTORY: Directory to upload. Required.
    """
    conn = S3Connection()
    bucket = get_or_create_bucket(conn, AWS_BUCKET)

    files = []
    total_size = 0

    for (dirpath, dirnames, filenames) in os.walk(directory):
        files.extend([f for f in filenames if not f[0] == '.'])

    print 'Found', len(files), 'files to upload to s3://' + AWS_BUCKET

    for path in files:
        full_path = os.path.join(directory, path)
        filesize = os.path.getsize(full_path)
        total_size += filesize

        print 'Uploading', path, '-', sizeof_fmt(filesize)

        k = Key(bucket)
        k.key = path
        k.set_contents_from_filename(full_path)

    print 'Done. Uploaded', sizeof_fmt(total_size)

if __name__ == '__main__':
    upload()
<commit_msg>add check for bucket variable<commit_after>
import os

from boto.s3.connection import S3Connection
from boto.s3.key import Key
import click

AWS_BUCKET = os.environ.get('AWS_BUCKET')

def get_or_create_bucket(conn, bucket_name):
    """Get or create an S3 bucket.

    :param conn: boto.s3.connection.S3Connection
    :param bucket_name: string
    :returns: boto.s3.bucket.Bucket
    """
    try:
        bucket = conn.get_bucket(bucket_name)
    except ValueError:
        bucket = conn.create_bucket(bucket_name)

    return bucket

def sizeof_fmt(num):
    """Human readable file size.

    Modified from http://stackoverflow.com/a/1094933/1377021

    :param num: float
    :returns: string
    """
    for unit in ['', 'k', 'm', 'g', 't', 'p', 'e', 'z']:
        if abs(num) < 1024.0:
            return "%.0f%s%s" % (num, unit, 'b')
        num /= 1024.0

    return "%.f%s%s" % (num, 'y', 'b')

@click.command()
@click.argument('directory', type=click.Path(exists=True))
def upload(directory):
    """Upload a directory to S3.

    DIRECTORY: Directory to upload. Required.
    """
    if not AWS_BUCKET:
        print 'AWS_BUCKET environment variable not set. Exiting.'
        return

    conn = S3Connection()
    bucket = get_or_create_bucket(conn, AWS_BUCKET)

    files = []
    total_size = 0

    for (dirpath, dirnames, filenames) in os.walk(directory):
        files.extend([f for f in filenames if not f[0] == '.'])

    print 'Found', len(files), 'files to upload to s3://' + AWS_BUCKET

    for path in files:
        full_path = os.path.join(directory, path)
        filesize = os.path.getsize(full_path)
        total_size += filesize

        print 'Uploading', path, '-', sizeof_fmt(filesize)

        k = Key(bucket)
        k.key = path
        k.set_contents_from_filename(full_path)

    print 'Done. Uploaded', sizeof_fmt(total_size)

if __name__ == '__main__':
    upload()
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-

"""
This modules provides functions to deal with scores stored on the Web server
"""

import os.path
import re
import json

def init_scores(name):
    """
    Initialize the local scores file
    """
    if not os.path.isfile(name):
        dirname = os.path.dirname(name)
        if not os.path.exists(dirname):
            try:
                os.makedirs(dirname)
            except OSError as e:
                pass

        with open(name, 'w') as f:
            f.write(json.dumps([]))

def sanitize_score(sc):
    """
    Sanitize a score from an external source. The score should be a tuple of
    ``user``, ``score``, and ``text``.
    """
    u, s, t = sc[:3]

    try:
        s = int(s)
    except ValueError:
        s = 0
    except TypeError:
        s = 0

    u = re.sub(r'\W+', '', str(u).strip())[:40]
    t = re.sub(r'[^\w\., ]+', '', str(t))[:80]
    return (u, s, t)

def sanitize_scores(scs):
    """
    Sanitize scores from an external source, and filter out those with an empty
    username and a null score.
    """
    return [s for s in map(sanitize_score, scs) if s[0] and s[1] > 0]

def merge_scores(scs, fname):
    """
    Merge local scores with the given ones, and save the new list in the local
    file. Duplicate scores (same user, score and text) are not preserved, and
    if a new score is the same than a previous one, the new one is taking
    precedence over the former one (i.e. it'll be the first in the
    leaderboard).
    """
    scs = list(map(list, sanitize_scores(scs)))
    scs.sort(key=lambda s: s[1], reverse=True)

    init_scores(fname)
    with open(fname, 'r') as f:
        scores = json.loads(f.read())

    final_scores = []
    while scores or scs:
        if not scores:
            final_scores += scs
            break

        if not scs:
            final_scores += scores
            break

        s1 = scores[0]
        s2 = scs[0]

        if s1 == s2:
            # don't add duplicates in the file
            scs.pop(0)
            continue

        coll = scores if s1[1] > s2[1] else scs
        final_scores.append(coll.pop(0))

    with open(fname, 'w') as f:
        f.write(json.dumps(final_scores))
<commit_msg>Fail if os.makedirs fails.<commit_after># -*- coding: UTF-8 -*-

"""
This modules provides functions to deal with scores stored on the Web server
"""

import os.path
import re
import json

def init_scores(name):
    """
    Initialize the local scores file
    """
    if not os.path.isfile(name):
        dirname = os.path.dirname(name)
        if not os.path.exists(dirname):
            os.makedirs(dirname)

        with open(name, 'w') as f:
            f.write(json.dumps([]))

def sanitize_score(sc):
    """
    Sanitize a score from an external source. The score should be a tuple of
    ``user``, ``score``, and ``text``.
    """
    u, s, t = sc[:3]

    try:
        s = int(s)
    except ValueError:
        s = 0
    except TypeError:
        s = 0

    u = re.sub(r'\W+', '', str(u).strip())[:40]
    t = re.sub(r'[^\w\., ]+', '', str(t))[:80]
    return (u, s, t)

def sanitize_scores(scs):
    """
    Sanitize scores from an external source, and filter out those with an empty
    username and a null score.
    """
    return [s for s in map(sanitize_score, scs) if s[0] and s[1] > 0]

def merge_scores(scs, fname):
    """
    Merge local scores with the given ones, and save the new list in the local
    file. Duplicate scores (same user, score and text) are not preserved, and
    if a new score is the same than a previous one, the new one is taking
    precedence over the former one (i.e. it'll be the first in the
    leaderboard).
    """
    scs = list(map(list, sanitize_scores(scs)))
    scs.sort(key=lambda s: s[1], reverse=True)

    init_scores(fname)
    with open(fname, 'r') as f:
        scores = json.loads(f.read())

    final_scores = []
    while scores or scs:
        if not scores:
            final_scores += scs
            break

        if not scs:
            final_scores += scores
            break

        s1 = scores[0]
        s2 = scs[0]

        if s1 == s2:
            # don't add duplicates in the file
            scs.pop(0)
            continue

        coll = scores if s1[1] > s2[1] else scs
        final_scores.append(coll.pop(0))

    with open(fname, 'w') as f:
        f.write(json.dumps(final_scores))
<|endoftext|>


<commit_before>import datetime
import pytz
from   Cogs import FuzzySearch

def setup(bot):
	# This module isn't actually a cog
    return

def getClockForTime(time_string):
	# Assumes a HH:MM PP format
	print("Clock called: " + time_string)
	try:
		time = time_string.split(" ")
		time = time[0].split(":")
		hour = int(time[0])
		minute = int(time[1])
	except:
		return ""
	print(hour, minute)
	clock_string = ""
	if min > 44:
		clock_string = str(hour + 1) if hour < 12 else "1"
	elif min > 14:
		clock_string = str(hour) + "30"
	else:
		clock_string = str(hour)
	print(clock_string)
	return time_string +" :clock" + clock_string + ":"

def getUserTime(member, settings, time = None, strft = "%Y-%m-%d %I:%M %p", clock = True):
	# Returns a dict representing the time from the passed member's perspective
	offset = settings.getGlobalUserStat(member, "TimeZone")
	if offset == None:
		offset = settings.getGlobalUserStat(member, "UTCOffset")
	if offset == None:
		# No offset or tz - return UTC
		if clock:
			t = getClockForTime(time.strftime(strft))
		else:
			t = time.strftime(strft)
		return { "zone" : 'UTC', "time" : t }

	# At this point - we need to determine if we have an offset - or possibly a timezone passed
	t = getTimeFromTZ(offset, time, strft, clock)
	if t == None:
		# We did not get a zone
		t = getTimeFromOffset(offset, time, strft, clock)
	return t

def getTimeFromOffset(offset, t = None, strft = "%Y-%m-%d %I:%M %p", clock = True):
	offset = offset.replace('+', '')
	# Split time string by : and get hour/minute values
	try:
		hours, minutes = map(int, offset.split(':'))
	except Exception:
		try:
			hours = int(offset)
			minutes = 0
		except Exception:
			return None
			# await ctx.channel.send('Offset has to be in +-H:M!')
			# return
	msg = 'UTC'
	# Get the time
	if t == None:
		t = datetime.datetime.utcnow()
	# Apply offset
	if hours > 0:
		# Apply positive offset
		msg += '+{}'.format(offset)
		td = datetime.timedelta(hours=hours, minutes=minutes)
		newTime = t + td
	elif hours < 0:
		# Apply negative offset
		msg += '{}'.format(offset)
		td = datetime.timedelta(hours=(-1*hours), minutes=(-1*minutes))
		newTime = t - td
	else:
		# No offset
		newTime = t
	if clock:
		ti = getClockForTime(newTime.strftime(strft))
	else:
		ti = newTime.strftime(strft)
	return { "zone" : msg, "time" : ti }

def getTimeFromTZ(tz, t = None, strft = "%Y-%m-%d %I:%M %p", clock = True):
	# Assume sanitized zones - as they're pulled from pytz
	# Let's get the timezone list
	tz_list = FuzzySearch.search(tz, pytz.all_timezones, None, 3)
	if not tz_list[0]['Ratio'] == 1:
		# We didn't find a complete match
		return None
	zone = pytz.timezone(tz_list[0]['Item'])
	if t == None:
		zone_now = datetime.datetime.now(zone)
	else:
		zone_now = pytz.utc.localize(t, is_dst=None).astimezone(zone)
		#zone_now = t.astimezone(zone)
	if clock:
		ti = getClockForTime(zone_now.strftime(strft))
	else:
		ti = zone_now.strftime(strft)
	return { "zone" : tz_list[0]['Item'], "time" : ti}
<commit_msg>Update UserTime.py<commit_after>import datetime
import pytz
from   Cogs import FuzzySearch

def setup(bot):
	# This module isn't actually a cog
    return

def getClockForTime(time_string):
	# Assumes a HH:MM PP format
	print("Clock called: " + time_string)
	try:
		time = time_string.split(" ")
		if len(time) == 2:
			time = time[0].split(":")
		elif len(time) == 3:
			time = time[1].split(":")
		else:
			return time_string
		hour = int(time[0])
		minute = int(time[1])
	except:
		return time_string
	print(hour, minute)
	clock_string = ""
	if min > 44:
		clock_string = str(hour + 1) if hour < 12 else "1"
	elif min > 14:
		clock_string = str(hour) + "30"
	else:
		clock_string = str(hour)
	print(clock_string)
	return time_string +" :clock" + clock_string + ":"

def getUserTime(member, settings, time = None, strft = "%Y-%m-%d %I:%M %p", clock = True):
	# Returns a dict representing the time from the passed member's perspective
	offset = settings.getGlobalUserStat(member, "TimeZone")
	if offset == None:
		offset = settings.getGlobalUserStat(member, "UTCOffset")
	if offset == None:
		# No offset or tz - return UTC
		if clock:
			t = getClockForTime(time.strftime(strft))
		else:
			t = time.strftime(strft)
		return { "zone" : 'UTC', "time" : t }

	# At this point - we need to determine if we have an offset - or possibly a timezone passed
	t = getTimeFromTZ(offset, time, strft, clock)
	if t == None:
		# We did not get a zone
		t = getTimeFromOffset(offset, time, strft, clock)
	return t

def getTimeFromOffset(offset, t = None, strft = "%Y-%m-%d %I:%M %p", clock = True):
	offset = offset.replace('+', '')
	# Split time string by : and get hour/minute values
	try:
		hours, minutes = map(int, offset.split(':'))
	except Exception:
		try:
			hours = int(offset)
			minutes = 0
		except Exception:
			return None
			# await ctx.channel.send('Offset has to be in +-H:M!')
			# return
	msg = 'UTC'
	# Get the time
	if t == None:
		t = datetime.datetime.utcnow()
	# Apply offset
	if hours > 0:
		# Apply positive offset
		msg += '+{}'.format(offset)
		td = datetime.timedelta(hours=hours, minutes=minutes)
		newTime = t + td
	elif hours < 0:
		# Apply negative offset
		msg += '{}'.format(offset)
		td = datetime.timedelta(hours=(-1*hours), minutes=(-1*minutes))
		newTime = t - td
	else:
		# No offset
		newTime = t
	if clock:
		ti = getClockForTime(newTime.strftime(strft))
	else:
		ti = newTime.strftime(strft)
	return { "zone" : msg, "time" : ti }

def getTimeFromTZ(tz, t = None, strft = "%Y-%m-%d %I:%M %p", clock = True):
	# Assume sanitized zones - as they're pulled from pytz
	# Let's get the timezone list
	tz_list = FuzzySearch.search(tz, pytz.all_timezones, None, 3)
	if not tz_list[0]['Ratio'] == 1:
		# We didn't find a complete match
		return None
	zone = pytz.timezone(tz_list[0]['Item'])
	if t == None:
		zone_now = datetime.datetime.now(zone)
	else:
		zone_now = pytz.utc.localize(t, is_dst=None).astimezone(zone)
		#zone_now = t.astimezone(zone)
	if clock:
		ti = getClockForTime(zone_now.strftime(strft))
	else:
		ti = zone_now.strftime(strft)
	return { "zone" : tz_list[0]['Item'], "time" : ti}
<|endoftext|>