<commit_before># Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.

# Python
import re

# Django
from django.core.exceptions import FieldError, ValidationError
from django.db import models
from django.db.models import Q
from django.db.models.fields import FieldDoesNotExist
from django.db.models.fields.related import ForeignObjectRel, ManyToManyField, ForeignKey
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericForeignKey
from django.utils.encoding import force_text
from django.utils.translation import ugettext_lazy as _

# Django REST Framework
from rest_framework.exceptions import ParseError, PermissionDenied
from rest_framework.filters import BaseFilterBackend

# Ansible Tower
from awx.main.utils import get_type_for_model, to_python_boolean
from awx.main.models.rbac import RoleAncestorEntry

class MongoFilterBackend(BaseFilterBackend):

    # FIX: Note that MongoEngine can't use the filter backends from DRF
    def filter_queryset(self, request, queryset, view):
        return queryset

class TypeFilterBackend(BaseFilterBackend):
    '''
    Filter on type field now returned with all objects.
    '''

    def filter_queryset(self, request, queryset, view):
        try:
            types = None
            for key, value in request.query_params.items():
                if key == 'type':
                    if ',' in value:
                        types = value.split(',')
                    else:
                        types = (value,)
            if types:
                types_map = {}
                for ct in ContentType.objects.filter(Q(app_label='main') | Q(app_label='auth', model='user')):
                    ct_model = ct.model_class()
                    if not ct_model:
                        continue
                    ct_type = get_type_for_model(ct_model)
                    types_map[ct_type] = ct.pk
                model = queryset.model
                model_type = get_type_for_model(model)
                if 'polymorphic_ctype' in model._meta.get_all_field_names():
                    types_pks = set([v for k,v in types_map.items() if k in types])
                    queryset = queryset.filter(polymorphic_ctype_id__in=types_pks)
                elif model_type in types:
                    queryset = queryset
                else:
                    queryset = queryset.none()
            return queryset
        except FieldError as e:
            # Return a 400 for invalid field names.
            raise ParseError(*e.args)

class FieldLookupBackend(BaseFilterBackend):
    '''
    Filter using field lookups provided via query string parameters.
    '''

    RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by',
                      'search', 'type')

    SUPPORTED_LOOKUPS = ('exact', 'iexact', 'contains', 'icontains',
                         'startswith', 'istartswith', 'endswith', 'iendswith',
                         'regex', 'iregex', 'gt', 'gte', 'lt', 'lte', 'in',
                         'isnull', 'search')

    def get_field_from_lookup(self, model, lookup):
        field = None
        parts = lookup.split('__')
        if parts and parts[-1] not in self.SUPPORTED_LOOKUPS:
            parts.append('exact')
        # FIXME: Could build up a list of models used across relationships, use
        # those lookups combined with request.user.get_queryset(Model) to make
        # sure user cannot query using objects he could not view.
        new_parts = []

        for name in parts[:-1]:
            # HACK: Make project and inventory source filtering by old field names work for backwards compatibility.
            if model._meta.object_name in ('Project', 'InventorySource'):
                name = {
                    'current_update': 'current_job',
                    'last_update': 'last_job',
                    'last_update_failed': 'last_job_failed',
                    'last_updated': 'last_job_run',
                }.get(name, name)

            if name == 'type' and 'polymorphic_ctype' in model._meta.get_all_field_names():
                name = 'polymorphic_ctype'
                new_parts.append('polymorphic_ctype__model')
            else:
                new_parts.append(name)

            if name in getattr(model, 'PASSWORD_FIELDS', ()):
                raise PermissionDenied(_('Filtering on password fields is not allowed.'))
            elif name == 'pk':
                field = model._meta.pk
            else:
                name_alt = name.replace("_", "")
                if name_alt in model._meta.fields_map.keys():
                    field = model._meta.fields_map[name_alt]
                    new_parts.pop()
                    new_parts.append(name_alt)
                else:
                    field = model._meta.get_field_by_name(name)[0]
                if isinstance(field, ForeignObjectRel) and getattr(field.field, '__prevent_search__', False):
                    raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
                elif getattr(field, '__prevent_search__', False):
                    raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
            model = getattr(field, 'related_model', None) or field.model

        if parts:
            new_parts.append(parts[-1])
        new_lookup = '__'.join(new_parts)
        return field, new_lookup

    def to_python_related(self, value):
        value = force_text(value)
        if value.lower() in ('none', 'null'):
            return None
        else:
            return int(value)

    def value_to_python_for_field(self, field, value):
        if isinstance(field, models.NullBooleanField):
            return to_python_boolean(value, allow_none=True)
        elif isinstance(field, models.BooleanField):
            return to_python_boolean(value)
        elif isinstance(field, (ForeignObjectRel, ManyToManyField, GenericForeignKey, ForeignKey)):
            return self.to_python_related(value)
        else:
            return field.to_python(value)

    def value_to_python(self, model, lookup, value):
        field, new_lookup = self.get_field_from_lookup(model, lookup)

        # Type names are stored without underscores internally, but are presented and
        # and serialized over the API containing underscores so we remove `_`
        # for polymorphic_ctype__model lookups.
        if new_lookup.startswith('polymorphic_ctype__model'):
            value = value.replace('_','')
        elif new_lookup.endswith('__isnull'):
            value = to_python_boolean(value)
        elif new_lookup.endswith('__in'):
            items = []
            if not value:
                raise ValueError('cannot provide empty value for __in')
            for item in value.split(','):
                items.append(self.value_to_python_for_field(field, item))
            value = items
        elif new_lookup.endswith('__regex') or new_lookup.endswith('__iregex'):
            try:
                re.compile(value)
            except re.error as e:
                raise ValueError(e.args[0])
        elif new_lookup.endswith('__search'):
            related_model = getattr(field, 'related_model', None)
            if not related_model:
                raise ValueError('%s is not searchable' % new_lookup[:-8])
            new_lookups = []
            for rm_field in related_model._meta.fields:
                if rm_field.name in ('username', 'first_name', 'last_name', 'email', 'name', 'description'):
                    new_lookups.append('{}__{}__icontains'.format(new_lookup[:-8], rm_field.name))
            return value, new_lookups
        else:
            value = self.value_to_python_for_field(field, value)
        return value, new_lookup

    def filter_queryset(self, request, queryset, view):
        try:
            # Apply filters specified via query_params. Each entry in the lists
            # below is (negate, field, value).
            and_filters = []
            or_filters = []
            chain_filters = []
            role_filters = []
            search_filters = []
            for key, values in request.query_params.lists():
                if key in self.RESERVED_NAMES:
                    continue

                # HACK: Make job event filtering by host name mostly work even
                # when not capturing job event hosts M2M.
                if queryset.model._meta.object_name == 'JobEvent' and key.startswith('hosts__name'):
                    key = key.replace('hosts__name', 'or__host__name')
                    or_filters.append((False, 'host__name__isnull', True))

                # Custom __int filter suffix (internal use only).
                q_int = False
                if key.endswith('__int'):
                    key = key[:-5]
                    q_int = True

                # RBAC filtering
                if key == 'role_level':
                    role_filters.append(values[0])
                    continue

                # Search across related objects.
                if key.endswith('__search'):
                    for value in values:
                        for search_term in force_text(value).replace(',', ' ').split():
                            search_value, new_keys = self.value_to_python(queryset.model, key, search_term)
                            assert isinstance(new_keys, list)
                            for new_key in new_keys:
                                search_filters.append((new_key, search_value))
                    continue

                # Custom chain__ and or__ filters, mutually exclusive (both can
                # precede not__).
                q_chain = False
                q_or = False
                if key.startswith('chain__'):
                    key = key[7:]
                    q_chain = True
                elif key.startswith('or__'):
                    key = key[4:]
                    q_or = True

                # Custom not__ filter prefix.
                q_not = False
                if key.startswith('not__'):
                    key = key[5:]
                    q_not = True

                # Convert value(s) to python and add to the appropriate list.
                for value in values:
                    if q_int:
                        value = int(value)
                    value, new_key = self.value_to_python(queryset.model, key, value)
                    if q_chain:
                        chain_filters.append((q_not, new_key, value))
                    elif q_or:
                        or_filters.append((q_not, new_key, value))
                    else:
                        and_filters.append((q_not, new_key, value))

            # Now build Q objects for database query filter.
            if and_filters or or_filters or chain_filters or role_filters or search_filters:
                args = []
                for n, k, v in and_filters:
                    if n:
                        args.append(~Q(**{k:v}))
                    else:
                        args.append(Q(**{k:v}))
                for role_name in role_filters:
                    args.append(
                        Q(pk__in=RoleAncestorEntry.objects.filter(
                            ancestor__in=request.user.roles.all(),
                            content_type_id=ContentType.objects.get_for_model(queryset.model).id,
                            role_field=role_name
                        ).values_list('object_id').distinct())
                    )
                if or_filters:
                    q = Q()
                    for n,k,v in or_filters:
                        if n:
                            q |= ~Q(**{k:v})
                        else:
                            q |= Q(**{k:v})
                    args.append(q)
                if search_filters:
                    q = Q()
                    for k,v in search_filters:
                        q |= Q(**{k:v})
                    args.append(q)
                for n,k,v in chain_filters:
                    if n:
                        q = ~Q(**{k:v})
                    else:
                        q = Q(**{k:v})
                    queryset = queryset.filter(q)
                queryset = queryset.filter(*args).distinct()
            return queryset
        except (FieldError, FieldDoesNotExist, ValueError, TypeError) as e:
            raise ParseError(e.args[0])
        except ValidationError as e:
            raise ParseError(e.messages)

class OrderByBackend(BaseFilterBackend):
    '''
    Filter to apply ordering based on query string parameters.
    '''

    def filter_queryset(self, request, queryset, view):
        try:
            order_by = None
            for key, value in request.query_params.items():
                if key in ('order', 'order_by'):
                    order_by = value
                    if ',' in value:
                        order_by = value.split(',')
                    else:
                        order_by = (value,)
            if order_by:
                # Special handling of the type field for ordering. In this
                # case, we're not sorting exactly on the type field, but
                # given the limited number of views with multiple types,
                # sorting on polymorphic_ctype.model is effectively the same.
                new_order_by = []
                if 'polymorphic_ctype' in queryset.model._meta.get_all_field_names():
                    for field in order_by:
                        if field == 'type':
                            new_order_by.append('polymorphic_ctype__model')
                        elif field == '-type':
                            new_order_by.append('-polymorphic_ctype__model')
                        else:
                            new_order_by.append(field)
                else:
                    for field in order_by:
                        if field not in ('type', '-type'):
                            new_order_by.append(field)
                queryset = queryset.order_by(*new_order_by)
            return queryset
        except FieldError as e:
            # Return a 400 for invalid field names.
            raise ParseError(*e.args)
<commit_msg>Remove unicode marker from filter error message.<commit_after># Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.

# Python
import re
import json

# Django
from django.core.exceptions import FieldError, ValidationError
from django.db import models
from django.db.models import Q
from django.db.models.fields import FieldDoesNotExist
from django.db.models.fields.related import ForeignObjectRel, ManyToManyField, ForeignKey
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericForeignKey
from django.utils.encoding import force_text
from django.utils.translation import ugettext_lazy as _

# Django REST Framework
from rest_framework.exceptions import ParseError, PermissionDenied
from rest_framework.filters import BaseFilterBackend

# Ansible Tower
from awx.main.utils import get_type_for_model, to_python_boolean
from awx.main.models.rbac import RoleAncestorEntry

class MongoFilterBackend(BaseFilterBackend):

    # FIX: Note that MongoEngine can't use the filter backends from DRF
    def filter_queryset(self, request, queryset, view):
        return queryset

class TypeFilterBackend(BaseFilterBackend):
    '''
    Filter on type field now returned with all objects.
    '''

    def filter_queryset(self, request, queryset, view):
        try:
            types = None
            for key, value in request.query_params.items():
                if key == 'type':
                    if ',' in value:
                        types = value.split(',')
                    else:
                        types = (value,)
            if types:
                types_map = {}
                for ct in ContentType.objects.filter(Q(app_label='main') | Q(app_label='auth', model='user')):
                    ct_model = ct.model_class()
                    if not ct_model:
                        continue
                    ct_type = get_type_for_model(ct_model)
                    types_map[ct_type] = ct.pk
                model = queryset.model
                model_type = get_type_for_model(model)
                if 'polymorphic_ctype' in model._meta.get_all_field_names():
                    types_pks = set([v for k,v in types_map.items() if k in types])
                    queryset = queryset.filter(polymorphic_ctype_id__in=types_pks)
                elif model_type in types:
                    queryset = queryset
                else:
                    queryset = queryset.none()
            return queryset
        except FieldError as e:
            # Return a 400 for invalid field names.
            raise ParseError(*e.args)

class FieldLookupBackend(BaseFilterBackend):
    '''
    Filter using field lookups provided via query string parameters.
    '''

    RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by',
                      'search', 'type')

    SUPPORTED_LOOKUPS = ('exact', 'iexact', 'contains', 'icontains',
                         'startswith', 'istartswith', 'endswith', 'iendswith',
                         'regex', 'iregex', 'gt', 'gte', 'lt', 'lte', 'in',
                         'isnull', 'search')

    def get_field_from_lookup(self, model, lookup):
        field = None
        parts = lookup.split('__')
        if parts and parts[-1] not in self.SUPPORTED_LOOKUPS:
            parts.append('exact')
        # FIXME: Could build up a list of models used across relationships, use
        # those lookups combined with request.user.get_queryset(Model) to make
        # sure user cannot query using objects he could not view.
        new_parts = []

        for name in parts[:-1]:
            # HACK: Make project and inventory source filtering by old field names work for backwards compatibility.
            if model._meta.object_name in ('Project', 'InventorySource'):
                name = {
                    'current_update': 'current_job',
                    'last_update': 'last_job',
                    'last_update_failed': 'last_job_failed',
                    'last_updated': 'last_job_run',
                }.get(name, name)

            if name == 'type' and 'polymorphic_ctype' in model._meta.get_all_field_names():
                name = 'polymorphic_ctype'
                new_parts.append('polymorphic_ctype__model')
            else:
                new_parts.append(name)

            if name in getattr(model, 'PASSWORD_FIELDS', ()):
                raise PermissionDenied(_('Filtering on password fields is not allowed.'))
            elif name == 'pk':
                field = model._meta.pk
            else:
                name_alt = name.replace("_", "")
                if name_alt in model._meta.fields_map.keys():
                    field = model._meta.fields_map[name_alt]
                    new_parts.pop()
                    new_parts.append(name_alt)
                else:
                    field = model._meta.get_field_by_name(name)[0]
                if isinstance(field, ForeignObjectRel) and getattr(field.field, '__prevent_search__', False):
                    raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
                elif getattr(field, '__prevent_search__', False):
                    raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
            model = getattr(field, 'related_model', None) or field.model

        if parts:
            new_parts.append(parts[-1])
        new_lookup = '__'.join(new_parts)
        return field, new_lookup

    def to_python_related(self, value):
        value = force_text(value)
        if value.lower() in ('none', 'null'):
            return None
        else:
            return int(value)

    def value_to_python_for_field(self, field, value):
        if isinstance(field, models.NullBooleanField):
            return to_python_boolean(value, allow_none=True)
        elif isinstance(field, models.BooleanField):
            return to_python_boolean(value)
        elif isinstance(field, (ForeignObjectRel, ManyToManyField, GenericForeignKey, ForeignKey)):
            return self.to_python_related(value)
        else:
            return field.to_python(value)

    def value_to_python(self, model, lookup, value):
        field, new_lookup = self.get_field_from_lookup(model, lookup)

        # Type names are stored without underscores internally, but are presented and
        # and serialized over the API containing underscores so we remove `_`
        # for polymorphic_ctype__model lookups.
        if new_lookup.startswith('polymorphic_ctype__model'):
            value = value.replace('_','')
        elif new_lookup.endswith('__isnull'):
            value = to_python_boolean(value)
        elif new_lookup.endswith('__in'):
            items = []
            if not value:
                raise ValueError('cannot provide empty value for __in')
            for item in value.split(','):
                items.append(self.value_to_python_for_field(field, item))
            value = items
        elif new_lookup.endswith('__regex') or new_lookup.endswith('__iregex'):
            try:
                re.compile(value)
            except re.error as e:
                raise ValueError(e.args[0])
        elif new_lookup.endswith('__search'):
            related_model = getattr(field, 'related_model', None)
            if not related_model:
                raise ValueError('%s is not searchable' % new_lookup[:-8])
            new_lookups = []
            for rm_field in related_model._meta.fields:
                if rm_field.name in ('username', 'first_name', 'last_name', 'email', 'name', 'description'):
                    new_lookups.append('{}__{}__icontains'.format(new_lookup[:-8], rm_field.name))
            return value, new_lookups
        else:
            value = self.value_to_python_for_field(field, value)
        return value, new_lookup

    def filter_queryset(self, request, queryset, view):
        try:
            # Apply filters specified via query_params. Each entry in the lists
            # below is (negate, field, value).
            and_filters = []
            or_filters = []
            chain_filters = []
            role_filters = []
            search_filters = []
            for key, values in request.query_params.lists():
                if key in self.RESERVED_NAMES:
                    continue

                # HACK: Make job event filtering by host name mostly work even
                # when not capturing job event hosts M2M.
                if queryset.model._meta.object_name == 'JobEvent' and key.startswith('hosts__name'):
                    key = key.replace('hosts__name', 'or__host__name')
                    or_filters.append((False, 'host__name__isnull', True))

                # Custom __int filter suffix (internal use only).
                q_int = False
                if key.endswith('__int'):
                    key = key[:-5]
                    q_int = True

                # RBAC filtering
                if key == 'role_level':
                    role_filters.append(values[0])
                    continue

                # Search across related objects.
                if key.endswith('__search'):
                    for value in values:
                        for search_term in force_text(value).replace(',', ' ').split():
                            search_value, new_keys = self.value_to_python(queryset.model, key, search_term)
                            assert isinstance(new_keys, list)
                            for new_key in new_keys:
                                search_filters.append((new_key, search_value))
                    continue

                # Custom chain__ and or__ filters, mutually exclusive (both can
                # precede not__).
                q_chain = False
                q_or = False
                if key.startswith('chain__'):
                    key = key[7:]
                    q_chain = True
                elif key.startswith('or__'):
                    key = key[4:]
                    q_or = True

                # Custom not__ filter prefix.
                q_not = False
                if key.startswith('not__'):
                    key = key[5:]
                    q_not = True

                # Convert value(s) to python and add to the appropriate list.
                for value in values:
                    if q_int:
                        value = int(value)
                    value, new_key = self.value_to_python(queryset.model, key, value)
                    if q_chain:
                        chain_filters.append((q_not, new_key, value))
                    elif q_or:
                        or_filters.append((q_not, new_key, value))
                    else:
                        and_filters.append((q_not, new_key, value))

            # Now build Q objects for database query filter.
            if and_filters or or_filters or chain_filters or role_filters or search_filters:
                args = []
                for n, k, v in and_filters:
                    if n:
                        args.append(~Q(**{k:v}))
                    else:
                        args.append(Q(**{k:v}))
                for role_name in role_filters:
                    args.append(
                        Q(pk__in=RoleAncestorEntry.objects.filter(
                            ancestor__in=request.user.roles.all(),
                            content_type_id=ContentType.objects.get_for_model(queryset.model).id,
                            role_field=role_name
                        ).values_list('object_id').distinct())
                    )
                if or_filters:
                    q = Q()
                    for n,k,v in or_filters:
                        if n:
                            q |= ~Q(**{k:v})
                        else:
                            q |= Q(**{k:v})
                    args.append(q)
                if search_filters:
                    q = Q()
                    for k,v in search_filters:
                        q |= Q(**{k:v})
                    args.append(q)
                for n,k,v in chain_filters:
                    if n:
                        q = ~Q(**{k:v})
                    else:
                        q = Q(**{k:v})
                    queryset = queryset.filter(q)
                queryset = queryset.filter(*args).distinct()
            return queryset
        except (FieldError, FieldDoesNotExist, ValueError, TypeError) as e:
            raise ParseError(e.args[0])
        except ValidationError as e:
            raise ParseError(json.dumps(e.messages, ensure_ascii=False))

class OrderByBackend(BaseFilterBackend):
    '''
    Filter to apply ordering based on query string parameters.
    '''

    def filter_queryset(self, request, queryset, view):
        try:
            order_by = None
            for key, value in request.query_params.items():
                if key in ('order', 'order_by'):
                    order_by = value
                    if ',' in value:
                        order_by = value.split(',')
                    else:
                        order_by = (value,)
            if order_by:
                # Special handling of the type field for ordering. In this
                # case, we're not sorting exactly on the type field, but
                # given the limited number of views with multiple types,
                # sorting on polymorphic_ctype.model is effectively the same.
                new_order_by = []
                if 'polymorphic_ctype' in queryset.model._meta.get_all_field_names():
                    for field in order_by:
                        if field == 'type':
                            new_order_by.append('polymorphic_ctype__model')
                        elif field == '-type':
                            new_order_by.append('-polymorphic_ctype__model')
                        else:
                            new_order_by.append(field)
                else:
                    for field in order_by:
                        if field not in ('type', '-type'):
                            new_order_by.append(field)
                queryset = queryset.order_by(*new_order_by)
            return queryset
        except FieldError as e:
            # Return a 400 for invalid field names.
            raise ParseError(*e.args)
<|endoftext|>


<commit_before>
## Notes

# Url to connect tht bot to a server: https://discordapp.com/oauth2/authorize?&client_id=324471059816382464&scope=bot
# id 324471059816382464

# Basic informations. To change if you want to setup your own Bot.

__program__ = "FoxliBot"
__version__ = "2.0b"

from pprint import pprint
from inspect import getmembers
import discord
import asyncio
import aiohttp
from discord import opus
from discord.ext import commands
from bs4 import BeautifulSoup as BS
import random
import os
from json import load as jload

OPUS_LIBS = ['libopus-0.x86.dll', 'libopus-0.x64.dll', 'libopus-0.dll', 'libopus.so.0', 'libopus.0.dylib']

description = '''An example bot to showcase the discord.ext.commands extension
module.
There are a number of utility commands being showcased here.'''

name='foxlibot'

prefix = '**'

def read_key():
    """
    Read a bot's key JSON to get it's token
    Keys must be stored in the key folder and have a basic 'key':'<keytext>' object
    """
    with open('keys/foxlibot.key', 'r') as f:
        datum = jload(f)
        key = datum.get("key", "")
        if not key:
            raise IOError("Key not found in JSON keyfile")
        return key
    return None

global datadir
datadir = os.path.dirname("./data/")
if not os.path.exists(datadir):
    os.makedirs(datadir)
audio = os.path.dirname("./data/audio")
if not os.path.exists(audio):
    os.makedirs(audio)

def logMsg():
    dir = os.path.dirname("./logs/")
    if not os.path.exists(dir):
            os.makedirs(dir)
    today = datetime.datetime.now()
    logsFile=open(dir+"/"+str(today)+".txt","a")
    log = "oui"
    logsFile.write(log)
    print(log,end='')

def get_cmd_message(uid=None):
    """
    Search the given channel for the last message
    aka: the command that was given to the bot
    """
    if len(bot.messages) == 0:
        raise Exception("Wat")
    c_uid = lambda u, v: True
    if uid is not None:
        c_uid = lambda u, v: u == v
    res = [msg for msg in bot.messages
            if msg.channel == chan
            and msg.author.id != bot.user.id
            and c_uid(uid, msg.author.id)]
    return res[-1]

def get_last_message(uid=None):
    """
    Search the given channel for the second-to-last message
    aka: the message before the command was given to the bot
    """
    if len(bot.messages) == 0:
        raise Exception("Wat")
    if len(bot.messages) == 1:
        return None
    c_uid = lambda u, v: True
    if uid is not None:
        c_uid = lambda u, v: u == v
    res = [msg for msg in bot.messages
            if  msg.author.id != bot.user.id
            and c_uid(uid, msg.author.id)]
    if len(res) <= 1:
        return None
    return res[-2]

def load_opus_lib(opus_libs=OPUS_LIBS):
    """
    Load opus libs for voice handling
    """
    if opus.is_loaded():
        return True

    for opus_lib in opus_libs:
        try:
            opus.load_opus(opus_lib)
            return
        except OSError:
            pass

##Many Functions :

URLMAP = {"+": "%2B",
            " ": "+",
            "%": "%25",
            "&": "%26",
            "@": "%40",
            "#": "%23",
            "$": "%24",
            "=": "%3D"}

def replace(string="", char_map=URLMAP):
    """
    Used to convert special chars in links for the ytsearch
    """
    "Replace a string with URL safe characters (' ' => '%20')"
    s = string
    for k, v in char_map.items():
        s = s.replace(k, v)
    return s

    return voice
#https://youtu.be/I67v_OaB3k8

def closePlayer(player, voice):
    """
    Close player
        player : player to close
        voice = deprecated
    """
    print('Closing player...')
    action = player.stop()
    print('Player closed.')

##ASYNC joinChannel
async def joinChannel(channel):
    """
    Automaticly join a voice channel
        channel = the voice channel to join
    """
    print('Joining '+str(channel))
    try:
        voice = await bot.join_voice_channel(channel)
        print('CONNECTED')
    except Exception as error:
        print('TimeoutError : '+str(error))
    return voice

##ASYNC playYtVid
async def playYtVid(target, link):
    """
    Play a youtube video's audio
    Will play a sound if the link is incorrect
        target = channel to join
        link = video link
    """
    voice = await joinChannel(target)
    print('Launching audio stream...')
    try :
        player = await voice.create_ytdl_player(link, use_avconv=False)
    except Exception as error:
        try:
            print(error)
            print('==PLAYING FIRST ERROR FILE==')
            file = 'Spy_no0'+str(random.randint(1,3))+'.wav'
            print(file)
            player = voice.create_ffmpeg_player(file, use_avconv=False)
        except Exception as error:
            print(error)
            print('==PRINT DEFAULT NOPE==')
            player = await voice.create_ytdl_player('https://youtu.be/fxYOC3gDe7k', use_avconv=False)
    player.volume= 0.5
    #pprint(getmembers(player))
    print('Starting player')
    player.start()
    try:
        waittime = player.duration
    except:
        waittime = 1
    print(waittime)
    while not player.is_done():
        await asyncio.sleep(waittime)
    player.stop()
    print ('Disconnecting...')
    await voice.disconnect()
    print('DISCONNECTED')

##ASYNC playAudioFile
async def playAudioFile(target, file='yee.wav'):
    """
    Plays and audio file located in data/audio
        target = channel to join
        file = played file
    """
    voice = await joinChannel(target)
    dict = 'data/audio/'
    print('Launching audio stream...')
    try:
        player = voice.create_ffmpeg_player(dict+file, use_avconv=False, after=lambda: closePlayer(player, voice))
    except:
        player = voice.create_ffmpeg_player(dict+'yee.wav', use_avconv=False, after=lambda: closePlayer(player, voice))
    print('Starting player')
    player.volume=0.5
    player.start()
    while not player.is_done():
        await asyncio.sleep(1)
    print ('Disconnecting...')
    await voice.disconnect()
    print('DISCONNECTED')

##ASYNC searchvid
async def searchvid(ctx, src):
    if not src:
        return bot.say("I can't search an empty text")
    print(src)
    tube = "https://www.youtube.com"
    query = tube + "/results?search_query=" + replace(src)
    print(query)
    async with aiohttp.get(query) as resp:
        if resp.status != 200:
            return await bot.say("Failed to retrieve search. STATUS:"+str(resp.status))

    # Build a BS parser and find all Youtube links on the page
        txt = await resp.text()
        bs = BS(txt, "html.parser")
        main_d = bs.find('div', id='results')
        if not main_d:
            return bot.say('Failed to find results')
        items = main_d.find_all("div", class_="yt-lockup-content")
        if not items:
            return await bot.say("No videos found")
        # Loop until we find a valid non-advertisement link
        for container in items:
            href = container.find('a', class_='yt-uix-sessionlink')['href']
            if href.startswith('/watch'):
                return await bot.say(tube+href)
        return await bot.say("No YouTube video found")

##BOT and async functions

bot = commands.Bot(command_prefix='!', description=description)

@bot.event
async def on_ready():
    print('Logged in as')
    print(bot.user.name)
    print(bot.user.id)
    print('------')

##CMD roll
@bot.command()
async def roll(dice : str):
    """Rolls a dice in NdN format."""
    try:
        rolls, limit = map(int, dice.split('d'))
    except Exception:
        await bot.say('Format has to be in NdN!')
        return

    result = ', '.join(str(random.randint(1, limit)) for r in range(rolls))
    await bot.say(result)

##CMD choose
@bot.command(description='For when you wanna settle the score some other way')
async def choose(*choices : str):
    """Chooses between multiple choices."""
    await bot.say(random.choice(choices))

##CMD joined
@bot.command()
async def joined(member : discord.Member):
    """Says when a member joined."""
    await bot.say('{0.name} joined in {0.joined_at}'.format(member))
##CMD status
@bot.command()
async def status(status='I am DLBot but better'):
    """Set a new status"""
    await bot.change_presence(game=discord.Game(name=status))

##CMD cool
@bot.group(pass_context=True)
async def cool(ctx):
    """Says if a user is cool.
    In reality this just checks if a subcommand is being invoked.
    """
    if ctx.invoked_subcommand is None:
        await bot.say('No, {0.subcommand_passed} is not cool'.format(ctx))
@cool.command(name='bot')
async def _bot():
    """Is the bot cool?"""
    await bot.say('Yes, the bot is cool.')

##CMD shutdown
@bot.command()
async def shutdown():
    """
    Kill the process
    """
    bot.logout()
    bot.close()
    exit()

##CMD ytplay
@bot.command(pass_context=True)
async def ytplay(ctx, src=""):
    """
    Will play a youtube video's audio
    """
    if not src or src=="":
        return bot.say("I can't search an empty text")
    if "youtu" not in src:
        return bot.say('Use a valid link')
    print(src)
    target = ctx.message.author.voice_channel
    load_opus_lib()
    await playYtVid(target, src)

##CMD audioplay
@bot.command(pass_context=True)
async def audioplay(ctx, src=""):
    """
    Will play a file
    """
    if not src or src=="":
        return bot.say("I can't play an empty text")
    target = ctx.message.author.voice_channel
    load_opus_lib()
    await playAudioFile(target, src)

##CMD addaudio
@bot.command(pass_context=True)
async def addaudio(ctx, src=""):
    """
    Will add an audio file
    """
    #pprint(getmembers(ctx.message.attachments))
    #print(ctx.message.attachments)
    for att in ctx.message.attachments:
        print(att['url'])
        link = att['url']
        async with aiohttp.get(link) as response:
            filename = att['filename']
            with open('data/audio/'+filename, 'wb') as f_handle:
                while True:
                    chunk = await response.content.read(1024)
                    if not chunk:
                        break
                    f_handle.write(chunk)
            await bot.say(filename + ' created ! Call it using `!audioplay '+filename+'`')
            return await response.release()

#FIXME broken
##CMD ytsearch
@bot.command(pass_context=True)
async def ytsearch(ctx, src=""):
    """
    Get the first Youtube search result video
    Example: !yt how do I take a screenshot
    """
    origChan = ctx.message.channel
    await bot.send_typing(origChan)
    src = ctx.message.content.replace('!ytsearch ','')
    await searchvid(ctx, src)

##CMD ytsearchplay
@bot.command(pass_context=True)
async def ytsearchplay(ctx, src=""):
    """
    Get the first Youtube search result video and play it
    Example: !yt how do I take a screenshot
    """
    bot.delete_message(ctx.message)
    target = ctx.message.author.voice_channel
    origChan = ctx.message.channel
    await bot.send_typing(origChan)
    src = ctx.message.content.replace('!ytsearchplay ','')
    mgs = await searchvid(ctx, src)
    load_opus_lib()
    src = mgs.content
    bot.delete_message(mgs)
    await playYtVid(target, src)

bot.run(read_key())
<commit_msg>Update 2.0b<commit_after>
## Notes

# Url to connect tht bot to a server: https://discordapp.com/oauth2/authorize?&client_id=324471059816382464&scope=bot
# id 324471059816382464

# Basic informations. To change if you want to setup your own Bot.

__program__ = "FoxliBot"
__version__ = "2.0b"

from pprint import pprint
from inspect import getmembers
import discord
import asyncio
import aiohttp
from discord import opus
from discord.ext import commands
from bs4 import BeautifulSoup as BS
import random
import os
from json import load as jload

OPUS_LIBS = ['libopus-0.x86.dll', 'libopus-0.x64.dll', 'libopus-0.dll', 'libopus.so.0', 'libopus.0.dylib']

description = '''An example bot to showcase the discord.ext.commands extension
module.
There are a number of utility commands being showcased here.'''

name='foxlibot'

prefix = '**'

def read_key():
    """
    Read a bot's key JSON to get it's token
    Keys must be stored in the key folder and have a basic 'key':'<keytext>' object
    """
    with open('keys/foxlibot.key', 'r') as f:
        datum = jload(f)
        key = datum.get("key", "")
        if not key:
            raise IOError("Key not found in JSON keyfile")
        return key
    return None

global datadir
datadir = os.path.dirname("./data/")
if not os.path.exists(datadir):
    os.makedirs(datadir)
audio = os.path.dirname("./data/audio")
if not os.path.exists(audio):
    os.makedirs(audio)

def logMsg():
    dir = os.path.dirname("./logs/")
    if not os.path.exists(dir):
            os.makedirs(dir)
    today = datetime.datetime.now()
    logsFile=open(dir+"/"+str(today)+".txt","a")
    log = "oui"
    logsFile.write(log)
    print(log,end='')

def get_cmd_message(uid=None):
    """
    Search the given channel for the last message
    aka: the command that was given to the bot
    """
    if len(bot.messages) == 0:
        raise Exception("Wat")
    c_uid = lambda u, v: True
    if uid is not None:
        c_uid = lambda u, v: u == v
    res = [msg for msg in bot.messages
            if msg.channel == chan
            and msg.author.id != bot.user.id
            and c_uid(uid, msg.author.id)]
    return res[-1]

def get_last_message(uid=None):
    """
    Search the given channel for the second-to-last message
    aka: the message before the command was given to the bot
    """
    if len(bot.messages) == 0:
        raise Exception("Wat")
    if len(bot.messages) == 1:
        return None
    c_uid = lambda u, v: True
    if uid is not None:
        c_uid = lambda u, v: u == v
    res = [msg for msg in bot.messages
            if  msg.author.id != bot.user.id
            and c_uid(uid, msg.author.id)]
    if len(res) <= 1:
        return None
    return res[-2]

def load_opus_lib(opus_libs=OPUS_LIBS):
    """
    Load opus libs for voice handling
    """
    if opus.is_loaded():
        return True

    for opus_lib in opus_libs:
        try:
            opus.load_opus(opus_lib)
            return
        except OSError:
            pass

##Many Functions :

URLMAP = {"+": "%2B",
            " ": "+",
            "%": "%25",
            "&": "%26",
            "@": "%40",
            "#": "%23",
            "$": "%24",
            "=": "%3D"}

def replace(string="", char_map=URLMAP):
    """
    Used to convert special chars in links for the ytsearch
    """
    "Replace a string with URL safe characters (' ' => '%20')"
    s = string
    for k, v in char_map.items():
        s = s.replace(k, v)
    return s

    return voice
#https://youtu.be/I67v_OaB3k8

def closePlayer(player, voice):
    """
    Close player
        player : player to close
        voice = deprecated
    """
    print('Closing player...')
    action = player.stop()
    print('Player closed.')

##ASYNC joinChannel
async def joinChannel(channel):
    """
    Automaticly join a voice channel
        channel = the voice channel to join
    """
    print('Joining '+str(channel))
    try:
        voice = await bot.join_voice_channel(channel)
        print('CONNECTED')
    except Exception as error:
        print('TimeoutError : '+str(error))
    return voice

##ASYNC playYtVid
async def playYtVid(target, link):
    """
    Play a youtube video's audio
    Will play a sound if the link is incorrect
        target = channel to join
        link = video link
    """
    voice = await joinChannel(target)
    print('Launching audio stream...')
    try :
        player = await voice.create_ytdl_player(link, use_avconv=False)
    except Exception as error:
        try:
            print(error)
            print('==PLAYING FIRST ERROR FILE==')
            file = 'Spy_no0'+str(random.randint(1,3))+'.wav'
            print(file)
            player = voice.create_ffmpeg_player(file, use_avconv=False)
        except Exception as error:
            print(error)
            print('==PRINT DEFAULT NOPE==')
            player = await voice.create_ytdl_player('https://youtu.be/fxYOC3gDe7k', use_avconv=False)
    player.volume= 0.5
    #pprint(getmembers(player))
    print('Starting player')
    player.start()
    try:
        waittime = player.duration
    except:
        waittime = 1
    print(waittime)
    while not player.is_done():
        await asyncio.sleep(waittime)
    player.stop()
    print ('Disconnecting...')
    await voice.disconnect()
    print('DISCONNECTED')

##ASYNC playAudioFile
async def playAudioFile(target, file='yee.wav'):
    """
    Plays and audio file located in data/audio
        target = channel to join
        file = played file
    """
    voice = await joinChannel(target)
    dict = 'data/audio/'
    print('Launching audio stream...')
    try:
        player = voice.create_ffmpeg_player(dict+file, use_avconv=False)
    except:
        player = voice.create_ffmpeg_player(dict+'yee.wav', use_avconv=False)
    print('Starting player')
    player.volume=0.5
    player.start()
    while not player.is_done():
        await asyncio.sleep(1)
    action = player.stop()
    print ('Disconnecting...')
    await voice.disconnect()
    print('DISCONNECTED')

##ASYNC searchvid
async def searchvid(ctx, src):
    if not src:
        return bot.say("I can't search an empty text")
    print(src)
    tube = "https://www.youtube.com"
    query = tube + "/results?search_query=" + replace(src)
    print(query)
    async with aiohttp.get(query) as resp:
        if resp.status != 200:
            return await bot.say("Failed to retrieve search. STATUS:"+str(resp.status))

    # Build a BS parser and find all Youtube links on the page
        txt = await resp.text()
        bs = BS(txt, "html.parser")
        main_d = bs.find('div', id='results')
        if not main_d:
            return bot.say('Failed to find results')
        items = main_d.find_all("div", class_="yt-lockup-content")
        if not items:
            return await bot.say("No videos found")
        # Loop until we find a valid non-advertisement link
        for container in items:
            href = container.find('a', class_='yt-uix-sessionlink')['href']
            if href.startswith('/watch'):
                return await bot.say(tube+href)
        return await bot.say("No YouTube video found")

##BOT and async functions

bot = commands.Bot(command_prefix='!', description=description)

@bot.event
async def on_ready():
    print('Logged in as')
    print(bot.user.name)
    print(bot.user.id)
    print('------')

##CMD roll
@bot.command()
async def roll(dice : str):
    """Rolls a dice in NdN format."""
    try:
        rolls, limit = map(int, dice.split('d'))
    except Exception:
        await bot.say('Format has to be in NdN!')
        return

    result = ', '.join(str(random.randint(1, limit)) for r in range(rolls))
    await bot.say(result)

##CMD choose
@bot.command(description='For when you wanna settle the score some other way')
async def choose(*choices : str):
    """Chooses between multiple choices."""
    await bot.say(random.choice(choices))

##CMD joined
@bot.command()
async def joined(member : discord.Member):
    """Says when a member joined."""
    await bot.say('{0.name} joined in {0.joined_at}'.format(member))
##CMD status
@bot.command()
async def status(status='I am DLBot but better'):
    """Set a new status"""
    await bot.change_presence(game=discord.Game(name=status))

##CMD cool
@bot.group(pass_context=True)
async def cool(ctx):
    """Says if a user is cool.
    In reality this just checks if a subcommand is being invoked.
    """
    if ctx.invoked_subcommand is None:
        await bot.say('No, {0.subcommand_passed} is not cool'.format(ctx))
@cool.command(name='bot')
async def _bot():
    """Is the bot cool?"""
    await bot.say('Yes, the bot is cool.')

##CMD shutdown
@bot.command()
async def shutdown():
    """
    Kill the process
    """
    bot.logout()
    bot.close()
    exit()

##CMD ytplay
@bot.command(pass_context=True)
async def ytplay(ctx, src=""):
    """
    Will play a youtube video's audio
    """
    if not src or src=="":
        return bot.say("I can't search an empty text")
    if "youtu" not in src:
        return bot.say('Use a valid link')
    print(src)
    target = ctx.message.author.voice_channel
    load_opus_lib()
    await playYtVid(target, src)

##CMD audioplay
@bot.command(pass_context=True)
async def audioplay(ctx, src=""):
    """
    Will play a file
    """
    if not src or src=="":
        return bot.say("I can't play an empty text")
    target = ctx.message.author.voice_channel
    load_opus_lib()
    await playAudioFile(target, src)

##CMD addaudio
@bot.command(pass_context=True)
async def addaudio(ctx, src=""):
    """
    Will add an audio file
    """
    #pprint(getmembers(ctx.message.attachments))
    #print(ctx.message.attachments)
    for att in ctx.message.attachments:
        print(att['url'])
        link = att['url']
        async with aiohttp.get(link) as response:
            filename = att['filename']
            with open('data/audio/'+filename, 'wb') as f_handle:
                while True:
                    chunk = await response.content.read(1024)
                    if not chunk:
                        break
                    f_handle.write(chunk)
            await bot.say(filename + ' created ! Call it using `!audioplay '+filename+'`')
            return await response.release()

#FIXME broken
##CMD ytsearch
@bot.command(pass_context=True)
async def ytsearch(ctx, src=""):
    """
    Get the first Youtube search result video
    Example: !yt how do I take a screenshot
    """
    origChan = ctx.message.channel
    await bot.send_typing(origChan)
    src = ctx.message.content.replace('!ytsearch ','')
    await searchvid(ctx, src)

##CMD ytsearchplay
@bot.command(pass_context=True)
async def ytsearchplay(ctx, src=""):
    """
    Get the first Youtube search result video and play it
    Example: !yt how do I take a screenshot
    """
    bot.delete_message(ctx.message)
    target = ctx.message.author.voice_channel
    origChan = ctx.message.channel
    await bot.send_typing(origChan)
    src = ctx.message.content.replace('!ytsearchplay ','')
    mgs = await searchvid(ctx, src)
    load_opus_lib()
    src = mgs.content
    bot.delete_message(mgs)
    await playYtVid(target, src)

bot.run(read_key())
<|endoftext|>


<commit_before>import wnck, gtk
from collections import deque
import time
import keybinder
import configuration

class Azulejo:

    maximized_window_geometry = gtk.gdk.get_default_root_window().property_get('_NET_WORKAREA')[2][:4]
    upper_corner = maximized_window_geometry[:2]
    screen_width = maximized_window_geometry[2]
    screen_height = maximized_window_geometry[3]

    #because window resizing is not acurate we need a quick dirty workaround
    window_geometry_error_margin = 30

    #variable to hold the amount of windows since the last arrangement
    arrangement_size = 0

    def get_all_windows(self):
        def f_normal_window(window):
            if window.get_window_type() == wnck.WindowType.__enum_values__[0]:
                return True
            return False

        s = wnck.screen_get_default()

        while gtk.events_pending():
            gtk.main_iteration()

        windows = s.get_windows_stacked()
        filtered_windows = filter(f_normal_window, windows)
        filtered_windows.reverse()
        return filtered_windows

    def parse_simple_math_expressions(self, expression):
        expression = str(expression)
        expression = expression.replace('w', str(self.screen_width))
        expression = expression.replace('h', str(self.screen_height))
        return eval(expression)

    def parse_geometry(self, geometry):
        return map(self.parse_simple_math_expressions, geometry)

    def parse_arrangement(self, arrangement):
        return map(self.parse_geometry, arrangement)

    def resize_windows(self, arrangement):
        arrangement_numeric = self.parse_arrangement(arrangement)
        #print arrangement_numeric
        filtered_windows = self.get_all_windows()
        amount_of_windows = len(filtered_windows)

        if amount_of_windows < len(arrangement_numeric):
            arrangement_numeric = arrangement_numeric[:amount_of_windows]

        i = 0
        self.arrangement_size = len(arrangement_numeric) #global scope variable, also used to rotate windows
        while i < self.arrangement_size:
            geometry_list_args = [0, 255]
            index = self.arrangement_size - (i + 1) #we must start in the end in order to keep window order correct
            geometry_list_args.extend(map (int, arrangement_numeric[index]))
            filtered_windows[index].unmaximize()
            filtered_windows[index].set_geometry(*geometry_list_args)
            i += 1

    def resize_single_window(self, geometries):

        def similar_geometries(ga, gb):
            for i in range(4):
                if abs(ga[i] - gb[i]) >= self.window_geometry_error_margin:
                    return False
            return True

        window = wnck.screen_get_default().get_active_window()
        window_original_geometry = window.get_geometry()

        #not an arrangement, but a list of geometires for that matter
        geometries_numeric = self.parse_arrangement(geometries)
        geometry_list_args = [0, 255]

        i = 1
        geometry_to_use_index = 0
        for geometry_numeric in geometries_numeric:
            if similar_geometries(geometry_numeric, window_original_geometry):
                geometry_to_use_index = i % len(geometries_numeric)
                break
            i += 1

        geometry_list_args.extend(map (int, geometries_numeric[geometry_to_use_index]))
        window.unmaximize()
        window.set_geometry(*geometry_list_args)

    def rotate_windows(self, dummy):
        windows = self.get_all_windows()
        amount_of_windows = len(windows)

        if amount_of_windows > self.arrangement_size:
            windows = windows[:self.arrangement_size]

        geos = []
        for window in windows:
            window_geo = window.get_geometry()
            window_geo = window_geo[:4]
            geos.append(window_geo)

            #do the actual rotations, lets use deque as it's dramatically more efficient than a trivial shift implementation
        windows_deq = deque(windows)
        windows_deq.rotate(1)

        rotation_len = len(windows_deq)
        i = 0
        while i < rotation_len:
            geometry_list_args = [0, 255]
            index = rotation_len - (i + 1) #again, start by the tail
            geometry_list_args.extend(map (int, geos[index]))
            windows_deq[index].unmaximize()
            windows_deq[index].set_geometry(*geometry_list_args)
            i += 1

        #(windows_deq[0]).activate(int(time.time())) #not sure why it doesn't work. if uncommented causes other windows beyond the rotated ones to hide behind current ones even after pressing ctrl+tab

    def maximize(self, dummy):
        windows = self.get_all_windows()
        curwin = windows[0]
        curwin.maximize()

    callable_actions = dict(\
        resize_single_window=resize_single_window, \
        resize_windows=resize_windows, \
        rotate_windows=rotate_windows, \
        maximize=maximize
    )

def dispatcher(dis_param):

    (func, azulejo_obj, param) = dis_param

    wnck.screen_get_default().force_update() #doesn't apear to do much
    func(azulejo_obj, param)

def run():

    azulejo_obj = Azulejo()

    for action in configuration.conf_data:
        keybind = action['keybind']
        function_name = action['function']
        function = azulejo_obj.callable_actions[function_name]
        parameters = action['parameters']

        dispacher_parameters = (
            function,
            azulejo_obj,
            parameters
        )

        keybinder.bind(keybind, dispatcher, dispacher_parameters)

    gtk.main()
<commit_msg>Remove intermediate vars for keybind<commit_after>import wnck, gtk
from collections import deque
import time
import keybinder
import configuration

class Azulejo:

    maximized_window_geometry = gtk.gdk.get_default_root_window().property_get('_NET_WORKAREA')[2][:4]
    upper_corner = maximized_window_geometry[:2]
    screen_width = maximized_window_geometry[2]
    screen_height = maximized_window_geometry[3]

    #because window resizing is not acurate we need a quick dirty workaround
    window_geometry_error_margin = 30

    #variable to hold the amount of windows since the last arrangement
    arrangement_size = 0

    def get_all_windows(self):
        def f_normal_window(window):
            if window.get_window_type() == wnck.WindowType.__enum_values__[0]:
                return True
            return False

        s = wnck.screen_get_default()

        while gtk.events_pending():
            gtk.main_iteration()

        windows = s.get_windows_stacked()
        filtered_windows = filter(f_normal_window, windows)
        filtered_windows.reverse()
        return filtered_windows

    def parse_simple_math_expressions(self, expression):
        expression = str(expression)
        expression = expression.replace('w', str(self.screen_width))
        expression = expression.replace('h', str(self.screen_height))
        return eval(expression)

    def parse_geometry(self, geometry):
        return map(self.parse_simple_math_expressions, geometry)

    def parse_arrangement(self, arrangement):
        return map(self.parse_geometry, arrangement)

    def resize_windows(self, arrangement):
        arrangement_numeric = self.parse_arrangement(arrangement)
        #print arrangement_numeric
        filtered_windows = self.get_all_windows()
        amount_of_windows = len(filtered_windows)

        if amount_of_windows < len(arrangement_numeric):
            arrangement_numeric = arrangement_numeric[:amount_of_windows]

        i = 0
        self.arrangement_size = len(arrangement_numeric) #global scope variable, also used to rotate windows
        while i < self.arrangement_size:
            geometry_list_args = [0, 255]
            index = self.arrangement_size - (i + 1) #we must start in the end in order to keep window order correct
            geometry_list_args.extend(map (int, arrangement_numeric[index]))
            filtered_windows[index].unmaximize()
            filtered_windows[index].set_geometry(*geometry_list_args)
            i += 1

    def resize_single_window(self, geometries):

        def similar_geometries(ga, gb):
            for i in range(4):
                if abs(ga[i] - gb[i]) >= self.window_geometry_error_margin:
                    return False
            return True

        window = wnck.screen_get_default().get_active_window()
        window_original_geometry = window.get_geometry()

        #not an arrangement, but a list of geometires for that matter
        geometries_numeric = self.parse_arrangement(geometries)
        geometry_list_args = [0, 255]

        i = 1
        geometry_to_use_index = 0
        for geometry_numeric in geometries_numeric:
            if similar_geometries(geometry_numeric, window_original_geometry):
                geometry_to_use_index = i % len(geometries_numeric)
                break
            i += 1

        geometry_list_args.extend(map (int, geometries_numeric[geometry_to_use_index]))
        window.unmaximize()
        window.set_geometry(*geometry_list_args)

    def rotate_windows(self, dummy):
        windows = self.get_all_windows()
        amount_of_windows = len(windows)

        if amount_of_windows > self.arrangement_size:
            windows = windows[:self.arrangement_size]

        geos = []
        for window in windows:
            window_geo = window.get_geometry()
            window_geo = window_geo[:4]
            geos.append(window_geo)

            #do the actual rotations, lets use deque as it's dramatically more efficient than a trivial shift implementation
        windows_deq = deque(windows)
        windows_deq.rotate(1)

        rotation_len = len(windows_deq)
        i = 0
        while i < rotation_len:
            geometry_list_args = [0, 255]
            index = rotation_len - (i + 1) #again, start by the tail
            geometry_list_args.extend(map (int, geos[index]))
            windows_deq[index].unmaximize()
            windows_deq[index].set_geometry(*geometry_list_args)
            i += 1

        #(windows_deq[0]).activate(int(time.time())) #not sure why it doesn't work. if uncommented causes other windows beyond the rotated ones to hide behind current ones even after pressing ctrl+tab

    def maximize(self, dummy):
        windows = self.get_all_windows()
        curwin = windows[0]
        curwin.maximize()

    callable_actions = dict(\
        resize_single_window=resize_single_window, \
        resize_windows=resize_windows, \
        rotate_windows=rotate_windows, \
        maximize=maximize
    )

def dispatcher(dis_param):

    (func, azulejo_obj, param) = dis_param

    wnck.screen_get_default().force_update() #doesn't apear to do much
    func(azulejo_obj, param)

def run():

    azulejo_obj = Azulejo()

    for action in configuration.conf_data:

        keybinder.bind(
            action['keybind'],
            dispatcher,
            (
                azulejo_obj.callable_actions[action['function']],
                azulejo_obj,
                action['parameters']
            )
        )

    gtk.main()
<|endoftext|>


<commit_before>#!/usr/bin/python
import re
import sys

input = "\n".join([l for l in open(sys.argv[1])])

class Node:
  def __init__(self, name, args):
    assert isinstance(args, list)
    self.name = name
    self.args = args

  def tostr(self, indent):
    indentation = indent * "  "
    result = ""

    result = indentation + self.name
    for arg in self.args:
      result += "\n" + arg.tostr(indent + 1)

    return result

  def __repr__(self):
    return self.tostr(0)

  def wrap(self, str):
    return "(function(){%s;})()" % (str)

  def compile(self):
    ops = ["+", "/", "*", "-", "||", "&&", "===", "!==", "!=", "==", "+=", "-=", "/=", "*=", "instanceof", "<", ">", "<=", ">="]
    unary = ["!", "+", "-", "new", "typeof"]

    name = self.name
    if name == "=":
      return "(%s = %s)" % (self.args[0].compile(), self.args[1].compile())
    elif name == "root":
      return ";\n".join([arg.compile() for arg in self.args])
    elif name == "call":
      return "(%s)()" % self.args[0].compile()
    elif name == "try":
      return "try {\n%s\n} catch %s  {\n%s} finally {\n%s}" % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile(), self.args[3].compile())
    elif name == "do":
      if len(self.args) == 0:
        return self.wrap("return void 0");
      return self.wrap(";\n".join([arg.compile() for arg in self.args[:-1]]) + "\nreturn %s;" % (self.args[-1].compile()))
    elif name == "var":
      return "var %s" % ",".join([arg.compile() for arg in self.args])
    elif name == "root":
      return ";\n".join([arg.compile() for arg in self.args])
    elif name == "==":
      return "%s == %s" % (self.args[0].compile(), self.args[1].compile())
    elif name == "if":
      while len(self.args) < 3:
        self.args.append(Node("void", [Atom("0")])) # Append empty bodies to unfilled parts of the if

      return self.wrap("if (%s) {return %s;} else {return %s;}" % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile()))
      # return ("if (%s) {%s;} else {%s;}") % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile())
    elif name == "function":
      return "function %s { return %s }" % (self.args[0].compile(), ";".join([arg.compile() for arg in self.args[1:]]))
    elif name == "arglist":
      return "(" + ",".join([arg.compile() for arg in self.args]) + ")"

    for op in ops:
      if name == op and len(self.args) == 2:
        return "(%s %s %s)" % (self.args[0].compile(), op, self.args[1].compile())

    for op in unary:
      if name == op and len(self.args) == 1:
        return "(%s %s)" % (op, self.args[0].compile())

    return "%s(%s)" % (self.name, ", ".join([arg.compile() for arg in self.args]))

class Atom:
  def __init__(self, contents):
    self.contents = contents

  def tostr(self, indent):
    return indent * "  " + self.contents

  def compile(self):
    return self.contents

  def __repr__(self):
    return "Atom " + self.contents

def is_paren(str):
  return str == "(" or str == ")"

def tokenize(string):
  in_string = False
  string_opener = ""
  tokens = [""]
  for ch in string:
    if ch == "'" or ch == '"':
      in_string = not in_string
      tokens[-1] += ch
      if not in_string: tokens.append("")
      continue

    if not in_string:
      if ch == " ":
        tokens.append("")
        continue

      if is_paren(ch):
        tokens.append(ch)
        tokens.append("")
        continue

    tokens[-1] += ch

  tokens = [tok.strip() for tok in tokens[:-1] if tok.strip() != ""]
  return tokens

def parse(string):
  string = string.strip()
  string = "(root " + string + ")"
  tokens = tokenize(string)

  def helper(tokens):
    node_name = tokens[1]
    node_args = [[]]
    tokens = tokens[2:-1] #ignore (, name, and ).

    depth = 0
    for token in tokens:
      if is_paren(token):
        depth = depth + (1 if token == "(" else -1)
        node_args[-1].append(token)

        if depth == 0:
          node_args[-1] = helper(node_args[-1])
          node_args.append([])
        continue

      if depth == 0:
        node_args[-1] = Atom(token)
        node_args.append([])
      else:
        node_args[-1].append(token)

    node_args = node_args[:-1]
    return Node(node_name, node_args)

  return helper(tokens)

ast = parse(input)

output = sys.argv[1].split(".")[0] + ".js" #same name as input, but .js instead of .sc
open(output, 'w').write(ast.compile())
<commit_msg>Why is it eating everything?!?<commit_after>#!/usr/bin/python
import re
import sys

input = "\n".join([l for l in open(sys.argv[1])])

class Node:
  def __init__(self, name, args):
    assert isinstance(args, list)
    self.name = name
    self.args = args

  def tostr(self, indent):
    indentation = indent * "  "
    result = ""

    result = indentation + self.name
    for arg in self.args:
      result += "\n" + arg.tostr(indent + 1)

    return result

  def __repr__(self):
    return self.tostr(0)

  def wrap(self, str):
    return "(function(){%s;})()" % (str)

  def compile(self):
    ops = ["+", "/", "*", "-", "||", "&&", "===", "!==", "!=", "==", "+=", "-=", "/=", "*=", "instanceof", "<", ">", "<=", ">="]
    unary = ["!", "+", "-", "new", "typeof"]

    name = self.name
    if name == "=":
      return "(%s = %s)" % (self.args[0].compile(), self.args[1].compile())
    elif name == "root":
      return ";\n".join([arg.compile() for arg in self.args])
    elif name == "call":
      return "(%s)()" % self.args[0].compile()
    elif name == "try":
      return "try {\n%s\n} catch %s  {\n%s} finally {\n%s}" % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile(), self.args[3].compile())
    elif name == "do":
      if len(self.args) == 0:
        return "(void 0)"
      return self.wrap(",\n".join([arg.compile() for arg in self.args]))
    elif name == "var":
      return "var %s" % ",".join([arg.compile() for arg in self.args])
    elif name == "root":
      return ";\n".join([arg.compile() for arg in self.args])
    elif name == "==":
      return "%s == %s" % (self.args[0].compile(), self.args[1].compile())
    elif name == "ternary":
      assert len(self.args) == 3
      return "(%s ? %s : %s)" % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile())
    elif name == "if":
      while len(self.args) < 3:
        self.args.append(Node("void", [Atom("0")])) # Append empty bodies to unfilled parts of the if

      return self.wrap("if (%s) {return %s;} else {return %s;}" % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile()))
      # return ("if (%s) {%s;} else {%s;}") % (self.args[0].compile(), self.args[1].compile(), self.args[2].compile())
    elif name == "function":
      return "function %s { return %s }" % (self.args[0].compile(), ";".join([arg.compile() for arg in self.args[1:]]))
    elif name == "arglist":
      return "(" + ",".join([arg.compile() for arg in self.args]) + ")"

    for op in ops:
      if name == op and len(self.args) == 2:
        return "(%s %s %s)" % (self.args[0].compile(), op, self.args[1].compile())

    for op in unary:
      if name == op and len(self.args) == 1:
        return "(%s %s)" % (op, self.args[0].compile())

    return "%s(%s)" % (self.name, ", ".join([arg.compile() for arg in self.args]))

class Atom:
  def __init__(self, contents):
    self.contents = contents

  def tostr(self, indent):
    return indent * "  " + self.contents

  def compile(self):
    return self.contents

  def __repr__(self):
    return "Atom " + self.contents

def is_paren(str):
  return str == "(" or str == ")"

def tokenize(string):
  in_string = False
  string_opener = ""
  tokens = [""]
  for ch in string:
    if ch == "'" or ch == '"':
      in_string = not in_string
      tokens[-1] += ch
      if not in_string: tokens.append("")
      continue

    if not in_string:
      if ch == " ":
        tokens.append("")
        continue

      if is_paren(ch):
        tokens.append(ch)
        tokens.append("")
        continue

    tokens[-1] += ch

  tokens = [tok.strip() for tok in tokens[:-1] if tok.strip() != ""]
  return tokens

def parse(string):
  string = string.strip()
  string = "(root " + string + ")"
  tokens = tokenize(string)

  def helper(tokens):
    node_name = tokens[1]
    node_args = [[]]
    tokens = tokens[2:-1] #ignore (, name, and ).

    depth = 0
    for token in tokens:
      if is_paren(token):
        depth = depth + (1 if token == "(" else -1)
        node_args[-1].append(token)

        if depth == 0:
          node_args[-1] = helper(node_args[-1])
          node_args.append([])
        continue

      if depth == 0:
        node_args[-1] = Atom(token)
        node_args.append([])
      else:
        node_args[-1].append(token)

    node_args = node_args[:-1]
    return Node(node_name, node_args)

  return helper(tokens)

ast = parse(input)

output = sys.argv[1].split(".")[0] + ".js" #same name as input, but .js instead of .sc
open(output, 'w').write(ast.compile())
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""scholarly.py"""

from __future__ import absolute_import, division, print_function, unicode_literals

from bs4 import BeautifulSoup

import arrow
import bibtexparser
import codecs
import hashlib
import pprint
import random
import re
import requests
import sys
import time

_GOOGLEID = hashlib.md5(str(random.random()).encode('utf-8')).hexdigest()[:16]
_COOKIES = {'GSP': 'ID={0}:CF=4'.format(_GOOGLEID)}
_HEADERS = {
    'accept-language': 'en-US,en',
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/41.0.2272.76 Chrome/41.0.2272.76 Safari/537.36',
    'accept': 'text/html,application/xhtml+xml,application/xml'
    }
_HOST = 'https://scholar.google.com'
_AUTHSEARCH = '/citations?view_op=search_authors&hl=en&mauthors={0}'
_CITATIONAUTH = '/citations?user={0}&hl=en'
_CITATIONPUB = '/citations?view_op=view_citation&citation_for_view={0}'
_KEYWORDSEARCH = '/citations?view_op=search_authors&hl=en&mauthors=label:{0}'
_PUBSEARCH = '/scholar?q={0}'
_SCHOLARPUB = '/scholar?oi=bibs&hl=en&cites={0}'

_CITATIONAUTHRE = r'user=([\w-]*)'
_CITATIONPUBRE = r'citation_for_view=([\w-]*:[\w-]*)'
_SCHOLARCITERE = r'gs_ocit\(event,\'([\w-]*)\''
_SCHOLARPUBRE = r'cites=([\w-]*)'

_SESSION = requests.Session()
_PAGESIZE = 100

def _handle_captcha(url):
    # TODO: PROBLEMS HERE! NEEDS ATTENTION
    # Get the captcha image
    captcha_url = _HOST + '/sorry/image?id={0}'.format(g_id)
    captcha = _SESSION.get(captcha_url, headers=_HEADERS)
    # Upload to remote host and display to user for human verification
    img_upload = requests.post('http://postimage.org/',
        files={'upload[]': ('scholarly_captcha.jpg', captcha.text)})
    print(img_upload.text)
    img_url_soup = BeautifulSoup(img_upload.text, 'html.parser')
    img_url = img_url_soup.find_all(alt='scholarly_captcha')[0].get('src')
    print('CAPTCHA image URL: {0}'.format(img_url))
    # Need to check Python version for input
    if sys.version[0]=="3":
        g_response = input('Enter CAPTCHA: ')
    else:
        g_response = raw_input('Enter CAPTCHA: ')
    # Once we get a response, follow through and load the new page.
    url_response = _HOST+'/sorry/CaptchaRedirect?continue={0}&id={1}&captcha={2}&submit=Submit'.format(dest_url, g_id, g_response)
    resp_captcha = _SESSION.get(url_response, headers=_HEADERS, cookies=_COOKIES)
    print('Forwarded to {0}'.format(resp_captcha.url))
    return resp_captcha.url

def _get_page(pagerequest):
    """Return the data for a page on scholar.google.com"""
    # Note that we include a sleep to avoid overloading the scholar server
    time.sleep(5+random.uniform(0, 5))
    resp = _SESSION.get(pagerequest, headers=_HEADERS, cookies=_COOKIES)
    if resp.status_code == 200:
        return resp.text
    if resp.status_code == 503:
        # Inelegant way of dealing with the G captcha
        raise Exception('Error: {0} {1}'.format(resp.status_code, resp.reason))
        # TODO: Need to fix captcha handling
        # dest_url = requests.utils.quote(_SCHOLARHOST+pagerequest)
        # soup = BeautifulSoup(resp.text, 'html.parser')
        # captcha_url = soup.find('img').get('src')
        # resp = _handle_captcha(captcha_url)
        # return _get_page(re.findall(r'https:\/\/(?:.*?)(\/.*)', resp)[0])
    else:
        raise Exception('Error: {0} {1}'.format(resp.status_code, resp.reason))

def _get_soup(pagerequest):
    """Return the BeautifulSoup for a page on scholar.google.com"""
    html = _get_page(pagerequest)
    return BeautifulSoup(html, 'html.parser')

def _search_scholar_soup(soup):
    """Generator that returns Publication objects from the search page"""
    while True:
        for row in soup.find_all('div', 'gs_r'):
            yield Publication(row, 'scholar')
        if soup.find(class_='gs_ico gs_ico_nav_next'):
            url = soup.find(class_='gs_ico gs_ico_nav_next').parent['href']
            soup = _get_soup(_HOST+url)
        else:
            break

def _search_citation_soup(soup):
    """Generator that returns Author objects from the author search page"""
    while True:
        for row in soup.find_all('div', 'gsc_1usr'):
            yield Author(row)
        nextbutton = soup.find(class_='gs_btnPR gs_in_ib gs_btn_half gs_btn_srt')
        if nextbutton and 'disabled' not in nextbutton.attrs:
            url = nextbutton['onclick'][17:-1]
            url = codecs.getdecoder("unicode_escape")(url)[0]
            soup = _get_soup(_HOST+url)
        else:
            break

class Publication(object):
    """Returns an object for a single publication"""
    def __init__(self, __data, pubtype=None):
        self.bib = dict()
        self.source = pubtype
        if self.source == 'citations':
            self.bib['title'] = __data.find('a', class_='gsc_a_at').text
            self.id_citations = re.findall(_CITATIONPUBRE, __data.find('a', class_='gsc_a_at')['href'])[0]
            citedby = __data.find(class_='gsc_a_ac')
            if citedby and not citedby.text.isspace():
                self.citedby = int(citedby.text)
            year = __data.find(class_='gsc_a_h')
            if year and year.text and not year.text.isspace() and len(year.text)>0:
                self.bib['year'] = int(year.text)
        elif self.source == 'scholar':
            databox = __data.find('div', class_='gs_ri')
            title = databox.find('h3', class_='gs_rt')
            if title.find('span', class_='gs_ctu'): # A citation
                title.span.extract()
            elif title.find('span', class_='gs_ctc'): # A book or PDF
                title.span.extract()
            self.bib['title'] = title.text.strip()
            if title.find('a'):
                self.bib['url'] = title.find('a')['href']
            authorinfo = databox.find('div', class_='gs_a')
            self.bib['author'] = ' and '.join([i.strip() for i in authorinfo.text.split(' - ')[0].split(',')])
            if databox.find('div', class_='gs_rs'):
                self.bib['abstract'] = databox.find('div', class_='gs_rs').text
                if self.bib['abstract'][0:8].lower() == 'abstract':
                    self.bib['abstract'] = self.bib['abstract'][9:].strip()
            lowerlinks = databox.find('div', class_='gs_fl').find_all('a')
            for link in lowerlinks:
                if 'Import into BibTeX' in link.text:
                    self.url_scholarbib = link['href']
                if 'Cited by' in link.text:
                    self.citedby = int(re.findall(r'\d+', link.text)[0])
                    self.id_scholarcitedby = re.findall(_SCHOLARPUBRE, link['href'])[0]
            if __data.find('div', class_='gs_ggs gs_fl'):
                self.bib['eprint'] = _HOST + __data.find('div', class_='gs_ggs gs_fl').a['href']
        self._filled = False

    def fill(self):
        """Populate the Publication with information from its profile"""
        if self.source == 'citations':
            url = _CITATIONPUB.format(self.id_citations)
            soup = _get_soup(_HOST+url)
            self.bib['title'] = soup.find('div', id='gsc_title').text
            if soup.find('a', class_='gsc_title_link'):
                self.bib['url'] = soup.find('a', class_='gsc_title_link')['href']
            for item in soup.find_all('div', class_='gs_scl'):
                key = item.find(class_='gsc_field').text
                val = item.find(class_='gsc_value')
                if key == 'Authors':
                    self.bib['author'] = ' and '.join([i.strip() for i in val.text.split(',')])
                elif key == 'Journal':
                    self.bib['journal'] = val.text
                elif key == 'Volume':
                    self.bib['volume'] = val.text
                elif key == 'Issue':
                    self.bib['number'] = val.text
                elif key == 'Pages':
                    self.bib['pages'] = val.text
                elif key == 'Publisher':
                    self.bib['publisher'] = val.text
                elif key == 'Publication date':
                    self.bib['year'] = arrow.get(val.text).year
                elif key == 'Description':
                    if val.text[0:8].lower() == 'abstract':
                        val = val.text[9:].strip()
                    self.bib['abstract'] = val
                elif key == 'Total citations':
                    self.id_scholarcitedby = re.findall(_SCHOLARPUBRE, val.a['href'])[0]
            if soup.find('div', class_='gsc_title_ggi'):
                self.bib['eprint'] = _HOST + soup.find('div', class_='gsc_title_ggi').a['href']
            self._filled = True
        elif self.source == 'scholar':
            bibtex = _get_page(self.url_scholarbib)
            self.bib.update(bibtexparser.loads(bibtex).entries[0])
            self._filled = True
        return self

    def get_citedby(self):
        """Searches GScholar for other articles that cite this Publication and
        returns a Publication generator.
        """
        if not hasattr(self, 'id_scholarcitedby'):
            self.fill()
        if hasattr(self, 'id_scholarcitedby'):
            url = _SCHOLARPUB.format(requests.utils.quote(self.id_scholarcitedby))
            soup = _get_soup(_HOST+url)
            return _search_scholar_soup(soup)
        else:
            return []

    def __str__(self):
        return pprint.pformat(self.__dict__)

class Author(object):
    """Returns an object for a single author"""
    def __init__(self, __data):
        if isinstance(__data, str):
            self.id = __data
        else:
            self.id = re.findall(_CITATIONAUTHRE, __data('a')[0]['href'])[0]
            self.url_picture = __data('img')[0]['src']
            self.name = __data.find('h3', class_='gsc_1usr_name').text
            affiliation = __data.find('div', class_='gsc_1usr_aff')
            if affiliation:
                self.affiliation = affiliation.text
            email = __data.find('div', class_='gsc_1usr_emlb')
            if email:
                self.email = email.text
            self.interests = [i.text.strip() for i in
                              __data.find_all('a', class_='gsc_co_int')]
            citedby = __data.find('div', class_='gsc_1usr_cby')
            if citedby:
                self.citedby = int(citedby.text[9:])
        self._filled = False

    def fill(self):
        """Populate the Author with information from their profile"""
        url_citations = _CITATIONAUTH.format(self.id)
        url = '{0}&pagesize={1}'.format(url_citations, _PAGESIZE)
        soup = _get_soup(_HOST+url)
        self.name = soup.find('div', id='gsc_prf_in').text
        self.affiliation = soup.find('div', class_='gsc_prf_il').text
        self.interests = [i.text.strip() for i in soup.find_all('a', class_='gsc_prf_ila')]
        self.url_picture = soup.find('img')['src']

        # h-index, i10-index and h-index, i10-index in the last 5 years
        index = soup.find_all('td', class_='gsc_rsb_std')
        self.hindex = int(index[2].text)
        self.hindex5y = int(index[3].text)
        self.i10index = int(index[4].text)
        self.i10index5y = int(index[5].text)

        # number of citations per year
        years = [int(y.text) for y in soup.find_all('span', class_='gsc_g_t')]
        cites = [int(c.text) for c in soup.find_all('span', class_='gsc_g_al')]
        self.cites_per_year = dict(zip(years, cites))

        self.publications = list()
        pubstart = 0
        while True:
            for row in soup.find_all('tr', class_='gsc_a_tr'):
                new_pub = Publication(row, 'citations')
                self.publications.append(new_pub)
            if 'disabled' not in soup.find('button', id='gsc_bpf_next').attrs:
                pubstart += _PAGESIZE
                url = '{0}&cstart={1}&pagesize={2}'.format(url_citations, pubstart, _PAGESIZE)
                soup = _get_soup(_HOST+url)
            else:
                break
        self._filled = True
        return self

    def __str__(self):
        return pprint.pformat(self.__dict__)

def search_pubs_query(query):
    """Search by scholar query and return a generator of Publication objects"""
    url = _PUBSEARCH.format(requests.utils.quote(query))
    soup = _get_soup(_HOST+url)
    return _search_scholar_soup(soup)

def search_author(name):
    """Search by author name and return a generator of Author objects"""
    url = _AUTHSEARCH.format(requests.utils.quote(name))
    soup = _get_soup(_HOST+url)
    return _search_citation_soup(soup)

def search_keyword(keyword):
    """Search by keyword and return a generator of Author objects"""
    url = _KEYWORDSEARCH.format(requests.utils.quote(keyword))
    soup = _get_soup(_HOST+url)
    return _search_citation_soup(soup)

def search_pubs_custom_url(url):
    """Search by custom URL and return a generator of Publication objects
    URL should be of the form '/scholar?q=...'"""
    soup = _get_soup(_HOST+url)
    return _search_scholar_soup(soup)

def search_author_custom_url(url):
    """Search by custom URL and return a generator of Publication objects
    URL should be of the form '/citation?q=...'"""
    soup = _get_soup(_HOST+url)
    return _search_citation_soup(soup)

if __name__ == "__main__":
    author = next(search_author('Steven A. Cholewiak')).fill()
    print(author)
<commit_msg>Changes contributed by Eitan Frachtenberg<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""scholarly.py"""

from __future__ import absolute_import, division, print_function, unicode_literals

from bs4 import BeautifulSoup

import arrow
import bibtexparser
import codecs
import hashlib
import pprint
import random
import re
import requests
import sys
import time

_GOOGLEID = hashlib.md5(str(random.random()).encode('utf-8')).hexdigest()[:16]
_COOKIES = {'GSP': 'ID={0}:CF=4'.format(_GOOGLEID)}
_HEADERS = {
    'accept-language': 'en-US,en',
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/41.0.2272.76 Chrome/41.0.2272.76 Safari/537.36',
    'accept': 'text/html,application/xhtml+xml,application/xml'
    }
_HOST = 'https://scholar.google.com'
_AUTHSEARCH = '/citations?view_op=search_authors&hl=en&mauthors={0}'
_CITATIONAUTH = '/citations?user={0}&hl=en'
_CITATIONPUB = '/citations?view_op=view_citation&citation_for_view={0}'
_KEYWORDSEARCH = '/citations?view_op=search_authors&hl=en&mauthors=label:{0}'
_PUBSEARCH = '/scholar?q={0}'
_SCHOLARPUB = '/scholar?oi=bibs&hl=en&cites={0}'

_CITATIONAUTHRE = r'user=([\w-]*)'
_CITATIONPUBRE = r'citation_for_view=([\w-]*:[\w-]*)'
_SCHOLARCITERE = r'gs_ocit\(event,\'([\w-]*)\''
_SCHOLARPUBRE = r'cites=([\w-]*)'
_EMAILAUTHORRE = r'Verified email at '

_SESSION = requests.Session()
_PAGESIZE = 100

def _handle_captcha(url):
    # TODO: PROBLEMS HERE! NEEDS ATTENTION
    # Get the captcha image
    captcha_url = _HOST + '/sorry/image?id={0}'.format(g_id)
    captcha = _SESSION.get(captcha_url, headers=_HEADERS)
    # Upload to remote host and display to user for human verification
    img_upload = requests.post('http://postimage.org/',
        files={'upload[]': ('scholarly_captcha.jpg', captcha.text)})
    print(img_upload.text)
    img_url_soup = BeautifulSoup(img_upload.text, 'html.parser')
    img_url = img_url_soup.find_all(alt='scholarly_captcha')[0].get('src')
    print('CAPTCHA image URL: {0}'.format(img_url))
    # Need to check Python version for input
    if sys.version[0]=="3":
        g_response = input('Enter CAPTCHA: ')
    else:
        g_response = raw_input('Enter CAPTCHA: ')
    # Once we get a response, follow through and load the new page.
    url_response = _HOST+'/sorry/CaptchaRedirect?continue={0}&id={1}&captcha={2}&submit=Submit'.format(dest_url, g_id, g_response)
    resp_captcha = _SESSION.get(url_response, headers=_HEADERS, cookies=_COOKIES)
    print('Forwarded to {0}'.format(resp_captcha.url))
    return resp_captcha.url

def _get_page(pagerequest):
    """Return the data for a page on scholar.google.com"""
    # Note that we include a sleep to avoid overloading the scholar server
    time.sleep(5+random.uniform(0, 5))
    resp = _SESSION.get(pagerequest, headers=_HEADERS, cookies=_COOKIES)
    if resp.status_code == 200:
        return resp.text
    if resp.status_code == 503:
        # Inelegant way of dealing with the G captcha
        raise Exception('Error: {0} {1}'.format(resp.status_code, resp.reason))
        # TODO: Need to fix captcha handling
        # dest_url = requests.utils.quote(_SCHOLARHOST+pagerequest)
        # soup = BeautifulSoup(resp.text, 'html.parser')
        # captcha_url = soup.find('img').get('src')
        # resp = _handle_captcha(captcha_url)
        # return _get_page(re.findall(r'https:\/\/(?:.*?)(\/.*)', resp)[0])
    else:
        raise Exception('Error: {0} {1}'.format(resp.status_code, resp.reason))

def _get_soup(pagerequest):
    """Return the BeautifulSoup for a page on scholar.google.com"""
    html = _get_page(pagerequest)
    return BeautifulSoup(html, 'html.parser')

def _search_scholar_soup(soup):
    """Generator that returns Publication objects from the search page"""
    while True:
        for row in soup.find_all('div', 'gs_r'):
            yield Publication(row, 'scholar')
        if soup.find(class_='gs_ico gs_ico_nav_next'):
            url = soup.find(class_='gs_ico gs_ico_nav_next').parent['href']
            soup = _get_soup(_HOST+url)
        else:
            break

def _search_citation_soup(soup):
    """Generator that returns Author objects from the author search page"""
    while True:
        for row in soup.find_all('div', 'gsc_1usr'):
            yield Author(row)
        nextbutton = soup.find(class_='gs_btnPR gs_in_ib gs_btn_half gs_btn_srt')
        if nextbutton and 'disabled' not in nextbutton.attrs:
            url = nextbutton['onclick'][17:-1]
            url = codecs.getdecoder("unicode_escape")(url)[0]
            soup = _get_soup(_HOST+url)
        else:
            break

class Publication(object):
    """Returns an object for a single publication"""
    def __init__(self, __data, pubtype=None):
        self.bib = dict()
        self.source = pubtype
        if self.source == 'citations':
            self.bib['title'] = __data.find('a', class_='gsc_a_at').text
            self.id_citations = re.findall(_CITATIONPUBRE, __data.find('a', class_='gsc_a_at')['data-href'])[0]
            citedby = __data.find(class_='gsc_a_ac')
            if citedby and not (citedby.text.isspace() or citedby.text == ''):
                self.citedby = int(citedby.text)
            year = __data.find(class_='gsc_a_h')
            if year and year.text and not year.text.isspace() and len(year.text)>0:
                self.bib['year'] = int(year.text)
        elif self.source == 'scholar':
            databox = __data.find('div', class_='gs_ri')
            title = databox.find('h3', class_='gs_rt')
            if title.find('span', class_='gs_ctu'): # A citation
                title.span.extract()
            elif title.find('span', class_='gs_ctc'): # A book or PDF
                title.span.extract()
            self.bib['title'] = title.text.strip()
            if title.find('a'):
                self.bib['url'] = title.find('a')['href']
            authorinfo = databox.find('div', class_='gs_a')
            self.bib['author'] = ' and '.join([i.strip() for i in authorinfo.text.split(' - ')[0].split(',')])
            if databox.find('div', class_='gs_rs'):
                self.bib['abstract'] = databox.find('div', class_='gs_rs').text
                if self.bib['abstract'][0:8].lower() == 'abstract':
                    self.bib['abstract'] = self.bib['abstract'][9:].strip()
            lowerlinks = databox.find('div', class_='gs_fl').find_all('a')
            for link in lowerlinks:
                if 'Import into BibTeX' in link.text:
                    self.url_scholarbib = link['href']
                if 'Cited by' in link.text:
                    self.citedby = int(re.findall(r'\d+', link.text)[0])
                    self.id_scholarcitedby = re.findall(_SCHOLARPUBRE, link['href'])[0]
            if __data.find('div', class_='gs_ggs gs_fl'):
                self.bib['eprint'] = _HOST + __data.find('div', class_='gs_ggs gs_fl').a['href']
        self._filled = False

    def fill(self):
        """Populate the Publication with information from its profile"""
        if self.source == 'citations':
            url = _CITATIONPUB.format(self.id_citations)
            soup = _get_soup(_HOST+url)
            self.bib['title'] = soup.find('div', id='gsc_vcd_title').text
            if soup.find('a', class_='gsc_vcd_title_link'):
                self.bib['url'] = soup.find('a', class_='gsc_vcd_title_link')['href']
            for item in soup.find_all('div', class_='gs_scl'):
                key = item.find(class_='gsc_vcd_field').text
                val = item.find(class_='gsc_vcd_value')
                if key == 'Authors':
                    self.bib['author'] = ' and '.join([i.strip() for i in val.text.split(',')])
                elif key == 'Journal':
                    self.bib['journal'] = val.text
                elif key == 'Volume':
                    self.bib['volume'] = val.text
                elif key == 'Issue':
                    self.bib['number'] = val.text
                elif key == 'Pages':
                    self.bib['pages'] = val.text
                elif key == 'Publisher':
                    self.bib['publisher'] = val.text
                elif key == 'Publication date':
                    self.bib['year'] = arrow.get(val.text).year
                elif key == 'Description':
                    if val.text[0:8].lower() == 'abstract':
                        val = val.text[9:].strip()
                    self.bib['abstract'] = val
                elif key == 'Total citations':
                    self.id_scholarcitedby = re.findall(_SCHOLARPUBRE, val.a['href'])[0]
            if soup.find('div', class_='gsc_vcd_title_ggi'):
                self.bib['eprint'] = _HOST + soup.find('div', class_='gsc_vcd_title_ggi').a['href']
            self._filled = True
        elif self.source == 'scholar':
            bibtex = _get_page(self.url_scholarbib)
            self.bib.update(bibtexparser.loads(bibtex).entries[0])
            self._filled = True
        return self

    def get_citedby(self):
        """Searches GScholar for other articles that cite this Publication and
        returns a Publication generator.
        """
        if not hasattr(self, 'id_scholarcitedby'):
            self.fill()
        if hasattr(self, 'id_scholarcitedby'):
            url = _SCHOLARPUB.format(requests.utils.quote(self.id_scholarcitedby))
            soup = _get_soup(_HOST+url)
            return _search_scholar_soup(soup)
        else:
            return []

    def __str__(self):
        return pprint.pformat(self.__dict__)

class Author(object):
    """Returns an object for a single author"""
    def __init__(self, __data):
        if isinstance(__data, str):
            self.id = __data
        else:
            self.id = re.findall(_CITATIONAUTHRE, __data('a')[0]['href'])[0]
            self.url_picture = __data('img')[0]['src']
            self.name = __data.find('h3', class_='gsc_oai_name').text
            affiliation = __data.find('div', class_='gsc_oai_aff')
            if affiliation:
                self.affiliation = affiliation.text
            email = __data.find('div', class_='gsc_oai_eml')
            if email:
                self.email = re.sub(_EMAILAUTHORRE, r'@', email.text)
            self.interests = [i.text.strip() for i in
                              __data.find_all('a', class_='gsc_oai_one_int')]
            citedby = __data.find('div', class_='gsc_oai_cby')
            if citedby and citedby.text != '':
                self.citedby = int(citedby.text[9:])
        self._filled = False

    def fill(self):
        """Populate the Author with information from their profile"""
        url_citations = _CITATIONAUTH.format(self.id)
        url = '{0}&pagesize={1}'.format(url_citations, _PAGESIZE)
        soup = _get_soup(_HOST+url)
        self.name = soup.find('div', id='gsc_prf_in').text
        self.affiliation = soup.find('div', class_='gsc_prf_il').text
        self.interests = [i.text.strip() for i in soup.find_all('a', class_='gsc_prf_inta')]
        self.url_picture = soup.find('img')['src']

        # h-index, i10-index and h-index, i10-index in the last 5 years
        index = soup.find_all('td', class_='gsc_rsb_std')
        if index:
            self.hindex = int(index[2].text)
            self.hindex5y = int(index[3].text)
            self.i10index = int(index[4].text)
            self.i10index5y = int(index[5].text)
        else:
            self.hindex = self.hindex5y = self.i10index = self.i10index5y = 0

        # number of citations per year
        years = [int(y.text) for y in soup.find_all('span', class_='gsc_g_t')]
        cites = [int(c.text) for c in soup.find_all('span', class_='gsc_g_al')]
        self.cites_per_year = dict(zip(years, cites))

        self.publications = list()
        pubstart = 0
        while True:
            for row in soup.find_all('tr', class_='gsc_a_tr'):
                new_pub = Publication(row, 'citations')
                self.publications.append(new_pub)
            if 'disabled' not in soup.find('button', id='gsc_bpf_more').attrs:
                pubstart += _PAGESIZE
                url = '{0}&cstart={1}&pagesize={2}'.format(url_citations, pubstart, _PAGESIZE)
                soup = _get_soup(_HOST+url)
            else:
                break
        self._filled = True
        return self

    def __str__(self):
        return pprint.pformat(self.__dict__)

def search_pubs_query(query):
    """Search by scholar query and return a generator of Publication objects"""
    url = _PUBSEARCH.format(requests.utils.quote(query))
    soup = _get_soup(_HOST+url)
    return _search_scholar_soup(soup)

def search_author(name):
    """Search by author name and return a generator of Author objects"""
    url = _AUTHSEARCH.format(requests.utils.quote(name))
    soup = _get_soup(_HOST+url)
    return _search_citation_soup(soup)

def search_keyword(keyword):
    """Search by keyword and return a generator of Author objects"""
    url = _KEYWORDSEARCH.format(requests.utils.quote(keyword))
    soup = _get_soup(_HOST+url)
    return _search_citation_soup(soup)

def search_pubs_custom_url(url):
    """Search by custom URL and return a generator of Publication objects
    URL should be of the form '/scholar?q=...'"""
    soup = _get_soup(_HOST+url)
    return _search_scholar_soup(soup)

def search_author_custom_url(url):
    """Search by custom URL and return a generator of Publication objects
    URL should be of the form '/citation?q=...'"""
    soup = _get_soup(_HOST+url)
    return _search_citation_soup(soup)

if __name__ == "__main__":
    author = next(search_author('Steven A. Cholewiak')).fill()
    print(author)
<|endoftext|>