<commit_before># -*- coding: utf-8 -*-

"""
The MIT License (MIT)

Copyright (c) 2017 SML

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""

import os
import discord
import datetime as dt
from collections import defaultdict
from discord.ext import commands

from __main__ import send_cmd_help
from cogs.utils import checks
from cogs.utils.chat_formatting import pagify, bold, escape_mass_mentions
from cogs.utils.dataIO import dataIO

PATH = os.path.join("data", "reactionmanager")
JSON = os.path.join(PATH, "settings.json")

def nested_dict():
    """Recursively nested defaultdict."""
    return defaultdict(nested_dict)

class ReactionManager:

    def __init__(self, bot):
        """Reaction Management."""
        self.bot = bot
        self.settings = dataIO.load_json(JSON)

    @commands.group(aliases=['rm'], pass_context=True, no_pm=True)
    async def reactionmanager(self, ctx):
        """Reaction Manager."""
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)

    @reactionmanager.command(name='add', pass_context=True, no_pm=True)
    @checks.mod_or_permissions(manage_messages=True)
    async def rm_add(self, ctx, *args):
        """Add reactions to a message by message id.

        Add reactions to a specific message id
        [p]rm add 123456 :white_check_mark: :x: :zzz:

        Add reactions to the last message in channel
        [p]rm add :white_check_mark: :x: :zzz:
        """
        channel = ctx.message.channel

        if not len(args):
            await send_cmd_help(ctx)
            return

        has_message_id = args[0].isdigit()

        emojis = args[1:] if has_message_id else args
        message_id = args[0] if has_message_id else None

        if has_message_id:
            try:
                message = await self.bot.get_message(channel, message_id)
            except discord.NotFound:
                await self.bot.say("Cannot find message with that id.")
                return
        else:
            # use the 2nd last message because the last message would be the command
            messages = []
            async for m in self.bot.logs_from(channel, limit=2):
                messages.append(m)

            # messages = [m async for m in self.bot.logs_from(channel, limit=2)]
            message = messages[1]

        for emoji in emojis:
            try:
                await self.bot.add_reaction(message, emoji)
            except discord.HTTPException:
                # reaction add failed
                pass
            except discord.Forbidden:
                await self.bot.say(
                    "I don’t have permission to react to that message.")
                break
            except discord.InvalidArgument:
                await self.bot.say("Invalid arguments for emojis")
                break

        await self.bot.delete_message(ctx.message)

    @reactionmanager.command(name="remove", pass_context=True, no_pm=True)
    @checks.mod_or_permissions(manage_messages=True)
    async def rm_remove(self, ctx, number: int):
        """Remove reactions from last X messages in the channel."""
        channel = ctx.message.channel
        author = ctx.message.author
        server = author.server
        has_permissions = channel.permissions_for(server.me).manage_messages
        to_manage = []

        if not has_permissions:
            await self.bot.say("I’m not allowed to remove reactions.")
            return

        async for message in self.bot.logs_from(channel, limit=number + 1):
            to_manage.append(message)

        await self.remove_reactions(to_manage)

    async def remove_reactions(self, messages):
        """Remove reactions."""
        for message in messages:
            await self.bot.clear_reactions(message)

    @reactionmanager.command(name="get", pass_context=True, no_pm=True)
    @checks.mod_or_permissions(manage_messages=True)
    async def rm_get(self, ctx, channel: discord.Channel, message_id, exclude_self=True):
        """Display list of reactions added by users."""
        message = await self.bot.get_message(channel, message_id)

        if message is None:
            await self.bot.say("Cannot find that message id.")
            return

        out = await self.get_reactions(message, exclude_self=exclude_self)

        for page in pagify('\n'.join(out), shorten_by=24):
            await self.bot.say(page)

    @reactionmanager.command(name="getserver", pass_context=True, no_pm=True)
    @checks.is_owner()
    async def rm_getserver(self, ctx, server_name, channel_name, message_id, exclude_self=True):
        """Display list of reactions added by users."""
        server = discord.utils.get(self.bot.servers, name=server_name)
        channel = discord.utils.get(server.channels, name=channel_name)

        message = await self.bot.get_message(channel, message_id)
        if message is None:
            await self.bot.say("Cannot find that message id.")
            return

        out = await self.get_reactions(message, exclude_self=exclude_self)

        for page in pagify('\n'.join(out), shorten_by=24):
            await self.bot.say(page)

    async def get_reactions(self, message, exclude_self=True):
        title = message.channel.name
        description = message.content

        out = [
            bold('Channel: {}'.format(title)),
            escape_mass_mentions(description)
        ]

        for reaction in message.reactions:
            if reaction.custom_emoji:
                # <:emoji_name:emoji_id>
                emoji = '<:{}:{}>'.format(
                    reaction.emoji.name,
                    reaction.emoji.id)
            else:
                emoji = reaction.emoji

            reaction_users = await self.bot.get_reaction_users(reaction)
            valid_users = []
            for u in reaction_users:
                if exclude_self and u == self.bot.user:
                    continue
                valid_users.append(u)
                valid_users = sorted(valid_users, key=lambda u: u.display_name.lower())
            users_str = ', '.join([u.display_name for u in valid_users])
            count = len(valid_users)
            value = '{}: {}: {}'.format(emoji, count, users_str)

            out.append(value)

        return out

def check_folder():
    """Check folder."""
    if not os.path.exists(PATH):
        os.makedirs(PATH)

def check_file():
    """Check files."""
    defaults = {}
    if not dataIO.is_valid_json(JSON):
        dataIO.save_json(JSON, defaults)

def setup(bot):
    """Setup bot."""
    check_folder()
    check_file()
    n = ReactionManager(bot)
    bot.add_cog(n)
<commit_msg>[reactionmanager] force server display name in results<commit_after># -*- coding: utf-8 -*-

"""
The MIT License (MIT)

Copyright (c) 2017 SML

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""

import os
import discord
import datetime as dt
from collections import defaultdict
from discord.ext import commands

from __main__ import send_cmd_help
from cogs.utils import checks
from cogs.utils.chat_formatting import pagify, bold, escape_mass_mentions
from cogs.utils.dataIO import dataIO

PATH = os.path.join("data", "reactionmanager")
JSON = os.path.join(PATH, "settings.json")

def nested_dict():
    """Recursively nested defaultdict."""
    return defaultdict(nested_dict)

class ReactionManager:

    def __init__(self, bot):
        """Reaction Management."""
        self.bot = bot
        self.settings = dataIO.load_json(JSON)

    @commands.group(aliases=['rm'], pass_context=True, no_pm=True)
    async def reactionmanager(self, ctx):
        """Reaction Manager."""
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)

    @reactionmanager.command(name='add', pass_context=True, no_pm=True)
    @checks.mod_or_permissions(manage_messages=True)
    async def rm_add(self, ctx, *args):
        """Add reactions to a message by message id.

        Add reactions to a specific message id
        [p]rm add 123456 :white_check_mark: :x: :zzz:

        Add reactions to the last message in channel
        [p]rm add :white_check_mark: :x: :zzz:
        """
        channel = ctx.message.channel

        if not len(args):
            await send_cmd_help(ctx)
            return

        has_message_id = args[0].isdigit()

        emojis = args[1:] if has_message_id else args
        message_id = args[0] if has_message_id else None

        if has_message_id:
            try:
                message = await self.bot.get_message(channel, message_id)
            except discord.NotFound:
                await self.bot.say("Cannot find message with that id.")
                return
        else:
            # use the 2nd last message because the last message would be the command
            messages = []
            async for m in self.bot.logs_from(channel, limit=2):
                messages.append(m)

            # messages = [m async for m in self.bot.logs_from(channel, limit=2)]
            message = messages[1]

        for emoji in emojis:
            try:
                await self.bot.add_reaction(message, emoji)
            except discord.HTTPException:
                # reaction add failed
                pass
            except discord.Forbidden:
                await self.bot.say(
                    "I don’t have permission to react to that message.")
                break
            except discord.InvalidArgument:
                await self.bot.say("Invalid arguments for emojis")
                break

        await self.bot.delete_message(ctx.message)

    @reactionmanager.command(name="remove", pass_context=True, no_pm=True)
    @checks.mod_or_permissions(manage_messages=True)
    async def rm_remove(self, ctx, number: int):
        """Remove reactions from last X messages in the channel."""
        channel = ctx.message.channel
        author = ctx.message.author
        server = author.server
        has_permissions = channel.permissions_for(server.me).manage_messages
        to_manage = []

        if not has_permissions:
            await self.bot.say("I’m not allowed to remove reactions.")
            return

        async for message in self.bot.logs_from(channel, limit=number + 1):
            to_manage.append(message)

        await self.remove_reactions(to_manage)

    async def remove_reactions(self, messages):
        """Remove reactions."""
        for message in messages:
            await self.bot.clear_reactions(message)

    @reactionmanager.command(name="get", pass_context=True, no_pm=True)
    @checks.mod_or_permissions(manage_messages=True)
    async def rm_get(self, ctx, channel: discord.Channel, message_id, exclude_self=True):
        """Display list of reactions added by users."""
        message = await self.bot.get_message(channel, message_id)

        if message is None:
            await self.bot.say("Cannot find that message id.")
            return

        out = await self.get_reactions(message, exclude_self=exclude_self)

        for page in pagify('\n'.join(out), shorten_by=24):
            await self.bot.say(page)

    @reactionmanager.command(name="getserver", pass_context=True, no_pm=True)
    @checks.is_owner()
    async def rm_getserver(self, ctx, server_name, channel_name, message_id, exclude_self=True):
        """Display list of reactions added by users."""
        server = discord.utils.get(self.bot.servers, name=server_name)
        channel = discord.utils.get(server.channels, name=channel_name)

        message = await self.bot.get_message(channel, message_id)
        if message is None:
            await self.bot.say("Cannot find that message id.")
            return

        out = await self.get_reactions(message, exclude_self=exclude_self)

        for page in pagify('\n'.join(out), shorten_by=24):
            await self.bot.say(page)

    async def get_reactions(self, message, exclude_self=True):
        title = message.channel.name
        description = message.content

        out = [
            bold('Channel: {}'.format(title)),
            escape_mass_mentions(description)
        ]

        server = message.server

        for reaction in message.reactions:
            if reaction.custom_emoji:
                # <:emoji_name:emoji_id>
                emoji = '<:{}:{}>'.format(
                    reaction.emoji.name,
                    reaction.emoji.id)
            else:
                emoji = reaction.emoji

            reaction_users = await self.bot.get_reaction_users(reaction)
            valid_users = []
            for u in reaction_users:
                if exclude_self and u == self.bot.user:
                    continue
                valid_users.append(u)
                valid_users = sorted(valid_users, key=lambda u: u.display_name.lower())
            user_ids = [u.id for u in valid_users]
            members = []
            for uid in user_ids:
                member = server.get_member(uid)
                if member:
                    members.append(member)
            users_str = ', '.join([m.display_name for m in members])
            count = len(valid_users)
            value = '{}: {}: {}'.format(emoji, count, users_str)

            out.append(value)

        return out

def check_folder():
    """Check folder."""
    if not os.path.exists(PATH):
        os.makedirs(PATH)

def check_file():
    """Check files."""
    defaults = {}
    if not dataIO.is_valid_json(JSON):
        dataIO.save_json(JSON, defaults)

def setup(bot):
    """Setup bot."""
    check_folder()
    check_file()
    n = ReactionManager(bot)
    bot.add_cog(n)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Core views, including the main homepage,

documentation and header rendering, and server errors.
"""

from __future__ import absolute_import
from __future__ import division
import os
import logging

from django.conf import settings
from django.http import HttpResponseRedirect, Http404, JsonResponse, HttpResponse
from django.shortcuts import render, get_object_or_404, redirect
from django.views.generic import TemplateView

from readthedocs.builds.models import Version
from readthedocs.core.resolver import resolve_path
from readthedocs.core.symlink import PrivateSymlink, PublicSymlink
from readthedocs.core.utils import broadcast
from readthedocs.projects.constants import PRIVATE
from readthedocs.projects.models import Project, ImportedFile
from readthedocs.projects.tasks import remove_dirs
from readthedocs.redirects.utils import get_redirect_response, project_and_path_from_request

log = logging.getLogger(__name__)

class NoProjectException(Exception):
    pass

class HomepageView(TemplateView):

    template_name = 'homepage.html'

    def get_context_data(self, **kwargs):
        """Add latest builds and featured projects."""
        context = super().get_context_data(**kwargs)
        context['featured_list'] = Project.objects.filter(featured=True)
        context['projects_count'] = Project.objects.count()
        return context

class SupportView(TemplateView):
    template_name = 'support.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        support_email = getattr(settings, 'SUPPORT_EMAIL', None)
        if not support_email:
            support_email = 'support@{domain}'.format(
                domain=getattr(
                    settings,
                    'PRODUCTION_DOMAIN',
                    'readthedocs.org',
                ),
            )

        context['support_email'] = support_email
        return context

def random_page(request, project_slug=None):  # pylint: disable=unused-argument
    imported_file = ImportedFile.objects.order_by('?')
    if project_slug:
        imported_file = imported_file.filter(project__slug=project_slug)
    imported_file = imported_file.first()
    if imported_file is None:
        raise Http404
    url = imported_file.get_absolute_url()
    return HttpResponseRedirect(url)

def wipe_version(request, project_slug, version_slug):
    version = get_object_or_404(
        Version,
        project__slug=project_slug,
        slug=version_slug,
    )
    # We need to check by ``for_admin_user`` here to allow members of the
    # ``Admin`` team (which doesn't own the project) under the corporate site.
    if version.project not in Project.objects.for_admin_user(user=request.user):
        raise Http404('You must own this project to wipe it.')

    if request.method == 'POST':
        del_dirs = [
            os.path.join(version.project.doc_path, 'checkouts', version.slug),
            os.path.join(version.project.doc_path, 'envs', version.slug),
            os.path.join(version.project.doc_path, 'conda', version.slug),
        ]
        for del_dir in del_dirs:
            broadcast(type='build', task=remove_dirs, args=[(del_dir,)])
        return redirect('project_version_list', project_slug)
    return render(
        request,
        'wipe_version.html',
        {'version': version, 'project': version.project},
    )

def server_error_500(request, template_name='500.html'):
    """A simple 500 handler so we get media."""
    r = render(request, template_name)
    r.status_code = 500
    return r

def server_error_404(request, exception=None, template_name='404.html'):  # pylint: disable=unused-argument  # noqa
    """
    A simple 404 handler so we get media.

    .. note::

        Marking exception as optional to make /404/ testing page to work.
    """
    response = get_redirect_response(request, full_path=request.get_full_path())

    # Return a redirect response if there is one
    if response:
        if response.url == request.build_absolute_uri():
            # check that we do have a response and avoid infinite redirect
            log.warning(
                'Infinite Redirect: FROM URL is the same than TO URL. url=%s',
                response.url,
            )
        else:
            return response

    project, path = project_and_path_from_request(request, request.get_full_path())
    version_slug = project.get_default_version()
    version = project.versions.get(slug=version_slug)

    filename = resolve_path(
        project,
        version_slug=version_slug,
        filename='404.html',
        subdomain=True,  # subdomain will make it a "full" path without a URL prefix
    )

    # This breaks path joining, by ignoring the root when given an "absolute" path
    if filename[0] == '/':
        filename = filename[1:]

    if version.privacy_level == PRIVATE:
        symlink = PrivateSymlink(project)
    else:
        symlink = PublicSymlink(project)
    basepath = symlink.project_root
    fullpath = os.path.join(basepath, filename)

    if os.path.exists(fullpath):
        log.debug('Serving custom 404.html page for project: %', project.slug)
        r = HttpResponse(open(fullpath).read(), content_type='text/html')
    else:
        # Return the default 404 page generated by Read the Docs
        r = render(request, template_name)

    r.status_code = 404
    return r

def do_not_track(request):
    dnt_header = request.META.get('HTTP_DNT')

    # https://w3c.github.io/dnt/drafts/tracking-dnt.html#status-representation
    return JsonResponse(  # pylint: disable=redundant-content-type-for-json-response
        {
            'policy': 'https://docs.readthedocs.io/en/latest/privacy-policy.html',
            'same-party': [
                'readthedocs.org',
                'readthedocs.com',
                'readthedocs.io',           # .org Documentation Sites
                'readthedocs-hosted.com',   # .com Documentation Sites
            ],
            'tracking': 'N' if dnt_header == '1' else 'T',
        }, content_type='application/tracking-status+json',
    )
<commit_msg>Use _serve_file to serve custom 404 page<commit_after># -*- coding: utf-8 -*-

"""
Core views, including the main homepage,

documentation and header rendering, and server errors.
"""

from __future__ import absolute_import
from __future__ import division
import os
import logging

from django.conf import settings
from django.http import HttpResponseRedirect, Http404, JsonResponse
from django.shortcuts import render, get_object_or_404, redirect
from django.views.generic import TemplateView

from readthedocs.builds.models import Version
from readthedocs.core.resolver import resolve_path
from readthedocs.core.symlink import PrivateSymlink, PublicSymlink
from readthedocs.core.utils import broadcast
from readthedocs.core.views.serve import _serve_file
from readthedocs.projects.constants import PRIVATE
from readthedocs.projects.models import Project, ImportedFile
from readthedocs.projects.tasks import remove_dirs
from readthedocs.redirects.utils import get_redirect_response, project_and_path_from_request

log = logging.getLogger(__name__)

class NoProjectException(Exception):
    pass

class HomepageView(TemplateView):

    template_name = 'homepage.html'

    def get_context_data(self, **kwargs):
        """Add latest builds and featured projects."""
        context = super().get_context_data(**kwargs)
        context['featured_list'] = Project.objects.filter(featured=True)
        context['projects_count'] = Project.objects.count()
        return context

class SupportView(TemplateView):
    template_name = 'support.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        support_email = getattr(settings, 'SUPPORT_EMAIL', None)
        if not support_email:
            support_email = 'support@{domain}'.format(
                domain=getattr(
                    settings,
                    'PRODUCTION_DOMAIN',
                    'readthedocs.org',
                ),
            )

        context['support_email'] = support_email
        return context

def random_page(request, project_slug=None):  # pylint: disable=unused-argument
    imported_file = ImportedFile.objects.order_by('?')
    if project_slug:
        imported_file = imported_file.filter(project__slug=project_slug)
    imported_file = imported_file.first()
    if imported_file is None:
        raise Http404
    url = imported_file.get_absolute_url()
    return HttpResponseRedirect(url)

def wipe_version(request, project_slug, version_slug):
    version = get_object_or_404(
        Version,
        project__slug=project_slug,
        slug=version_slug,
    )
    # We need to check by ``for_admin_user`` here to allow members of the
    # ``Admin`` team (which doesn't own the project) under the corporate site.
    if version.project not in Project.objects.for_admin_user(user=request.user):
        raise Http404('You must own this project to wipe it.')

    if request.method == 'POST':
        del_dirs = [
            os.path.join(version.project.doc_path, 'checkouts', version.slug),
            os.path.join(version.project.doc_path, 'envs', version.slug),
            os.path.join(version.project.doc_path, 'conda', version.slug),
        ]
        for del_dir in del_dirs:
            broadcast(type='build', task=remove_dirs, args=[(del_dir,)])
        return redirect('project_version_list', project_slug)
    return render(
        request,
        'wipe_version.html',
        {'version': version, 'project': version.project},
    )

def server_error_500(request, template_name='500.html'):
    """A simple 500 handler so we get media."""
    r = render(request, template_name)
    r.status_code = 500
    return r

def server_error_404(request, exception=None, template_name='404.html'):  # pylint: disable=unused-argument  # noqa
    """
    A simple 404 handler so we get media.

    .. note::

        Marking exception as optional to make /404/ testing page to work.
    """
    response = get_redirect_response(request, full_path=request.get_full_path())

    # Return a redirect response if there is one
    if response:
        if response.url == request.build_absolute_uri():
            # check that we do have a response and avoid infinite redirect
            log.warning(
                'Infinite Redirect: FROM URL is the same than TO URL. url=%s',
                response.url,
            )
        else:
            return response

    project, path = project_and_path_from_request(request, request.get_full_path())
    version_slug = project.get_default_version()
    version = project.versions.get(slug=version_slug)

    filename = resolve_path(
        project,
        version_slug=version_slug,
        filename='404.html',
        subdomain=True,  # subdomain will make it a "full" path without a URL prefix
    )

    # This breaks path joining, by ignoring the root when given an "absolute" path
    if filename[0] == '/':
        filename = filename[1:]

    if version.privacy_level == PRIVATE:
        symlink = PrivateSymlink(project)
    else:
        symlink = PublicSymlink(project)
    basepath = symlink.project_root
    fullpath = os.path.join(basepath, filename)

    if os.path.exists(fullpath):
        log.info('Serving custom 404.html page for project: %', project.slug)
        r = _serve_file(request, filename, basepath)
    else:
        # Return the default 404 page generated by Read the Docs
        r = render(request, template_name)

    r.status_code = 404
    return r

def do_not_track(request):
    dnt_header = request.META.get('HTTP_DNT')

    # https://w3c.github.io/dnt/drafts/tracking-dnt.html#status-representation
    return JsonResponse(  # pylint: disable=redundant-content-type-for-json-response
        {
            'policy': 'https://docs.readthedocs.io/en/latest/privacy-policy.html',
            'same-party': [
                'readthedocs.org',
                'readthedocs.com',
                'readthedocs.io',           # .org Documentation Sites
                'readthedocs-hosted.com',   # .com Documentation Sites
            ],
            'tracking': 'N' if dnt_header == '1' else 'T',
        }, content_type='application/tracking-status+json',
    )
<|endoftext|>


<commit_before>from flask import Flask, render_template, request, redirect, abort

from mmmpaste import db
from mmmpaste.forms import NewPaste

app = Flask(__name__)

@app.teardown_request
def shutdown_session(exception = None):
    db.session.remove()

@app.route("/")
def root():
    pass

@app.route("/p/<id>")
def get_paste(id):
    paste = db.get_paste(id)
    if paste is None:
        abort(404)
    return render_template("paste.html", paste = paste)

@app.route("/history")
def history():
    pass

@app.route("/about")
def about():
    pass
<commit_msg>Added some more dummy routes that we will need later.<commit_after>from flask import Flask, render_template, request, redirect, abort

from mmmpaste import db
from mmmpaste.forms import NewPaste

app = Flask(__name__)

@app.teardown_request
def shutdown_session(exception = None):
    db.session.remove()

@app.route("/")
def root():
    pass

@app.route("/p/<id>")
def get_paste(id):
    paste = db.get_paste(id)
    if paste is None:
        abort(404)
    return render_template("paste.html", paste = paste)

@app.route("/history")
def history():
    pass

@app.route("/about")
def about():
    pass

@app.route("/p/<id>/raw")
def get_raw_paste(id):
    pass

@app.route("/p/<id>/download")
def download_paste(id):
    pass

@app.route("/p/<id>/clone")
def clone_paste(id):
    pass
<|endoftext|>


<commit_before>"""Binary for decoding from translation models based on tensorflow/models/rnn/translate/translate.py.

Note that this decoder is greedy and very basic. For a better decoder, see http://ucam-smt.github.io/sgnmt/html/tutorial.html
which supports decoding from tensorflow models.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import cPickle
import sys
import numpy as np
import datetime
import logging

from six.moves import xrange  # pylint: disable=redefined-builtin
import tensorflow as tf

if __name__ == "__main__":
  from tensorflow.models.rnn.translate.train import FLAGS as train_flags # get default model settings from train.py
from tensorflow.models.rnn.translate.utils import data_utils, model_utils

# Decoder settings
tf.app.flags.DEFINE_string("test_src_idx", "/tmp/in.txt", "An integer-encoded input file")
tf.app.flags.DEFINE_string("test_out_idx", "/tmp/out.txt", "Output file for decoder output")
tf.app.flags.DEFINE_string("test_hidden_state", "/tmp/hidden.txt", "Output file for hidden state")
tf.app.flags.DEFINE_integer("max_sentences", 0, "The maximum number of sentences to translate (all if set to 0)")
tf.app.flags.DEFINE_string("decode_hidden", None, "Decode from hidden layers in file")
tf.app.flags.DEFINE_boolean("interactive", False, "Decode from command line")
tf.app.flags.DEFINE_string("model_path", None, "Path to trained model")
FLAGS = tf.app.flags.FLAGS

def decode(config, input=None, output=None, max_sentences=0):
  if input and output:
    inp = input
    out = output
  else:
    inp = config['test_src_idx']
    out = config['test_out_idx']
  if 'test_hidden_state' in config:
    hidden = config['test_hidden_state']
  else:
    hidden = '/tmp/hidden'

  if config['decode_hidden']:
    decode_hidden(config, out)
  else:
    # Find longest input to create suitable bucket
    max_input_length = 0
    with open(inp) as f_in:
      for sentence in f_in:
        token_ids = [ int(tok) for tok in sentence.strip().split() ]
        if config['add_src_eos']:
          token_ids.append(data_utils.EOS_ID)
        if len(token_ids) > max_input_length:
          max_input_length = len(token_ids)
    bucket = model_utils.make_bucket(max_input_length, greedy_decoder=True)
    buckets = list(model_utils._buckets)
    buckets.append(bucket)
    logging.info("Add new bucket={}".format(bucket))

    with tf.Session() as session:
      # Create model and load parameters: uses the training graph for decoding
      model = model_utils.create_model(session, config, forward_only=True,
                                          buckets=buckets)
      model.batch_size = 1  # We decode one sentence at a time.

      # Decode input file
      num_sentences = 0
      max_sents = 0
      if 'max_sentences' in config:
        max_sents = config and config['max_sentences']
      if max_sentences > 0:
        max_sents = max_sentences

      logging.info("Start decoding, max_sentences=%i" % max_sents)
      with open(inp) as f_in, open(out, 'w') as f_out, open(hidden, 'wb') as f_hidden:
        pickler = cPickle.Pickler(f_hidden)
        for sentence in f_in:
          outputs, states = get_outputs(session, config, model, sentence, buckets)
          logging.info("Output: {}".format(outputs))
          #logging.info("States: {}".format(states))

          # If there is an EOS symbol in outputs, cut them at that point.
          if data_utils.EOS_ID in outputs:
            outputs = outputs[:outputs.index(data_utils.EOS_ID)]
          print(" ".join([str(tok) for tok in outputs]), file=f_out)
          pickler.dump(states)
          num_sentences += 1
          if max_sents > 0 and num_sentences >= max_sents:
            break
  logging.info("Decoding completed.")

def decode_hidden(config, out):
  with tf.Session() as session:
    model = model_utils.create_model(session, config,
                                     forward_only=True, hidden=True)
    model.batch_size = 1  # We decode one sentence at a time.
    with open(config['decode_hidden'], 'rb') as f_in:
      unpickler = cPickle.Unpickler(f_in)
      num_layers = config['num_layers']
      hidden_size = config['hidden_size']
      while True:
        try:
          #hidden = np.random.randn(1, 1000)
          hidden = np.array(unpickler.load())
          hidden = hidden.reshape(model.batch_size, 2*hidden_size)
          #logging.info("Hidden: {}".format(hidden))
          #inp = ' '.join(list(str(num) for num in np.random.randint(3, high=100, size=7))) # random dummy input
          outputs, states = get_outputs(session, config, model,
                                    sentence='', hidden=hidden)
          logging.info("Output: {}".format(outputs))
        except (EOFError):
          break

def decode_interactive(config):
  with tf.Session() as session:
    # Create model and load parameters: uses the training graph for decoding
    config['batch_size'] = 1 # We decode one sentence at a time.
    model = model_utils.create_model(session, config, forward_only=True)

    # Decode from standard input.
    sys.stdout.write("> ")
    sys.stdout.flush()
    sentence = sys.stdin.readline()
    while sentence:
      outputs, states = get_outputs(session, config, model, sentence)
      print("Output: %s" % " ".join([str(tok) for tok in outputs]))
      print("> ", end="")
      sys.stdout.flush()
      sentence = sys.stdin.readline()

def get_outputs(session, config, model, sentence, buckets=None, hidden=None):
  # Get token-ids for the input sentence.
  token_ids = [ int(tok) for tok in sentence.strip().split() ]
  token_ids = [ w if w < config['src_vocab_size'] else data_utils.UNK_ID
                for w in token_ids ]
  if config['add_src_eos']:
    token_ids.append(data_utils.EOS_ID)

  if not buckets:
    buckets = model_utils._buckets
  bucket_id = min([b for b in xrange(len(buckets))
                  if buckets[b][0] >= len(token_ids)])
  logging.info("Bucket {}".format(buckets[bucket_id]))
  logging.info("Input: {}".format(token_ids))

  # Get a 1-element batch to feed the sentence to the model.
  encoder_inputs, decoder_inputs, target_weights, sequence_length, src_mask, bow_mask = model.get_batch(
    {bucket_id: [(token_ids, [])]}, bucket_id, config['encoder'])

  # Get output logits for the sentence.
  _, _, output_logits, hidden_states = model.get_state_step(
    session, encoder_inputs,
    decoder_inputs,
    target_weights, bucket_id,
    forward_only=True,
    sequence_length=sequence_length,
    src_mask=src_mask, bow_mask=bow_mask, hidden=hidden)

  # This is a greedy decoder - outputs are just argmaxes of output_logits.
  outputs = [int(np.argmax(logit, axis=1)) for logit in output_logits]
  return outputs, hidden_states

def main(_):
  config = model_utils.process_args(FLAGS, train=False, greedy_decoder=True)
  if FLAGS.interactive:
    decode_interactive(config)
  else:
    decode(config)

if __name__ == "__main__":
  logging.getLogger().setLevel(logging.INFO)
  logging.info("Start: {}".format(datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S')))
  tf.app.run()
  logging.info("End: {}".format(datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S')))

<commit_msg>save output decoded from hidden state to file<commit_after>"""Binary for decoding from translation models based on tensorflow/models/rnn/translate/translate.py.

Note that this decoder is greedy and very basic. For a better decoder, see http://ucam-smt.github.io/sgnmt/html/tutorial.html
which supports decoding from tensorflow models.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import cPickle
import sys
import numpy as np
import datetime
import logging

from six.moves import xrange  # pylint: disable=redefined-builtin
import tensorflow as tf

if __name__ == "__main__":
  from tensorflow.models.rnn.translate.train import FLAGS as train_flags # get default model settings from train.py
from tensorflow.models.rnn.translate.utils import data_utils, model_utils

# Decoder settings
tf.app.flags.DEFINE_string("test_src_idx", "/tmp/in.txt", "An integer-encoded input file")
tf.app.flags.DEFINE_string("test_out_idx", "/tmp/out.txt", "Output file for decoder output")
tf.app.flags.DEFINE_string("test_hidden_state", "/tmp/hidden.txt", "Output file for hidden state")
tf.app.flags.DEFINE_integer("max_sentences", 0, "The maximum number of sentences to translate (all if set to 0)")
tf.app.flags.DEFINE_string("decode_hidden", None, "Decode from hidden layers in file")
tf.app.flags.DEFINE_boolean("interactive", False, "Decode from command line")
tf.app.flags.DEFINE_string("model_path", None, "Path to trained model")
FLAGS = tf.app.flags.FLAGS

def decode(config, input=None, output=None, max_sentences=0):
  if input and output:
    inp = input
    out = output
  else:
    inp = config['test_src_idx']
    out = config['test_out_idx']
  if 'test_hidden_state' in config:
    hidden = config['test_hidden_state']
  else:
    hidden = '/tmp/hidden'

  if config['decode_hidden']:
    decode_hidden(config, out)
  else:
    # Find longest input to create suitable bucket
    max_input_length = 0
    with open(inp) as f_in:
      for sentence in f_in:
        token_ids = [ int(tok) for tok in sentence.strip().split() ]
        if config['add_src_eos']:
          token_ids.append(data_utils.EOS_ID)
        if len(token_ids) > max_input_length:
          max_input_length = len(token_ids)
    bucket = model_utils.make_bucket(max_input_length, greedy_decoder=True)
    buckets = list(model_utils._buckets)
    buckets.append(bucket)
    logging.info("Add new bucket={}".format(bucket))

    with tf.Session() as session:
      # Create model and load parameters: uses the training graph for decoding
      model = model_utils.create_model(session, config, forward_only=True,
                                          buckets=buckets)
      model.batch_size = 1  # We decode one sentence at a time.

      # Decode input file
      num_sentences = 0
      max_sents = 0
      if 'max_sentences' in config:
        max_sents = config and config['max_sentences']
      if max_sentences > 0:
        max_sents = max_sentences

      logging.info("Start decoding, max_sentences=%i" % max_sents)
      with open(inp) as f_in, open(out, 'w') as f_out, open(hidden, 'wb') as f_hidden:
        pickler = cPickle.Pickler(f_hidden)
        for sentence in f_in:
          outputs, states = get_outputs(session, config, model, sentence, buckets)
          logging.info("Output: {}".format(outputs))
          # If there is an EOS symbol in outputs, cut them at that point.
          if data_utils.EOS_ID in outputs:
            outputs = outputs[:outputs.index(data_utils.EOS_ID)]
          print(" ".join([str(tok) for tok in outputs]), file=f_out)
          pickler.dump(states)
          num_sentences += 1
          if max_sents > 0 and num_sentences >= max_sents:
            break
  logging.info("Decoding completed.")

def decode_hidden(config, out):
  with tf.Session() as session:
    model = model_utils.create_model(session, config,
                                     forward_only=True, hidden=True)
    model.batch_size = 1  # We decode one sentence at a time.
    with open(config['decode_hidden'], 'rb') as f_in, open(config['test_out_idx'], 'w') as f_out:
      unpickler = cPickle.Unpickler(f_in)
      num_layers = config['num_layers']
      hidden_size = config['hidden_size']
      while True:
        try:
          #hidden = np.random.randn(1, 1000)
          hidden = np.array(unpickler.load())
          hidden = hidden.reshape(model.batch_size, 2*hidden_size)
          #logging.info("Hidden: {}".format(hidden))
          #inp = ' '.join(list(str(num) for num in np.random.randint(3, high=100, size=7))) # random dummy input
          outputs, states = get_outputs(session, config, model,
                                    sentence='', hidden=hidden)
          logging.info("Output: {}".format(outputs))
          if data_utils.EOS_ID in outputs:
            outputs = outputs[:outputs.index(data_utils.EOS_ID)]
          print(" ".join([str(tok) for tok in outputs]), file=f_out)
        except (EOFError):
          break

def decode_interactive(config):
  with tf.Session() as session:
    # Create model and load parameters: uses the training graph for decoding
    config['batch_size'] = 1 # We decode one sentence at a time.
    model = model_utils.create_model(session, config, forward_only=True)

    # Decode from standard input.
    sys.stdout.write("> ")
    sys.stdout.flush()
    sentence = sys.stdin.readline()
    while sentence:
      outputs, states = get_outputs(session, config, model, sentence)
      print("Output: %s" % " ".join([str(tok) for tok in outputs]))
      print("> ", end="")
      sys.stdout.flush()
      sentence = sys.stdin.readline()

def get_outputs(session, config, model, sentence, buckets=None, hidden=None):
  # Get token-ids for the input sentence.
  token_ids = [ int(tok) for tok in sentence.strip().split() ]
  token_ids = [ w if w < config['src_vocab_size'] else data_utils.UNK_ID
                for w in token_ids ]
  if config['add_src_eos']:
    token_ids.append(data_utils.EOS_ID)

  if not buckets:
    buckets = model_utils._buckets
  bucket_id = min([b for b in xrange(len(buckets))
                  if buckets[b][0] >= len(token_ids)])
  logging.info("Bucket {}".format(buckets[bucket_id]))
  logging.info("Input: {}".format(token_ids))

  # Get a 1-element batch to feed the sentence to the model.
  encoder_inputs, decoder_inputs, target_weights, sequence_length, src_mask, bow_mask = model.get_batch(
    {bucket_id: [(token_ids, [])]}, bucket_id, config['encoder'])

  # Get output logits for the sentence.
  _, _, output_logits, hidden_states = model.get_state_step(
    session, encoder_inputs,
    decoder_inputs,
    target_weights, bucket_id,
    forward_only=True,
    sequence_length=sequence_length,
    src_mask=src_mask, bow_mask=bow_mask, hidden=hidden)

  # This is a greedy decoder - outputs are just argmaxes of output_logits.
  outputs = [int(np.argmax(logit, axis=1)) for logit in output_logits]
  return outputs, hidden_states

def main(_):
  config = model_utils.process_args(FLAGS, train=False, greedy_decoder=True)
  if FLAGS.interactive:
    decode_interactive(config)
  else:
    decode(config)

if __name__ == "__main__":
  logging.getLogger().setLevel(logging.INFO)
  logging.info("Start: {}".format(datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S')))
  tf.app.run()
  logging.info("End: {}".format(datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S')))

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""Machine learning model which uses Wikipedia data to predicts ILI levels.

Usage:
  model.py <year_start> <year_end> <dataset_path> <incidence_path> <keywords_file> [--p] [--f] [--v] [--d=<directory>] [--no-future]

  <year_start>      The first influenza season we want to predict.
  <year_end>        The last influenza season we want to predict.
  <dataset_path>    The path to dataset files (the ones which contain Wikipedia pageviews).
  <incidence_path>  The path to the files which specify the ILI levels for each week.
  <keywords_file>   The path to the file which contains the Wikipedia pages used.
  -p, --poisson     Use the Poisson model + LASSO instead of the linear one.
  -f, --file        Write informations to file
  -v, --verbose     Output more informations
  -d, --directory   Select a directory in which save your files
  -n, --no-future   Use a different method to train the model (avoid using seasonal influenza which are
                    later than the one we want to predict)
  -h, --help        Print this help message
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import csv
from sklearn.linear_model import LassoCV
from sklearn.metrics import mean_squared_error
from tabulate import tabulate
from docopt import docopt

from models_utils import generate, generate_labels, generate_labels_one_year, generate_one_year, standardize_data, generate_keywords, stz, get_important_pages, correlation_matrix

from cvglmnetCoef import cvglmnetCoef
from cvglmnetPredict import cvglmnetPredict
from cvglmnet import cvglmnet

#########################
#### INITIALIZATION #####
#########################

# Parse the command line
arguments = docopt(__doc__)

# Directory where to save files
save_directory="./"
if arguments["--d"]:
    save_directory = arguments["--d"]

# Year selected
year_start = int(arguments["<year_start>"])
year_end = int(arguments["<year_end>"])+1

# Model type
model_type = "Linear Model"
using_poisson = False
if arguments["--p"]:
    using_poisson = True
    model_type="Poisson Model"

# Feature path and labels
path_features = arguments["<dataset_path>"]
path_labels = arguments["<incidence_path>"]
keywords = arguments["<keywords_file>"]

# Selected columns that will be extracted from the dataframes
selected_columns = generate_keywords(keywords)

all_features_values = pd.DataFrame()
all_predicted_values = []
all_true_labels = pd.DataFrame()
total_weeks = []
all_weighted_feature=dict()

# The pair start-end seasons
year_sel = (year_start, year_end)

# Useful information which can be written to file
peaks = []
predicted_peaks = []
peaks_value = []
predicted_peaks_value = []
mse_seasons = []

######################
##### ALGORITHM ######
######################

# For each season, train a model with the complete dataset (without the season)
# and predict the ILI incidence.
for year_selected in range(year_sel[0], year_sel[1]):

    print("------------")
    print("[*] ", year_selected )

    # Generate the list of excluded years
    excluded=[]
    if arguments["--no-future"]:
        for i in range(2007, 2018):
            if i>year_selected:
                excluded.append(i)

    # Data generation from data files
    dataset = generate(year_selected, excluded, path_features)[selected_columns]
    labels = generate_labels(year_selected, excluded, path_labels)
    data = generate_one_year(year_selected, path_features)[selected_columns]
    labels_test = generate_labels_one_year(year_selected, path_labels)
    weeks = labels_test["week"]

    # Create a copy of the dataset where we fills NaN values
    # with 0.
    dataset_zero = dataset.fillna(0)
    data_zero = data.fillna(0)

    # Standardize data
    train, test = standardize_data(dataset_zero, data_zero)

    # Create a Lasso Cross-Validation instance which will be
    # trained on the dataset in which NaN values are replaced
    # with the column mean.
    if not using_poisson:
        model = LassoCV(max_iter=100000, n_jobs=-1, n_alphas=1000, random_state=1)
        model.fit(train, labels["incidence"].fillna(0))
        result = model.predict(test)
    else:
        model = cvglmnet(x=train.as_matrix().copy(), y=labels["incidence"].fillna(0).copy().as_matrix(), family='poisson', alpha=1.0,
                       ptype="mse", parallel=True, nfolds=10)
        result = cvglmnetPredict(model, test.as_matrix(), ptype='response', s="lambda_min")
        coeff = cvglmnetCoef(model, s="lambda_min")

    # Get the feature coefficients
    if not using_poisson:
        coeff = model.coef_
    else:
        coeff = cvglmnetCoef(model, s="lambda_min")

    # Add the pair (value, coeff) to a dictionary
    for i in list(zip(coeff, selected_columns)):
        if (i[0] != 0):
            if all_weighted_feature.get(i[1], None) is None:
                all_weighted_feature[i[1]] = [i[0]]
            else:
                all_weighted_feature[i[1]].append(i[0])

    # Add the value to global variables
    all_features_values= all_features_values.append(data.fillna(0))
    all_predicted_values.extend(result)
    all_true_labels = all_true_labels.append(labels_test)
    total_weeks.extend(weeks)

    mse_seasons.append(mean_squared_error(labels_test['incidence'].fillna(0), result))

    # If we are verbose print more informations
    index_peak = labels_test['incidence'].fillna(0).idxmax(axis=1)
    index_peak_m = np.argmax(result)
    peaks.append(labels_test.iloc[index_peak]["week"])
    predicted_peaks.append(labels_test.iloc[index_peak_m]["week"])
    peaks_value.append(labels_test.iloc[index_peak]["incidence"])
    predicted_peaks_value.append(result[index_peak_m])
    if arguments["--v"]:
        print("[*] Influenza Season Peak (Week):", labels_test.iloc[index_peak]["week"])
        print("[*] Influenza Seasons Predicted Peak (Week): ", labels_test.iloc[index_peak_m]["week"])
        print("[*] Influenza Season Peak Value: ", labels_test.iloc[index_peak]["incidence"])
        print("[*] Influenza Season Predicted Peak Value: ", result[index_peak_m])
        print("------------")

# Get important pages to generate the plot we need
important_pages = get_important_pages(all_weighted_feature, 5)

# Print MSE of the two models
mse = mean_squared_error(all_true_labels["incidence"].fillna(0), all_predicted_values)
print("------------")
print("MSE: ", mse)
print("------------")

# Print Pearson Coefficient
pcc = np.corrcoef(all_predicted_values, all_true_labels["incidence"].fillna(0), rowvar=False)[0][1]
print("------------")
print("Pearson Correlation Coeff: ", pcc)
print("------------")

# Print important pages
print(tabulate(important_pages, headers=["Page name", "Weight"]))

###########################
#### GRAPHS GENERATION ####
###########################

# Plot some informations
fig = plt.figure(3, figsize=(15, 6))
ax = fig.add_subplot(111)

# Set up the graph title, x and y axis titles
plt.title("Influenza Seasons "+str(year_sel[0]-1)+" - "+str(year_sel[1]-1), fontsize=18)
plt.ylabel("Incidence on 100000 people", fontsize=17)
plt.xlabel("Year-Week", fontsize=17)

# Generate the x axis labels in such way to
# have the year-week pair only every two ticks.
weeks_used = []
for k, v in enumerate(total_weeks):
    if k%2==0:
        weeks_used.append(v)
    else:
        weeks_used.append(" ")

# Set up the axes ticks
plt.xticks(range(0, len(weeks_used)), weeks_used, rotation="vertical", fontsize=15)
plt.yticks(fontsize=15)

# Plot the model result and the incidence
plt.plot(range(0, len(all_predicted_values)), all_predicted_values, 'r-', label=model_type, linewidth=3)
plt.plot(range(0, len(all_true_labels["incidence"])), all_true_labels["incidence"].fillna(0), 'k-', label="Incidence", linewidth=3)

# Add dotted line into the graph to delimit the influenza seasons.
for i in range(1, (year_sel[1]-year_sel[0])):
    plt.axvline(26*i, linestyle="dashed", color="k", linewidth=3)

# Update some graphical options
plt.grid()
plt.legend(fontsize=16, loc="upper right")
plt.tight_layout()

# Save the graph on file
plt.savefig(save_directory+"season_"+str(year_sel[0]-1)+"-"+str(year_sel[1]-1)+".png", dpi=150)

# Feature plot
figf = plt.figure(4, figsize=(15, 6))
axf = fig.add_subplot(111)

# Generate the graph showing the pageview variation of the top 5 Wikipedia's pages
# choosen by the model.
plt.title("Pageview variation "+str(year_sel[0]-1)+" - "+str(year_sel[1]-1), fontsize=18)
plt.ylabel("Pageview value", fontsize=17)
plt.xlabel("Year-Week", fontsize=17)

# Plot the axes labels
plt.xticks(range(0, len(weeks_used)), weeks_used, rotation="vertical", fontsize=15)
plt.yticks(fontsize=15)

# Plot all the pageview data and the ILI incidence
std_all_features_values = stz(all_features_values)
for key, value in important_pages:
 plt.plot(range(0, len(std_all_features_values[key])), std_all_features_values[key], '-', label=key, linewidth=3)
plt.plot(range(0, len(all_true_labels["incidence"])), stz(all_true_labels["incidence"]), 'k-', label="Incidence", linewidth=3)

# Add dotted line into the graph to delimit the influenza seasons.
for i in range(1, (year_sel[1]-year_sel[0])):
    plt.axvline(26*i, linestyle="dashed", color="k", linewidth=3)

# Update some graphical options
plt.grid()
plt.legend(fontsize=16, loc="best")
plt.tight_layout()

# Save again the graph on file
plt.savefig(save_directory+"season_"+str(year_sel[0]-1)+"-"+str(year_sel[1]-1)+"_features.png", dpi=150)

# If --f then we write some information to file
if arguments["--f"]:
    with open(save_directory+str(year_sel[0]-1)+"-"+str(year_sel[1]-1)+"_information.csv", 'w', newline='') as csvfile:
        spamwriter = csv.writer(csvfile, delimiter=',',
                                quotechar='|', quoting=csv.QUOTE_MINIMAL)
        spamwriter.writerow(['real_influenza_peak', peaks])
        spamwriter.writerow(['predicted_influenza_peak', predicted_peaks])
        spamwriter.writerow(['real_influenza_peak_value', peaks_value])
        spamwriter.writerow(['predicted_influenza_peak_value', predicted_peaks_value])
        spamwriter.writerow(['mse_all_seasons', mse_seasons])
        spamwriter.writerow(['mse', mean_squared_error(all_true_labels["incidence"].fillna(0), all_predicted_values)])
        spamwriter.writerow(['pearson', pcc])
        for p in important_pages:
            spamwriter.writerow([p[0], float(p[1])])

# Correlation Matrix
new_pages=[]
for p in important_pages:
    new_pages.append(p[0])
first = pd.DataFrame(all_features_values[new_pages])
second = pd.DataFrame(all_true_labels["incidence"]).set_index(first.index.values)
first['incidence'] = second
first = first.loc[:, (first != 0).any(axis=0)]
new_pages = list(first.columns.values)
correlation_matrix(first, "Correlation Matrix "+str(year_sel[0]-1)+"-"+str(year_sel[1]-1), new_pages, save_directory+"cmatrix_"+str(year_sel[0]-1)+"-"+str(year_sel[1]-1)+".png")<commit_msg>[model] Add --no-images function to avoid printing all the graphs.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""Machine learning model which uses Wikipedia data to predicts ILI levels.

Usage:
  model.py <year_start> <year_end> <dataset_path> <incidence_path> <keywords_file> [--p] [--f] [--v] [--d=<directory>] [--no-future] [--no-images]

  <year_start>      The first influenza season we want to predict.
  <year_end>        The last influenza season we want to predict.
  <dataset_path>    The path to dataset files (the ones which contain Wikipedia pageviews).
  <incidence_path>  The path to the files which specify the ILI levels for each week.
  <keywords_file>   The path to the file which contains the Wikipedia pages used.
  -p, --poisson     Use the Poisson model + LASSO instead of the linear one.
  -f, --file        Write informations to file
  -v, --verbose     Output more informations
  -d, --directory   Select a directory in which save your files
  -n, --no-future   Use a different method to train the model (avoid using seasonal influenza which are
                    later than the one we want to predict)
  --no-images       Do not generate any graphs. Print only to screen.
  -h, --help        Print this help message
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import csv
from sklearn.linear_model import LassoCV
from sklearn.metrics import mean_squared_error
from tabulate import tabulate
from docopt import docopt

from models_utils import generate, generate_labels, generate_labels_one_year, generate_one_year, standardize_data, generate_keywords, stz, get_important_pages, correlation_matrix

from cvglmnetCoef import cvglmnetCoef
from cvglmnetPredict import cvglmnetPredict
from cvglmnet import cvglmnet

#########################
#### INITIALIZATION #####
#########################

# Parse the command line
arguments = docopt(__doc__)

# Directory where to save files
save_directory="./"
if arguments["--d"]:
    save_directory = arguments["--d"]

# Year selected
year_start = int(arguments["<year_start>"])
year_end = int(arguments["<year_end>"])+1

# Model type
model_type = "Linear Model"
using_poisson = False
if arguments["--p"]:
    using_poisson = True
    model_type="Poisson Model"

# Feature path and labels
path_features = arguments["<dataset_path>"]
path_labels = arguments["<incidence_path>"]
keywords = arguments["<keywords_file>"]

# Selected columns that will be extracted from the dataframes
selected_columns = generate_keywords(keywords)

all_features_values = pd.DataFrame()
all_predicted_values = []
all_true_labels = pd.DataFrame()
total_weeks = []
all_weighted_feature=dict()

# The pair start-end seasons
year_sel = (year_start, year_end)

# Useful information which can be written to file
peaks = []
predicted_peaks = []
peaks_value = []
predicted_peaks_value = []
mse_seasons = []

######################
##### ALGORITHM ######
######################

# For each season, train a model with the complete dataset (without the season)
# and predict the ILI incidence.
for year_selected in range(year_sel[0], year_sel[1]):

    print("------------")
    print("[*] ", year_selected )

    # Generate the list of excluded years
    excluded=[]
    if arguments["--no-future"]:
        for i in range(2007, 2018):
            if i>year_selected:
                excluded.append(i)

    # Data generation from data files
    dataset = generate(year_selected, excluded, path_features)[selected_columns]
    labels = generate_labels(year_selected, excluded, path_labels)
    data = generate_one_year(year_selected, path_features)[selected_columns]
    labels_test = generate_labels_one_year(year_selected, path_labels)
    weeks = labels_test["week"]

    # Create a copy of the dataset where we fills NaN values
    # with 0.
    dataset_zero = dataset.fillna(0)
    data_zero = data.fillna(0)

    # Standardize data
    train, test = standardize_data(dataset_zero, data_zero)

    # Create a Lasso Cross-Validation instance which will be
    # trained on the dataset in which NaN values are replaced
    # with the column mean.
    if not using_poisson:
        model = LassoCV(max_iter=100000, n_jobs=-1, n_alphas=1000, random_state=1)
        model.fit(train, labels["incidence"].fillna(0))
        result = model.predict(test)
    else:
        model = cvglmnet(x=train.values.copy(), y=labels["incidence"].fillna(0).copy().values, family='poisson', alpha=1.0,
                       ptype="mse", parallel=True, nfolds=10)
        result = cvglmnetPredict(model, test.values, ptype='response', s="lambda_min")
        coeff = cvglmnetCoef(model, s="lambda_min")

    # Get the feature coefficients
    if not using_poisson:
        coeff = model.coef_
    else:
        coeff = cvglmnetCoef(model, s="lambda_min")

    # Add the pair (value, coeff) to a dictionary
    for i in list(zip(coeff, selected_columns)):
        if (i[0] != 0):
            if all_weighted_feature.get(i[1], None) is None:
                all_weighted_feature[i[1]] = [i[0]]
            else:
                all_weighted_feature[i[1]].append(i[0])

    # Add the value to global variables
    all_features_values= all_features_values.append(data.fillna(0))
    all_predicted_values.extend(result)
    all_true_labels = all_true_labels.append(labels_test)
    total_weeks.extend(weeks)

    mse_seasons.append(mean_squared_error(labels_test['incidence'].fillna(0), result))

    # If we are verbose print more informations
    index_peak = labels_test['incidence'].fillna(0).idxmax(axis=1)
    index_peak_m = np.argmax(result)
    peaks.append(labels_test.iloc[index_peak]["week"])
    predicted_peaks.append(labels_test.iloc[index_peak_m]["week"])
    peaks_value.append(labels_test.iloc[index_peak]["incidence"])
    predicted_peaks_value.append(result[index_peak_m])
    if arguments["--v"]:
        print("[*] Influenza Season Peak (Week):", labels_test.iloc[index_peak]["week"])
        print("[*] Influenza Seasons Predicted Peak (Week): ", labels_test.iloc[index_peak_m]["week"])
        print("[*] Influenza Season Peak Value: ", labels_test.iloc[index_peak]["incidence"])
        print("[*] Influenza Season Predicted Peak Value: ", result[index_peak_m])
        print("------------")

# Get important pages to generate the plot we need
important_pages = get_important_pages(all_weighted_feature, 5)

# Print MSE of the two models
mse = mean_squared_error(all_true_labels["incidence"].fillna(0), all_predicted_values)
print("------------")
print("MSE: ", mse)
print("------------")

# Print Pearson Coefficient
pcc = np.corrcoef(all_predicted_values, all_true_labels["incidence"].fillna(0), rowvar=False)[0][1]
print("------------")
print("Pearson Correlation Coeff: ", pcc)
print("------------")

# Print important pages
print(tabulate(important_pages, headers=["Page name", "Weight"]))

###########################
#### GRAPHS GENERATION ####
###########################

if not arguments["--no-images"]:
    # Plot some informations
    fig = plt.figure(3, figsize=(15, 6))
    ax = fig.add_subplot(111)

    # Set up the graph title, x and y axis titles
    plt.title("Influenza Seasons "+str(year_sel[0]-1)+" - "+str(year_sel[1]-1), fontsize=18)
    plt.ylabel("Incidence on 100000 people", fontsize=17)
    plt.xlabel("Year-Week", fontsize=17)

    # Generate the x axis labels in such way to
    # have the year-week pair only every two ticks.
    weeks_used = []
    for k, v in enumerate(total_weeks):
        if k%2==0:
            weeks_used.append(v)
        else:
            weeks_used.append(" ")

    # Set up the axes ticks
    plt.xticks(range(0, len(weeks_used)), weeks_used, rotation="vertical", fontsize=15)
    plt.yticks(fontsize=15)

    # Plot the model result and the incidence
    plt.plot(range(0, len(all_predicted_values)), all_predicted_values, 'r-', label=model_type, linewidth=3)
    plt.plot(range(0, len(all_true_labels["incidence"])), all_true_labels["incidence"].fillna(0), 'k-', label="Incidence", linewidth=3)

    # Add dotted line into the graph to delimit the influenza seasons.
    for i in range(1, (year_sel[1]-year_sel[0])):
        plt.axvline(26*i, linestyle="dashed", color="k", linewidth=3)

    # Update some graphical options
    plt.grid()
    plt.legend(fontsize=16, loc="upper right")
    plt.tight_layout()

    # Save the graph on file
    plt.savefig(save_directory+"season_"+str(year_sel[0]-1)+"-"+str(year_sel[1]-1)+".png", dpi=150)

    # Feature plot
    figf = plt.figure(4, figsize=(15, 6))
    axf = figf.add_subplot(111)

    # Generate the graph showing the pageview variation of the top 5 Wikipedia's pages
    # choosen by the model.
    plt.title("Pageview variation "+str(year_sel[0]-1)+" - "+str(year_sel[1]-1), fontsize=18)
    plt.ylabel("Pageview value", fontsize=17)
    plt.xlabel("Year-Week", fontsize=17)

    # Plot the axes labels
    plt.xticks(range(0, len(weeks_used)), weeks_used, rotation="vertical", fontsize=15)
    plt.yticks(fontsize=15)

    # Plot all the pageview data and the ILI incidence
    std_all_features_values = stz(all_features_values)
    for key, value in important_pages:
     plt.plot(range(0, len(std_all_features_values[key])), std_all_features_values[key], '-', label=key, linewidth=3)
    plt.plot(range(0, len(all_true_labels["incidence"])), stz(all_true_labels["incidence"]), 'k-', label="Incidence", linewidth=3)

    # Add dotted line into the graph to delimit the influenza seasons.
    for i in range(1, (year_sel[1]-year_sel[0])):
        plt.axvline(26*i, linestyle="dashed", color="k", linewidth=3)

    # Update some graphical options
    plt.grid()
    plt.legend(fontsize=16, loc="best")
    plt.tight_layout()

    # Save again the graph on file
    plt.savefig(save_directory+"season_"+str(year_sel[0]-1)+"-"+str(year_sel[1]-1)+"_features.png", dpi=150)

    # Correlation Matrix
    new_pages=[]
    for p in important_pages:
        new_pages.append(p[0])
    first = pd.DataFrame(all_features_values[new_pages])
    second = pd.DataFrame(all_true_labels["incidence"]).set_index(first.index.values)
    first['incidence'] = second
    first = first.loc[:, (first != 0).any(axis=0)]
    new_pages = list(first.columns.values)
    correlation_matrix(first, "Correlation Matrix "+str(year_sel[0]-1)+"-"+str(year_sel[1]-1), new_pages, save_directory+"cmatrix_"+str(year_sel[0]-1)+"-"+str(year_sel[1]-1)+".png")

###################
#  SAVE TO FILE   #
###################

# If --f then we write some information to file
if arguments["--f"]:
    with open(save_directory+str(year_sel[0]-1)+"-"+str(year_sel[1]-1)+"_information.csv", 'w', newline='') as csvfile:
        spamwriter = csv.writer(csvfile, delimiter=',',
                                quotechar='|', quoting=csv.QUOTE_MINIMAL)
        spamwriter.writerow(['real_influenza_peak', peaks])
        spamwriter.writerow(['predicted_influenza_peak', predicted_peaks])
        spamwriter.writerow(['real_influenza_peak_value', peaks_value])
        spamwriter.writerow(['predicted_influenza_peak_value', predicted_peaks_value])
        spamwriter.writerow(['mse_all_seasons', mse_seasons])
        spamwriter.writerow(['mse', mean_squared_error(all_true_labels["incidence"].fillna(0), all_predicted_values)])
        spamwriter.writerow(['pearson', pcc])
        for p in important_pages:
            spamwriter.writerow([p[0], float(p[1])])

<|endoftext|>


<commit_before># -*- mode: python; indent-tabs-mode: nil -*-

__all__ = ('decode_id13',)

def _make_upper_table():
    ut = []
    for i in range(32):
        v = 0
        id13 = i << 8
        if id13 & 0x1000:
            v |= 0x0010  # C1
        if id13 & 0x0800:
            v |= 0x1000  # A1
        if id13 & 0x0400:
            v |= 0x0020  # C2
        if id13 & 0x0200:
            v |= 0x2000  # A2
        if id13 & 0x0100:
            v |= 0x0040  # C4
        ut.append(v)
    return ut

def _make_lower_table():
    lt = []
    for id13 in range(256):
        v = 0
        if id13 & 0x0080:
            v |= 0x4000  # A4
        # 0040 unused (M/X)
        if id13 & 0x0020:
            v |= 0x0100  # B1
        if id13 & 0x0010:
            v |= 0x0001  # D1/Q
        if id13 & 0x0008:
            v |= 0x0200  # B2
        if id13 & 0x0004:
            v |= 0x0002  # D2
        if id13 & 0x0002:
            v |= 0x0400  # B4
        if id13 & 0x0001:
            v |= 0x0004  # D4
        lt.append(v)

    return lt

def decode_id13(id13):
    """Decode a 13-bit Mode A squawk.

    The expected ordering is that from Annex 10 vol 4 3.1.2.6.7.1:

      C1, A1, C2, A2, C4, A4, ZERO, B1, D1, B2, D2, B4, D4

    Returns the squawk as a 4-character string."""

    return '{0:04x}'.format(_lt[id13 & 255] | _ut[id13 >> 8])

if __name__ == '__main__':
    _lt = _make_lower_table()
    _ut = _make_upper_table()

    print('# -*- mode: python; indent-tabs-mode: nil -*-')
    print('# generated by modes.squawk: python3 -m modes.squawk')
    print()
    print('lt = (')
    for i in range(0, len(_lt), 8):
        print('    ' + ', '.join(['0x{0:04x}'.format(v) for v in _lt[i:i+8]]) + ',')
    print(')')
    print()
    print('ut = (')
    for i in range(0, len(_ut), 8):
        print('    ' + ', '.join(['0x{0:04x}'.format(v) for v in _ut[i:i+8]]) + ',')
    print(')')

else:
    try:
        from .squawk_lookup import ut as _ut, lt as _lt
    except ImportError:
        _lt = _make_lower_table()
        _ut = _make_upper_table()
<commit_msg>Squawk tables can be smaller (32+256 -> 64+64)<commit_after># -*- mode: python; indent-tabs-mode: nil -*-

__all__ = ('decode_id13',)

def _make_upper_table():
    ut = []
    for i in range(64):
        v = 0
        id13 = i << 7
        if id13 & 0x1000:
            v |= 0x0010  # C1
        if id13 & 0x0800:
            v |= 0x1000  # A1
        if id13 & 0x0400:
            v |= 0x0020  # C2
        if id13 & 0x0200:
            v |= 0x2000  # A2
        if id13 & 0x0100:
            v |= 0x0040  # C4
        if id13 & 0x0080:
            v |= 0x4000  # A4
        ut.append(v)
    return ut

def _make_lower_table():
    lt = []
    for id13 in range(64):
        v = 0
        # 0040 unused (M/X)
        if id13 & 0x0020:
            v |= 0x0100  # B1
        if id13 & 0x0010:
            v |= 0x0001  # D1/Q
        if id13 & 0x0008:
            v |= 0x0200  # B2
        if id13 & 0x0004:
            v |= 0x0002  # D2
        if id13 & 0x0002:
            v |= 0x0400  # B4
        if id13 & 0x0001:
            v |= 0x0004  # D4
        lt.append(v)

    return lt

def decode_id13(id13):
    """Decode a 13-bit Mode A squawk.

    The expected ordering is that from Annex 10 vol 4 3.1.2.6.7.1:

      C1, A1, C2, A2, C4, A4, ZERO, B1, D1, B2, D2, B4, D4

    Returns the squawk as a 4-character string."""

    return '{0:04x}'.format(_lt[id13 & 63] | _ut[id13 >> 7])

if __name__ == '__main__':
    _lt = _make_lower_table()
    _ut = _make_upper_table()

    print('# -*- mode: python; indent-tabs-mode: nil -*-')
    print('# generated by modes.squawk: python3 -m modes.squawk')
    print()
    print('lt = (')
    for i in range(0, len(_lt), 8):
        print('    ' + ', '.join(['0x{0:04x}'.format(v) for v in _lt[i:i+8]]) + ',')
    print(')')
    print()
    print('ut = (')
    for i in range(0, len(_ut), 8):
        print('    ' + ', '.join(['0x{0:04x}'.format(v) for v in _ut[i:i+8]]) + ',')
    print(')')

else:
    try:
        from .squawk_lookup import ut as _ut, lt as _lt
    except ImportError:
        _lt = _make_lower_table()
        _ut = _make_upper_table()
<|endoftext|>


<commit_before>import math
import operator
import re
import subprocess

class Calculator:
	"""A postfix calculator engine."""
	opers = [
		{ # no operands
			"e": math.e, "pi": math.pi, "i": lambda: 1j
		},
		{ # 1 operand
			"~": lambda x: ~x, "sqrt": math.sqrt, "ln": math.log, "log": math.log10,
			"sin": math.sin, "cos": math.cos, "tan": math.tan, "exp": math.exp,
			"asin": math.asin, "acos": math.acos, "atan": math.atan,
			"rad": math.radians, "deg": math.degrees, "floor": math.floor,
			"ceil": math.ceil, "abs": math.fabs, "!": math.factorial,
			"1/x": lambda x: 1/x, "inv": lambda x: 1/x,
		},
		{ # 2 operands
			"+": operator.add, "-": operator.sub, "*": operator.mul,
			"/": operator.truediv, "\\": operator.floordiv, "%": operator.mod,
			"**": operator.pow, "^": operator.xor, "&": operator.and_,
			"|": operator.or_,
		}
	]
	opers_ = {
		"_": lambda self: self.stack[-1],
	}
	bases = {"b": 2, "o": 8, "h": 16}

	def __init__(self):
		self.stack = []

	def calc(self, expr):
		self.stack = []
		for token in expr.split():
			if token in self.opers[0]:
				self.stack.append(self.opers[0][token]())
			elif token in self.opers[1]:
				self.stack.append(self.opers[1][token](self.stack.pop()))
			elif token in self.opers[2]:
				self.stack[-2:] = [self.opers[2][token](*self.stack[-2:])]
			elif token in self.opers_:
				self.stack.append(self.opers_[token](self))
			else:
				try:
					val = None
					if token[-1] in "boh":
						val = int(token[:-1], self.bases[token[-1]])
					else:
						val = float(token)
					self.stack.append(val)
				except ValueError:
					pass
		self.stack = [int(i) for i in self.stack if float.is_integer(float(i))]

class CalcModule:
	require = "cmd"

	def __init__(self, circa):
		self.circa = circa
		self.contexts = {}

		self.events = {
			"cmd.calc": [self.calc],
			"cmd.bcalc": [self.bcalc],
			"cmd.ocalc": [self.ocalc],
			"cmd.hcalc": [self.hcalc],
		}
		self.docs = {
			"calc": "calc [expr] → evaluate the postfix expression. More info in the online docs.",
			"bcalc": "bcalc [expr] → like calc, but returns answers in binary",
			"ocalc": "ocalc [expr] → like calc, but returns answers in octal",
			"hcalc": "hcalc [expr] → like calc, but returns answers in hexadecimal",
		}

	def ensure_context(self, chan):
		if chan not in self.contexts:
			self.contexts[chan] = Calculator()

	def _calc(self, to, expr):
		self.ensure_context(to)
		self.contexts[to].calc(expr)
		return self.contexts[to].stack

	def calc(self, fr, to, expr, m):
		results = self._calc(to, expr)
		self.circa.say(to, fr + ": " + ", ".join(map(str, results)))

	def bcalc(self, fr, to, expr, m):
		results = map(lambda x: bin(x)[2:] + "b", self._calc(to, expr))
		self.circa.say(to, fr + ": " + ", ".join(map(str, results)))

	def ocalc(self, fr, to, expr, m):
		results = map(lambda x: oct(x)[2:] + "o", self._calc(to, expr))
		self.circa.say(to, fr + ": " + ", ".join(map(str, results)))

	def hcalc(self, fr, to, expr, m):
		results = map(lambda x: hex(x)[2:] + "h", self._calc(to, expr))
		self.circa.say(to, fr + ": " + ", ".join(map(str, results)))

module = CalcModule
<commit_msg>sorting and shifting operators<commit_after>import math
import operator
import re
import subprocess

class Calculator:
	"""A postfix calculator engine."""
	opers = [
		{ # no operands
			"e": math.e, "pi": math.pi, "i": lambda: 1j
		},
		{ # 1 operand
			"~": lambda x: ~x, "sqrt": math.sqrt, "ln": math.log, "log": math.log10,
			"sin": math.sin, "cos": math.cos, "tan": math.tan, "exp": math.exp,
			"asin": math.asin, "acos": math.acos, "atan": math.atan,
			"rad": math.radians, "deg": math.degrees, "floor": math.floor,
			"ceil": math.ceil, "abs": math.fabs, "!": math.factorial,
			"1/x": lambda x: 1/x, "inv": lambda x: 1/x,
		},
		{ # 2 operands
			"+": operator.add, "-": operator.sub, "*": operator.mul,
			"/": operator.truediv, "\\": operator.floordiv, "%": operator.mod,
			"**": operator.pow, "^": operator.xor, "&": operator.and_,
			"|": operator.or_, ">>": operator.rshift, "<<": operator.lshift,
		}
	]
	opers_ = {
		"_": lambda self: self.stack[-1],
		"@<": lambda self: self.sort(),
		"@>": lambda self: self.sort(True),
	}
	bases = {"b": 2, "o": 8, "h": 16}

	def __init__(self):
		self.stack = []

	def sort(self, rev=False):
		self.stack = sorted(self.stack, reverse=rev)

	def calc(self, expr):
		self.stack = []
		for token in expr.split():
			if token in self.opers[0]:
				self.stack.append(self.opers[0][token]())
			elif token in self.opers[1]:
				self.stack.append(self.opers[1][token](self.stack.pop()))
			elif token in self.opers[2]:
				self.stack[-2:] = [self.opers[2][token](*self.stack[-2:])]
			elif token in self.opers_:
				self.stack.append(self.opers_[token](self))
			else:
				try:
					val = None
					if token[-1] in "boh":
						val = int(token[:-1], self.bases[token[-1]])
					else:
						val = float(token)
					self.stack.append(val)
				except ValueError:
					pass
		self.stack = [int(i) for i in self.stack if float.is_integer(float(i))]

class CalcModule:
	require = "cmd"

	def __init__(self, circa):
		self.circa = circa
		self.contexts = {}

		self.events = {
			"cmd.calc": [self.calc],
			"cmd.bcalc": [self.bcalc],
			"cmd.ocalc": [self.ocalc],
			"cmd.hcalc": [self.hcalc],
		}
		self.docs = {
			"calc": "calc [expr] → evaluate the postfix expression. More info in the online docs.",
			"bcalc": "bcalc [expr] → like calc, but returns answers in binary",
			"ocalc": "ocalc [expr] → like calc, but returns answers in octal",
			"hcalc": "hcalc [expr] → like calc, but returns answers in hexadecimal",
		}

	def ensure_context(self, chan):
		if chan not in self.contexts:
			self.contexts[chan] = Calculator()

	def _calc(self, to, expr):
		self.ensure_context(to)
		self.contexts[to].calc(expr)
		return self.contexts[to].stack

	def calc(self, fr, to, expr, m):
		results = self._calc(to, expr)
		self.circa.say(to, fr + ": " + ", ".join(map(str, results)))

	def bcalc(self, fr, to, expr, m):
		results = map(lambda x: bin(x)[2:] + "b", self._calc(to, expr))
		self.circa.say(to, fr + ": " + ", ".join(map(str, results)))

	def ocalc(self, fr, to, expr, m):
		results = map(lambda x: oct(x)[2:] + "o", self._calc(to, expr))
		self.circa.say(to, fr + ": " + ", ".join(map(str, results)))

	def hcalc(self, fr, to, expr, m):
		results = map(lambda x: hex(x)[2:] + "h", self._calc(to, expr))
		self.circa.say(to, fr + ": " + ", ".join(map(str, results)))

module = CalcModule
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
find.py - Jenni Spell Checking Module
Copyright 2011, Michael Yanovich, yanovich.net
Licensed under the Eiffel Forum License 2.

More info:
 * Jenni: https://github.com/myano/jenni/
 * Phenny: http://inamidst.com/phenny/

Contributions from: Matt Meinwald and Morgan Goose
This module will fix spelling errors if someone corrects them
using the sed notation (s///) commonly found in vi/vim.
"""

import os, re

def give_me_unicode(obj, encoding="utf-8"):
    if isinstance(obj, basestring):
        if not isinstance(obj, unicode):
            obj = unicode(obj, encoding)
    return obj

def load_db():
    """ load lines from find.txt to search_dict """
    if not os.path.isfile("find.txt"):
        f = open("find.txt", "w")
        f.write("#test,yano,foobar\n")
        f.close()
    search_file = open("find.txt", "r")
    lines = search_file.readlines()
    search_file.close()
    search_dict = dict()
    for line in lines:
        line = give_me_unicode(line)
        a = line.replace(r'\n', '')
        new = a.split(r',')
        if len(new) < 3: continue
        channel = new[0]
        nick = new[1]
        if len(new) < 2: continue
        if channel not in search_dict:
            search_dict[channel] = dict()
        if nick not in search_dict[channel]:
            search_dict[channel][nick] = list()
        if len(new) > 3:
            result = ",".join(new[2:])
            result = result.replace('\n','')
        elif len(new) == 3:
            result = new[-1]
            if len(result) > 0:
                result = result[:-1]
        if result:
            search_dict[channel][nick].append(result)
    return search_dict

def save_db(search_dict):
    """ save search_dict to find.txt """
    search_file = open("find.txt", "w")
    for channel in search_dict:
        if channel is not "":
            for nick in search_dict[channel]:
                for line in search_dict[channel][nick]:
                    channel_utf = (channel).encode("utf-8")
                    search_file.write(channel)
                    search_file.write(",")
                    nick = (nick).encode("utf-8")
                    search_file.write(nick)
                    search_file.write(",")
                    line_utf = (line).encode("utf-8")
                    search_file.write(line_utf)
                    search_file.write("\n")
    search_file.close()

# Create a temporary log of the most recent thing anyone says.
def collectlines(jenni, input):
    # don't log things in PM
    channel = (input.sender).encode("utf-8")
    nick = (input.nick).encode("utf-8")
    if not channel.startswith('#'): return
    search_dict = load_db()
    if channel not in search_dict:
        search_dict[channel] = dict()
    if nick not in search_dict[channel]:
        search_dict[channel][nick] = list()
    templist = search_dict[channel][nick]
    line = input.group()
    if line.startswith("s/"):
        return
    else:
        templist.append(line)
    del templist[:-10]
    search_dict[channel][nick] = templist
    save_db(search_dict)
collectlines.rule = r'.*'
collectlines.priority = 'low'

def findandreplace(jenni, input):
    # don't bother in PM
    channel = (input.sender).encode("utf-8")
    nick = (input.nick).encode("utf-8")

    if not channel.startswith('#'): return

    search_dict = load_db()

    rnick = input.group(1) or nick # Correcting other person vs self.

    # only do something if there is conversation to work with
    if channel not in search_dict or rnick not in search_dict[channel]: return

    sep = input.group(2)
    rest = input.group(3).split(sep)
    me = False # /me command
    flags = ''
    if len(rest) < 2:
        return # need at least a find and replacement value
    elif len(rest) > 2:
        # Word characters immediately after the second separator
        # are considered flags (only g and i now have meaning)
        flags = re.match(r'\w*',rest[2], re.U).group(0)
    #else (len == 2) do nothing special

    count = 'g' in flags and -1 or 1 # Replace unlimited times if /g, else once
    if 'i' in flags:
        regex = re.compile(re.escape(rest[0]),re.U|re.I)
        repl = lambda s: re.sub(regex,rest[1],s,count == 1)
    else:
        repl = lambda s: s.replace(rest[0],rest[1],count)

    for line in reversed(search_dict[channel][rnick]):
        if line.startswith("\x01ACTION"):
            me = True # /me command
            line = line[8:]
        else:
            me = False
        new_phrase = repl(line)
        if new_phrase != line: # we are done
            break

    if not new_phrase or new_phrase == line: return # Didn't find anything

    # Save the new "edited" message.
    templist = search_dict[channel][rnick]
    templist.append((me and '\x01ACTION ' or '') + new_phrase)
    search_dict[channel][rnick] = templist
    save_db(search_dict)
    #search_file = open("find.txt","w")
    #pickle.dump(search_dict, search_file)
    #search_file.close()

    # output
    phrase = nick + (input.group(1) and ' thinks ' + rnick or '') + (me and ' ' or " \x02meant\x02 to say: ") + new_phrase
    if me and not input.group(1): phrase = '\x02' + phrase + '\x02'
    jenni.say(phrase)

# Matches optional whitespace + 's' + optional whitespace + separator character
findandreplace.rule = r'(?u)(?:([^\s:]+)[\s:])?\s*s\s*([^\s\w])(.*)' # May work for both this and "meant" (requires input.group(i+1))
findandreplace.priority = 'high'

if __name__ == '__main__':
    print __doc__.strip()
<commit_msg>fixed a trailing ^A on /me's s///es<commit_after>#!/usr/bin/env python
"""
find.py - Jenni Spell Checking Module
Copyright 2011, Michael Yanovich, yanovich.net
Licensed under the Eiffel Forum License 2.

More info:
 * Jenni: https://github.com/myano/jenni/
 * Phenny: http://inamidst.com/phenny/

Contributions from: Matt Meinwald and Morgan Goose
This module will fix spelling errors if someone corrects them
using the sed notation (s///) commonly found in vi/vim.
"""

import os, re

def give_me_unicode(obj, encoding="utf-8"):
    if isinstance(obj, basestring):
        if not isinstance(obj, unicode):
            obj = unicode(obj, encoding)
    return obj

def load_db():
    """ load lines from find.txt to search_dict """
    if not os.path.isfile("find.txt"):
        f = open("find.txt", "w")
        f.write("#test,yano,foobar\n")
        f.close()
    search_file = open("find.txt", "r")
    lines = search_file.readlines()
    search_file.close()
    search_dict = dict()
    for line in lines:
        line = give_me_unicode(line)
        a = line.replace(r'\n', '')
        new = a.split(r',')
        if len(new) < 3: continue
        channel = new[0]
        nick = new[1]
        if len(new) < 2: continue
        if channel not in search_dict:
            search_dict[channel] = dict()
        if nick not in search_dict[channel]:
            search_dict[channel][nick] = list()
        if len(new) > 3:
            result = ",".join(new[2:])
            result = result.replace('\n','')
        elif len(new) == 3:
            result = new[-1]
            if len(result) > 0:
                result = result[:-1]
        if result:
            search_dict[channel][nick].append(result)
    return search_dict

def save_db(search_dict):
    """ save search_dict to find.txt """
    search_file = open("find.txt", "w")
    for channel in search_dict:
        if channel is not "":
            for nick in search_dict[channel]:
                for line in search_dict[channel][nick]:
                    channel_utf = (channel).encode("utf-8")
                    search_file.write(channel)
                    search_file.write(",")
                    nick = (nick).encode("utf-8")
                    search_file.write(nick)
                    search_file.write(",")
                    line_utf = (line).encode("utf-8")
                    search_file.write(line_utf)
                    search_file.write("\n")
    search_file.close()

# Create a temporary log of the most recent thing anyone says.
def collectlines(jenni, input):
    # don't log things in PM
    channel = (input.sender).encode("utf-8")
    nick = (input.nick).encode("utf-8")
    if not channel.startswith('#'): return
    search_dict = load_db()
    if channel not in search_dict:
        search_dict[channel] = dict()
    if nick not in search_dict[channel]:
        search_dict[channel][nick] = list()
    templist = search_dict[channel][nick]
    line = input.group()
    if line.startswith("s/"):
        return
    elif line.startswith("\x01ACTION"):
        line = line[:-1]
        templist.append(line)
    else:
        templist.append(line)
    del templist[:-10]
    search_dict[channel][nick] = templist
    save_db(search_dict)
collectlines.rule = r'.*'
collectlines.priority = 'low'

def findandreplace(jenni, input):
    # don't bother in PM
    channel = (input.sender).encode("utf-8")
    nick = (input.nick).encode("utf-8")

    if not channel.startswith('#'): return

    search_dict = load_db()

    rnick = input.group(1) or nick # Correcting other person vs self.

    # only do something if there is conversation to work with
    if channel not in search_dict or rnick not in search_dict[channel]: return

    sep = input.group(2)
    rest = input.group(3).split(sep)
    me = False # /me command
    flags = ''
    if len(rest) < 2:
        return # need at least a find and replacement value
    elif len(rest) > 2:
        # Word characters immediately after the second separator
        # are considered flags (only g and i now have meaning)
        flags = re.match(r'\w*',rest[2], re.U).group(0)
    #else (len == 2) do nothing special

    count = 'g' in flags and -1 or 1 # Replace unlimited times if /g, else once
    if 'i' in flags:
        regex = re.compile(re.escape(rest[0]),re.U|re.I)
        repl = lambda s: re.sub(regex,rest[1],s,count == 1)
    else:
        repl = lambda s: s.replace(rest[0],rest[1],count)

    for line in reversed(search_dict[channel][rnick]):
        if line.startswith("\x01ACTION"):
            me = True # /me command
            line = line[8:]
        else:
            me = False
        new_phrase = repl(line)
        if new_phrase != line: # we are done
            break

    if not new_phrase or new_phrase == line: return # Didn't find anything

    # Save the new "edited" message.
    templist = search_dict[channel][rnick]
    templist.append((me and '\x01ACTION ' or '') + new_phrase)
    search_dict[channel][rnick] = templist
    save_db(search_dict)

    # output
    phrase = nick + (input.group(1) and ' thinks ' + rnick or '') + (me and ' ' or " \x02meant\x02 to say: ") + new_phrase
    if me and not input.group(1): phrase = '\x02' + phrase + '\x02'
    jenni.say(phrase)

# Matches optional whitespace + 's' + optional whitespace + separator character
findandreplace.rule = r'(?u)(?:([^\s:]+)[\s:])?\s*s\s*([^\s\w])(.*)' # May work for both this and "meant" (requires input.group(i+1))
findandreplace.priority = 'high'

if __name__ == '__main__':
    print __doc__.strip()
<|endoftext|>


<commit_before>#Simple library call
import fileinput
#import os

#Open the file
#file = open('Configuration\config.json','w')
#message = file.read()
#print("Going to print the contents of the config")
#print(message)
#file.close()

#Start editing method 1
hostingurl = input("What is the hosting url for your bot (this is where the bot will say the address is. Include an address and port): ")
for line in fileinput.input("Configuration\config.json", inplace=True):
    print hostingurl % (fileinput.filelineno(), 4),

<commit_msg>error with printing to file<commit_after>#Simple library call
import fileinput
#import os

#Open the file
#file = open('Configuration\config.json','w')
#message = file.read()
#print("Going to print the contents of the config")
#print(message)
#file.close()

#Start editing method 1
hostingurl = int(input("What is the hosting url for your bot (this is where the bot will say the address is. Include an address and port): "))
for line in fileinput.input("Configuration\config.json", inplace=True):
    print "hostingurl" % (fileinput.filelineno(), 4),

<|endoftext|>