<commit_before>import asyncio
import aiohttp
import logging
import typing

from analyticord import errors

logger = logging.getLogger("analyticord")

base_address = "https://analyticord.solutions"
login_address = base_address + "/api/botLogin"
send_address = base_address + "/api/submit"
get_address = base_address + "/api/getData"
botinfo_address = base_address + "/api/botinfo"
botlist_address = base_address + "/api/botlist"

def _make_error(error, **kwargs) -> errors.ApiError:
    name = error.get("error", "")  # type: str
    name = name[:1].upper() + name[1:]

    err = getattr(errors, name, None)

    if err is None:
        return errors.ApiError(**error, **kwargs)

    return err(**error, **kwargs)

class EventProxy:
    def __init__(self, analytics, anal_name: str):
        """
        Proxy class to make events and actions acessible through dot notation

        :param analytics: The :class:`analyticord.AnalytiCord` this proxy is tied to.
        :param anal_name: The analyticord name of the event.
        """
        self.analytics = analytics
        self.anal_name = anal_name
        self.lock = asyncio.Lock()
        self.counter = 0

    def send(self, count: int):
        """Invoke this events send message."""
        return self.analytics.send(self.anal_name, count)

    def hook_bot(self, dpy_name: str, bot):
        """Hook the event of a discord.py bot to this event.

        :param dpy_name: Name of discord.py event.
        :param bot: An instance of a discord.py :class:`discord.ext.commands.Bot`.
        """
        async def _hook(*_, **__):
            await self.increment()

        bot.add_listener(_hook, dpy_name)

    async def increment(self):
        """Increment this events counter."""
        async with self.lock:
            self.counter += 1

    async def update_now(self):
        "Trigger an update of this event, resetting the counter."
        async with self.lock:
            resp = await self.send(self.counter)
            self.counter = 0
            return resp

class AnalytiCord:
    """Represents an AnalytiCord api object.

    Instantiation example:

    .. code-block:: python3

        analytics = AnalytiCord("token", "user_token")
        await analytics.start()

    """

    #: Default listeners in format (discord event, analyticord event)
    default_listens = ("messages", "guildJoin", "error", "guildLeave", "disconnect",
                       "voiceChannelJoin", "guildDetails", "mentions", "commands_used")

    def __init__(self,
                 token: str,
                 user_token: str=None,
                 event_interval: int=60,
                 session: aiohttp.ClientSession=None,
                 loop=None):
        """
        :param token: Your AnalytiCord bot token.
        :param user_token:
            Your AnalytiCord user token.
            This is not required unless you wish to use endpoints that require User auth.
        :param event_interval: The interval between sending event updates in seconds.

        """

        #: Your AnalytiCord bot token.
        self.token = "bot {}".format(token)

        self.loop = loop or asyncio.get_event_loop()
        self.session = session or aiohttp.ClientSession()

        #: Interval between sending event updates
        self.event_interval = event_interval

        if user_token is not None:
            self.user_token = "user {}".format(user_token)

        self.events = {i: EventProxy(self, i) for i in self.default_listens}

        self.updater = None  # placeholder for updater event

    def __getattr__(self, attr):
        return self.events[attr]

    @property
    def _auth(self):
        return {"Authorization": self.token}

    @property
    def _user_auth(self):
        if not hasattr(self, "user_token"):
            raise Exception("user_token must be set to use this feature.")
        return {"Authorization": self.user_token}

    def register(self, anal_name: str):
        """Register an event.

        Once registered, AnalytiCord.<anal_name> will return a :class:`analyticord.EventProxy`
        for the given <anal_name>.

        This allows you to do:
        await AnalytiCord.event.increment()  # isend me inncrement the event counter
        AnalytiCord.event.hook_bot(bot)  # hook the event to a bot

        :param anal_name: The AnalytiCord event name, for example: messages, guildJoin.
        """
        self.events[anal_name] = EventProxy(self, anal_name)

    async def _do_request(self, rtype: str, endpoint: str, auth, **kwargs):
        req = getattr(self.session, rtype)
        async with req(endpoint, headers=auth, **kwargs) as resp:
            body = await resp.json()
            if resp.status != 200:
                raise _make_error(body, status=resp.status)
            return body

    async def start(self):
        """Fire a login event.
        Also runs the event updater loop

        :raises: :class:`analyticord.errors.ApiError`.
        """
        resp = await self._do_request("get", login_address, self._auth)
        self.updater = self.loop.create_task(self._update_events_loop())
        return resp

    async def stop(self):
        """Update all events and stop the analyticord updater loop."""
        self.updater.cancel()
        await self._update_once()

    async def send(self, event_type: str, data: str) -> dict:
        """Send data to analiticord.

        :param event_type: Event type to send.
        :param data: Data to send.

        :return: Dict response from api.

        :raises: :class:`analyticord.errors.ApiError`.
        """
        return await self._do_request("post", send_address, self._auth,
                                      data=dict(eventType=event_type, data=data))

    async def get(self, **attrs) -> list:
        """Get data from the api.

        :param attrs: Kwarg attributes passed to get request params.

        :return: Response list on success.

        :raises:  :class:`analyticord.errors.ApiError`.
        """
        return await self._do_request("get", get_address, self._user_auth, params=attrs)

    async def bot_info(self, id: int) -> dict:
        """Get info for a bot id.

        :param id: The ID of the bot to get info for.

        :return: Bot info data.

        :raises: :class:`analyticord.errors.ApiError`.
        """
        return await self._do_request("get", botinfo_address, self._user_auth,
                                      params={"id": id})

    async def bot_list(self) -> list:
        """Get list of bots owned by this auth.

        :return: list of bot info data.

        :raises: :class:`analyticord.errors.ApiError`.
        """
        return await self._do_request("get", botlist_address, self._user_auth)

    async def _update_once(self):
        for event in self.events.values():
            if event.counter:
                try:
                    await event.update_now()
                except errors.ApiError as e:
                    logger.error(str(e))

    async def _update_events_loop(self):
        while True:
            await asyncio.sleep(self.event_interval)
            await self._update_once()
<commit_msg>Move out different events into their own classes<commit_after>import asyncio
import aiohttp
import logging
import typing

from analyticord import errors

logger = logging.getLogger("analyticord")

base_address = "https://analyticord.solutions"
login_address = base_address + "/api/botLogin"
send_address = base_address + "/api/submit"
get_address = base_address + "/api/getData"
botinfo_address = base_address + "/api/botinfo"
botlist_address = base_address + "/api/botlist"

def _make_error(error, **kwargs) -> errors.ApiError:
    name = error.get("error", "")  # type: str
    name = name[:1].upper() + name[1:]

    err = getattr(errors, name, None)

    if err is None:
        return errors.ApiError(**error, **kwargs)

    return err(**error, **kwargs)

class EventProxy:
    def __init__(self, analytics, anal_name: str):
        """
        Proxy class to make events and actions acessible through dot notation

        :param analytics: The :class:`analyticord.AnalytiCord` this proxy is tied to.
        :param anal_name: The analyticord name of the event.
        """
        self.analytics = analytics
        self.anal_name = anal_name

    def send(self, value: typing.Any):
        """Invoke this events send message."""
        return self.analytics.send(self.anal_name, value)

    def hook_bot(self, bot, dpy_name: str):
        """Hook a discord event to commiting the relevent action for this event.

        :param dpy_name: name of discord.py event.
        :param bot: an instance of a discord.py :class:`discord.ext.commands.bot`.
        """
        async def _hook(*_, **__):
            await self.send(True)

        bot.add_listener(_hook, dpy_name)

class MessageEventProxy(EventProxy):
    """Basically, only message event takes a delta value.
    Everything else is exact, so half the stuff in EventProxy is useless for anything but `messages`
    """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.lock = asyncio.Lock()
        self.counter = 0

    def hook_bot(self, bot, dpy_name: str="on_message"):
        """Hook a discord event to increment the message count.

        :param dpy_name: Name of discord.py event.
        :param bot: An instance of a discord.py :class:`discord.ext.commands.Bot`.
        """
        async def _hook(*_, **__):
            await self.increment()

        bot.add_listener(_hook, dpy_name)

    async def increment(self):
        """Increment this events counter."""
        async with self.lock:
            self.counter += 1

    async def update_now(self):
        "Trigger an update of this event, resetting the counter."
        async with self.lock:
            resp = await self.send(self.counter)
            self.counter = 0
            return resp

    async def _update_once(self):
        if self.counter:
            try:
                await self.update_now()
            except errors.ApiError as e:
                logger.error(str(e))

    async def _update_events_loop(self):
        while True:
            await asyncio.sleep(self.event_interval)
            await self._update_once()

class ErrorEventProxy(EventProxy):
    """Proxy class for the error event."""

    def hook_bot(self, bot, dpy_name: str="on_command_error"):
        """Hook a discord event to send an error event.

        :param dpy_name: name of discord.py event.
        :param bot: an instance of a discord.py :class:`discord.ext.commands.bot`.
        """
        async def _hook(ctx, exception):
            await self.send("command: {}. error: {}.".format(ctx, exception))

        bot.add_listener(_hook, dpy_name)

class GuildJoinEventProxy(EventProxy):
    """Proxy class for the guild join event."""

    def hook_bot(self, bot, dpy_name: str="on_guild_join"):
        """Hook a discord event to send a guildJoin event.

        :param dpy_name: name of discord.py event.
        :param bot: an instance of a discord.py :class:`discord.ext.commands.bot`.
        """
        async def _hook(*_, **__):
            await self.send(len(bot.guilds))

        bot.add_listener(_hook, dpy_name)

class CommandUsedEventProxy(EventProxy):
    """Proxy used for the command used event."""

    def hook_bot(self, bot, dpy_name: str="on_command_completion"):
        """Hook a discord event to send a commandUsed event.

        :param dpy_name: name of discord.py event.
        :param bot: an instance of a discord.py :class:`discord.ext.commands.bot`.
        """

        async def _hook(ctx):
            await self.send(ctx.command.name)

        bot.add_listener(_hook, dpy_name)

class AnalytiCord:
    """Represents an AnalytiCord api object.

    Instantiation example:

    .. code-block:: python3

        analytics = AnalytiCord("token", "user_token")
        await analytics.start()

    """

    #: Default listeners in format (discord event, Proxy class)
    default_listens = (("messages", MessageEventProxy),
                       ("guildJoin", GuildJoinEventProxy),
                       ("error", ErrorEventProxy),
                       ("guildLeave", EventProxy),
                       ("disconnect", EventProxy),
                       ("voiceChannelJoin", EventProxy),
                       ("guildDetails", EventProxy),
                       ("mentions", EventProxy),
                       ("commands_used", CommandUsedEventProxy))

    def __init__(self,
                 token: str,
                 user_token: str=None,
                 event_interval: int=60,
                 session: aiohttp.ClientSession=None,
                 loop=None):
        """
        :param token: Your AnalytiCord bot token.
        :param user_token:
            Your AnalytiCord user token.
            This is not required unless you wish to use endpoints that require User auth.
        :param event_interval: The interval between sending event updates in seconds.

        """

        #: Your AnalytiCord bot token.
        self.token = "bot {}".format(token)

        self.loop = loop or asyncio.get_event_loop()
        self.session = session or aiohttp.ClientSession()

        #: Interval between sending event updates
        self.event_interval = event_interval

        if user_token is not None:
            self.user_token = "user {}".format(user_token)

        self.events = {i: e(self, i) for i, e in self.default_listens}

    def __getattr__(self, attr):
        return self.events[attr]

    @property
    def _auth(self):
        return {"Authorization": self.token}

    @property
    def _user_auth(self):
        if not hasattr(self, "user_token"):
            raise Exception("user_token must be set to use this feature.")
        return {"Authorization": self.user_token}

    def register(self, anal_name: str, proxy_type: EventProxy=EventProxy):
        """Register an event.

        Once registered, AnalytiCord.<anal_name> will return a :class:`analyticord.EventProxy`
        for the given <anal_name>.

        This allows you to do:
        await AnalytiCord.event.increment()  # isend me inncrement the event counter
        AnalytiCord.event.hook_bot(bot)  # hook the event to a bot

        :param anal_name: The AnalytiCord event name, for example: messages, guildJoin.
        :parap proxy_type: The event proxy to use. Should be a subclass of :class:`analyticord.EventProxy`.
        """
        self.events[anal_name] = proxy_type(self, anal_name)

    async def _do_request(self, rtype: str, endpoint: str, auth, **kwargs):
        req = getattr(self.session, rtype)
        async with req(endpoint, headers=auth, **kwargs) as resp:
            body = await resp.json()
            if resp.status != 200:
                raise _make_error(body, status=resp.status)
            return body

    async def start(self):
        """Fire a login event.
        Also runs the event updater loop

        :raises: :class:`analyticord.errors.ApiError`.
        """
        resp = await self._do_request("get", login_address, self._auth)
        self.updater = self.loop.create_task(
            self.messages._update_events_loop())
        return resp

    async def stop(self):
        """Update all events and stop the analyticord updater loop."""
        self.updater.cancel()
        await self.messages._update_once()

    async def send(self, event_type: str, data: str) -> dict:
        """Send data to analiticord.

        :param event_type: Event type to send.
        :param data: Data to send.

        :return: Dict response from api.

        :raises: :class:`analyticord.errors.ApiError`.
        """
        return await self._do_request("post", send_address, self._auth,
                                      data=dict(eventType=event_type, data=data))

    async def get(self, **attrs) -> list:
        """Get data from the api.

        :param attrs: Kwarg attributes passed to get request params.

        :return: Response list on success.

        :raises:  :class:`analyticord.errors.ApiError`.
        """
        return await self._do_request("get", get_address, self._user_auth, params=attrs)

    async def bot_info(self, id: int) -> dict:
        """Get info for a bot id.

        :param id: The ID of the bot to get info for.

        :return: Bot info data.

        :raises: :class:`analyticord.errors.ApiError`.
        """
        return await self._do_request("get", botinfo_address, self._user_auth,
                                      params={"id": id})

    async def bot_list(self) -> list:
        """Get list of bots owned by this auth.

        :return: list of bot info data.

        :raises: :class:`analyticord.errors.ApiError`.
        """
        return await self._do_request("get", botlist_address, self._user_auth)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
        Copyright 2016 Stuart Freeman Licensed under the
	Educational Community License, Version 2.0 (the "License"); you may
	not use this file except in compliance with the License. You may
	obtain a copy of the License at

http://www.osedu.org/licenses/ECL-2.0

	Unless required by applicable law or agreed to in writing,
	software distributed under the License is distributed on an "AS IS"
	BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
	or implied. See the License for the specific language governing
	permissions and limitations under the License.
"""

import csv, datetime, errno, getopt, io, os, re, sqlite3, sys, zipfile

def eprint(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)

def main(argv):
    # If start and end date aren't specified we'll use the first and last days of the previous month
    startDate = (datetime.date.today().replace(day=1) - datetime.timedelta(days=1)).replace(day=1).strftime('%Y-%m-%d')
    endDate = (datetime.date.today().replace(day=1) - datetime.timedelta(days=1)).strftime('%Y-%m-%d')

    # Set up command line parsing
    usage = sys.argv[0] + ' -i <inputfile> -s <YYYY-MM-DD> -e <YYYY-MM-DD>'
    try:
        opts, args = getopt.getopt(argv, "hi:s:e:", ['ifile', 'startDate', 'endDate'])
    except getopt.GetoptError:
        print(usage)
        sys.exit(2)
    if len(opts) == 0:
        print(usage)
        sys.exit()
    for opt, arg in opts:
        if opt == '-h':
            print(usage)
            sys.exit()
        elif opt == '-i':
            inputfile = arg
        elif opt == '-s':
            if re.fullmatch('\d{4}-\d{2}-\d{2}', arg):
                startDate = arg
            else:
                eprint('Dates should be in the format YYYY-MM-DD')
                sys.exit(2)
        elif opt == '-e':
            if re.fullmatch('\d{4}-\d{2}-\d{2}', arg):
                endDate = arg
            else:
                eprint('Dates should be in the format YYYY-MM-DD')
                sys.exit(2)

    # Make the results dir
    resultspath = os.path.dirname(os.path.realpath(__file__)) + '/results'
    try:
        os.makedirs(resultspath)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

    dbfile = resultspath + '/coursera.sqlite3'
    dbExists = os.path.isfile(dbfile)
    dbOld = getFileModDate(inputfile) > getFileModDate(dbfile) if dbExists else False

    # Our cached DB is old, delete it so we can rebuild
    if dbOld:
        os.remove(dbfile)

    # Connect to the cached DB or create an empty one
    db = sqlite3.connect(dbfile)
    cur = db.cursor()

    # If we just created an empty DB populate it from the zip file
    if not dbExists or dbOld:
        with zipfile.ZipFile(inputfile) as myzip:
            parse('courses', cur, myzip)
            parse('course_memberships', cur, myzip)
            parse('course_grades', cur, myzip)
            parse('users_courses__certificate_payments', cur, myzip)

        cur.execute('CREATE INDEX course_idx1 ON courses(course_id);')
        cur.execute('CREATE INDEX course_idx2 ON course_memberships(course_id);')
        cur.execute('CREATE INDEX course_idx3 ON course_grades(course_id);')
        cur.execute('CREATE INDEX date_idx1 ON course_memberships(course_membership_ts);')
        cur.execute('CREATE VIEW completed_count (course_id, completed) AS SELECT course_id, count(course_id) FROM course_grades WHERE course_passing_state_id="1" OR course_passing_state_id="2" GROUP BY course_id')
        db.commit()

    with open('{0}/report.{1}.{2}.csv'.format(resultspath, startDate, endDate), 'w') as reportfile:
        writer = csv.writer(reportfile)
        cur.execute("""SELECT courses.course_name,
        count(course_memberships.gatech_user_id) AS members,
        count(CASE course_memberships.course_membership_role WHEN 'LEARNER' THEN 1 ELSE null END) AS active,
        completed_count.completed AS completed
        FROM courses
                JOIN course_memberships ON courses.course_id = course_memberships.course_id
                JOIN completed_count ON courses.course_id = completed_count.course_id
                WHERE course_memberships.course_membership_ts BETWEEN ? AND ? GROUP BY courses.course_id;""", (startDate, endDate))
        writer.writerow([d[0] for d in cur.description])
        writer.writerows(cur.fetchall())

def parse(name, cur, zipfile):
    """Extract a csv from the zip and read it into a table"""
    csvfile = io.TextIOWrapper(zipfile.open(name + '.csv'), encoding='utf-8')
    dr = csv.DictReader(csvfile)
    cur.execute('CREATE TABLE {0} ({1});'.format(name, ', '.join(dr.fieldnames)))
    for row in dr:
        columns = ', '.join([':' + key for key in filter(None, row.keys())])
        cur.execute('INSERT INTO {0} ({1}) VALUES ({2});'.format(name, ', '.join(filter(None,row.keys())), columns), row)

def getFileModDate(filename):
    """Get the datetime when a file was modified"""
    t = os.path.getmtime(filename)
    return datetime.datetime.fromtimestamp(t)

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>add paid count<commit_after>#!/usr/bin/env python3

"""
        Copyright 2016 Stuart Freeman Licensed under the
	Educational Community License, Version 2.0 (the "License"); you may
	not use this file except in compliance with the License. You may
	obtain a copy of the License at

http://www.osedu.org/licenses/ECL-2.0

	Unless required by applicable law or agreed to in writing,
	software distributed under the License is distributed on an "AS IS"
	BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
	or implied. See the License for the specific language governing
	permissions and limitations under the License.
"""

import csv, datetime, errno, getopt, io, os, re, sqlite3, sys, zipfile

def eprint(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)

def main(argv):
    # If start and end date aren't specified we'll use the first and last days of the previous month
    startDate = (datetime.date.today().replace(day=1) - datetime.timedelta(days=1)).replace(day=1).strftime('%Y-%m-%d')
    endDate = (datetime.date.today().replace(day=1) - datetime.timedelta(days=1)).strftime('%Y-%m-%d')

    # Set up command line parsing
    usage = sys.argv[0] + ' -i <inputfile> -s <YYYY-MM-DD> -e <YYYY-MM-DD>'
    try:
        opts, args = getopt.getopt(argv, "hi:s:e:", ['ifile', 'startDate', 'endDate'])
    except getopt.GetoptError:
        print(usage)
        sys.exit(2)
    if len(opts) == 0:
        print(usage)
        sys.exit()
    for opt, arg in opts:
        if opt == '-h':
            print(usage)
            sys.exit()
        elif opt == '-i':
            inputfile = arg
        elif opt == '-s':
            if re.fullmatch('\d{4}-\d{2}-\d{2}', arg):
                startDate = arg
            else:
                eprint('Dates should be in the format YYYY-MM-DD')
                sys.exit(2)
        elif opt == '-e':
            if re.fullmatch('\d{4}-\d{2}-\d{2}', arg):
                endDate = arg
            else:
                eprint('Dates should be in the format YYYY-MM-DD')
                sys.exit(2)

    # Make the results dir
    resultspath = os.path.dirname(os.path.realpath(__file__)) + '/results'
    try:
        os.makedirs(resultspath)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

    dbfile = resultspath + '/coursera.sqlite3'
    dbExists = os.path.isfile(dbfile)
    dbOld = getFileModDate(inputfile) > getFileModDate(dbfile) if dbExists else False

    # Our cached DB is old, delete it so we can rebuild
    if dbOld:
        os.remove(dbfile)

    # Connect to the cached DB or create an empty one
    db = sqlite3.connect(dbfile)
    cur = db.cursor()

    # If we just created an empty DB populate it from the zip file
    if not dbExists or dbOld:
        with zipfile.ZipFile(inputfile) as myzip:
            parse('courses', cur, myzip)
            parse('course_memberships', cur, myzip)
            parse('course_grades', cur, myzip)
            parse('users_courses__certificate_payments', cur, myzip)

        cur.execute('CREATE INDEX course_idx1 ON courses(course_id);')
        cur.execute('CREATE INDEX course_idx2 ON course_memberships(course_id);')
        cur.execute('CREATE INDEX course_idx3 ON course_grades(course_id);')
        cur.execute('CREATE INDEX date_idx1 ON course_memberships(course_membership_ts);')
        cur.execute('CREATE VIEW payment_count (course_id, paid) AS SELECT course_id, count(course_id) FROM users_courses__certificate_payments GROUP BY course_id;')
        cur.execute('CREATE VIEW completed_count (course_id, completed) AS SELECT course_id, count(course_id) FROM course_grades WHERE course_passing_state_id="1" OR course_passing_state_id="2" GROUP BY course_id')
        db.commit()

    with open('{0}/report.{1}.{2}.csv'.format(resultspath, startDate, endDate), 'w') as reportfile:
        writer = csv.writer(reportfile)
        cur.execute("""SELECT courses.course_name,
        count(course_memberships.gatech_user_id) AS members,
        count(CASE course_memberships.course_membership_role WHEN 'LEARNER' THEN 1 ELSE null END) AS active,
        payment_count.paid AS paid,
        completed_count.completed AS completed
        FROM courses
                JOIN course_memberships ON courses.course_id = course_memberships.course_id
                JOIN payment_count ON courses.course_id = payment_count.course_id
                JOIN completed_count ON courses.course_id = completed_count.course_id
                WHERE course_memberships.course_membership_ts BETWEEN ? AND ? GROUP BY courses.course_id;""", (startDate, endDate))
        writer.writerow([d[0] for d in cur.description])
        writer.writerows(cur.fetchall())

def parse(name, cur, zipfile):
    """Extract a csv from the zip and read it into a table"""
    csvfile = io.TextIOWrapper(zipfile.open(name + '.csv'), encoding='utf-8')
    dr = csv.DictReader(csvfile)
    cur.execute('CREATE TABLE {0} ({1});'.format(name, ', '.join(dr.fieldnames)))
    for row in dr:
        columns = ', '.join([':' + key for key in filter(None, row.keys())])
        cur.execute('INSERT INTO {0} ({1}) VALUES ({2});'.format(name, ', '.join(filter(None,row.keys())), columns), row)

def getFileModDate(filename):
    """Get the datetime when a file was modified"""
    t = os.path.getmtime(filename)
    return datetime.datetime.fromtimestamp(t)

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before># Create your views here.

import os
import re
import operator
import io
import datetime
import hashlib

from django.contrib.sites.models import Site
from django.shortcuts import render, redirect
from django.http import HttpResponse
from django.conf import settings
from django.http import HttpResponseNotFound, HttpResponse, Http404, QueryDict
from django.views.decorators.csrf import csrf_protect
from django.core.urlresolvers import reverse
from django.db.models import Count
from django.utils.html import escape
from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.views.decorators.csrf import csrf_exempt

from toolbar_buttons.config.settings import config
from toolbar_buttons.builder import button, locales, util, build

from toolbar_buttons.toolbar_buttons_web.tbutton_maker.models import Application, Button, DownloadSession

class WebButton(button.SimpleButton):
    def __init__(self, folders, buttons, settings, applications):
        button.SimpleButton.__init__(self, folders, buttons, settings, applications)
        self._description = {}
        self._icons = {}

        for folder, buttonId, files in self._info:
            if "description" in files:
                with open(os.path.join(folder, "description"), "r") as description:
                    self._description[buttonId] = description.read()
                    if not self._description[buttonId].strip():
                        print buttonId
        for buttonId, icons in self._button_image.iteritems():
            for image, selector in icons:
                if image and not selector:
                    self._icons[buttonId] = image
                    break

    def get_xul_files(self, button):
        return self._xul_files[button]

    def get_icons(self, button):
        return self._icons[button]

    def description(self, button):
        return self._description.get(button)

def locale_string(button_locale, locale_name, buttons_obj):
    def locale_str(str_type, buttonId):
        default_local = "en-US" #TODO: make a setting for this
        value = button_locale.get_dtd_value(locale_name, "%s.%s" % (buttonId, str_type), buttons_obj)
        if value is None:
            if str_type == "label":
                regexp = r'label="&(.*\.label);"'
            else:
                regexp = r'tooltiptext="&(.*\.tooltip);"'
            with open(buttons_obj.get_xul_files(buttonId)[0]) as fp:
                data = fp.read()
                match = re.search(regexp, data)
                value = button_locale.get_dtd_value(locale_name, match.group(1), buttons_obj)
            if value is None:
                value = button_locale.get_dtd_value(default_local, match.group(1), buttons_obj)
        if value is None:
            return '' #print buttonId
        return value.replace("&brandShortName;", "").replace("'", "'")
    return locale_str

def get_extenion_data(locale_name=None, applications=None):
    extension_settings = dict(config)
    extension_settings["project_root"] = settings.TBUTTON_DATA
    default_local = extension_settings.get("default_locale")
    if locale_name == None:
        locale_name = default_local
    if applications == None:
        applications = extension_settings.get("applications_data").keys()
    else:
        applications = applications.split(',')
    locale_folder, locale = util.get_locale_folders(set([locale_name, default_local]), extension_settings)
    if locale:
        button_locale = locales.Locale(config, locale_folder,
            locale, load_properites=False)
    else:
        raise Http404
    button_folders, buttons = util.get_button_folders("all", extension_settings)
    buttons_obj = WebButton(button_folders, buttons, extension_settings, applications)
    locale_str = locale_string(button_locale=button_locale, locale_name=locale_name, buttons_obj=buttons_obj)
    return buttons_obj, locale_str, extension_settings, locale_name, applications

def list_buttons(request, locale_name=None, applications=None, template_name='tbutton_maker/list.html'):
    return index(request, locale_name, applications, template_name)

def index(request, locale_name=None, applications=None, template_name='tbutton_maker/index.html'):
    buttons_obj, locale_str, extension_settings, locale_name, applications = get_extenion_data(locale_name, applications)

    button_data = []
    for buttonId, apps in buttons_obj.button_applications().items():
        button_data.append((buttonId, sorted(list(apps)), locale_str("label", buttonId),
                locale_str("tooltip", buttonId), buttons_obj.get_icons(buttonId),
                buttons_obj.description(buttonId)))
    def button_key(item):
        return item[2].lower() if item[2] else None
    locale_folders, locales_names = util.get_locale_folders(extension_settings.get("locale"), extension_settings)
    locale_meta = locales.Locale(extension_settings, locale_folders,
                locales_names, only_meta=True)
    local_data = []
    for locales_name in locales_names:
        local_data.append((
            locales_name,
            locale_meta.get_dtd_value(locales_name, 'name'),
            locale_meta.get_dtd_value(locales_name, 'native_name'),
            locale_meta.get_dtd_value(locales_name, 'country'),
        ))
    button_data.sort(key=button_key)
    local_data.sort(key=button_key)
    application_data = extension_settings.get("applications_data")
    application_names = dict((key, [item[0] for item in value]) for key, value in application_data.iteritems())
    #TODO: add option for what update "stream" to follow, the size of icon,
    # and the icon set (if I get more then one)
    data = {
        "locale": locale_name,
        "all_applications": sorted(application_data.keys()),
        "applications": applications,
        "button_data": button_data,
        "application_names": application_names,
        "local_data": local_data,
        "icon_sets": [(name, label) for name, (label, _) in settings.TBUTTON_ICONS.items()],
        "default_icons": settings.TBUTTON_DEFAULT_ICONS,
    }
    if template_name is None:
        return data
    return render(request, template_name, data)

def create_buttons(request, query):
    buttons = query.getlist("button")
    locale = query.get("button-locale", query.get("locale", "all"))

    extension_settings = dict(config)
    extension_settings.update({
        "show_updated_prompt": False,
        "project_root": settings.TBUTTON_DATA,
        "icon": os.path.join(settings.TBUTTON_DATA, extension_settings.get("icon")),
        "licence": os.path.join(settings.TBUTTON_DATA, extension_settings.get("licence")),
        "buttons": buttons,
    })

    if not locale or query.get("include-all-locales") == "true":
        locale = "all"
    if query.get("create-toolbars") == "true":
        extension_settings["put_button_on_toolbar"] = True
        extension_settings["include_toolbars"] = -1
    else:
        extension_settings["include_toolbars"] = 0
    extension_settings["create_menu"] = query.get("create-menu") == "true"
    extension_settings["locale"] = locale
    applications = query.getlist("button-application")
    if not applications:
        applications = query.get("application", "all").split(",")
    extension_settings["applications"] = applications
    update_query = query.copy()
    update_query["application"] = ",".join(applications)
    update_query["locale"] = locale
    allowed_options = set(("application", "locale", "button", "create-menu", "create-toolbars", "icon-size"))
    for key in update_query.keys():
        if key not in allowed_options:
            del update_query[key]
    icons_size = settings.TBUTTON_ICON_SET_SIZES.get(settings.TBUTTON_DEFAULT_ICONS).get(query.get("icon-size"))
    if icons_size:
        extension_settings["icon_size"] = icons_size
    extension_settings["extension_id"] = "%s@button.codefisher.org" % hashlib.md5("_".join(sorted(buttons))).hexdigest()
    extension_settings["update_url"] = "https://%s%s?%s" % (Site.objects.get_current().domain,
            reverse("tbutton-update"), escape(update_query.urlencode()))

    output = io.BytesIO()
    buttons, button_locales = build.build_extension(extension_settings, output=output)
    responce = HttpResponse(output.getvalue(), content_type="application/x-xpinstall")
    output.close()
    if request.POST.get("offer-download") == "true" or ("browser" not in applications):
        responce['Content-Disposition'] = 'attachment; filename=%s' % (extension_settings.get("output_file") % extension_settings)
    else:
        responce['Content-Disposition'] = 'filename=%s' % (extension_settings.get("output_file") % extension_settings)
    session = DownloadSession()
    session.query_string = query.urlencode()
    session.save()
    for button in buttons.buttons():
        Button.objects.create(name=button, session=session)
    for button in buttons.applications():
        Application.objects.create(name=button, session=session)
    return responce

@csrf_exempt
def create(request):
    #if request.method != "POST":
    #    raise Http404()
    buttons = request.GET.getlist("button")
    if not buttons or "update-submit" in request.GET:
        applications = ",".join(request.GET.getlist("button-application"))
        locale = request.GET.get("button-locale")
        kwargs = {}
        if locale:
            kwargs["locale_name"] = locale
        if applications:
            kwargs["applications"] = applications
        return redirect(reverse("tbutton-custom", kwargs=kwargs))
    return create_buttons(request, request.GET)

def statistics(request, days=30, template_name='tbutton_maker/statistics.html'):
    locale_str = get_extenion_data()[1]

    time = datetime.datetime.now() - datetime.timedelta(days)
    buttons = Button.objects.filter(session__time__gt=time)
    stats = list(buttons.values('name').annotate(downloads=Count('name')).order_by("-downloads"))
    sum = buttons.count()
    total = 0
    for item in stats:
        count = item["downloads"]
        total += count
        item.update({
            "label": locale_str('label', item["name"]),
            "average": (float(count) / days),
            "percent": (float(count) / sum * 100),
            "total": (float(total) / sum * 100)
        })
    data = {
        "stats": stats,
        "count": sum,
        "average": float(sum)/(len(stats)*days) if stats else 0
    }
    return render_to_response(template_name, data, RequestContext(request))

def old_update(request):
    query = QueryDict('').copy()
    query.setlist('button', request.GET.get('buttons').split('_'))
    return redirect("%s?%s" % (reverse('tbutton-update'), query.urlencode()))

def update(request):
    def flat(l):
        return [item for sublist in l for item in sublist]
    buttons = request.GET.getlist("button")
    applications = request.GET.getlist("button-application")
    if not applications:
        applications = request.GET.get("application", "all").split(",")
    if applications == ["all"]:
        app_data = flat(config.get("applications_data").values())
    else:
        app_data = flat([config.get("applications_data").get(app) for app in applications])
    update_url = "https://%s%s?%s" % (Site.objects.get_current().domain,
            reverse("tbutton-make-button"), request.GET.urlencode())

    data = {
        "applications": app_data,
        "version": config.get("version"),
        "update_url": update_url,
        "extension_id": "%s@button.codefisher.org" % hashlib.md5("_".join(sorted(buttons))).hexdigest(),
    }

    return render(request, "tbutton_maker/update.rdf", data, content_type="application/xml+rdf")

def make(request):
    return create_buttons(request, request.GET)

def page_it(request, entries_list):
    paginator = Paginator(entries_list, 10)
    # todo: this does not work now
    try:
        page = int(request.GET.get('page', '1'))
    except ValueError:
        page = 1

    try:
        entries = paginator.page(page)
    except (EmptyPage, InvalidPage):
        # todo: better raise a redirect
        entries = paginator.page(paginator.num_pages)
    return entries

def list_app_buttons(request, app_name, days=30, template_name='tbutton_maker/app_list.html'):
    app_data = config.get("applications_data")
    if app_name not in app_data:
        for key, items in app_data.iteritems():
            if app_name.lower() in [item[0].lower() for item in items]:
                app_name = key
                break
        else:
            raise Http404
    time = datetime.datetime.now() - datetime.timedelta(days)
    buttons = Button.objects.filter(session__time__gt=time)
    stats = dict((item["name"], item["downloads"]) for item in buttons.values('name').annotate(downloads=Count('name')).order_by("-downloads"))
    def button_key(item):
        stats.get(item[0], 0)
    data = index(request, applications=app_name, template_name=None)
    data["button_data"].sort(key=button_key)
    data["entries"] = page_it(request, data["button_data"])
    data["application"] = app_name
    return render(request, template_name, data)
<commit_msg>fixed some of the string handeling, and other a few other updates<commit_after># Create your views here.

import os
import re
import operator
import io
import datetime
import hashlib

from django.contrib.sites.models import Site
from django.shortcuts import render, redirect
from django.http import HttpResponse
from django.conf import settings
from django.http import HttpResponseNotFound, HttpResponse, Http404, QueryDict
from django.views.decorators.csrf import csrf_protect
from django.core.urlresolvers import reverse
from django.db.models import Count
from django.utils.html import escape
from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.views.decorators.csrf import csrf_exempt

from toolbar_buttons.config.settings import config
from toolbar_buttons.builder import button, locales, util, build

from toolbar_buttons.toolbar_buttons_web.tbutton_maker.models import Application, Button, DownloadSession

class WebButton(button.SimpleButton):
    def __init__(self, folders, buttons, settings, applications):
        button.SimpleButton.__init__(self, folders, buttons, settings, applications)
        self._description = {}
        self._icons = {}

        for folder, buttonId, files in self._info:
            if "description" in files:
                with open(os.path.join(folder, "description"), "r") as description:
                    self._description[buttonId] = description.read()
                    if not self._description[buttonId].strip():
                        print buttonId
        for buttonId, icons in self._button_image.iteritems():
            for image, selector in icons:
                if image and not selector:
                    self._icons[buttonId] = image
                    break

    def get_xul_files(self, button):
        return self._xul_files[button]

    def get_icons(self, button):
        return self._icons[button]

    def description(self, button):
        return self._description.get(button)

def locale_string(button_locale, locale_name, buttons_obj):
    def locale_str(str_type, buttonId):
        default_locale = config.get('default_locale', 'en-US')
        value = button_locale.get_dtd_value(locale_name, "%s.%s" % (buttonId, str_type), buttons_obj)
        if value is None:
            if str_type == "label":
                regexp = r'label="&(.*\.label);"'
            else:
                regexp = r'tooltiptext="&(.*\.tooltip);"'
            with open(buttons_obj.get_xul_files(buttonId)[0]) as fp:
                data = fp.read()
                match = re.search(regexp, data)
                value = button_locale.get_dtd_value(locale_name, match.group(1), buttons_obj)
            if value is None:
                value = button_locale.get_dtd_value(default_locale, match.group(1), buttons_obj)
        if value is None:
            return '' #print buttonId
        return value.replace("&brandShortName;", "").replace("'", "'")
    return locale_str

def get_extenion_data(locale_name=None, applications=None):
    extension_settings = dict(config)
    extension_settings["project_root"] = settings.TBUTTON_DATA
    default_local = extension_settings.get("default_locale")
    if locale_name == None:
        locale_name = default_local
    if applications == None:
        applications = extension_settings.get("applications_data").keys()
    else:
        applications = applications.split(',')
    locale_folder, locale = util.get_locale_folders(set([locale_name, default_local]), extension_settings)
    if locale:
        button_locale = locales.Locale(config, locale_folder,
            locale, load_properites=False)
    else:
        raise Http404
    button_folders, buttons = util.get_button_folders("all", extension_settings)
    buttons_obj = WebButton(button_folders, buttons, extension_settings, applications)
    locale_str = locale_string(button_locale=button_locale, locale_name=locale_name, buttons_obj=buttons_obj)
    return buttons_obj, locale_str, extension_settings, locale_name, applications

def list_buttons(request, locale_name=None, applications=None, template_name='tbutton_maker/list.html'):
    return index(request, locale_name, applications, template_name)

def index(request, locale_name=None, applications=None, template_name='tbutton_maker/index.html'):
    buttons_obj, locale_str, extension_settings, locale_name, applications = get_extenion_data(locale_name, applications)

    button_data = []
    for buttonId, apps in buttons_obj.button_applications().items():
        button_data.append((buttonId, sorted(list(apps)), locale_str("label", buttonId),
                locale_str("tooltip", buttonId), buttons_obj.get_icons(buttonId),
                buttons_obj.description(buttonId)))
    def button_key(item):
        return item[2].lower() if item[2] else None
    locale_folders, locales_names = util.get_locale_folders(extension_settings.get("locale"), extension_settings)
    locale_meta = locales.Locale(extension_settings, locale_folders,
                locales_names, only_meta=True)
    local_data = []
    for locales_name in locales_names:
        local_data.append((
            locales_name,
            locale_meta.get_dtd_value(locales_name, 'name'),
            locale_meta.get_dtd_value(locales_name, 'native_name'),
            locale_meta.get_dtd_value(locales_name, 'country'),
        ))
    button_data.sort(key=button_key)
    local_data.sort(key=button_key)
    application_data = extension_settings.get("applications_data")
    application_names = dict((key, [item[0] for item in value]) for key, value in application_data.iteritems())
    #TODO: add option for what update "stream" to follow, the size of icon,
    # and the icon set (if I get more then one)
    data = {
        "locale": locale_name,
        "all_applications": sorted(application_data.keys()),
        "applications": applications,
        "button_data": button_data,
        "application_names": application_names,
        "local_data": local_data,
        "icon_sets": [(name, label) for name, (label, _) in settings.TBUTTON_ICONS.items()],
        "default_icons": settings.TBUTTON_DEFAULT_ICONS,
    }
    if template_name is None:
        return data
    return render(request, template_name, data)

def create_buttons(request, query):
    buttons = query.getlist("button")
    locale = query.get("button-locale", query.get("locale", "all"))

    extension_settings = dict(config)
    extension_settings.update({
        "show_updated_prompt": False,
        "project_root": settings.TBUTTON_DATA,
        "icon": os.path.join(settings.TBUTTON_DATA, extension_settings.get("icon")),
        "licence": os.path.join(settings.TBUTTON_DATA, extension_settings.get("licence")),
        "buttons": buttons,
    })

    if not locale or query.get("include-all-locales") == "true":
        locale = "all"
    if query.get("create-toolbars") == "true":
        extension_settings["put_button_on_toolbar"] = True
        extension_settings["include_toolbars"] = -1
    else:
        extension_settings["include_toolbars"] = 0
    extension_settings["create_menu"] = query.get("create-menu") == "true"
    extension_settings["locale"] = locale
    applications = query.getlist("button-application")
    if not applications:
        applications = query.get("application", "all").split(",")
    extension_settings["applications"] = applications
    update_query = query.copy()
    update_query["application"] = ",".join(applications)
    update_query["locale"] = locale
    allowed_options = set(("application", "locale", "button", "create-menu", "create-toolbars", "icon-size"))
    for key in update_query.keys():
        if key not in allowed_options:
            del update_query[key]
    icons_size = settings.TBUTTON_ICON_SET_SIZES.get(settings.TBUTTON_DEFAULT_ICONS).get(query.get("icon-size"))
    if icons_size:
        extension_settings["icon_size"] = icons_size
    extension_settings["extension_id"] = "%s@button.codefisher.org" % hashlib.md5("_".join(sorted(buttons))).hexdigest()
    extension_settings["update_url"] = "https://%s%s?%s" % (Site.objects.get_current().domain,
            reverse("tbutton-update"), escape(update_query.urlencode()))

    output = io.BytesIO()
    buttons, button_locales = build.build_extension(extension_settings, output=output)
    responce = HttpResponse(output.getvalue(), content_type="application/x-xpinstall")
    output.close()
    if request.POST.get("offer-download") == "true" or ("browser" not in applications):
        responce['Content-Disposition'] = 'attachment; filename=%s' % (extension_settings.get("output_file") % extension_settings)
    else:
        responce['Content-Disposition'] = 'filename=%s' % (extension_settings.get("output_file") % extension_settings)
    session = DownloadSession()
    session.query_string = query.urlencode()
    session.save()
    for button in buttons.buttons():
        Button.objects.create(name=button, session=session)
    for button in buttons.applications():
        Application.objects.create(name=button, session=session)
    return responce

@csrf_exempt
def create(request):
    #if request.method != "POST":
    #    raise Http404()
    buttons = request.GET.getlist("button")
    if not buttons or "update-submit" in request.GET:
        applications = ",".join(request.GET.getlist("button-application"))
        locale = request.GET.get("button-locale")
        kwargs = {}
        if locale:
            kwargs["locale_name"] = locale
        if applications:
            kwargs["applications"] = applications
        return redirect(reverse("tbutton-custom", kwargs=kwargs))
    return create_buttons(request, request.GET)

def statistics(request, days=30, template_name='tbutton_maker/statistics.html'):
    locale_str = get_extenion_data()[1]

    time = datetime.datetime.now() - datetime.timedelta(days)
    buttons = Button.objects.filter(session__time__gt=time)
    stats = list(buttons.values('name').annotate(downloads=Count('name')).order_by("-downloads"))
    sum = buttons.count()
    total = 0
    for item in stats:
        count = item["downloads"]
        total += count
        item.update({
            "label": locale_str('label', item["name"]),
            "average": (float(count) / days),
            "percent": (float(count) / sum * 100),
            "total": (float(total) / sum * 100)
        })
    data = {
        "stats": stats,
        "count": sum,
        "average": float(sum)/(len(stats)*days) if stats else 0
    }
    return render_to_response(template_name, data, RequestContext(request))

def old_update(request):
    query = QueryDict('').copy()
    query.setlist('button', request.GET.get('buttons').split('_'))
    return redirect("%s?%s" % (reverse('tbutton-update'), query.urlencode()))

def update(request):
    def flat(l):
        return [item for sublist in l for item in sublist]
    buttons = request.GET.getlist("button")
    applications = request.GET.getlist("button-application")
    if not applications:
        applications = request.GET.get("application", "all").split(",")
    if applications == ["all"]:
        app_data = flat(config.get("applications_data").values())
    else:
        app_data = flat([config.get("applications_data").get(app) for app in applications])
    update_url = "https://%s%s?%s" % (Site.objects.get_current().domain,
            reverse("tbutton-make-button"), request.GET.urlencode())

    data = {
        "applications": app_data,
        "version": config.get("version"),
        "update_url": update_url,
        "extension_id": "%s@button.codefisher.org" % hashlib.md5("_".join(sorted(buttons))).hexdigest(),
    }

    return render(request, "tbutton_maker/update.rdf", data, content_type="application/xml+rdf")

def make(request):
    return create_buttons(request, request.GET)

def page_it(request, entries_list):
    paginator = Paginator(entries_list, 10)
    # todo: this does not work now
    try:
        page = int(request.GET.get('page', '1'))
    except ValueError:
        page = 1

    try:
        entries = paginator.page(page)
    except (EmptyPage, InvalidPage):
        # todo: better raise a redirect
        entries = paginator.page(paginator.num_pages)
    return entries

def list_app_buttons(request, app_name, days=30, template_name='tbutton_maker/app_list.html'):
    app_data = config.get("applications_data")
    if app_name not in app_data:
        for key, items in app_data.iteritems():
            if app_name.lower() in [item[0].lower() for item in items]:
                app_name = key
                break
        else:
            raise Http404
    time = datetime.datetime.now() - datetime.timedelta(days)
    buttons = Button.objects.filter(session__time__gt=time)
    stats = dict((item["name"], item["downloads"]) for item in buttons.values('name').annotate(downloads=Count('name')).order_by("-downloads"))
    def button_key(item):
        stats.get(item[0], 0)
    data = index(request, applications=app_name, template_name=None)
    data["button_data"].sort(key=button_key)
    data["entries"] = page_it(request, data["button_data"])
    data["application"] = app_name
    return render(request, template_name, data)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import factory
import names

from factory.django import DjangoModelFactory

from . import models

JOURNAL_CHOICES = [
    ('Bioinformatics', 'Bioinformatics'),
    ('BMC Bioinf.', 'BMC Bioinformatics'),
    ('JACS', 'Journal of the American Chemical Society'),
    ('J. Comput. Chem.', 'Journal of Computational Chemistry'),
    ('Nat. Biotechnol.', 'Nature Biotechnology'),
    ('Nucleic Acids Res.', 'Nucleic Acids Research'),
    ('PNAS', 'Proceedings of the National Academy of Sciences of the United States of America'),  # NOPEP8
    ('Proteins Struct. Funct. Bioinf.', 'Proteins: Structure, Function, and Bioinformatics'),  # NOPEP8
]

class AbstractHuman(DjangoModelFactory):
    FACTORY_FOR = models.AbstractHuman
    ABSTRACT_FACTORY = True

    first_name = names.get_first_name()
    last_name = names.get_last_name()

class AuthorFactory(AbstractHuman):
    FACTORY_FOR = models.Author

class EditorFactory(AbstractHuman):
    FACTORY_FOR = models.Editor

class AbstractEntityFactory(DjangoModelFactory):
    FACTORY_FOR = models.AbstractEntity
    ABSTRACT_FACTORY = True

class JournalFactory(AbstractEntityFactory):
    FACTORY_FOR = models.Journal
    FACTORY_DJANGO_GET_OR_CREATE = ('abbreviation',)

    name = factory.Iterator(JOURNAL_CHOICES, getter=lambda c: c[1])
    abbreviation = factory.Iterator(JOURNAL_CHOICES, getter=lambda c: c[0])

class PublisherFactory(AbstractEntityFactory):
    FACTORY_FOR = models.Publisher

class EntryFactory(DjangoModelFactory):
    FACTORY_FOR = models.Entry

class CollectionFactory(DjangoModelFactory):
    FACTORY_FOR = models.Collection
<commit_msg>Implement the EntryFactory<commit_after># -*- coding: utf-8 -*-
import datetime
import factory
import names
import random

from factory.django import DjangoModelFactory
from factory.fuzzy import BaseFuzzyAttribute

from . import models

JOURNAL_CHOICES = [
    ('Bioinformatics', 'Bioinformatics'),
    ('BMC Bioinf.', 'BMC Bioinformatics'),
    ('JACS', 'Journal of the American Chemical Society'),
    ('J. Comput. Chem.', 'Journal of Computational Chemistry'),
    ('Nat. Biotechnol.', 'Nature Biotechnology'),
    ('Nucleic Acids Res.', 'Nucleic Acids Research'),
    ('PNAS', 'Proceedings of the National Academy of Sciences of the United States of America'),  # NOPEP8
    ('Proteins Struct. Funct. Bioinf.', 'Proteins: Structure, Function, and Bioinformatics'),  # NOPEP8
]

ENTRY_TYPES_RAW_CHOICES = [c[0] for c in models.Entry.ENTRY_TYPES_CHOICES]

# Custom fuzzy attributes definition
#
class FuzzyFirstName(BaseFuzzyAttribute):
    """Random first name"""

    def fuzz(self):
        return names.get_first_name()

class FuzzyLastName(BaseFuzzyAttribute):
    """Random last name"""

    def fuzz(self):
        return names.get_last_name()

class FuzzyPages(BaseFuzzyAttribute):
    """Random pages numbers separated by double-hyphens"""

    def __init__(self, low, high=None, **kwargs):
        if high is None:
            high = low
            low = 1

        self.low = low
        self.high = high

        super(FuzzyPages, self).__init__(**kwargs)

    def fuzz(self):
        start = random.randint(self.low, self.high)
        end = random.randint(start, self.high)
        return "%d--%d" % (start, end)

# Factories
#
class AbstractHuman(DjangoModelFactory):
    FACTORY_FOR = models.AbstractHuman
    ABSTRACT_FACTORY = True

    first_name = FuzzyFirstName()
    last_name = FuzzyLastName()

class AuthorFactory(AbstractHuman):
    FACTORY_FOR = models.Author

class EditorFactory(AbstractHuman):
    FACTORY_FOR = models.Editor

class AbstractEntityFactory(DjangoModelFactory):
    FACTORY_FOR = models.AbstractEntity
    ABSTRACT_FACTORY = True

class JournalFactory(AbstractEntityFactory):
    FACTORY_FOR = models.Journal
    FACTORY_DJANGO_GET_OR_CREATE = ('abbreviation',)

    name = factory.Iterator(JOURNAL_CHOICES, getter=lambda c: c[1])
    abbreviation = factory.Iterator(JOURNAL_CHOICES, getter=lambda c: c[0])

class PublisherFactory(AbstractEntityFactory):
    FACTORY_FOR = models.Publisher

class EntryFactory(DjangoModelFactory):
    FACTORY_FOR = models.Entry

    type = factory.fuzzy.FuzzyChoice(ENTRY_TYPES_RAW_CHOICES)
    title = factory.Sequence(lambda n: 'Entry title %s' % n)
    journal = factory.SubFactory(JournalFactory)
    publication_date = factory.fuzzy.FuzzyDate(datetime.date(1942, 1, 1))
    volume = factory.fuzzy.FuzzyInteger(1, 10)
    number = factory.fuzzy.FuzzyInteger(1, 50)
    pages = FuzzyPages(1, 2000)

    # Many to many
    @factory.post_generation
    def authors(self, create, extracted, **kwargs):
        if not create:
            # Simple build, do nothing.
            return

        if extracted:
            # A list of groups were passed in, use them
            for author in extracted:
                self.authors.add(author)

class CollectionFactory(DjangoModelFactory):
    FACTORY_FOR = models.Collection
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# MIT License

from sqlalchemy import Column
from sqlalchemy import ForeignKey
from sqlalchemy import UniqueConstraint
from sqlalchemy import CheckConstraint
from sqlalchemy import Integer
from sqlalchemy.orm import relationship
from .BaseMixin import BASE

class TagWord(BASE):
    '''
    Tag instances are composed of a list of TagWord instances.
    Tags with spaces are composed of multiple TagWord instances.
    Each TagWord maps a Word to a position and a specific Tag.
    position is always 0 unless the tag is composed of multiple TagWord instances
    The Slide/Bullett example:
        http://docs.sqlalchemy.org/en/latest/orm/extensions/orderinglist.html
        seems to confirm that TagWords are necessary because in this instance
        many Slides(Tags) can have the same Bullet(Word), and in the example
        a Bullet can only exist on one Slide.

    The tagword table has a row count:
    of (4 col) that are parsed by the ORM to reconstruct each Tag's __repr__.
    * row_count = count of every word in every tag including duplicates
    * row_count = tags + spaces
    Does not appear to be a big deal because it's not exp and most tags have 0 spaces

    '''
    __table_args__ = (UniqueConstraint('tag_id', 'word_id', 'position', 'previous_position'),) #previous_position?
    tag_id = Column(Integer,
                    ForeignKey("tag.id"),
                    unique=False,
                    primary_key=True)
    word_id = Column(Integer,
                     ForeignKey("word.id"),
                     unique=False,
                     primary_key=True)
    # Must be signed int because -1 has special meaning
    position_constraint = 'position<100' #limit words/tag to 100
    position = Column(Integer,
                      CheckConstraint(position_constraint),
                      unique=False,
                      primary_key=True)
    previous_position_constraint = \
        '(previous_position IS NULL AND position = 0) ' + \
        'OR ((previous_position = position - 1) IS TRUE)'
    previous_position = Column(Integer,
                               CheckConstraint(previous_position_constraint),
                               primary_key=False,
                               nullable=True)
    # collection_class=set?
    word = relationship("Word", backref='tag_words')
    def __repr__(self):
        return str(self.word)

#    def __repr__(self):
#        return 'TagWord<' + \
#            'word: ' + str(self.word) + \
#            ', tag_id: ' + str(self.tag_id) + \
#            ', word_id: ' + str(self.word_id) + \
#            ', position: ' + str(self.position) + '>'

<commit_msg>auto-commit<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# MIT License

from sqlalchemy import Column
from sqlalchemy import ForeignKey
from sqlalchemy import UniqueConstraint
from sqlalchemy import CheckConstraint
from sqlalchemy import Integer
from sqlalchemy.orm import relationship
from .BaseMixin import BASE

class TagWord(BASE):
    '''
    Tag instances are composed of a list of TagWord instances.
    Tags with spaces are composed of multiple TagWord instances.
    Each TagWord maps a Word to a position and a specific Tag.
    position is always 0 unless the tag is composed of multiple TagWord instances
    The Slide/Bullett example:
        http://docs.sqlalchemy.org/en/latest/orm/extensions/orderinglist.html
        seems to confirm that TagWords are necessary because in this instance
        many Slides(Tags) can have the same Bullet(Word), and in the example
        a Bullet can only exist on one Slide.

    The tagword table has a row count:
    of (4 col) that are parsed by the ORM to reconstruct each Tag's __repr__.
    * row_count = count of every word in every tag including duplicates
    * row_count = tags + spaces
    Does not appear to be a big deal because it's not exp and most tags have 0 spaces

    '''
    __table_args__ = (UniqueConstraint('tag_id', 'word_id', 'position', 'previous_position'),) #previous_position?
    tag_id = Column(Integer,
                    ForeignKey("tag.id"),
                    unique=False,
                    primary_key=True)
    word_id = Column(Integer,
                     ForeignKey("word.id"),
                     unique=False,
                     primary_key=True)
    # Must be signed int because -1 has special meaning
    position_constraint = 'position<100' #limit words/tag to 100
    position = Column(Integer,
                      CheckConstraint(position_constraint),
                      unique=False,
                      primary_key=True)
    previous_position_constraint = \
        '(previous_position IS NULL AND position = 0) ' + \
        'OR ((previous_position = position - 1) IS TRUE)'
    previous_position = Column(Integer,
                               CheckConstraint(previous_position_constraint),
                               primary_key=False,
                               nullable=True)
    # collection_class=set?
    word = relationship("Word", backref='tag_words')

    def __repr__(self):
        return str(self.word)

#    def __repr__(self):
#        return 'TagWord<' + \
#            'word: ' + str(self.word) + \
#            ', tag_id: ' + str(self.tag_id) + \
#            ', word_id: ' + str(self.word_id) + \
#            ', position: ' + str(self.position) + '>'

<|endoftext|>


<commit_before>import logging
from werkzeug.datastructures import MultiDict

_log = logging.getLogger(__name__)

def _replace_named_sets_in_ids(v):
  """
  replaces magic named sets references with their ids
  :param v:
  :return:
  """
  import storage
  import phovea_server.plugin

  manager = phovea_server.plugin.lookup('idmanager')

  union = set()

  def add_namedset(vi):
    # convert named sets to the primary ids
    namedset_id = vi
    namedset = storage.get_namedset_by_id(namedset_id)
    uids = namedset['ids']
    id_type = namedset['idType']
    ids = manager.unmap(uids, id_type)
    for id in ids:
      union.add(id)

  if isinstance(v, list):
    for vi in v:
      add_namedset(vi)
  else:
    add_namedset(v)
  return list(union)

def _replace_range_in_ids(v, id_type, target_id_type):
  from phovea_server.dataset import get_mappingmanager, get_idmanager
  from phovea_server.range import parse

  manager = get_idmanager()
  mappingmanager = get_mappingmanager()

  union = set()

  def add_range(r):
    # convert named sets to the primary ids
    uids = parse(r)[0].tolist()
    ids = manager.unmap(uids, id_type)
    if id_type != target_id_type:
      # need to map the ids
      mapped_ids = mappingmanager(id_type, target_id_type, ids)
      for id in mapped_ids:
        if id is not None and len(id) > 0:
          union.add(id[0])  # just the first one for now
    else:
      for id in ids:
        union.add(id)

  if isinstance(v, list):
    for vi in v:
      add_range(vi)
  else:
    add_range(v)
  return list(union)

def filter_logic(view, args):
  """
  parses the request arguments for filter
  :param view:
  :return:
  """
  processed_args = MultiDict()
  extra_args = MultiDict()
  where_clause = {}
  for k, v in args.lists():
    if k.endswith('[]'):
      k = k[:-2]
    if k.startswith('filter_'):
      where_clause[k[7:]] = v  # remove filter_
    else:
      processed_args.setlist(k, v)

  # handle special namedset4 filter types by resolve them and and the real ids as filter
  for k, v in where_clause.items():
    if k.startswith('namedset4'):
      del where_clause[k]  # delete value
      real_key = k[9:]  # remove the namedset4 part
      ids = _replace_named_sets_in_ids(v)
      if real_key not in where_clause:
        where_clause[real_key] = ids
      else:
        where_clause[real_key].extend(ids)
    if k.startswith('rangeOf'):
      del where_clause[k]  # delete value
      id_type_and_key = k[7:]
      id_type = id_type_and_key[:id_type_and_key.index('4')]
      real_key = id_type_and_key[id_type_and_key.index('4') + 1:]  # remove the range4 part
      ids = _replace_range_in_ids(v, id_type, view.idtype)
      if real_key not in where_clause:
        where_clause[real_key] = ids
      else:
        where_clause[real_key].extend(ids)

  def to_clause(k, v):
    l = len(v)
    kp = k.replace('.', '_')
    if l == 1:  # single value
      extra_args[kp] = v[0]
      operator = '='
    else:
      extra_args.setlist(kp, list(v))
      operator = 'IN'
    # find the sub query to replace, can be injected for more complex filter operations based on the input
    sub_query = view.get_filter_subquery(k)
    return sub_query.format(operator=operator, value=':' + kp)

  for key in where_clause.keys():
    if not view.is_valid_filter(key):
      _log.warn('invalid filter key detected for view "%s" and key "%s"', view.query, key)
      del where_clause[key]

  where_default_clause = []
  where_group_clauses = {group: [] for group in view.filter_groups()}
  for k, v in where_clause.items():
    if len(v) <= 0:
      continue
    clause = to_clause(k, v)
    group = view.get_filter_group(k)
    join = view.get_filter_subjoin(k)
    if group is None:
      where_default_clause.append((clause, join))
    else:
      where_group_clauses[group].append((clause, join))

  replacements = dict()
  replacements['and_where'] = (' AND ' + ' AND '.join(c for c, _ in where_default_clause)) if where_default_clause else ''
  replacements['where'] = (' WHERE ' + ' AND '.join(c for c, _ in where_default_clause)) if where_default_clause else ''
  # unique joins
  replacements['joins'] = ' '.join(set(j for _, j in where_default_clause if j is not None))
  for group, v in where_group_clauses.items():
    replacements['and_' + group + '_where'] = (' AND ' + ' AND '.join(c for c, _ in v)) if v else ''
    replacements[group + '_where'] = (' WHERE ' + ' AND '.join(c for c, _ in v)) if v else ''
    replacements[group + '_joins'] = ' '.join(set(j for _, j in v if j is not None))

  return replacements, processed_args, extra_args, where_clause
<commit_msg>fix #93 array for non arrays<commit_after>import logging
from werkzeug.datastructures import MultiDict

_log = logging.getLogger(__name__)

def _replace_named_sets_in_ids(v):
  """
  replaces magic named sets references with their ids
  :param v:
  :return:
  """
  import storage
  import phovea_server.plugin

  manager = phovea_server.plugin.lookup('idmanager')

  union = set()

  def add_namedset(vi):
    # convert named sets to the primary ids
    namedset_id = vi
    namedset = storage.get_namedset_by_id(namedset_id)
    uids = namedset['ids']
    id_type = namedset['idType']
    ids = manager.unmap(uids, id_type)
    for id in ids:
      union.add(id)

  if isinstance(v, list):
    for vi in v:
      add_namedset(vi)
  else:
    add_namedset(v)
  return list(union)

def _replace_range_in_ids(v, id_type, target_id_type):
  from phovea_server.dataset import get_mappingmanager, get_idmanager
  from phovea_server.range import parse

  manager = get_idmanager()
  mappingmanager = get_mappingmanager()

  union = set()

  def add_range(r):
    # convert named sets to the primary ids
    uids = parse(r)[0].tolist()
    ids = manager.unmap(uids, id_type)
    if id_type != target_id_type:
      # need to map the ids
      mapped_ids = mappingmanager(id_type, target_id_type, ids)
      for id in mapped_ids:
        if id is not None and len(id) > 0:
          union.add(id[0])  # just the first one for now
    else:
      for id in ids:
        union.add(id)

  if isinstance(v, list):
    for vi in v:
      add_range(vi)
  else:
    add_range(v)
  return list(union)

def filter_logic(view, args):
  """
  parses the request arguments for filter
  :param view:
  :return:
  """
  processed_args = MultiDict()
  extra_args = dict()
  where_clause = {}
  for k, v in args.lists():
    if k.endswith('[]'):
      k = k[:-2]
    if k.startswith('filter_'):
      where_clause[k[7:]] = v  # remove filter_
    else:
      processed_args.setlist(k, v)

  # handle special namedset4 filter types by resolve them and and the real ids as filter
  for k, v in where_clause.items():
    if k.startswith('namedset4'):
      del where_clause[k]  # delete value
      real_key = k[9:]  # remove the namedset4 part
      ids = _replace_named_sets_in_ids(v)
      if real_key not in where_clause:
        where_clause[real_key] = ids
      else:
        where_clause[real_key].extend(ids)
    if k.startswith('rangeOf'):
      del where_clause[k]  # delete value
      id_type_and_key = k[7:]
      id_type = id_type_and_key[:id_type_and_key.index('4')]
      real_key = id_type_and_key[id_type_and_key.index('4') + 1:]  # remove the range4 part
      ids = _replace_range_in_ids(v, id_type, view.idtype)
      if real_key not in where_clause:
        where_clause[real_key] = ids
      else:
        where_clause[real_key].extend(ids)

  def to_clause(k, v):
    l = len(v)
    kp = k.replace('.', '_')
    if l == 1:  # single value
      extra_args[kp] = v[0]
      operator = '='
    else:
      extra_args[kp] = list(v)
      operator = 'IN'
    # find the sub query to replace, can be injected for more complex filter operations based on the input
    sub_query = view.get_filter_subquery(k)
    return sub_query.format(operator=operator, value=':' + kp)

  for key in where_clause.keys():
    if not view.is_valid_filter(key):
      _log.warn('invalid filter key detected for view "%s" and key "%s"', view.query, key)
      del where_clause[key]

  where_default_clause = []
  where_group_clauses = {group: [] for group in view.filter_groups()}
  for k, v in where_clause.items():
    if len(v) <= 0:
      continue
    clause = to_clause(k, v)
    group = view.get_filter_group(k)
    join = view.get_filter_subjoin(k)
    if group is None:
      where_default_clause.append((clause, join))
    else:
      where_group_clauses[group].append((clause, join))

  replacements = dict()
  replacements['and_where'] = (' AND ' + ' AND '.join(c for c, _ in where_default_clause)) if where_default_clause else ''
  replacements['where'] = (' WHERE ' + ' AND '.join(c for c, _ in where_default_clause)) if where_default_clause else ''
  # unique joins
  replacements['joins'] = ' '.join(set(j for _, j in where_default_clause if j is not None))
  for group, v in where_group_clauses.items():
    replacements['and_' + group + '_where'] = (' AND ' + ' AND '.join(c for c, _ in v)) if v else ''
    replacements[group + '_where'] = (' WHERE ' + ' AND '.join(c for c, _ in v)) if v else ''
    replacements[group + '_joins'] = ' '.join(set(j for _, j in v if j is not None))

  return replacements, processed_args, extra_args, where_clause
<|endoftext|>


<commit_before>#
# Copyright (C) 2011 - 2013 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
import anyconfig.backend.base as Base

import logging
import sys

SUPPORTED = True
try:
    import json
except ImportError:
    try:
        import simplejson as json
    except ImportError:
        sys.stderr.write(
            "JSON module is not available. Disabled its support.\n"
        )
        SUPPORTED = False

def dict_to_container(json_obj_dict):
    return JsonConfigParser.container().create(json_obj_dict)

class JsonConfigParser(Base.ConfigParser):
    _type = "json"
    _extensions = ["json", "jsn"]

    @classmethod
    def loads(cls, config_content, *args, **kwargs):
        return json.loads(config_content, object_hook=dict_to_container)

    @classmethod
    def load(cls, config_path, *args, **kwargs):
        return json.load(open(config_path), object_hook=dict_to_container)

    @classmethod
    def dumps(cls, data, *args, **kwargs):
        return json.dumps(data, indent=2)

    @classmethod
    def dump(cls, data, config_path, *args, **kwargs):
        json.dump(data, open(config_path, "w"), indent=2)

# vim:sw=4:ts=4:et:
<commit_msg>remove lines to import unused modules<commit_after>#
# Copyright (C) 2011 - 2013 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
import anyconfig.backend.base as Base
import sys

SUPPORTED = True
try:
    import json
except ImportError:
    try:
        import simplejson as json
    except ImportError:
        sys.stderr.write(
            "JSON module is not available. Disabled its support.\n"
        )
        SUPPORTED = False

def dict_to_container(json_obj_dict):
    return JsonConfigParser.container().create(json_obj_dict)

class JsonConfigParser(Base.ConfigParser):
    _type = "json"
    _extensions = ["json", "jsn"]

    @classmethod
    def loads(cls, config_content, *args, **kwargs):
        return json.loads(config_content, object_hook=dict_to_container)

    @classmethod
    def load(cls, config_path, *args, **kwargs):
        return json.load(open(config_path), object_hook=dict_to_container)

    @classmethod
    def dumps(cls, data, *args, **kwargs):
        return json.dumps(data, indent=2)

    @classmethod
    def dump(cls, data, config_path, *args, **kwargs):
        json.dump(data, open(config_path, "w"), indent=2)

# vim:sw=4:ts=4:et:
<|endoftext|>