<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Runs the development server of the downstream_node app.
# Not for production use.

import os
import argparse
import csv
import time
import base58
from flask import Flask, jsonify
from werkzeug.serving import run_simple
from werkzeug.wsgi import DispatcherMiddleware
from datetime import datetime, timedelta
from sqlalchemy import select, engine, update, insert, bindparam, true, func, and_

from downstream_node.startup import app, db
from downstream_node.models import Contract, Address, Token, File, Chunk, update_uptime_summary
from downstream_node import node
from downstream_node.utils import MonopolyDistribution, Distribution

def initdb():
    db.create_all()

def cleandb():
    # update uptime summary
    update_uptime_summary()

    # delete expired contracts and files
    s = Contract.__table__.delete().where(Contract.cached == true())

    db.engine.execute(s)

    # and delete unreferenced files
    s = File.__table__.delete().where(~File.__table__.c.id.in_(select([Contract.__table__.c.file_id])
                                                               .union(
                                                               select([Chunk.__table__.c.file_id]))))

    db.engine.execute(s)

def get_available_sizes():
    available_sizes_stmt = select([File.__table__.c.size]).select_from(Chunk.__table__.join(File.__table__))
    available_sizes_result = db.engine.execute(available_sizes_stmt).fetchall()
    available_sizes = [a[0] for a in available_sizes_result]
    return available_sizes

def maintain_capacity(min_chunk_size, max_chunk_size, size, base=1024):
    # maintains a certain size of available chunks
    while(1):
        available_sizes = get_available_sizes()
        available_dist = Distribution(from_list=available_sizes)
        # print('Sizes already available: {0}'.format(available_dist))
        # print('Total size available: {0}'.format(available_dist.get_total()))
        dist = MonopolyDistribution(min_chunk_size, max_chunk_size, size, base)
        # print('Desired distribution: {0}'.format(dist))
        missing = dist.subtract(available_dist)
        # print('Missing: {0}'.format(missing))
        missing_list = missing.get_list()
        if (len(missing_list) > 0):
            print('Generating chunks: {0}'.format(missing_list))
        for chunk_size in sorted(missing_list, reverse=True):
            generate_chunks(chunk_size)
        if (len(missing_list) > 0):
            print('Done.')
        time.sleep(2)

def generate_chunks(size, number=1):
    # generates a test chunk
    for i in range(0,number):
        node.generate_test_file(size)

def clear_chunks():
    s = Chunk.__table__.delete()

    db.engine.execute(s)

    cleandb()

def updatewhitelist(arg):
    if (not os.path.isfile(arg)):
        # arg is not a file, see if it is an SJCX address
        try:
            base58.b58decode_check(arg)
        except:
            raise RuntimeError(
                'Invalid address given: "{0}" is not a valid SJCX address '
                'or path to whitelist file.'.format(arg))
        s = Address.__table__.select().where(Address.address == arg)
        result = db.engine.execute(s).first()
        bal = app.config['MIN_SJCX_BALANCE']
        if (result is not None):
            if (result.crowdsale_balance <= bal):
                db.engine.execute(Address.__table__.update().\
                    where(Address.id == result.id).\
                    values(crowdsale_balance = bal))
            else:
                print('{0} is already in the whitelist.'.format(arg))
        else:
            db.engine.execute(Address.__table__.insert().\
                values(address=arg,
                       crowdsale_balance=bal))
        return
    with open(arg,'r') as f:
        r = csv.reader(f)
        next(r)
        updated=list()
        for l in r:
            s = Address.__table__.select().where(Address.address == l[0])
            result = db.engine.execute(s).first()
            if (result is not None):
                db.engine.execute(Address.__table__.update().\
                    where(Address.id == result.id).\
                    values(crowdsale_balance=int(l[1])))
            else:
                db.engine.execute(Address.__table__.insert().\
                    values(address=l[0], crowdsale_balance=l[1]))
                result = db.engine.execute(Address.__table__.select().\
                    where(Address.address == l[0])).first()
            updated.append(result.id)
        all = db.engine.execute(Address.__table__.select()).fetchall()
        for row in all:
            if (row.id not in updated):
                # also recursively delete all tokens associated with that address
                tbd_tokens = db.engine.execute(Token.__table__.select().\
                    where(Token.address_id == row.id)).fetchall()
                for t in tbd_tokens:
                    # and all contracts associated with that address
                    db.engine.execute(Contract.__table__.delete().\
                        where(Contract.token_id == t.id))
                    db.engine.execute(Token.__table__.delete().\
                        where(Token.id == t.id))
                db.engine.execute(Address.__table__.delete().\
                    where(Address.id == row.id))

def eval_args(args):
    if args.initdb:
        initdb()
    elif args.cleandb:
        cleandb()
    elif args.clearchunks:
        clear_chunks()
    elif (args.whitelist is not None):
        updatewhitelist(args.whitelist)
    elif (args.generate_chunk is not None):
        generate_chunks(args.generate_chunk)
    elif (args.maintain is not None):
        print('Maintaining total size: {0}, min chunk size: {1}, max chunk size: {2}'.format(
            args.maintain[2],
            args.maintain[0],
            args.maintain[1]))
        maintain_capacity(int(args.maintain[0]), int(args.maintain[1]), int(args.maintain[2]))
    else:
        debug_root = Flask(__name__)
        debug_root.debug = True
        debug_root.add_url_rule('/','index',lambda: jsonify(msg='debugging'))
        prefixed_app = DispatcherMiddleware(debug_root, {app.config['APPLICATION_ROOT']:app})
        run_simple('localhost', 5000, prefixed_app, use_reloader=True)

def parse_args():
    parser = argparse.ArgumentParser('downstream')
    parser.add_argument('--initdb', action='store_true')
    parser.add_argument('--cleandb', action='store_true')
    parser.add_argument('--whitelist', help='updates the white list '
        'in the db and exits from a whitelist csv file.  each row except'
        'the first should be in the format\n'
        '"address","crowdsale_balance",...\n'
        'and the first row will be skipped.  If argument is not a file name'
        ' it will attempt to parse the argument as an SJCX address.'
        'If successful, it will add that SJCX address to the in database'
        'whitelist.')
    parser.add_argument('--generate-chunk', help='Generates a test chunk of'
        'specified size.', type=int)
    parser.add_argument('--maintain', help='Maintain available chunk capacity'
        'Specify three values (min chunk size, max chunk size, total pre-gen '
        'size)', nargs=3)
    parser.add_argument('--clearchunks', help='Removes all chunks from '
                        ' the database', action='store_true')
    return parser.parse_args()

def main():
    args = parse_args()
    eval_args(args)

if __name__ == '__main__':
    main()
<commit_msg>make base an option for maintenance size for tweaking chunk distribution<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Runs the development server of the downstream_node app.
# Not for production use.

import os
import argparse
import csv
import time
import base58
from flask import Flask, jsonify
from werkzeug.serving import run_simple
from werkzeug.wsgi import DispatcherMiddleware
from datetime import datetime, timedelta
from sqlalchemy import select, engine, update, insert, bindparam, true, func, and_

from downstream_node.startup import app, db
from downstream_node.models import Contract, Address, Token, File, Chunk, update_uptime_summary
from downstream_node import node
from downstream_node.utils import MonopolyDistribution, Distribution

def initdb():
    db.create_all()

def cleandb():
    # update uptime summary
    update_uptime_summary()

    # delete expired contracts and files
    s = Contract.__table__.delete().where(Contract.cached == true())

    db.engine.execute(s)

    # and delete unreferenced files
    s = File.__table__.delete().where(~File.__table__.c.id.in_(select([Contract.__table__.c.file_id])
                                                               .union(
                                                               select([Chunk.__table__.c.file_id]))))

    db.engine.execute(s)

def get_available_sizes():
    available_sizes_stmt = select([File.__table__.c.size]).select_from(Chunk.__table__.join(File.__table__))
    available_sizes_result = db.engine.execute(available_sizes_stmt).fetchall()
    available_sizes = [a[0] for a in available_sizes_result]
    return available_sizes

def maintain_capacity(min_chunk_size, max_chunk_size, size, base=1024):
    # maintains a certain size of available chunks
    while(1):
        available_sizes = get_available_sizes()
        available_dist = Distribution(from_list=available_sizes)
        # print('Sizes already available: {0}'.format(available_dist))
        # print('Total size available: {0}'.format(available_dist.get_total()))
        dist = MonopolyDistribution(min_chunk_size, max_chunk_size, size, base)
        # print('Desired distribution: {0}'.format(dist))
        missing = dist.subtract(available_dist)
        # print('Missing: {0}'.format(missing))
        missing_list = missing.get_list()
        if (len(missing_list) > 0):
            print('Generating chunks: {0}'.format(missing_list))
        for chunk_size in sorted(missing_list, reverse=True):
            generate_chunks(chunk_size)
        if (len(missing_list) > 0):
            print('Done.')
        time.sleep(2)

def generate_chunks(size, number=1):
    # generates a test chunk
    for i in range(0,number):
        node.generate_test_file(size)

def clear_chunks():
    s = Chunk.__table__.delete()

    db.engine.execute(s)

    cleandb()

def updatewhitelist(arg):
    if (not os.path.isfile(arg)):
        # arg is not a file, see if it is an SJCX address
        try:
            base58.b58decode_check(arg)
        except:
            raise RuntimeError(
                'Invalid address given: "{0}" is not a valid SJCX address '
                'or path to whitelist file.'.format(arg))
        s = Address.__table__.select().where(Address.address == arg)
        result = db.engine.execute(s).first()
        bal = app.config['MIN_SJCX_BALANCE']
        if (result is not None):
            if (result.crowdsale_balance <= bal):
                db.engine.execute(Address.__table__.update().\
                    where(Address.id == result.id).\
                    values(crowdsale_balance = bal))
            else:
                print('{0} is already in the whitelist.'.format(arg))
        else:
            db.engine.execute(Address.__table__.insert().\
                values(address=arg,
                       crowdsale_balance=bal))
        return
    with open(arg,'r') as f:
        r = csv.reader(f)
        next(r)
        updated=list()
        for l in r:
            s = Address.__table__.select().where(Address.address == l[0])
            result = db.engine.execute(s).first()
            if (result is not None):
                db.engine.execute(Address.__table__.update().\
                    where(Address.id == result.id).\
                    values(crowdsale_balance=int(l[1])))
            else:
                db.engine.execute(Address.__table__.insert().\
                    values(address=l[0], crowdsale_balance=l[1]))
                result = db.engine.execute(Address.__table__.select().\
                    where(Address.address == l[0])).first()
            updated.append(result.id)
        all = db.engine.execute(Address.__table__.select()).fetchall()
        for row in all:
            if (row.id not in updated):
                # also recursively delete all tokens associated with that address
                tbd_tokens = db.engine.execute(Token.__table__.select().\
                    where(Token.address_id == row.id)).fetchall()
                for t in tbd_tokens:
                    # and all contracts associated with that address
                    db.engine.execute(Contract.__table__.delete().\
                        where(Contract.token_id == t.id))
                    db.engine.execute(Token.__table__.delete().\
                        where(Token.id == t.id))
                db.engine.execute(Address.__table__.delete().\
                    where(Address.id == row.id))

def eval_args(args):
    if args.initdb:
        initdb()
    elif args.cleandb:
        cleandb()
    elif args.clearchunks:
        clear_chunks()
    elif (args.whitelist is not None):
        updatewhitelist(args.whitelist)
    elif (args.generate_chunk is not None):
        generate_chunks(args.generate_chunk)
    elif (args.maintain is not None):
        print('Maintaining total size: {0}, min chunk size: {1}, max chunk size: {2}'.format(
            args.maintain[2],
            args.maintain[0],
            args.maintain[1]))
        maintain_capacity(int(args.maintain[0]), int(args.maintain[1]), int(args.maintain[2]), int(args.maintain[3]))
    else:
        debug_root = Flask(__name__)
        debug_root.debug = True
        debug_root.add_url_rule('/','index',lambda: jsonify(msg='debugging'))
        prefixed_app = DispatcherMiddleware(debug_root, {app.config['APPLICATION_ROOT']:app})
        run_simple('localhost', 5000, prefixed_app, use_reloader=True)

def parse_args():
    parser = argparse.ArgumentParser('downstream')
    parser.add_argument('--initdb', action='store_true')
    parser.add_argument('--cleandb', action='store_true')
    parser.add_argument('--whitelist', help='updates the white list '
        'in the db and exits from a whitelist csv file.  each row except'
        'the first should be in the format\n'
        '"address","crowdsale_balance",...\n'
        'and the first row will be skipped.  If argument is not a file name'
        ' it will attempt to parse the argument as an SJCX address.'
        'If successful, it will add that SJCX address to the in database'
        'whitelist.')
    parser.add_argument('--generate-chunk', help='Generates a test chunk of'
        'specified size.', type=int)
    parser.add_argument('--maintain', help='Maintain available chunk capacity'
        'Specify three values (min chunk size, max chunk size, total pre-gen '
        'size)', nargs=4)
    parser.add_argument('--clearchunks', help='Removes all chunks from '
                        ' the database', action='store_true')
    return parser.parse_args()

def main():
    args = parse_args()
    eval_args(args)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
These models are used in the databases of content that get imported from Studio.
Any fields added here (and not in base_models.py) are assumed to be locally
calculated cached fields. If a field is intended to be imported from a content
database generated by Studio, it should be added in base_models.py.

*DEVELOPER WARNING regarding updates to these models*

If you modify the schema here, it has implications for the content import pipeline
because we will need to calculate these values during content import (as we they will
not be present in the content databases distributed by Studio).

In the case where new fields are added that do not need to be added to an export schema
the generate_schema command should be run like this:

    `kolibri manage generate_schema current`

This will just regenerate the current schema for SQLAlchemy, so that we can use SQLAlchemy
to calculate these fields if needed (this can frequently be more efficient than using the
Django ORM for these calculations).
"""
from __future__ import print_function

import os
from gettext import gettext as _

from django.core.urlresolvers import reverse
from django.db import connection
from django.db import models
from django.db.models import Min
from django.db.models import Q
from django.db.models import QuerySet
from django.utils.encoding import python_2_unicode_compatible
from django.utils.text import get_valid_filename
from le_utils.constants import content_kinds
from le_utils.constants import format_presets
from mptt.managers import TreeManager
from mptt.querysets import TreeQuerySet

from .utils import paths
from kolibri.core.content import base_models
from kolibri.core.content.errors import InvalidStorageFilenameError
from kolibri.core.device.models import ContentCacheKey
from kolibri.core.mixins import FilterByUUIDQuerysetMixin

PRESET_LOOKUP = dict(format_presets.choices)

@python_2_unicode_compatible
class ContentTag(base_models.ContentTag):
    def __str__(self):
        return self.tag_name

class ContentNodeQueryset(TreeQuerySet, FilterByUUIDQuerysetMixin):
    def dedupe_by_content_id(self, use_distinct=True):
        # Cannot use distinct if queryset is also going to use annotate,
        # so optional use_distinct flag can be used to fallback to a subquery
        # remove duplicate content nodes based on content_id
        if connection.vendor == "sqlite" or not use_distinct:
            if connection.vendor == "postgresql":
                # Create a subquery of all contentnodes deduped by content_id
                # to avoid calling distinct on an annotated queryset.
                deduped_ids = self.model.objects.order_by("content_id").distinct(
                    "content_id"
                )
            else:
                # adapted from https://code.djangoproject.com/ticket/22696
                deduped_ids = (
                    self.values("content_id")
                    .annotate(node_id=Min("id"))
                    .values_list("node_id", flat=True)
                )
            return self.filter_by_uuids(deduped_ids)

        # when using postgres, we can call distinct on a specific column
        elif connection.vendor == "postgresql":
            return self.order_by("content_id").distinct("content_id")

    def filter_by_content_ids(self, content_ids, validate=True):
        return self._by_uuids(content_ids, validate, "content_id", True)

    def exclude_by_content_ids(self, content_ids, validate=True):
        return self._by_uuids(content_ids, validate, "content_id", False)

class ContentNodeManager(
    models.Manager.from_queryset(ContentNodeQueryset), TreeManager
):
    def get_queryset(self, *args, **kwargs):
        """
        Ensures that this manager always returns nodes in tree order.
        """
        return (
            super(TreeManager, self)
            .get_queryset(*args, **kwargs)
            .order_by(self.tree_id_attr, self.left_attr)
        )

    def build_tree_nodes(self, data, target=None, position="last-child"):
        """
        vendored from:
        https://github.com/django-mptt/django-mptt/blob/fe2b9cc8cfd8f4b764d294747dba2758147712eb/mptt/managers.py#L614
        """
        opts = self.model._mptt_meta
        if target:
            tree_id = target.tree_id
            if position in ("left", "right"):
                level = getattr(target, opts.level_attr)
                if position == "left":
                    cursor = getattr(target, opts.left_attr)
                else:
                    cursor = getattr(target, opts.right_attr) + 1
            else:
                level = getattr(target, opts.level_attr) + 1
                if position == "first-child":
                    cursor = getattr(target, opts.left_attr) + 1
                else:
                    cursor = getattr(target, opts.right_attr)
        else:
            tree_id = self._get_next_tree_id()
            cursor = 1
            level = 0

        stack = []

        def treeify(data, cursor=1, level=0):
            data = dict(data)
            children = data.pop("children", [])
            node = self.model(**data)
            stack.append(node)
            setattr(node, opts.tree_id_attr, tree_id)
            setattr(node, opts.level_attr, level)
            setattr(node, opts.left_attr, cursor)
            for child in children:
                cursor = treeify(child, cursor=cursor + 1, level=level + 1)
            cursor += 1
            setattr(node, opts.right_attr, cursor)
            return cursor

        treeify(data, cursor=cursor, level=level)

        if target:
            self._create_space(2 * len(stack), cursor - 1, tree_id)

        return stack

@python_2_unicode_compatible
class ContentNode(base_models.ContentNode):
    """
    The primary object type in a content database. Defines the properties that are shared
    across all content types.

    It represents videos, exercises, audio, documents, and other 'content items' that
    exist as nodes in content channels.
    """

    # Fields used only on Kolibri and not imported from a content database
    # Total number of coach only resources for this node
    num_coach_contents = models.IntegerField(default=0, null=True, blank=True)
    # Total number of available resources on the device under this topic - if this is not a topic
    # then it is 1 or 0 depending on availability
    on_device_resources = models.IntegerField(default=0, null=True, blank=True)

    objects = ContentNodeManager()

    class Meta:
        ordering = ("lft",)
        index_together = [
            ["level", "channel_id", "kind"],
            ["level", "channel_id", "available"],
        ]

    def __str__(self):
        return self.title

    def get_descendant_content_ids(self):
        """
        Retrieve a queryset of content_ids for non-topic content nodes that are
        descendants of this node.
        """
        return (
            ContentNode.objects.filter(lft__gte=self.lft, lft__lte=self.rght)
            .exclude(kind=content_kinds.TOPIC)
            .values_list("content_id", flat=True)
        )

@python_2_unicode_compatible
class Language(base_models.Language):
    def __str__(self):
        return self.lang_name or ""

def get_download_filename(title, preset, extension):
    """
    Return a valid filename to be downloaded as.
    """
    filename = "{} ({}).{}".format(title, preset, extension)
    valid_filename = get_valid_filename(filename)
    return valid_filename

class File(base_models.File):
    """
    The second to bottom layer of the contentDB schema, defines the basic building brick for content.
    Things it can represent are, for example, mp4, avi, mov, html, css, jpeg, pdf, mp3...
    """

    class Meta:
        ordering = ["priority"]

    class Admin:
        pass

    def get_extension(self):
        return self.local_file.extension

    def get_file_size(self):
        return self.local_file.file_size

    def get_storage_url(self):
        return self.local_file.get_storage_url()

    def get_preset(self):
        """
        Return the preset.
        """
        return PRESET_LOOKUP.get(self.preset, _("Unknown format"))

    def get_download_filename(self):
        """
        Return a valid filename to be downloaded as.
        """
        return get_download_filename(
            self.contentnode.title, self.get_preset(), self.get_extension()
        )

    def get_download_url(self):
        """
        Return the download url.
        """
        new_filename = self.get_download_filename()
        return reverse(
            "kolibri:core:downloadcontent",
            kwargs={
                "filename": self.local_file.get_filename(),
                "new_filename": new_filename,
            },
        )

class LocalFileQueryset(models.QuerySet, FilterByUUIDQuerysetMixin):
    def delete_unused_files(self):
        for file in self.get_unused_files():
            try:
                os.remove(paths.get_content_storage_file_path(file.get_filename()))
                yield True, file
            except (IOError, OSError, InvalidStorageFilenameError):
                yield False, file
        self.get_unused_files().update(available=False)

    def get_orphan_files(self):
        return self.filter(files__isnull=True)

    def delete_orphan_file_objects(self):
        return self.filter(files__isnull=True).delete()

    def get_unused_files(self):
        return self.filter(
            Q(files__contentnode__available=False) | Q(files__isnull=True)
        ).filter(available=True)

@python_2_unicode_compatible
class LocalFile(base_models.LocalFile):
    """
    The bottom layer of the contentDB schema, defines the local state of files on the device storage.
    """

    objects = LocalFileQueryset.as_manager()

    class Admin:
        pass

    def __str__(self):
        return paths.get_content_file_name(self)

    def get_filename(self):
        return self.__str__()

    def get_storage_url(self):
        """
        Return a url for the client side to retrieve the content file.
        The same url will also be exposed by the file serializer.
        """
        return paths.get_local_content_storage_file_url(self)

    def delete_stored_file(self):
        """
        Delete the stored file from disk.
        """
        deleted = False

        try:
            os.remove(paths.get_content_storage_file_path(self.get_filename()))
            deleted = True
        except (IOError, OSError, InvalidStorageFilenameError):
            deleted = False

        self.available = False
        self.save()
        return deleted

class AssessmentMetaData(base_models.AssessmentMetaData):
    """
    A model to describe additional metadata that characterizes assessment behaviour in Kolibri.
    This model contains additional fields that are only revelant to content nodes that probe a
    user's state of knowledge and allow them to practice to Mastery.
    ContentNodes with this metadata may also be able to be used within quizzes and exams.
    """

    pass

class ChannelMetadataQueryset(QuerySet, FilterByUUIDQuerysetMixin):
    pass

@python_2_unicode_compatible
class ChannelMetadata(base_models.ChannelMetadata):
    """
    Holds metadata about all existing content databases that exist locally.
    """

    # precalculated fields during annotation/migration
    published_size = models.BigIntegerField(default=0, null=True, blank=True)
    total_resource_count = models.IntegerField(default=0, null=True, blank=True)
    included_languages = models.ManyToManyField(
        "Language", related_name="channels", verbose_name="languages", blank=True
    )
    order = models.PositiveIntegerField(default=0, null=True, blank=True)
    public = models.NullBooleanField()

    objects = ChannelMetadataQueryset.as_manager()

    class Admin:
        pass

    class Meta:
        ordering = ["order"]

    def __str__(self):
        return self.name

    def delete_content_tree_and_files(self):
        # Use Django ORM to ensure cascading delete:
        self.root.delete()
        ContentCacheKey.update_cache_key()
<commit_msg>shortcircuit bad performance query<commit_after>"""
These models are used in the databases of content that get imported from Studio.
Any fields added here (and not in base_models.py) are assumed to be locally
calculated cached fields. If a field is intended to be imported from a content
database generated by Studio, it should be added in base_models.py.

*DEVELOPER WARNING regarding updates to these models*

If you modify the schema here, it has implications for the content import pipeline
because we will need to calculate these values during content import (as we they will
not be present in the content databases distributed by Studio).

In the case where new fields are added that do not need to be added to an export schema
the generate_schema command should be run like this:

    `kolibri manage generate_schema current`

This will just regenerate the current schema for SQLAlchemy, so that we can use SQLAlchemy
to calculate these fields if needed (this can frequently be more efficient than using the
Django ORM for these calculations).
"""
from __future__ import print_function

import os
from gettext import gettext as _

from django.core.urlresolvers import reverse
from django.db import connection
from django.db import models
from django.db.models import Min
from django.db.models import Q
from django.db.models import QuerySet
from django.utils.encoding import python_2_unicode_compatible
from django.utils.text import get_valid_filename
from le_utils.constants import content_kinds
from le_utils.constants import format_presets
from mptt.managers import TreeManager
from mptt.querysets import TreeQuerySet

from .utils import paths
from kolibri.core.content import base_models
from kolibri.core.content.errors import InvalidStorageFilenameError
from kolibri.core.device.models import ContentCacheKey
from kolibri.core.mixins import FilterByUUIDQuerysetMixin

PRESET_LOOKUP = dict(format_presets.choices)

@python_2_unicode_compatible
class ContentTag(base_models.ContentTag):
    def __str__(self):
        return self.tag_name

class ContentNodeQueryset(TreeQuerySet, FilterByUUIDQuerysetMixin):
    def dedupe_by_content_id(self, use_distinct=True):
        # Cannot use distinct if queryset is also going to use annotate,
        # so optional use_distinct flag can be used to fallback to a subquery
        # remove duplicate content nodes based on content_id
        if connection.vendor == "sqlite" or not use_distinct:
            if connection.vendor == "postgresql":
                # Create a subquery of all contentnodes deduped by content_id
                # to avoid calling distinct on an annotated queryset.
                deduped_ids = self.model.objects.order_by("content_id").distinct(
                    "content_id"
                )
            else:
                # adapted from https://code.djangoproject.com/ticket/22696
                deduped_ids = (
                    self.values("content_id")
                    .annotate(node_id=Min("id"))
                    .values_list("node_id", flat=True)
                )
            return self.filter_by_uuids(deduped_ids)

        # when using postgres, we can call distinct on a specific column
        elif connection.vendor == "postgresql":
            return self.order_by("content_id").distinct("content_id")

    def filter_by_content_ids(self, content_ids, validate=True):
        return self._by_uuids(content_ids, validate, "content_id", True)

    def exclude_by_content_ids(self, content_ids, validate=True):
        return self._by_uuids(content_ids, validate, "content_id", False)

class ContentNodeManager(
    models.Manager.from_queryset(ContentNodeQueryset), TreeManager
):
    def get_queryset(self, *args, **kwargs):
        """
        Ensures that this manager always returns nodes in tree order.
        """
        return (
            super(TreeManager, self)
            .get_queryset(*args, **kwargs)
            .order_by(self.tree_id_attr, self.left_attr)
        )

    def build_tree_nodes(self, data, target=None, position="last-child"):
        """
        vendored from:
        https://github.com/django-mptt/django-mptt/blob/fe2b9cc8cfd8f4b764d294747dba2758147712eb/mptt/managers.py#L614
        """
        opts = self.model._mptt_meta
        if target:
            tree_id = target.tree_id
            if position in ("left", "right"):
                level = getattr(target, opts.level_attr)
                if position == "left":
                    cursor = getattr(target, opts.left_attr)
                else:
                    cursor = getattr(target, opts.right_attr) + 1
            else:
                level = getattr(target, opts.level_attr) + 1
                if position == "first-child":
                    cursor = getattr(target, opts.left_attr) + 1
                else:
                    cursor = getattr(target, opts.right_attr)
        else:
            tree_id = self._get_next_tree_id()
            cursor = 1
            level = 0

        stack = []

        def treeify(data, cursor=1, level=0):
            data = dict(data)
            children = data.pop("children", [])
            node = self.model(**data)
            stack.append(node)
            setattr(node, opts.tree_id_attr, tree_id)
            setattr(node, opts.level_attr, level)
            setattr(node, opts.left_attr, cursor)
            for child in children:
                cursor = treeify(child, cursor=cursor + 1, level=level + 1)
            cursor += 1
            setattr(node, opts.right_attr, cursor)
            return cursor

        treeify(data, cursor=cursor, level=level)

        if target:
            self._create_space(2 * len(stack), cursor - 1, tree_id)

        return stack

@python_2_unicode_compatible
class ContentNode(base_models.ContentNode):
    """
    The primary object type in a content database. Defines the properties that are shared
    across all content types.

    It represents videos, exercises, audio, documents, and other 'content items' that
    exist as nodes in content channels.
    """

    # Fields used only on Kolibri and not imported from a content database
    # Total number of coach only resources for this node
    num_coach_contents = models.IntegerField(default=0, null=True, blank=True)
    # Total number of available resources on the device under this topic - if this is not a topic
    # then it is 1 or 0 depending on availability
    on_device_resources = models.IntegerField(default=0, null=True, blank=True)

    objects = ContentNodeManager()

    class Meta:
        ordering = ("lft",)
        index_together = [
            ["level", "channel_id", "kind"],
            ["level", "channel_id", "available"],
        ]

    def __str__(self):
        return self.title

    def get_descendant_content_ids(self):
        """
        Retrieve a queryset of content_ids for non-topic content nodes that are
        descendants of this node.
        """
        return (
            ContentNode.objects.filter(lft__gte=self.lft, lft__lte=self.rght)
            .exclude(kind=content_kinds.TOPIC)
            .values_list("content_id", flat=True)
        )

@python_2_unicode_compatible
class Language(base_models.Language):
    def __str__(self):
        return self.lang_name or ""

def get_download_filename(title, preset, extension):
    """
    Return a valid filename to be downloaded as.
    """
    filename = "{} ({}).{}".format(title, preset, extension)
    valid_filename = get_valid_filename(filename)
    return valid_filename

class File(base_models.File):
    """
    The second to bottom layer of the contentDB schema, defines the basic building brick for content.
    Things it can represent are, for example, mp4, avi, mov, html, css, jpeg, pdf, mp3...
    """

    class Meta:
        ordering = ["priority"]

    class Admin:
        pass

    def get_extension(self):
        return self.local_file.extension

    def get_file_size(self):
        return self.local_file.file_size

    def get_storage_url(self):
        return self.local_file.get_storage_url()

    def get_preset(self):
        """
        Return the preset.
        """
        return PRESET_LOOKUP.get(self.preset, _("Unknown format"))

    def get_download_filename(self):
        """
        Return a valid filename to be downloaded as.
        """
        return get_download_filename(
            self.contentnode.title, self.get_preset(), self.get_extension()
        )

    def get_download_url(self):
        """
        Return the download url.
        """
        new_filename = self.get_download_filename()
        return reverse(
            "kolibri:core:downloadcontent",
            kwargs={
                "filename": self.local_file.get_filename(),
                "new_filename": new_filename,
            },
        )

class LocalFileQueryset(models.QuerySet, FilterByUUIDQuerysetMixin):
    def delete_unused_files(self):
        for file in self.get_unused_files():
            try:
                os.remove(paths.get_content_storage_file_path(file.get_filename()))
                yield True, file
            except (IOError, OSError, InvalidStorageFilenameError):
                yield False, file
        self.get_unused_files().update(available=False)

    def get_orphan_files(self):
        return self.filter(files__isnull=True)

    def delete_orphan_file_objects(self):
        return self.filter(files__isnull=True).delete()

    def get_unused_files(self):
        ids = LocalFile.objects.filter(
            Q(files__contentnode__available=False) | Q(files__isnull=True)
        )
        return (
            self.filter(id__in=ids)
            .exclude(files__contentnode__available=True)
            .filter(available=True)
        )

@python_2_unicode_compatible
class LocalFile(base_models.LocalFile):
    """
    The bottom layer of the contentDB schema, defines the local state of files on the device storage.
    """

    objects = LocalFileQueryset.as_manager()

    class Admin:
        pass

    def __str__(self):
        return paths.get_content_file_name(self)

    def get_filename(self):
        return self.__str__()

    def get_storage_url(self):
        """
        Return a url for the client side to retrieve the content file.
        The same url will also be exposed by the file serializer.
        """
        return paths.get_local_content_storage_file_url(self)

    def delete_stored_file(self):
        """
        Delete the stored file from disk.
        """
        deleted = False

        try:
            os.remove(paths.get_content_storage_file_path(self.get_filename()))
            deleted = True
        except (IOError, OSError, InvalidStorageFilenameError):
            deleted = False

        self.available = False
        self.save()
        return deleted

class AssessmentMetaData(base_models.AssessmentMetaData):
    """
    A model to describe additional metadata that characterizes assessment behaviour in Kolibri.
    This model contains additional fields that are only revelant to content nodes that probe a
    user's state of knowledge and allow them to practice to Mastery.
    ContentNodes with this metadata may also be able to be used within quizzes and exams.
    """

    pass

class ChannelMetadataQueryset(QuerySet, FilterByUUIDQuerysetMixin):
    pass

@python_2_unicode_compatible
class ChannelMetadata(base_models.ChannelMetadata):
    """
    Holds metadata about all existing content databases that exist locally.
    """

    # precalculated fields during annotation/migration
    published_size = models.BigIntegerField(default=0, null=True, blank=True)
    total_resource_count = models.IntegerField(default=0, null=True, blank=True)
    included_languages = models.ManyToManyField(
        "Language", related_name="channels", verbose_name="languages", blank=True
    )
    order = models.PositiveIntegerField(default=0, null=True, blank=True)
    public = models.NullBooleanField()

    objects = ChannelMetadataQueryset.as_manager()

    class Admin:
        pass

    class Meta:
        ordering = ["order"]

    def __str__(self):
        return self.name

    def delete_content_tree_and_files(self):
        # Use Django ORM to ensure cascading delete:
        self.root.delete()
        ContentCacheKey.update_cache_key()
<|endoftext|>


<commit_before># Copyright (c) 2001-2015, Canal TP and/or its affiliates. All rights reserved.
#
# This file is part of Navitia,
#     the software to build cool stuff with public transport.
#
# Hope you'll enjoy and contribute to this project,
#     powered by Canal TP (www.canaltp.fr).
# Help us simplify mobility and open public transport:
#     a non ending quest to the responsive locomotion way of traveling!
#
# LICENCE: This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Stay tuned using
# twitter @navitia
# IRC #navitia on freenode
# https://groups.google.com/d/forum/navitia
# www.navitia.io
from itertools import izip
import logging
import itertools
import datetime
from jormungandr.scenarios.utils import compare, get_pseudo_duration
from navitiacommon import response_pb2

#we can't use reverse(enumerate(list)) without creating a temporary list, so we define our own reverse enumerate
reverse_enumerate = lambda l: izip(xrange(len(l)-1, -1, -1), reversed(l))

def _delete_journeys(responses, request):

    if request.get('debug', False):
        return

    nb_deleted = 0
    for r in responses:
        for idx, j in reverse_enumerate(r.journeys):
            if not _to_be_deleted(j):
                continue

            del r.journeys[idx]
            nb_deleted += 1

    if nb_deleted:
        logging.getLogger(__name__).info('filtering {} journeys'.format(nb_deleted))

def filter_journeys(response_list, instance, request, original_request):
    """
    Filter by side effect the list of pb responses's journeys

    first draft, we only remove the journeys with the same vjs
    """

    # for clarity purpose we build a temporary list
    journeys = [j for r in response_list for j in r.journeys]

    #DEBUG
    for j in journeys:
        _debug_journey(j)

    _filter_similar_journeys(journeys, request)

    _filter_not_coherent_journeys(journeys, instance, request, original_request)

    _delete_journeys(response_list, request)

    return response_list

def _get_worst_similar_vjs(j1, j2, request):
    """
    Decide which is the worst journey between 2 similar journeys.

    The choice is made on:
     - asap
     - less fallback
     - duration
    """
    if request.get('clockwise', True):
        if j1.arrival_date_time != j2.arrival_date_time:
            return j1 if j1.arrival_date_time < j2.arrival_date_time else j2
    else:
        if j1.departure_date_time != j2.departure_date_time:
            return j1 if j1.departure_date_time > j2.departure_date_time else j2

    fallback1 = fallback_duration(j1)
    fallback2 = fallback_duration(j2)

    if fallback1 != fallback2:
        return j1 if fallback1 < fallback2 else j2

    return j1 if j1.duration < j2.duration else j2

def _to_be_deleted(journey):
    return 'to_delete' in journey.tags

def _mark_as_dead(journey, *reasons):
    journey.tags.append('to_delete')
    for reason in reasons:
        journey.tags.append('deleted_because_' + reason)

def _filter_similar_journeys(journeys, request):
    """
    for the moment very simple filter.

    we filter the journeys with the same vjs

    in case of similar journeys we let _get_worst_similar_vjs decide which one to delete
    """
    logger = logging.getLogger(__name__)
    for j1, j2 in itertools.combinations(journeys, 2):
        if _to_be_deleted(j1) or _to_be_deleted(j2):
            continue
        if compare(j1, j2, similar_journeys_generator):
            #chose the best
            worst = _get_worst_similar_vjs(j1, j2, request)
            logger.debug("the journeys {}, {} are similar, we delete {}".format(j1.internal_id,
                                                                                j2.internal_id,
                                                                                worst.internal_id))

            _mark_as_dead(worst, 'duplicate_journey', 'similar_to_{other}'
                          .format(other=j1.internal_id if worst == j2 else j2.internal_id))

def way_later(journey, asap_journey, original_request):
    """
    to check if a journey is way later than the asap journey
    we check for each journey the difference between the requested datetime and the arrival datetime
    (the other way around for non clockwise)

    requested dt
    *
                   |=============>
                          asap

                                           |=============>
                                                 journey

    -------------------------------
             asap pseudo duration

    ------------------------------------------------------
                       journey pseudo duration

    """
    requested_dt = original_request['datetime']
    is_clockwise = original_request.get('clockwise', True)

    pseudo_asap_duration = get_pseudo_duration(asap_journey, requested_dt, is_clockwise)
    pseudo_journey_duration = get_pseudo_duration(journey, requested_dt, is_clockwise)

    max_factor = 3  #TODO get it in instance
    base_factor = 60*60  #TODO get it in instance, for the moment 1h

    max_value = pseudo_asap_duration * max_factor + base_factor
    return pseudo_journey_duration > max_value

def _filter_not_coherent_journeys(journeys, instance, request, original_request):
    """
    Filter not coherent journeys

    The aim is to keep that as simple as possible
    """

    logger = logging.getLogger(__name__)
    # we filter journeys that arrive way later than other ones
    if request.get('clockwise', True):
        comp_value = lambda j: j.arrival_date_time
        comp_func = min
    else:
        comp_value = lambda j: j.departure_date_time
        comp_func = max

    asap_journey = comp_func((j for j in journeys if not _to_be_deleted(j)), key=comp_value)

    for j in journeys:
        if _to_be_deleted(j):
            continue
        if way_later(j, asap_journey, original_request):
            logger.debug("the journey {} is too long compared to {}, we delete it"
                         .format(j.internal_id, asap_journey.internal_id))

            _mark_as_dead(j, 'too_long', 'too_long_compared_to_{}'.format(asap_journey.internal_id))

def similar_journeys_generator(journey):
    for s in journey.sections:
        if s.type == response_pb2.PUBLIC_TRANSPORT:
            yield s.pt_display_informations.uris.vehicle_journey

def fallback_duration(journey):
    duration = 0
    for section in journey.sections:
        if section.type in (response_pb2.STREET_NETWORK, response_pb2.CROW_FLY):
            duration += section.duration

    return duration

def _debug_journey(journey):
    """
    For debug purpose print the journey
    """
    logger = logging.getLogger(__name__)

    __shorten = {
        'Walking': 'W',
        'Bike': 'B',
        'Car': 'C',
        'BSS_RENT': '/',
        'BSS_PUT_BACK': '\\',
        'PARK': '/',
        'LEAVE_PARKING': '\\',
        'TRANSFER': 'T',
        'WAITING': '.',
    }

    def shorten(name):
        return __shorten.get(name, name)

    sections = []
    for s in journey.sections:
        if s.type == response_pb2.PUBLIC_TRANSPORT:
            sections.append(u"{line} ({vj})".format(line=s.pt_display_informations.uris.line,
                                                   vj=s.pt_display_informations.uris.vehicle_journey))
        elif s.type == response_pb2.STREET_NETWORK:
            sections.append(shorten(response_pb2.StreetNetworkMode.Name(s.street_network.mode)))
        else:
            sections.append(shorten(response_pb2.SectionType.Name(s.type)))

    def _datetime_to_str(ts):
        #DEBUG, no tz handling
        dt = datetime.datetime.utcfromtimestamp(ts)
        return dt.strftime("%dT%H:%M:%S")

    logger.debug(u"journey {id}: {dep} -> {arr} - {duration} ({fallback} map) | ({sec})".format(
        id=journey.internal_id,
        dep=_datetime_to_str(journey.departure_date_time),
        arr=_datetime_to_str(journey.arrival_date_time),
        duration=datetime.timedelta(seconds=journey.duration),
        fallback=datetime.timedelta(seconds=fallback_duration(journey)),
        sec=" - ".join(sections)))
<commit_msg>remove useless blank line<commit_after># Copyright (c) 2001-2015, Canal TP and/or its affiliates. All rights reserved.
#
# This file is part of Navitia,
#     the software to build cool stuff with public transport.
#
# Hope you'll enjoy and contribute to this project,
#     powered by Canal TP (www.canaltp.fr).
# Help us simplify mobility and open public transport:
#     a non ending quest to the responsive locomotion way of traveling!
#
# LICENCE: This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Stay tuned using
# twitter @navitia
# IRC #navitia on freenode
# https://groups.google.com/d/forum/navitia
# www.navitia.io
from itertools import izip
import logging
import itertools
import datetime
from jormungandr.scenarios.utils import compare, get_pseudo_duration
from navitiacommon import response_pb2

#we can't use reverse(enumerate(list)) without creating a temporary list, so we define our own reverse enumerate
reverse_enumerate = lambda l: izip(xrange(len(l)-1, -1, -1), reversed(l))

def _delete_journeys(responses, request):

    if request.get('debug', False):
        return

    nb_deleted = 0
    for r in responses:
        for idx, j in reverse_enumerate(r.journeys):
            if not _to_be_deleted(j):
                continue

            del r.journeys[idx]
            nb_deleted += 1

    if nb_deleted:
        logging.getLogger(__name__).info('filtering {} journeys'.format(nb_deleted))

def filter_journeys(response_list, instance, request, original_request):
    """
    Filter by side effect the list of pb responses's journeys

    first draft, we only remove the journeys with the same vjs
    """

    # for clarity purpose we build a temporary list
    journeys = [j for r in response_list for j in r.journeys]

    #DEBUG
    for j in journeys:
        _debug_journey(j)

    _filter_similar_journeys(journeys, request)

    _filter_not_coherent_journeys(journeys, instance, request, original_request)

    _delete_journeys(response_list, request)

    return response_list

def _get_worst_similar_vjs(j1, j2, request):
    """
    Decide which is the worst journey between 2 similar journeys.

    The choice is made on:
     - asap
     - less fallback
     - duration
    """
    if request.get('clockwise', True):
        if j1.arrival_date_time != j2.arrival_date_time:
            return j1 if j1.arrival_date_time < j2.arrival_date_time else j2
    else:
        if j1.departure_date_time != j2.departure_date_time:
            return j1 if j1.departure_date_time > j2.departure_date_time else j2

    fallback1 = fallback_duration(j1)
    fallback2 = fallback_duration(j2)

    if fallback1 != fallback2:
        return j1 if fallback1 < fallback2 else j2

    return j1 if j1.duration < j2.duration else j2

def _to_be_deleted(journey):
    return 'to_delete' in journey.tags

def _mark_as_dead(journey, *reasons):
    journey.tags.append('to_delete')
    for reason in reasons:
        journey.tags.append('deleted_because_' + reason)

def _filter_similar_journeys(journeys, request):
    """
    for the moment very simple filter.

    we filter the journeys with the same vjs

    in case of similar journeys we let _get_worst_similar_vjs decide which one to delete
    """
    logger = logging.getLogger(__name__)
    for j1, j2 in itertools.combinations(journeys, 2):
        if _to_be_deleted(j1) or _to_be_deleted(j2):
            continue
        if compare(j1, j2, similar_journeys_generator):
            #chose the best
            worst = _get_worst_similar_vjs(j1, j2, request)
            logger.debug("the journeys {}, {} are similar, we delete {}".format(j1.internal_id,
                                                                                j2.internal_id,
                                                                                worst.internal_id))

            _mark_as_dead(worst, 'duplicate_journey', 'similar_to_{other}'
                          .format(other=j1.internal_id if worst == j2 else j2.internal_id))

def way_later(journey, asap_journey, original_request):
    """
    to check if a journey is way later than the asap journey
    we check for each journey the difference between the requested datetime and the arrival datetime
    (the other way around for non clockwise)

    requested dt
    *
                   |=============>
                          asap

                                           |=============>
                                                 journey

    -------------------------------
             asap pseudo duration

    ------------------------------------------------------
                       journey pseudo duration

    """
    requested_dt = original_request['datetime']
    is_clockwise = original_request.get('clockwise', True)

    pseudo_asap_duration = get_pseudo_duration(asap_journey, requested_dt, is_clockwise)
    pseudo_journey_duration = get_pseudo_duration(journey, requested_dt, is_clockwise)

    max_factor = 3  #TODO get it in instance
    base_factor = 60*60  #TODO get it in instance, for the moment 1h

    max_value = pseudo_asap_duration * max_factor + base_factor
    return pseudo_journey_duration > max_value

def _filter_not_coherent_journeys(journeys, instance, request, original_request):
    """
    Filter not coherent journeys

    The aim is to keep that as simple as possible
    """

    logger = logging.getLogger(__name__)
    # we filter journeys that arrive way later than other ones
    if request.get('clockwise', True):
        comp_value = lambda j: j.arrival_date_time
        comp_func = min
    else:
        comp_value = lambda j: j.departure_date_time
        comp_func = max

    asap_journey = comp_func((j for j in journeys if not _to_be_deleted(j)), key=comp_value)

    for j in journeys:
        if _to_be_deleted(j):
            continue
        if way_later(j, asap_journey, original_request):
            logger.debug("the journey {} is too long compared to {}, we delete it"
                         .format(j.internal_id, asap_journey.internal_id))

            _mark_as_dead(j, 'too_long', 'too_long_compared_to_{}'.format(asap_journey.internal_id))

def similar_journeys_generator(journey):
    for s in journey.sections:
        if s.type == response_pb2.PUBLIC_TRANSPORT:
            yield s.pt_display_informations.uris.vehicle_journey

def fallback_duration(journey):
    duration = 0
    for section in journey.sections:
        if section.type in (response_pb2.STREET_NETWORK, response_pb2.CROW_FLY):
            duration += section.duration

    return duration

def _debug_journey(journey):
    """
    For debug purpose print the journey
    """
    logger = logging.getLogger(__name__)

    __shorten = {
        'Walking': 'W',
        'Bike': 'B',
        'Car': 'C',
        'BSS_RENT': '/',
        'BSS_PUT_BACK': '\\',
        'PARK': '/',
        'LEAVE_PARKING': '\\',
        'TRANSFER': 'T',
        'WAITING': '.',
    }

    def shorten(name):
        return __shorten.get(name, name)

    sections = []
    for s in journey.sections:
        if s.type == response_pb2.PUBLIC_TRANSPORT:
            sections.append(u"{line} ({vj})".format(line=s.pt_display_informations.uris.line,
                                                   vj=s.pt_display_informations.uris.vehicle_journey))
        elif s.type == response_pb2.STREET_NETWORK:
            sections.append(shorten(response_pb2.StreetNetworkMode.Name(s.street_network.mode)))
        else:
            sections.append(shorten(response_pb2.SectionType.Name(s.type)))

    def _datetime_to_str(ts):
        #DEBUG, no tz handling
        dt = datetime.datetime.utcfromtimestamp(ts)
        return dt.strftime("%dT%H:%M:%S")

    logger.debug(u"journey {id}: {dep} -> {arr} - {duration} ({fallback} map) | ({sec})".format(
        id=journey.internal_id,
        dep=_datetime_to_str(journey.departure_date_time),
        arr=_datetime_to_str(journey.arrival_date_time),
        duration=datetime.timedelta(seconds=journey.duration),
        fallback=datetime.timedelta(seconds=fallback_duration(journey)),
        sec=" - ".join(sections)))
<|endoftext|>


<commit_before># strategy.py
# Strategy pattern -- function-based implementation

"""
    >>> joe = Customer('John Doe', 0)  # <1>
    >>> ann = Customer('Ann Smith', 1100)
    >>> cart = [LineItem('banana', 4, .5),
    ...         LineItem('apple', 10, 1.5),
    ...         LineItem('watermellon', 5, 5.0)]
    >>> Order(joe, cart, fidelity_promo)  # <2>
    <Order total: 42.00 due: 42.00>
    >>> Order(ann, cart, fidelity_promo)
    <Order total: 42.00 due: 39.90>
    >>> banana_cart = [LineItem('banana', 30, .5),
    ...                LineItem('apple', 10, 1.5)]
    >>> Order(joe, banana_cart, bulk_item_promo)  # <3>
    <Order total: 30.00 due: 28.50>
    >>> long_order = [LineItem(str(item_code), 1, 1.0)
    ...               for item_code in range(10)]
    >>> Order(joe, long_order, large_order_promo)
    <Order total: 10.00 due: 9.30>
    >>> Order(joe, cart, large_order_promo)
    <Order total: 42.00 due: 42.00>
"""

from abc import ABC, abstractmethod  # DELETE THIS IMPORT. Yay!
from collections import namedtuple

Customer = namedtuple('Customer', 'name fidelity')

class LineItem:

    def __init__(self, product, quantity, price):
        self.product = product
        self.quantity = quantity
        self.price = price

    def total(self):
        return self.price * self.quantity

class Order:

    def __init__(self, customer, cart, promotion=None):
        self.customer = customer
        self.cart = list(cart)
        self.promotion = promotion

    def total(self):
        if not hasattr(self, '__total'):
            self.__total = sum(item.total() for item in self.cart)
        return self.__total

    def due(self):
        if self.promotion is None:
            discount = 0
        else:
            discount = self.promotion(self)  # invoke the promotion function
        return self.total() - discount

    def __repr__(self):
        fmt = '<Order total: {:.2f} due: {:.2f}>'
        return fmt.format(self.total(), self.due())

class Promotion(ABC):  # DELETE THIS CLASS. HOORAY!

    @abstractmethod
    def discount(self, order):
        """Return discount as a positive dollar amount"""

class FidelityPromo(Promotion):  # REFACTOR INTO A FUNCTION
    """5% discount for customers with 1000 or more fidelity points"""

    def discount(self, order):
        return order.total() * .05 if order.customer.fidelity >= 1000 else 0

class BulkItemPromo(Promotion):  # REFACTOR INTO ANOTHER FUNCTION
    """10% discount for each LineItem with 20 or more units"""

    def discount(self, order):
        discount = 0
        for item in order.cart:
            if item.quantity >= 20:
                discount += item.total() * .1
        return discount

class LargeOrderPromo(Promotion):  # REFACTOR INTO A THIRD FUNCTION
    """7% discount for orders with 10 or more distinct items"""

    def discount(self, order):
        distinct_items = {item.product for item in order.cart}
        if len(distinct_items) >= 10:
            return order.total() * .07
        return 0
<commit_msg>strategy.py<commit_after># strategy.py
# Strategy pattern -- function-based implementation

"""
This is a simple implementation of the classic Strategy

    >>> joe = Customer('John Doe', 0)  # <1>
    >>> ann = Customer('Ann Smith', 1100)
    >>> cart = [LineItem('banana', 4, .5),
    ...         LineItem('apple', 10, 1.5),
    ...         LineItem('watermellon', 5, 5.0)]
    >>> Order(joe, cart, fidelity_promo)  # <2>
    <Order total: 42.00 due: 42.00>
    >>> Order(ann, cart, fidelity_promo)
    <Order total: 42.00 due: 39.90>
    >>> banana_cart = [LineItem('banana', 30, .5),
    ...                LineItem('apple', 10, 1.5)]
    >>> Order(joe, banana_cart, bulk_item_promo)  # <3>
    <Order total: 30.00 due: 28.50>
    >>> long_order = [LineItem(str(item_code), 1, 1.0)
    ...               for item_code in range(10)]
    >>> Order(joe, long_order, large_order_promo)
    <Order total: 10.00 due: 9.30>
    >>> Order(joe, cart, large_order_promo)
    <Order total: 42.00 due: 42.00>
"""

from abc import ABC, abstractmethod  # DELETE THIS IMPORT. Yay!
from collections import namedtuple

Customer = namedtuple('Customer', 'name fidelity')

class LineItem:

    def __init__(self, product, quantity, price):
        self.product = product
        self.quantity = quantity
        self.price = price

    def total(self):
        return self.price * self.quantity

class Order:

    def __init__(self, customer, cart, promotion=None):
        self.customer = customer
        self.cart = list(cart)
        self.promotion = promotion

    def total(self):
        if not hasattr(self, '__total'):
            self.__total = sum(item.total() for item in self.cart)
        return self.__total

    def due(self):
        if self.promotion is None:
            discount = 0
        else:
            discount = self.promotion(self)  # invoke the promotion function
        return self.total() - discount

    def __repr__(self):
        fmt = '<Order total: {:.2f} due: {:.2f}>'
        return fmt.format(self.total(), self.due())

class Promotion(ABC):  # DELETE THIS CLASS. HOORAY!

    @abstractmethod
    def discount(self, order):
        """Return discount as a positive dollar amount"""

class FidelityPromo(Promotion):  # REFACTOR INTO A FUNCTION
    """5% discount for customers with 1000 or more fidelity points"""

    def discount(self, order):
        return order.total() * .05 if order.customer.fidelity >= 1000 else 0

class BulkItemPromo(Promotion):  # REFACTOR INTO ANOTHER FUNCTION
    """10% discount for each LineItem with 20 or more units"""

    def discount(self, order):
        discount = 0
        for item in order.cart:
            if item.quantity >= 20:
                discount += item.total() * .1
        return discount

class LargeOrderPromo(Promotion):  # REFACTOR INTO A THIRD FUNCTION
    """7% discount for orders with 10 or more distinct items"""

    def discount(self, order):
        distinct_items = {item.product for item in order.cart}
        if len(distinct_items) >= 10:
            return order.total() * .07
        return 0
<|endoftext|>


<commit_before>import tensorflow as tf
import numpy as np

def orthogonal_initializer(scale = 1.1):
    def _initializer(shape, dtype=tf.float32):
        '''
        from keras https://github.com/fchollet/keras/blob/master/keras/initializations.py
        '''
        flat_shape = (shape[0], np.prod(shape[1:]))
        a = np.random.normal(0.0, 1.0, flat_shape)
        u, _, v = np.linalg.svd(a, full_matrices=False)
        # pick the one with the correct shape
        q = u if u.shape == flat_shape else v
        q = q.reshape(shape)
        return tf.constant(scale * q[:shape[0], :shape[1]], dtype=tf.float32)
    return _initializer

class AlternatingAttention(object):
    """Iterative Alternating Attention Network"""
    def __init__(self, batch_size, vocab_size, document_size, query_size, encoding_size, embedding_size,
                    num_glimpses = 8,
                    grad_norm_clip = 5.,
                    l2_reg_coef=1e-4,
                    session=tf.Session(),
                    name='AlternatingAttention'):
        """
        Creates an iterative alternating attention network as described in https://arxiv.org/abs/1606.02245
        """
        self._batch_size = batch_size
        self._vocab_size = vocab_size
        self._doc_len = document_size
        self._query_len = query_size
        self._encode_size = encoding_size
        self._infer_size = 4 * encoding_size
        self._embedding_size = embedding_size
        self._num_glimpses = num_glimpses
        self._sess = session
        self._name = name

        self._build_placeholders()
        self._build_variables()

        # Regularization
        tf.contrib.layers.apply_regularization(tf.contrib.layers.l2_regularizer(l2_reg_coef), [self._embeddings])

        # Answer probability
        doc_attentions = self._inference(self._docs, self._queries)
        P_a = tf.pack([tf.reduce_sum(tf.gather(tf.squeeze(doc_attentions)[i, :], tf.where(tf.equal(a, self._docs[i, :])))) for i, a in enumerate(tf.unpack(self._answers))])
        loss_op = -tf.reduce_mean(tf.log(tf.clip_by_value(P_a,1e-10,1.0)))
        self._loss_op = loss_op
        tf.scalar_summary('loss', loss_op)

        self._doc_attentions = doc_attentions

        # Optimizer and gradients
        with tf.variable_scope("optimizer"):
            self._opt = tf.train.AdamOptimizer(learning_rate=self._learning_rate)
            grads_and_vars = self._opt.compute_gradients(loss_op)
            capped_grads_and_vars = [(tf.clip_by_norm(g, grad_norm_clip), v) for g,v in grads_and_vars]
            self._global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), dtype=tf.int32, trainable=False)
            self._train_op = self._opt.apply_gradients(grads_and_vars, global_step=self._global_step)

        self._summary_op = tf.merge_all_summaries()

        self._sess.run(tf.initialize_all_variables())

    def _build_placeholders(self):
        """
        Adds tensorflow placeholders for inputs to the model: documents, queries, answers.
        keep_prob and learning_rate are hyperparameters that we might like to adjust while training.
        """
        batch_size, doc_len, query_len = self._batch_size, self._doc_len, self._query_len
        self._docs = tf.placeholder(tf.int32, [self._batch_size, self._doc_len])
        self._queries = tf.placeholder(tf.int32, [self._batch_size, self._query_len])
        self._answers = tf.placeholder(tf.int32, [self._batch_size])
        self._keep_prob = tf.placeholder(tf.float32)
        self._learning_rate = tf.placeholder(tf.float32)

    def _build_variables(self):
        with tf.variable_scope(self._name, initializer=tf.random_normal_initializer(mean=0.0, stddev=0.22, dtype=tf.float32)):
            self._embeddings = tf.get_variable("embeddings", [self._vocab_size, self._embedding_size], dtype=tf.float32)
            self._A_q = tf.get_variable("A_q", [self._batch_size, 2*self._encode_size, self._infer_size], dtype=tf.float32)
            self._a_q = tf.get_variable("a_q", [self._batch_size, 2*self._encode_size, 1], dtype=tf.float32)

            self._A_d = tf.get_variable("A_d", [self._batch_size, 2*self._encode_size, self._infer_size + 2*self._encode_size], dtype=tf.float32)
            self._a_d = tf.get_variable("a_d", [self._batch_size, 2*self._encode_size, 1], dtype=tf.float32)

            self._g_q = tf.get_variable("g_q", [self._batch_size, self._infer_size + 6 * self._encode_size, 2 * self._encode_size])
            self._g_d = tf.get_variable("g_d", [self._batch_size, self._infer_size + 6 * self._encode_size, 2 * self._encode_size])

    def _embed(self, sequence, seq_len):
        """
        performs embedding lookups for every word in the sequence
        """
        with tf.variable_scope('embed'):
            embedded = tf.nn.embedding_lookup(self._embeddings, sequence)
            return embedded

    def _bidirectional_encode(self, sequence, seq_len, size):
        """
        Encodes sequence with two GRUs, one forward, one backward, and returns the concatenation
        """
        with tf.variable_scope('encode'):
            with tf.variable_scope("_FW") as fw_scope:
                fw_encode = tf.nn.rnn_cell.GRUCell(size)
                fw_state = tf.get_variable("gru_state", [self._batch_size, fw_encode.state_size], initializer=orthogonal_initializer())
                output_fw, output_state_fw = tf.nn.dynamic_rnn(
                    cell=fw_encode, inputs=sequence, initial_state=fw_state, scope=fw_scope, swap_memory=True)
            with tf.variable_scope("_BW") as bw_scope:
                bw_encode = tf.nn.rnn_cell.GRUCell(size)
                bw_state = tf.get_variable("gru_state", [self._batch_size, bw_encode.state_size], initializer=orthogonal_initializer())
                inputs_reverse = tf.reverse_sequence(
                    input=sequence, seq_lengths=seq_len,
                    seq_dim=1, batch_dim=0)
                tmp, output_state_bw = tf.nn.dynamic_rnn(
                    cell=bw_encode, inputs=inputs_reverse, sequence_length=seq_len, initial_state = bw_state, scope=bw_scope, swap_memory=True)
                output_bw = tf.reverse_sequence(
                  input=tmp, seq_lengths=seq_len,
                  seq_dim=1, batch_dim=0)

            outputs = (output_fw, output_bw)
            encoded = tf.concat(2, outputs)
            return encoded

    def _glimpse(self, weights, bias, encodings, inputs):
        """
        Computes glimpse over an encoding. Attention weights are computed based on the bilinear product of
        the encodings, weight matrix, and inputs.

        Returns attention weights and computed glimpse
        """
        tf.nn.dropout(weights, self._keep_prob)
        tf.nn.dropout(inputs, self._keep_prob)
        attention = tf.batch_matmul(weights, inputs) + bias
        attention = tf.squeeze(tf.batch_matmul(encodings, attention))
        attention = tf.expand_dims(tf.nn.softmax(attention), -1)
        return attention, tf.reduce_sum(attention * encodings, 1)

    def _inference(self, docs, queries):
        """
        Computes document attentions given a document batch and query batch.
        """
        with tf.variable_scope(self._name):
            # Encode Document / Query
            with tf.variable_scope('docs'):
                encoded_docs = tf.nn.dropout(self._embed(docs, self._doc_len), self._keep_prob)
                encoded_docs = self._bidirectional_encode(encoded_docs, self._batch_size * [self._doc_len], self._encode_size)
            with tf.variable_scope('queries'):
                encoded_queries = tf.nn.dropout(self._embed(queries, self._query_len), self._keep_prob)
                encoded_queries = self._bidirectional_encode(encoded_queries, self._batch_size * [self._query_len], self._encode_size)

            with tf.variable_scope('attend') as scope:
                infer_gru = tf.nn.rnn_cell.GRUCell(self._infer_size)
                infer_state = infer_gru.zero_state(self._batch_size, tf.float32)
                for iter_step in range(self._num_glimpses):
                    if iter_step > 0:
                        scope.reuse_variables()

                    # Glimpse query and document
                    _, q_glimpse = self._glimpse(self._A_q, self._a_q, encoded_queries, tf.expand_dims(infer_state, -1))
                    d_attention, d_glimpse = self._glimpse(self._A_d, self._a_d, encoded_docs, tf.concat(1, [tf.expand_dims(infer_state, -1), tf.expand_dims(q_glimpse, -1)]))

                    # Search Gates
                    gate_concat = tf.concat(1, [tf.squeeze(infer_state), q_glimpse, d_glimpse, q_glimpse * d_glimpse])
                    gate_concat = tf.expand_dims(gate_concat, 1)

                    r_d = tf.sigmoid(tf.squeeze(tf.batch_matmul(gate_concat, self._g_d)))
                    tf.nn.dropout(r_d, self._keep_prob)
                    r_q = tf.sigmoid(tf.squeeze(tf.batch_matmul(gate_concat, self._g_q)))
                    tf.nn.dropout(r_q, self._keep_prob)

                    _, infer_state = infer_gru(tf.concat(1, [r_q * q_glimpse, r_d * d_glimpse]), tf.squeeze(infer_state))
            return d_attention

    def batch_fit(self, docs, queries, answers, learning_rate=1e-3, run_options=None, run_metadata=None):
        """
        Perform a batch training iteration
        """
        feed_dict = { self._docs: docs, self._queries: queries, self._answers: answers, self._keep_prob: 0.8, self._learning_rate: learning_rate}
        loss, summary, _, step = self._sess.run([self._loss_op, self._summary_op, self._train_op, self._global_step], feed_dict=feed_dict, options=run_options, run_metadata=run_metadata)
        return loss, summary, step

    def batch_predict(self, docs, queries, answers):
        """
        Perform batch prediction. Computes accuracy of batch predictions.
        """
        feed_dict = { self._docs: docs, self._queries: queries, self._answers: answers, self._keep_prob: 1. }
        loss, summary, attentions = self._sess.run([self._loss_op, self._summary_op, self._doc_attentions], feed_dict=feed_dict)
        correct = 0.
        for i in range(docs.shape[0]):
            words = np.asarray(list(set(docs[i,:])))
            accuracies = np.zeros(words.shape)
            for j,w in enumerate(words):
                accuracy = np.sum(attentions[i, docs[i,:] == w])
                accuracies[j] = accuracy
            if words[np.argmax(accuracies)] == answers[i]:
                correct += 1
        return loss, summary, correct / docs.shape[0]
<commit_msg>Compute document/query lengths of each batch for zeroing out encoders and attentions<commit_after>import tensorflow as tf
import numpy as np

def orthogonal_initializer(scale = 1.1):
    def _initializer(shape, dtype=tf.float32):
        '''
        from keras https://github.com/fchollet/keras/blob/master/keras/initializations.py
        '''
        flat_shape = (shape[0], np.prod(shape[1:]))
        a = np.random.normal(0.0, 1.0, flat_shape)
        u, _, v = np.linalg.svd(a, full_matrices=False)
        # pick the one with the correct shape
        q = u if u.shape == flat_shape else v
        q = q.reshape(shape)
        return tf.constant(scale * q[:shape[0], :shape[1]], dtype=tf.float32)
    return _initializer

class AlternatingAttention(object):
    """Iterative Alternating Attention Network"""
    def __init__(self, batch_size, vocab_size, document_size, query_size, encoding_size, embedding_size,
                    num_glimpses = 8,
                    grad_norm_clip = 5.,
                    l2_reg_coef=1e-4,
                    session=tf.Session(),
                    name='AlternatingAttention'):
        """
        Creates an iterative alternating attention network as described in https://arxiv.org/abs/1606.02245
        """
        self._batch_size = batch_size
        self._vocab_size = vocab_size
        self._doc_len = document_size
        self._query_len = query_size
        self._encode_size = encoding_size
        self._infer_size = 4 * encoding_size
        self._embedding_size = embedding_size
        self._num_glimpses = num_glimpses
        self._sess = session
        self._name = name

        self._build_placeholders()
        self._build_variables()

        # Regularization
        tf.contrib.layers.apply_regularization(tf.contrib.layers.l2_regularizer(l2_reg_coef), [self._embeddings])

        # Answer probability
        doc_attentions = self._inference(self._docs, self._queries)
        P_a = tf.pack([tf.reduce_sum(tf.gather(tf.squeeze(doc_attentions)[i, :], tf.where(tf.equal(a, self._docs[i, :])))) for i, a in enumerate(tf.unpack(self._answers))])
        loss_op = -tf.reduce_mean(tf.log(tf.clip_by_value(P_a,1e-10,1.0)))
        self._loss_op = loss_op
        tf.scalar_summary('loss', loss_op)

        self._doc_attentions = doc_attentions

        # Optimizer and gradients
        with tf.variable_scope("optimizer"):
            self._opt = tf.train.AdamOptimizer(learning_rate=self._learning_rate)
            grads_and_vars = self._opt.compute_gradients(loss_op)
            capped_grads_and_vars = [(tf.clip_by_norm(g, grad_norm_clip), v) for g,v in grads_and_vars]
            self._global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), dtype=tf.int32, trainable=False)
            self._train_op = self._opt.apply_gradients(grads_and_vars, global_step=self._global_step)

        self._summary_op = tf.merge_all_summaries()

        self._sess.run(tf.initialize_all_variables())

    def _build_placeholders(self):
        """
        Adds tensorflow placeholders for inputs to the model: documents, queries, answers.
        keep_prob and learning_rate are hyperparameters that we might like to adjust while training.
        """
        self._docs = tf.placeholder(tf.int32, [self._batch_size, self._doc_len])
        self._queries = tf.placeholder(tf.int32, [self._batch_size, self._query_len])
        self._answers = tf.placeholder(tf.int32, [self._batch_size])

        self._keep_prob = tf.placeholder(tf.float32)
        self._learning_rate = tf.placeholder(tf.float32)

    def _build_variables(self):
        with tf.variable_scope(self._name, initializer=tf.random_normal_initializer(mean=0.0, stddev=0.22, dtype=tf.float32)):
            self._embeddings = tf.get_variable("embeddings", [self._vocab_size, self._embedding_size], dtype=tf.float32)
            self._A_q = tf.get_variable("A_q", [self._batch_size, 2*self._encode_size, self._infer_size], dtype=tf.float32)
            self._a_q = tf.get_variable("a_q", [self._batch_size, 2*self._encode_size, 1], dtype=tf.float32)

            self._A_d = tf.get_variable("A_d", [self._batch_size, 2*self._encode_size, self._infer_size + 2*self._encode_size], dtype=tf.float32)
            self._a_d = tf.get_variable("a_d", [self._batch_size, 2*self._encode_size, 1], dtype=tf.float32)

            self._g_q = tf.get_variable("g_q", [self._batch_size, self._infer_size + 6 * self._encode_size, 2 * self._encode_size])
            self._g_d = tf.get_variable("g_d", [self._batch_size, self._infer_size + 6 * self._encode_size, 2 * self._encode_size])

    def _embed(self, sequence):
        """
        performs embedding lookups for every word in the sequence
        """
        with tf.variable_scope('embed'):
            embedded = tf.nn.embedding_lookup(self._embeddings, sequence)
            return embedded

    def _bidirectional_encode(self, sequence, seq_lens, size):
        """
        Encodes sequence with two GRUs, one forward, one backward, and returns the concatenation
        """
        with tf.variable_scope('encode'):
            with tf.variable_scope("_FW") as fw_scope:
                fw_encode = tf.nn.rnn_cell.GRUCell(size)
                fw_state = tf.get_variable("gru_state", [self._batch_size, fw_encode.state_size], initializer=orthogonal_initializer())
                output_fw, output_state_fw = tf.nn.dynamic_rnn(
                    cell=fw_encode, inputs=sequence,
                    initial_state=fw_state, scope=fw_scope,
                    sequence_lengths=seq_lens, swap_memory=True)

            with tf.variable_scope("_BW") as bw_scope:
                bw_encode = tf.nn.rnn_cell.GRUCell(size)
                bw_state = tf.get_variable("gru_state", [self._batch_size, bw_encode.state_size], initializer=orthogonal_initializer())
                inputs_reverse = tf.reverse_sequence(
                    input=sequence, seq_lengths=seq_lens,
                    seq_dim=1, batch_dim=0)
                tmp, output_state_bw = tf.nn.dynamic_rnn(
                    cell=bw_encode, inputs=inputs_reverse,
                    sequence_lengths=seq_len, initial_state = bw_state,
                    scope=bw_scope, swap_memory=True)
                output_bw = tf.reverse_sequence(
                  input=tmp, seq_lengths=seq_lens,
                  seq_dim=1, batch_dim=0)

            outputs = (output_fw, output_bw)
            encoded = tf.concat(2, outputs)
            return encoded

    def _glimpse(self, weights, bias, encodings, inputs):
        """
        Computes glimpse over an encoding. Attention weights are computed based on the bilinear product of
        the encodings, weight matrix, and inputs.

        Returns attention weights and computed glimpse
        """
        tf.nn.dropout(weights, self._keep_prob)
        tf.nn.dropout(inputs, self._keep_prob)
        attention = tf.batch_matmul(weights, inputs) + bias
        attention = tf.squeeze(tf.batch_matmul(encodings, attention))
        attention = tf.expand_dims(tf.nn.softmax(attention), -1)
        return attention, tf.reduce_sum(attention * encodings, 1)

    def _inference(self, docs, queries):
        """
        Computes document attentions given a document batch and query batch.
        """
        with tf.variable_scope(self._name):
            # Compute document lengths / query lengths for batch
            doc_lens = tf.argmin(docs, 1)
            doc_seq_mask = tf.sequence_mask(doc_lens, self._doc_len)

            query_lens = tf.argmin(docs, 1)
            query_seq_mask = tf.sequence_mask(query_lens, self._query_len)

            # Encode Document / Query
            with tf.variable_scope('docs'):
                encoded_docs = tf.nn.dropout(self._embed(docs), self._keep_prob)
                encoded_docs = self._bidirectional_encode(encoded_docs, doc_lens, self._encode_size)
            with tf.variable_scope('queries'):
                encoded_queries = tf.nn.dropout(self._embed(queries), self._keep_prob)
                encoded_queries = self._bidirectional_encode(encoded_queries, query_lens, self._encode_size)

            with tf.variable_scope('attend') as scope:
                infer_gru = tf.nn.rnn_cell.GRUCell(self._infer_size)
                infer_state = infer_gru.zero_state(self._batch_size, tf.float32)
                for iter_step in range(self._num_glimpses):
                    if iter_step > 0:
                        scope.reuse_variables()

                    # Glimpse query and document
                    _, q_glimpse = self._glimpse(self._A_q, self._a_q, encoded_queries, tf.expand_dims(infer_state, -1))
                    d_attention, d_glimpse = self._glimpse(self._A_d, self._a_d, encoded_docs, tf.concat(1, [tf.expand_dims(infer_state, -1), tf.expand_dims(q_glimpse, -1)]))

                    # Search Gates
                    gate_concat = tf.concat(1, [tf.squeeze(infer_state), q_glimpse, d_glimpse, q_glimpse * d_glimpse])
                    gate_concat = tf.expand_dims(gate_concat, 1)

                    r_d = tf.sigmoid(tf.squeeze(tf.batch_matmul(gate_concat, self._g_d)))
                    tf.nn.dropout(r_d, self._keep_prob)
                    r_q = tf.sigmoid(tf.squeeze(tf.batch_matmul(gate_concat, self._g_q)))
                    tf.nn.dropout(r_q, self._keep_prob)

                    _, infer_state = infer_gru(tf.concat(1, [r_q * q_glimpse, r_d * d_glimpse]), tf.squeeze(infer_state))
            return d_attention

    def batch_fit(self, docs, queries, answers, learning_rate=1e-3, run_options=None, run_metadata=None):
        """
        Perform a batch training iteration
        """
        feed_dict = {
            self._docs: docs,
            self._queries: queries,
            self._answers: answers,
            self._keep_prob: 0.8,
            self._learning_rate: learning_rate
            }
        loss, summary, _, step = self._sess.run([self._loss_op, self._summary_op, self._train_op, self._global_step], feed_dict=feed_dict, options=run_options, run_metadata=run_metadata)
        return loss, summary, step

    def batch_predict(self, docs, queries, answers):
        """
        Perform batch prediction. Computes accuracy of batch predictions.
        """
        feed_dict = {
            self._docs: docs,
            self._queries: queries,
            self._answers: answers,
            self._keep_prob: 1.
            }
        loss, summary, attentions = self._sess.run([self._loss_op, self._summary_op, self._doc_attentions], feed_dict=feed_dict)
        correct = 0.
        for i in range(docs.shape[0]):
            words = np.asarray(list(set(docs[i,:])))
            accuracies = np.zeros(words.shape)
            for j,w in enumerate(words):
                accuracy = np.sum(attentions[i, docs[i,:] == w])
                accuracies[j] = accuracy
            if words[np.argmax(accuracies)] == answers[i]:
                correct += 1
        return loss, summary, correct / docs.shape[0]
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Run the bot, as well as the web server."""

import sys
import argparse
import os
import pycon_bot.driver
import pycon_bot.mongo
from pycon_bot.web.app import app as webapp
from twisted.web.server import Site
from twisted.web.wsgi import WSGIResource
from twisted.python import log
from twisted.internet import reactor

def run_bot(irc_server, irc_port, irc_channel, bot_name, http_port, logfile):
    log.startLogging(logfile)
    if irc_server is not None:
        bot = pycon_bot.driver.PyConBotFactory([irc_channel], bot_name)
        reactor.connectTCP(irc_server, irc_port, bot)
    if http_port is not None:
        web = WSGIResource(reactor, reactor.getThreadPool(), webapp)
        reactor.listenTCP(http_port, Site(web))
    reactor.run()

if __name__ == '__main__':
    p = argparse.ArgumentParser()
    p.add_argument('--irc-server', default='irc.freenode.net')
    p.add_argument('--irc-port', type=int, default=6667)
    p.add_argument('--irc-channel', default=os.environ.get('PYCONBOT_CHANNEL', '#pycon-pc'))
    p.add_argument('--irc-nickname', default=os.environ.get('PYCONBOT_NICK', 'pycon_bot'))
    p.add_argument('--http-port', type=int, default=8000)
    p.add_argument('--no-irc', dest='run_irc', action='store_false', default=True)
    p.add_argument('--no-web', dest='run_web', action='store_false', default=True)
    p.add_argument('--dsn', default=None)
    args = p.parse_args()

    # attempt to connect to the mongo db using either
    # the argument passed or the built-in environment variable
    if not pycon_bot.mongo.connect(args.dsn):
        sys.stderr.write("Need to pass --dsn or set env[MONGO_DSN].")
        sys.exit(1)

    # Run ze bot!
    run_bot(
        irc_server=args.irc_server if args.run_irc else None,
        irc_port=args.irc_port,
        irc_channel=args.irc_channel,
        bot_name=args.irc_nickname,
        http_port=args.http_port if args.run_web else None,
        logfile=sys.stderr,
    )
<commit_msg>Trivial alteration so I can bounce the bot on Heroku<commit_after>#!/usr/bin/env python
"""Run the bot, as well as the web server."""
import sys
import argparse
import os
import pycon_bot.driver
import pycon_bot.mongo
from pycon_bot.web.app import app as webapp
from twisted.web.server import Site
from twisted.web.wsgi import WSGIResource
from twisted.python import log
from twisted.internet import reactor

def run_bot(irc_server, irc_port, irc_channel, bot_name, http_port, logfile):
    log.startLogging(logfile)
    if irc_server is not None:
        bot = pycon_bot.driver.PyConBotFactory([irc_channel], bot_name)
        reactor.connectTCP(irc_server, irc_port, bot)
    if http_port is not None:
        web = WSGIResource(reactor, reactor.getThreadPool(), webapp)
        reactor.listenTCP(http_port, Site(web))
    reactor.run()

if __name__ == '__main__':
    p = argparse.ArgumentParser()
    p.add_argument('--irc-server', default='irc.freenode.net')
    p.add_argument('--irc-port', type=int, default=6667)
    p.add_argument('--irc-channel', default=os.environ.get('PYCONBOT_CHANNEL', '#pycon-pc'))
    p.add_argument('--irc-nickname', default=os.environ.get('PYCONBOT_NICK', 'pycon_bot'))
    p.add_argument('--http-port', type=int, default=8000)
    p.add_argument('--no-irc', dest='run_irc', action='store_false', default=True)
    p.add_argument('--no-web', dest='run_web', action='store_false', default=True)
    p.add_argument('--dsn', default=None)
    args = p.parse_args()

    # attempt to connect to the mongo db using either
    # the argument passed or the built-in environment variable
    if not pycon_bot.mongo.connect(args.dsn):
        sys.stderr.write("Need to pass --dsn or set env[MONGO_DSN].")
        sys.exit(1)

    # Run ze bot!
    run_bot(
        irc_server=args.irc_server if args.run_irc else None,
        irc_port=args.irc_port,
        irc_channel=args.irc_channel,
        bot_name=args.irc_nickname,
        http_port=args.http_port if args.run_web else None,
        logfile=sys.stderr,
    )
<|endoftext|>