<commit_before># -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil: coding: utf-8 -*-
# ex:ts=4:sw=4:sts=4:et:fenc=utf-8
from __future__ import absolute_import
import sys
import logging
import re
import unicodedata
import platform
from operator import itemgetter

try:
    import HTMLParser
except ImportError:
    # pylint: disable-msg=import-error
    import html.parser as HTMLParser
try:
    from requests import Session
except ImportError:
    print("You need to install python-requests to use this script")
    sys.exit(3)

from svtplay_dl import error

is_py2 = (sys.version_info[0] == 2)
is_py3 = (sys.version_info[0] == 3)
is_py2_old = (sys.version_info < (2, 7))

# Used for UA spoofing in get_http_data()
FIREFOX_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.3'

# TODO: should be set as the default option in the argument parsing?
DEFAULT_PROTOCOL_PRIO = ["dash", "hls", "hds", "http", "rtmp"]

log = logging.getLogger('svtplay_dl')
progress_stream = sys.stderr

class HTTP(Session):
    def __init__(self, options, *args, **kwargs):
        Session.__init__(self, *args, **kwargs)
        self.verify = options.ssl_verify
        if options.http_headers:
            self.headers.update(self.split_header(options.http_headers))
        self.headers.update({"User-Agent": FIREFOX_UA})

    def check_redirect(self, url):
        return self.get(url, stream=True).url

    def request(self, method, url, *args, **kwargs):
        headers = kwargs.pop("headers", None)
        if headers:
            for i in headers.keys():
                self.headers[i] = headers[i]

        log.debug("HTTP getting %r", url)
        res = Session.request(self, method, url, verify=self.verify, *args, **kwargs)
        return res

    def split_header(self, headers):
        return dict(x.split('=') for x in headers.split(';'))

def sort_quality(data):
    data = sorted(data, key=lambda x: (x.bitrate, x.name()), reverse=True)
    datas = []
    for i in data:
        datas.append([i.bitrate, i.name()])
    return datas

def list_quality(videos):
    data = sort_quality(videos)
    log.info("Quality\tMethod")
    for i in data:
        log.info("%s\t%s" % (i[0], i[1].upper()))

def prio_streams(streams, protocol_prio):
    # Map score's to the reverse of the list's index values
    proto_score = dict(zip(protocol_prio, range(len(protocol_prio), 0, -1)))
    log.debug("Protocol priority scores (higher is better): %s",
        str(proto_score))

    # Build a tuple (bitrate, proto_score, stream), and use it
    # for sorting.
    prioritized = [(s.bitrate, proto_score[s.name()], s) for
                   s in streams if s.name() in proto_score]
    return [x[2] for
            x in sorted(prioritized, key=itemgetter(0,1), reverse=True)]

def select_quality(options, streams):
    # Extract protocol prio, in the form of "hls,hds,http,rtmp",
    # we want it as a list
    proto_prio = DEFAULT_PROTOCOL_PRIO
    if options.stream_prio:
        proto_prio = options.stream_prio.split(',')

    # Filter away any unwanted protocols, and prioritize
    # based on --stream-priority.
    streams = prio_streams(streams, proto_prio)

    if len(streams) == 0:
        raise error.NoRequestedProtocols(
            requested=proto_prio,
            found=list(set([s.name() for s in streams]))
        )

    available = sorted(int(x.bitrate) for x in streams)
    try:
        optq = int(options.quality)
    except ValueError:
        log.error("Requested quality need to be a number")
        sys.exit(4)
    if optq:
        try:
            optf = int(options.flexibleq)
        except ValueError:
            log.error("Flexible-quality need to be a number")
            sys.exit(4)
        if not optf:
            wanted = [optq]
        else:
            wanted = range(optq-optf, optq+optf+1)
    else:
        wanted = [available[-1]]

    selected = None
    for q in available:
        if q in wanted:
            selected = q
            break
    if not selected and selected != 0:
        data = sort_quality(streams)
        quality = ", ".join("%s (%s)" % (str(x), str(y)) for x, y in data)
        log.error("Can't find that quality. Try one of: %s (or try --flexible-quality)", quality)

        sys.exit(4)

    for s in streams:
        if s.bitrate == selected:
            return s

def ensure_unicode(s):
    """
    Ensure string is a unicode string. If it isn't it assumed it is
    utf-8 and decodes it to a unicode string.
    """
    if (is_py2 and isinstance(s, str)) or (is_py3 and isinstance(s, bytes)):
        s = s.decode('utf-8', 'replace')
    return s

def decode_html_entities(s):
    """
    Replaces html entities with the character they represent.

        >>> print(decode_html_entities("<3 &"))
        <3 &
    """
    parser = HTMLParser.HTMLParser()
    def unesc(m):
        return parser.unescape(m.group())
    return re.sub(r'(&[^;]+;)', unesc, ensure_unicode(s))

def filenamify(title):
    """
    Convert a string to something suitable as a file name. E.g.

     Matlagning del 1 av 10 - Räksmörgås | SVT Play
       ->  matlagning.del.1.av.10.-.raksmorgas.svt.play
    """
    # ensure it is unicode
    title = ensure_unicode(title)

    # NFD decomposes chars into base char and diacritical mark, which
    # means that we will get base char when we strip out non-ascii.
    title = unicodedata.normalize('NFD', title)

    # Convert to lowercase
    # Drop any non ascii letters/digits
    # Drop any leading/trailing whitespace that may have appeared
    title = re.sub(r'[^a-z0-9 .-]', '', title.lower().strip())

    # Replace whitespace with dot
    title = re.sub(r'\s+', '.', title)
    title = re.sub(r'\.-\.', '-', title)

    return title

def download_thumbnail(options, url):
    data = Session.get(url).content

    filename = re.search(r"(.*)\.[a-z0-9]{2,3}$", options.output)
    tbn = "%s.tbn" % filename.group(1)
    log.info("Thumbnail: %s", tbn)

    fd = open(tbn, "wb")
    fd.write(data)
    fd.close()

def which(program):
    import os

    if platform.system() == "Windows":
        program = "{0}.exe".format(program)

    def is_exe(fpath):
        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

    fpath, fname = os.path.split(program)
    if fpath:
        if is_exe(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file
        if os.path.isfile(program):
            exe_file = os.path.join(os.getcwd(), program)
            if is_exe(exe_file):
                return exe_file
    return None
<commit_msg>select_quality: Replace sys.exits with UIExceptions<commit_after># -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil: coding: utf-8 -*-
# ex:ts=4:sw=4:sts=4:et:fenc=utf-8
from __future__ import absolute_import
import sys
import logging
import re
import unicodedata
import platform
from operator import itemgetter

try:
    import HTMLParser
except ImportError:
    # pylint: disable-msg=import-error
    import html.parser as HTMLParser
try:
    from requests import Session
except ImportError:
    print("You need to install python-requests to use this script")
    sys.exit(3)

from svtplay_dl import error

is_py2 = (sys.version_info[0] == 2)
is_py3 = (sys.version_info[0] == 3)
is_py2_old = (sys.version_info < (2, 7))

# Used for UA spoofing in get_http_data()
FIREFOX_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.3'

# TODO: should be set as the default option in the argument parsing?
DEFAULT_PROTOCOL_PRIO = ["dash", "hls", "hds", "http", "rtmp"]

log = logging.getLogger('svtplay_dl')
progress_stream = sys.stderr

class HTTP(Session):
    def __init__(self, options, *args, **kwargs):
        Session.__init__(self, *args, **kwargs)
        self.verify = options.ssl_verify
        if options.http_headers:
            self.headers.update(self.split_header(options.http_headers))
        self.headers.update({"User-Agent": FIREFOX_UA})

    def check_redirect(self, url):
        return self.get(url, stream=True).url

    def request(self, method, url, *args, **kwargs):
        headers = kwargs.pop("headers", None)
        if headers:
            for i in headers.keys():
                self.headers[i] = headers[i]

        log.debug("HTTP getting %r", url)
        res = Session.request(self, method, url, verify=self.verify, *args, **kwargs)
        return res

    def split_header(self, headers):
        return dict(x.split('=') for x in headers.split(';'))

def sort_quality(data):
    data = sorted(data, key=lambda x: (x.bitrate, x.name()), reverse=True)
    datas = []
    for i in data:
        datas.append([i.bitrate, i.name()])
    return datas

def list_quality(videos):
    data = sort_quality(videos)
    log.info("Quality\tMethod")
    for i in data:
        log.info("%s\t%s" % (i[0], i[1].upper()))

def prio_streams(streams, protocol_prio):
    # Map score's to the reverse of the list's index values
    proto_score = dict(zip(protocol_prio, range(len(protocol_prio), 0, -1)))
    log.debug("Protocol priority scores (higher is better): %s",
        str(proto_score))

    # Build a tuple (bitrate, proto_score, stream), and use it
    # for sorting.
    prioritized = [(s.bitrate, proto_score[s.name()], s) for
                   s in streams if s.name() in proto_score]
    return [x[2] for
            x in sorted(prioritized, key=itemgetter(0,1), reverse=True)]

def select_quality(options, streams):
    # Extract protocol prio, in the form of "hls,hds,http,rtmp",
    # we want it as a list
    proto_prio = DEFAULT_PROTOCOL_PRIO
    if options.stream_prio:
        proto_prio = options.stream_prio.split(',')

    # Filter away any unwanted protocols, and prioritize
    # based on --stream-priority.
    streams = prio_streams(streams, proto_prio)

    if len(streams) == 0:
        raise error.NoRequestedProtocols(
            requested=proto_prio,
            found=list(set([s.name() for s in streams]))
        )

    available = sorted(int(x.bitrate) for x in streams)
    try:
        optq = int(options.quality)
    except ValueError:
        raise error.UIException("Requested quality needs to be a number")
    if optq:
        try:
            optf = int(options.flexibleq)
        except ValueError:
            raise error.UIException("Flexible-quality needs to be a number")
        if not optf:
            wanted = [optq]
        else:
            wanted = range(optq-optf, optq+optf+1)
    else:
        wanted = [available[-1]]

    selected = None
    for q in available:
        if q in wanted:
            selected = q
            break
    if not selected and selected != 0:
        data = sort_quality(streams)
        quality = ", ".join("%s (%s)" % (str(x), str(y)) for x, y in data)
        raise error.UIException("Can't find that quality. Try one of: %s (or "
                                "try --flexible-quality)" % quality)

    for s in streams:
        if s.bitrate == selected:
            return s

def ensure_unicode(s):
    """
    Ensure string is a unicode string. If it isn't it assumed it is
    utf-8 and decodes it to a unicode string.
    """
    if (is_py2 and isinstance(s, str)) or (is_py3 and isinstance(s, bytes)):
        s = s.decode('utf-8', 'replace')
    return s

def decode_html_entities(s):
    """
    Replaces html entities with the character they represent.

        >>> print(decode_html_entities("<3 &"))
        <3 &
    """
    parser = HTMLParser.HTMLParser()
    def unesc(m):
        return parser.unescape(m.group())
    return re.sub(r'(&[^;]+;)', unesc, ensure_unicode(s))

def filenamify(title):
    """
    Convert a string to something suitable as a file name. E.g.

     Matlagning del 1 av 10 - Räksmörgås | SVT Play
       ->  matlagning.del.1.av.10.-.raksmorgas.svt.play
    """
    # ensure it is unicode
    title = ensure_unicode(title)

    # NFD decomposes chars into base char and diacritical mark, which
    # means that we will get base char when we strip out non-ascii.
    title = unicodedata.normalize('NFD', title)

    # Convert to lowercase
    # Drop any non ascii letters/digits
    # Drop any leading/trailing whitespace that may have appeared
    title = re.sub(r'[^a-z0-9 .-]', '', title.lower().strip())

    # Replace whitespace with dot
    title = re.sub(r'\s+', '.', title)
    title = re.sub(r'\.-\.', '-', title)

    return title

def download_thumbnail(options, url):
    data = Session.get(url).content

    filename = re.search(r"(.*)\.[a-z0-9]{2,3}$", options.output)
    tbn = "%s.tbn" % filename.group(1)
    log.info("Thumbnail: %s", tbn)

    fd = open(tbn, "wb")
    fd.write(data)
    fd.close()

def which(program):
    import os

    if platform.system() == "Windows":
        program = "{0}.exe".format(program)

    def is_exe(fpath):
        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

    fpath, fname = os.path.split(program)
    if fpath:
        if is_exe(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            path = path.strip('"')
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file
        if os.path.isfile(program):
            exe_file = os.path.join(os.getcwd(), program)
            if is_exe(exe_file):
                return exe_file
    return None
<|endoftext|>


<commit_before># Author: Braedy Kuzma
from django.conf.urls import url
from . import views

urlpatterns = [
    url(r'^posts/$', views.PostsView.as_view(), name='posts'),
    url(r'^posts/(?P<pid>[0-9a-fA-F\-]+)/$', views.PostView.as_view(),
        name='post'),
    url(r'^posts/(?P<pid>[0-9a-fA-F\-]+)/comments/$',
        views.CommentView.as_view(), name='comments'),
    url(r'^author/posts/$', views.PostView.as_view(), name='authorpost'),
    url(r'^author/(?P<aid>[0-9a-fA-F\-]+)/$', views.AuthorView.as_view(),
        name='author'),
    url(r'^author/(?P<aid>[0-9a-fA-F\-]+)/friends/$',
        views.AuthorFriendsView.as_view(), name='friends')
]
<commit_msg>Fix spelling.<commit_after># Author: Braedy Kuzma
from django.conf.urls import url
from . import views

urlpatterns = [
    url(r'^posts/$', views.PostsView.as_view(), name='posts'),
    url(r'^posts/(?P<pid>[0-9a-fA-F\-]+)/$', views.PostView.as_view(),
        name='post'),
    url(r'^posts/(?P<pid>[0-9a-fA-F\-]+)/comments/$',
        views.CommentView.as_view(), name='comments'),
    url(r'^author/posts/$', views.PostsView.as_view(), name='authorpost'),
    url(r'^author/(?P<aid>[0-9a-fA-F\-]+)/$', views.AuthorView.as_view(),
        name='author'),
    url(r'^author/(?P<aid>[0-9a-fA-F\-]+)/friends/$',
        views.AuthorFriendsView.as_view(), name='friends')
]
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import pytoml as toml
import mastodon
import os
import pickle
import re
import time
import trigger
import logger
import sendmail
import traceback
import sys

class RetootBot(object):
    def __init__(self, config, filter, logger):
        self.config = config
        self.filter = filter
        self.client_id = self.register()
        self.m = self.login()

        # load state
        try:
            with open('seen_toots.pickle', 'rb') as f:
                self.seen_toots = pickle.load(f)
        except IOError:
            self.seen_toots = set()

        # intialize shutdown contact
        try:
            self.no_shutdown_contact = False
            self.contact = self.config['mail']['contact']
        except KeyError:
            self.no_shutdown_contact = True

        self.logger = logger

    def register(self):
        client_id = os.path.join(
            'appkeys',
            self.config['mapp']['name'] +
            '@' + self.config['muser']['server']
        )

        if not os.path.isfile(client_id):
            mastodon.Mastodon.create_app(
                self.config['mapp']['name'],
                api_base_url=self.config['muser']['server'],
                to_file=client_id
            )
        return self.client_id

    def login(self):
        m = mastodon.Mastodon(
            client_id=self.client_id,
            api_base_url=self.config['muser']['server']
        )
        m.log_in(
            self.config['muser']['email'],
            self.config['muser']['password']
        )
        return m

    def retoot(self, toots=()):
        # toot external provided messages
        for toot in toots:
            self.m.toot(toot)

        # boost mentions
        retoots = []
        for notification in self.m.notifications():
            if (notification['type'] == 'mention'
                    and notification['status']['id'] not in self.seen_toots):
                self.seen_toots.add(notification['status']['id'])
                text_content = re.sub(r'<[^>]*>', '',
                                      notification['status']['content'])
                if not self.filter.is_ok(text_content):
                    continue
                self.logger.log('Boosting toot from %s: %s' % (
                    # notification['status']['id'],
                    notification['status']['account']['acct'],
                    notification['status']['content']))
                self.m.status_reblog(notification['status']['id'])
                retoots.append('%s: %s' % (
                    notification['status']['account']['acct'],
                    re.sub(r'@\S*', '', text_content)))
        # If the Mastodon instance returns interesting Errors, add them here:

        # save state
        with os.fdopen(os.open('seen_toots.pickle.part', os.O_WRONLY | os.O_EXCL | os.O_CREAT), 'wb') as f:
            pickle.dump(self.seen_toots, f)
        os.rename('seen_toots.pickle.part', 'seen_toots.pickle')

        # return mentions for mirroring
        return retoots

    def shutdown(self, tb):
        """ If something breaks, it shuts down the bot and messages the owner.
        """
        logmessage = "Shit went wrong, closing down.\n" + tb + "\n\n"
        if self.no_shutdown_contact:
            self.logger.log(logmessage)
            return
        logmessage = logmessage + " Sending message to " + self.contact
        self.logger.log(logmessage)
        # feature yet only in RetweetBot:
        # self.save_last_mention()
        try:
            mailer = sendmail.Mailer(self.config)
            mailer.send(tb, self.contact, "Ticketfrei Crash Report")
        except:
            # traceback.print_exc()
            self.logger.log(traceback.extract_tb(sys.exc_info()[2]))
            print()

if __name__ == '__main__':
    # read config in TOML format (https://github.com/toml-lang/toml#toml)
    with open('config.toml') as configfile:
        config = toml.load(configfile)

    trigger = trigger.Trigger(config)
    logger = logger.Logger()

    bot = RetootBot(config, trigger, logger)

    try:
        while True:
            bot.retoot()
            time.sleep(1)
    except KeyboardInterrupt:
        print("Good bye! Remember to restart the bot.")
    except:
        tb = traceback.extract_tb(sys.exc_info()[2])
        bot.logger.log(tb)
        print()
        bot.shutdown(tb)
<commit_msg>small fix<commit_after>#!/usr/bin/env python3

import pytoml as toml
import mastodon
import os
import pickle
import re
import time
import trigger
import logger
import sendmail
import traceback
import sys

class RetootBot(object):
    def __init__(self, config, filter, logger):
        self.config = config
        self.filter = filter
        self.client_id = self.register()
        self.m = self.login()

        # load state
        try:
            with open('seen_toots.pickle', 'rb') as f:
                self.seen_toots = pickle.load(f)
        except IOError:
            self.seen_toots = set()

        # intialize shutdown contact
        try:
            self.no_shutdown_contact = False
            self.contact = self.config['mail']['contact']
        except KeyError:
            self.no_shutdown_contact = True

        self.logger = logger

    def register(self):
        client_id = os.path.join(
            'appkeys',
            self.config['mapp']['name'] +
            '@' + self.config['muser']['server']
        )

        if not os.path.isfile(client_id):
            mastodon.Mastodon.create_app(
                self.config['mapp']['name'],
                api_base_url=self.config['muser']['server'],
                to_file=client_id
            )
        return client_id

    def login(self):
        m = mastodon.Mastodon(
            client_id=self.client_id,
            api_base_url=self.config['muser']['server']
        )
        m.log_in(
            self.config['muser']['email'],
            self.config['muser']['password']
        )
        return m

    def retoot(self, toots=()):
        # toot external provided messages
        for toot in toots:
            self.m.toot(toot)

        # boost mentions
        retoots = []
        for notification in self.m.notifications():
            if (notification['type'] == 'mention'
                    and notification['status']['id'] not in self.seen_toots):
                self.seen_toots.add(notification['status']['id'])
                text_content = re.sub(r'<[^>]*>', '',
                                      notification['status']['content'])
                if not self.filter.is_ok(text_content):
                    continue
                self.logger.log('Boosting toot from %s: %s' % (
                    # notification['status']['id'],
                    notification['status']['account']['acct'],
                    notification['status']['content']))
                self.m.status_reblog(notification['status']['id'])
                retoots.append('%s: %s' % (
                    notification['status']['account']['acct'],
                    re.sub(r'@\S*', '', text_content)))
        # If the Mastodon instance returns interesting Errors, add them here:

        # save state
        with os.fdopen(os.open('seen_toots.pickle.part', os.O_WRONLY | os.O_EXCL | os.O_CREAT), 'wb') as f:
            pickle.dump(self.seen_toots, f)
        os.rename('seen_toots.pickle.part', 'seen_toots.pickle')

        # return mentions for mirroring
        return retoots

    def shutdown(self, tb):
        """ If something breaks, it shuts down the bot and messages the owner.
        """
        logmessage = "Shit went wrong, closing down.\n" + tb + "\n\n"
        if self.no_shutdown_contact:
            self.logger.log(logmessage)
            return
        logmessage = logmessage + " Sending message to " + self.contact
        self.logger.log(logmessage)
        # feature yet only in RetweetBot:
        # self.save_last_mention()
        try:
            mailer = sendmail.Mailer(self.config)
            mailer.send(tb, self.contact, "Ticketfrei Crash Report")
        except:
            # traceback.print_exc()
            self.logger.log(traceback.extract_tb(sys.exc_info()[2]))
            print()

if __name__ == '__main__':
    # read config in TOML format (https://github.com/toml-lang/toml#toml)
    with open('config.toml') as configfile:
        config = toml.load(configfile)

    trigger = trigger.Trigger(config)
    logger = logger.Logger()

    bot = RetootBot(config, trigger, logger)

    try:
        while True:
            bot.retoot()
            time.sleep(1)
    except KeyboardInterrupt:
        print("Good bye! Remember to restart the bot.")
    except:
        tb = traceback.extract_tb(sys.exc_info()[2])
        bot.logger.log(tb)
        print()
        bot.shutdown(tb)
<|endoftext|>


<commit_before>"""Basic template tags library
"""
import collections
import itertools
from template import Template
from tag import tag_manager, parse_token

def block(token, block, template, loader):
    """Block tag
    """
    # Create inner template for blocks
    tmpl = Template(loader)
    tmpl.commands = block

    # Add template block into list
    if not hasattr(template, 'blocks'):
        template.blocks = {}
    is_new = token not in template.blocks
    template.blocks[token] = tmpl

    # Add function that executes inner template into commands
    if is_new:
        return template.blocks[token]
    else:
        index = template.commands.index(template.parent.blocks[token])
        template.commands[index] = tmpl
        return lambda context: ''

tag_manager.register(
        name='block',
        tag=block,
        is_block_tag=True,
        context_required=False,
        template_required=True,
        loader_required=True,
        is_lazy_tag=False
)

def extend(token, template, loader):
    """Tag used to create tamplates hierarhy
    """
    tokens = parse_token(token)[0]
    template.parent = loader.get_template(tokens[0])
    if not hasattr(template, 'blocks'):
        template.blocks = template.parent.blocks.copy()
    else:
        template.blocks.update(template.parent.blocks)
    template.commands.extend(template.parent.commands)
    return None

tag_manager.register(
        name='extend',
        tag=extend,
        template_required=True,
        loader_required=True,
        is_lazy_tag=False
)

def spaceless(token, block, context):
    """This tag removes unused spaces

    Template

        {% spaceless %}
            Some
                    text
        {% endspaceless %}

    will be rendered to:

        Some text

    """
    results = [command(context).split('\n') for command in block]
    return "".join([line.lstrip() for line in itertools.chain(*results)])

tag_manager.register(
        name='spaceless',
        tag=spaceless,
        is_block_tag=True,
        context_required=True,
        template_required=False,
        loader_required=False,
        is_lazy_tag=True
)

def with_tag(token, block, context):
    """With tag can be used to set the shorter name for variable used few times

    Example:

        {% with request.context.user.name as user_name %}

{{ user_name }}'s profile

            Hello, {{ user_name }}
            <form action="update_profile" method="post">
                Your name:
                <input type="text" name="user_name" value="{{ user_name }}" />
                <input type="submit" value="Update profile" />
            </form>
        {% endwith %}
    """
    data_field, _, var_name = token.split(' ')
    if '.' in data_field:
        fields = data_field.split('.')
        value = reduce(Template.get_field, [context[fields[0]]] + fields[1:])
    else:
        value = context[data_field]
    old_value = context[var_name] if var_name in context else None
    context[var_name] = value
    result = "".join([command(context) for command in block])
    if old_value:
        command[var_name] = old_value
    return result

tag_manager.register(
        name='with',
        tag=with_tag,
        is_block_tag=True,
        context_required=True,
        template_required=False,
        loader_required=False,
        is_lazy_tag=True
)

def if_tag(token, block, context):
    """If tag can brings some logic into template.

    Example:

        {% if user.is_authenticated %}Hello, {{ user.name }}!{% endif %}

    TODO:

        - add else
        - add conditions
    """
    if '.' in token:
        fields = token.split('.')
        value = reduce(Template.get_field, [context[fields[0]]] + fields[1:])
    else:
        value = context[token]
    if value:
        return "".join([command(context) for command in block])
    return ''

tag_manager.register(
        name='if',
        tag=if_tag,
        is_block_tag=True,
        context_required=True,
        template_required=False,
        loader_required=False,
        is_lazy_tag=True
)

def for_tag(token, block, context):
    """For tag

    Example:

        {% for a in items %}{{ a }}{% endfor %}

    returns for items = [1, 2, 3]:

        123

    Also forloop variable will be added into scope. It contains few flags can
    be used to render customized templates

        {% for a in items %}
            {% spaceless %}
                {{ forloop.counter0 }}.
                {{ forloop.counter }} from {{ forloop.total }}
            {% endspaceless %}
        {% endfor %}

    returns

        0. 1 from 3
        1. 2 from 3
        2. 3 from 3

    """
    var_name, _, data_field = token.split(' ')
    if '.' in data_field:
        fields = data_field.split('.')
        values = reduce(Template.get_field, [context[fields[0]]] + fields[1:])
    else:
        values = context[data_field]
    # Check values
    if not isinstance(values, collections.Iterable):
        raise ValueError('%s: "%s" is not iterable' % (data_field, values))
    length = len(values)
    forloop = {'first': True, 'last': length == 1, 'total': length,
               'counter0': 0, 'counter': 1}
    old_value = context[values] if var_name in context else None
    old_forloop = context['forloop'] if 'forloop' in context else None
    results = []
    for v in values:
        context[var_name] = v
        forloop['counter'] += 1
        forloop['counter0'] += 1
        forloop['first'] = False
        forloop['last'] = forloop['counter'] < length
        results.append("".join([command(context) for command in block]))
    context[var_name] = old_value
    context['forloop'] = old_forloop
    return "".join(results)

tag_manager.register(
        name='for',
        tag=for_tag,
        is_block_tag=True,
        context_required=True,
        template_required=False,
        loader_required=False,
        is_lazy_tag=True
)
<commit_msg>Templatetags refactoring: add function for variable resolving.<commit_after>"""Basic template tags library
"""
import collections
import itertools
from template import Template
from tag import tag_manager, parse_token

def get_value(var_name, context):
    if var_name in context:
        return context[var_name]
    raise Exception('"%s" not in context' % var_name)

def resolve(var_name, context):
    if '.' in var_name:
        fields = var_name.split('.')
        return reduce(Template.get_field,
                      [get_value(fields[0], context)] + fields[1:])
    return get_value(var_name, context)

def block(token, block, template, loader):
    """Block tag
    """
    # Create inner template for blocks
    tmpl = Template(loader)
    tmpl.commands = block

    # Add template block into list
    if not hasattr(template, 'blocks'):
        template.blocks = {}
    is_new = token not in template.blocks
    template.blocks[token] = tmpl

    # Add function that executes inner template into commands
    if is_new:
        return template.blocks[token]
    else:
        index = template.commands.index(template.parent.blocks[token])
        template.commands[index] = tmpl
        return lambda context: ''

tag_manager.register(
        name='block',
        tag=block,
        is_block_tag=True,
        context_required=False,
        template_required=True,
        loader_required=True,
        is_lazy_tag=False
)

def extend(token, template, loader):
    """Tag used to create tamplates hierarhy
    """
    tokens = parse_token(token)[0]
    template.parent = loader.get_template(tokens[0])
    if not hasattr(template, 'blocks'):
        template.blocks = template.parent.blocks.copy()
    else:
        template.blocks.update(template.parent.blocks)
    template.commands.extend(template.parent.commands)
    return None

tag_manager.register(
        name='extend',
        tag=extend,
        template_required=True,
        loader_required=True,
        is_lazy_tag=False
)

def spaceless(token, block, context):
    """This tag removes unused spaces

    Template

        {% spaceless %}
            Some
                    text
        {% endspaceless %}

    will be rendered to:

        Some text

    """
    results = [command(context).split('\n') for command in block]
    return "".join([line.lstrip() for line in itertools.chain(*results)])

tag_manager.register(
        name='spaceless',
        tag=spaceless,
        is_block_tag=True,
        context_required=True,
        template_required=False,
        loader_required=False,
        is_lazy_tag=True
)

def with_tag(token, block, context):
    """With tag can be used to set the shorter name for variable used few times

    Example:

        {% with request.context.user.name as user_name %}

{{ user_name }}'s profile

            Hello, {{ user_name }}
            <form action="update_profile" method="post">
                Your name:
                <input type="text" name="user_name" value="{{ user_name }}" />
                <input type="submit" value="Update profile" />
            </form>
        {% endwith %}
    """
    data_field, _, var_name = token.split(' ')
    value = resolve(data_field, context)
    old_value = context[var_name] if var_name in context else None
    context[var_name] = value
    result = "".join([command(context) for command in block])
    if old_value:
        command[var_name] = old_value
    return result

tag_manager.register(
        name='with',
        tag=with_tag,
        is_block_tag=True,
        context_required=True,
        template_required=False,
        loader_required=False,
        is_lazy_tag=True
)

def if_tag(token, block, context):
    """If tag can brings some logic into template.

    Example:

        {% if user.is_authenticated %}Hello, {{ user.name }}!{% endif %}

    TODO:

        - add else
        - add conditions
    """
    if resolve(token, context):
        return "".join([command(context) for command in block])
    return ''

tag_manager.register(
        name='if',
        tag=if_tag,
        is_block_tag=True,
        context_required=True,
        template_required=False,
        loader_required=False,
        is_lazy_tag=True
)

def for_tag(token, block, context):
    """For tag

    Example:

        {% for a in items %}{{ a }}{% endfor %}

    returns for items = [1, 2, 3]:

        123

    Also forloop variable will be added into scope. It contains few flags can
    be used to render customized templates

        {% for a in items %}
            {% spaceless %}
                {{ forloop.counter0 }}.
                {{ forloop.counter }} from {{ forloop.total }}
            {% endspaceless %}
        {% endfor %}

    returns

        0. 1 from 3
        1. 2 from 3
        2. 3 from 3

    """
    var_name, _, data_field = token.split(' ')
    values = resolve(data_field, context)
    # Check values
    if not isinstance(values, collections.Iterable):
        raise ValueError('%s: "%s" is not iterable' % (data_field, values))
    length = len(values)
    forloop = {'first': True, 'last': length == 1, 'total': length,
               'counter0': 0, 'counter': 1}
    old_value = context[var_name] if var_name in context else None
    old_forloop = context['forloop'] if 'forloop' in context else None
    results = []
    for v in values:
        context[var_name] = v
        forloop['counter'] += 1
        forloop['counter0'] += 1
        forloop['first'] = False
        forloop['last'] = forloop['counter'] < length
        results.append("".join([command(context) for command in block]))
    context[var_name] = old_value
    context['forloop'] = old_forloop
    return "".join(results)

tag_manager.register(
        name='for',
        tag=for_tag,
        is_block_tag=True,
        context_required=True,
        template_required=False,
        loader_required=False,
        is_lazy_tag=True
)
<|endoftext|>


<commit_before>"""Views for the branding app. """
from __future__ import absolute_import

import logging

import six
from django.conf import settings
from django.contrib.staticfiles.storage import staticfiles_storage
from django.core.cache import cache
from django.db import transaction
from django.http import Http404, HttpResponse
from django.shortcuts import redirect
from django.urls import reverse
from django.utils import translation
from django.utils.translation.trans_real import get_supported_language_variant
from django.views.decorators.cache import cache_control
from django.views.decorators.csrf import ensure_csrf_cookie

import branding.api as branding_api
import lms.djangoapps.courseware.views.views as courseware_views
import student.views
from edxmako.shortcuts import marketing_link, render_to_response
from openedx.core.djangoapps.lang_pref.api import released_languages
from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers
from util.cache import cache_if_anonymous
from util.json_request import JsonResponse

log = logging.getLogger(__name__)

@ensure_csrf_cookie
@transaction.non_atomic_requests
@cache_if_anonymous()
def index(request):
    """
    Redirects to main page -- info page if user authenticated, or marketing if not
    """
    if request.user.is_authenticated:
        # Only redirect to dashboard if user has
        # courses in his/her dashboard. Otherwise UX is a bit cryptic.
        # In this case, we want to have the user stay on a course catalog
        # page to make it easier to browse for courses (and register)
        if configuration_helpers.get_value(
                'ALWAYS_REDIRECT_HOMEPAGE_TO_DASHBOARD_FOR_AUTHENTICATED_USER',
                settings.FEATURES.get('ALWAYS_REDIRECT_HOMEPAGE_TO_DASHBOARD_FOR_AUTHENTICATED_USER', True)):
            return redirect(reverse('dashboard'))

    enable_mktg_site = configuration_helpers.get_value(
        'ENABLE_MKTG_SITE',
        settings.FEATURES.get('ENABLE_MKTG_SITE', False)
    )

    if enable_mktg_site:
        marketing_urls = configuration_helpers.get_value(
            'MKTG_URLS',
            settings.MKTG_URLS
        )
        return redirect(marketing_urls.get('ROOT'))

    domain = request.META.get('HTTP_HOST')

    # keep specialized logic for Edge until we can migrate over Edge to fully use
    # configuration.
    if domain and 'edge.edx.org' in domain:
        return redirect(reverse("signin_user"))

    #  we do not expect this case to be reached in cases where
    #  marketing and edge are enabled
    return student.views.index(request, user=request.user)

@ensure_csrf_cookie
@cache_if_anonymous()
def courses(request):
    """
    Render the "find courses" page. If the marketing site is enabled, redirect
    to that. Otherwise, if subdomain branding is on, this is the university
    profile page. Otherwise, it's the edX courseware.views.views.courses page
    """
    enable_mktg_site = configuration_helpers.get_value(
        'ENABLE_MKTG_SITE',
        settings.FEATURES.get('ENABLE_MKTG_SITE', False)
    )

    if enable_mktg_site:
        return redirect(marketing_link('COURSES'), permanent=True)

    if not settings.FEATURES.get('COURSES_ARE_BROWSABLE'):
        raise Http404

    #  we do not expect this case to be reached in cases where
    #  marketing is enabled or the courses are not browsable
    return courseware_views.courses(request)

def _footer_static_url(request, name):
    """Construct an absolute URL to a static asset. """
    return request.build_absolute_uri(staticfiles_storage.url(name))

def _footer_css_urls(request, package_name):
    """Construct absolute URLs to CSS assets in a package. """
    # We need this to work both in local development and in production.
    # Unfortunately, in local development we don't run the full asset pipeline,
    # so fully processed output files may not exist.
    # For this reason, we use the *css package* name(s), rather than the static file name
    # to identify the CSS file name(s) to include in the footer.
    # We then construct an absolute URI so that external sites (such as the marketing site)
    # can locate the assets.
    package = settings.PIPELINE['STYLESHEETS'].get(package_name, {})
    paths = [package['output_filename']] if not settings.DEBUG else package['source_filenames']
    return [
        _footer_static_url(request, path)
        for path in paths
    ]

def _render_footer_html(request, show_openedx_logo, include_dependencies, include_language_selector, language):
    """Render the footer as HTML.

    Arguments:
        show_openedx_logo (bool): If True, include the OpenEdX logo in the rendered HTML.
        include_dependencies (bool): If True, include JavaScript and CSS dependencies.
        include_language_selector (bool): If True, include a language selector with all supported languages.

    Returns: unicode

    """
    bidi = 'rtl' if translation.get_language_bidi() else 'ltr'
    css_name = settings.FOOTER_CSS['openedx'][bidi]

    context = {
        'hide_openedx_link': not show_openedx_logo,
        'footer_js_url': _footer_static_url(request, 'js/footer-edx.js'),
        'footer_css_urls': _footer_css_urls(request, css_name),
        'bidi': bidi,
        'include_dependencies': include_dependencies,
        'include_language_selector': include_language_selector,
        'language': language
    }

    return render_to_response("footer.html", context)

@cache_control(must_revalidate=True, max_age=settings.FOOTER_BROWSER_CACHE_MAX_AGE)
def footer(request):
    """Retrieve the branded footer.

    This end-point provides information about the site footer,
    allowing for consistent display of the footer across other sites
    (for example, on the marketing site and blog).

    It can be used in one of two ways:
    1) A client renders the footer from a JSON description.
    2) A browser loads an HTML representation of the footer
        and injects it into the DOM.  The HTML includes
        CSS and JavaScript links.

    In case (2), we assume that the following dependencies
    are included on the page:
    a) JQuery (same version as used in edx-platform)
    b) font-awesome (same version as used in edx-platform)
    c) Open Sans web fonts

    Example: Retrieving the footer as JSON

        GET /api/branding/v1/footer
        Accepts: application/json

        {
            "navigation_links": [
                {
                  "url": "http://example.com/about",
                  "name": "about",
                  "title": "About"
                },
                # ...
            ],
            "social_links": [
                {
                    "url": "http://example.com/social",
                    "name": "facebook",
                    "icon-class": "fa-facebook-square",
                    "title": "Facebook",
                    "action": "Sign up on Facebook!"
                },
                # ...
            ],
            "mobile_links": [
                {
                    "url": "http://example.com/android",
                    "name": "google",
                    "image": "http://example.com/google.png",
                    "title": "Google"
                },
                # ...
            ],
            "legal_links": [
                {
                    "url": "http://example.com/terms-of-service.html",
                    "name": "terms_of_service",
                    "title': "Terms of Service"
                },
                # ...
            ],
            "openedx_link": {
                "url": "http://open.edx.org",
                "title": "Powered by Open edX",
                "image": "http://example.com/openedx.png"
            },
            "logo_image": "http://example.com/static/images/logo.png",
            "copyright": "edX, Open edX and their respective logos are registered trademarks of edX Inc."
        }

    Example: Retrieving the footer as HTML

        GET /api/branding/v1/footer
        Accepts: text/html

    Example: Including the footer with the "Powered by Open edX" logo

        GET /api/branding/v1/footer?show-openedx-logo=1
        Accepts: text/html

    Example: Retrieving the footer in a particular language

        GET /api/branding/v1/footer?language=en
        Accepts: text/html

    Example: Retrieving the footer with a language selector

        GET /api/branding/v1/footer?include-language-selector=1
        Accepts: text/html

    Example: Retrieving the footer with all JS and CSS dependencies (for testing)

        GET /api/branding/v1/footer?include-dependencies=1
        Accepts: text/html

    """
    if not branding_api.is_enabled():
        raise Http404

    # Use the content type to decide what representation to serve
    accepts = request.META.get('HTTP_ACCEPT', '*/*')

    # Show the OpenEdX logo in the footer
    show_openedx_logo = bool(request.GET.get('show-openedx-logo', False))

    # Include JS and CSS dependencies
    # This is useful for testing the end-point directly.
    include_dependencies = bool(request.GET.get('include-dependencies', False))

    # Override the language if necessary
    language = request.GET.get('language', translation.get_language())
    try:
        language = get_supported_language_variant(language)
    except LookupError:
        language = settings.LANGUAGE_CODE

    # Include a language selector
    include_language_selector = request.GET.get('include-language-selector', '') == '1'

    # Render the footer information based on the extension
    if 'text/html' in accepts or '*/*' in accepts:
        cache_params = {
            'language': language,
            'show_openedx_logo': show_openedx_logo,
            'include_dependencies': include_dependencies
        }
        if include_language_selector:
            cache_params['language_selector_options'] = ','.join(sorted([lang.code for lang in released_languages()]))
        cache_key = u"branding.footer.{params}.html".format(params=six.moves.urllib.parse.urlencode(cache_params))

        content = cache.get(cache_key)
        if content is None:
            with translation.override(language):
                content = _render_footer_html(
                    request, show_openedx_logo, include_dependencies, include_language_selector, language
                )
                cache.set(cache_key, content, settings.FOOTER_CACHE_TIMEOUT)
        return HttpResponse(content, status=200, content_type="text/html; charset=utf-8")

    elif 'application/json' in accepts:
        cache_key = u"branding.footer.{params}.json".format(
            params=six.moves.urllib.parse.urlencode({
                'language': language,
                'is_secure': request.is_secure(),
            })
        )
        footer_dict = cache.get(cache_key)
        if footer_dict is None:
            with translation.override(language):
                footer_dict = branding_api.get_footer(is_secure=request.is_secure())
                cache.set(cache_key, footer_dict, settings.FOOTER_CACHE_TIMEOUT)
        return JsonResponse(footer_dict, 200, content_type="application/json; charset=utf-8")

    else:
        return HttpResponse(status=406)
<commit_msg>Use the view names for redirect.<commit_after>"""Views for the branding app. """
from __future__ import absolute_import

import logging

import six
from django.conf import settings
from django.contrib.staticfiles.storage import staticfiles_storage
from django.core.cache import cache
from django.db import transaction
from django.http import Http404, HttpResponse
from django.shortcuts import redirect
from django.urls import reverse
from django.utils import translation
from django.utils.translation.trans_real import get_supported_language_variant
from django.views.decorators.cache import cache_control
from django.views.decorators.csrf import ensure_csrf_cookie

import branding.api as branding_api
import lms.djangoapps.courseware.views.views as courseware_views
import student.views
from edxmako.shortcuts import marketing_link, render_to_response
from openedx.core.djangoapps.lang_pref.api import released_languages
from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers
from util.cache import cache_if_anonymous
from util.json_request import JsonResponse

log = logging.getLogger(__name__)

@ensure_csrf_cookie
@transaction.non_atomic_requests
@cache_if_anonymous()
def index(request):
    """
    Redirects to main page -- info page if user authenticated, or marketing if not
    """
    if request.user.is_authenticated:
        # Only redirect to dashboard if user has
        # courses in his/her dashboard. Otherwise UX is a bit cryptic.
        # In this case, we want to have the user stay on a course catalog
        # page to make it easier to browse for courses (and register)
        if configuration_helpers.get_value(
                'ALWAYS_REDIRECT_HOMEPAGE_TO_DASHBOARD_FOR_AUTHENTICATED_USER',
                settings.FEATURES.get('ALWAYS_REDIRECT_HOMEPAGE_TO_DASHBOARD_FOR_AUTHENTICATED_USER', True)):
            return redirect('dashboard')

    enable_mktg_site = configuration_helpers.get_value(
        'ENABLE_MKTG_SITE',
        settings.FEATURES.get('ENABLE_MKTG_SITE', False)
    )

    if enable_mktg_site:
        marketing_urls = configuration_helpers.get_value(
            'MKTG_URLS',
            settings.MKTG_URLS
        )
        return redirect(marketing_urls.get('ROOT'))

    domain = request.META.get('HTTP_HOST')

    # keep specialized logic for Edge until we can migrate over Edge to fully use
    # configuration.
    if domain and 'edge.edx.org' in domain:
        return redirect("signin_user")

    #  we do not expect this case to be reached in cases where
    #  marketing and edge are enabled
    return student.views.index(request, user=request.user)

@ensure_csrf_cookie
@cache_if_anonymous()
def courses(request):
    """
    Render the "find courses" page. If the marketing site is enabled, redirect
    to that. Otherwise, if subdomain branding is on, this is the university
    profile page. Otherwise, it's the edX courseware.views.views.courses page
    """
    enable_mktg_site = configuration_helpers.get_value(
        'ENABLE_MKTG_SITE',
        settings.FEATURES.get('ENABLE_MKTG_SITE', False)
    )

    if enable_mktg_site:
        return redirect(marketing_link('COURSES'), permanent=True)

    if not settings.FEATURES.get('COURSES_ARE_BROWSABLE'):
        raise Http404

    #  we do not expect this case to be reached in cases where
    #  marketing is enabled or the courses are not browsable
    return courseware_views.courses(request)

def _footer_static_url(request, name):
    """Construct an absolute URL to a static asset. """
    return request.build_absolute_uri(staticfiles_storage.url(name))

def _footer_css_urls(request, package_name):
    """Construct absolute URLs to CSS assets in a package. """
    # We need this to work both in local development and in production.
    # Unfortunately, in local development we don't run the full asset pipeline,
    # so fully processed output files may not exist.
    # For this reason, we use the *css package* name(s), rather than the static file name
    # to identify the CSS file name(s) to include in the footer.
    # We then construct an absolute URI so that external sites (such as the marketing site)
    # can locate the assets.
    package = settings.PIPELINE['STYLESHEETS'].get(package_name, {})
    paths = [package['output_filename']] if not settings.DEBUG else package['source_filenames']
    return [
        _footer_static_url(request, path)
        for path in paths
    ]

def _render_footer_html(request, show_openedx_logo, include_dependencies, include_language_selector, language):
    """Render the footer as HTML.

    Arguments:
        show_openedx_logo (bool): If True, include the OpenEdX logo in the rendered HTML.
        include_dependencies (bool): If True, include JavaScript and CSS dependencies.
        include_language_selector (bool): If True, include a language selector with all supported languages.

    Returns: unicode

    """
    bidi = 'rtl' if translation.get_language_bidi() else 'ltr'
    css_name = settings.FOOTER_CSS['openedx'][bidi]

    context = {
        'hide_openedx_link': not show_openedx_logo,
        'footer_js_url': _footer_static_url(request, 'js/footer-edx.js'),
        'footer_css_urls': _footer_css_urls(request, css_name),
        'bidi': bidi,
        'include_dependencies': include_dependencies,
        'include_language_selector': include_language_selector,
        'language': language
    }

    return render_to_response("footer.html", context)

@cache_control(must_revalidate=True, max_age=settings.FOOTER_BROWSER_CACHE_MAX_AGE)
def footer(request):
    """Retrieve the branded footer.

    This end-point provides information about the site footer,
    allowing for consistent display of the footer across other sites
    (for example, on the marketing site and blog).

    It can be used in one of two ways:
    1) A client renders the footer from a JSON description.
    2) A browser loads an HTML representation of the footer
        and injects it into the DOM.  The HTML includes
        CSS and JavaScript links.

    In case (2), we assume that the following dependencies
    are included on the page:
    a) JQuery (same version as used in edx-platform)
    b) font-awesome (same version as used in edx-platform)
    c) Open Sans web fonts

    Example: Retrieving the footer as JSON

        GET /api/branding/v1/footer
        Accepts: application/json

        {
            "navigation_links": [
                {
                  "url": "http://example.com/about",
                  "name": "about",
                  "title": "About"
                },
                # ...
            ],
            "social_links": [
                {
                    "url": "http://example.com/social",
                    "name": "facebook",
                    "icon-class": "fa-facebook-square",
                    "title": "Facebook",
                    "action": "Sign up on Facebook!"
                },
                # ...
            ],
            "mobile_links": [
                {
                    "url": "http://example.com/android",
                    "name": "google",
                    "image": "http://example.com/google.png",
                    "title": "Google"
                },
                # ...
            ],
            "legal_links": [
                {
                    "url": "http://example.com/terms-of-service.html",
                    "name": "terms_of_service",
                    "title': "Terms of Service"
                },
                # ...
            ],
            "openedx_link": {
                "url": "http://open.edx.org",
                "title": "Powered by Open edX",
                "image": "http://example.com/openedx.png"
            },
            "logo_image": "http://example.com/static/images/logo.png",
            "copyright": "edX, Open edX and their respective logos are registered trademarks of edX Inc."
        }

    Example: Retrieving the footer as HTML

        GET /api/branding/v1/footer
        Accepts: text/html

    Example: Including the footer with the "Powered by Open edX" logo

        GET /api/branding/v1/footer?show-openedx-logo=1
        Accepts: text/html

    Example: Retrieving the footer in a particular language

        GET /api/branding/v1/footer?language=en
        Accepts: text/html

    Example: Retrieving the footer with a language selector

        GET /api/branding/v1/footer?include-language-selector=1
        Accepts: text/html

    Example: Retrieving the footer with all JS and CSS dependencies (for testing)

        GET /api/branding/v1/footer?include-dependencies=1
        Accepts: text/html

    """
    if not branding_api.is_enabled():
        raise Http404

    # Use the content type to decide what representation to serve
    accepts = request.META.get('HTTP_ACCEPT', '*/*')

    # Show the OpenEdX logo in the footer
    show_openedx_logo = bool(request.GET.get('show-openedx-logo', False))

    # Include JS and CSS dependencies
    # This is useful for testing the end-point directly.
    include_dependencies = bool(request.GET.get('include-dependencies', False))

    # Override the language if necessary
    language = request.GET.get('language', translation.get_language())
    try:
        language = get_supported_language_variant(language)
    except LookupError:
        language = settings.LANGUAGE_CODE

    # Include a language selector
    include_language_selector = request.GET.get('include-language-selector', '') == '1'

    # Render the footer information based on the extension
    if 'text/html' in accepts or '*/*' in accepts:
        cache_params = {
            'language': language,
            'show_openedx_logo': show_openedx_logo,
            'include_dependencies': include_dependencies
        }
        if include_language_selector:
            cache_params['language_selector_options'] = ','.join(sorted([lang.code for lang in released_languages()]))
        cache_key = u"branding.footer.{params}.html".format(params=six.moves.urllib.parse.urlencode(cache_params))

        content = cache.get(cache_key)
        if content is None:
            with translation.override(language):
                content = _render_footer_html(
                    request, show_openedx_logo, include_dependencies, include_language_selector, language
                )
                cache.set(cache_key, content, settings.FOOTER_CACHE_TIMEOUT)
        return HttpResponse(content, status=200, content_type="text/html; charset=utf-8")

    elif 'application/json' in accepts:
        cache_key = u"branding.footer.{params}.json".format(
            params=six.moves.urllib.parse.urlencode({
                'language': language,
                'is_secure': request.is_secure(),
            })
        )
        footer_dict = cache.get(cache_key)
        if footer_dict is None:
            with translation.override(language):
                footer_dict = branding_api.get_footer(is_secure=request.is_secure())
                cache.set(cache_key, footer_dict, settings.FOOTER_CACHE_TIMEOUT)
        return JsonResponse(footer_dict, 200, content_type="application/json; charset=utf-8")

    else:
        return HttpResponse(status=406)
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# File:  mitx/lib/loncapa/loncapa_check.py
#
# Python functions which duplicate the standard comparison functions available to LON-CAPA problems.
# Used in translating LON-CAPA problems to i4x problem specification language.

import random

def lc_random(lower,upper,stepsize):
    '''
    like random.randrange but lower and upper can be non-integer
    '''
    nstep = int((upper-lower)/(1.0*stepsize))
    choices = [lower+x*stepsize for x in range(nstep)]
    return random.choice(choices)

<commit_msg>add more loncapa check functions (choose, rad2deg, deg2rad)<commit_after>#!/usr/bin/python
#
# File:  mitx/lib/loncapa/loncapa_check.py
#
# Python functions which duplicate the standard comparison functions available to LON-CAPA problems.
# Used in translating LON-CAPA problems to i4x problem specification language.

from __future__ import division
import random
import math

def lc_random(lower,upper,stepsize):
    '''
    like random.randrange but lower and upper can be non-integer
    '''
    nstep = int((upper-lower)/(1.0*stepsize))
    choices = [lower+x*stepsize for x in range(nstep)]
    return random.choice(choices)

def lc_choose(index,*args):
    '''
    return args[index]
    '''
    try:
        return args[int(index)-1]
    except Exception,err:
        pass
    if len(args):
        return args[0]
    raise Exception,"loncapa_check.lc_choose error, index=%s, args=%s" % (index,args)

deg2rad = 180.0*math.pi
rad2deg = math.pi * 180

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
rio.tasks
~~~~~~~~~~

Implement of rio tasks based on celery.
"""

from time import time

from celery import chord
from requests import ConnectionError

from rio.core import celery
from rio.utils.http import dispatch_webhook_request
from rio.utils.http import FailureWebhookError

@celery.task()
def call_webhook(event, webhook, payload):
    """Build request from event,webhook,payoad and parse response."""
    started_at = time()
    request = {'url': webhook['url'], 'method': webhook['method']}
    request['headers'] = {
        'X-RIO-EVENT': 'uuid=%s,project=%s,topic=%s' % (
            str(event['uuid']),
            event['project'],
            event['topic']
        ),
    }
    if webhook['method'] == 'GET':
        request['params'] = payload
    else:
        request['json'] = payload

    try:
        content = dispatch_webhook_request(**request)
    except (FailureWebhookError, ConnectionError) as exception:
        # TODO: propagate to sentry if configured.
        return dict(
            parent=str(event['uuid']),
            error=exception.message,
            started_at=started_at,
            ended_at=time(),
        )
    return dict(
        parent=str(event['uuid']),
        content=content,
        started_at=started_at,
        ended_at=time()
    )

@celery.task()
def merge_webhooks_runset(runset):
    """Make some statistics on the run set.
    """
    min_started_at = min([w['started_at'] for w in runset])
    max_ended_at = max([w['ended_at'] for w in runset])
    ellapse = max_ended_at - min_started_at
    errors_count = sum(1 for w in runset if 'error' in w)
    total_count = len(runset)
    return dict(
        ellapse=ellapse,
        errors_count=errors_count,
        total_count=total_count,
    )

def exec_event(event, webhooks, payload):
    """Execute event.

    Merge webhooks run set to do some stats after all
    of the webhooks been responded successfully.

    Error webhook will be propagated. Note that other webhook
    calls will still execute.
    """
    calls = (
        call_webhook.s(event, webhook, payload)
        for webhook in webhooks
    )
    callback = merge_webhooks_runset.s()

    call_promise = chord(calls)
    return call_promise(callback)
<commit_msg>simplify code.<commit_after># -*- coding: utf-8 -*-
"""
rio.tasks
~~~~~~~~~~

Implement of rio tasks based on celery.
"""

from time import time

from celery import chord
from requests import ConnectionError

from rio.core import celery
from rio.utils.http import dispatch_webhook_request
from rio.utils.http import FailureWebhookError

@celery.task()
def call_webhook(event, webhook, payload):
    """Build request from event,webhook,payoad and parse response."""
    started_at = time()
    event_identity = 'uuid=%s,project=%s,topic=%s' % (
        str(event['uuid']), event['project'], event['topic']
    )

    request = {
        'url': webhook['url'],
        'method': webhook['method'],
        'headers': {
            'X-RIO-EVENT': event_identity,
        }
    }

    if webhook['method'] == 'GET':
        request['params'] = payload
    else:
        request['json'] = payload

    try:
        content = dispatch_webhook_request(**request)
    except (FailureWebhookError, ConnectionError) as exception:
        # TODO: propagate to sentry if configured with request context.
        return dict(
            parent=str(event['uuid']),
            error=exception.message,
            started_at=started_at,
            ended_at=time(),
        )
    return dict(
        parent=str(event['uuid']),
        content=content,
        started_at=started_at,
        ended_at=time()
    )

@celery.task()
def merge_webhooks_runset(runset):
    """Make some statistics on the run set.
    """
    min_started_at = min([w['started_at'] for w in runset])
    max_ended_at = max([w['ended_at'] for w in runset])
    ellapse = max_ended_at - min_started_at
    errors_count = sum(1 for w in runset if 'error' in w)
    total_count = len(runset)
    return dict(
        ellapse=ellapse,
        errors_count=errors_count,
        total_count=total_count,
    )

def exec_event(event, webhooks, payload):
    """Execute event.

    Merge webhooks run set to do some stats after all
    of the webhooks been responded successfully.

    Error webhook will be propagated. Note that other webhook
    calls will still execute.
    """
    calls = (
        call_webhook.s(event, webhook, payload)
        for webhook in webhooks
    )
    callback = merge_webhooks_runset.s()

    call_promise = chord(calls)
    return call_promise(callback)
<|endoftext|>


<commit_before>import sys
from math import pi
from astropy.io import fits
import numpy as np
import matplotlib.pyplot as plt
import stsci.convolve._lineshape as ls

# In KLpy
import fitpy.fittools as ft

# Utility function to open and plot original spectrum
def openNplot1d (filename, extname=('SCI',1)):
    hdulist = fits.open(filename, 'readonly')
    sp = hdulist[extname].data
    x = np.arange(sp.shape[0])
    plt.clf()
    plt.plot (x, sp)

    return hdulist

# Interactive specification of the section around the feature to work on
def getsubspec (sp):
    # here it should be graphical, but I'm still working on that
    x1 = input("Left edge pixel: ")
    x2 = input("Right edge pixel: ")

    flux = sp[x1:x2]
    pixel = np.arange(x1,x2,1)

    sub = np.zeros((2,flux.shape[0]))
    sub[0] = pixel
    sub[1] = flux

    #plt.clf()
    #plt.plot (pixel, flux)

    #input('continue')

    return sub

def plotresult (sp, bf, nsp):
    plt.clf()
    x = np.arange(0,sp.shape[0],1)
    plt.plot (x, sp)
    plt.plot (x, nsp)
    x = np.arange(0,bf.shape[0],1)
    plt.plot (x, bf)

def rmfeature (inspec, outspec, params=None, profile='voigt'):
    #---- plot and get data
    spin = openNplot1d(inspec)
    specdata = spin['SCI'].data
    spin.close()

    #---- Get data for section around feature
    linedata = getsubspec(specdata)

    #---- Calculate and set initial parameter from linedata

    if params==None:
        contslope = (linedata[1][0] - linedata[1][-1]) / \
                    (linedata[0][0] - linedata[0][-1])
        contlevel = linedata[1][0] - (contslope * linedata[0][0])
        lineindex = linedata.argmin(1)[1]
        lineposition = linedata[0][lineindex]
        linestrength = linedata[1][lineindex] - \
                       ((contslope*linedata[0][lineindex]) + contlevel)
        linewidth = 20.   # pixels.  should find a better way.

        cte = ft.Parameter(contlevel)
        m = ft.Parameter(contslope)
        A = ft.Parameter(linestrength)
        mu = ft.Parameter(lineposition)
        fwhmL = ft.Parameter(linewidth)
        fwhmD = ft.Parameter(linewidth)
    else:
        cte = ft.Parameter(params[0])
        m = ft.Parameter(params[1])
        A = ft.Parameter(params[2])
        mu = ft.Parameter(params[3])
        fwhmL = ft.Parameter(params[4])
        fwhmD = ft.Parameter(params[5])

    #---- Define function [linear (continuum) + lorentz (feature)]
    #     I don't know where the factor 10 I need to apply to A() comes from.
    #     I'll need to figure it out.
    #
    #     Also, those functions apparently need to be defined after the
    #     cte(), m(), etc. Parameter instances are defined.

    def line(x):
        return cte() + m()*x

    def lorentz(x):     # The version in numarray.convolve._lineshape is wrong
        amp = A() * fwhmL() * pi / 2.
        return amp * (fwhmL()/(2.*pi))/((x-mu())**2. + (fwhmL()/2.)**2.)

    def voigt(x):   # import numarray.convolve._lineshape as ls
    #    a = sqrt(log(2.)) * fwhmL() / (2. * fwhmD())
    #    b = 2. * sqrt(log(2.)) * (x - mu()) / fwhmD()
    #    H = exp(-(b**2)) + a/(sqrt(pi)*b**2.)
        amp = A() * 1.      # not right
#        amp = A() * (1. + (fwhmL()/fwhmD())*(fwhmL()*pi/2.))
        return amp * ls.voigt(x, (fwhmD(),fwhmL()), mu())

    def contlorentz(x):
        return line(x) + lorentz(x)

    def contvoigt(x):
        return line(x) + voigt(x)

    #---- Non-linear least square fit (optimize.leastsq)
    if (params==None):
        if profile=='voigt':    # Get initial params from Lorentz fit.
            ft.nlfit(contlorentz, [cte, m, A, mu, fwhmL], linedata[1], x=linedata[0])
            ft.nlfit(contvoigt, [cte, m, A, mu, fwhmD, fwhmL], linedata[1], x=linedata[0])
        elif profile=='lorentz':
            ft.nlfit(contlorentz, [cte, m, A, mu, fwhmL], linedata[1], x=linedata[0])
            fwhmD=ft.Parameter(None)
    else:
        pass

    #---- retrieve line profile parameters only and create a profile
    #     with zero continuum for the entire range for the original spectrum
    #     Then remove the feature

    if profile=='voigt':
        newspecdata = specdata - voigt(np.arange(0,specdata.shape[0],1))
        bestfit = contvoigt(np.arange(0,specdata.shape[0],1))
    elif profile=='lorentz':
        newspecdata = specdata - lorentz(np.arange(0,specdata.shape[0],1))
        bestfit = contlorentz(np.arange(0,specdata.shape[0],1))

    #---- display the original spectrum, the best fit and the
    #     new spectrum.  The feature should be gone

    plotresult(specdata, bestfit, newspecdata)
    print "Best Fit Parameters:"
    print " section = ",linedata[0][0],",",linedata[0][-1]+1
    print "     cte = ",cte()
    print "       m = ",m()
    print "       A = ",A()
    print "      mu = ",mu()
    print "   fwhmL = ",fwhmL()
    print "   fwhmD = ",fwhmD()

    write = raw_input('Write corrected spectrum to '+outspec+'? (y/n): ')

    #---- write output spectrum
    if write=='y':
        spout = fits.open(inspec,'readonly')  # just to create copy of HDUList
        spout['SCI'].data = newspecdata
        spout.writeto(outspec, output_verify='ignore')
        #print ("Not implemented yet, but it isn't the app cool!")
    else:
        print ("Too bad.")

if __name__ == '__main__':
    if len(sys.argv) == 3:
        rmfeature(sys.argv[1], sys.argv[2])
    elif len(sys.argv) > 3:
        params = sys.argv[3:size(sys.argv)]
        rmfeature(sys.argv[1], sys.argv[2], params=params)
    else:
        print "Usage: rmfeature in.fits out.fits <params>"

<commit_msg>fix the fittools import to import from KLpy<commit_after>import sys
from math import pi
from astropy.io import fits
import numpy as np
import matplotlib.pyplot as plt
import stsci.convolve._lineshape as ls

# In KLpy
from KLpy.fitpy import fittools as ft

# Utility function to open and plot original spectrum
def openNplot1d (filename, extname=('SCI',1)):
    hdulist = fits.open(filename, 'readonly')
    sp = hdulist[extname].data
    x = np.arange(sp.shape[0])
    plt.clf()
    plt.plot (x, sp)

    return hdulist

# Interactive specification of the section around the feature to work on
def getsubspec (sp):
    # here it should be graphical, but I'm still working on that
    x1 = input("Left edge pixel: ")
    x2 = input("Right edge pixel: ")

    flux = sp[x1:x2]
    pixel = np.arange(x1,x2,1)

    sub = np.zeros((2,flux.shape[0]))
    sub[0] = pixel
    sub[1] = flux

    #plt.clf()
    #plt.plot (pixel, flux)

    #input('continue')

    return sub

def plotresult (sp, bf, nsp):
    plt.clf()
    x = np.arange(0,sp.shape[0],1)
    plt.plot (x, sp)
    plt.plot (x, nsp)
    x = np.arange(0,bf.shape[0],1)
    plt.plot (x, bf)

def rmfeature (inspec, outspec, params=None, profile='voigt'):
    #---- plot and get data
    spin = openNplot1d(inspec)
    specdata = spin['SCI'].data
    spin.close()

    #---- Get data for section around feature
    linedata = getsubspec(specdata)

    #---- Calculate and set initial parameter from linedata

    if params==None:
        contslope = (linedata[1][0] - linedata[1][-1]) / \
                    (linedata[0][0] - linedata[0][-1])
        contlevel = linedata[1][0] - (contslope * linedata[0][0])
        lineindex = linedata.argmin(1)[1]
        lineposition = linedata[0][lineindex]
        linestrength = linedata[1][lineindex] - \
                       ((contslope*linedata[0][lineindex]) + contlevel)
        linewidth = 20.   # pixels.  should find a better way.

        cte = ft.Parameter(contlevel)
        m = ft.Parameter(contslope)
        A = ft.Parameter(linestrength)
        mu = ft.Parameter(lineposition)
        fwhmL = ft.Parameter(linewidth)
        fwhmD = ft.Parameter(linewidth)
    else:
        cte = ft.Parameter(params[0])
        m = ft.Parameter(params[1])
        A = ft.Parameter(params[2])
        mu = ft.Parameter(params[3])
        fwhmL = ft.Parameter(params[4])
        fwhmD = ft.Parameter(params[5])

    #---- Define function [linear (continuum) + lorentz (feature)]
    #     I don't know where the factor 10 I need to apply to A() comes from.
    #     I'll need to figure it out.
    #
    #     Also, those functions apparently need to be defined after the
    #     cte(), m(), etc. Parameter instances are defined.

    def line(x):
        return cte() + m()*x

    def lorentz(x):     # The version in numarray.convolve._lineshape is wrong
        amp = A() * fwhmL() * pi / 2.
        return amp * (fwhmL()/(2.*pi))/((x-mu())**2. + (fwhmL()/2.)**2.)

    def voigt(x):   # import numarray.convolve._lineshape as ls
    #    a = sqrt(log(2.)) * fwhmL() / (2. * fwhmD())
    #    b = 2. * sqrt(log(2.)) * (x - mu()) / fwhmD()
    #    H = exp(-(b**2)) + a/(sqrt(pi)*b**2.)
        amp = A() * 1.      # not right
#        amp = A() * (1. + (fwhmL()/fwhmD())*(fwhmL()*pi/2.))
        return amp * ls.voigt(x, (fwhmD(),fwhmL()), mu())

    def contlorentz(x):
        return line(x) + lorentz(x)

    def contvoigt(x):
        return line(x) + voigt(x)

    #---- Non-linear least square fit (optimize.leastsq)
    if (params==None):
        if profile=='voigt':    # Get initial params from Lorentz fit.
            ft.nlfit(contlorentz, [cte, m, A, mu, fwhmL], linedata[1], x=linedata[0])
            ft.nlfit(contvoigt, [cte, m, A, mu, fwhmD, fwhmL], linedata[1], x=linedata[0])
        elif profile=='lorentz':
            ft.nlfit(contlorentz, [cte, m, A, mu, fwhmL], linedata[1], x=linedata[0])
            fwhmD=ft.Parameter(None)
    else:
        pass

    #---- retrieve line profile parameters only and create a profile
    #     with zero continuum for the entire range for the original spectrum
    #     Then remove the feature

    if profile=='voigt':
        newspecdata = specdata - voigt(np.arange(0,specdata.shape[0],1))
        bestfit = contvoigt(np.arange(0,specdata.shape[0],1))
    elif profile=='lorentz':
        newspecdata = specdata - lorentz(np.arange(0,specdata.shape[0],1))
        bestfit = contlorentz(np.arange(0,specdata.shape[0],1))

    #---- display the original spectrum, the best fit and the
    #     new spectrum.  The feature should be gone

    plotresult(specdata, bestfit, newspecdata)
    print "Best Fit Parameters:"
    print " section = ",linedata[0][0],",",linedata[0][-1]+1
    print "     cte = ",cte()
    print "       m = ",m()
    print "       A = ",A()
    print "      mu = ",mu()
    print "   fwhmL = ",fwhmL()
    print "   fwhmD = ",fwhmD()

    write = raw_input('Write corrected spectrum to '+outspec+'? (y/n): ')

    #---- write output spectrum
    if write=='y':
        spout = fits.open(inspec,'readonly')  # just to create copy of HDUList
        spout['SCI'].data = newspecdata
        spout.writeto(outspec, output_verify='ignore')
        #print ("Not implemented yet, but it isn't the app cool!")
    else:
        print ("Too bad.")

if __name__ == '__main__':
    if len(sys.argv) == 3:
        rmfeature(sys.argv[1], sys.argv[2])
    elif len(sys.argv) > 3:
        params = sys.argv[3:size(sys.argv)]
        rmfeature(sys.argv[1], sys.argv[2], params=params)
    else:
        print "Usage: rmfeature in.fits out.fits <params>"

<|endoftext|>


<commit_before># encoding: UTF-8
# Copyright 2017 Google.com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import tensorflow as tf
from tensorflow.contrib import layers
from tensorflow.contrib import rnn  # rnn stuff temporarily in contrib, moving back to code in TF 1.1
import os
import time
import math
import numpy as np
import my_txtutils as txt
tf.set_random_seed(0)

# model parameters
#
# Usage:
#   Training only:
#         Leave all the parameters as they are
#         Disable validation to run a bit faster (set validation=False below)
#         You can follow progress in Tensorboard: tensorboard --log-dir=log
#   Training and experimentation (default):
#         Keep validation enabled
#         You can now play with the parameters anf follow the effects in Tensorboard
#         A good choice of parameters ensures that the testing and validation curves stay close
#         To see the curves drift apart ("overfitting") try to use an insufficient amount of
#         training data (shakedir = "shakespeare/t*.txt" for example)
#
SEQLEN = 30
BATCHSIZE = 200
ALPHASIZE = txt.ALPHASIZE
INTERNALSIZE = 512
NLAYERS = 3
learning_rate = 0.001  # fixed learning rate
dropout_pkeep = 0.8    # some dropout

# load data, either shakespeare, or the Python source of Tensorflow itself
shakedir = "shakespeare/*.txt"
#shakedir = "../tensorflow/**/*.py"
codetext, valitext, bookranges = txt.read_data_files(shakedir, validation=True)

# display some stats on the data
epoch_size = len(codetext) // (BATCHSIZE * SEQLEN)
txt.print_data_stats(len(codetext), len(valitext), epoch_size)

#
# the model (see FAQ in README.md)
#
lr = tf.placeholder(tf.float32, name='lr')  # learning rate
pkeep = tf.placeholder(tf.float32, name='pkeep')  # dropout parameter
batchsize = tf.placeholder(tf.int32, name='batchsize')

# inputs
X = tf.placeholder(tf.uint8, [None, None], name='X')    # [ BATCHSIZE, SEQLEN ]
Xo = tf.one_hot(X, ALPHASIZE, 1.0, 0.0)                 # [ BATCHSIZE, SEQLEN, ALPHASIZE ]
# expected outputs = same sequence shifted by 1 since we are trying to predict the next character
Y_ = tf.placeholder(tf.uint8, [None, None], name='Y_')  # [ BATCHSIZE, SEQLEN ]
Yo_ = tf.one_hot(Y_, ALPHASIZE, 1.0, 0.0)               # [ BATCHSIZE, SEQLEN, ALPHASIZE ]
# input state
Hin = tf.placeholder(tf.float32, [None, INTERNALSIZE*NLAYERS], name='Hin')  # [ BATCHSIZE, INTERNALSIZE * NLAYERS]

# using a NLAYERS=3 layers of GRU cells, unrolled SEQLEN=30 times
# dynamic_rnn infers SEQLEN from the size of the inputs Xo

# How to properly apply dropout in RNNs: see README.md
cells = [rnn.GRUCell(INTERNALSIZE) for _ in range(NLAYERS)]

# variational dropout implementation (does not work)
#dropcells  = [rnn.DropoutWrapper(cells[0],  input_keep_prob=pkeep, output_keep_prob=1.0,   state_keep_prob=pkeep, variational_recurrent=True, input_size=ALPHASIZE, dtype=tf.float32)]
#dropcells += [rnn.DropoutWrapper(cell,      input_keep_prob=pkeep, output_keep_prob=1.0,   state_keep_prob=pkeep, variational_recurrent=True, input_size=INTERNALSIZE, dtype=tf.float32) for cell in cells[1:-1]]
#dropcells += [rnn.DropoutWrapper(cells[-1], input_keep_prob=pkeep, output_keep_prob=pkeep, state_keep_prob=pkeep, variational_recurrent=True, input_size=INTERNALSIZE, dtype=tf.float32)]
#multicell = rnn.MultiRNNCell(dropcells, state_is_tuple=False)
#"naive dropout" implementation
dropcells = [rnn.DropoutWrapper(cell,input_keep_prob=pkeep) for cell in cells]
multicell = rnn.MultiRNNCell(dropcells, state_is_tuple=False)
multicell = rnn.DropoutWrapper(multicell, output_keep_prob=pkeep)  # dropout for the softmax layer

Yr, H = tf.nn.dynamic_rnn(multicell, Xo, dtype=tf.float32, initial_state=Hin)
# Yr: [ BATCHSIZE, SEQLEN, INTERNALSIZE ]
# H:  [ BATCHSIZE, INTERNALSIZE*NLAYERS ] # this is the last state in the sequence

H = tf.identity(H, name='H')  # just to give it a name

# Softmax layer implementation:
# Flatten the first two dimension of the output [ BATCHSIZE, SEQLEN, ALPHASIZE ] => [ BATCHSIZE x SEQLEN, ALPHASIZE ]
# then apply softmax readout layer. This way, the weights and biases are shared across unrolled time steps.
# From the readout point of view, a value coming from a sequence time step or a minibatch item is the same thing.

Yflat = tf.reshape(Yr, [-1, INTERNALSIZE])    # [ BATCHSIZE x SEQLEN, INTERNALSIZE ]
Ylogits = layers.linear(Yflat, ALPHASIZE)     # [ BATCHSIZE x SEQLEN, ALPHASIZE ]
Yflat_ = tf.reshape(Yo_, [-1, ALPHASIZE])     # [ BATCHSIZE x SEQLEN, ALPHASIZE ]
loss = tf.nn.softmax_cross_entropy_with_logits(logits=Ylogits, labels=Yflat_)  # [ BATCHSIZE x SEQLEN ]
loss = tf.reshape(loss, [batchsize, -1])      # [ BATCHSIZE, SEQLEN ]
Yo = tf.nn.softmax(Ylogits, name='Yo')        # [ BATCHSIZE x SEQLEN, ALPHASIZE ]
Y = tf.argmax(Yo, 1)                          # [ BATCHSIZE x SEQLEN ]
Y = tf.reshape(Y, [batchsize, -1], name="Y")  # [ BATCHSIZE, SEQLEN ]
train_step = tf.train.AdamOptimizer(lr).minimize(loss)

# stats for display
seqloss = tf.reduce_mean(loss, 1)
batchloss = tf.reduce_mean(seqloss)
accuracy = tf.reduce_mean(tf.cast(tf.equal(Y_, tf.cast(Y, tf.uint8)), tf.float32))
loss_summary = tf.summary.scalar("batch_loss", batchloss)
acc_summary = tf.summary.scalar("batch_accuracy", accuracy)
summaries = tf.summary.merge([loss_summary, acc_summary])

# Init Tensorboard stuff. This will save Tensorboard information into a different
# folder at each run named 'log/<timestamp>/'. Two sets of data are saved so that
# you can compare training and validation curves visually in Tensorboard.
timestamp = str(math.trunc(time.time()))
summary_writer = tf.summary.FileWriter("log/" + timestamp + "-training")
validation_writer = tf.summary.FileWriter("log/" + timestamp + "-validation")

# Init for saving models. They will be saved into a directory named 'checkpoints'.
# Only the last checkpoint is kept.
if not os.path.exists("checkpoints"):
    os.mkdir("checkpoints")
saver = tf.train.Saver(max_to_keep=1000)

# for display: init the progress bar
DISPLAY_FREQ = 50
_50_BATCHES = DISPLAY_FREQ * BATCHSIZE * SEQLEN
progress = txt.Progress(DISPLAY_FREQ, size=111+2, msg="Training on next "+str(DISPLAY_FREQ)+" batches")

# init
istate = np.zeros([BATCHSIZE, INTERNALSIZE*NLAYERS])  # initial zero input state
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
step = 0

# training loop
for x, y_, epoch in txt.rnn_minibatch_sequencer(codetext, BATCHSIZE, SEQLEN, nb_epochs=10):

    # train on one minibatch
    feed_dict = {X: x, Y_: y_, Hin: istate, lr: learning_rate, pkeep: dropout_pkeep, batchsize: BATCHSIZE}
    _, y, ostate = sess.run([train_step, Y, H], feed_dict=feed_dict)

    # log training data for Tensorboard display a mini-batch of sequences (every 50 batches)
    if step % _50_BATCHES == 0:
        feed_dict = {X: x, Y_: y_, Hin: istate, pkeep: 1.0, batchsize: BATCHSIZE}  # no dropout for validation
        y, l, bl, acc, smm = sess.run([Y, seqloss, batchloss, accuracy, summaries], feed_dict=feed_dict)
        txt.print_learning_learned_comparison(x, y, l, bookranges, bl, acc, epoch_size, step, epoch)
        summary_writer.add_summary(smm, step)

    # run a validation step every 50 batches
    # The validation text should be a single sequence but that's too slow (1s per 1024 chars!),
    # so we cut it up and batch the pieces (slightly inaccurate)
    # tested: validating with 5K sequences instead of 1K is only slightly more accurate, but a lot slower.
    if step % _50_BATCHES == 0 and len(valitext) > 0:
        VALI_SEQLEN = 1*1024  # Sequence length for validation. State will be wrong at the start of each sequence.
        bsize = len(valitext) // VALI_SEQLEN
        txt.print_validation_header(len(codetext), bookranges)
        vali_x, vali_y, _ = next(txt.rnn_minibatch_sequencer(valitext, bsize, VALI_SEQLEN, 1))  # all data in 1 batch
        vali_nullstate = np.zeros([bsize, INTERNALSIZE*NLAYERS])
        feed_dict = {X: vali_x, Y_: vali_y, Hin: vali_nullstate, pkeep: 1.0,  # no dropout for validation
                     batchsize: bsize}
        ls, acc, smm = sess.run([batchloss, accuracy, summaries], feed_dict=feed_dict)
        txt.print_validation_stats(ls, acc)
        # save validation data for Tensorboard
        validation_writer.add_summary(smm, step)

    # display a short text generated with the current weights and biases (every 150 batches)
    if step // 3 % _50_BATCHES == 0:
        txt.print_text_generation_header()
        ry = np.array([[txt.convert_from_alphabet(ord("K"))]])
        rh = np.zeros([1, INTERNALSIZE * NLAYERS])
        for k in range(1000):
            ryo, rh = sess.run([Yo, H], feed_dict={X: ry, pkeep: 1.0, Hin: rh, batchsize: 1})
            rc = txt.sample_from_probabilities(ryo, topn=10 if epoch <= 1 else 2)
            print(chr(txt.convert_to_alphabet(rc)), end="")
            ry = np.array([[rc]])
        txt.print_text_generation_footer()

    # save a checkpoint (every 500 batches)
    if step // 10 % _50_BATCHES == 0:
        saved_file = saver.save(sess, 'checkpoints/rnn_train_' + timestamp, global_step=step)
        print("Saved file: " + saved_file)

    # display progress bar
    progress.step(reset=step % _50_BATCHES == 0)

    # loop state around
    istate = ostate
    step += BATCHSIZE * SEQLEN

# all runs: SEQLEN = 30, BATCHSIZE = 100, ALPHASIZE = 98, INTERNALSIZE = 512, NLAYERS = 3
# run 1477669632 decaying learning rate 0.001-0.0001-1e7 dropout 0.5: not good
# run 1477670023 lr=0.001 no dropout: very good

# Tensorflow runs:
# 1485434262
#   trained on shakespeare/t*.txt only. Validation on 1K sequences
#   validation loss goes up from step 5M (overfitting because of small dataset)
# 1485436038
#   trained on shakespeare/t*.txt only. Validation on 5K sequences
#   On 5K sequences validation accuracy is slightly higher and loss slightly lower
#   => sequence breaks do introduce inaccuracies but the effect is small
# 1485437956
#   Trained on shakespeare/*.txt. Validation on 1K sequences
#   On this much larger dataset, validation loss still decreasing after 6 epochs (step 35M)
# 1495447371
#   Trained on shakespeare/*.txt no dropout, 30 epochs
#   Validation loss starts going up after 10 epochs (overfitting)
# 1495440473
#   Trained on shakespeare/*.txt "naive dropout" pkeep=0.8, 30 epochs
#   Dropout brings the validation loss under control, preventing it from
#   going up but the effect is small.
<commit_msg>Tensorflow 1.1 updates<commit_after># encoding: UTF-8
# Copyright 2017 Google.com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import tensorflow as tf
from tensorflow.contrib import layers
from tensorflow.contrib import rnn  # rnn stuff temporarily in contrib, moving back to code in TF 1.1
import os
import time
import math
import numpy as np
import my_txtutils as txt
tf.set_random_seed(0)

# model parameters
#
# Usage:
#   Training only:
#         Leave all the parameters as they are
#         Disable validation to run a bit faster (set validation=False below)
#         You can follow progress in Tensorboard: tensorboard --log-dir=log
#   Training and experimentation (default):
#         Keep validation enabled
#         You can now play with the parameters anf follow the effects in Tensorboard
#         A good choice of parameters ensures that the testing and validation curves stay close
#         To see the curves drift apart ("overfitting") try to use an insufficient amount of
#         training data (shakedir = "shakespeare/t*.txt" for example)
#
SEQLEN = 30
BATCHSIZE = 200
ALPHASIZE = txt.ALPHASIZE
INTERNALSIZE = 512
NLAYERS = 3
learning_rate = 0.001  # fixed learning rate
dropout_pkeep = 0.8    # some dropout

# load data, either shakespeare, or the Python source of Tensorflow itself
shakedir = "shakespeare/*.txt"
#shakedir = "../tensorflow/**/*.py"
codetext, valitext, bookranges = txt.read_data_files(shakedir, validation=True)

# display some stats on the data
epoch_size = len(codetext) // (BATCHSIZE * SEQLEN)
txt.print_data_stats(len(codetext), len(valitext), epoch_size)

#
# the model (see FAQ in README.md)
#
lr = tf.placeholder(tf.float32, name='lr')  # learning rate
pkeep = tf.placeholder(tf.float32, name='pkeep')  # dropout parameter
batchsize = tf.placeholder(tf.int32, name='batchsize')

# inputs
X = tf.placeholder(tf.uint8, [None, None], name='X')    # [ BATCHSIZE, SEQLEN ]
Xo = tf.one_hot(X, ALPHASIZE, 1.0, 0.0)                 # [ BATCHSIZE, SEQLEN, ALPHASIZE ]
# expected outputs = same sequence shifted by 1 since we are trying to predict the next character
Y_ = tf.placeholder(tf.uint8, [None, None], name='Y_')  # [ BATCHSIZE, SEQLEN ]
Yo_ = tf.one_hot(Y_, ALPHASIZE, 1.0, 0.0)               # [ BATCHSIZE, SEQLEN, ALPHASIZE ]
# input state
Hin = tf.placeholder(tf.float32, [None, INTERNALSIZE*NLAYERS], name='Hin')  # [ BATCHSIZE, INTERNALSIZE * NLAYERS]

# using a NLAYERS=3 layers of GRU cells, unrolled SEQLEN=30 times
# dynamic_rnn infers SEQLEN from the size of the inputs Xo

# How to properly apply dropout in RNNs: see README.md
cells = [rnn.GRUCell(INTERNALSIZE) for _ in range(NLAYERS)]
# "naive dropout" implementation
dropcells = [rnn.DropoutWrapper(cell,input_keep_prob=pkeep) for cell in cells]
multicell = rnn.MultiRNNCell(dropcells, state_is_tuple=False)
multicell = rnn.DropoutWrapper(multicell, output_keep_prob=pkeep)  # dropout for the softmax layer

Yr, H = tf.nn.dynamic_rnn(multicell, Xo, dtype=tf.float32, initial_state=Hin)
# Yr: [ BATCHSIZE, SEQLEN, INTERNALSIZE ]
# H:  [ BATCHSIZE, INTERNALSIZE*NLAYERS ] # this is the last state in the sequence

H = tf.identity(H, name='H')  # just to give it a name

# Softmax layer implementation:
# Flatten the first two dimension of the output [ BATCHSIZE, SEQLEN, ALPHASIZE ] => [ BATCHSIZE x SEQLEN, ALPHASIZE ]
# then apply softmax readout layer. This way, the weights and biases are shared across unrolled time steps.
# From the readout point of view, a value coming from a sequence time step or a minibatch item is the same thing.

Yflat = tf.reshape(Yr, [-1, INTERNALSIZE])    # [ BATCHSIZE x SEQLEN, INTERNALSIZE ]
Ylogits = layers.linear(Yflat, ALPHASIZE)     # [ BATCHSIZE x SEQLEN, ALPHASIZE ]
Yflat_ = tf.reshape(Yo_, [-1, ALPHASIZE])     # [ BATCHSIZE x SEQLEN, ALPHASIZE ]
loss = tf.nn.softmax_cross_entropy_with_logits(logits=Ylogits, labels=Yflat_)  # [ BATCHSIZE x SEQLEN ]
loss = tf.reshape(loss, [batchsize, -1])      # [ BATCHSIZE, SEQLEN ]
Yo = tf.nn.softmax(Ylogits, name='Yo')        # [ BATCHSIZE x SEQLEN, ALPHASIZE ]
Y = tf.argmax(Yo, 1)                          # [ BATCHSIZE x SEQLEN ]
Y = tf.reshape(Y, [batchsize, -1], name="Y")  # [ BATCHSIZE, SEQLEN ]
train_step = tf.train.AdamOptimizer(lr).minimize(loss)

# stats for display
seqloss = tf.reduce_mean(loss, 1)
batchloss = tf.reduce_mean(seqloss)
accuracy = tf.reduce_mean(tf.cast(tf.equal(Y_, tf.cast(Y, tf.uint8)), tf.float32))
loss_summary = tf.summary.scalar("batch_loss", batchloss)
acc_summary = tf.summary.scalar("batch_accuracy", accuracy)
summaries = tf.summary.merge([loss_summary, acc_summary])

# Init Tensorboard stuff. This will save Tensorboard information into a different
# folder at each run named 'log/<timestamp>/'. Two sets of data are saved so that
# you can compare training and validation curves visually in Tensorboard.
timestamp = str(math.trunc(time.time()))
summary_writer = tf.summary.FileWriter("log/" + timestamp + "-training")
validation_writer = tf.summary.FileWriter("log/" + timestamp + "-validation")

# Init for saving models. They will be saved into a directory named 'checkpoints'.
# Only the last checkpoint is kept.
if not os.path.exists("checkpoints"):
    os.mkdir("checkpoints")
saver = tf.train.Saver(max_to_keep=1000)

# for display: init the progress bar
DISPLAY_FREQ = 50
_50_BATCHES = DISPLAY_FREQ * BATCHSIZE * SEQLEN
progress = txt.Progress(DISPLAY_FREQ, size=111+2, msg="Training on next "+str(DISPLAY_FREQ)+" batches")

# init
istate = np.zeros([BATCHSIZE, INTERNALSIZE*NLAYERS])  # initial zero input state
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
step = 0

# training loop
for x, y_, epoch in txt.rnn_minibatch_sequencer(codetext, BATCHSIZE, SEQLEN, nb_epochs=10):

    # train on one minibatch
    feed_dict = {X: x, Y_: y_, Hin: istate, lr: learning_rate, pkeep: dropout_pkeep, batchsize: BATCHSIZE}
    _, y, ostate = sess.run([train_step, Y, H], feed_dict=feed_dict)

    # log training data for Tensorboard display a mini-batch of sequences (every 50 batches)
    if step % _50_BATCHES == 0:
        feed_dict = {X: x, Y_: y_, Hin: istate, pkeep: 1.0, batchsize: BATCHSIZE}  # no dropout for validation
        y, l, bl, acc, smm = sess.run([Y, seqloss, batchloss, accuracy, summaries], feed_dict=feed_dict)
        txt.print_learning_learned_comparison(x, y, l, bookranges, bl, acc, epoch_size, step, epoch)
        summary_writer.add_summary(smm, step)

    # run a validation step every 50 batches
    # The validation text should be a single sequence but that's too slow (1s per 1024 chars!),
    # so we cut it up and batch the pieces (slightly inaccurate)
    # tested: validating with 5K sequences instead of 1K is only slightly more accurate, but a lot slower.
    if step % _50_BATCHES == 0 and len(valitext) > 0:
        VALI_SEQLEN = 1*1024  # Sequence length for validation. State will be wrong at the start of each sequence.
        bsize = len(valitext) // VALI_SEQLEN
        txt.print_validation_header(len(codetext), bookranges)
        vali_x, vali_y, _ = next(txt.rnn_minibatch_sequencer(valitext, bsize, VALI_SEQLEN, 1))  # all data in 1 batch
        vali_nullstate = np.zeros([bsize, INTERNALSIZE*NLAYERS])
        feed_dict = {X: vali_x, Y_: vali_y, Hin: vali_nullstate, pkeep: 1.0,  # no dropout for validation
                     batchsize: bsize}
        ls, acc, smm = sess.run([batchloss, accuracy, summaries], feed_dict=feed_dict)
        txt.print_validation_stats(ls, acc)
        # save validation data for Tensorboard
        validation_writer.add_summary(smm, step)

    # display a short text generated with the current weights and biases (every 150 batches)
    if step // 3 % _50_BATCHES == 0:
        txt.print_text_generation_header()
        ry = np.array([[txt.convert_from_alphabet(ord("K"))]])
        rh = np.zeros([1, INTERNALSIZE * NLAYERS])
        for k in range(1000):
            ryo, rh = sess.run([Yo, H], feed_dict={X: ry, pkeep: 1.0, Hin: rh, batchsize: 1})
            rc = txt.sample_from_probabilities(ryo, topn=10 if epoch <= 1 else 2)
            print(chr(txt.convert_to_alphabet(rc)), end="")
            ry = np.array([[rc]])
        txt.print_text_generation_footer()

    # save a checkpoint (every 500 batches)
    if step // 10 % _50_BATCHES == 0:
        saved_file = saver.save(sess, 'checkpoints/rnn_train_' + timestamp, global_step=step)
        print("Saved file: " + saved_file)

    # display progress bar
    progress.step(reset=step % _50_BATCHES == 0)

    # loop state around
    istate = ostate
    step += BATCHSIZE * SEQLEN

# all runs: SEQLEN = 30, BATCHSIZE = 100, ALPHASIZE = 98, INTERNALSIZE = 512, NLAYERS = 3
# run 1477669632 decaying learning rate 0.001-0.0001-1e7 dropout 0.5: not good
# run 1477670023 lr=0.001 no dropout: very good

# Tensorflow runs:
# 1485434262
#   trained on shakespeare/t*.txt only. Validation on 1K sequences
#   validation loss goes up from step 5M (overfitting because of small dataset)
# 1485436038
#   trained on shakespeare/t*.txt only. Validation on 5K sequences
#   On 5K sequences validation accuracy is slightly higher and loss slightly lower
#   => sequence breaks do introduce inaccuracies but the effect is small
# 1485437956
#   Trained on shakespeare/*.txt. Validation on 1K sequences
#   On this much larger dataset, validation loss still decreasing after 6 epochs (step 35M)
# 1495447371
#   Trained on shakespeare/*.txt no dropout, 30 epochs
#   Validation loss starts going up after 10 epochs (overfitting)
# 1495440473
#   Trained on shakespeare/*.txt "naive dropout" pkeep=0.8, 30 epochs
#   Dropout brings the validation loss under control, preventing it from
#   going up but the effect is small.
<|endoftext|>


<commit_before>import math
from config import *

def get_area(level):
    """Get the section of the game a current level is in."""
    if level >= 15:
        return 4
    return int(math.floor(level / 4.0))

def action_color(key):
    return COLORS[key]

def level_color(level):
    return LEVEL_COLORS[get_area(level)]

def get_x(keys):
    if LEFT in keys and RIGHT in keys:
        return 0
    elif LEFT in keys:
        return -1.4142 if UP in keys or DOWN in keys else -1
    elif RIGHT in keys:
        return 1.4142 if UP in keys or DOWN in keys else 1
    else:
        return 0

def get_y(keys):
    if UP in keys and DOWN in keys:
        return 0
    elif DOWN in keys:
        return -1.4142 if LEFT in keys or RIGHT in keys else -1
    elif UP in keys:
        return 1.4142 if LEFT in keys or RIGHT in keys else 1
    else:
        return 0<commit_msg>Fix some math suckage.<commit_after>import math
from config import *

def get_area(level):
    """Get the section of the game a current level is in."""
    if level >= 15:
        return 4
    return int(math.floor(level / 4.0))

def action_color(key):
    return COLORS[key]

def level_color(level):
    return LEVEL_COLORS[get_area(level)]

def get_x(keys):
    diag = UP in keys or DOWN in keys
    if LEFT in keys and RIGHT in keys:
        return 0
    elif LEFT in keys:
        return -0.707 if diag else -1
    elif RIGHT in keys:
        return 0.707 if diag in keys else 1
    else:
        return 0

def get_y(keys):
    diag = LEFT in keys or RIGHT in keys
    if UP in keys and DOWN in keys:
        return 0
    elif DOWN in keys:
        return -0.707 if diag else -1
    elif UP in keys:
        return 0.707 if diag in keys else 1
    else:
        return 0<|endoftext|>