<commit_before># vim: set fileencoding=utf-8 :

import sys, socket, time
from helper import *
import config

GZIP = True
try:
    import StringIO, gzip
except:
    GZIP = False

if 'MSG_WAITALL' in socket.__dict__:
    EASY_RECV = True
else:
    EASY_RECV = False

def extract_host_page_port(url):
    ''' returns tuple (host, page, port) '''
    page = ''
    if url.startswith('http://'): # we don't need this
        url = url[7:]
    p = url.find(':')   # port
    br = url.find('/')  # get request
    if br == -1:
        br = url.find('?') # get request 2
    if br == -1:
        host = url
        page = '/'
        br = 999999
    else:
        host = url[:br]
        page = url[br:]
    port = 80
    if p != -1:
        port = int(url[p+1:br])
        host = url[:p]
    return (host, page, port)

C_OPEN   = 1
C_IN_USE = 2
class http(object):
    conns = {} # this will store all keep-alive connections in form (host, state)

    def __init__(self, url, log = None):
        self.host, self.page, self.port = extract_host_page_port(url)
        self.request = {}
        self.request['http_version'] = '1.1'
        self.request['method']       = 'GET'
        self.request['header']       = [] # can be set from outside
        if GZIP:
            self.request['header'].append('Accept-Encoding: gzip')
        self.log = log
        self.redirection = ''

    def connect(self, force = False):
        if self.request['http_version'] == '1.1' and config.keepalive:
            self.keepalive = True
        else:
            self.keepalive = False
        if self.keepalive and self.host in http.conns and not force:
            if http.conns[self.host][0] == C_OPEN:
                return http.conns[self.host][1]
        c = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            c.connect((self.host, self.port))
        except socket.gaierror, (e, txt):
            # socket.gaierror: (-2, 'Name or service not known')
            if self.log:
                self.log.bug('error in connect to %s:%d errorcode:%d and %s' % (self.host, self.port, e, txt.decode('utf-8')))
            if self.host in http.conns:
                del http.conns[self.host]
        else:
            if self.keepalive:
                http.conns[self.host] = [c, C_IN_USE]
        return c

    def open(self, post = ''):
        self.c = self.connect()
        header = []
        if post:
            self.request['method'] = 'POST'
        header.append('%s %s HTTP/%s' % (self.request['method'], self.page, self.request['http_version']))
        header.append('HOST: %s' % self.host)
        for i in self.request['header']:
            header.append(i)
        if post:
            header.append('Content-Type: application/x-www-form-urlencoded')
            header.append('Content-Length: %d' % len(post))
            header.append('\r\n%s' % post)
        send = '\r\n'.join(header)
        if not post:
            send += '\r\n\r\n'
        self.c.sendall(send)
        self.get_head()

    def recv(self, size = 4096, precision = False):
        ''' a blocking recv function - which should also work on windows and solaris
            this is the lowest level of recv, which i can call from this class '''
        data = ''
        if self.buf:
            data = self.buf[:size]
            self.buf = self.buf[size:]
            size -= len(data)
            if size == 0:
                return data
        if EASY_RECV:
            data += self.c.recv(size, socket.MSG_WAITALL)
        else:
            while size > 0:
                chunk = self.c.recv(size)
                if chunk == '':
                    break
                data += chunk
                size -= len(chunk)
        if precision:
            self.buf = data[size:]
            return data[:size]
        return data

    def recv_with_reconnect(self, size = 4096):
        ''' a wrapper around the socketrecv to allow reconnect on closed sockets '''
        try:
            return self.c.recv(size)
        except socket.error, (e, err):
            # error: (104, 'Die Verbindung wurde vom Kommunikationspartner zur\xc3\xbcckgeset
            # gaierror: (-2,eerror: (104, 'Die Verbindung wurde vom Kommunikationspartner zur\xc3\xbcckgesetzt')
            if e == 104:
                self.c = self.connect(True)
                return self.c.recv(size)
            else:
                if self.host in http.conns:
                    del http.conns[self.host] # we have a strange error here, so we just delete this host - cause it will surely produce more errors
                self.log.bug('crecv has a problem with %d, %d, %s' % (e, err.eerror[0], err.eerror[1]))

    def get_chunks(self):
        ''' recursively getting chunks to_end is an internally used bool, to determine if we received already the full body '''
        body = self.buf
        # first we download the whole file
        while True:
            if body.endswith('\n0\r\n\r\n'):
                break
            body += self.c.recv(4096)
        body = body[:-5]

        # after that we create a new return string and eliminate all chunk-trash
        x = body.find('\r\n')
        body2 = ''
        while x > 0:
            length = int(body[:x], 16)
            body2 += body[(x + 2):(x + 2 + length)]
            body = body[x + 4 + length:]
            if not body:
                return body2
            x = body.find('\r\n')
        # self.log.bug('strange chunked response')
        return ''

    def get_head(self):
        # TODO add a nonblocking recv here, cause we can be quite sure, that after the recv we want to read at least the return-header
        # maybe make an argument, here
        # TODO check for keep-alive here
        ''' just get the answering head - we need at least this, to receive the body (else we won't know if the body is chunked and so on)
        also returns all already gathered pieces of the body '''
        self.buf = None # reset it first
        self.buf = self.recv_with_reconnect()
        x = self.buf.find('\r\n\r\n')
        while x == -1:
            self.buf += self.recv()
            x = self.buf.find('\r\n\r\n')
        self.head = header(self.buf[:x+2]) # keep the \r\n at the end, so we can search easier
        self.buf = self.buf[x+4:]
        if self.head.status == 301 or self.head.status == 302 or self.head.status == 303: # 302 == found, 303 == see other
            self.redirection = self.head.get('Location')
            self.host, self.page, self.port = extract_host_page_port(self.redirection)
            self.open()
        # open(self.host,'w').writelines(self.head.plain())

    def finnish(self):
        ''' when a download gets ended, this function will mark the connection as free for future requests '''
        if self.keepalive:
            http.conns[self.host] = (self.c, C_OPEN)
        else:
            self.c.close()

    def get(self):
        if self.head.get('Transfer-Encoding') == 'chunked':
            body = self.get_chunks()
        else:
            # http://code.activestate.com/recipes/408859/
            # for recv-all ideas - i use the simple method where i expect the server to close - merged with the content-length field
            length = self.head.get('Content-Length')
            if not length:
                length = 9999999 # very big - to make sure we download everything
                if self.log:
                    self.log.warning('there was no content length in the header')
            else:
                length = int(length)
            # if delta > 0: - i think this isn't needed
            body = self.recv(length)

        self.finnish() # close connection or free it for future requests

        if GZIP and self.head.get('Content-Encoding') == 'gzip':
            compressedstream = StringIO.StringIO(body)
            gzipper   = gzip.GzipFile(fileobj = compressedstream)
            body = gzipper.read()
        return body

    def get_redirection(self):
        if self.redirection:
            return self.redirection
        return ''

    def __del__(self):
        # TODO - look if this is realy ok.. for instance someone could request just the header and ignore the body part
        # i don't know what happens if the next download will reuse this connection, where the body-part is still in
        # -> i tested it with google and it seems ok
        if self.keepalive:
            if self.host in http.conns and http.conns[self.host][1] != C_OPEN:
                if self.log:
                    self.log.debug('creating a dirty connection')
                self.finnish()

class header(object):
    def __init__(self, head):
        self.head = head
        # HTTP/1.0 200 OK
        self.version = head[5:8] # 1.0
        self.status  = int(head[9:12]) # 200
        self.cached = {}
    def get(self, what):
        if what not in self.cached:
            self.cached[what] = textextract(self.head, what + ': ', '\r\n')
        return self.cached[what]
    def plain(self):
        return self.head

if __name__ == '__main__':
    def tick_time(t):
        for i in xrange(0, t):
            time.sleep(1)
            print i

    a = http('http://dtwow.eu')
    a.header.append('User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008062417 (Gentoo) Iceweasel/3.0.1')
    a.header.append('Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
    a.header.append('Accept-Language: en-us,en;q=0.5')
    a.header.append('Accept-Charset: utf-8,ISO-8859-1;q=0.7,*;q=0.7')
    a.open()
    a.get()
    #print a.head.plain()
    #print a.get()
    tick_time(270)
    a = http('http://dtwow.eu')
    a.header.append('User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008062417 (Gentoo) Iceweasel/3.0.1')
    a.header.append('Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
    a.header.append('Accept-Language: en-us,en;q=0.5')
    a.header.append('Accept-Charset: utf-8,ISO-8859-1;q=0.7,*;q=0.7')
    a.open()
    print a.get()
<commit_msg>added dns_cache to http-class<commit_after># vim: set fileencoding=utf-8 :

import sys, socket, time
from helper import *
import config

GZIP = True
try:
    import StringIO, gzip
except:
    GZIP = False

if 'MSG_WAITALL' in socket.__dict__:
    EASY_RECV = True
else:
    EASY_RECV = False

def extract_host_page_port(url):
    ''' returns tuple (host, page, port) '''
    page = ''
    if url.startswith('http://'): # we don't need this
        url = url[7:]
    p = url.find(':')   # port
    br = url.find('/')  # get request
    if br == -1:
        br = url.find('?') # get request 2
    if br == -1:
        host = url
        page = '/'
        br = 999999
    else:
        host = url[:br]
        page = url[br:]
    port = 80
    if p != -1:
        port = int(url[p+1:br])
        host = url[:p]
    return (host, page, port)

C_OPEN   = 1
C_IN_USE = 2
class http(object):
    conns = {} # this will store all keep-alive connections in form (host, state)
    dns_cache = {} # will translate host to ip ... 'dns_name.org': (ip, timestamp)

    def __init__(self, url, log = None):
        self.host, self.page, self.port = extract_host_page_port(url)
        self.request = {}
        self.request['http_version'] = '1.1'
        self.request['method']       = 'GET'
        self.request['header']       = [] # can be set from outside
        if GZIP:
            self.request['header'].append('Accept-Encoding: gzip')
        self.log = log
        self.redirection = ''

    @classmethod
    def get_ip(cls, host, force = False):
        if force or host not in cls.dns_cache:
            ip = socket.gethostbyname(host)
            cls.dns_cache[host] = (ip, time.time())
        else:
            ip, last_update = cls.dns_cache[host]
            if last_update < time.time() + (60 * 60 * 8): # after 8h
                ip = cls.get_ip(host, True)
        return ip

    def connect(self, force = False):
        if self.request['http_version'] == '1.1' and config.keepalive:
            self.keepalive = True
        else:
            self.keepalive = False
        if self.keepalive and self.host in http.conns and not force:
            if http.conns[self.host][0] == C_OPEN:
                return http.conns[self.host][1]
        c = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            self.ip = http.get_ip(self.host)
            c.connect((self.ip, self.port))
        except socket.gaierror, (e, txt):
            # socket.gaierror: (-2, 'Name or service not known')
            if self.log:
                self.log.bug('error in connect to %s:%d errorcode:%d and %s' % (self.host, self.port, e, txt.decode('utf-8')))
            if self.host in http.conns:
                del http.conns[self.host]
        else:
            if self.keepalive:
                http.conns[self.host] = [c, C_IN_USE]
        return c

    def open(self, post = ''):
        self.c = self.connect()
        header = []
        if post:
            self.request['method'] = 'POST'
        header.append('%s %s HTTP/%s' % (self.request['method'], self.page, self.request['http_version']))
        header.append('HOST: %s' % self.host)
        for i in self.request['header']:
            header.append(i)
        if post:
            header.append('Content-Type: application/x-www-form-urlencoded')
            header.append('Content-Length: %d' % len(post))
            header.append('\r\n%s' % post)
        send = '\r\n'.join(header)
        if not post:
            send += '\r\n\r\n'
        self.c.sendall(send)
        self.get_head()

    def recv(self, size = 4096, precision = False):
        ''' a blocking recv function - which should also work on windows and solaris
            this is the lowest level of recv, which i can call from this class '''
        data = ''
        if self.buf:
            data = self.buf[:size]
            self.buf = self.buf[size:]
            size -= len(data)
            if size == 0:
                return data
        if EASY_RECV:
            data += self.c.recv(size, socket.MSG_WAITALL)
        else:
            while size > 0:
                chunk = self.c.recv(size)
                if chunk == '':
                    break
                data += chunk
                size -= len(chunk)
        if precision:
            self.buf = data[size:]
            return data[:size]
        return data

    def recv_with_reconnect(self, size = 4096):
        ''' a wrapper around the socketrecv to allow reconnect on closed sockets '''
        try:
            return self.c.recv(size)
        except socket.error, (e, err):
            # error: (104, 'Die Verbindung wurde vom Kommunikationspartner zur\xc3\xbcckgeset
            # gaierror: (-2,eerror: (104, 'Die Verbindung wurde vom Kommunikationspartner zur\xc3\xbcckgesetzt')
            if e == 104:
                self.c = self.connect(True)
                return self.c.recv(size)
            else:
                if self.host in http.conns:
                    del http.conns[self.host] # we have a strange error here, so we just delete this host - cause it will surely produce more errors
                self.log.bug('crecv has a problem with %d, %d, %s' % (e, err.eerror[0], err.eerror[1]))

    def get_chunks(self):
        ''' getting chunks - through some strange implementation, i will first recv everything and then just strip off the chunk-informations '''
        # TODO implement it better - currently it is quite slow
        body = self.buf
        # first we download the whole file
        while True:
            if body.endswith('\n0\r\n\r\n'):
                break
            body += self.c.recv(4096)
        body = body[:-5]

        # after that we create a new return string and eliminate all chunk-trash
        x = body.find('\r\n')
        body2 = ''
        while x > 0:
            length = int(body[:x], 16)
            body2 += body[(x + 2):(x + 2 + length)]
            body = body[x + 4 + length:]
            if not body:
                return body2
            x = body.find('\r\n')
        # self.log.bug('strange chunked response')
        return ''

    def get_head(self):
        ''' just get the answering head - we need at least this, to receive the body (else we won't know if the body is chunked and so on)
        also returns all already gathered pieces of the body '''
        self.buf = None # reset it first (important)
        self.buf = self.recv_with_reconnect()
        x = self.buf.find('\r\n\r\n')
        while x == -1:
            self.buf += self.recv()
            x = self.buf.find('\r\n\r\n')
        self.head = header(self.buf[:x+2]) # keep the \r\n at the end, so we can search easier
        self.buf = self.buf[x+4:]
        if self.head.status == 301 or self.head.status == 302 or self.head.status == 303: # 302 == found, 303 == see other
            self.redirection = self.head.get('Location')
            self.host, self.page, self.port = extract_host_page_port(self.redirection)
            self.open()

    def finnish(self):
        ''' when a download gets ended, this function will mark the connection as free for future requests '''
        if self.keepalive:
            http.conns[self.host] = (self.c, C_OPEN)
        else:
            self.c.close()

    def get(self):
        if self.head.get('Transfer-Encoding') == 'chunked':
            body = self.get_chunks()
        else:
            length = self.head.get('Content-Length')
            if not length:
                length = 9999999 # very big - to make sure we download everything
                if self.log:
                    self.log.warning('there was no content length in the header')
            else:
                length = int(length)
            # if delta > 0: - i think this isn't needed
            body = self.recv(length)

        self.finnish() # close connection or free it for future requests

        if GZIP and self.head.get('Content-Encoding') == 'gzip':
            compressedstream = StringIO.StringIO(body)
            gzipper   = gzip.GzipFile(fileobj = compressedstream)
            body = gzipper.read()
        return body

    def get_redirection(self):
        if self.redirection:
            return self.redirection
        return ''

    def __del__(self):
        # TODO - look if this is realy ok.. for instance someone could request just the header and ignore the body part
        # i don't know what happens if the next download will reuse this connection, where the body-part is still in
        # -> i tested it with google and it seems ok
        if self.keepalive:
            if self.host in http.conns and http.conns[self.host][1] != C_OPEN:
                if self.log:
                    self.log.debug('creating a dirty connection')
                self.finnish()

class header(object):
    def __init__(self, head):
        self.head = head
        # HTTP/1.0 200 OK
        self.version = head[5:8] # 1.0
        self.status  = int(head[9:12]) # 200
        self.cached = {}
    def get(self, what):
        if what not in self.cached:
            self.cached[what] = textextract(self.head, what + ': ', '\r\n')
        return self.cached[what]
    def plain(self):
        return self.head

if __name__ == '__main__':
    def tick_time(t):
        for i in xrange(0, t):
            time.sleep(1)
            print i

    a = http('http://dtwow.eu')
    a.header.append('User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008062417 (Gentoo) Iceweasel/3.0.1')
    a.header.append('Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
    a.header.append('Accept-Language: en-us,en;q=0.5')
    a.header.append('Accept-Charset: utf-8,ISO-8859-1;q=0.7,*;q=0.7')
    a.open()
    a.get()
    #print a.head.plain()
    #print a.get()
    tick_time(270)
    a = http('http://dtwow.eu')
    a.header.append('User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008062417 (Gentoo) Iceweasel/3.0.1')
    a.header.append('Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
    a.header.append('Accept-Language: en-us,en;q=0.5')
    a.header.append('Accept-Charset: utf-8,ISO-8859-1;q=0.7,*;q=0.7')
    a.open()
    print a.get()
<|endoftext|>


<commit_before>import sqlite3
import os, glob, re

try:
    from lsst.pipe.analysis.utils import Filenamer
except ImportError:
    logging.warning('Pipe analysis not available.')

def get_visits_sql(field, tract, filt, sqlitedir='/scratch/hchiang2/parejko/'):
    with sqlite3.connect(os.path.join(sqlitedir, 'db{}.sqlite3'.format(field))) as conn:
        cursor = conn.cursor()
        cmd = "select distinct visit from calexp where tract=:tract and filter=:filt"
        cursor.execute(cmd, dict(tract=tract, filt=filt))
        result = cursor.fetchall()
    return [x[0] for x in result]
    # return '^'.join(str(x[0]) for x in result)

def result(df):
    """Returns in-memory dataframe or series, getting result if necessary from dask
    """
    if hasattr(df, 'compute'):
        return df.compute()
    elif hasattr(df, 'result'):
        return df.result()
    else:
        return df

def get_tracts(butler):
    """Reads tracts for which plots are available.

    Decently hack-y pseudo-butler activity here.
    """
    dataId = {'tract':0, 'filter':'HSC-I'}
    filenamer = Filenamer(butler, 'plotCoadd', dataId)

    fake_filename = butler.get(filenamer.dataset + '_filename', dataId, description='foo', style='bar')[0]
    m = re.search('(.+/plots)/.+', fake_filename)
    plot_rootdir = m.group(1)
    filters = os.listdir(plot_rootdir)
    tracts = []
    for f in filters:
        dirs = os.listdir(os.path.join(plot_rootdir, f))
        for d in dirs:
            files = glob.glob('{}/{}/{}/*.png'.format(plot_rootdir,f,d))
            files += glob.glob('{}/{}/{}/*.parq'.format(plot_rootdir,f,d))
            if len() > 0:
                tracts.append(d)
    tracts = list(set([int(t.replace('tract-', '')) for t in tracts]))
    tracts.sort()
    return tracts

def get_visits(butler, tract, filt):
    """Returns visits for which plots exist, for given tract and filt
    """
    dataId = {'tract':tract, 'filter':filt}
    filenamer = Filenamer(butler, 'plotCoadd', dataId)

    fake_filename = butler.get(filenamer.dataset + '_filename', dataId, description='foo', style='bar')[0]
    tract_dir = os.path.dirname(fake_filename)
    visit_dirs = glob.glob(os.path.join(tract_dir, 'visit*'))
    visits = [int(re.search('visit-(\d+)', d).group(1)) for d in visit_dirs if len(os.listdir(d)) > 0]
    visits.sort()
    return visits<commit_msg>whoops<commit_after>import sqlite3
import os, glob, re

try:
    from lsst.pipe.analysis.utils import Filenamer
except ImportError:
    logging.warning('Pipe analysis not available.')

def get_visits_sql(field, tract, filt, sqlitedir='/scratch/hchiang2/parejko/'):
    with sqlite3.connect(os.path.join(sqlitedir, 'db{}.sqlite3'.format(field))) as conn:
        cursor = conn.cursor()
        cmd = "select distinct visit from calexp where tract=:tract and filter=:filt"
        cursor.execute(cmd, dict(tract=tract, filt=filt))
        result = cursor.fetchall()
    return [x[0] for x in result]
    # return '^'.join(str(x[0]) for x in result)

def result(df):
    """Returns in-memory dataframe or series, getting result if necessary from dask
    """
    if hasattr(df, 'compute'):
        return df.compute()
    elif hasattr(df, 'result'):
        return df.result()
    else:
        return df

def get_tracts(butler):
    """Reads tracts for which plots are available.

    Decently hack-y pseudo-butler activity here.
    """
    dataId = {'tract':0, 'filter':'HSC-I'}
    filenamer = Filenamer(butler, 'plotCoadd', dataId)

    fake_filename = butler.get(filenamer.dataset + '_filename', dataId, description='foo', style='bar')[0]
    m = re.search('(.+/plots)/.+', fake_filename)
    plot_rootdir = m.group(1)
    filters = os.listdir(plot_rootdir)
    tracts = []
    for f in filters:
        dirs = os.listdir(os.path.join(plot_rootdir, f))
        for d in dirs:
            files = glob.glob('{}/{}/{}/*.png'.format(plot_rootdir,f,d))
            files += glob.glob('{}/{}/{}/*.parq'.format(plot_rootdir,f,d))
            if len(files) > 0:
                tracts.append(d)
    tracts = list(set([int(t.replace('tract-', '')) for t in tracts]))
    tracts.sort()
    return tracts

def get_visits(butler, tract, filt):
    """Returns visits for which plots exist, for given tract and filt
    """
    dataId = {'tract':tract, 'filter':filt}
    filenamer = Filenamer(butler, 'plotCoadd', dataId)

    fake_filename = butler.get(filenamer.dataset + '_filename', dataId, description='foo', style='bar')[0]
    tract_dir = os.path.dirname(fake_filename)
    visit_dirs = glob.glob(os.path.join(tract_dir, 'visit*'))
    visits = [int(re.search('visit-(\d+)', d).group(1)) for d in visit_dirs if len(os.listdir(d)) > 0]
    visits.sort()
    return visits<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright: (c) 2008, Jarek Zgoda <jarek.zgoda@gmail.com>

__revision__ = '$Id: models.py 28 2009-10-22 15:03:02Z jarek.zgoda $'

import datetime

from django.db import models
from django.core.urlresolvers import reverse
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericForeignKey
from django.http import HttpRequest, HttpResponse
from django.shortcuts import render
from django.utils.timezone import now as timezone_now

from zerver.lib.send_email import send_email
from zerver.lib.utils import generate_random_token
from zerver.models import PreregistrationUser, EmailChangeStatus, MultiuseInvite, UserProfile
from random import SystemRandom
import string
from typing import Any, Dict, Optional, Text, Union

class ConfirmationKeyException(Exception):
    WRONG_LENGTH = 1
    EXPIRED = 2
    DOES_NOT_EXIST = 3

    def __init__(self, error_type: int) -> None:
        super().__init__()
        self.error_type = error_type

def render_confirmation_key_error(request: HttpRequest, exception: ConfirmationKeyException) -> HttpResponse:
    if exception.error_type == ConfirmationKeyException.WRONG_LENGTH:
        return render(request, 'confirmation/link_malformed.html')
    if exception.error_type == ConfirmationKeyException.EXPIRED:
        return render(request, 'confirmation/link_expired.html')
    return render(request, 'confirmation/link_does_not_exist.html')

def generate_key() -> str:
    generator = SystemRandom()
    # 24 characters * 5 bits of entropy/character = 120 bits of entropy
    return ''.join(generator.choice(string.ascii_lowercase + string.digits) for _ in range(24))

ConfirmationObjT = Union[MultiuseInvite, PreregistrationUser, EmailChangeStatus]
def get_object_from_key(confirmation_key: str,
                        confirmation_type: int) -> ConfirmationObjT:
    # Confirmation keys used to be 40 characters
    if len(confirmation_key) not in (24, 40):
        raise ConfirmationKeyException(ConfirmationKeyException.WRONG_LENGTH)
    try:
        confirmation = Confirmation.objects.get(confirmation_key=confirmation_key,
                                                type=confirmation_type)
    except Confirmation.DoesNotExist:
        raise ConfirmationKeyException(ConfirmationKeyException.DOES_NOT_EXIST)

    time_elapsed = timezone_now() - confirmation.date_sent
    if time_elapsed.total_seconds() > _properties[confirmation.type].validity_in_days * 24 * 3600:
        raise ConfirmationKeyException(ConfirmationKeyException.EXPIRED)

    obj = confirmation.content_object
    if hasattr(obj, "status"):
        obj.status = getattr(settings, 'STATUS_ACTIVE', 1)
        obj.save(update_fields=['status'])
    return obj

def create_confirmation_link(obj: Union[ContentType, int], host: str,
                             confirmation_type: int,
                             url_args: Optional[Dict[str, str]]=None) -> str:
    key = generate_key()
    Confirmation.objects.create(content_object=obj, date_sent=timezone_now(), confirmation_key=key,
                                type=confirmation_type)
    return confirmation_url(key, host, confirmation_type, url_args)

def confirmation_url(confirmation_key: str, host: str,
                     confirmation_type: int,
                     url_args: Optional[Dict[str, str]]=None) -> str:
    if url_args is None:
        url_args = {}
    url_args['confirmation_key'] = confirmation_key
    return '%s%s%s' % (settings.EXTERNAL_URI_SCHEME, host,
                       reverse(_properties[confirmation_type].url_name, kwargs=url_args))

class Confirmation(models.Model):
    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField()  # type: int
    content_object = GenericForeignKey('content_type', 'object_id')
    date_sent = models.DateTimeField()  # type: datetime.datetime
    confirmation_key = models.CharField(max_length=40)  # type: str

    # The following list is the set of valid types
    USER_REGISTRATION = 1
    INVITATION = 2
    EMAIL_CHANGE = 3
    UNSUBSCRIBE = 4
    SERVER_REGISTRATION = 5
    MULTIUSE_INVITE = 6
    type = models.PositiveSmallIntegerField()  # type: int

    def __str__(self) -> Text:
        return '<Confirmation: %s>' % (self.content_object,)

class ConfirmationType:
    def __init__(self, url_name: str,
                 validity_in_days: int=settings.CONFIRMATION_LINK_DEFAULT_VALIDITY_DAYS) -> None:
        self.url_name = url_name
        self.validity_in_days = validity_in_days

_properties = {
    Confirmation.USER_REGISTRATION: ConfirmationType('confirmation.views.confirm'),
    Confirmation.INVITATION: ConfirmationType('confirmation.views.confirm',
                                              validity_in_days=settings.INVITATION_LINK_VALIDITY_DAYS),
    Confirmation.EMAIL_CHANGE: ConfirmationType('zerver.views.user_settings.confirm_email_change'),
    Confirmation.UNSUBSCRIBE: ConfirmationType('zerver.views.unsubscribe.email_unsubscribe',
                                               validity_in_days=1000000),  # should never expire
    Confirmation.MULTIUSE_INVITE: ConfirmationType(
        'zerver.views.registration.accounts_home_from_multiuse_invite',
        validity_in_days=settings.INVITATION_LINK_VALIDITY_DAYS)
}

# Confirmation pathways for which there is no content_object that we need to
# keep track of.

def check_key_is_valid(creation_key: Text) -> bool:
    if not RealmCreationKey.objects.filter(creation_key=creation_key).exists():
        return False
    days_sofar = (timezone_now() - RealmCreationKey.objects.get(creation_key=creation_key).date_created).days
    # Realm creation link expires after settings.REALM_CREATION_LINK_VALIDITY_DAYS
    if days_sofar <= settings.REALM_CREATION_LINK_VALIDITY_DAYS:
        return True
    return False

def generate_realm_creation_url() -> Text:
    key = generate_key()
    RealmCreationKey.objects.create(creation_key=key, date_created=timezone_now())
    return u'%s%s%s' % (settings.EXTERNAL_URI_SCHEME,
                        settings.EXTERNAL_HOST,
                        reverse('zerver.views.create_realm',
                                kwargs={'creation_key': key}))

class RealmCreationKey(models.Model):
    creation_key = models.CharField('activation key', max_length=40)
    date_created = models.DateTimeField('created', default=timezone_now)
<commit_msg>confirmation: Fix time enforcement in check_key_is_valid.<commit_after># -*- coding: utf-8 -*-

# Copyright: (c) 2008, Jarek Zgoda <jarek.zgoda@gmail.com>

__revision__ = '$Id: models.py 28 2009-10-22 15:03:02Z jarek.zgoda $'

import datetime

from django.db import models
from django.core.urlresolvers import reverse
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericForeignKey
from django.http import HttpRequest, HttpResponse
from django.shortcuts import render
from django.utils.timezone import now as timezone_now

from zerver.lib.send_email import send_email
from zerver.lib.utils import generate_random_token
from zerver.models import PreregistrationUser, EmailChangeStatus, MultiuseInvite, UserProfile
from random import SystemRandom
import string
from typing import Any, Dict, Optional, Text, Union

class ConfirmationKeyException(Exception):
    WRONG_LENGTH = 1
    EXPIRED = 2
    DOES_NOT_EXIST = 3

    def __init__(self, error_type: int) -> None:
        super().__init__()
        self.error_type = error_type

def render_confirmation_key_error(request: HttpRequest, exception: ConfirmationKeyException) -> HttpResponse:
    if exception.error_type == ConfirmationKeyException.WRONG_LENGTH:
        return render(request, 'confirmation/link_malformed.html')
    if exception.error_type == ConfirmationKeyException.EXPIRED:
        return render(request, 'confirmation/link_expired.html')
    return render(request, 'confirmation/link_does_not_exist.html')

def generate_key() -> str:
    generator = SystemRandom()
    # 24 characters * 5 bits of entropy/character = 120 bits of entropy
    return ''.join(generator.choice(string.ascii_lowercase + string.digits) for _ in range(24))

ConfirmationObjT = Union[MultiuseInvite, PreregistrationUser, EmailChangeStatus]
def get_object_from_key(confirmation_key: str,
                        confirmation_type: int) -> ConfirmationObjT:
    # Confirmation keys used to be 40 characters
    if len(confirmation_key) not in (24, 40):
        raise ConfirmationKeyException(ConfirmationKeyException.WRONG_LENGTH)
    try:
        confirmation = Confirmation.objects.get(confirmation_key=confirmation_key,
                                                type=confirmation_type)
    except Confirmation.DoesNotExist:
        raise ConfirmationKeyException(ConfirmationKeyException.DOES_NOT_EXIST)

    time_elapsed = timezone_now() - confirmation.date_sent
    if time_elapsed.total_seconds() > _properties[confirmation.type].validity_in_days * 24 * 3600:
        raise ConfirmationKeyException(ConfirmationKeyException.EXPIRED)

    obj = confirmation.content_object
    if hasattr(obj, "status"):
        obj.status = getattr(settings, 'STATUS_ACTIVE', 1)
        obj.save(update_fields=['status'])
    return obj

def create_confirmation_link(obj: Union[ContentType, int], host: str,
                             confirmation_type: int,
                             url_args: Optional[Dict[str, str]]=None) -> str:
    key = generate_key()
    Confirmation.objects.create(content_object=obj, date_sent=timezone_now(), confirmation_key=key,
                                type=confirmation_type)
    return confirmation_url(key, host, confirmation_type, url_args)

def confirmation_url(confirmation_key: str, host: str,
                     confirmation_type: int,
                     url_args: Optional[Dict[str, str]]=None) -> str:
    if url_args is None:
        url_args = {}
    url_args['confirmation_key'] = confirmation_key
    return '%s%s%s' % (settings.EXTERNAL_URI_SCHEME, host,
                       reverse(_properties[confirmation_type].url_name, kwargs=url_args))

class Confirmation(models.Model):
    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField()  # type: int
    content_object = GenericForeignKey('content_type', 'object_id')
    date_sent = models.DateTimeField()  # type: datetime.datetime
    confirmation_key = models.CharField(max_length=40)  # type: str

    # The following list is the set of valid types
    USER_REGISTRATION = 1
    INVITATION = 2
    EMAIL_CHANGE = 3
    UNSUBSCRIBE = 4
    SERVER_REGISTRATION = 5
    MULTIUSE_INVITE = 6
    type = models.PositiveSmallIntegerField()  # type: int

    def __str__(self) -> Text:
        return '<Confirmation: %s>' % (self.content_object,)

class ConfirmationType:
    def __init__(self, url_name: str,
                 validity_in_days: int=settings.CONFIRMATION_LINK_DEFAULT_VALIDITY_DAYS) -> None:
        self.url_name = url_name
        self.validity_in_days = validity_in_days

_properties = {
    Confirmation.USER_REGISTRATION: ConfirmationType('confirmation.views.confirm'),
    Confirmation.INVITATION: ConfirmationType('confirmation.views.confirm',
                                              validity_in_days=settings.INVITATION_LINK_VALIDITY_DAYS),
    Confirmation.EMAIL_CHANGE: ConfirmationType('zerver.views.user_settings.confirm_email_change'),
    Confirmation.UNSUBSCRIBE: ConfirmationType('zerver.views.unsubscribe.email_unsubscribe',
                                               validity_in_days=1000000),  # should never expire
    Confirmation.MULTIUSE_INVITE: ConfirmationType(
        'zerver.views.registration.accounts_home_from_multiuse_invite',
        validity_in_days=settings.INVITATION_LINK_VALIDITY_DAYS)
}

# Confirmation pathways for which there is no content_object that we need to
# keep track of.

def check_key_is_valid(creation_key: Text) -> bool:
    if not RealmCreationKey.objects.filter(creation_key=creation_key).exists():
        return False
    time_elapsed = timezone_now() - RealmCreationKey.objects.get(creation_key=creation_key).date_created
    if time_elapsed.total_seconds() > settings.REALM_CREATION_LINK_VALIDITY_DAYS * 24 * 3600:
        return False
    return True

def generate_realm_creation_url() -> Text:
    key = generate_key()
    RealmCreationKey.objects.create(creation_key=key, date_created=timezone_now())
    return u'%s%s%s' % (settings.EXTERNAL_URI_SCHEME,
                        settings.EXTERNAL_HOST,
                        reverse('zerver.views.create_realm',
                                kwargs={'creation_key': key}))

class RealmCreationKey(models.Model):
    creation_key = models.CharField('activation key', max_length=40)
    date_created = models.DateTimeField('created', default=timezone_now)
<|endoftext|>


<commit_before>from django.http.response import HttpResponseRedirect
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.views.generic.base import View
from django.views.generic import ListView
from django.views.generic.edit import CreateView, DeleteView
from django.views.decorators.http import require_POST, require_GET
from django.utils.decorators import method_decorator
from django.core.urlresolvers import reverse_lazy
from django.forms.models import model_to_dict
from django.http import HttpResponse

from explorer.models import Query, QueryLog
from explorer import app_settings
from explorer.forms import QueryForm
from explorer.utils import url_get_rows,\
    url_get_query_id,\
    url_get_log_id,\
    schema_info,\
    url_get_params,\
    safe_admin_login_prompt,\
    build_download_response,\
    build_stream_response,\
    user_can_see_query,\
    fmt_sql

try:
    from collections import Counter
except:
    from counter import Counter

import re
import json
from functools import wraps

def view_permission(f):
    @wraps(f)
    def wrap(request, *args, **kwargs):
        if not app_settings.EXPLORER_PERMISSION_VIEW(request.user)\
                and not user_can_see_query(request, kwargs)\
                and not (app_settings.EXPLORER_TOKEN_AUTH_ENABLED()
                         and request.META.get('HTTP_X_API_TOKEN') == app_settings.EXPLORER_TOKEN):
            return safe_admin_login_prompt(request)
        return f(request, *args, **kwargs)
    return wrap

def change_permission(f):
    @wraps(f)
    def wrap(request, *args, **kwargs):
        if not app_settings.EXPLORER_PERMISSION_CHANGE(request.user):
            return safe_admin_login_prompt(request)
        return f(request, *args, **kwargs)
    return wrap

class ExplorerContextMixin(object):

    def gen_ctx(self):
        return {'can_view': app_settings.EXPLORER_PERMISSION_VIEW(self.request.user),
                'can_change': app_settings.EXPLORER_PERMISSION_CHANGE(self.request.user)}

    def get_context_data(self, **kwargs):
        ctx = super(ExplorerContextMixin, self).get_context_data(**kwargs)
        ctx.update(self.gen_ctx())
        return ctx

    def render_template(self, template, ctx):
        ctx.update(self.gen_ctx())
        return render_to_response(template, ctx)

@view_permission
@require_GET
def download_query(request, query_id):
    return _csv_response(request, query_id, False)

@view_permission
@require_GET
def view_csv_query(request, query_id):
    return _csv_response(request, query_id, True)

def _csv_response(request, query_id, stream=False):
    query = get_object_or_404(Query, pk=query_id)
    query.params = url_get_params(request)
    return build_stream_response(query) if stream else build_download_response(query)

@change_permission
@require_POST
def download_csv_from_sql(request):
    sql = request.POST.get('sql')
    return build_download_response(Query(sql=sql, title="Playground", params=url_get_params(request)))

@change_permission
@require_GET
def schema(request):
    return render_to_response('explorer/schema.html', {'schema': schema_info()})

@require_POST
def format_sql(request):
    sql = request.POST.get('sql', '')
    formatted = fmt_sql(sql)
    return HttpResponse(json.dumps({"formatted": formatted}), content_type="application/json")

class ListQueryView(ExplorerContextMixin, ListView):

    @method_decorator(view_permission)
    def dispatch(self, *args, **kwargs):
        return super(ListQueryView, self).dispatch(*args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(ListQueryView, self).get_context_data(**kwargs)
        context['object_list'] = self._build_queries_and_headers()
        context['recent_queries'] = Query.objects.all().order_by('-last_run_date')[:app_settings.EXPLORER_RECENT_QUERY_COUNT]
        return context

    def _build_queries_and_headers(self):
        """
        Build a list of query information and headers (pseudo-folders) for consumption by the template.

        Strategy: Look for queries with titles of the form "something - else" (eg. with a ' - ' in the middle)
                  and split on the ' - ', treating the left side as a "header" (or folder). Interleave the
                  headers into the ListView's object_list as appropriate. Ignore headers that only have one
                  child. The front end uses bootstrap's JS Collapse plugin, which necessitates generating CSS
                  classes to map the header onto the child rows, hence the collapse_target variable.

                  To make the return object homogeneous, convert the object_list models into dictionaries for
                  interleaving with the header "objects". This necessitates special handling of 'created_at'
                  and 'created_by_user' because model_to_dict doesn't include non-editable fields (created_at)
                  and will give the int representation of the user instead of the string representation.

        :return: A list of model dictionaries representing all the query objects, interleaved with header dictionaries.
        """

        dict_list = []
        rendered_headers = []
        pattern = re.compile('[\W_]+')

        headers = Counter([q.title.split(' - ')[0] for q in self.object_list])

        for q in self.object_list:
            model_dict = model_to_dict(q)
            header = q.title.split(' - ')[0]
            collapse_target = pattern.sub('', header)

            if headers[header] > 1 and header not in rendered_headers:
                dict_list.append({'title': header,
                                  'is_header': True,
                                  'collapse_target': collapse_target,
                                  'count': headers[header]})
                rendered_headers.append(header)

            model_dict.update({'is_in_category': headers[header] > 1,
                               'collapse_target': collapse_target,
                               'created_at': q.created_at,
                               'created_by_user': q.created_by_user.__unicode__ if q.created_by_user else None})
            dict_list.append(model_dict)
        return dict_list

    model = Query

class ListQueryLogView(ExplorerContextMixin, ListView):

    @method_decorator(view_permission)
    def dispatch(self, *args, **kwargs):
        return super(ListQueryLogView, self).dispatch(*args, **kwargs)

    context_object_name = "recent_logs"
    model = QueryLog
    paginate_by = 20

class CreateQueryView(ExplorerContextMixin, CreateView):

    @method_decorator(change_permission)
    def dispatch(self, *args, **kwargs):
        return super(CreateQueryView, self).dispatch(*args, **kwargs)

    def form_valid(self, form):
        form.instance.created_by_user = self.request.user
        return super(CreateQueryView, self).form_valid(form)

    form_class = QueryForm
    template_name = 'explorer/query.html'

class DeleteQueryView(ExplorerContextMixin, DeleteView):

    @method_decorator(change_permission)
    def dispatch(self, *args, **kwargs):
        return super(DeleteQueryView, self).dispatch(*args, **kwargs)

    model = Query
    success_url = reverse_lazy("explorer_index")

class PlayQueryView(ExplorerContextMixin, View):

    @method_decorator(change_permission)
    def dispatch(self, *args, **kwargs):
        return super(PlayQueryView, self).dispatch(*args, **kwargs)

    def get(self, request):
        if url_get_query_id(request):
            query = get_object_or_404(Query, pk=url_get_query_id(request))
            return self.render_with_sql(request, query)

        if url_get_log_id(request):
            log = get_object_or_404(QueryLog, pk=url_get_log_id(request))
            query = Query(sql=log.sql, title="Playground")
            return self.render_with_sql(request, query)

        return self.render(request)

    def post(self, request):
        sql = request.POST.get('sql')
        query = Query(sql=sql, title="Playground")
        query.log(request.user)
        return self.render_with_sql(request, query)

    def render(self, request):
        return self.render_template('explorer/play.html', RequestContext(request, {'title': 'Playground'}))

    def render_with_sql(self, request, query):
        return self.render_template('explorer/play.html', query_viewmodel(request, query, title="Playground"))

class QueryView(ExplorerContextMixin, View):

    @method_decorator(view_permission)
    def dispatch(self, *args, **kwargs):
        return super(QueryView, self).dispatch(*args, **kwargs)

    def get(self, request, query_id):
        query, form = QueryView.get_instance_and_form(request, query_id)
        query.save()  # updates the modified date
        vm = query_viewmodel(request, query, form=form)
        return self.render_template('explorer/query.html', vm)

    def post(self, request, query_id):
        if not app_settings.EXPLORER_PERMISSION_CHANGE(request.user):
            return HttpResponseRedirect(
                reverse_lazy('query_detail', kwargs={'query_id': query_id})
            )

        query, form = QueryView.get_instance_and_form(request, query_id)
        success = form.is_valid() and form.save()
        if form.has_changed():
            query.log(request.user)
        vm = query_viewmodel(request, query, form=form, message="Query saved." if success else None)
        return self.render_template('explorer/query.html', vm)

    @staticmethod
    def get_instance_and_form(request, query_id):
        query = get_object_or_404(Query, pk=query_id)
        query.params = url_get_params(request)
        form = QueryForm(request.POST if len(request.POST) else None, instance=query)
        return query, form

def query_viewmodel(request, query, title=None, form=None, message=None):
    rows = url_get_rows(request)
    res = query.headers_and_data()
    return RequestContext(request, {
            'error': res.error,
            'params': query.available_params(),
            'title': title,
            'query': query,
            'form': form,
            'message': message,
            'data': res.data[:rows],
            'headers': res.headers,
            'duration': res.duration,
            'rows': rows,
            'total_rows': len(res.data),
            'dataUrl': reverse_lazy('query_csv', kwargs={'query_id': query.id}) if query.id else ''})
<commit_msg>reducing sql calls when rendering list<commit_after>from django.http.response import HttpResponseRedirect
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.views.generic.base import View
from django.views.generic import ListView
from django.views.generic.edit import CreateView, DeleteView
from django.views.decorators.http import require_POST, require_GET
from django.utils.decorators import method_decorator
from django.core.urlresolvers import reverse_lazy
from django.forms.models import model_to_dict
from django.http import HttpResponse

from explorer.models import Query, QueryLog
from explorer import app_settings
from explorer.forms import QueryForm
from explorer.utils import url_get_rows,\
    url_get_query_id,\
    url_get_log_id,\
    schema_info,\
    url_get_params,\
    safe_admin_login_prompt,\
    build_download_response,\
    build_stream_response,\
    user_can_see_query,\
    fmt_sql

try:
    from collections import Counter
except:
    from counter import Counter

import re
import json
from functools import wraps

def view_permission(f):
    @wraps(f)
    def wrap(request, *args, **kwargs):
        if not app_settings.EXPLORER_PERMISSION_VIEW(request.user)\
                and not user_can_see_query(request, kwargs)\
                and not (app_settings.EXPLORER_TOKEN_AUTH_ENABLED()
                         and request.META.get('HTTP_X_API_TOKEN') == app_settings.EXPLORER_TOKEN):
            return safe_admin_login_prompt(request)
        return f(request, *args, **kwargs)
    return wrap

def change_permission(f):
    @wraps(f)
    def wrap(request, *args, **kwargs):
        if not app_settings.EXPLORER_PERMISSION_CHANGE(request.user):
            return safe_admin_login_prompt(request)
        return f(request, *args, **kwargs)
    return wrap

class ExplorerContextMixin(object):

    def gen_ctx(self):
        return {'can_view': app_settings.EXPLORER_PERMISSION_VIEW(self.request.user),
                'can_change': app_settings.EXPLORER_PERMISSION_CHANGE(self.request.user)}

    def get_context_data(self, **kwargs):
        ctx = super(ExplorerContextMixin, self).get_context_data(**kwargs)
        ctx.update(self.gen_ctx())
        return ctx

    def render_template(self, template, ctx):
        ctx.update(self.gen_ctx())
        return render_to_response(template, ctx)

@view_permission
@require_GET
def download_query(request, query_id):
    return _csv_response(request, query_id, False)

@view_permission
@require_GET
def view_csv_query(request, query_id):
    return _csv_response(request, query_id, True)

def _csv_response(request, query_id, stream=False):
    query = get_object_or_404(Query, pk=query_id)
    query.params = url_get_params(request)
    return build_stream_response(query) if stream else build_download_response(query)

@change_permission
@require_POST
def download_csv_from_sql(request):
    sql = request.POST.get('sql')
    return build_download_response(Query(sql=sql, title="Playground", params=url_get_params(request)))

@change_permission
@require_GET
def schema(request):
    return render_to_response('explorer/schema.html', {'schema': schema_info()})

@require_POST
def format_sql(request):
    sql = request.POST.get('sql', '')
    formatted = fmt_sql(sql)
    return HttpResponse(json.dumps({"formatted": formatted}), content_type="application/json")

class ListQueryView(ExplorerContextMixin, ListView):

    @method_decorator(view_permission)
    def dispatch(self, *args, **kwargs):
        return super(ListQueryView, self).dispatch(*args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(ListQueryView, self).get_context_data(**kwargs)
        context['object_list'] = self._build_queries_and_headers()
        context['recent_queries'] = Query.objects.all().order_by('-last_run_date')[:app_settings.EXPLORER_RECENT_QUERY_COUNT]
        return context

    def get_queryset(self):
        return Query.objects.prefetch_related('created_by_user').all()

    def _build_queries_and_headers(self):
        """
        Build a list of query information and headers (pseudo-folders) for consumption by the template.

        Strategy: Look for queries with titles of the form "something - else" (eg. with a ' - ' in the middle)
                  and split on the ' - ', treating the left side as a "header" (or folder). Interleave the
                  headers into the ListView's object_list as appropriate. Ignore headers that only have one
                  child. The front end uses bootstrap's JS Collapse plugin, which necessitates generating CSS
                  classes to map the header onto the child rows, hence the collapse_target variable.

                  To make the return object homogeneous, convert the object_list models into dictionaries for
                  interleaving with the header "objects". This necessitates special handling of 'created_at'
                  and 'created_by_user' because model_to_dict doesn't include non-editable fields (created_at)
                  and will give the int representation of the user instead of the string representation.

        :return: A list of model dictionaries representing all the query objects, interleaved with header dictionaries.
        """

        dict_list = []
        rendered_headers = []
        pattern = re.compile('[\W_]+')

        headers = Counter([q.title.split(' - ')[0] for q in self.object_list])

        for q in self.object_list:
            model_dict = model_to_dict(q)
            header = q.title.split(' - ')[0]
            collapse_target = pattern.sub('', header)

            if headers[header] > 1 and header not in rendered_headers:
                dict_list.append({'title': header,
                                  'is_header': True,
                                  'collapse_target': collapse_target,
                                  'count': headers[header]})
                rendered_headers.append(header)

            model_dict.update({'is_in_category': headers[header] > 1,
                               'collapse_target': collapse_target,
                               'created_at': q.created_at,
                               'created_by_user': unicode(q.created_by_user) if q.created_by_user else None})
            dict_list.append(model_dict)
        return dict_list

    model = Query

class ListQueryLogView(ExplorerContextMixin, ListView):

    @method_decorator(view_permission)
    def dispatch(self, *args, **kwargs):
        return super(ListQueryLogView, self).dispatch(*args, **kwargs)

    context_object_name = "recent_logs"
    model = QueryLog
    paginate_by = 20

class CreateQueryView(ExplorerContextMixin, CreateView):

    @method_decorator(change_permission)
    def dispatch(self, *args, **kwargs):
        return super(CreateQueryView, self).dispatch(*args, **kwargs)

    def form_valid(self, form):
        form.instance.created_by_user = self.request.user
        return super(CreateQueryView, self).form_valid(form)

    form_class = QueryForm
    template_name = 'explorer/query.html'

class DeleteQueryView(ExplorerContextMixin, DeleteView):

    @method_decorator(change_permission)
    def dispatch(self, *args, **kwargs):
        return super(DeleteQueryView, self).dispatch(*args, **kwargs)

    model = Query
    success_url = reverse_lazy("explorer_index")

class PlayQueryView(ExplorerContextMixin, View):

    @method_decorator(change_permission)
    def dispatch(self, *args, **kwargs):
        return super(PlayQueryView, self).dispatch(*args, **kwargs)

    def get(self, request):
        if url_get_query_id(request):
            query = get_object_or_404(Query, pk=url_get_query_id(request))
            return self.render_with_sql(request, query)

        if url_get_log_id(request):
            log = get_object_or_404(QueryLog, pk=url_get_log_id(request))
            query = Query(sql=log.sql, title="Playground")
            return self.render_with_sql(request, query)

        return self.render(request)

    def post(self, request):
        sql = request.POST.get('sql')
        query = Query(sql=sql, title="Playground")
        query.log(request.user)
        return self.render_with_sql(request, query)

    def render(self, request):
        return self.render_template('explorer/play.html', RequestContext(request, {'title': 'Playground'}))

    def render_with_sql(self, request, query):
        return self.render_template('explorer/play.html', query_viewmodel(request, query, title="Playground"))

class QueryView(ExplorerContextMixin, View):

    @method_decorator(view_permission)
    def dispatch(self, *args, **kwargs):
        return super(QueryView, self).dispatch(*args, **kwargs)

    def get(self, request, query_id):
        query, form = QueryView.get_instance_and_form(request, query_id)
        query.save()  # updates the modified date
        vm = query_viewmodel(request, query, form=form)
        return self.render_template('explorer/query.html', vm)

    def post(self, request, query_id):
        if not app_settings.EXPLORER_PERMISSION_CHANGE(request.user):
            return HttpResponseRedirect(
                reverse_lazy('query_detail', kwargs={'query_id': query_id})
            )

        query, form = QueryView.get_instance_and_form(request, query_id)
        success = form.is_valid() and form.save()
        if form.has_changed():
            query.log(request.user)
        vm = query_viewmodel(request, query, form=form, message="Query saved." if success else None)
        return self.render_template('explorer/query.html', vm)

    @staticmethod
    def get_instance_and_form(request, query_id):
        query = get_object_or_404(Query, pk=query_id)
        query.params = url_get_params(request)
        form = QueryForm(request.POST if len(request.POST) else None, instance=query)
        return query, form

def query_viewmodel(request, query, title=None, form=None, message=None):
    rows = url_get_rows(request)
    res = query.headers_and_data()
    return RequestContext(request, {
            'error': res.error,
            'params': query.available_params(),
            'title': title,
            'query': query,
            'form': form,
            'message': message,
            'data': res.data[:rows],
            'headers': res.headers,
            'duration': res.duration,
            'rows': rows,
            'total_rows': len(res.data),
            'dataUrl': reverse_lazy('query_csv', kwargs={'query_id': query.id}) if query.id else ''})
<|endoftext|>


<commit_before>"""
Tinman Test Application

"""
__author__ = 'Gavin M. Roy'
__email__ = 'gmr@myyearbook.com'
__since__ = '2011-06-06'

import logging
import tinman
from tornado import web

CONFIG = {'Application': {'debug': True,
                          'xsrf_cookies': False},
         'HTTPServer': {'no_keep_alive': False,
                        'ports': [8000],
                        'xheaders': False},
         'Logging': {'filename': 'log.txt',
                     'format': "%(module)-12s# %(lineno)-5d %(levelname) -10s\
%(asctime)s  %(message)s",
                     'level': logging.DEBUG},
         'Routes': [("/", "tinman.test.DefaultHandler")]}

class DefaultHandler(web.RequestHandler):

    def get(self):

        # Send a JSON string for our test
        self.write({"message": "Hello World",
                    "request": {"method": self.request.method,
                                "protocol": self.request.protocol,
                                "path": self.request.path,
                                "query": self.request.query,
                                "remote_ip": self.request.remote_ip,
                                "version": self.request.version},
                    "tinman": {"version":  "%i.%i.%i" % tinman.__version__}})
<commit_msg>Fix the test apps use of the old __version__ format<commit_after>"""
Tinman Test Application

"""
__author__ = 'Gavin M. Roy'
__email__ = 'gmr@myyearbook.com'
__since__ = '2011-06-06'

import logging
import tinman
from tornado import web

CONFIG = {'Application': {'debug': True,
                          'xsrf_cookies': False},
         'HTTPServer': {'no_keep_alive': False,
                        'ports': [8000],
                        'xheaders': False},
         'Logging': {'filename': 'log.txt',
                     'format': "%(module)-12s# %(lineno)-5d %(levelname) -10s\
%(asctime)s  %(message)s",
                     'level': logging.DEBUG},
         'Routes': [("/", "tinman.test.DefaultHandler")]}

class DefaultHandler(web.RequestHandler):

    def get(self):

        # Send a JSON string for our test
        self.write({"message": "Hello World",
                    "request": {"method": self.request.method,
                                "protocol": self.request.protocol,
                                "path": self.request.path,
                                "query": self.request.query,
                                "remote_ip": self.request.remote_ip,
                                "version": self.request.version},
                    "tinman": {"version":  tinman.__version__}})
<|endoftext|>


<commit_before>#!/usr/bin/python3
#
# Adapted from Apertium
# http://wiki.apertium.org/wiki/Tools_for_TMX
#

from xml.sax import make_parser
from xml.sax.handler import ContentHandler

import sys
import codecs

class TMXHandler(ContentHandler):
    def __init__ (self, slang, tlang, sfile, tfile):
        self.pair = set([slang, tlang])
        self.files = {}
        self.files[slang] = sfile
        self.files[tlang] = tfile
        self.inTag = ''
        self.note = ''
        self.tuid = ''
        self.type = ''
        self.cur_pair = set()
        self.cur_lang = ''
        self.seg = {}
        self.seg[slang] = ''
        self.seg[tlang] = ''

    def startElement(self, name, attrs):

        if name == 'tu':
            self.cur_pair = set()
            self.inTag = 'tu'
            self.tuid = attrs.get('tuid','')
            self.type = attrs.get('datatype','')
        elif name == 'note':
            self.inTag = 'note'
            self.note = ""
        elif name == 'tuv':
            self.inTag = 'tuv'
            self.cur_lang = attrs.get('xml:lang', '')
            self.cur_pair.add(self.cur_lang)
        elif name == 'seg':
            self.inTag = 'seg'
            if self.cur_lang in self.pair:
                self.seg[self.cur_lang] = ''

    def characters (self, c):
        if self.inTag == 'note':
            self.note += c
        elif self.inTag == 'seg' and self.cur_lang in self.pair:
            self.seg[self.cur_lang] += c

    def endElement(self, name):
        if name == 'tu' and self.pair == self.cur_pair:
            for lang in self.cur_pair:
                self.files[lang].write("{}\n".format(self.seg[lang].strip()))

if __name__ == "__main__":
    parser = make_parser()

    if len(sys.argv) < 3:
        print('Usage: tmx-extract.py <file> <slang> <tlang>')
        print('')
        sys.exit(-1)

    sfile_path = sys.argv[1] + "." + sys.argv[2]
    tfile_path = sys.argv[1] + "." + sys.argv[3]

    with open(sfile_path, 'w+') as sfile, open(tfile_path, 'w+') as tfile:
        curHandler = TMXHandler(sys.argv[2], sys.argv[3], sfile, tfile)
        parser.setContentHandler(curHandler)
        with open(sys.argv[1], 'r') as tmx_file:
            parser.parse(tmx_file)

<commit_msg>tmx-extract.py: replace newlines<commit_after>#!/usr/bin/python3
#
# Adapted from Apertium
# http://wiki.apertium.org/wiki/Tools_for_TMX
#

from xml.sax import make_parser
from xml.sax.handler import ContentHandler

import sys
import codecs

class TMXHandler(ContentHandler):
    def __init__ (self, slang, tlang, sfile, tfile):
        self.pair = set([slang, tlang])
        self.files = {}
        self.files[slang] = sfile
        self.files[tlang] = tfile
        self.inTag = ''
        self.note = ''
        self.tuid = ''
        self.type = ''
        self.cur_pair = set()
        self.cur_lang = ''
        self.seg = {}
        self.seg[slang] = ''
        self.seg[tlang] = ''

    def startElement(self, name, attrs):

        if name == 'tu':
            self.cur_pair = set()
            self.inTag = 'tu'
            self.tuid = attrs.get('tuid','')
            self.type = attrs.get('datatype','')
        elif name == 'note':
            self.inTag = 'note'
            self.note = ""
        elif name == 'tuv':
            self.inTag = 'tuv'
            self.cur_lang = attrs.get('xml:lang', '')
            self.cur_pair.add(self.cur_lang)
        elif name == 'seg':
            self.inTag = 'seg'
            if self.cur_lang in self.pair:
                self.seg[self.cur_lang] = ''

    def characters (self, c):
        if self.inTag == 'note':
            self.note += c
        elif self.inTag == 'seg' and self.cur_lang in self.pair:
            self.seg[self.cur_lang] += c

    def endElement(self, name):
        if name == 'tu' and self.pair == self.cur_pair:
            for lang in self.cur_pair:
                self.files[lang].write("{}\n".format(self.seg[lang].replace("\n", " ").strip()))

if __name__ == "__main__":
    parser = make_parser()

    if len(sys.argv) < 3:
        print('Usage: tmx-extract.py <file> <slang> <tlang>')
        print('')
        sys.exit(-1)

    sfile_path = sys.argv[1] + "." + sys.argv[2]
    tfile_path = sys.argv[1] + "." + sys.argv[3]

    with open(sfile_path, 'w+') as sfile, open(tfile_path, 'w+') as tfile:
        curHandler = TMXHandler(sys.argv[2], sys.argv[3], sfile, tfile)
        parser.setContentHandler(curHandler)
        with open(sys.argv[1], 'r') as tmx_file:
            parser.parse(tmx_file)

<|endoftext|>


<commit_before>import argparse
import json
import os
import logging

from c7n.commands import policy_command
from c7n.credentials import SessionFactory
from c7n import mu, resources

log = logging.getLogger('resources')

@policy_command
def resources_gc_prefix(options, policy_collection):
    """Garbage collect old custodian policies based on prefix.

    We attempt to introspect to find the event sources for a policy
    but without the old configuration this is implicit.
    """
    session_factory = SessionFactory(
        options.region, options.profile, options.assume_role)

    manager = mu.LambdaManager(session_factory)
    funcs = list(manager.list_functions('custodian-'))

    client = session_factory().client('lambda')

    remove = []
    current_policies = [p.name for p in policy_collection]
    for f in funcs:
        pn = f['FunctionName'].split('-', 1)[1]
        if pn not in current_policies:
            remove.append(f)

    for n in remove:
        log.info("Removing %s" % n['FunctionName'])

    for func in remove:
        events = []
        result = client.get_policy(FunctionName=func['FunctionName'])
        if 'Policy' not in result:
            pass
        else:
            p = json.loads(result['Policy'])
            for s in p['Statement']:
                principal = s.get('Principal')
                if not isinstance(principal, dict):
                    log.info("Skipping function %s" % func['FunctionName'])
                    continue
                if principal == {'Service': 'events.amazonaws.com'}:
                    events.append(
                        mu.CloudWatchEventSource({}, session_factory))

        f = mu.LambdaFunction({
            'name': n['FunctionName'],
            'role': n['Role'],
            'handler': n['Handler'],
            'timeout': n['Timeout'],
            'memory_size': n['MemorySize'],
            'description': n['Description'],
            'runtime': n['Runtime'],
            'events': events}, None)
        log.info("Removing %s" % f)

        if options.dryrun:
            log.info("Dryrun skipping")
            continue
        manager.remove(f)

def setup_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--config', required=True)
    parser.add_argument(
        '-r', '--region', default=os.environ.get(
            'AWS_DEFAULT_REGION', 'us-east-1'))
    parser.add_argument('--dryrun', action="store_true", default=False)
    parser.add_argument(
        "--profile", default=os.environ.get('AWS_PROFILE'),
        help="AWS Account Config File Profile to utilize")
    parser.add_argument(
        "--assume", default=None, dest="assume_role",
        help="Role to assume")
    return parser

def main():
    parser = setup_parser()
    options = parser.parse_args()
    options.policy_filter = None
    options.log_group = None
    options.cache_period = 0
    options.cache = None
    logging.basicConfig(
        level=logging.DEBUG,
        format="%(asctime)s: %(name)s:%(levelname)s %(message)s")
    logging.getLogger('botocore').setLevel(logging.ERROR)

    resources.load_resources()
    resources_gc_prefix(options)

if __name__ == '__main__':
    main()
<commit_msg>support multiple config files (#423)<commit_after>import argparse
import json
import os
import logging

from c7n.credentials import SessionFactory
from c7n.policy import load as policy_load
from c7n import mu, resources

log = logging.getLogger('resources')

def load_policies(options):
    policies = []
    for f in options.config_files:
        for collection in policy_load(options, f):
            policies.extend(collection.filter(options.policy_filter))
    return policies

def resources_gc_prefix(options, policy_collection):
    """Garbage collect old custodian policies based on prefix.

    We attempt to introspect to find the event sources for a policy
    but without the old configuration this is implicit.
    """
    session_factory = SessionFactory(
        options.region, options.profile, options.assume_role)

    manager = mu.LambdaManager(session_factory)
    funcs = list(manager.list_functions('custodian-'))

    client = session_factory().client('lambda')

    remove = []
    current_policies = [p.name for p in policy_collection]
    for f in funcs:
        pn = f['FunctionName'].split('-', 1)[1]
        if pn not in current_policies:
            remove.append(f)

    for n in remove:
        log.info("Removing %s" % n['FunctionName'])

    for func in remove:
        events = []
        result = client.get_policy(FunctionName=func['FunctionName'])
        if 'Policy' not in result:
            pass
        else:
            p = json.loads(result['Policy'])
            for s in p['Statement']:
                principal = s.get('Principal')
                if not isinstance(principal, dict):
                    log.info("Skipping function %s" % func['FunctionName'])
                    continue
                if principal == {'Service': 'events.amazonaws.com'}:
                    events.append(
                        mu.CloudWatchEventSource({}, session_factory))

        f = mu.LambdaFunction({
            'name': n['FunctionName'],
            'role': n['Role'],
            'handler': n['Handler'],
            'timeout': n['Timeout'],
            'memory_size': n['MemorySize'],
            'description': n['Description'],
            'runtime': n['Runtime'],
            'events': events}, None)
        log.info("Removing %s" % f)

        if options.dryrun:
            log.info("Dryrun skipping")
            continue
        manager.remove(f)

def setup_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-c', '--config',
        required=True, dest="config_files", action="append")
    parser.add_argument(
        '-r', '--region', default=os.environ.get(
            'AWS_DEFAULT_REGION', 'us-east-1'))
    parser.add_argument('--dryrun', action="store_true", default=False)
    parser.add_argument(
        "--profile", default=os.environ.get('AWS_PROFILE'),
        help="AWS Account Config File Profile to utilize")
    parser.add_argument(
        "--assume", default=None, dest="assume_role",
        help="Role to assume")
    return parser

def main():
    parser = setup_parser()
    options = parser.parse_args()
    options.policy_filter = None
    options.log_group = None
    options.cache_period = 0
    options.cache = None
    logging.basicConfig(
        level=logging.DEBUG,
        format="%(asctime)s: %(name)s:%(levelname)s %(message)s")
    logging.getLogger('botocore').setLevel(logging.ERROR)

    resources.load_resources()

    policies = load_policies(options.config_files)
    resources_gc_prefix(options, policies)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
For CommonsCloud copyright information please see the LICENSE document
(the "License") included with this software package. This file may not
be used in any manner except in compliance with the License

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

"""
Import Flask Dependencies
"""
from flask import request

from flask.ext.security import current_user

"""
Import Application Module Dependencies
"""
from CommonsCloudAPI.extensions import oauth
from CommonsCloudAPI.extensions import status as status_

from CommonsCloudAPI.models.application import Application

from . import module

from .permissions import permission_required

@module.route('/v2/applications.<string:extension>', methods=['OPTIONS'])
def application_preflight(extension):
  arguments = {
    'the_content': {
      "preflight": "Request successful"
    },
    'extension': extension
  }

  return Application_.endpoint_response(**arguments)

@module.route('/v2/applications.<string:extension>', methods=['GET'])
@oauth.require_oauth()
def application_list(oauth_request, extension):

  Application_ = Application()
  Application_.current_user = oauth_request.user

  applications_ = Application_.application_list()

  arguments = {
    'the_content': applications_,
    'list_name': 'applications',
    'extension': extension
  }

  return Application_.endpoint_response(**arguments)

@module.route('/v2/applications.<string:extension>', methods=['POST'])
@oauth.require_oauth()
def application_post(oauth_request, extension):

  Application_ = Application()
  Application_.current_user = oauth_request.user

  new_application = Application_.application_create(request)

  arguments = {
    'the_content': new_application,
    'extension': extension,
    'code': 201
  }

  return Application_.endpoint_response(**arguments)

@module.route('/v2/applications/<int:application_id>.<string:extension>', methods=['GET'])
@oauth.require_oauth()
# @permission_required('can_view')
def application_get(oauth_request, application_id, extension):

  Application_ = Application()
  Application_.current_user = oauth_request.user

  this_application = Application_.application_get(application_id)

  if type(this_application) is 'Response':
    return this_application, this_application.code

  arguments = {
    'the_content': this_application,
    'extension': extension
  }

  return Application_.endpoint_response(**arguments)

@module.route('/v2/applications/<int:application_id>.<string:extension>', methods=['PUT', 'PATCH'])
@oauth.require_oauth()
# @permission_required('can_edit')
def application_update(oauth_request, application_id, extension):

  Application_ = Application()
  Application_.current_user = oauth_request.user

  updated_application = Application_.application_update(application_id, request)

  arguments = {
    'the_content': updated_application,
    'extension': extension
  }

  return Application_.endpoint_response(**arguments)

@module.route('/v2/applications/<int:application_id>.<string:extension>', methods=['DELETE'])
# @oauth.require_oauth()
# @permission_required('can_delete')
def application_delete(oauth_request, application_id, extension):

  Application_ = Application()
  Application_.current_user = oauth_request.user

  Application_.application_delete(application_id)

  return status_.status_204(), 204

<commit_msg>Added temporary/test OPTIONS to application.json<commit_after>"""
For CommonsCloud copyright information please see the LICENSE document
(the "License") included with this software package. This file may not
be used in any manner except in compliance with the License

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

"""
Import Flask Dependencies
"""
from flask import request

from flask.ext.security import current_user

"""
Import Application Module Dependencies
"""
from CommonsCloudAPI.extensions import oauth
from CommonsCloudAPI.extensions import status as status_

from CommonsCloudAPI.models.application import Application

from . import module

from .permissions import permission_required

@module.route('/v2/applications.<string:extension>', methods=['OPTIONS'])
def application_preflight(extension):

  Application_ = Application()

  arguments = {
    'the_content': {
      "preflight": "Request successful"
    },
    'extension': extension
  }

  return Application_.endpoint_response(**arguments)

@module.route('/v2/applications.<string:extension>', methods=['GET'])
@oauth.require_oauth()
def application_list(oauth_request, extension):

  Application_ = Application()
  Application_.current_user = oauth_request.user

  applications_ = Application_.application_list()

  arguments = {
    'the_content': applications_,
    'list_name': 'applications',
    'extension': extension
  }

  return Application_.endpoint_response(**arguments)

@module.route('/v2/applications.<string:extension>', methods=['POST'])
@oauth.require_oauth()
def application_post(oauth_request, extension):

  Application_ = Application()
  Application_.current_user = oauth_request.user

  new_application = Application_.application_create(request)

  arguments = {
    'the_content': new_application,
    'extension': extension,
    'code': 201
  }

  return Application_.endpoint_response(**arguments)

@module.route('/v2/applications/<int:application_id>.<string:extension>', methods=['GET'])
@oauth.require_oauth()
# @permission_required('can_view')
def application_get(oauth_request, application_id, extension):

  Application_ = Application()
  Application_.current_user = oauth_request.user

  this_application = Application_.application_get(application_id)

  if type(this_application) is 'Response':
    return this_application, this_application.code

  arguments = {
    'the_content': this_application,
    'extension': extension
  }

  return Application_.endpoint_response(**arguments)

@module.route('/v2/applications/<int:application_id>.<string:extension>', methods=['PUT', 'PATCH'])
@oauth.require_oauth()
# @permission_required('can_edit')
def application_update(oauth_request, application_id, extension):

  Application_ = Application()
  Application_.current_user = oauth_request.user

  updated_application = Application_.application_update(application_id, request)

  arguments = {
    'the_content': updated_application,
    'extension': extension
  }

  return Application_.endpoint_response(**arguments)

@module.route('/v2/applications/<int:application_id>.<string:extension>', methods=['DELETE'])
# @oauth.require_oauth()
# @permission_required('can_delete')
def application_delete(oauth_request, application_id, extension):

  Application_ = Application()
  Application_.current_user = oauth_request.user

  Application_.application_delete(application_id)

  return status_.status_204(), 204

<|endoftext|>


<commit_before>from scoring_engine.celery_app import celery_app
from celery.exceptions import SoftTimeLimitExceeded
import subprocess

from scoring_engine.logger import logger

@celery_app.task(name='execute_command', soft_time_limit=30)
def execute_command(job):
    output = ""
    # Disable duplicate celery log messages
    if logger.propagate:
        logger.propagate = False
    logger.info("Running cmd for " + str(job))
    try:
        cmd_result = subprocess.run(
            job['command'],
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT
        )
        output = cmd_result.stdout.decode("utf-8")
        job['errored_out'] = False
    except SoftTimeLimitExceeded:
        job['errored_out'] = True
    job['output'] = output
    return job
<commit_msg>Modify worker to requeue jobs if worker is killed<commit_after>from scoring_engine.celery_app import celery_app
from celery.exceptions import SoftTimeLimitExceeded
import subprocess

from scoring_engine.logger import logger

@celery_app.task(name='execute_command', acks_late=True, reject_on_worker_lost=True, soft_time_limit=30)
def execute_command(job):
    output = ""
    # Disable duplicate celery log messages
    if logger.propagate:
        logger.propagate = False
    logger.info("Running cmd for " + str(job))
    try:
        cmd_result = subprocess.run(
            job['command'],
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT
        )
        output = cmd_result.stdout.decode("utf-8")
        job['errored_out'] = False
    except SoftTimeLimitExceeded:
        job['errored_out'] = True
    job['output'] = output
    return job
<|endoftext|>