<commit_before># -*- coding:utf-8 -*-

import json
import os
import re
import requests

import sys

from web.models import Chairman

reload(sys)
sys.setdefaultencoding('utf-8')

class Fetcher():

    def __init__(self):
        self.chairmans = []

    def fetch_douyu(self):
        print 'fetch douyu'

        url = 'http://www.douyu.com/directory/game/How'

        session = requests.Session()
        response = session.get(url)

        base_url = 'http://www.douyu.com/'
        # print response.content.decode('utf8')
        for each_content in re.finditer('([\s\S]*?)<\/a>', response.content.decode('utf8')):
            chairman = Chairman()
            chairman.type = 'douyu'
            group = each_content.group()
            # print group
            href = re.search('href=".*?"', group).group().lstrip('href="').rstrip('"')
            chairman.href = base_url + href
            chairman.set_id(chairman.type + str("_") + href.lstrip('/'))

            title = re.search('title=".*?"', group).group().lstrip('title="').rstrip('"')
            chairman.title = title

            img = re.search('data-original=".*?"', group).group().lstrip('data-original="').rstrip('"')
            chairman.img = img

            name = re.search('.*?', group).group().lstrip('').rstrip('')
            chairman.name = name

            num = re.search('.*?', group).group().lstrip('').rstrip('')
            chairman.set_num(num)

            self.chairmans.append(chairman)

    def fetch_xiongmao(self):
        print 'fetch xiongmao'

        url = 'http://www.panda.tv/cate/hearthstone'

        session = requests.Session()
        response = session.get(url)

        base_url = 'http://www.panda.tv/'
        for each_content in re.finditer('<a href=".*?" class="video-list-item-wrap"([\s\S]*?)<\/a>', response.content.decode('utf8')):
            chairman = Chairman()
            chairman.type = 'panda'
            group = each_content.group()
            # print group

            href = re.search('href=".*?"', group).group().lstrip('href="').rstrip('"')
            chairman.href = base_url + href

            chairman.set_id(chairman.type + str("_") + href.lstrip('/'))

            title = re.search('title=".*?"', group).group().lstrip('title="').rstrip('"')
            chairman.title = title

            img = re.search('data-original=".*?"', group).group().lstrip('data-original="').rstrip('"')
            chairman.img = img

            name = re.search('.*?', group).group().lstrip(
                '').rstrip('')
            chairman.name = name

            num = re.search('.*?', group).group().lstrip(
                '').rstrip('')
            chairman.set_num(num)

            self.chairmans.append(chairman)

    def fetch_quanmin(self):
        print 'fetch quanmin'

        url = 'http://www.quanmin.tv/json/categories/heartstone/list.json?t=24468018'

        session = requests.Session()
        response = session.get(url)

        base_url = 'http://www.quanmin.tv/v/'
        for each in response.json()['data']:
            chairman = Chairman()
            chairman.type = 'quanmin'

            chairman.set_id(chairman.type + str("_") + each['uid'])

            chairman.title = each['title']
            chairman.href = base_url + each['uid']
            chairman.img = each['thumb']
            chairman.name = each['nick']
            chairman.set_num(str(each['follow']))

            self.chairmans.append(chairman)

    def fetch_zhanqi(self):
        print 'fetch zhangqi'

        url = 'http://www.zhanqi.tv/chns/blizzard/how'

        session = requests.Session()
        response = session.get(url)

        base_url = 'http://www.zhanqi.tv/'
        for each_content in re.finditer('([\s\S]*?)<\/a>',
                                        response.content.decode('utf8')):
            group = each_content.group()
            href = re.search('href=".*?"', group).group().lstrip('href="').rstrip('"')
            if href != '${url}':
                chairman = Chairman()
                chairman.type = 'zhanqi'

                # print group

                chairman.href = base_url + href

                chairman.set_id(chairman.type + str("_") + href.lstrip('/'))

                title = re.search('.*?', group).group().lstrip('').rstrip('')
                chairman.title = title

                img = re.search('<img src=".*?"', group).group().lstrip('<img src="').rstrip('"')
                chairman.img = img

                name = re.search('.*?', group).group().lstrip(
                    '').rstrip('')
                chairman.name = name

                num = re.search('.*?', group).group().lstrip(
                    '').rstrip('')
                chairman.set_num(num)

                self.chairmans.append(chairman)

    # def fetch_huomao(self):
    #     url = 'http://www.zhanqi.tv/chns/blizzard/how'
    #
    #     session = requests.Session()
    #     response = session.get(url)
    #
    #     for each_content in re.finditer('([\s\S]*?)<\/a>',
    #                                     response.content.decode('utf8')):
    #         chairman = Chairman()
    #         group = each_content.group()
    #         # print group
    #
    #         href = re.search('href=".*?"', group).group().lstrip('href="').rstrip('"')
    #         chairman.href = base_url + href
    #
    #         title = re.search('.*?', group).group().lstrip('').rstrip(
    #             '')
    #         chairman.title = title
    #
    #         img = re.search('<img src=".*?"', group).group().lstrip('<img src="').rstrip('"')
    #         chairman.img = img
    #
    #         name = re.search('.*?', group).group().lstrip(
    #             '').rstrip('')
    #         chairman.name = name
    #
    #         num = re.search('.*?', group).group().lstrip(
    #             '').rstrip('')
    #         chairman.set_num(num)
    #
    #         print chairman

    def fetch_huya(self):
        print 'fetch huya'

        url = 'http://www.huya.com/g/hearthstone'

        session = requests.Session()
        response = session.get(url)
        # print response.content
        for each_content in re.finditer('
([\s\S]*?)<\/li>',
                                        response.content.decode('utf8')):
            chairman = Chairman()
            chairman.type = 'huya'

            group = each_content.group()
            # print group
            href = re.search('href=".*?"', group).group().lstrip('href=').strip('"')
            chairman.href = href
            chairman.set_id(chairman.type + str("_") + href.lstrip('http://www.huya.com/'))

            title = re.search('>.*?', group).group().lstrip('eid="click/gamelist/card/hearthstone" eid_desc="点击/游戏列表页/卡片/炉石传说">').rstrip(
                '')
            chairman.title = title
        #
            img = re.search('<img class="pic" src=".*?"', group).group().lstrip('<img class="pic" src="').rstrip('"')
            chairman.img = img
        #
            name = re.search('', group).group().lstrip(
                '')
            chairman.name = name

            num = re.search('.*?', group).group().lstrip(
                '').rstrip('')
            chairman.set_num(num)

            self.chairmans.append(chairman)

    def fetch_longzhu(self):
        print 'fetch longzhu'

        url = 'http://longzhu.com/channels/hs?from=figame'

        session = requests.Session()
        response = session.get(url)
        # print response.content.decode('utf8')
        for each_content in re.finditer('<a href=".*? class="livecard"([\s\S]*?)<\/a>',
                                        response.content.decode('utf8')):
            chairman = Chairman()
            chairman.type = 'longzhu'

            group = each_content.group()
            # print group

            href = re.search('href=".*?"', group).group().lstrip('href=').strip('"')
            chairman.href = href

            chairman.set_id(chairman.type + str("_") + href.replace('/', '').lstrip('http://star.longzhu.com/').rstrip('?from=challcontent'))

            title = re.search('title=".*?"', group).group().lstrip('title="').rstrip('"')
            chairman.title = title

            img = re.search('<img src=".*?"', group).group().lstrip('<img src="').rstrip('"')
            chairman.img = img

            name = re.search('.*?', group).group().lstrip(
                '').rstrip('')
            chairman.name = name

            num = re.search('.*?', group).group().lstrip(
                '').rstrip('')
            chairman.set_num(num)

            self.chairmans.append(chairman)

    def fetch_cc(self):
        print 'fetch cc'
        url = 'http://cc.163.com/category/list/?gametype=1005'

        session = requests.Session()
        response = session.get(url)

        base_url = 'http://cc.163.com/'

        for each_content in re.finditer('
([\s\S]*?)<\/li>',
                                        response.content.decode('utf8')):
            group = each_content.group()
            href = re.search('href=".*?"', group).group().lstrip('href="/').rstrip('/"')

            if href != '{[value.ccid]}':
                chairman = Chairman()
                chairman.type = 'cc'

                # print group

                chairman.href = base_url + href

                chairman.set_id(chairman.type + str("_") + href)

                title = re.search('title=".*?"', group).group().lstrip('title="').rstrip('"')
                chairman.title = title

                img = re.search('<img src=".*?"', group).group().lstrip('<img src="').rstrip('"')
                chairman.img = img

                name = re.search('', group).group().lstrip(
                    '')
                chairman.name = name

                num = re.search('([\s\S]*?)', group).group().lstrip(
                    '').rstrip('')
                chairman.set_num(num)

                self.chairmans.append(chairman)

if __name__ == "__main__":
    fetcher = Fetcher()
    # fetcher.fetch_douyu()
    # fetcher.fetch_xiongmao()
    # fetcher.fetch_quanmin()
    # fetcher.fetch_zhanqi()
    # fetcher.fetch_huomao()
    # fetcher.fetch_longzhu()
    # fetcher.fetch_cc()
    fetcher.fetch_huya()

    print fetcher.chairmans<commit_msg>fix panda issue<commit_after># -*- coding:utf-8 -*-

import json
import os
import re
import requests

import sys

from web.models import Chairman

reload(sys)
sys.setdefaultencoding('utf-8')

class Fetcher():

    def __init__(self):
        self.chairmans = []

    def fetch_douyu(self):
        print 'fetch douyu'

        url = 'http://www.douyu.com/directory/game/How'

        session = requests.Session()
        response = session.get(url)

        base_url = 'http://www.douyu.com/'
        # print response.content.decode('utf8')
        for each_content in re.finditer('([\s\S]*?)<\/a>', response.content.decode('utf8')):
            chairman = Chairman()
            chairman.type = 'douyu'
            group = each_content.group()
            # print group
            href = re.search('href=".*?"', group).group().lstrip('href="').rstrip('"')
            chairman.href = base_url + href
            chairman.set_id(chairman.type + str("_") + href.lstrip('/'))

            title = re.search('title=".*?"', group).group().lstrip('title="').rstrip('"')
            chairman.title = title

            img = re.search('data-original=".*?"', group).group().lstrip('data-original="').rstrip('"')
            chairman.img = img

            name = re.search('.*?', group).group().lstrip('').rstrip('')
            chairman.name = name

            num = re.search('.*?', group).group().lstrip('').rstrip('')
            chairman.set_num(num)

            self.chairmans.append(chairman)

    def fetch_xiongmao(self):
        print 'fetch xiongmao'

        url = 'http://www.panda.tv/cate/hearthstone'

        session = requests.Session()
        response = session.get(url)

        base_url = 'http://www.panda.tv/'
        for each_content in re.finditer('<a href=".*?" class="video-list-item-wrap"([\s\S]*?)<\/a>', response.content.decode('utf8')):
            chairman = Chairman()
            chairman.type = 'panda'
            group = each_content.group()
            # print group

            href = re.search('href=".*?"', group).group().lstrip('href="/').rstrip('"')
            chairman.href = base_url + href

            chairman.set_id(chairman.type + str("_") + href.lstrip('/'))

            title = re.search('title=".*?"', group).group().lstrip('title="').rstrip('"')
            chairman.title = title

            img = re.search('data-original=".*?"', group).group().lstrip('data-original="').rstrip('"')
            chairman.img = img

            name = re.search('.*?', group).group().lstrip(
                '').rstrip('')
            chairman.name = name

            num = re.search('.*?', group).group().lstrip(
                '').rstrip('')
            chairman.set_num(num)

            self.chairmans.append(chairman)

    def fetch_quanmin(self):
        print 'fetch quanmin'

        url = 'http://www.quanmin.tv/json/categories/heartstone/list.json?t=24468018'

        session = requests.Session()
        response = session.get(url)

        base_url = 'http://www.quanmin.tv/v/'
        for each in response.json()['data']:
            chairman = Chairman()
            chairman.type = 'quanmin'

            chairman.set_id(chairman.type + str("_") + each['uid'])

            chairman.title = each['title']
            chairman.href = base_url + each['uid']
            chairman.img = each['thumb']
            chairman.name = each['nick']
            chairman.set_num(str(each['follow']))

            self.chairmans.append(chairman)

    def fetch_zhanqi(self):
        print 'fetch zhangqi'

        url = 'http://www.zhanqi.tv/chns/blizzard/how'

        session = requests.Session()
        response = session.get(url)

        base_url = 'http://www.zhanqi.tv/'
        for each_content in re.finditer('([\s\S]*?)<\/a>',
                                        response.content.decode('utf8')):
            group = each_content.group()
            href = re.search('href=".*?"', group).group().lstrip('href="').rstrip('"')
            if href != '${url}':
                chairman = Chairman()
                chairman.type = 'zhanqi'

                # print group

                chairman.href = base_url + href

                chairman.set_id(chairman.type + str("_") + href.lstrip('/'))

                title = re.search('.*?', group).group().lstrip('').rstrip('')
                chairman.title = title

                img = re.search('<img src=".*?"', group).group().lstrip('<img src="').rstrip('"')
                chairman.img = img

                name = re.search('.*?', group).group().lstrip(
                    '').rstrip('')
                chairman.name = name

                num = re.search('.*?', group).group().lstrip(
                    '').rstrip('')
                chairman.set_num(num)

                self.chairmans.append(chairman)

    # def fetch_huomao(self):
    #     url = 'http://www.zhanqi.tv/chns/blizzard/how'
    #
    #     session = requests.Session()
    #     response = session.get(url)
    #
    #     for each_content in re.finditer('([\s\S]*?)<\/a>',
    #                                     response.content.decode('utf8')):
    #         chairman = Chairman()
    #         group = each_content.group()
    #         # print group
    #
    #         href = re.search('href=".*?"', group).group().lstrip('href="').rstrip('"')
    #         chairman.href = base_url + href
    #
    #         title = re.search('.*?', group).group().lstrip('').rstrip(
    #             '')
    #         chairman.title = title
    #
    #         img = re.search('<img src=".*?"', group).group().lstrip('<img src="').rstrip('"')
    #         chairman.img = img
    #
    #         name = re.search('.*?', group).group().lstrip(
    #             '').rstrip('')
    #         chairman.name = name
    #
    #         num = re.search('.*?', group).group().lstrip(
    #             '').rstrip('')
    #         chairman.set_num(num)
    #
    #         print chairman

    def fetch_huya(self):
        print 'fetch huya'

        url = 'http://www.huya.com/g/hearthstone'

        session = requests.Session()
        response = session.get(url)
        # print response.content
        for each_content in re.finditer('
([\s\S]*?)<\/li>',
                                        response.content.decode('utf8')):
            chairman = Chairman()
            chairman.type = 'huya'

            group = each_content.group()
            # print group
            href = re.search('href=".*?"', group).group().lstrip('href=').strip('"')
            chairman.href = href
            chairman.set_id(chairman.type + str("_") + href.lstrip('http://www.huya.com/'))

            title = re.search('>.*?', group).group().lstrip('eid="click/gamelist/card/hearthstone" eid_desc="点击/游戏列表页/卡片/炉石传说">').rstrip(
                '')
            chairman.title = title
        #
            img = re.search('<img class="pic" src=".*?"', group).group().lstrip('<img class="pic" src="').rstrip('"')
            chairman.img = img
        #
            name = re.search('', group).group().lstrip(
                '')
            chairman.name = name

            num = re.search('.*?', group).group().lstrip(
                '').rstrip('')
            chairman.set_num(num)

            self.chairmans.append(chairman)

    def fetch_longzhu(self):
        print 'fetch longzhu'

        url = 'http://longzhu.com/channels/hs?from=figame'

        session = requests.Session()
        response = session.get(url)
        # print response.content.decode('utf8')
        for each_content in re.finditer('<a href=".*? class="livecard"([\s\S]*?)<\/a>',
                                        response.content.decode('utf8')):
            chairman = Chairman()
            chairman.type = 'longzhu'

            group = each_content.group()
            # print group

            href = re.search('href=".*?"', group).group().lstrip('href=').strip('"')
            chairman.href = href

            chairman.set_id(chairman.type + str("_") + href.replace('/', '').lstrip('http://star.longzhu.com/').rstrip('?from=challcontent'))

            title = re.search('title=".*?"', group).group().lstrip('title="').rstrip('"')
            chairman.title = title

            img = re.search('<img src=".*?"', group).group().lstrip('<img src="').rstrip('"')
            chairman.img = img

            name = re.search('.*?', group).group().lstrip(
                '').rstrip('')
            chairman.name = name

            num = re.search('.*?', group).group().lstrip(
                '').rstrip('')
            chairman.set_num(num)

            self.chairmans.append(chairman)

    def fetch_cc(self):
        print 'fetch cc'
        url = 'http://cc.163.com/category/list/?gametype=1005'

        session = requests.Session()
        response = session.get(url)

        base_url = 'http://cc.163.com/'

        for each_content in re.finditer('
([\s\S]*?)<\/li>',
                                        response.content.decode('utf8')):
            group = each_content.group()
            href = re.search('href=".*?"', group).group().lstrip('href="/').rstrip('/"')

            if href != '{[value.ccid]}':
                chairman = Chairman()
                chairman.type = 'cc'

                # print group

                chairman.href = base_url + href

                chairman.set_id(chairman.type + str("_") + href)

                title = re.search('title=".*?"', group).group().lstrip('title="').rstrip('"')
                chairman.title = title

                img = re.search('<img src=".*?"', group).group().lstrip('<img src="').rstrip('"')
                chairman.img = img

                name = re.search('', group).group().lstrip(
                    '')
                chairman.name = name

                num = re.search('([\s\S]*?)', group).group().lstrip(
                    '').rstrip('')
                chairman.set_num(num)

                self.chairmans.append(chairman)

if __name__ == "__main__":
    fetcher = Fetcher()
    # fetcher.fetch_douyu()
    # fetcher.fetch_xiongmao()
    # fetcher.fetch_quanmin()
    # fetcher.fetch_zhanqi()
    # fetcher.fetch_huomao()
    # fetcher.fetch_longzhu()
    # fetcher.fetch_cc()
    fetcher.fetch_huya()

    print fetcher.chairmans<|endoftext|>


<commit_before>#!/usr/bin/env python3
import argparse
import logging
import signal
import os
import sys
import asyncio

import aiohttp

class Crawler:

    def __init__(self, identifiers, loop, done_callback=None, maxtasks=None):
        maxtasks = 100 if not maxtasks else maxtasks

        self.identifiers = identifiers
        self.loop = loop
        self.todo = set()
        self.busy = set()
        self.tasks = set()
        self.done_callback = done_callback
        self.sem = asyncio.Semaphore(maxtasks)

        # connector stores cookies between requests and uses connection pool
        self.connector = aiohttp.TCPConnector(share_cookies=True, loop=loop)

    @asyncio.coroutine
    def run(self):
        asyncio.Task(self.addurls(self.identifiers))  # Set initial work.
        yield from asyncio.sleep(1)
        while self.busy:
            yield from asyncio.sleep(1)

        self.connector.close()
        self.loop.stop()

    def _done_callback(self, future):
        resp = future.result()
        resp.content = resp._content
        if self.done_callback:
            self.done_callback(resp)
        else:
            print(resp.content.decode('utf-8'))

    @asyncio.coroutine
    def addurls(self, identifiers):
        for identifier in identifiers:
            if not identifier.strip():
                continue
            url = 'http://archive.org/metadata/{}'.format(identifier.strip())
            self.todo.add(url)
            yield from self.sem.acquire()
            task = asyncio.Task(self.process(url))
            task.add_done_callback(lambda t: self.sem.release())
            task.add_done_callback(self.tasks.remove)
            task.add_done_callback(self._done_callback)
            self.tasks.add(task)

    @asyncio.coroutine
    def process(self, url):
        self.todo.remove(url)
        self.busy.add(url)
        resp = None
        data = None
        i = 0
        max_retries = 10
        while True:
            i += 1
            try:
                resp = yield from aiohttp.request(
                    'get', url, connector=self.connector)
                break
            except Exception as exc:
                sys.stderr.write('{0} has error {1}\n'.format(url, repr(exc)))
                if i >= max_retries:
                    break
                sys.stderr.write('... retry {0}/{1} {2}\n'.format(i, max_retries, url))
                yield from asyncio.sleep(1)

        if resp:
            yield from resp.read()
            resp.close()
        self.busy.remove(url)
        return resp

def main():
    parser = argparse.ArgumentParser(description='''Concurrently retrieve metadata from
                                                    Archive.org items.''')

    parser.add_argument('itemlist', nargs='?', default=sys.stdin,
                        help='''A file containing Archive.org identifiers, one per line,
                                for which to retrieve metadata from. If no itemlist is
                                provided, identifiers will be read from stdin.''')
    parser.add_argument('--workers', type=int,
                        help='The maximum number of tasks to run at once.')
    args = parser.parse_args()

    if args.itemlist is sys.stdin:
        if not (os.fstat(sys.stdin.fileno()).st_size > 0):
            # Exit with 2 if stdin appears to be empty.
            sys.exit(2)

    loop = asyncio.get_event_loop()

    if not hasattr(args.itemlist, 'read'):
        itemlist = open(args.itemlist)
    else:
        itemlist = args.itemlist

    c = Crawler(itemlist, loop, maxtasks=args.workers)
    asyncio.Task(c.run())

    try:
        loop.add_signal_handler(signal.SIGINT, loop.stop)
    except RuntimeError:
        pass

    loop.run_forever()

if __name__ == '__main__':
    main()
<commit_msg>Renamed Crawler to Miner<commit_after>#!/usr/bin/env python3
import argparse
import logging
import signal
import os
import sys
import asyncio

import aiohttp

class Miner:

    def __init__(self, identifiers, loop, done_callback=None, maxtasks=None):
        maxtasks = 100 if not maxtasks else maxtasks

        self.identifiers = identifiers
        self.loop = loop
        self.todo = set()
        self.busy = set()
        self.tasks = set()
        self.done_callback = done_callback
        self.sem = asyncio.Semaphore(maxtasks)

        # connector stores cookies between requests and uses connection pool
        self.connector = aiohttp.TCPConnector(share_cookies=True, loop=loop)

    @asyncio.coroutine
    def run(self):
        asyncio.Task(self.addurls(self.identifiers))  # Set initial work.
        yield from asyncio.sleep(1)
        while self.busy:
            yield from asyncio.sleep(1)

        self.connector.close()
        self.loop.stop()

    def _done_callback(self, future):
        resp = future.result()
        resp.content = resp._content
        if self.done_callback:
            self.done_callback(resp)
        else:
            print(resp.content.decode('utf-8'))

    @asyncio.coroutine
    def addurls(self, identifiers):
        for identifier in identifiers:
            if not identifier.strip():
                continue
            url = 'http://archive.org/metadata/{}'.format(identifier.strip())
            self.todo.add(url)
            yield from self.sem.acquire()
            task = asyncio.Task(self.process(url))
            task.add_done_callback(lambda t: self.sem.release())
            task.add_done_callback(self.tasks.remove)
            task.add_done_callback(self._done_callback)
            self.tasks.add(task)

    @asyncio.coroutine
    def process(self, url):
        self.todo.remove(url)
        self.busy.add(url)
        resp = None
        data = None
        i = 0
        max_retries = 10
        while True:
            i += 1
            try:
                resp = yield from aiohttp.request(
                    'get', url, connector=self.connector)
                break
            except Exception as exc:
                sys.stderr.write('{0} has error {1}\n'.format(url, repr(exc)))
                if i >= max_retries:
                    break
                sys.stderr.write('... retry {0}/{1} {2}\n'.format(i, max_retries, url))
                yield from asyncio.sleep(1)

        if resp:
            yield from resp.read()
            resp.close()
        self.busy.remove(url)
        return resp

def main():
    parser = argparse.ArgumentParser(description='''Concurrently retrieve metadata from
                                                    Archive.org items.''')

    parser.add_argument('itemlist', nargs='?', default=sys.stdin,
                        help='''A file containing Archive.org identifiers, one per line,
                                for which to retrieve metadata from. If no itemlist is
                                provided, identifiers will be read from stdin.''')
    parser.add_argument('--workers', type=int,
                        help='The maximum number of tasks to run at once.')
    args = parser.parse_args()

    if args.itemlist is sys.stdin:
        if not (os.fstat(sys.stdin.fileno()).st_size > 0):
            # Exit with 2 if stdin appears to be empty.
            sys.exit(2)

    loop = asyncio.get_event_loop()

    if not hasattr(args.itemlist, 'read'):
        itemlist = open(args.itemlist)
    else:
        itemlist = args.itemlist

    c = Miner(itemlist, loop, maxtasks=args.workers)
    asyncio.Task(c.run())

    try:
        loop.add_signal_handler(signal.SIGINT, loop.stop)
    except RuntimeError:
        pass

    loop.run_forever()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#:coding=utf8:

from django.contrib.auth.models import User
from tagging.models import *
from django.db.models import *
from django.utils.translation import ugettext_lazy as _

from tagging.fields import TagField

from datetime import datetime

BLOG_LOCALES = (
    ('ja', u'日本語'),
    ('en', u'English'),
)

class PostManager(Manager):
    def active(self):
        return self.filter(pub_date__lt = datetime.now(), active=True)

class Post(Model):
    author = ForeignKey(User, verbose_name=u"author")
    slug = SlugField(u"slug", max_length=50)
    title = TextField(u"title")
    content = TextField(u"content")
    markup_type = models.CharField(max_length=10, choices=(
        ("html", "HTML"),
        ("rst", "reStructuredText"),
    ), default="html")
    locale = CharField(u'locale', max_length=20, choices=BLOG_LOCALES, default="en")
    tags = TagField()

    active = BooleanField(u'active', default=False)
    pub_date = DateTimeField(u'date', default=datetime.now)
    create_date = DateTimeField(u'created', default=datetime.now)

    objects = PostManager()

    class Meta:
        verbose_name = _("post")
        verbose_name_plural = _("posts")
        ordering = ('-pub_date',)
<commit_msg>Updated language<commit_after>#:coding=utf8:

from django.contrib.auth.models import User
from tagging.models import *
from django.db.models import *
from django.utils.translation import ugettext_lazy as _

from tagging.fields import TagField

from datetime import datetime

BLOG_LOCALES = (
    ('jp', u'日本語'),
    ('en', u'English'),
)

class PostManager(Manager):
    def active(self):
        return self.filter(pub_date__lt = datetime.now(), active=True)

class Post(Model):
    author = ForeignKey(User, verbose_name=u"author")
    slug = SlugField(u"slug", max_length=50)
    title = TextField(u"title")
    content = TextField(u"content")
    markup_type = models.CharField(max_length=10, choices=(
        ("html", "HTML"),
        ("rst", "reStructuredText"),
    ), default="html")
    locale = CharField(u'locale', max_length=20, choices=BLOG_LOCALES, default="en")
    tags = TagField()

    active = BooleanField(u'active', default=False)
    pub_date = DateTimeField(u'date', default=datetime.now)
    create_date = DateTimeField(u'created', default=datetime.now)

    objects = PostManager()

    class Meta:
        verbose_name = _("post")
        verbose_name_plural = _("posts")
        ordering = ('-pub_date',)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
constructs the Theano optimization and trains a learning model,
optionally by initializing the network with pre-trained weights.
"""
from __future__ import absolute_import, division, print_function
from ibeis_cnn import utils
from ibeis_cnn import models
from ibeis_cnn import ibsplugin

import time
import theano
import numpy as np
import cPickle as pickle
from lasagne import layers
from sklearn import preprocessing

import utool as ut
import six
from os.path import join, abspath, dirname, exists

def train(data_file, labels_file, model, weights_file, results_path,
          pretrained_weights_file=None, pretrained_kwargs=False, **kwargs):
    """
    Driver function

    Args:
        data_file (?):
        labels_file (?):
        model (?):
        weights_file (?):
        pretrained_weights_file (None):
        pretrained_kwargs (bool):
    """

    ######################################################################################

    # Load the data
    print('\n[data] loading data...')
    print('data_file = %r' % (data_file,))
    print('labels_file = %r' % (labels_file,))
    data, labels = utils.load(data_file, labels_file)

    # Ensure results dir
    weights_path = dirname(abspath(weights_file))
    ut.ensuredir(weights_path)
    ut.ensuredir(results_path)
    if pretrained_weights_file is not None:
        pretrained_weights_path = dirname(abspath(pretrained_weights_file))
        ut.ensuredir(pretrained_weights_path)

    # Training parameters defaults
    utils._update(kwargs, 'center',         True)
    utils._update(kwargs, 'encode',         True)
    utils._update(kwargs, 'learning_rate',  0.01)
    utils._update(kwargs, 'momentum',       0.9)
    utils._update(kwargs, 'batch_size',     128)
    utils._update(kwargs, 'patience',       10)
    utils._update(kwargs, 'test',           5)  # Test every X epochs
    utils._update(kwargs, 'max_epochs',     kwargs.get('patience') * 10)
    utils._update(kwargs, 'regularization', None)
    utils._update(kwargs, 'test_time_augmentation',  False)
    utils._update(kwargs, 'output_dims',    None)
    utils._update(kwargs, 'show_features',  True)

    # Automatically figure out how many classes
    if kwargs.get('output_dims') is None:
        kwargs['output_dims'] = len(list(np.unique(labels)))

    # print('[load] adding channels...')
    # data = utils.add_channels(data)
    print('[train]     data.shape = %r' % (data.shape,))
    print('[train]     data.dtype = %r' % (data.dtype,))
    print('[train]     labels.shape = %r' % (labels.shape,))
    print('[train]     labels.dtype = %r' % (labels.dtype,))

    labelhist = {key: len(val) for key, val in six.iteritems(ut.group_items(labels, labels))}
    print('label stats = \n' + ut.dict_str(labelhist))
    print('train kwargs = \n' + (ut.dict_str(kwargs)))

    # Encoding labels
    kwargs['encoder'] = None
    if kwargs.get('encode', False):
        kwargs['encoder'] = preprocessing.LabelEncoder()
        kwargs['encoder'].fit(labels)

    # utils.show_image_from_data(data)

    # Split the dataset into training and validation
    print('[train] creating train, validation datasaets...')
    dataset = utils.train_test_split(data, labels, eval_size=0.2)
    X_train, y_train, X_valid, y_valid = dataset
    dataset = utils.train_test_split(X_train, y_train, eval_size=0.1)
    X_train, y_train, X_test, y_test = dataset

    # Build and print the model
    print('\n[model] building model...')
    input_cases, input_channels, input_height, input_width = data.shape
    output_layer = model.build_model(
        kwargs.get('batch_size'), input_width, input_height,
        input_channels, kwargs.get('output_dims'))
    utils.print_layer_info(output_layer)

    # Create the Theano primitives
    print('[model] creating Theano primitives...')
    learning_rate_theano = theano.shared(utils.float32(kwargs.get('learning_rate')))
    all_iters = utils.create_iter_funcs(learning_rate_theano, output_layer, **kwargs)
    train_iter, valid_iter, test_iter = all_iters

    # Load the pretrained model if specified
    if pretrained_weights_file is not None and exists(pretrained_weights_file):
        print('[model] loading pretrained weights from %s' % (pretrained_weights_file))
        with open(pretrained_weights_file, 'rb') as pfile:
            kwargs_ = pickle.load(pfile)
            pretrained_weights = kwargs_.get('best_weights', None)
            layers.set_all_param_values(output_layer, pretrained_weights)
            if pretrained_kwargs:
                kwargs = kwargs_

    # Center the data by subtracting the mean (AFTER KWARGS UPDATE)
    if kwargs.get('center'):
        print('[train] applying data centering...')
        utils._update(kwargs, 'center_mean', np.mean(X_train, axis=0))
        # utils._update(kwargs, 'center_std', np.std(X_train, axis=0))
        utils._update(kwargs, 'center_std', 255.0)
    else:
        utils._update(kwargs, 'center_mean', 0.0)
        utils._update(kwargs, 'center_std', 255.0)

    # Begin training the neural network
    vals = (utils.get_current_time(), kwargs.get('learning_rate'), )
    print('\n[train] starting training at %s with learning rate %.9f' % vals)
    utils.print_header_columns()

    utils._update(kwargs, 'best_weights',        None)
    utils._update(kwargs, 'best_valid_accuracy', 0.0)
    utils._update(kwargs, 'best_test_accuracy',  0.0)
    utils._update(kwargs, 'best_train_loss',     np.inf)
    utils._update(kwargs, 'best_valid_loss',     np.inf)
    utils._update(kwargs, 'best_valid_accuracy', 0.0)
    utils._update(kwargs, 'best_test_accuracy',  0.0)
    try:
        epoch = 0
        epoch_marker = epoch
        while True:
            try:
                # Start timer
                t0 = time.time()

                # compute the loss over all training and validation batches
                augment_fn = getattr(model, 'augment', None)
                avg_train_loss = utils.forward_train(X_train, y_train, train_iter, rand=True,
                                                     augment=augment_fn, **kwargs)
                if kwargs.get('test_time_augmentation', False):
                    avg_valid_data = utils.forward_valid(X_valid, y_valid, valid_iter,
                                                         augment=augment_fn, **kwargs)
                else:
                    avg_valid_data = utils.forward_valid(X_valid, y_valid, valid_iter, **kwargs)
                avg_valid_loss, avg_valid_accuracy = avg_valid_data

                # If the training loss is nan, the training has diverged
                if np.isnan(avg_train_loss):
                    print('\n[train] training diverged\n')
                    break

                # Is this model the best we've ever seen?
                best_found = avg_valid_loss < kwargs.get('best_valid_loss')
                if best_found:
                    kwargs['best_epoch'] = epoch
                    epoch_marker = epoch
                    kwargs['best_weights'] = layers.get_all_param_values(output_layer)

                # compute the loss over all testing batches
                request_test = kwargs.get('test') is not None and epoch % kwargs.get('test') == 0
                if best_found or request_test:
                    avg_test_accuracy = utils.forward_test(X_test, y_test, test_iter,
                                                           results_path, **kwargs)
                    # Output the layer 1 features
                    if kwargs.get('show_features'):
                        utils.show_convolutional_features(output_layer, results_path, color=True, target=0)
                else:
                    avg_test_accuracy = None

                # Running tab for what the best model
                if avg_train_loss < kwargs.get('best_train_loss'):
                    kwargs['best_train_loss'] = avg_train_loss
                if avg_valid_loss < kwargs.get('best_valid_loss'):
                    kwargs['best_valid_loss'] = avg_valid_loss
                if avg_valid_accuracy > kwargs.get('best_valid_accuracy'):
                    kwargs['best_valid_accuracy'] = avg_valid_accuracy
                if avg_test_accuracy > kwargs.get('best_test_accuracy'):
                    kwargs['best_test_accuracy'] = avg_test_accuracy

                # Learning rate schedule update
                if epoch >= epoch_marker + kwargs.get('patience'):
                    epoch_marker = epoch
                    utils.set_learning_rate(learning_rate_theano, model.learning_rate_update)

                # End timer
                t1 = time.time()

                # Increment the epoch
                epoch += 1
                utils.print_epoch_info(avg_train_loss, avg_valid_loss,
                                       avg_valid_accuracy, avg_test_accuracy,
                                       epoch, t1 - t0, **kwargs)

                # Break on max epochs
                if epoch >= kwargs.get('max_epochs'):
                    print('\n[train] maximum number of epochs reached\n')
                    break
            except KeyboardInterrupt:
                # We have caught the Keyboard Interrupt, figure out what resolution mode
                print('\n[train] Caught CRTL+C')
                resolution = ''
                while not (resolution.isdigit() and int(resolution) in [1, 2, 3]):
                    print('\n[train] What do you want to do?')
                    print('[train]     1 - Shock weights')
                    print('[train]     2 - Save best weights')
                    print('[train]     3 - Stop network training')
                    resolution = raw_input('[train] Resolution: ')
                resolution = int(resolution)
                # We have a resolution
                if resolution == 1:
                    # Shock the weights of the network
                    utils.shock_network(output_layer)
                    epoch_marker = epoch
                    utils.set_learning_rate(learning_rate_theano, model.learning_rate_shock)
                elif resolution == 2:
                    # Save the weights of the network
                    utils.save_model(kwargs, weights_file)
                else:
                    # Terminate the network training
                    raise KeyboardInterrupt
    except KeyboardInterrupt:
        print('\n\n[train] ...stopping network training\n')

    # Save the best network
    utils.save_model(kwargs, weights_file)

#@ibeis.register_plugin()
def train_identification_pz():
    r"""

    CommandLine:
        python -m ibeis_cnn.train --test-train_identification_pz

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.train import *  # NOQA
        >>> train_identification_pz()
    """
    print('get_identification_decision_training_data')
    import ibeis
    ibs = ibeis.opendb('NNP_Master3')
    base_size = 64
    #max_examples = 1001
    max_examples = None
    data_file, labels_file = ibsplugin.get_identify_training_fpaths(ibs, base_size=base_size, max_examples=max_examples)

    model = models.IdentificationModel()
    config = dict(
        batch_size=8,
        learning_rate=.003,
    )
    nets_dir = ut.unixjoin(ibs.get_cachedir(), 'nets')
    ut.ensuredir(nets_dir)
    weights_file = join(nets_dir, 'ibeis_cnn_weights.pickle')
    train(data_file, labels_file, model, weights_file, **config)
    #X = k

def train_pz():
    r"""
    CommandLine:
        python -m ibeis_cnn.train --test-train_pz

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.train import *  # NOQA
        >>> result = train_pz()
        >>> print(result)
    """
    project_name            = 'plains'
    model                   = models.PZ_Model()

    root                    = abspath(join('..', 'data'))
    train_data_file         = join(root, 'numpy', project_name, 'X.npy')
    train_labels_file       = join(root, 'numpy', project_name, 'y.npy')
    results_path            = join(root, 'results', project_name)
    weights_file            = join(root, 'nets', project_name, 'ibeis_cnn_weights.pickle')
    pretrained_weights_file = join(root, 'nets', project_name, 'ibeis_cnn_weights.pickle')  # NOQA

    config                  = {
        'patience':   5,
        'max_epochs': 100,
        'test_time_augmentation': True,
        'regularization': 0.0001,
        'pretrained_weights_file': pretrained_weights_file,
    }

    train(train_data_file, train_labels_file, model, weights_file, results_path, **config)

def train_pz_girm():
    r"""
    CommandLine:
        python -m ibeis_cnn.train --test-train_pz_girm

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.train import *  # NOQA
        >>> result = train_pz_girm()
        >>> print(result)
    """
    project_name            = 'viewpoint'
    model                   = models.PZ_GIRM_Model()

    root                    = abspath(join('..', 'data'))
    train_data_file         = join(root, 'numpy', project_name, 'X.npy')
    train_labels_file       = join(root, 'numpy', project_name, 'y.npy')
    results_path            = join(root, 'results', project_name)
    weights_file            = join(root, 'nets', project_name, 'ibeis_cnn_weights.pickle')
    pretrained_weights_file = join(root, 'nets', project_name, 'ibeis_cnn_weights.pickle')  # NOQA

    config                  = {
        'patience':   5,
        'max_epochs': 100,
        'test_time_augmentation': True,
        'regularization': 0.0001,
        # 'pretrained_weights_file': pretrained_weights_file,
    }

    train(train_data_file, train_labels_file, model, weights_file, results_path, **config)

if __name__ == '__main__':
    """
    CommandLine:
        python -m ibeis_cnn.train
        python -m ibeis_cnn.train --allexamples
        python -m ibeis_cnn.train --allexamples --noface --nosrc

    CommandLine:
        cd %CODE_DIR%/ibies_cnn/code
        cd $CODE_DIR/ibies_cnn/code
        code
        cd ibeis_cnn/code
        python train.py

    PythonPrereqs:
        pip install theano
        pip install git+https://github.com/Lasagne/Lasagne.git
        pip install git+git://github.com/lisa-lab/pylearn2.git
        #pip install lasagne
        #pip install pylearn2
        git clone git://github.com/lisa-lab/pylearn2.git
        git clone https://github.com/Lasagne/Lasagne.git
        cd pylearn2
        python setup.py develop
        cd ..
        cd Lesagne
        git checkout 8758ac1434175159e5c1f30123041799c2b6098a
        python setup.py develop
    """
    #train_pz()
    import multiprocessing
    multiprocessing.freeze_support()  # for win32
    import utool as ut  # NOQA
    ut.doctest_funcs()
<commit_msg>patience increase<commit_after>#!/usr/bin/env python
"""
constructs the Theano optimization and trains a learning model,
optionally by initializing the network with pre-trained weights.
"""
from __future__ import absolute_import, division, print_function
from ibeis_cnn import utils
from ibeis_cnn import models
from ibeis_cnn import ibsplugin

import time
import theano
import numpy as np
import cPickle as pickle
from lasagne import layers
from sklearn import preprocessing

import utool as ut
import six
from os.path import join, abspath, dirname, exists

def train(data_file, labels_file, model, weights_file, results_path,
          pretrained_weights_file=None, pretrained_kwargs=False, **kwargs):
    """
    Driver function

    Args:
        data_file (?):
        labels_file (?):
        model (?):
        weights_file (?):
        pretrained_weights_file (None):
        pretrained_kwargs (bool):
    """

    ######################################################################################

    # Load the data
    print('\n[data] loading data...')
    print('data_file = %r' % (data_file,))
    print('labels_file = %r' % (labels_file,))
    data, labels = utils.load(data_file, labels_file)

    # Ensure results dir
    weights_path = dirname(abspath(weights_file))
    ut.ensuredir(weights_path)
    ut.ensuredir(results_path)
    if pretrained_weights_file is not None:
        pretrained_weights_path = dirname(abspath(pretrained_weights_file))
        ut.ensuredir(pretrained_weights_path)

    # Training parameters defaults
    utils._update(kwargs, 'center',         True)
    utils._update(kwargs, 'encode',         True)
    utils._update(kwargs, 'learning_rate',  0.01)
    utils._update(kwargs, 'momentum',       0.9)
    utils._update(kwargs, 'batch_size',     128)
    utils._update(kwargs, 'patience',       10)
    utils._update(kwargs, 'test',           5)  # Test every X epochs
    utils._update(kwargs, 'max_epochs',     kwargs.get('patience') * 10)
    utils._update(kwargs, 'regularization', None)
    utils._update(kwargs, 'test_time_augmentation',  False)
    utils._update(kwargs, 'output_dims',    None)
    utils._update(kwargs, 'show_features',  True)

    # Automatically figure out how many classes
    if kwargs.get('output_dims') is None:
        kwargs['output_dims'] = len(list(np.unique(labels)))

    # print('[load] adding channels...')
    # data = utils.add_channels(data)
    print('[train]     data.shape = %r' % (data.shape,))
    print('[train]     data.dtype = %r' % (data.dtype,))
    print('[train]     labels.shape = %r' % (labels.shape,))
    print('[train]     labels.dtype = %r' % (labels.dtype,))

    labelhist = {key: len(val) for key, val in six.iteritems(ut.group_items(labels, labels))}
    print('label stats = \n' + ut.dict_str(labelhist))
    print('train kwargs = \n' + (ut.dict_str(kwargs)))

    # Encoding labels
    kwargs['encoder'] = None
    if kwargs.get('encode', False):
        kwargs['encoder'] = preprocessing.LabelEncoder()
        kwargs['encoder'].fit(labels)

    # utils.show_image_from_data(data)

    # Split the dataset into training and validation
    print('[train] creating train, validation datasaets...')
    dataset = utils.train_test_split(data, labels, eval_size=0.2)
    X_train, y_train, X_valid, y_valid = dataset
    dataset = utils.train_test_split(X_train, y_train, eval_size=0.1)
    X_train, y_train, X_test, y_test = dataset

    # Build and print the model
    print('\n[model] building model...')
    input_cases, input_channels, input_height, input_width = data.shape
    output_layer = model.build_model(
        kwargs.get('batch_size'), input_width, input_height,
        input_channels, kwargs.get('output_dims'))
    utils.print_layer_info(output_layer)

    # Create the Theano primitives
    print('[model] creating Theano primitives...')
    learning_rate_theano = theano.shared(utils.float32(kwargs.get('learning_rate')))
    all_iters = utils.create_iter_funcs(learning_rate_theano, output_layer, **kwargs)
    train_iter, valid_iter, test_iter = all_iters

    # Load the pretrained model if specified
    if pretrained_weights_file is not None and exists(pretrained_weights_file):
        print('[model] loading pretrained weights from %s' % (pretrained_weights_file))
        with open(pretrained_weights_file, 'rb') as pfile:
            kwargs_ = pickle.load(pfile)
            pretrained_weights = kwargs_.get('best_weights', None)
            layers.set_all_param_values(output_layer, pretrained_weights)
            if pretrained_kwargs:
                kwargs = kwargs_

    # Center the data by subtracting the mean (AFTER KWARGS UPDATE)
    if kwargs.get('center'):
        print('[train] applying data centering...')
        utils._update(kwargs, 'center_mean', np.mean(X_train, axis=0))
        # utils._update(kwargs, 'center_std', np.std(X_train, axis=0))
        utils._update(kwargs, 'center_std', 255.0)
    else:
        utils._update(kwargs, 'center_mean', 0.0)
        utils._update(kwargs, 'center_std', 255.0)

    # Begin training the neural network
    vals = (utils.get_current_time(), kwargs.get('learning_rate'), )
    print('\n[train] starting training at %s with learning rate %.9f' % vals)
    utils.print_header_columns()

    utils._update(kwargs, 'best_weights',        None)
    utils._update(kwargs, 'best_valid_accuracy', 0.0)
    utils._update(kwargs, 'best_test_accuracy',  0.0)
    utils._update(kwargs, 'best_train_loss',     np.inf)
    utils._update(kwargs, 'best_valid_loss',     np.inf)
    utils._update(kwargs, 'best_valid_accuracy', 0.0)
    utils._update(kwargs, 'best_test_accuracy',  0.0)
    try:
        epoch = 0
        epoch_marker = epoch
        while True:
            try:
                # Start timer
                t0 = time.time()

                # compute the loss over all training and validation batches
                augment_fn = getattr(model, 'augment', None)
                avg_train_loss = utils.forward_train(X_train, y_train, train_iter, rand=True,
                                                     augment=augment_fn, **kwargs)
                if kwargs.get('test_time_augmentation', False):
                    avg_valid_data = utils.forward_valid(X_valid, y_valid, valid_iter,
                                                         augment=augment_fn, **kwargs)
                else:
                    avg_valid_data = utils.forward_valid(X_valid, y_valid, valid_iter, **kwargs)
                avg_valid_loss, avg_valid_accuracy = avg_valid_data

                # If the training loss is nan, the training has diverged
                if np.isnan(avg_train_loss):
                    print('\n[train] training diverged\n')
                    break

                # Is this model the best we've ever seen?
                best_found = avg_valid_loss < kwargs.get('best_valid_loss')
                if best_found:
                    kwargs['best_epoch'] = epoch
                    epoch_marker = epoch
                    kwargs['best_weights'] = layers.get_all_param_values(output_layer)

                # compute the loss over all testing batches
                request_test = kwargs.get('test') is not None and epoch % kwargs.get('test') == 0
                if best_found or request_test:
                    avg_test_accuracy = utils.forward_test(X_test, y_test, test_iter,
                                                           results_path, **kwargs)
                    # Output the layer 1 features
                    if kwargs.get('show_features'):
                        utils.show_convolutional_features(output_layer, results_path, color=True, target=0)
                else:
                    avg_test_accuracy = None

                # Running tab for what the best model
                if avg_train_loss < kwargs.get('best_train_loss'):
                    kwargs['best_train_loss'] = avg_train_loss
                if avg_valid_loss < kwargs.get('best_valid_loss'):
                    kwargs['best_valid_loss'] = avg_valid_loss
                if avg_valid_accuracy > kwargs.get('best_valid_accuracy'):
                    kwargs['best_valid_accuracy'] = avg_valid_accuracy
                if avg_test_accuracy > kwargs.get('best_test_accuracy'):
                    kwargs['best_test_accuracy'] = avg_test_accuracy

                # Learning rate schedule update
                if epoch >= epoch_marker + kwargs.get('patience'):
                    epoch_marker = epoch
                    utils.set_learning_rate(learning_rate_theano, model.learning_rate_update)

                # End timer
                t1 = time.time()

                # Increment the epoch
                epoch += 1
                utils.print_epoch_info(avg_train_loss, avg_valid_loss,
                                       avg_valid_accuracy, avg_test_accuracy,
                                       epoch, t1 - t0, **kwargs)

                # Break on max epochs
                if epoch >= kwargs.get('max_epochs'):
                    print('\n[train] maximum number of epochs reached\n')
                    break
            except KeyboardInterrupt:
                # We have caught the Keyboard Interrupt, figure out what resolution mode
                print('\n[train] Caught CRTL+C')
                resolution = ''
                while not (resolution.isdigit() and int(resolution) in [1, 2, 3]):
                    print('\n[train] What do you want to do?')
                    print('[train]     1 - Shock weights')
                    print('[train]     2 - Save best weights')
                    print('[train]     3 - Stop network training')
                    resolution = raw_input('[train] Resolution: ')
                resolution = int(resolution)
                # We have a resolution
                if resolution == 1:
                    # Shock the weights of the network
                    utils.shock_network(output_layer)
                    epoch_marker = epoch
                    utils.set_learning_rate(learning_rate_theano, model.learning_rate_shock)
                elif resolution == 2:
                    # Save the weights of the network
                    utils.save_model(kwargs, weights_file)
                else:
                    # Terminate the network training
                    raise KeyboardInterrupt
    except KeyboardInterrupt:
        print('\n\n[train] ...stopping network training\n')

    # Save the best network
    utils.save_model(kwargs, weights_file)

#@ibeis.register_plugin()
def train_identification_pz():
    r"""

    CommandLine:
        python -m ibeis_cnn.train --test-train_identification_pz

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.train import *  # NOQA
        >>> train_identification_pz()
    """
    print('get_identification_decision_training_data')
    import ibeis
    ibs = ibeis.opendb('NNP_Master3')
    base_size = 64
    #max_examples = 1001
    max_examples = None
    data_file, labels_file = ibsplugin.get_identify_training_fpaths(ibs, base_size=base_size, max_examples=max_examples)

    model = models.IdentificationModel()
    config = dict(
        batch_size=8,
        learning_rate=.003,
    )
    nets_dir = ut.unixjoin(ibs.get_cachedir(), 'nets')
    ut.ensuredir(nets_dir)
    weights_file = join(nets_dir, 'ibeis_cnn_weights.pickle')
    train(data_file, labels_file, model, weights_file, **config)
    #X = k

def train_pz():
    r"""
    CommandLine:
        python -m ibeis_cnn.train --test-train_pz

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.train import *  # NOQA
        >>> result = train_pz()
        >>> print(result)
    """
    project_name            = 'plains'
    model                   = models.PZ_Model()

    root                    = abspath(join('..', 'data'))
    train_data_file         = join(root, 'numpy', project_name, 'X.npy')
    train_labels_file       = join(root, 'numpy', project_name, 'y.npy')
    results_path            = join(root, 'results', project_name)
    weights_file            = join(root, 'nets', project_name, 'ibeis_cnn_weights.pickle')
    pretrained_weights_file = join(root, 'nets', project_name, 'ibeis_cnn_weights.pickle')  # NOQA

    config                  = {
        'patience': 10,
        'max_epochs': 100,
        'regularization': 0.0001,
        'test_time_augmentation': True,
        'pretrained_weights_file': pretrained_weights_file,
    }

    train(train_data_file, train_labels_file, model, weights_file, results_path, **config)

def train_pz_girm():
    r"""
    CommandLine:
        python -m ibeis_cnn.train --test-train_pz_girm

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.train import *  # NOQA
        >>> result = train_pz_girm()
        >>> print(result)
    """
    project_name            = 'viewpoint'
    model                   = models.PZ_GIRM_Model()

    root                    = abspath(join('..', 'data'))
    train_data_file         = join(root, 'numpy', project_name, 'X.npy')
    train_labels_file       = join(root, 'numpy', project_name, 'y.npy')
    results_path            = join(root, 'results', project_name)
    weights_file            = join(root, 'nets', project_name, 'ibeis_cnn_weights.pickle')
    pretrained_weights_file = join(root, 'nets', project_name, 'ibeis_cnn_weights.pickle')  # NOQA

    config                  = {
        'patience': 10,
        'max_epochs': 100,
        'regularization': 0.0001,
        'test_time_augmentation': True,
        'pretrained_weights_file': pretrained_weights_file,
    }

    train(train_data_file, train_labels_file, model, weights_file, results_path, **config)

if __name__ == '__main__':
    """
    CommandLine:
        python -m ibeis_cnn.train
        python -m ibeis_cnn.train --allexamples
        python -m ibeis_cnn.train --allexamples --noface --nosrc

    CommandLine:
        cd %CODE_DIR%/ibies_cnn/code
        cd $CODE_DIR/ibies_cnn/code
        code
        cd ibeis_cnn/code
        python train.py

    PythonPrereqs:
        pip install theano
        pip install git+https://github.com/Lasagne/Lasagne.git
        pip install git+git://github.com/lisa-lab/pylearn2.git
        #pip install lasagne
        #pip install pylearn2
        git clone git://github.com/lisa-lab/pylearn2.git
        git clone https://github.com/Lasagne/Lasagne.git
        cd pylearn2
        python setup.py develop
        cd ..
        cd Lesagne
        git checkout 8758ac1434175159e5c1f30123041799c2b6098a
        python setup.py develop
    """
    #train_pz()
    import multiprocessing
    multiprocessing.freeze_support()  # for win32
    import utool as ut  # NOQA
    ut.doctest_funcs()
<|endoftext|>


<commit_before># parsetree.py
# Copyright (C) 2006, 2007 Michael Bayer mike_mp@zzzcomputing.com
#
# This module is part of Mako and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php

"""defines the parse tree components for Mako templates."""

from mako import exceptions, ast, util, filters
import re

class Node(object):
    """base class for a Node in the parse tree."""
    def __init__(self, lineno, pos, filename):
        self.lineno = lineno
        self.pos = pos
        self.filename = filename
    def get_children(self):
        return []
    def accept_visitor(self, visitor):
        def traverse(node):
            for n in node.get_children():
                n.accept_visitor(visitor)
        method = getattr(visitor, "visit" + self.__class__.__name__, traverse)
        method(self)

class TemplateNode(Node):
    """a 'container' node that stores the overall collection of nodes."""
    def __init__(self, filename):
        super(TemplateNode, self).__init__(0, 0, filename)
        self.nodes = []
        self.page_attributes = {}
    def get_children(self):
        return self.nodes
    def __repr__(self):
        return "TemplateNode(%s, %s)" % (repr(self.page_attributes), repr(self.nodes))

class ControlLine(Node):
    """defines a control line, a line-oriented python line or end tag.

    % if foo:
        (markup)
    % endif
    """
    def __init__(self, keyword, isend, text, **kwargs):
        super(ControlLine, self).__init__(**kwargs)
        self.text = text
        self.keyword = keyword
        self.isend = isend
        self.is_primary = keyword in ['for','if', 'while', 'try']
        if self.isend:
            self._declared_identifiers = []
            self._undeclared_identifiers = []
        else:
            code = ast.PythonFragment(text, self.lineno, self.pos, self.filename)
            (self._declared_identifiers, self._undeclared_identifiers) = (code.declared_identifiers, code.undeclared_identifiers)
    def declared_identifiers(self):
        return self._declared_identifiers
    def undeclared_identifiers(self):
        return self._undeclared_identifiers
    def is_ternary(self, keyword):
        """return true if the given keyword is a ternary keyword for this ControlLine"""
        return keyword in {
            'if':util.Set(['else', 'elif']),
            'try':util.Set(['except', 'finally']),
            'for':util.Set(['else'])
        }.get(self.keyword, [])
    def __repr__(self):
        return "ControlLine(%s, %s, %s, %s)" % (repr(self.keyword), repr(self.text), repr(self.isend), repr((self.lineno, self.pos)))

class Text(Node):
    """defines plain text in the template."""
    def __init__(self, content, **kwargs):
        super(Text, self).__init__(**kwargs)
        self.content = content
    def __repr__(self):
        return "Text(%s, %s)" % (repr(self.content), repr((self.lineno, self.pos)))

class Code(Node):
    """defines a Python code block, either inline or module level.

    inline:
    <%
        x = 12
    %>

    module level:
    <%!
        import logger
    %>

    """
    def __init__(self, text, ismodule, **kwargs):
        super(Code, self).__init__(**kwargs)
        self.text = text
        self.ismodule = ismodule
        self.code = ast.PythonCode(text, self.lineno, self.pos, self.filename)
    def declared_identifiers(self):
        return self.code.declared_identifiers
    def undeclared_identifiers(self):
        return self.code.undeclared_identifiers
    def __repr__(self):
        return "Code(%s, %s, %s)" % (repr(self.text), repr(self.ismodule), repr((self.lineno, self.pos)))

class Comment(Node):
    """defines a comment line.

    # this is a comment

    """
    def __init__(self, text, **kwargs):
        super(Comment, self).__init__(**kwargs)
        self.text = text
    def __repr__(self):
        return "Comment(%s, %s)" % (repr(self.text), repr((self.lineno, self.pos)))

class Expression(Node):
    """defines an inline expression.

    ${x+y}

    """
    def __init__(self, text, escapes, **kwargs):
        super(Expression, self).__init__(**kwargs)
        self.text = text
        self.escapes = escapes
        self.escapes_code = ast.ArgumentList(escapes, self.lineno, self.pos, self.filename)
        self.code = ast.PythonCode(text, self.lineno, self.pos, self.filename)
    def declared_identifiers(self):
        return []
    def undeclared_identifiers(self):
        # TODO: make the "filter" shortcut list configurable at parse/gen time
        return self.code.undeclared_identifiers.union(self.escapes_code.undeclared_identifiers.difference(util.Set(filters.DEFAULT_ESCAPES.keys())))
    def __repr__(self):
        return "Expression(%s, %s, %s)" % (repr(self.text), repr(self.escapes_code.args), repr((self.lineno, self.pos)))

class _TagMeta(type):
    """metaclass to allow Tag to produce a subclass according to its keyword"""
    _classmap = {}
    def __init__(cls, clsname, bases, dict):
        if cls.__keyword__ is not None:
            cls._classmap[cls.__keyword__] = cls
            super(_TagMeta, cls).__init__(clsname, bases, dict)
    def __call__(cls, keyword, attributes, **kwargs):
        try:
            cls = _TagMeta._classmap[keyword]
        except KeyError:
            raise exceptions.CompileException("No such tag: '%s'" % keyword, kwargs['lineno'], kwargs['pos'], kwargs['filename'])
        return type.__call__(cls, keyword, attributes, **kwargs)

class Tag(Node):
    """abstract base class for tags.

    <%sometag/>

    <%someothertag>
        stuff
    </%someothertag>
    """
    __metaclass__ = _TagMeta
    __keyword__ = None
    def __init__(self, keyword, attributes, expressions, nonexpressions, required, **kwargs):
        """construct a new Tag instance.

        this constructor not called directly, and is only called by subclasses.

        keyword - the tag keyword

        attributes - raw dictionary of attribute key/value pairs

        expressions - a util.Set of identifiers that are legal attributes, which can also contain embedded expressions

        nonexpressions - a util.Set of identifiers that are legal attributes, which cannot contain embedded expressions

        **kwargs - other arguments passed to the Node superclass (lineno, pos)"""
        super(Tag, self).__init__(**kwargs)
        self.keyword = keyword
        self.attributes = attributes
        self._parse_attributes(expressions, nonexpressions)
        missing = [r for r in required if r not in self.parsed_attributes]
        if len(missing):
            raise exceptions.CompileException("Missing attribute(s): %s" % ",".join([repr(m) for m in missing]), self.lineno, self.pos, self.filename)
        self.parent = None
        self.nodes = []
    def is_root(self):
        return self.parent is None
    def get_children(self):
        return self.nodes
    def _parse_attributes(self, expressions, nonexpressions):
        undeclared_identifiers = util.Set()
        self.parsed_attributes = {}
        for key in self.attributes:
            if key in expressions:
                expr = []
                for x in re.split(r'(\${.+?})', self.attributes[key]):
                    m = re.match(r'^\${(.+?)}$', x)
                    if m:
                        code = ast.PythonCode(m.group(1), self.lineno, self.pos, self.filename)
                        undeclared_identifiers = undeclared_identifiers.union(code.undeclared_identifiers)
                        expr.append(m.group(1))
                    else:
                        if x:
                            expr.append(repr(x))
                self.parsed_attributes[key] = " + ".join(expr)
            elif key in nonexpressions:
                if re.search(r'${.+?}', self.attributes[key]):
                    raise exceptions.CompileException("Attibute '%s' in tag '%s' does not allow embedded expressions"  %(key, self.keyword), self.lineno, self.pos, self.filename)
                self.parsed_attributes[key] = repr(self.attributes[key])
            else:
                raise exceptions.CompileException("Invalid attribute for tag '%s': '%s'" %(self.keyword, key), self.lineno, self.pos, self.filename)
        self.expression_undeclared_identifiers = undeclared_identifiers
    def declared_identifiers(self):
        return []
    def undeclared_identifiers(self):
        return self.expression_undeclared_identifiers
    def __repr__(self):
        return "%s(%s, %s, %s, %s)" % (self.__class__.__name__, repr(self.keyword), repr(self.attributes), repr((self.lineno, self.pos)), repr([repr(x) for x in self.nodes]))

class IncludeTag(Tag):
    __keyword__ = 'include'
    def __init__(self, keyword, attributes, **kwargs):
        super(IncludeTag, self).__init__(keyword, attributes, ('file', 'import', 'args'), (), ('file',), **kwargs)
        self.page_args = ast.PythonCode("foo(%s)" % attributes.get('args', ''), self.lineno, self.pos, self.filename)
    def declared_identifiers(self):
        return []
    def undeclared_identifiers(self):
        return self.page_args.undeclared_identifiers

class NamespaceTag(Tag):
    __keyword__ = 'namespace'
    def __init__(self, keyword, attributes, **kwargs):
        super(NamespaceTag, self).__init__(keyword, attributes, (), ('name','inheritable','file','import','module'), (), **kwargs)
        self.name = attributes.get('name', '__anon_%s' % hex(abs(id(self))))
        if not 'name' in attributes and not 'import' in attributes:
            raise exceptions.CompileException("'name' and/or 'import' attributes are required for <%namespace>", self.lineno, self.pos, self.filename)
    def declared_identifiers(self):
        return []

class TextTag(Tag):
    __keyword__ = 'text'
    def __init__(self, keyword, attributes, **kwargs):
        super(TextTag, self).__init__(keyword, attributes, (), ('filter'), (), **kwargs)
        self.filter_args = ast.ArgumentList(attributes.get('filter', ''), self.lineno, self.pos, self.filename)

class DefTag(Tag):
    __keyword__ = 'def'
    def __init__(self, keyword, attributes, **kwargs):
        super(DefTag, self).__init__(keyword, attributes, ('buffered', 'cached', 'cache_key', 'cache_timeout', 'cache_type', 'cache_dir', 'cache_url'), ('name','filter'), ('name',), **kwargs)
        name = attributes['name']
        if re.match(r'^[\w_]+$',name):
            raise exceptions.CompileException("Missing parenthesis in %def", self.lineno, self.pos, self.filename)
        self.function_decl = ast.FunctionDecl("def " + name + ":pass", self.lineno, self.pos, self.filename)
        self.name = self.function_decl.funcname
        self.filter_args = ast.ArgumentList(attributes.get('filter', ''), self.lineno, self.pos, self.filename)
    def declared_identifiers(self):
        return self.function_decl.argnames
    def undeclared_identifiers(self):
        res = []
        for c in self.function_decl.defaults:
            res += list(ast.PythonCode(c, self.lineno, self.pos, self.filename).undeclared_identifiers)
        return res + list(self.filter_args.undeclared_identifiers.difference(util.Set(filters.DEFAULT_ESCAPES.keys())))

class CallTag(Tag):
    __keyword__ = 'call'
    def __init__(self, keyword, attributes, **kwargs):
        super(CallTag, self).__init__(keyword, attributes, ('args'), ('expr',), ('expr',), **kwargs)
        self.code = ast.PythonCode(attributes['expr'], self.lineno, self.pos, self.filename)
        self.body_decl = ast.FunctionArgs(attributes.get('args', ''), self.lineno, self.pos, self.filename)
    def declared_identifiers(self):
        return self.code.declared_identifiers.union(self.body_decl.argnames)
    def undeclared_identifiers(self):
        return self.code.undeclared_identifiers

class InheritTag(Tag):
    __keyword__ = 'inherit'
    def __init__(self, keyword, attributes, **kwargs):
        super(InheritTag, self).__init__(keyword, attributes, ('file',), (), ('file',), **kwargs)

class PageTag(Tag):
    __keyword__ = 'page'
    def __init__(self, keyword, attributes, **kwargs):
        super(PageTag, self).__init__(keyword, attributes, ('cached', 'cache_key', 'cache_timeout', 'cache_type', 'cache_dir', 'args', 'expression_filter'), (), (), **kwargs)
        self.body_decl = ast.FunctionArgs(attributes.get('args', ''), self.lineno, self.pos, self.filename)
        self.filter_args = ast.ArgumentList(attributes.get('expression_filter', ''), self.lineno, self.pos, self.filename)
    def declared_identifiers(self):
        return self.body_decl.argnames

<commit_msg>cache_url attribute on page tag<commit_after># parsetree.py
# Copyright (C) 2006, 2007 Michael Bayer mike_mp@zzzcomputing.com
#
# This module is part of Mako and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php

"""defines the parse tree components for Mako templates."""

from mako import exceptions, ast, util, filters
import re

class Node(object):
    """base class for a Node in the parse tree."""
    def __init__(self, lineno, pos, filename):
        self.lineno = lineno
        self.pos = pos
        self.filename = filename
    def get_children(self):
        return []
    def accept_visitor(self, visitor):
        def traverse(node):
            for n in node.get_children():
                n.accept_visitor(visitor)
        method = getattr(visitor, "visit" + self.__class__.__name__, traverse)
        method(self)

class TemplateNode(Node):
    """a 'container' node that stores the overall collection of nodes."""
    def __init__(self, filename):
        super(TemplateNode, self).__init__(0, 0, filename)
        self.nodes = []
        self.page_attributes = {}
    def get_children(self):
        return self.nodes
    def __repr__(self):
        return "TemplateNode(%s, %s)" % (repr(self.page_attributes), repr(self.nodes))

class ControlLine(Node):
    """defines a control line, a line-oriented python line or end tag.

    % if foo:
        (markup)
    % endif
    """
    def __init__(self, keyword, isend, text, **kwargs):
        super(ControlLine, self).__init__(**kwargs)
        self.text = text
        self.keyword = keyword
        self.isend = isend
        self.is_primary = keyword in ['for','if', 'while', 'try']
        if self.isend:
            self._declared_identifiers = []
            self._undeclared_identifiers = []
        else:
            code = ast.PythonFragment(text, self.lineno, self.pos, self.filename)
            (self._declared_identifiers, self._undeclared_identifiers) = (code.declared_identifiers, code.undeclared_identifiers)
    def declared_identifiers(self):
        return self._declared_identifiers
    def undeclared_identifiers(self):
        return self._undeclared_identifiers
    def is_ternary(self, keyword):
        """return true if the given keyword is a ternary keyword for this ControlLine"""
        return keyword in {
            'if':util.Set(['else', 'elif']),
            'try':util.Set(['except', 'finally']),
            'for':util.Set(['else'])
        }.get(self.keyword, [])
    def __repr__(self):
        return "ControlLine(%s, %s, %s, %s)" % (repr(self.keyword), repr(self.text), repr(self.isend), repr((self.lineno, self.pos)))

class Text(Node):
    """defines plain text in the template."""
    def __init__(self, content, **kwargs):
        super(Text, self).__init__(**kwargs)
        self.content = content
    def __repr__(self):
        return "Text(%s, %s)" % (repr(self.content), repr((self.lineno, self.pos)))

class Code(Node):
    """defines a Python code block, either inline or module level.

    inline:
    <%
        x = 12
    %>

    module level:
    <%!
        import logger
    %>

    """
    def __init__(self, text, ismodule, **kwargs):
        super(Code, self).__init__(**kwargs)
        self.text = text
        self.ismodule = ismodule
        self.code = ast.PythonCode(text, self.lineno, self.pos, self.filename)
    def declared_identifiers(self):
        return self.code.declared_identifiers
    def undeclared_identifiers(self):
        return self.code.undeclared_identifiers
    def __repr__(self):
        return "Code(%s, %s, %s)" % (repr(self.text), repr(self.ismodule), repr((self.lineno, self.pos)))

class Comment(Node):
    """defines a comment line.

    # this is a comment

    """
    def __init__(self, text, **kwargs):
        super(Comment, self).__init__(**kwargs)
        self.text = text
    def __repr__(self):
        return "Comment(%s, %s)" % (repr(self.text), repr((self.lineno, self.pos)))

class Expression(Node):
    """defines an inline expression.

    ${x+y}

    """
    def __init__(self, text, escapes, **kwargs):
        super(Expression, self).__init__(**kwargs)
        self.text = text
        self.escapes = escapes
        self.escapes_code = ast.ArgumentList(escapes, self.lineno, self.pos, self.filename)
        self.code = ast.PythonCode(text, self.lineno, self.pos, self.filename)
    def declared_identifiers(self):
        return []
    def undeclared_identifiers(self):
        # TODO: make the "filter" shortcut list configurable at parse/gen time
        return self.code.undeclared_identifiers.union(self.escapes_code.undeclared_identifiers.difference(util.Set(filters.DEFAULT_ESCAPES.keys())))
    def __repr__(self):
        return "Expression(%s, %s, %s)" % (repr(self.text), repr(self.escapes_code.args), repr((self.lineno, self.pos)))

class _TagMeta(type):
    """metaclass to allow Tag to produce a subclass according to its keyword"""
    _classmap = {}
    def __init__(cls, clsname, bases, dict):
        if cls.__keyword__ is not None:
            cls._classmap[cls.__keyword__] = cls
            super(_TagMeta, cls).__init__(clsname, bases, dict)
    def __call__(cls, keyword, attributes, **kwargs):
        try:
            cls = _TagMeta._classmap[keyword]
        except KeyError:
            raise exceptions.CompileException("No such tag: '%s'" % keyword, kwargs['lineno'], kwargs['pos'], kwargs['filename'])
        return type.__call__(cls, keyword, attributes, **kwargs)

class Tag(Node):
    """abstract base class for tags.

    <%sometag/>

    <%someothertag>
        stuff
    </%someothertag>
    """
    __metaclass__ = _TagMeta
    __keyword__ = None
    def __init__(self, keyword, attributes, expressions, nonexpressions, required, **kwargs):
        """construct a new Tag instance.

        this constructor not called directly, and is only called by subclasses.

        keyword - the tag keyword

        attributes - raw dictionary of attribute key/value pairs

        expressions - a util.Set of identifiers that are legal attributes, which can also contain embedded expressions

        nonexpressions - a util.Set of identifiers that are legal attributes, which cannot contain embedded expressions

        **kwargs - other arguments passed to the Node superclass (lineno, pos)"""
        super(Tag, self).__init__(**kwargs)
        self.keyword = keyword
        self.attributes = attributes
        self._parse_attributes(expressions, nonexpressions)
        missing = [r for r in required if r not in self.parsed_attributes]
        if len(missing):
            raise exceptions.CompileException("Missing attribute(s): %s" % ",".join([repr(m) for m in missing]), self.lineno, self.pos, self.filename)
        self.parent = None
        self.nodes = []
    def is_root(self):
        return self.parent is None
    def get_children(self):
        return self.nodes
    def _parse_attributes(self, expressions, nonexpressions):
        undeclared_identifiers = util.Set()
        self.parsed_attributes = {}
        for key in self.attributes:
            if key in expressions:
                expr = []
                for x in re.split(r'(\${.+?})', self.attributes[key]):
                    m = re.match(r'^\${(.+?)}$', x)
                    if m:
                        code = ast.PythonCode(m.group(1), self.lineno, self.pos, self.filename)
                        undeclared_identifiers = undeclared_identifiers.union(code.undeclared_identifiers)
                        expr.append(m.group(1))
                    else:
                        if x:
                            expr.append(repr(x))
                self.parsed_attributes[key] = " + ".join(expr)
            elif key in nonexpressions:
                if re.search(r'${.+?}', self.attributes[key]):
                    raise exceptions.CompileException("Attibute '%s' in tag '%s' does not allow embedded expressions"  %(key, self.keyword), self.lineno, self.pos, self.filename)
                self.parsed_attributes[key] = repr(self.attributes[key])
            else:
                raise exceptions.CompileException("Invalid attribute for tag '%s': '%s'" %(self.keyword, key), self.lineno, self.pos, self.filename)
        self.expression_undeclared_identifiers = undeclared_identifiers
    def declared_identifiers(self):
        return []
    def undeclared_identifiers(self):
        return self.expression_undeclared_identifiers
    def __repr__(self):
        return "%s(%s, %s, %s, %s)" % (self.__class__.__name__, repr(self.keyword), repr(self.attributes), repr((self.lineno, self.pos)), repr([repr(x) for x in self.nodes]))

class IncludeTag(Tag):
    __keyword__ = 'include'
    def __init__(self, keyword, attributes, **kwargs):
        super(IncludeTag, self).__init__(keyword, attributes, ('file', 'import', 'args'), (), ('file',), **kwargs)
        self.page_args = ast.PythonCode("foo(%s)" % attributes.get('args', ''), self.lineno, self.pos, self.filename)
    def declared_identifiers(self):
        return []
    def undeclared_identifiers(self):
        return self.page_args.undeclared_identifiers

class NamespaceTag(Tag):
    __keyword__ = 'namespace'
    def __init__(self, keyword, attributes, **kwargs):
        super(NamespaceTag, self).__init__(keyword, attributes, (), ('name','inheritable','file','import','module'), (), **kwargs)
        self.name = attributes.get('name', '__anon_%s' % hex(abs(id(self))))
        if not 'name' in attributes and not 'import' in attributes:
            raise exceptions.CompileException("'name' and/or 'import' attributes are required for <%namespace>", self.lineno, self.pos, self.filename)
    def declared_identifiers(self):
        return []

class TextTag(Tag):
    __keyword__ = 'text'
    def __init__(self, keyword, attributes, **kwargs):
        super(TextTag, self).__init__(keyword, attributes, (), ('filter'), (), **kwargs)
        self.filter_args = ast.ArgumentList(attributes.get('filter', ''), self.lineno, self.pos, self.filename)

class DefTag(Tag):
    __keyword__ = 'def'
    def __init__(self, keyword, attributes, **kwargs):
        super(DefTag, self).__init__(keyword, attributes, ('buffered', 'cached', 'cache_key', 'cache_timeout', 'cache_type', 'cache_dir', 'cache_url'), ('name','filter'), ('name',), **kwargs)
        name = attributes['name']
        if re.match(r'^[\w_]+$',name):
            raise exceptions.CompileException("Missing parenthesis in %def", self.lineno, self.pos, self.filename)
        self.function_decl = ast.FunctionDecl("def " + name + ":pass", self.lineno, self.pos, self.filename)
        self.name = self.function_decl.funcname
        self.filter_args = ast.ArgumentList(attributes.get('filter', ''), self.lineno, self.pos, self.filename)
    def declared_identifiers(self):
        return self.function_decl.argnames
    def undeclared_identifiers(self):
        res = []
        for c in self.function_decl.defaults:
            res += list(ast.PythonCode(c, self.lineno, self.pos, self.filename).undeclared_identifiers)
        return res + list(self.filter_args.undeclared_identifiers.difference(util.Set(filters.DEFAULT_ESCAPES.keys())))

class CallTag(Tag):
    __keyword__ = 'call'
    def __init__(self, keyword, attributes, **kwargs):
        super(CallTag, self).__init__(keyword, attributes, ('args'), ('expr',), ('expr',), **kwargs)
        self.code = ast.PythonCode(attributes['expr'], self.lineno, self.pos, self.filename)
        self.body_decl = ast.FunctionArgs(attributes.get('args', ''), self.lineno, self.pos, self.filename)
    def declared_identifiers(self):
        return self.code.declared_identifiers.union(self.body_decl.argnames)
    def undeclared_identifiers(self):
        return self.code.undeclared_identifiers

class InheritTag(Tag):
    __keyword__ = 'inherit'
    def __init__(self, keyword, attributes, **kwargs):
        super(InheritTag, self).__init__(keyword, attributes, ('file',), (), ('file',), **kwargs)

class PageTag(Tag):
    __keyword__ = 'page'
    def __init__(self, keyword, attributes, **kwargs):
        super(PageTag, self).__init__(keyword, attributes, ('cached', 'cache_key', 'cache_timeout', 'cache_type', 'cache_dir', 'cache_url', 'args', 'expression_filter'), (), (), **kwargs)
        self.body_decl = ast.FunctionArgs(attributes.get('args', ''), self.lineno, self.pos, self.filename)
        self.filter_args = ast.ArgumentList(attributes.get('expression_filter', ''), self.lineno, self.pos, self.filename)
    def declared_identifiers(self):
        return self.body_decl.argnames

<|endoftext|>


<commit_before>#!/usr/bin/env python3.5
# -*- coding: utf-8 -*-
#

import os.path
from flask import Flask, render_template, url_for, request, g, redirect
from flask.ext.httpauth import HTTPDigestAuth
#from flask.ext.uploads import delete, init, save, Upload
#from flask.ext.sqlalchemy import SQLAlchemy
#from flask.ext.storage import get_default_storage_class
from werkzeug.utils import secure_filename
import redis
import requests
import json
users = {
    "admin": "test",
}

ALLOWED_EXTENSIONS = set(['applet', 'bin', 'dll', 'doc', 'exe', 'html', 'ie', 'jar', 'pdf', 'vbs', 'xls', 'zip', 'jpg', 'jpeg', 'gif', 'png', 'tif', 'tiff', 'apk', 'cmd', 'bat', 'infected'])

# Configurables

BASE_URL = "http://localhost:8090"
TASKS_VIEW = "/tasks/view/"
TASKS_REPORT = "/tasks/report/"
MACHINES_LIST = "/machines/list"

app = Flask(__name__, static_folder='static', static_url_path='/static')
app.config.from_object(__name__)
#app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///home/cuckoo/tmp/test.db'
app.config['DEBUG'] = True
#app.config['PROPAGATE_EXCEPTIONS'] = True
app.config['DEFAULT_FILE_STORAGE'] = 'filesystem'
app.config['UPLOAD_FOLDER']  = '/home/cuckoo/dma-frontend/web/static/upload'
app.config['SECRET_KEY'] = 'put your secret key'
#init(SQLAlchemy(app), get_default_storage_class(app))
auth = HTTPDigestAuth()

def status(username, retmax=20):
    red = redis.StrictRedis(host='localhost', port=6379, db=5)
    print(username)
    t = red.smembers("t:"+username)
    x = []
    at = list(t)
    at = [a for a in at if a != 'null']
    for task in sorted(at, key=lambda x: float(x), reverse=True)[:retmax]:
        r = requests.get(BASE_URL+TASKS_VIEW+task)
        j = json.loads(r.text)
        x.append(j)
    return x

def machines():
    r = requests.get(BASE_URL+MACHINES_LIST)
    return json.loads(r.text)

@auth.get_password
def get_pw(username):
    if username in users:
        return users[username]
    return None

@app.route('/')
@auth.login_required
def index():
    s = status(auth.username())
    m = machines()
    return render_template('main.html', auth=auth, s=s, machines=m)

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS

@app.route('/upload', methods=['GET', 'POST'])
@auth.login_required
def upload():
    s = status(auth.username())
    m = machines()
    if request.method == 'POST' and request.files['sample'] and request.form['machine'] and request.form['package']:
        f = request.files['sample']
        if f and allowed_file(f.filename):
            sfname = secure_filename(f.filename)
            file.save(os.path.join(app.config['UPLOAD_FOLDER'], sfname))
        r = redis.StrictRedis(host='localhost', port=6379, db=5)
        r.rpush("submit", auth.username()+":"+app.config['UPLOADS_FOLDER']+"/"+request.files['sample'].filename+":"+request.form['machine']+":"+request.form['package'])
    return render_template('main.html', auth=auth, upload=request.files['sample'], s=s, machines=m)

@app.route('/rfetch/<int:taskid>', methods=['GET'])
@auth.login_required
def rfetch(taskid, auth=auth):
    red = redis.StrictRedis(host='localhost', port=6379, db=5)
    t = red.smembers("t:"+auth.username())
    if str(taskid) in t:
        r = requests.get(BASE_URL+TASKS_REPORT+str(taskid)+"/html")
        return r.text
    else:
        return "Not allowed"
if __name__ == '__main__':
        app.run(host='0.0.0.0')
<commit_msg>- amended upload folder and fixed file -> f typo<commit_after>#!/usr/bin/env python3.5
# -*- coding: utf-8 -*-
#

import os.path
from flask import Flask, render_template, url_for, request, g, redirect
from flask.ext.httpauth import HTTPDigestAuth
#from flask.ext.uploads import delete, init, save, Upload
#from flask.ext.sqlalchemy import SQLAlchemy
#from flask.ext.storage import get_default_storage_class
from werkzeug.utils import secure_filename
import redis
import requests
import json
users = {
    "admin": "test",
}

ALLOWED_EXTENSIONS = set(['applet', 'bin', 'dll', 'doc', 'exe', 'html', 'ie', 'jar', 'pdf', 'vbs', 'xls', 'zip', 'jpg', 'jpeg', 'gif', 'png', 'tif', 'tiff', 'apk', 'cmd', 'bat', 'infected'])

# Configurables

BASE_URL = "http://localhost:8090"
TASKS_VIEW = "/tasks/view/"
TASKS_REPORT = "/tasks/report/"
MACHINES_LIST = "/machines/list"

app = Flask(__name__, static_folder='static', static_url_path='/static')
app.config.from_object(__name__)
#app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///home/cuckoo/tmp/test.db'
app.config['DEBUG'] = True
#app.config['PROPAGATE_EXCEPTIONS'] = True
app.config['DEFAULT_FILE_STORAGE'] = 'filesystem'
app.config['UPLOAD_FOLDER']  = '/home/cuckoo/dma-frontend/web/static/upload'
app.config['SECRET_KEY'] = 'put your secret key'
#init(SQLAlchemy(app), get_default_storage_class(app))
auth = HTTPDigestAuth()

def status(username, retmax=20):
    red = redis.StrictRedis(host='localhost', port=6379, db=5)
    print(username)
    t = red.smembers("t:"+username)
    x = []
    at = list(t)
    at = [a for a in at if a != 'null']
    for task in sorted(at, key=lambda x: float(x), reverse=True)[:retmax]:
        r = requests.get(BASE_URL+TASKS_VIEW+task)
        j = json.loads(r.text)
        x.append(j)
    return x

def machines():
    r = requests.get(BASE_URL+MACHINES_LIST)
    return json.loads(r.text)

@auth.get_password
def get_pw(username):
    if username in users:
        return users[username]
    return None

@app.route('/')
@auth.login_required
def index():
    s = status(auth.username())
    m = machines()
    return render_template('main.html', auth=auth, s=s, machines=m)

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS

@app.route('/upload', methods=['GET', 'POST'])
@auth.login_required
def upload():
    s = status(auth.username())
    m = machines()
    if request.method == 'POST' and request.files['sample'] and request.form['machine'] and request.form['package']:
        f = request.files['sample']
        if f and allowed_file(f.filename):
            sfname = secure_filename(f.filename)
            f.save(os.path.join(app.config['UPLOAD_FOLDER'], sfname))
        r = redis.StrictRedis(host='localhost', port=6379, db=5)
        r.rpush("submit", auth.username()+":"+app.config['UPLOAD_FOLDER']+"/"+request.files['sample'].filename+":"+request.form['machine']+":"+request.form['package'])
    return render_template('main.html', auth=auth, upload=request.files['sample'], s=s, machines=m)

@app.route('/rfetch/<int:taskid>', methods=['GET'])
@auth.login_required
def rfetch(taskid, auth=auth):
    red = redis.StrictRedis(host='localhost', port=6379, db=5)
    t = red.smembers("t:"+auth.username())
    if str(taskid) in t:
        r = requests.get(BASE_URL+TASKS_REPORT+str(taskid)+"/html")
        return r.text
    else:
        return "Not allowed"
if __name__ == '__main__':
        app.run(host='0.0.0.0')
<|endoftext|>


<commit_before>from django.db import models

# Create your models here.
<commit_msg>Add BlogPost and Comment models<commit_after>from django.db import models
from django.contrib.auth.models import User

class BlogPost(models.Model):
    user = models.ForeignKey(User)
    title = models.CharField(max_lenght = 120)
    text = models.TextField()
    date = models.DateTimeField(auto_now_add = True)

class Comment(models.Model):
    user = models.ForeignKey(User)
    post = models.ForeignKey(BlogPost)
    text = models.TextField()
    date = models.DateTimeField(auto_now_add = True)
<|endoftext|>