<commit_before>#!/usr/bin/python

# Copyright (c) 2011 Jimmy Cao
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from oyoyo.client import IRCClient
from oyoyo.parse import parse_nick
import logging
import botconfig
import wolfgame

def on_privmsg(cli, rawnick, chan, msg):
    if chan != botconfig.NICK:  #not a PM
        for x in wolfgame.COMMANDS.keys():
            if not x or msg.lower().startswith(botconfig.CMD_CHAR+x):
                h = msg[len(x)+1:]
                if not h or h[0] == " " or not x:
                    for fn in wolfgame.COMMANDS[x]:
                        fn(cli, rawnick, chan, h.lstrip())
    else:
        for x in wolfgame.PM_COMMANDS.keys():
            if msg.lower().startswith(botconfig.CMD_CHAR+x):
                h = msg[len(x)+1:]
            elif not x or msg.lower().startswith(x):
                h = msg[len(x):]
            else:
                continue
            if not h or h[0] == " " or not x:
                for fn in wolfgame.PM_COMMANDS[x]:
                    fn(cli, rawnick, h.lstrip())

def __unhandled__(cli, prefix, cmd, *args):
    if cmd in wolfgame.HOOKS.keys():
        largs = list(args)
        for i,arg in enumerate(largs):
            if isinstance(arg, bytes): largs[i] = arg.decode('ascii')
        for fn in wolfgame.HOOKS[cmd]:
            fn(cli, prefix, *largs)
    else:
        logging.debug('Unhandled command {0}({1})'.format(cmd, [arg.decode('utf_8')
                                                              for arg in args
                                                              if isinstance(arg, bytes)]))

def main():
    logging.basicConfig(level=logging.DEBUG)
    cli = IRCClient(
                      {"privmsg":on_privmsg,
                       "":__unhandled__},
                     host=botconfig.HOST,
                     port=botconfig.PORT,
                     nickname=botconfig.NICK,
                     connect_cb=wolfgame.connect_callback
                    )
    cli.mainLoop()

if __name__ == "__main__":
    main()<commit_msg>test cia one more time<commit_after>#!/usr/bin/python

# Copyright (c) 2011 Jimmy Cao
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from oyoyo.client import IRCClient
from oyoyo.parse import parse_nick
import logging
import botconfig
import wolfgame

def on_privmsg(cli, rawnick, chan, msg):
    if chan != botconfig.NICK:  #not a PM
        for x in wolfgame.COMMANDS.keys():
            if not x or msg.lower().startswith(botconfig.CMD_CHAR+x):
                h = msg[len(x)+1:]
                if not h or h[0] == " " or not x:
                    for fn in wolfgame.COMMANDS[x]:
                        fn(cli, rawnick, chan, h.lstrip())
    else:
        for x in wolfgame.PM_COMMANDS.keys():
            if msg.lower().startswith(botconfig.CMD_CHAR+x):
                h = msg[len(x)+1:]
            elif not x or msg.lower().startswith(x):
                h = msg[len(x):]
            else:
                continue
            if not h or h[0] == " " or not x:
                for fn in wolfgame.PM_COMMANDS[x]:
                    fn(cli, rawnick, h.lstrip())

def __unhandled__(cli, prefix, cmd, *args):
    if cmd in wolfgame.HOOKS.keys():
        largs = list(args)
        for i,arg in enumerate(largs):
            if isinstance(arg, bytes): largs[i] = arg.decode('ascii')
        for fn in wolfgame.HOOKS[cmd]:
            fn(cli, prefix, *largs)
    else:
        logging.debug('Unhandled command {0}({1})'.format(cmd, [arg.decode('utf_8')
                                                              for arg in args
                                                              if isinstance(arg, bytes)]))

def main():
    logging.basicConfig(level=logging.DEBUG)
    cli = IRCClient(
                      {"privmsg":on_privmsg,
                       "":__unhandled__},
                     host=botconfig.HOST,
                     port=botconfig.PORT,
                     nickname=botconfig.NICK,
                     connect_cb=wolfgame.connect_callback
                    )
    cli.mainLoop()

if __name__ == "__main__":
    main()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import re
import glob
import codecs
import jinja2
import markdown
import pygments
import tempfile

from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter

from subprocess import *

class Generator:
    def __init__(self, options, args):
        self.configure(options, args)

    def configure(self, options, args):
        """
        Configures this generator from its properties. "args" are not used (yet?)
        """
        self.direct = options.direct
        self.encoding = options.encoding
        self.verbose = False if options.direct else options.verbose

        if (os.path.exists(options.destination_file) and not os.path.isfile(options.destination_file)):
            raise IOError(u"Destination %s exists and is not a file" % options.destination_file)
        else:
            self.destination_file = options.destination_file

        if (self.destination_file.endswith('.html')):
            self.file_type = 'html'
        elif (self.destination_file.endswith('.pdf')):
            self.file_type = 'pdf'
        else:
            raise IOError(u"This program can only write html or pdf files, "
                           "please use one of these file extensions in the "
                           "destination")

        if (os.path.exists(options.source)):
            self.source = options.source
        else:
            raise IOError(u"Source file/directory %s does not exist" % options.source)

        if (os.path.exists(options.template_file)):
            self.template_file = options.template_file
        else:
            raise IOError(u"Template file %s does not exist" % options.template_file)

    def execute(self):
        """
        Execute this generator regarding its current configuration
        """
        if (self.direct):
            if (self.file_type == 'pdf'):
                raise IOError(u"Direct output mode is not available for PDF export")
            else:
                print self.render()
        else:
            self.write()
            self.log(u"Generated file: %s" % self.destination_file)

    def fetch_md_contents(self, source):
        """
        Recursively fetches Markdown contents from a single file or directory
        containing Markdown files
        """
        self.log(u"Adding %s" % source)

        md_contents = ""

        if os.path.isdir(source):
            for entry in os.listdir(source):
                current = os.path.join(source, entry)
                if os.path.isdir(current):
                    md_contents = md_contents + self.fetch_md_contents(current)
                elif current.endswith('.md') or current.endswith('.markdown'):
                    md_contents = md_contents + self.fetch_md_contents(current)
        else:
            md_contents = codecs.open(source, encoding=self.encoding).read()

        return md_contents

    def get_template_vars(self, slides_src):
        """
        Computes template vars from slide source
        """
        head_title = slides_src[0].split('>')[1].split('<')[0]

        slides = []

        for slide_src in slides_src:
            header, content = slide_src.split('\n', 1)

            slides.append({'header': header,
                           'content': self.highlight_code(content)})

        return {'head_title': head_title, 'slides': slides}

    def highlight_code(self, content):
        """
        Performs syntax coloration in slide code blocks
        """
        while u'!' in content:
            lang_match = re.search('!(.+)\n', content)

            if lang_match:
                lang = lang_match.group(1)
                code = content.split(lang, 1)[1].split(u'</code', 1)[0]

                lexer = get_lexer_by_name(lang)

                formatter = HtmlFormatter(linenos='inline', noclasses=True,
                                          nobackground=True)

                pretty_code = pygments.highlight(code, lexer, formatter)
                pretty_code = pretty_code.replace('&', '&')

                before_code = content.split(u'', 1)[0]
                after_code = content.split(u'', 1)[1]

                content = before_code + pretty_code + after_code

        return content

    def log(self, message):
        if (self.verbose):
            print message

    def render(self):
        """
        Returns generated html code
        """
        md_src = self.fetch_md_contents(self.source)

        slides_src = markdown.markdown(md_src).split(u'
\n')

        template_src = codecs.open(self.template_file, encoding=self.encoding)
        template = jinja2.Template(template_src.read())

        return template.render(self.get_template_vars(slides_src))

    def write(self):
        """
        Writes generated presentation code into the destination file
        """
        html = self.render()

        if (self.file_type == 'pdf'):
            self.write_pdf(html)
        else:
            outfile = codecs.open(self.destination_file, 'w', encoding=self.encoding)
            outfile.write(html)

    def write_pdf(self, html):
        """
        Tries to write a PDF export from the command line using PrinceXML if
        available
        """
        try:
            f = tempfile.NamedTemporaryFile(delete=False, suffix='.html')
            f.write(html.encode(self.encoding))
            f.close()
        except Exception:
            raise IOError(u"Unable to create temporary file")

        try:
            command = Popen(["prince", f.name, self.destination_file])
        except Exception:
            raise EnvironmentError(u"Unable to generate PDF file using prince."
                                    "Is it installed and available?")
<commit_msg>Fixed multiple code block escaping problem<commit_after># -*- coding: utf-8 -*-

import os
import re
import glob
import codecs
import jinja2
import markdown
import pygments
import tempfile

from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter

from subprocess import *

class Generator:
    def __init__(self, options, args):
        self.configure(options, args)

    def configure(self, options, args):
        """
        Configures this generator from its properties. "args" are not used (yet?)
        """
        self.direct = options.direct
        self.encoding = options.encoding
        self.verbose = False if options.direct else options.verbose

        if (os.path.exists(options.destination_file) and not os.path.isfile(options.destination_file)):
            raise IOError(u"Destination %s exists and is not a file" % options.destination_file)
        else:
            self.destination_file = options.destination_file

        if (self.destination_file.endswith('.html')):
            self.file_type = 'html'
        elif (self.destination_file.endswith('.pdf')):
            self.file_type = 'pdf'
        else:
            raise IOError(u"This program can only write html or pdf files, "
                           "please use one of these file extensions in the "
                           "destination")

        if (os.path.exists(options.source)):
            self.source = options.source
        else:
            raise IOError(u"Source file/directory %s does not exist" % options.source)

        if (os.path.exists(options.template_file)):
            self.template_file = options.template_file
        else:
            raise IOError(u"Template file %s does not exist" % options.template_file)

    def execute(self):
        """
        Execute this generator regarding its current configuration
        """
        if (self.direct):
            if (self.file_type == 'pdf'):
                raise IOError(u"Direct output mode is not available for PDF export")
            else:
                print self.render()
        else:
            self.write()
            self.log(u"Generated file: %s" % self.destination_file)

    def fetch_md_contents(self, source):
        """
        Recursively fetches Markdown contents from a single file or directory
        containing Markdown files
        """
        self.log(u"Adding %s" % source)

        md_contents = ""

        if os.path.isdir(source):
            for entry in os.listdir(source):
                current = os.path.join(source, entry)
                if os.path.isdir(current):
                    md_contents = md_contents + self.fetch_md_contents(current)
                elif current.endswith('.md') or current.endswith('.markdown'):
                    md_contents = md_contents + self.fetch_md_contents(current)
        else:
            md_contents = codecs.open(source, encoding=self.encoding).read()

        return md_contents

    def get_template_vars(self, slides_src):
        """
        Computes template vars from slide source
        """
        head_title = slides_src[0].split('>')[1].split('<')[0]

        slides = []

        for slide_src in slides_src:
            header, content = slide_src.split('\n', 1)

            slides.append({'header': header,
                           'content': self.highlight_code(content)})

        return {'head_title': head_title, 'slides': slides}

    def highlight_code(self, content):
        """
        Performs syntax coloration in slide code blocks
        """

        while u'!' in content:
            code_match = re.search('!(.+?)\n(.+?)', content, re.DOTALL)

            if code_match:
                lang, code = code_match.groups()

                code = code.replace('<', '<').replace('>', '>')

                lexer = get_lexer_by_name(lang)

                formatter = HtmlFormatter(linenos='inline', noclasses=True,
                                          nobackground=True)

                pretty_code = pygments.highlight(code, lexer, formatter)
                pretty_code = pretty_code.replace('&', '&')

                before_code = content.split(u'', 1)[0]
                after_code = content.split(u'', 1)[1]

                content = before_code + pretty_code + after_code

        return content

    def log(self, message):
        if (self.verbose):
            print message

    def render(self):
        """
        Returns generated html code
        """
        md_src = self.fetch_md_contents(self.source)

        slides_src = markdown.markdown(md_src).split(u'
\n')

        template_src = codecs.open(self.template_file, encoding=self.encoding)
        template = jinja2.Template(template_src.read())

        return template.render(self.get_template_vars(slides_src))

    def write(self):
        """
        Writes generated presentation code into the destination file
        """
        html = self.render()

        if (self.file_type == 'pdf'):
            self.write_pdf(html)
        else:
            outfile = codecs.open(self.destination_file, 'w', encoding=self.encoding)
            outfile.write(html)

    def write_pdf(self, html):
        """
        Tries to write a PDF export from the command line using PrinceXML if
        available
        """
        try:
            f = tempfile.NamedTemporaryFile(delete=False, suffix='.html')
            f.write(html.encode(self.encoding))
            f.close()
        except Exception:
            raise IOError(u"Unable to create temporary file")

        try:
            command = Popen(["prince", f.name, self.destination_file])
        except Exception:
            raise EnvironmentError(u"Unable to generate PDF file using prince."
                                    "Is it installed and available?")
<|endoftext|>


<commit_before>#-*-coding: utf-8-*-
import json
import datetime
import base64
import zlib
from tornado import escape
from tornado import gen
#from tornado import httpclient
#from tornado.httpclient import HTTPError
from tornado.httpclient import AsyncHTTPClient
from tornado.options import parse_config_file
from tornado.options import options
from functools import wraps
import tornado.ioloop
from libs.client import GetPage, PatchPage

parse_config_file("config.py")
update_id = None
github-china = None
github-world = None
AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")

def loop_call(delta=60 * 1000):
    def wrap_loop(func):
        @wraps(func)
        def wrap_func(*args, **kwargs):
            func(*args, **kwargs)
            tornado.ioloop.IOLoop.instance().add_timeout(
                datetime.timeelta(milliseconds=delta),
                wrap_func)
        return wrap_func
    return wrap_loop

@gen.coroutine
def update_user():
    global update_id
    global github-china
    if last_users_file_num is None:
        print "fetching..."
        resp = yield GetPage(options.users_url)
        print "fetched..."
        while resp.code != 200:
            options.logger.error("first get user gist error %d %s" % (resp.code, resp.message))
            resp = yield GetPage(options.users_url)
        resp = escape.json_encode(resp.body)
        user_files = resp["files"].keys()
        files_num = [int(filename[-1]) for filename in user_files
                     if "users" in filename]
        last_users_file_num = max(files_num)
        options.logger.info("last user_id is %d" % last_users_file_num)
        remote_users_file = {}
        fetch_new_user_id = last_users_file_num * options.user_file_interval
    fetch_new_user_url = options.api_url + "/users?since=" + str(fetch_new_user_id)
    resp = yield GetPage(fetch_new_user_url)
    if "X-RateLimit-Remaining" in resp.headers:
        options.logger.warning(resp.headers["X-RateLimit-Remaining"])
    if resp.code == 200:
        users_json = escape.json_decode(resp.body)
        if users_json == []:
            options.logger.info("no more users")
            tornado.ioloop.IOLoop.instance().add_timeout(
                datetime.timedelta(milliseconds=3600 * 1000),
                loop_fetch_new_user)
        else:
            users_json = sorted(users_json, key=lambda d: d["id"])
            if fetch_new_user_id < users_json[-1]["id"]:
                fetch_new_user_id = users_json[-1]["id"]
                options.logger.info("new user_id is %d" % fetch_new_user_id)
            for user in users_json:
                if user["id"] not in remote_users_file:
                    remote_users_file[user["id"]] = {
                        "login": user["login"],
                        "id": user["id"],
                        "gravatar": user["avatar_url"],
                        "name": "",
                        "location": "",
                        "followers": 0,
                        "contributions": 0,
                        "activity": 1
                    }
            now_file_num = fetch_new_user_id / options.user_file_interval
            if now_file_num > last_users_file_num:
                try:
                    update_users_file = {
                        key: remote_users_file[key] for key in remote_users_file
                        if key > (now_file_num - 1) * options.user_file_interval
                        and key < now_file_num * options.user_file_interval + 1}
                except KeyError, e:
                    print e
                    options.logger.error("update file %d KeyError in %r" % (now_file_num, e))
                resp = yield update_file(update_users_file, now_file_num)
                print "update..."
                if resp.code == 200:
                    print "200"
                    last_users_file_num = now_file_num
                    keys = remote_users_file.keys()
                    try:
                        for key in keys:
                            if key <= now_file_num * options.user_file_interval:
                                del remote_users_file[key]
                    except Exception, e:
                        print e
                    print "del..."
                    resp = escape.json_decode(resp.body)
                    filename = "users%d" % now_file_num
                    options.logger.info("file %s size %d update success" %
                                        (resp["files"][filename]["filename"],
                                         resp["files"][filename]["size"]))
                else:
                    options.logger.error("update users file error %d, %r" %
                                         (resp.code, resp.message))

            tornado.ioloop.IOLoop.instance().add_timeout(
                datetime.timedelta(milliseconds=1 * 1000),
                loop_fetch_new_user)
    else:
        options.logger.error("fetch users error %d %r" % (resp.code, resp.message))
        tornado.ioloop.IOLoop.instance().add_timeout(
            datetime.timedelta(milliseconds=2 * 1000),
            loop_fetch_new_user)

@gen.coroutine
def update_file(update_users_file, now_file_num):
    filename = "users%d" % now_file_num
    try:
        body = json.dumps({
            "description": "update users file  %d" % now_file_num,
            "files": {
                filename: {
                    "content": base64.b64encode(
                        zlib.compress(json.dumps(update_users_file))
                    )
                }
            }
        })
    except Exception, e:
        options.logger.error("process body error %d %s" % (e.code, e.message))
    resp = yield PatchPage(options.users_url, body)
    raise gen.Return(resp)

if __name__ == "__main__":
    loop_fetch_new_user()
    tornado.ioloop.IOLoop.instance().start()
<commit_msg>change to simple version<commit_after>#-*-coding: utf-8-*-
import json
import datetime
import base64
import urllib
import zlib
from tornado import escape
from tornado import gen
from tornado.httpclient import AsyncHTTPClient
from tornado.options import parse_config_file
from tornado.options import options
from functools import wraps
import tornado.ioloop
from libs.client import GetPage, PatchPage, sync_loop_call, formula

parse_config_file("config.py")
github_china = []
github_world = []
current_china_page = 0
current_world_page = 0
AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")

def loop_call(delta=60 * 1000):
    def wrap_loop(func):
        @wraps(func)
        def wrap_func(*args, **kwargs):
            func(*args, **kwargs)
            tornado.ioloop.IOLoop.instance().add_timeout(
                datetime.timeelta(milliseconds=delta),
                wrap_func)
        return wrap_func
    return wrap_loop

@gen.coroutine
def contribute(login):
    resp = yield GetPage(options.contribution_url(login))
    if resp.code == 200:
        resp = escape.json_decode(resp.body)
        all_contribute = sum([day[1] for day in resp])
    else:
        all_contribute = 0
        options.logger.error("fetch contribution error %d, %s" %
                             (resp.code, resp.message))
    raise gen.Return(all_contribute)

@sync_loop_call(1 * 1000)
@gen.coroutine
def update_china_user():
    global github_china
    global current_china_page
    temp_github_china = []
    options.logger.info("current page is %d" % current_china_page)
    resp = yield search_china(current_china_page)
    if resp.code == 200:
        resp = escape.json_decode(resp.body)
        users = resp["users"]
        for user in users:
            contributions = yield contribute(user["login"])
            temp_github_china.append({
                "login": user["login"],
                "name": user["name"],
                "location": user["location"],
                "gravatar": "http://www.gravatar.com/avatar/" + user["gravatar_id"]
                + urllib.urlencode({"s": 48}),
                "language": user["language"],
                "contributions": contributions,
                "followers": user["followers"],
            })
        current_china_page += 1
    elif resp.code == 422:
        github_china = temp_github_china[:]
        github_china = sorted(github_china,
                              key=lambda d: d["contributions"] + formula(d["followers"]),
                              reverse=True)
        temp_github_china = []
        current_china_page = 0
        options.logger.info("china loop end")
    else:
        options.logger.error("get chine user error on page %d, error code %d, %s" %
                             (current_china_page, resp.code, resp.message))

@sync_loop_call(1 * 1000)
@gen.coroutine
def update_world_user():
    global github_world
    global current_world_page
    temp_github_world = []
    options.logger.info("current page is %d" % current_world_page)
    resp = yield search_world(current_world_page)
    if resp.code == 200:
        resp = escape.json_decode(resp.body)
        users = resp["users"]
        for user in users:
            contributions = yield contribute(user["login"])
            temp_github_world.append({
                "login": user["login"],
                "name": user["name"],
                "location": user["location"],
                "gravatar": "http://www.gravatar.com/avatar/" + user["gravatar_id"]
                + urllib.urlencode({"s": 48}),
                "language": user["language"],
                "contributions": contributions,
                "followers": user["followers"],
            })
        current_world_page += 1
    elif resp.code == 422:
        github_world = temp_github_world[:]
        github_world = sorted(github_world,
                              key=lambda d: d["contributions"] + formula(d["followers"]),
                              reverse=True)
        temp_github_world = []
        current_world_page = 0
        options.logger.info("world loop end")
    else:
        options.logger.error("get world user error on page %d, error code %d, %s" %
                             (current_world_page, resp.code, resp.message))

@gen.coroutine
def search_china(page):
    url = options.api_url + "/legacy/user/search/location:china?start_page=" + str(page) + "&sort=followers&order=desc"
    resp = yield GetPage(url)
    raise gen.Return(resp)

@gen.coroutine
def search_world(page):
    url = options.api_url + "/legacy/user/search/followers:>0?start_page=" + str(page) + "&sort=followers&order=desc"
    resp = yield GetPage(url)
    raise gen.Return(resp)

@gen.coroutine
def update_file(update_users_file, now_file_num):
    filename = "users%d" % now_file_num
    try:
        body = json.dumps({
            "description": "update users file  %d" % now_file_num,
            "files": {
                filename: {
                    "content": base64.b64encode(
                        zlib.compress(json.dumps(update_users_file))
                    )
                }
            }
        })
    except Exception, e:
        options.logger.error("Error: %s" % e)
    resp = yield PatchPage(options.users_url, body)
    raise gen.Return(resp)

if __name__ == "__main__":
    update_china_user()
    update_world_user()
    tornado.ioloop.IOLoop.instance().start()
<|endoftext|>


<commit_before>#-*-coding: utf-8-*-
import urllib
from tornado import escape
from tornado import gen
from tornado.httpclient import AsyncHTTPClient
#from tornado.options import parse_config_file
from tornado.options import options
import tornado.ioloop
from libs.client import GetPage, sync_loop_call, formula, update_file
from libs.geo import match_geoname, match_world_geoname
import time

#parse_config_file("config.py")
github_china = []
github_world = []
temp_github_world = []
temp_github_china = []
current_china_page = 1
current_world_page = 1

china_location_map = {}
china_map = {}
for city in options.city_list:
    china_map[city] = {"score": 0, "stateInitColor": 6}

world_location_map = {}
world_map = {}
for country_code in options.country_code_list:
    world_map[country_code] = {"score": 0, "staticInitColor": 6}

AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")

def wash(users):
    user_names = []
    new_users = []
    for user in users:
        if user['login'] not in user_names:
            user_names.append(user['login'])
            new_users.append(user)
    return new_users

@gen.coroutine
def contribute(login):
    resp = yield GetPage(options.contribution_url(login))
    if resp.code == 200:
        resp = escape.json_decode(resp.body)
        all_contribute = sum([day[1] for day in resp])
    else:
        all_contribute = 0
        options.logger.error("fetch contribution error %d, %s" %
                             (resp.code, resp.message))
    raise gen.Return(all_contribute)

@sync_loop_call(32 * 1000)
@gen.coroutine
def update_china_user():
    global github_china
    global current_china_page
    global temp_github_china
    options.logger.info("current page is %d" % current_china_page)
    resp = yield search_china(current_china_page)
    if resp.code == 200:
        resp = escape.json_decode(resp.body)
        users = resp["users"]
        for user in users:
            contributions = yield contribute(user["login"])
            temp_github_china.append({
                "login": user["login"],
                "name": user["name"] or "Unknown",
                "location": user["location"],
                "gravatar": "http://www.gravatar.com/avatar/" + user["gravatar_id"]
                + urllib.urlencode({"s": 48}),
                "language": user["language"],
                "contributions": contributions,
                "followers": user["followers"],
                "score": contributions + formula(user["followers"])
            })
        temp_github_china = wash(temp_github_china)
        current_china_page += 1
        if len(github_china) < len(temp_github_china):
            github_china = temp_github_china[:]
            github_china = sorted(github_china, key=lambda d: d['score'], reverse=True)
    elif resp.code == 422:
        github_china = temp_github_china[:]
        github_china = sorted(github_china, key=lambda d: d['score'], reverse=True)
        temp_github_china = []
        current_china_page = 1
        options.logger.info("china loop end")
    else:
        options.logger.error("get china user error on page %d, error code %d, %s" %
                             (current_china_page, resp.code, resp.message))

@sync_loop_call(64 * 1000)
@gen.coroutine
def update_world_user():
    global github_world
    global current_world_page
    global temp_github_world
    options.logger.info("current page is %d" % current_world_page)
    resp = yield search_world(current_world_page)
    if resp.code == 200:
        resp = escape.json_decode(resp.body)
        users = resp["users"]
        for user in users:
            contributions = yield contribute(user["login"])
            temp_github_world.append({
                "login": user["login"],
                "name": user["name"] or "Unknown",
                "location": user["location"],
                "gravatar": "http://www.gravatar.com/avatar/" + user["gravatar_id"]
                + urllib.urlencode({"s": 48}),
                "language": user["language"],
                "contributions": contributions,
                "followers": user["followers"],
                "score": contributions + formula(user["followers"])
            })
        temp_github_world = wash(temp_github_world)
        current_world_page += 1
        if len(github_world) < len(temp_github_world):
            github_world = temp_github_world[:]
            github_world = sorted(github_world, key=lambda d: d['score'], reverse=True)

    elif resp.code == 422:
        github_world = temp_github_world[:]
        github_world = sorted(github_world, key=lambda d: d['score'], reverse=True)
        temp_github_world = []
        current_world_page = 1
        options.logger.info("world loop end")
    else:
        options.logger.error("get world user error on page %d, error code %d, %s" %
                             (current_world_page, resp.code, resp.message))

@gen.coroutine
def search_china(page):
    url = options.api_url + "/legacy/user/search/location:china?start_page=" + str(page) + "&sort=followers&order=desc"
    resp = yield GetPage(url)
    options.logger.info("search china %s" % url)
    raise gen.Return(resp)

@gen.coroutine
def search_world(page):
    url = options.api_url + "/legacy/user/search/followers:>0?start_page=" + str(page) + "&sort=followers&order=desc"
    options.logger.info("search world %s" % url)
    resp = yield GetPage(url)
    raise gen.Return(resp)

@sync_loop_call(60 * 1000)
@gen.coroutine
def update_china_location():
    global china_location_map
    global china_map
    if not china_location_map:  # Fetch location_map file
        resp = yield GetPage(options.api_url + options.location_map_gist)
        if resp.code == 200:
            resp = escape.json_decode(resp.body)
            raw_url = resp['files']['location_map.json']['raw_url']
            resp = yield GetPage(raw_url)
            if resp.code == 200:
                china_location_map = escape.json_decode(resp.body)
            else:
                options.logger.error("Fetch raw data error %d, %s" %
                                     (resp.code, resp.message))
        else:
            options.logger.error("Get gist error %d, %s" % (resp.code, resp.message))
    else:  # update location_map file every hour
        if int(time.time()) % 36000 < 60:
            resp = yield update_file(options.api_url + options.location_map_gist,
                                     "location_map.json",
                                     china_location_map)
            if resp.code != 200:
                options.logger.error("update gists error %d, %s" % (resp.code, resp.message))

    temp_china_map = {}
    for city in options.city_list:
        temp_china_map[city] = {"score": 0, "stateInitColor": 6}

    for user in github_china:
        try:
            location = user["location"].lower()
            location = ','.join(location.strip().split())
        except Exception, e:
            options.logger.error("lower location error %s" % e)
            continue
        # because acording to geoname match china will match to shanghai
        if location == "china":
            continue
        if location in china_location_map:
            city = china_location_map[location]
        else:
            city = yield match_geoname(location)
        if city:
            temp_china_map[city]["score"] += 1
            china_location_map[location] = city
        else:
            options.logger.warning("location: %s can't match" % location)

    china_map = temp_china_map.copy()

@sync_loop_call(60 * 1000)
@gen.coroutine
def update_world_location():
    options.logger.info("start update world_location")
    global world_location_map
    global world_map
    if not world_location_map:  # get file into world_location_map
        resp = yield GetPage(options.api_url + options.world_location_map_gist)
        if resp.code == 200:
            resp = escape.json_decode(resp.body)
            raw_url = resp["files"]["world_location_map.json"]['raw_url']
            resp = yield GetPage(raw_url)
            if resp.code == 200:
                world_location_map = escape.json_decode(resp.body)
            else:
                options.logger.error("Fetch raw data error %d, %s" %
                                     (resp.code, resp.message))
        else:
            options.logger.error("Get gist error %d, %s" %
                                 (resp.code, resp.message))

    else:  # update world_location_map file every hour
        if int(time.time()) % 36000 < 60:
            resp = yield update_file(options.api_url + options.world_location_map_gist,
                                     "world_location_map.json",
                                     world_location_map)
            if resp.code != 200:
                options.logger.error("update gists error %d, %s" %
                                    (resp.code, resp.message))

    temp_world_map = {}
    for country_code in options.country_code_list:
        temp_world_map[country_code] = {"score": 0, "staticInitColor": 6}

    for user in github_world:
        if not user["location"]:
            continue
        try:
            location = user["location"].strip()
            location = ','.join(filter(lambda d: d,
                                ','.join(location.split()).split(',')))
        except Exception, e:
            options.logger.error("Error: %s" % e)
            continue
        if location in world_location_map:
            country_code = world_location_map[location]
        else:
            country_code = yield match_world_geoname(location)
        if country_code:
            temp_world_map[country_code]["score"] += 1
            world_location_map[location] = country_code
        else:
            options.logger.warning("location: %s can't match" % location)

    world_map = temp_world_map.copy()

if __name__ == "__main__":
    update_china_user()
    update_world_user()
    update_china_location()
    tornado.ioloop.IOLoop.instance().start()
<commit_msg>test update world location<commit_after>#-*-coding: utf-8-*-
import urllib
from tornado import escape
from tornado import gen
from tornado.httpclient import AsyncHTTPClient
#from tornado.options import parse_config_file
from tornado.options import options
import tornado.ioloop
from libs.client import GetPage, sync_loop_call, formula, update_file
from libs.geo import match_geoname, match_world_geoname
import time

#parse_config_file("config.py")
github_china = []
github_world = []
temp_github_world = []
temp_github_china = []
current_china_page = 1
current_world_page = 1

china_location_map = {}
china_map = {}
for city in options.city_list:
    china_map[city] = {"score": 0, "stateInitColor": 6}

world_location_map = {}
world_map = {}
for country_code in options.country_code_list:
    world_map[country_code] = {"score": 0, "staticInitColor": 6}

AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")

def wash(users):
    user_names = []
    new_users = []
    for user in users:
        if user['login'] not in user_names:
            user_names.append(user['login'])
            new_users.append(user)
    return new_users

@gen.coroutine
def contribute(login):
    resp = yield GetPage(options.contribution_url(login))
    if resp.code == 200:
        resp = escape.json_decode(resp.body)
        all_contribute = sum([day[1] for day in resp])
    else:
        all_contribute = 0
        options.logger.error("fetch contribution error %d, %s" %
                             (resp.code, resp.message))
    raise gen.Return(all_contribute)

@sync_loop_call(32 * 1000)
@gen.coroutine
def update_china_user():
    global github_china
    global current_china_page
    global temp_github_china
    options.logger.info("current page is %d" % current_china_page)
    resp = yield search_china(current_china_page)
    if resp.code == 200:
        resp = escape.json_decode(resp.body)
        users = resp["users"]
        for user in users:
            contributions = yield contribute(user["login"])
            temp_github_china.append({
                "login": user["login"],
                "name": user["name"] or "Unknown",
                "location": user["location"],
                "gravatar": "http://www.gravatar.com/avatar/" + user["gravatar_id"]
                + urllib.urlencode({"s": 48}),
                "language": user["language"],
                "contributions": contributions,
                "followers": user["followers"],
                "score": contributions + formula(user["followers"])
            })
        temp_github_china = wash(temp_github_china)
        current_china_page += 1
        if len(github_china) < len(temp_github_china):
            github_china = temp_github_china[:]
            github_china = sorted(github_china, key=lambda d: d['score'], reverse=True)
    elif resp.code == 422:
        github_china = temp_github_china[:]
        github_china = sorted(github_china, key=lambda d: d['score'], reverse=True)
        temp_github_china = []
        current_china_page = 1
        options.logger.info("china loop end")
    else:
        options.logger.error("get china user error on page %d, error code %d, %s" %
                             (current_china_page, resp.code, resp.message))

@sync_loop_call(64 * 1000)
@gen.coroutine
def update_world_user():
    global github_world
    global current_world_page
    global temp_github_world
    options.logger.info("current page is %d" % current_world_page)
    resp = yield search_world(current_world_page)
    if resp.code == 200:
        resp = escape.json_decode(resp.body)
        users = resp["users"]
        for user in users:
            contributions = yield contribute(user["login"])
            temp_github_world.append({
                "login": user["login"],
                "name": user["name"] or "Unknown",
                "location": user["location"],
                "gravatar": "http://www.gravatar.com/avatar/" + user["gravatar_id"]
                + urllib.urlencode({"s": 48}),
                "language": user["language"],
                "contributions": contributions,
                "followers": user["followers"],
                "score": contributions + formula(user["followers"])
            })
        temp_github_world = wash(temp_github_world)
        current_world_page += 1
        if len(github_world) < len(temp_github_world):
            github_world = temp_github_world[:]
            github_world = sorted(github_world, key=lambda d: d['score'], reverse=True)

    elif resp.code == 422:
        github_world = temp_github_world[:]
        github_world = sorted(github_world, key=lambda d: d['score'], reverse=True)
        temp_github_world = []
        current_world_page = 1
        options.logger.info("world loop end")
    else:
        options.logger.error("get world user error on page %d, error code %d, %s" %
                             (current_world_page, resp.code, resp.message))

@gen.coroutine
def search_china(page):
    url = options.api_url + "/legacy/user/search/location:china?start_page=" + str(page) + "&sort=followers&order=desc"
    resp = yield GetPage(url)
    options.logger.info("search china %s" % url)
    raise gen.Return(resp)

@gen.coroutine
def search_world(page):
    url = options.api_url + "/legacy/user/search/followers:>0?start_page=" + str(page) + "&sort=followers&order=desc"
    options.logger.info("search world %s" % url)
    resp = yield GetPage(url)
    raise gen.Return(resp)

@sync_loop_call(60 * 1000)
@gen.coroutine
def update_china_location():
    global china_location_map
    global china_map
    if not china_location_map:  # Fetch location_map file
        resp = yield GetPage(options.api_url + options.location_map_gist)
        if resp.code == 200:
            resp = escape.json_decode(resp.body)
            raw_url = resp['files']['location_map.json']['raw_url']
            resp = yield GetPage(raw_url)
            if resp.code == 200:
                china_location_map = escape.json_decode(resp.body)
            else:
                options.logger.error("Fetch raw data error %d, %s" %
                                     (resp.code, resp.message))
        else:
            options.logger.error("Get gist error %d, %s" % (resp.code, resp.message))
    else:  # update location_map file every hour
        if int(time.time()) % 36000 < 60:
            resp = yield update_file(options.api_url + options.location_map_gist,
                                     "location_map.json",
                                     china_location_map)
            if resp.code != 200:
                options.logger.error("update gists error %d, %s" % (resp.code, resp.message))

    temp_china_map = {}
    for city in options.city_list:
        temp_china_map[city] = {"score": 0, "stateInitColor": 6}

    for user in github_china:
        try:
            location = user["location"].lower()
            location = ','.join(location.strip().split())
        except Exception, e:
            options.logger.error("lower location error %s" % e)
            continue
        # because acording to geoname match china will match to shanghai
        if location == "china":
            continue
        if location in china_location_map:
            city = china_location_map[location]
        else:
            city = yield match_geoname(location)
        if city:
            temp_china_map[city]["score"] += 1
            china_location_map[location] = city
        else:
            options.logger.warning("location: %s can't match" % location)

    china_map = temp_china_map.copy()

@sync_loop_call(60 * 1000)
@gen.coroutine
def update_world_location():
    options.logger.info("start update world_location")
    global world_location_map
    global world_map
    if not world_location_map:  # get file into world_location_map
        resp = yield GetPage(options.api_url + options.world_location_map_gist)
        if resp.code == 200:
            resp = escape.json_decode(resp.body)
            raw_url = resp["files"]["world_location_map.json"]['raw_url']
            resp = yield GetPage(raw_url)
            if resp.code == 200:
                world_location_map = escape.json_decode(resp.body)
            else:
                options.logger.error("Fetch raw data error %d, %s" %
                                     (resp.code, resp.message))
        else:
            options.logger.error("Get gist error %d, %s" %
                                 (resp.code, resp.message))

    else:  # update world_location_map file every hour
        if int(time.time()) % 36000 < 60:
            resp = yield update_file(options.api_url + options.world_location_map_gist,
                                     "world_location_map.json",
                                     world_location_map)
            if resp.code != 200:
                options.logger.error("update gists error %d, %s" %
                                    (resp.code, resp.message))

    temp_world_map = {}
    for country_code in options.country_code_list:
        temp_world_map[country_code] = {"score": 0, "staticInitColor": 6}

    for user in github_world:
        if not user["location"]:
            continue
        try:
            location = user["location"].strip()
            location = ','.join(filter(lambda d: d,
                                ','.join(location.split()).split(',')))
        except Exception, e:
            options.logger.error("Error: %s" % e)
            continue
        if location in world_location_map:
            country_code = world_location_map[location]
        else:
            country_code = yield match_world_geoname(location)
        if country_code:
            temp_world_map[country_code]["score"] += 1
            world_location_map[location] = country_code
        else:
            options.logger.warning("location: %s can't match" % location)

    world_map = temp_world_map.copy()

if __name__ == "__main__":
    update_china_user()
    update_world_user()
    update_china_location()
    update_world_location()
    tornado.ioloop.IOLoop.instance().start()
<|endoftext|>


<commit_before>from __future__ import division, print_function, absolute_import
import itertools as itr
import numpy as np

def resize_by_factor(im, factor):
    """
    Resizes the image according to a factor. The image is pre-filtered
    with a Gaussian and then resampled with bilinear interpolation.

    This function uses scikit-image and essentially combines its
    `pyramid_reduce` with `pyramid_expand` into one function.

    Returns the same object if factor is 1, not a copy.

    Parameters
    ----------
    im : ndarray, ndim=2 or 3
        Image. Either 2D or 3D with 3 or 4 channels.
    factor : float
        Resize factor, e.g. a factor of 0.5 will halve both sides.
    """
    from skimage.transform.pyramids import pyramid_reduce, pyramid_expand
    if factor < 1:
        return pyramid_reduce(im, downscale=1/factor)
    elif factor > 1:
        return pyramid_expand(im, upscale=factor)
    else:
        return im

def asgray(im):
    """
    Takes an image and returns its grayscale version by averaging the color
    channels.  if an alpha channel is present, it will simply be ignored. If a
    grayscale image is given, the original image is returned.

    Parameters
    ----------
    image : ndarray, ndim 2 or 3
        RGB or grayscale image.

    Returns
    -------
    gray_image : ndarray, ndim 2
        Grayscale version of image.
    """
    if im.ndim == 2:
        return im
    elif im.ndim == 3 and im.shape[2] in (3, 4):
        return im[..., :3].mean(axis=-1)
    else:
        raise ValueError('Invalid image format')

def crop(im, size):
    """
    Crops an image in the center.

    Parameters
    ----------
    size : tuple, (height, width)
        Finally size after cropping.
    """
    diff = [im.shape[index] - size[index] for index in (0, 1)]
    im2 = im[diff[0]//2:diff[0]//2 + size[0], diff[1]//2:diff[1]//2 + size[1]]
    return im2

def crop_to_bounding_box(im, bb):
    """
    Crops according to a bounding box.

    Parameters
    ----------
    bounding_box : tuple, (top, left, bottom, right)
        Crops inclusively for top/left and exclusively for bottom/right.
    """
    return im[bb[0]:bb[2], bb[1]:bb[3]]

def load(path, dtype=np.float64):
    """
    Loads an image from file.

    Parameters
    ----------
    path : str
        Path to image file.
    dtype : np.dtype
        Defaults to ``np.float64``, which means the image will be returned as a
        float with values between 0 and 1. If ``np.uint8`` is specified, the
        values will be between 0 and 255 and no conversion cost will be
        incurred.
    """
    import skimage.io
    im = skimage.io.imread(path)
    if dtype == np.uint8:
        return im
    elif dtype in {np.float16, np.float32, np.float64}:
        return im.astype(dtype) / 255
    else:
        raise ValueError('Unsupported dtype')

def save(path, im):
    """
    Saves an image to file.

    If the image is type float, it will assume to have values in [0, 1].

    Parameters
    ----------
    path : str
        Path to which the image will be saved.
    im : ndarray (image)
        Image.
    """
    from PIL import Image
    if im.dtype == np.uint8:
        pil_im = Image.fromarray(im)
    else:
        pil_im = Image.fromarray((im*255).astype(np.uint8))
    pil_im.save(path)

def integrate(ii, r0, c0, r1, c1):
    """
    Use an integral image to integrate over a given window.

    Parameters
    ----------
    ii : ndarray
        Integral image.
    r0, c0 : int
        Top-left corner of block to be summed.
    r1, c1 : int
        Bottom-right corner of block to be summed.

    Returns
    -------
    S : int
        Integral (sum) over the given window.

    """
    # This line is modified
    S = np.zeros(ii.shape[-1])

    S += ii[r1, c1]

    if (r0 - 1 >= 0) and (c0 - 1 >= 0):
        S += ii[r0 - 1, c0 - 1]

    if (r0 - 1 >= 0):
        S -= ii[r0 - 1, c1]

    if (c0 - 1 >= 0):
        S -= ii[r1, c0 - 1]

    return S

def offset(img, offset, fill_value=0):
    """
    Moves the contents of image without changing the image size. The missing
    values are given a specified fill value.

    Parameters
    ----------
    img : array
        Image.
    offset : (vertical_offset, horizontal_offset)
        Tuple of length 2, specifying the offset along the two axes.
    fill_value : dtype of img
        Fill value. Defaults to 0.
    """

    sh = img.shape
    if sh == (0, 0):
        return img
    else:
        x = np.empty(sh)
        x[:] = fill_value
        x[max(offset[0], 0):min(sh[0]+offset[0], sh[0]),
          max(offset[1], 0):min(sh[1]+offset[1], sh[1])] = \
            img[max(-offset[0], 0):min(sh[0]-offset[0], sh[0]),
                max(-offset[1], 0):min(sh[1]-offset[1], sh[1])]
        return x

def bounding_box(alpha, threshold=0.1):
    """
    Returns a bounding box of the support.

    Parameters
    ----------
    alpha : ndarray, ndim=2
        Any one-channel image where the background has zero or low intensity.
    threshold : float
        The threshold that divides background from foreground.

    Returns
    -------
    bounding_box : (top, left, bottom, right)
        The bounding box describing the smallest rectangle containing the
        foreground object, as defined by the threshold.
    """
    assert alpha.ndim == 2

    # Take the bounding box of the support, with a certain threshold.
    supp_axs = [alpha.max(axis=1-i) for i in range(2)]

    # Check first and last value of that threshold
    bb = [np.where(supp_axs[i] > threshold)[0][[0, -1]] for i in range(2)]

    return (bb[0][0], bb[1][0], bb[0][1], bb[1][1])

def bounding_box_as_binary_map(alpha, threshold=0.1):
    """
    Similar to `bounding_box`, except returns the bounding box as a
    binary map the same size as the input.

    Same parameters as `bounding_box`.

    Returns
    -------
    binary_map : ndarray, ndim=2, dtype=np.bool_
        Binary map with True if object and False if background.
    """

    bb = bounding_box(alpha)
    x = np.zeros(alpha.shape, dtype=np.bool_)
    x[bb[0]:bb[2], bb[1]:bb[3]] = 1
    return x

def extract_patches(images, patch_shape, samples_per_image=40, seed=0,
                    cycle=True):
    """
    Takes a set of images and yields randomly chosen patches of specified size.

    Parameters
    ----------
    images : iterable
        The images have to be iterable, and each element must be a Numpy array
        with at least two spatial 2 dimensions as the first and second axis.
    patch_shape : tuple, length 2
        The spatial shape of the patches that should be extracted. If the
        images have further dimensions beyond the spatial, the patches will
        copy these too.
    samples_per_image : int
        Samples to extract before moving on to the next image.
    seed : int
        Seed with which to select the patches.
    cycle : bool
        If True, then the function will produce patches indefinitely, by going
        back to the first image when all are done. If False, the iteration will
        stop when there are no more images.

    Returns
    -------
    patch_generator
        This function returns a generator that will produce patches.

    Examples
    --------
    >>> import deepdish as dd
    >>> import matplotlib.pylab as plt
    >>> import itertools
    >>> images = ag.io.load_example('mnist')

    Now, let us say we want to exact patches from the these, where each patch
    has at least some activity.

    >>> gen = dd.image.extract_patches(images, (5, 5))
    >>> gen = (x for x in gen if x.mean() > 0.1)
    >>> patches = np.array(list(itertools.islice(gen, 25)))
    >>> patches.shape
    (25, 5, 5)
    >>> dd.plot.images(patches)
    >>> plt.show()

    """
    rs = np.random.RandomState(seed)
    for Xi in itr.cycle(images):
        # How many patches could we extract?
        w, h = [Xi.shape[i]-patch_shape[i] for i in range(2)]

        assert w > 0 and h > 0

        # Maybe shuffle an iterator of the indices?
        indices = np.asarray(list(itr.product(range(w), range(h))))
        rs.shuffle(indices)
        for x, y in indices[:samples_per_image]:
            yield Xi[x:x+patch_shape[0], y:y+patch_shape[1]]
<commit_msg>Added image.resize.<commit_after>from __future__ import division, print_function, absolute_import
import itertools as itr
import numpy as np

def resize_by_factor(im, factor):
    """
    Resizes the image according to a factor. The image is pre-filtered
    with a Gaussian and then resampled with bilinear interpolation.

    This function uses scikit-image and essentially combines its
    `pyramid_reduce` with `pyramid_expand` into one function.

    Returns the same object if factor is 1, not a copy.

    Parameters
    ----------
    im : ndarray, ndim=2 or 3
        Image. Either 2D or 3D with 3 or 4 channels.
    factor : float
        Resize factor, e.g. a factor of 0.5 will halve both sides.
    """
    from skimage.transform.pyramids import pyramid_reduce, pyramid_expand
    if factor < 1:
        return pyramid_reduce(im, downscale=1/factor)
    elif factor > 1:
        return pyramid_expand(im, upscale=factor)
    else:
        return im

def resize(im, shape=None, max_side=None, min_side=None):
    if min_side is not None:
        min = np.min(im.shape[:2])
        factor = min_side / min
        return resize_by_factor(im, factor)

    elif max_side is not None:
        max = np.max(im.shape[:2])
        factor = max_side / max
        return resize_by_factor(im, factor)

    else:
        factor_y = shape[0] / im.shape[0]
        factor_x = shape[1] / im.shape[1]
        assert np.fabs(factor_x - factor_y) < 0.5
        return resize_by_factor(im, factor_x)

def asgray(im):
    """
    Takes an image and returns its grayscale version by averaging the color
    channels.  if an alpha channel is present, it will simply be ignored. If a
    grayscale image is given, the original image is returned.

    Parameters
    ----------
    image : ndarray, ndim 2 or 3
        RGB or grayscale image.

    Returns
    -------
    gray_image : ndarray, ndim 2
        Grayscale version of image.
    """
    if im.ndim == 2:
        return im
    elif im.ndim == 3 and im.shape[2] in (3, 4):
        return im[..., :3].mean(axis=-1)
    else:
        raise ValueError('Invalid image format')

def crop(im, size):
    """
    Crops an image in the center.

    Parameters
    ----------
    size : tuple, (height, width)
        Finally size after cropping.
    """
    diff = [im.shape[index] - size[index] for index in (0, 1)]
    im2 = im[diff[0]//2:diff[0]//2 + size[0], diff[1]//2:diff[1]//2 + size[1]]
    return im2

def crop_to_bounding_box(im, bb):
    """
    Crops according to a bounding box.

    Parameters
    ----------
    bounding_box : tuple, (top, left, bottom, right)
        Crops inclusively for top/left and exclusively for bottom/right.
    """
    return im[bb[0]:bb[2], bb[1]:bb[3]]

def load(path, dtype=np.float64):
    """
    Loads an image from file.

    Parameters
    ----------
    path : str
        Path to image file.
    dtype : np.dtype
        Defaults to ``np.float64``, which means the image will be returned as a
        float with values between 0 and 1. If ``np.uint8`` is specified, the
        values will be between 0 and 255 and no conversion cost will be
        incurred.
    """
    import skimage.io
    im = skimage.io.imread(path)
    if dtype == np.uint8:
        return im
    elif dtype in {np.float16, np.float32, np.float64}:
        return im.astype(dtype) / 255
    else:
        raise ValueError('Unsupported dtype')

def save(path, im):
    """
    Saves an image to file.

    If the image is type float, it will assume to have values in [0, 1].

    Parameters
    ----------
    path : str
        Path to which the image will be saved.
    im : ndarray (image)
        Image.
    """
    from PIL import Image
    if im.dtype == np.uint8:
        pil_im = Image.fromarray(im)
    else:
        pil_im = Image.fromarray((im*255).astype(np.uint8))
    pil_im.save(path)

def integrate(ii, r0, c0, r1, c1):
    """
    Use an integral image to integrate over a given window.

    Parameters
    ----------
    ii : ndarray
        Integral image.
    r0, c0 : int
        Top-left corner of block to be summed.
    r1, c1 : int
        Bottom-right corner of block to be summed.

    Returns
    -------
    S : int
        Integral (sum) over the given window.

    """
    # This line is modified
    S = np.zeros(ii.shape[-1])

    S += ii[r1, c1]

    if (r0 - 1 >= 0) and (c0 - 1 >= 0):
        S += ii[r0 - 1, c0 - 1]

    if (r0 - 1 >= 0):
        S -= ii[r0 - 1, c1]

    if (c0 - 1 >= 0):
        S -= ii[r1, c0 - 1]

    return S

def offset(img, offset, fill_value=0):
    """
    Moves the contents of image without changing the image size. The missing
    values are given a specified fill value.

    Parameters
    ----------
    img : array
        Image.
    offset : (vertical_offset, horizontal_offset)
        Tuple of length 2, specifying the offset along the two axes.
    fill_value : dtype of img
        Fill value. Defaults to 0.
    """

    sh = img.shape
    if sh == (0, 0):
        return img
    else:
        x = np.empty(sh)
        x[:] = fill_value
        x[max(offset[0], 0):min(sh[0]+offset[0], sh[0]),
          max(offset[1], 0):min(sh[1]+offset[1], sh[1])] = \
            img[max(-offset[0], 0):min(sh[0]-offset[0], sh[0]),
                max(-offset[1], 0):min(sh[1]-offset[1], sh[1])]
        return x

def bounding_box(alpha, threshold=0.1):
    """
    Returns a bounding box of the support.

    Parameters
    ----------
    alpha : ndarray, ndim=2
        Any one-channel image where the background has zero or low intensity.
    threshold : float
        The threshold that divides background from foreground.

    Returns
    -------
    bounding_box : (top, left, bottom, right)
        The bounding box describing the smallest rectangle containing the
        foreground object, as defined by the threshold.
    """
    assert alpha.ndim == 2

    # Take the bounding box of the support, with a certain threshold.
    supp_axs = [alpha.max(axis=1-i) for i in range(2)]

    # Check first and last value of that threshold
    bb = [np.where(supp_axs[i] > threshold)[0][[0, -1]] for i in range(2)]

    return (bb[0][0], bb[1][0], bb[0][1], bb[1][1])

def bounding_box_as_binary_map(alpha, threshold=0.1):
    """
    Similar to `bounding_box`, except returns the bounding box as a
    binary map the same size as the input.

    Same parameters as `bounding_box`.

    Returns
    -------
    binary_map : ndarray, ndim=2, dtype=np.bool_
        Binary map with True if object and False if background.
    """

    bb = bounding_box(alpha)
    x = np.zeros(alpha.shape, dtype=np.bool_)
    x[bb[0]:bb[2], bb[1]:bb[3]] = 1
    return x

def extract_patches(images, patch_shape, samples_per_image=40, seed=0,
                    cycle=True):
    """
    Takes a set of images and yields randomly chosen patches of specified size.

    Parameters
    ----------
    images : iterable
        The images have to be iterable, and each element must be a Numpy array
        with at least two spatial 2 dimensions as the first and second axis.
    patch_shape : tuple, length 2
        The spatial shape of the patches that should be extracted. If the
        images have further dimensions beyond the spatial, the patches will
        copy these too.
    samples_per_image : int
        Samples to extract before moving on to the next image.
    seed : int
        Seed with which to select the patches.
    cycle : bool
        If True, then the function will produce patches indefinitely, by going
        back to the first image when all are done. If False, the iteration will
        stop when there are no more images.

    Returns
    -------
    patch_generator
        This function returns a generator that will produce patches.

    Examples
    --------
    >>> import deepdish as dd
    >>> import matplotlib.pylab as plt
    >>> import itertools
    >>> images = ag.io.load_example('mnist')

    Now, let us say we want to exact patches from the these, where each patch
    has at least some activity.

    >>> gen = dd.image.extract_patches(images, (5, 5))
    >>> gen = (x for x in gen if x.mean() > 0.1)
    >>> patches = np.array(list(itertools.islice(gen, 25)))
    >>> patches.shape
    (25, 5, 5)
    >>> dd.plot.images(patches)
    >>> plt.show()

    """
    rs = np.random.RandomState(seed)
    for Xi in itr.cycle(images):
        # How many patches could we extract?
        w, h = [Xi.shape[i]-patch_shape[i] for i in range(2)]

        assert w > 0 and h > 0

        # Maybe shuffle an iterator of the indices?
        indices = np.asarray(list(itr.product(range(w), range(h))))
        rs.shuffle(indices)
        for x, y in indices[:samples_per_image]:
            yield Xi[x:x+patch_shape[0], y:y+patch_shape[1]]
<|endoftext|>