<commit_before># vim:set et sts=4 sw=4:
#
# ibus - The Input Bus
#
# Copyright (c) 2007-2010 Peng Huang <shawn.p.huang@gmail.com>
# Copyright (c) 2007-2010 Red Hat, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330,
# Boston, MA  02111-1307  USA

__all__ = (
        "IBUS_IFACE_IBUS",
        "IBUS_SERVICE_IBUS",
        "IBUS_PATH_IBUS",
        "IBUS_IFACE_CONFIG",
        "IBUS_IFACE_PANEL",
        "IBUS_IFACE_ENGINE",
        "IBUS_IFACE_ENGINE_FACTORY",
        "IBUS_IFACE_INPUT_CONTEXT",
        "IBUS_IFACE_NOTIFICATIONS",
        "ORIENTATION_HORIZONTAL",
        "ORIENTATION_VERTICAL",
        "ORIENTATION_SYSTEM",
        "default_reply_handler",
        "default_error_handler",
        "DEFAULT_ASYNC_HANDLERS",
        "CONFIG_GENERAL_SHORTCUT_TRIGGER",
        "CONFIG_GENERAL_SHORTCUT_NEXT_ENGINE",
        "CONFIG_GENERAL_SHORTCUT_PREV_ENGINE",
        "CONFIG_GENERAL_SHORTCUT_TRIGGER_DEFAULT",
        "CONFIG_GENERAL_SHORTCUT_NEXT_ENGINE_DEFAULT",
        "CONFIG_GENERAL_SHORTCUT_PREV_ENGINE_DEFAULT",
        "main",
        "main_quit",
        "main_iteration",
        "get_address",
        "get_socket_path"
    )

import os
import sys
from xdg import BaseDirectory
import ctypes

# __display = os.environ["DISPLAY"]
# __hostname, __display_screen = __display.split(":", 1)
# if not __hostname.strip():
#     __hostname = "unix"
# __display_number = __display_screen.split(".")[0]
#
# __username = None
# try:
#     __username = os.getlogin()
# except:
#     pass
# if not __username:
#     __username = os.getenv ("LOGNAME")
# if not __username:
#     __username = os.getenv ("USER")
# if not __username:
#     __username = os.getenv ("LNAME")
# if not __username:
#     __username = os.getenv ("USERNAME")
# libibus = ctypes.CDLL("libibus.so")
# id = ctypes.c_char_p(libibus.ibus_get_local_machine_id()).value
#
# IBUS_SOCKET_FILE = os.path.join(BaseDirectory.xdg_config_home,
#                                 "ibus", "bus",
#                                 "%s-%s-%s"% (id, __hostname, __display_number))
# def get_address():
#     libibus = ctypes.CDLL("libibus.so")
#     address = ctypes.c_char_p(libibus.ibus_get_address()).value
#     return address
#
#     address = os.getenv("IBUS_ADDRESS")
#     if address:
#         return address
#     try:
#         for l in file(IBUS_SOCKET_FILE):
#             if not l.startswith("IBUS_ADDRESS="):
#                 continue
#             address = l[13:]
#             address = address.strip()
#             break
#     except:
#         return None
#     return address

libibus = ctypes.CDLL("libibus-1.0.so.0")
get_address = libibus.ibus_get_address
get_address.restype=ctypes.c_char_p

get_socket_path = libibus.ibus_get_socket_path
get_socket_path.restype=ctypes.c_char_p

# __session_id = os.getenv ("IBUS_SESSION_ID")
#
# IBUS_ADDR = "unix:path=/tmp/ibus-%s%s/ibus-%s-%s" % (__username,
#                                                      "-" + __session_id if __session_id else "",
#                                                      __hostname,
#                                                      __display_number)
# IBUS_ADDR  = "tcp:host=localhost,port=7799"

IBUS_IFACE_IBUS     = "org.freedesktop.IBus"
IBUS_PATH_IBUS      = "/org/freedesktop/IBus"
IBUS_SERVICE_IBUS   = "org.freedesktop.IBus"

IBUS_IFACE_PANEL            = "org.freedesktop.IBus.Panel"
IBUS_IFACE_CONFIG           = "org.freedesktop.IBus.Config"
IBUS_IFACE_ENGINE           = "org.freedesktop.IBus.Engine"
IBUS_IFACE_ENGINE_FACTORY   = "org.freedesktop.IBus.Factory"
IBUS_IFACE_INPUT_CONTEXT    = "org.freedesktop.IBus.InputContext"
IBUS_IFACE_NOTIFICATIONS    = "org.freedesktop.IBus.Notifications"

# define pre-edit commit mode when the focus is lost
IBUS_ENGINE_PREEDIT_CLEAR       = 0
IBUS_ENGINE_PREEDIT_COMMIT      = 1

# define orientation
ORIENTATION_HORIZONTAL  = 0
ORIENTATION_VERTICAL    = 1
ORIENTATION_SYSTEM      = 2

def default_reply_handler( *args):
    pass

def default_error_handler(e):
    print >> sys.stderr, e

DEFAULT_ASYNC_HANDLERS = {
    "reply_handler" : default_reply_handler,
    "error_handler" : default_error_handler
}

CONFIG_GENERAL_SHORTCUT_TRIGGER     = "/general/keyboard_shortcut_trigger"
CONFIG_GENERAL_SHORTCUT_TRIGGER_DEFAULT = [
    "Ctrl+space",
    "Zenkaku_Hankaku",
    "Hangul"]
CONFIG_GENERAL_SHORTCUT_NEXT_ENGINE = "/general/keyboard_shortcut_next_engine"
CONFIG_GENERAL_SHORTCUT_NEXT_ENGINE_DEFAULT = []
CONFIG_GENERAL_SHORTCUT_PREV_ENGINE = "/general/keyboard_shortcut_prev_engine"
CONFIG_GENERAL_SHORTCUT_PREV_ENGINE_DEFAULT = []

__mainloop = None

def __init_main_loop():
    global __mainloop
    if __mainloop == None:
        import gobject
        __mainloop = gobject.MainLoop()

def main():
    __init_main_loop()
    __mainloop.run()

def main_quit():
    global __mainloop
    if __mainloop:
        __mainloop.quit()

def main_iteration(may_block=False):
    __init_main_loop()
    return __mainloop.get_context().iteration(may_block)
<commit_msg>Fix typo in the default configuration of the trigger hotkeys.<commit_after># vim:set et sts=4 sw=4:
#
# ibus - The Input Bus
#
# Copyright (c) 2007-2010 Peng Huang <shawn.p.huang@gmail.com>
# Copyright (c) 2007-2010 Red Hat, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330,
# Boston, MA  02111-1307  USA

__all__ = (
        "IBUS_IFACE_IBUS",
        "IBUS_SERVICE_IBUS",
        "IBUS_PATH_IBUS",
        "IBUS_IFACE_CONFIG",
        "IBUS_IFACE_PANEL",
        "IBUS_IFACE_ENGINE",
        "IBUS_IFACE_ENGINE_FACTORY",
        "IBUS_IFACE_INPUT_CONTEXT",
        "IBUS_IFACE_NOTIFICATIONS",
        "ORIENTATION_HORIZONTAL",
        "ORIENTATION_VERTICAL",
        "ORIENTATION_SYSTEM",
        "default_reply_handler",
        "default_error_handler",
        "DEFAULT_ASYNC_HANDLERS",
        "CONFIG_GENERAL_SHORTCUT_TRIGGER",
        "CONFIG_GENERAL_SHORTCUT_NEXT_ENGINE",
        "CONFIG_GENERAL_SHORTCUT_PREV_ENGINE",
        "CONFIG_GENERAL_SHORTCUT_TRIGGER_DEFAULT",
        "CONFIG_GENERAL_SHORTCUT_NEXT_ENGINE_DEFAULT",
        "CONFIG_GENERAL_SHORTCUT_PREV_ENGINE_DEFAULT",
        "main",
        "main_quit",
        "main_iteration",
        "get_address",
        "get_socket_path"
    )

import os
import sys
from xdg import BaseDirectory
import ctypes

# __display = os.environ["DISPLAY"]
# __hostname, __display_screen = __display.split(":", 1)
# if not __hostname.strip():
#     __hostname = "unix"
# __display_number = __display_screen.split(".")[0]
#
# __username = None
# try:
#     __username = os.getlogin()
# except:
#     pass
# if not __username:
#     __username = os.getenv ("LOGNAME")
# if not __username:
#     __username = os.getenv ("USER")
# if not __username:
#     __username = os.getenv ("LNAME")
# if not __username:
#     __username = os.getenv ("USERNAME")
# libibus = ctypes.CDLL("libibus.so")
# id = ctypes.c_char_p(libibus.ibus_get_local_machine_id()).value
#
# IBUS_SOCKET_FILE = os.path.join(BaseDirectory.xdg_config_home,
#                                 "ibus", "bus",
#                                 "%s-%s-%s"% (id, __hostname, __display_number))
# def get_address():
#     libibus = ctypes.CDLL("libibus.so")
#     address = ctypes.c_char_p(libibus.ibus_get_address()).value
#     return address
#
#     address = os.getenv("IBUS_ADDRESS")
#     if address:
#         return address
#     try:
#         for l in file(IBUS_SOCKET_FILE):
#             if not l.startswith("IBUS_ADDRESS="):
#                 continue
#             address = l[13:]
#             address = address.strip()
#             break
#     except:
#         return None
#     return address

libibus = ctypes.CDLL("libibus-1.0.so.0")
get_address = libibus.ibus_get_address
get_address.restype=ctypes.c_char_p

get_socket_path = libibus.ibus_get_socket_path
get_socket_path.restype=ctypes.c_char_p

# __session_id = os.getenv ("IBUS_SESSION_ID")
#
# IBUS_ADDR = "unix:path=/tmp/ibus-%s%s/ibus-%s-%s" % (__username,
#                                                      "-" + __session_id if __session_id else "",
#                                                      __hostname,
#                                                      __display_number)
# IBUS_ADDR  = "tcp:host=localhost,port=7799"

IBUS_IFACE_IBUS     = "org.freedesktop.IBus"
IBUS_PATH_IBUS      = "/org/freedesktop/IBus"
IBUS_SERVICE_IBUS   = "org.freedesktop.IBus"

IBUS_IFACE_PANEL            = "org.freedesktop.IBus.Panel"
IBUS_IFACE_CONFIG           = "org.freedesktop.IBus.Config"
IBUS_IFACE_ENGINE           = "org.freedesktop.IBus.Engine"
IBUS_IFACE_ENGINE_FACTORY   = "org.freedesktop.IBus.Factory"
IBUS_IFACE_INPUT_CONTEXT    = "org.freedesktop.IBus.InputContext"
IBUS_IFACE_NOTIFICATIONS    = "org.freedesktop.IBus.Notifications"

# define pre-edit commit mode when the focus is lost
IBUS_ENGINE_PREEDIT_CLEAR       = 0
IBUS_ENGINE_PREEDIT_COMMIT      = 1

# define orientation
ORIENTATION_HORIZONTAL  = 0
ORIENTATION_VERTICAL    = 1
ORIENTATION_SYSTEM      = 2

def default_reply_handler( *args):
    pass

def default_error_handler(e):
    print >> sys.stderr, e

DEFAULT_ASYNC_HANDLERS = {
    "reply_handler" : default_reply_handler,
    "error_handler" : default_error_handler
}

CONFIG_GENERAL_SHORTCUT_TRIGGER     = "/general/keyboard_shortcut_trigger"
CONFIG_GENERAL_SHORTCUT_TRIGGER_DEFAULT = [
    "Control+space",
    "Zenkaku_Hankaku",
    "Hangul"]
CONFIG_GENERAL_SHORTCUT_NEXT_ENGINE = "/general/keyboard_shortcut_next_engine"
CONFIG_GENERAL_SHORTCUT_NEXT_ENGINE_DEFAULT = []
CONFIG_GENERAL_SHORTCUT_PREV_ENGINE = "/general/keyboard_shortcut_prev_engine"
CONFIG_GENERAL_SHORTCUT_PREV_ENGINE_DEFAULT = []

__mainloop = None

def __init_main_loop():
    global __mainloop
    if __mainloop == None:
        import gobject
        __mainloop = gobject.MainLoop()

def main():
    __init_main_loop()
    __mainloop.run()

def main_quit():
    global __mainloop
    if __mainloop:
        __mainloop.quit()

def main_iteration(may_block=False):
    __init_main_loop()
    return __mainloop.get_context().iteration(may_block)
<|endoftext|>


<commit_before>import sys
from datetime import datetime, time
import locale
from itertools import chain
import codecs
from common.utils import fetch_content_from_url, make_soup_from_html_content, remove_text_formatting_markup_from_fragments
from common.utils import extract_plaintext_urls_from_text
from csxj.common.tagging import tag_URL, classify_and_tag, make_tagged_url, TaggedURL
from csxj.db.article import ArticleData

# for datetime conversions
if sys.platform in ['linux2', 'cygwin']:
    locale.setlocale(locale.LC_TIME, 'fr_FR.UTF8')
elif sys.platform in [ 'darwin']:
    locale.setlocale(locale.LC_TIME, 'fr_BE')

RTLINFO_OWN_NETLOC = 'www.rtl.be'

RTLINFO_INTERNAL_SITES = {
    'blogs.rtlinfo.be':['internal', 'blog'],
}

SOURCE_TITLE = u"RTL Info"
SOURCE_NAME = u"rtlinfo"

def extract_title(main_article):
    left_column = main_article.find('div', {'id':'leftCol'})
    title = left_column.find('h1', {'class':'rtl_font_weight_normal'})

    return remove_text_formatting_markup_from_fragments(title.contents)

def extract_date_and_time(main_article):
    date_container = main_article.find('span', {'class':'date'})
    date_string = date_container.contents[0].strip()

    date_bytestring = codecs.encode(date_string, 'utf-8')
    pub_date = datetime.strptime(date_bytestring, '%d %B %Y')

    time_container = date_container.find('span', {'class':'time'})
    time_string=  time_container.contents[0]
    h, m = [int(i) for i in time_string.split('h')]
    pub_time = time(h, m)

    return pub_date, pub_time

def extract_category(main_article):
    breadcrumb_container = main_article.find('div', {'id':'breadcrumb'})
    breadcrumbs = [''.join(link.contents) for link in  breadcrumb_container.findAll('a')]

    return breadcrumbs

def extract_external_links(main_article):
    container = main_article.find('div', {'class':'art_ext_links'})

    if container:
        link_list = container.ul
        items = link_list.findAll('li')
        urls_and_titles = [(i.a.get('href'), remove_text_formatting_markup_from_fragments(i.a.contents))  for i in items]

        tagged_urls = list()
        for url, title in urls_and_titles:
            tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
            tagged_urls.append(make_tagged_url(url, title, tags))

        return tagged_urls

    else:
        return []

def extract_related_links(main_article):
    container = main_article.find('div', {'class':'relatedArticles'})

    if container:
        left_list, right_list = container.findAll('ul')
        all_list_items = [link_list.findAll('li', recursive=False) for link_list in (left_list, right_list)]

        tagged_urls = list()
        for item in chain(*all_list_items):
            url, title = item.a.get('href'), remove_text_formatting_markup_from_fragments(item.a.contents)
            tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
            tags.add('associated')

            tagged_urls.append(make_tagged_url(url, title, tags))

        return tagged_urls
    else:
        return []

def extract_links(main_article):
    external_links = extract_external_links(main_article)
    related_links = extract_related_links(main_article)

    return external_links + related_links

def extract_usable_links(container):
    """
    Extracts all <a> elements, then filters out anything with no title and target.

    Why, will you ask me? I'm going to tell you why. Because *sometimes* that
    stupid cms allows for broken empty links to appear in the source.
    It's not even clickable. It's just there, doing nothing.

    With no target.
    With no title.
    With no purpose.
    With no soul.

    This is the saddest thing ever.
    """
    def is_usable(link):
        return link.get('href') and link.contents

    all_links = container.findAll('a')
    return [l for l in  all_links if is_usable(l)]

def extract_embedded_links_from_articlebody(article_body):
    embedded_links = list()

    for link in extract_usable_links(article_body):
        url = link.get('href')
        title = remove_text_formatting_markup_from_fragments(link.contents)
        tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
        tags.add('in text')
        embedded_links.append(make_tagged_url(url, title, tags))

    for embedded_video_frame in article_body.findAll('iframe'):
        url = embedded_video_frame.get('src')
        title = '[Video] {0}'.format(url)
        tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
        tags.union(['in text', 'embedded'])
        embedded_links.append(make_tagged_url(url, title, tags))

    return embedded_links

def extract_links_and_text_content(main_article):
    article_body = main_article.find('div', {'class':'articleBody rtl_margin_top_25'})

    embedded_links = extract_embedded_links_from_articlebody(article_body)

    all_paragraphs = article_body.findAll('p', recursive=False)
    cleaned_up_paragraphs = list()
    all_plaintext_urls = list()

    for p in all_paragraphs:
        paragraph = remove_text_formatting_markup_from_fragments(p.contents)
        plaintext_urls = extract_plaintext_urls_from_text(paragraph)
        for url in plaintext_urls:
            tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
            tags.union(['in text', 'plaintext'])
            all_plaintext_urls.append(make_tagged_url(url, url, tags))

        cleaned_up_paragraphs.append(paragraph)

    all_links = embedded_links+all_plaintext_urls
    return all_links, cleaned_up_paragraphs

def extract_intro(main_article):
    left_column = main_article.find('div', {'id':'leftCol'})
    intro_container = left_column.find('h2', recursive=False)

    if intro_container:
        intro = remove_text_formatting_markup_from_fragments(intro_container.contents)
    else:
        intro = None

    return intro

def extract_article_data(source):
    if hasattr(source, 'read'):
        html_content = source.read()
    else:
        html_content = fetch_content_from_url(source)

    soup = make_soup_from_html_content(html_content)

    main_article= soup.find('div', {'id':'mainArticle'})

    if main_article:
        title = extract_title(main_article)
        category = extract_category(main_article)
        pub_date, pub_time = extract_date_and_time(main_article)
        fetched_datetime = datetime.now()

        links = extract_links(main_article)

        author = None
        embedded_links, content = extract_links_and_text_content(main_article)
        intro = extract_intro(main_article)

        all_links = links+embedded_links

        article_data = ArticleData(source, title, pub_date, pub_time, fetched_datetime, all_links, category, author, intro, content)
        return article_data, html_content

    else:
        return None, html_content

def extract_frontpage_title_and_url(link):
    title = remove_text_formatting_markup_from_fragments(link, ' ')
    return title, link.get('href')

def extract_headlines_from_module(module_container):
    """

    """
    body = module_container.find('div', {'class':'ModuleElementBody'})

    if body:
        headline_lists = body.findAll('div', {'class':'tot_widget-ul'})

        titles_and_urls = list()
        for l in headline_lists:
            for tag_name in ('h3', 'h5', 'li'):
                titles_and_urls.extend([extract_frontpage_title_and_url(h.a) for h in l.findAll(tag_name) if h.a])

        return titles_and_urls
    else:
        return []

def extract_headlines_from_modules(maincontent):
    module_zone = maincontent.find('div', {'id':'hp-zone-list'})

    all_modules = module_zone.findAll('div', {'class':'ModuleElement'})

    headlines = list()
    for module in all_modules:
        module_headlines = extract_headlines_from_module(module)
        headlines.extend(module_headlines)

    return headlines

def extract_small_articles(maincontent):
    small_articles_container = maincontent.find('div', {'id':'SmallArticles'})

    small_articles = small_articles_container.findAll('div', {'class':'SmallArticles-Items'})
    small_articles.extend(small_articles_container.findAll('div', {'class':'SmallArticles-Items SmallArticles-Lasts'}))

    return [extract_frontpage_title_and_url(item.h5.a) for item in small_articles ]

def extract_first_articles(maincontent):
    first_article = maincontent.find('div', {'id':'FirstArticlesLeft'})
    titles_and_urls = list()

    if first_article:
        if first_article.h1:
            titles_and_urls.append(extract_frontpage_title_and_url(first_article.h1.a))
        else:
            video_headline = first_article.find('div', {'class':'img_with_text_on_top_holder'})
            if video_headline:
                url = video_headline.a.get('href')
                title = video_headline.a.img.get('alt')
                titles_and_urls.append((title, url))

    articles_right = maincontent.findAll('div', {'class':'ArticlesRight '})
    articles_right.extend(maincontent.findAll('div', {'class':'ArticlesRight ArticlesRight-First'}))
    articles_right.extend(maincontent.findAll('div', {'class':'ArticlesRight ArticlesRight-Last'}))

    titles_and_urls.extend(extract_frontpage_title_and_url(article.h5.a) for article in articles_right)

    return titles_and_urls

def make_full_url((title, url)):
    if url.startswith('http://'):
        return title, url
    else:
        return title, 'http://www.rtl.be{0}'.format(url)

def separate_news_and_blogposts(titles_and_urls):
    all_items = set(titles_and_urls)
    blogposts = set([(title, url) for title, url in all_items if not url.startswith('/info')])
    news_items = all_items - blogposts

    return news_items, blogposts

def get_frontpage_toc():
    url = 'http://www.rtl.be/info/'
    html_content = fetch_content_from_url(url)
    soup = make_soup_from_html_content(html_content)

    maincontent = soup.find('div', {'class':'mainContent'})

    first_articles = extract_first_articles(maincontent)
    small_articles = extract_small_articles(maincontent)
    modules_articles = extract_headlines_from_modules(maincontent)

    all_articles = first_articles + small_articles + modules_articles

    news_items, blogposts = separate_news_and_blogposts(all_articles)

    return [make_full_url(title_and_url) for title_and_url in news_items], list(blogposts)

def test_sample_data():
    filename = '../../sample_data/rtlinfo_unicode_date.html'
    with open(filename) as f:
        article, raw = extract_article_data(f)
        article.print_summary()

def show_frontpage_news():
    toc, blogs = get_frontpage_toc()
    for t, u in toc:
        print u'fetching: {0}[{1}]'.format(t, u)
        article_data, raw_html = extract_article_data(u)
        if article_data:
            article_data.print_summary()
        else:
            print 'Was redirected to a blogpost'

if __name__=='__main__':
    #show_frontpage_news()
    test_sample_data()

<commit_msg>better tags<commit_after>import sys
from datetime import datetime, time
import locale
from itertools import chain
import codecs
from common.utils import fetch_content_from_url, make_soup_from_html_content, remove_text_formatting_markup_from_fragments
from common.utils import extract_plaintext_urls_from_text
from csxj.common.tagging import tag_URL, classify_and_tag, make_tagged_url, TaggedURL
from csxj.db.article import ArticleData

# for datetime conversions
if sys.platform in ['linux2', 'cygwin']:
    locale.setlocale(locale.LC_TIME, 'fr_FR.UTF8')
elif sys.platform in [ 'darwin']:
    locale.setlocale(locale.LC_TIME, 'fr_BE')

RTLINFO_OWN_NETLOC = 'www.rtl.be'

RTLINFO_INTERNAL_SITES = {
    'blogs.rtlinfo.be':['internal', 'blog'],
}

SOURCE_TITLE = u"RTL Info"
SOURCE_NAME = u"rtlinfo"

def extract_title(main_article):
    left_column = main_article.find('div', {'id':'leftCol'})
    title = left_column.find('h1', {'class':'rtl_font_weight_normal'})

    return remove_text_formatting_markup_from_fragments(title.contents)

def extract_date_and_time(main_article):
    date_container = main_article.find('span', {'class':'date'})
    date_string = date_container.contents[0].strip()

    date_bytestring = codecs.encode(date_string, 'utf-8')
    pub_date = datetime.strptime(date_bytestring, '%d %B %Y')

    time_container = date_container.find('span', {'class':'time'})
    time_string=  time_container.contents[0]
    h, m = [int(i) for i in time_string.split('h')]
    pub_time = time(h, m)

    return pub_date, pub_time

def extract_category(main_article):
    breadcrumb_container = main_article.find('div', {'id':'breadcrumb'})
    breadcrumbs = [''.join(link.contents) for link in  breadcrumb_container.findAll('a')]

    return breadcrumbs

def extract_external_links(main_article):
    container = main_article.find('div', {'class':'art_ext_links'})

    if container:
        link_list = container.ul
        items = link_list.findAll('li')
        urls_and_titles = [(i.a.get('href'), remove_text_formatting_markup_from_fragments(i.a.contents))  for i in items]

        tagged_urls = list()
        for url, title in urls_and_titles:
            tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
            tagged_urls.append(make_tagged_url(url, title, tags))

        return tagged_urls

    else:
        return []

def extract_related_links(main_article):
    container = main_article.find('div', {'class':'relatedArticles'})

    if container:
        left_list, right_list = container.findAll('ul')
        all_list_items = [link_list.findAll('li', recursive=False) for link_list in (left_list, right_list)]

        tagged_urls = list()
        for item in chain(*all_list_items):
            url, title = item.a.get('href'), remove_text_formatting_markup_from_fragments(item.a.contents)
            tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
            tags.add('associated')

            tagged_urls.append(make_tagged_url(url, title, tags))

        return tagged_urls
    else:
        return []

def extract_links(main_article):
    external_links = extract_external_links(main_article)
    related_links = extract_related_links(main_article)

    return external_links + related_links

def extract_usable_links(container):
    """
    Extracts all <a> elements, then filters out anything with no title and target.

    Why, will you ask me? I'm going to tell you why. Because *sometimes* that
    stupid cms allows for broken empty links to appear in the source.
    It's not even clickable. It's just there, doing nothing.

    With no target.
    With no title.
    With no purpose.
    With no soul.

    This is the saddest thing ever.
    """
    def is_usable(link):
        return link.get('href') and link.contents

    all_links = container.findAll('a')
    return [l for l in  all_links if is_usable(l)]

def extract_embedded_links_from_articlebody(article_body):
    embedded_links = list()

    for link in extract_usable_links(article_body):
        url = link.get('href')
        title = remove_text_formatting_markup_from_fragments(link.contents)
        tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
        tags.add('in text')
        embedded_links.append(make_tagged_url(url, title, tags))

    for embedded_video_frame in article_body.findAll('iframe'):
        url = embedded_video_frame.get('src')
        title = '[Video] {0}'.format(url)
        tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
        tags = tags.union(['in text', 'embedded'])
        embedded_links.append(make_tagged_url(url, title, tags))

    return embedded_links

def extract_links_and_text_content(main_article):
    article_body = main_article.find('div', {'class':'articleBody rtl_margin_top_25'})

    embedded_links = extract_embedded_links_from_articlebody(article_body)

    all_paragraphs = article_body.findAll('p', recursive=False)
    cleaned_up_paragraphs = list()
    all_plaintext_urls = list()

    for p in all_paragraphs:
        paragraph = remove_text_formatting_markup_from_fragments(p.contents)
        plaintext_urls = extract_plaintext_urls_from_text(paragraph)
        for url in plaintext_urls:
            tags = classify_and_tag(url, RTLINFO_OWN_NETLOC, RTLINFO_INTERNAL_SITES)
            tags = tags.union(['in text', 'plaintext'])
            all_plaintext_urls.append(make_tagged_url(url, url, tags))

        cleaned_up_paragraphs.append(paragraph)

    all_links = embedded_links+all_plaintext_urls
    return all_links, cleaned_up_paragraphs

def extract_intro(main_article):
    left_column = main_article.find('div', {'id':'leftCol'})
    intro_container = left_column.find('h2', recursive=False)

    if intro_container:
        intro = remove_text_formatting_markup_from_fragments(intro_container.contents)
    else:
        intro = None

    return intro

def extract_article_data(source):
    if hasattr(source, 'read'):
        html_content = source.read()
    else:
        html_content = fetch_content_from_url(source)

    soup = make_soup_from_html_content(html_content)

    main_article= soup.find('div', {'id':'mainArticle'})

    if main_article:
        title = extract_title(main_article)
        category = extract_category(main_article)
        pub_date, pub_time = extract_date_and_time(main_article)
        fetched_datetime = datetime.now()

        links = extract_links(main_article)

        author = None
        embedded_links, content = extract_links_and_text_content(main_article)
        intro = extract_intro(main_article)

        all_links = links+embedded_links

        article_data = ArticleData(source, title, pub_date, pub_time, fetched_datetime, all_links, category, author, intro, content)
        return article_data, html_content

    else:
        return None, html_content

def extract_frontpage_title_and_url(link):
    title = remove_text_formatting_markup_from_fragments(link, ' ')
    return title, link.get('href')

def extract_headlines_from_module(module_container):
    """

    """
    body = module_container.find('div', {'class':'ModuleElementBody'})

    if body:
        headline_lists = body.findAll('div', {'class':'tot_widget-ul'})

        titles_and_urls = list()
        for l in headline_lists:
            for tag_name in ('h3', 'h5', 'li'):
                titles_and_urls.extend([extract_frontpage_title_and_url(h.a) for h in l.findAll(tag_name) if h.a])

        return titles_and_urls
    else:
        return []

def extract_headlines_from_modules(maincontent):
    module_zone = maincontent.find('div', {'id':'hp-zone-list'})

    all_modules = module_zone.findAll('div', {'class':'ModuleElement'})

    headlines = list()
    for module in all_modules:
        module_headlines = extract_headlines_from_module(module)
        headlines.extend(module_headlines)

    return headlines

def extract_small_articles(maincontent):
    small_articles_container = maincontent.find('div', {'id':'SmallArticles'})

    small_articles = small_articles_container.findAll('div', {'class':'SmallArticles-Items'})
    small_articles.extend(small_articles_container.findAll('div', {'class':'SmallArticles-Items SmallArticles-Lasts'}))

    return [extract_frontpage_title_and_url(item.h5.a) for item in small_articles ]

def extract_first_articles(maincontent):
    first_article = maincontent.find('div', {'id':'FirstArticlesLeft'})
    titles_and_urls = list()

    if first_article:
        if first_article.h1:
            titles_and_urls.append(extract_frontpage_title_and_url(first_article.h1.a))
        else:
            video_headline = first_article.find('div', {'class':'img_with_text_on_top_holder'})
            if video_headline:
                url = video_headline.a.get('href')
                title = video_headline.a.img.get('alt')
                titles_and_urls.append((title, url))

    articles_right = maincontent.findAll('div', {'class':'ArticlesRight '})
    articles_right.extend(maincontent.findAll('div', {'class':'ArticlesRight ArticlesRight-First'}))
    articles_right.extend(maincontent.findAll('div', {'class':'ArticlesRight ArticlesRight-Last'}))

    titles_and_urls.extend(extract_frontpage_title_and_url(article.h5.a) for article in articles_right)

    return titles_and_urls

def make_full_url((title, url)):
    if url.startswith('http://'):
        return title, url
    else:
        return title, 'http://www.rtl.be{0}'.format(url)

def separate_news_and_blogposts(titles_and_urls):
    all_items = set(titles_and_urls)
    blogposts = set([(title, url) for title, url in all_items if not url.startswith('/info')])
    news_items = all_items - blogposts

    return news_items, blogposts

def get_frontpage_toc():
    url = 'http://www.rtl.be/info/'
    html_content = fetch_content_from_url(url)
    soup = make_soup_from_html_content(html_content)

    maincontent = soup.find('div', {'class':'mainContent'})

    first_articles = extract_first_articles(maincontent)
    small_articles = extract_small_articles(maincontent)
    modules_articles = extract_headlines_from_modules(maincontent)

    all_articles = first_articles + small_articles + modules_articles

    news_items, blogposts = separate_news_and_blogposts(all_articles)

    return [make_full_url(title_and_url) for title_and_url in news_items], list(blogposts)

def test_sample_data():
    filename = '../../sample_data/rtlinfo_unicode_date.html'
    with open(filename) as f:
        article, raw = extract_article_data(f)
        article.print_summary()

def show_frontpage_news():
    toc, blogs = get_frontpage_toc()
    for t, u in toc:
        print u'fetching: {0}[{1}]'.format(t, u)
        article_data, raw_html = extract_article_data(u)
        if article_data:
            article_data.print_summary()
        else:
            print 'Was redirected to a blogpost'

if __name__=='__main__':
    #show_frontpage_news()
    test_sample_data()

<|endoftext|>


<commit_before># Copyright (C) 2009  Renato Lima - Akretion
# Copyright (C) 2012  Raphaël Valyi - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

from odoo import models, fields, api
from odoo.addons import decimal_precision as dp

class SaleOrder(models.Model):
    _name = 'sale.order'
    _inherit = ['sale.order', 'l10n_br_fiscal.document.mixin']

    @api.multi
    @api.depends('order_line.price_unit', 'order_line.tax_id',
                 'order_line.discount', 'order_line.product_uom_qty')
    def _amount_all_wrapper(self):
        """ Wrapper because of direct method passing
        as parameter for function fields """
        return self._amount_all()

    @api.multi
    def _amount_all(self):
        for order in self:
            order.amount_untaxed = 0.0
            order.amount_tax = 0.0
            order.amount_total = 0.0
            order.amount_discount = 0.0
            order.amount_gross = 0.0
            amount_tax = amount_untaxed = \
                amount_discount = amount_gross = 0.0
            for line in order.order_line:
                amount_tax += self._amount_line_tax(line)
                amount_untaxed += line.price_subtotal
                amount_discount += line.discount_value
                amount_gross += line.price_gross

            order.amount_tax = order.pricelist_id.currency_id.round(
                amount_tax)
            order.amount_untaxed = order.pricelist_id.currency_id.round(
                amount_untaxed)
            order.amount_total = (order.amount_untaxed +
                                  order.amount_tax)
            order.amount_discount = order.pricelist_id.currency_id.round(
                amount_discount)
            order.amount_gross = order.pricelist_id.currency_id.round(
                amount_gross)

    @api.model
    def _amount_line_tax(self, line):
        value = 0.0
        price = line._calc_line_base_price()
        qty = line._calc_line_quantity()
        for computed in line.tax_id.compute_all(
                price, quantity=qty, product=line.product_id,
                partner=line.order_id.partner_invoice_id)['taxes']:
            tax = self.env['account.tax'].browse(computed['id'])
            if not tax.tax_group_id.fiscal_tax_group_id.tax_include:
                value += computed.get('amount', 0.0)
        return value

    copy_note = fields.Boolean(
        string='Copiar Observação no documentos fiscal')

    amount_discount = fields.Float(
        compute='_amount_all_wrapper',
        string='Desconto (-)',
        digits=dp.get_precision('Account'),
        store=True,
        help="The discount amount.")

    amount_gross = fields.Float(
        compute='_amount_all_wrapper',
        string='Vlr. Bruto',
        digits=dp.get_precision('Account'),
        store=True, help="The discount amount.")

    discount_rate = fields.Float(
        string='Desconto',
        readonly=True,
        states={'draft': [('readonly', False)]})

    cnpj_cpf = fields.Char(
        string=u'CNPJ/CPF',
        related='partner_id.cnpj_cpf')

    legal_name = fields.Char(
        string=u'Razão Social',
        related='partner_id.legal_name')

    ie = fields.Char(
        string=u'Inscrição Estadual',
        related='partner_id.inscr_est')

    @api.onchange('discount_rate')
    def onchange_discount_rate(self):
        for sale_order in self:
            for sale_line in sale_order.order_line:
                sale_line.discount = sale_order.discount_rate

    @api.model
    def _fiscal_comment(self, order):
        fp_comment = []
        fp_ids = []

        for line in order.order_line:
            if line.operation_line_id and \
                    line.operation_line_id.inv_copy_note and \
                    line.operation_line_id.note:
                if line.operation_line_id.id not in fp_ids:
                    fp_comment.append(line.operation_line_id.note)
                    fp_ids.append(line.operation_line_id.id)

        return fp_comment

    @api.multi
    def _prepare_invoice(self):
        self.ensure_one()
        result = super(SaleOrder, self)._prepare_invoice()
        context = self.env.context

        if (context.get('fiscal_type') == 'service' and
                self.order_line and self.order_line[0].operation_id):
            operation_id = self.order_line[0].operation_id.id
            result['operation_id'] = self.order_line.operation_id.id
        else:
            operation_id = self.operation_id
            result['operation_id'] = self.operation_id.id

        if operation_id:
            result['journal_id'] = operation_id.property_journal.id

        result['partner_shipping_id'] = self.partner_shipping_id.id

        comment = []
        if self.note and self.copy_note:
            comment.append(self.note)

        fiscal_comment = self._fiscal_comment(self)
        result['comment'] = " - ".join(comment)
        result['fiscal_comment'] = " - ".join(fiscal_comment)
        result['operation_id'] = operation_id.id

        return result
<commit_msg>[12.0][MIG][WIP] Code.<commit_after># Copyright (C) 2009  Renato Lima - Akretion
# Copyright (C) 2012  Raphaël Valyi - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

from odoo import models, fields, api
from odoo.addons import decimal_precision as dp

class SaleOrder(models.Model):
    _name = 'sale.order'
    _inherit = ['sale.order', 'l10n_br_fiscal.document.mixin']

    @api.multi
    @api.depends('order_line.price_unit', 'order_line.tax_id',
                 'order_line.discount', 'order_line.product_uom_qty')
    def _amount_all_wrapper(self):
        """ Wrapper because of direct method passing
        as parameter for function fields """
        return self._amount_all()

    @api.multi
    def _amount_all(self):
        for order in self:
            order.amount_untaxed = 0.0
            order.amount_tax = 0.0
            order.amount_total = 0.0
            order.amount_discount = 0.0
            order.amount_gross = 0.0
            amount_tax = amount_untaxed = \
                amount_discount = amount_gross = 0.0
            for line in order.order_line:
                amount_tax += self._amount_line_tax(line)
                amount_untaxed += line.price_subtotal
                amount_discount += line.discount_value
                amount_gross += line.price_gross

            order.amount_tax = order.pricelist_id.currency_id.round(
                amount_tax)
            order.amount_untaxed = order.pricelist_id.currency_id.round(
                amount_untaxed)
            order.amount_total = (order.amount_untaxed +
                                  order.amount_tax)
            order.amount_discount = order.pricelist_id.currency_id.round(
                amount_discount)
            order.amount_gross = order.pricelist_id.currency_id.round(
                amount_gross)

    @api.model
    def _amount_line_tax(self, line):
        value = 0.0
        price = line._calc_line_base_price()
        qty = line._calc_line_quantity()
        for computed in line.tax_id.compute_all(
                price, quantity=qty, product=line.product_id,
                partner=line.order_id.partner_invoice_id)['taxes']:
            tax = self.env['account.tax'].browse(computed['id'])
            if not tax.tax_group_id.fiscal_tax_group_id.tax_include:
                value += computed.get('amount', 0.0)
        return value

    copy_note = fields.Boolean(
        string='Copiar Observação no documentos fiscal')

    amount_discount = fields.Float(
        compute='_amount_all_wrapper',
        string='Desconto (-)',
        digits=dp.get_precision('Account'),
        store=True,
        help="The discount amount.")

    amount_gross = fields.Float(
        compute='_amount_all_wrapper',
        string='Vlr. Bruto',
        digits=dp.get_precision('Account'),
        store=True, help="The discount amount.")

    discount_rate = fields.Float(
        string='Desconto',
        readonly=True,
        states={'draft': [('readonly', False)]})

    cnpj_cpf = fields.Char(
        string=u'CNPJ/CPF',
        related='partner_id.cnpj_cpf')

    legal_name = fields.Char(
        string=u'Razão Social',
        related='partner_id.legal_name')

    ie = fields.Char(
        string=u'Inscrição Estadual',
        related='partner_id.inscr_est')

    @api.onchange('discount_rate')
    def onchange_discount_rate(self):
        for sale_order in self:
            for sale_line in sale_order.order_line:
                sale_line.discount = sale_order.discount_rate

    # TODO FIscal Comment
    # @api.model
    # def _fiscal_comment(self, order):
    #     fp_comment = []
    #     fp_ids = []
    #
    #     for line in order.order_line:
    #         if line.operation_line_id and \
    #                 line.operation_line_id.inv_copy_note and \
    #                 line.operation_line_id.note:
    #             if line.operation_line_id.id not in fp_ids:
    #                 fp_comment.append(line.operation_line_id.note)
    #                 fp_ids.append(line.operation_line_id.id)
    #
    #      return fp_comment

    @api.multi
    def _prepare_invoice(self):
        self.ensure_one()
        result = super(SaleOrder, self)._prepare_invoice()
        context = self.env.context

        if (context.get('fiscal_type') == 'service' and
                self.order_line and self.order_line[0].operation_id):
            operation_id = self.order_line[0].operation_id.id
            result['operation_id'] = self.order_line.operation_id.id
        else:
            operation_id = self.operation_id
            result['operation_id'] = self.operation_id.id

        # TODO check journal
        # if operation_id:
        #    result['journal_id'] = operation_id.property_journal.id

        result['partner_shipping_id'] = self.partner_shipping_id.id

        comment = []
        if self.note and self.copy_note:
            comment.append(self.note)

        # TODO FISCAL Commnet
        # fiscal_comment = self._fiscal_comment(self)
        result['comment'] = " - ".join(comment)
        # result['fiscal_comment'] = " - ".join(fiscal_comment)
        result['operation_id'] = operation_id.id

        return result
<|endoftext|>


<commit_before>import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import rgb2hex
from mpl_toolkits.basemap import Basemap
from numpy.random import uniform

# Plot a bunch of randomly distributed points on the earth.

# set up stereographic map centered on N. Pole.
m = Basemap(lon_0=-105,boundinglat=30.,
            resolution='l',area_thresh=10000.,projection='npstere')
# number of points to plot.
npts = 750
# generate random points on a sphere,
# so that every small area on the sphere is expected
# to have the same number of points.
# http://mathworld.wolfram.com/SpherePointPicking.html
u = uniform(0.,1.,size=npts)
v = uniform(0.,1.,size=npts)
z = uniform(0,100,size=npts)
lons = 360.*u
lats = (180./np.pi)*np.arccos(2*v-1) - 90.
# transform lons and lats to map coordinates.
x,y = m(lons,lats)
# plot them as filled circles on the map.
# first, create a figure.
fig=plt.figure()
# draw colored markers.
# use zorder=10 to make sure markers are drawn last.
# (otherwise they are covered up when continents are filled)
m.scatter(x,y,25,z,cmap=plt.cm.jet,marker='o',edgecolors='none',zorder=10)
# plot colorbar for markers.
plt.colorbar()
# create a list of strings containing z values
# or, plot actual numbers as color-coded text strings.
#zn = [ '%2i' % zz for zz in z ]
## plot numbers on map, colored by value.
#for numstr,zval,xpt,ypt in zip(zn,z,x,y):
#    # only plot values inside map region.
#    if xpt > m.xmin and xpt < m.xmax and ypt > m.ymin and ypt < m.ymax:
#        hexcolor = rgb2hex(plt.cm.jet(zval/100.)[:3])
#        plt.text(xpt,ypt,numstr,fontsize=9,weight='bold',color=hexcolor)
# draw coasts and fill continents/lakes.
m.drawcoastlines(linewidth=0.5)
m.fillcontinents(color='coral',lake_color='aqua')
# color ocean areas
m.drawmapboundary(fill_color='aqua')
# draw parallels and meridians.
delat = 20.
circles = np.arange(0.,90.,delat).tolist()+\
          np.arange(-delat,-90,-delat).tolist()
m.drawparallels(circles)
delon = 45.
meridians = np.arange(0,360,delon)
m.drawmeridians(meridians,labels=[1,0,1,1])
plt.title('Random Points',y=1.05)
plt.show()
<commit_msg>cosmetic changes<commit_after>import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import rgb2hex
from mpl_toolkits.basemap import Basemap
from numpy.random import uniform

# Plot a bunch of randomly distributed points on the earth.

# set up stereographic map centered on N. Pole.
m = Basemap(lon_0=-105,boundinglat=30.,
            resolution='l',area_thresh=10000.,projection='npstere')
# number of points to plot.
npts = 500
# generate random points on a sphere,
# so that every small area on the sphere is expected
# to have the same number of points.
# http://mathworld.wolfram.com/SpherePointPicking.html
u = uniform(0.,1.,size=npts)
v = uniform(0.,1.,size=npts)
lons = 360.*u
lats = (180./np.pi)*np.arccos(2*v-1) - 90.
z = uniform(0,100,size=npts) # this field controls color of dots.
# transform lons and lats to map coordinates.
x,y = m(lons,lats)
# plot them as filled circles on the map.
# first, create a figure.
fig=plt.figure()
# draw colored markers.
# use zorder=10 to make sure markers are drawn last.
# (otherwise they are covered up when continents are filled)
m.scatter(x,y,25,z,cmap=plt.cm.jet,marker='o',edgecolors='none',zorder=10)
# plot colorbar for markers.
plt.colorbar()
# create a list of strings containing z values
# or, plot actual numbers as color-coded text strings.
#zn = [ '%2i' % zz for zz in z ]
## plot numbers on map, colored by value.
#for numstr,zval,xpt,ypt in zip(zn,z,x,y):
#    # only plot values inside map region.
#    if xpt > m.xmin and xpt < m.xmax and ypt > m.ymin and ypt < m.ymax:
#        hexcolor = rgb2hex(plt.cm.jet(zval/100.)[:3])
#        plt.text(xpt,ypt,numstr,fontsize=9,weight='bold',color=hexcolor)
# draw coasts and fill continents/lakes.
m.drawcoastlines(linewidth=0.5)
m.fillcontinents(color='coral',lake_color='aqua')
# color ocean areas
m.drawmapboundary(fill_color='aqua')
# draw parallels and meridians.
delat = 20.
circles = np.arange(0.,90.,delat).tolist()+\
          np.arange(-delat,-90,-delat).tolist()
m.drawparallels(circles)
delon = 45.
meridians = np.arange(0,360,delon)
m.drawmeridians(meridians,labels=[1,0,1,1])
plt.title('Random Points',y=1.05)
plt.show()
<|endoftext|>


<commit_before>"""
Module with layers
"""

import itertools

import numpy as np

class Layer:

    def __init__(self):

        self.input_shape = None
        self.output_shape = None

        self.last_input = None
        self.last_output = None

    def build(self, input_shape):

        raise NotImplementedError()

    def forward(self, x):

        raise NotImplementedError()

    def train_forward(self, x):

        raise NotImplementedError()

    def train_backward(self, gradients, learning_rate=None):

        raise NotImplementedError()

class Input(Layer):
    """
    Input layer
    """

    def __init__(self, sample_shape):
        """
        Constructor
        :param sample_shape: 3 elements tuple (height, width, channels)
        """

        super().__init__()

        self.input_shape = (None,) + tuple(sample_shape)

    def build(self, input_shape=None):

        # Input layer's output shape is same as its input shape
        self.output_shape = self.input_shape

    def forward(self, x):

        if x.shape[1:] != self.input_shape[1:]:

            raise ValueError("{} shape expected, but input has shape {}".format(
                self.input_shape, x.shape))

        return x

    def train_forward(self, x):

        return self.forward(x)

    def train_backward(self, gradients, learning_rate=None):
        """
        For Input layer train backward is a no-op
        :return: None
        """

        return None

class Flatten(Layer):
    """
    Layer for flattening input. Outer dimension (batch size) is not affected
    """

    def __init__(self):

        super().__init__()

    def build(self, input_shape):

        self.input_shape = tuple(input_shape)

        squeezed_sample_shape = []

        for dimension in input_shape[1:]:

            if dimension != 1:

                squeezed_sample_shape.append(dimension)

        self.output_shape = (input_shape[0],) + tuple(squeezed_sample_shape)

    def forward(self, x):

        shape = (x.shape[0],) + self.output_shape[1:]
        return x.reshape(shape)

    def train_forward(self, x):

        self.last_input = x
        self.last_output = self.forward(x)

        return self.last_output

    def train_backward(self, gradients, learning_rate=None):

        return gradients.reshape(self.last_input.shape)

class Convolution2D(Layer):

    def __init__(self, nb_filter, nb_row, nb_col):
        """
        2D convolutional layer. Applies ReLU activation.
        :param nb_filter: number of filters
        :param nb_row: width of each filter
        :param nb_col: height of each filter
        """

        super().__init__()

        self.nb_filter = nb_filter
        self.nb_row = nb_row
        self.nb_col = nb_col

        self.kernels = None
        self.biases = None

        self.last_preactivation = None

    def build(self, input_shape):

        if len(input_shape) != 4:

            raise ValueError("Input shape {} isn't 4-dimensional".format(input_shape))

        self.input_shape = input_shape

        # Dimensions are [images, rows, columns, filters]
        self.output_shape = (None, input_shape[1] - self.nb_row + 1, input_shape[2] - self.nb_col + 1, self.nb_filter)

        input_channels = input_shape[-1]
        kernels_shape = (self.nb_filter, self.nb_row, self.nb_col, input_channels)

        scale = np.sqrt(2 / (self.nb_row * self.nb_col * input_channels))
        self.kernels = np.random.uniform(low=-scale, high=scale, size=kernels_shape)

        self.biases = np.zeros((self.nb_filter,), dtype=np.float32)

    def forward(self, input):

        preactivation = self._get_preactivation(input)

        # Apply ReLU activation
        return self._relu(preactivation)

    def _get_preactivation(self, input):

        preactivation = np.zeros(shape=(input.shape[0],) + self.output_shape[1:])

        # Kernels with kernels count dimension rolled to last dimension
        rolled_kernels = np.rollaxis(self.kernels, 0, 4)

        # For each image
        for sample_index, sample in enumerate(input):

            for row_index in range(0, input.shape[1] - self.nb_row + 1):

                for col_index in range(0, input.shape[2] - self.nb_col + 1):

                    # Select patch over which kernels will be applied
                    input_patch = sample[row_index: row_index + self.nb_row, col_index: col_index + self.nb_col, :]

                    # input patch is 3D, but we will convolve it with a series of 3D kernels, thus a 4D entity.
                    # Thus reshape input patch to have a 4th dimension. Numpy will then broadcast input patch
                    # along that dimension to match number of kernels
                    reshaped_input_patch = input_patch.reshape(input_patch.shape + (1,))

                    # Preactivation pixel value will a 1D array that holds results of convolutions over
                    # a selected patch with all kernels, one value for each kernel
                    preactivation_pixel_values = \
                        np.sum(reshaped_input_patch * rolled_kernels, axis=(0, 1, 2)) + self.biases

                    preactivation[sample_index, row_index, col_index, :] = preactivation_pixel_values

        return preactivation

    def _relu(self, x):

        return x * (x > 0)

    def _relu_derivative(self, x):

        return 1 * (x > 0)

    def train_forward(self, input):

        self.last_input = input

        self.last_preactivation = self._get_preactivation(input)
        self.last_output = self._relu(self.last_preactivation)

        return self.last_output

    def train_backward(self, gradients, learning_rate):

        preactivation_error_gradients = gradients * self._relu_derivative(self.last_preactivation)

        # Copy old kernels, we will use original values when computing image gradients
        old_kernels = self.kernels.copy()

        self._update_kernels(preactivation_error_gradients, learning_rate)
        self._update_biases(preactivation_error_gradients, learning_rate)

        return self._get_image_gradients(preactivation_error_gradients, old_kernels)

    def _update_biases(self, preactivation_error_gradients, learning_rate):

        for kernel_index in range(len(self.kernels)):

            # Sum contributions for each image
            bias_error_gradients_sums = np.sum(preactivation_error_gradients[:, :, :, kernel_index], axis=(1, 2))

            # And take mean across contributions to different images
            mean_bias_error_gradient = np.mean(bias_error_gradients_sums)

            self.biases[kernel_index] -= learning_rate * mean_bias_error_gradient

    def _update_kernels(self, preactivation_error_gradients, learning_rate):

        for kernel_index in range(len(self.kernels)):

            # Select gradients that were affected by current kernel
            # Dimensions are [image, rows, cols]
            kernel_preactivation_error_gradients = preactivation_error_gradients[:, :, :, kernel_index]

            for y in range(self.nb_row):

                input_row_start = y
                input_row_end = self.input_shape[1] - self.nb_row + y + 1

                for x in range(self.nb_col):

                    input_column_start = x
                    input_column_end = self.input_shape[2] - self.nb_col + x + 1

                    inputs_patches = self.last_input[
                                     :, input_row_start:input_row_end, input_column_start:input_column_end, :]

                    # Reshape kernel preactivation error gradients so they have a channel dimension
                    # Numpy broadcasting will then take care of matching it with number of channels of inputs patches
                    reshaped_kernel_preactivation_error_gradients = kernel_preactivation_error_gradients.reshape(
                        kernel_preactivation_error_gradients.shape + (1,))

                    total_kernel_weight_gradient = np.sum(
                        reshaped_kernel_preactivation_error_gradients * inputs_patches, axis=(0, 1, 2))

                    weight_index = (kernel_index, y, x)

                    self.kernels[weight_index] -= learning_rate * total_kernel_weight_gradient / len(inputs_patches)

    def _get_image_gradients(self, preactivation_error_gradients, kernels):

        # Tensor for storing error gradients on input image
        image_gradients = np.zeros_like(self.last_input, dtype=np.float32)

        images_count = self.last_input.shape[0]
        rows_count = self.last_input.shape[1]
        columns_count = self.last_input.shape[2]

        # For each column
        for y in range(rows_count):

            # For each row
            for x in range(columns_count):

                # Select all kernel weights this pixel was convolved with
                # kernels ordering is: (self.nb_filter, self.nb_row, self.nb_col, input_channels)

                # Get start and end indices for part of kernel tensor that pixels I[y,x] was convolved with
                subkernel_y_start = max(0, y - self.last_preactivation.shape[1] + 1)
                subkernel_x_start = max(0, x - self.last_preactivation.shape[2] + 1)

                subkernel_y_end = min(kernels.shape[1], y + 1)
                subkernel_x_end = min(kernels.shape[2], x + 1)

                # Select indices of all elements in preactivation errors that were affected by examined image pixel
                y_slice_end = None if y - subkernel_y_end == -1 else y - subkernel_y_end
                y_slice = slice(y - subkernel_y_start, y_slice_end, -1)

                x_slice_end = None if x - subkernel_x_end == -1 else x - subkernel_x_end
                x_slice = slice(x - subkernel_x_start, x_slice_end, -1)

                # Error patch has dimensions images, y, x, output_channels
                error_patch = preactivation_error_gradients[:, y_slice, x_slice, :]

                # Select part of kernels that were convolved with image region currently computed over
                # subkernel has order kernels count, y, x, input channels
                subkernel = kernels[:, subkernel_y_start:subkernel_y_end, subkernel_x_start:subkernel_x_end, :]

                # error_patch has output channel dimension in last axis, but subkernel has it in first axis, since
                # error/output channels are equal/result of number of kernels. Roll kernels count
                # axis to end so that error patch and subkernel have output channels dimension in the same location
                # subkernel will now have order: y, x, input_channels, output_channels/kernels count
                subkernel = np.rollaxis(subkernel, 0, 4)

                # Now roll input channel dimension to end
                # subkernel will now have order: y, x, output_channels, input channels
                subkernel = np.rollaxis(subkernel, 2, 4)

                # error patch is a 4D tensor with dimensions image, y, x, output channels.
                # subkernel is a 4D tensor with dimensions y, x, output_channels, input_channels.
                # To make them compatible for element wise multiplication, we need error patch to include
                # input channel dimension and subkernel to include images dimension.

                # Repeat subkernel for each image
                # subkernel now has dimensions images, y, x, output_channels, input_channels
                subkernel = np.array([subkernel] * images_count)

                # Now add input channels dimension to error patch
                # error patch now has dimensions images, y, x, output_channels, input_channels
                # Numpy will broadcast error patch along last dimension to match size of subkernel
                error_patch = error_patch.reshape(error_patch.shape + (1, ))

                pixel_gradient = np.sum(error_patch * subkernel, axis=(1, 2, 3))

                # Dimensions are images, y, x, input channels
                pixel_index = (slice(None), y, x, slice(None))

                image_gradients[pixel_index] = pixel_gradient

        return image_gradients

class Softmax:
    """
    Softmax layer
    """

    def __init__(self):

        self.input_shape = None
        self.output_shape = None

        self.last_output = None

    def build(self, input_shape):

        if len(input_shape) != 2:

            message = "Input shape must be 2D, but {}D input with shape {} was given".format(
                len(input_shape), input_shape)

            raise ValueError(message)

        if input_shape[1] == 1:

            message = "Last input shape dimension must be larger than 1, but provided input shape is: {}"\
                .format(input_shape)

            raise ValueError(message)

        self.input_shape = input_shape
        self.output_shape = input_shape

    def forward(self, x):

        if x.shape[1:] != self.input_shape[1:]:

            message = "Input shape incorrect, expected {} but {} was given".format(
                self.input_shape, x.shape)

            raise ValueError(message)

        # Clip values to sensible range for numerical stability
        clipped = np.clip(x, -50, 50)

        exponentials = np.exp(clipped)
        exponential_sums = np.sum(exponentials, axis=1).reshape((x.shape[0], 1))

        return exponentials / exponential_sums

    def train_forward(self, x):

        self.last_output = self.forward(x)
        return self.last_output

    def get_output_layer_error_gradients(self, labels):
        """
        Given correct labels for last predictions, compute error gradients
        :param labels: batch of correct labels
        :return: batch of error gradients
        """

        return self.last_output - labels

<commit_msg>Refactored _update_kernels<commit_after>"""
Module with layers
"""

import itertools

import numpy as np

class Layer:

    def __init__(self):

        self.input_shape = None
        self.output_shape = None

        self.last_input = None
        self.last_output = None

    def build(self, input_shape):

        raise NotImplementedError()

    def forward(self, x):

        raise NotImplementedError()

    def train_forward(self, x):

        raise NotImplementedError()

    def train_backward(self, gradients, learning_rate=None):

        raise NotImplementedError()

class Input(Layer):
    """
    Input layer
    """

    def __init__(self, sample_shape):
        """
        Constructor
        :param sample_shape: 3 elements tuple (height, width, channels)
        """

        super().__init__()

        self.input_shape = (None,) + tuple(sample_shape)

    def build(self, input_shape=None):

        # Input layer's output shape is same as its input shape
        self.output_shape = self.input_shape

    def forward(self, x):

        if x.shape[1:] != self.input_shape[1:]:

            raise ValueError("{} shape expected, but input has shape {}".format(
                self.input_shape, x.shape))

        return x

    def train_forward(self, x):

        return self.forward(x)

    def train_backward(self, gradients, learning_rate=None):
        """
        For Input layer train backward is a no-op
        :return: None
        """

        return None

class Flatten(Layer):
    """
    Layer for flattening input. Outer dimension (batch size) is not affected
    """

    def __init__(self):

        super().__init__()

    def build(self, input_shape):

        self.input_shape = tuple(input_shape)

        squeezed_sample_shape = []

        for dimension in input_shape[1:]:

            if dimension != 1:

                squeezed_sample_shape.append(dimension)

        self.output_shape = (input_shape[0],) + tuple(squeezed_sample_shape)

    def forward(self, x):

        shape = (x.shape[0],) + self.output_shape[1:]
        return x.reshape(shape)

    def train_forward(self, x):

        self.last_input = x
        self.last_output = self.forward(x)

        return self.last_output

    def train_backward(self, gradients, learning_rate=None):

        return gradients.reshape(self.last_input.shape)

class Convolution2D(Layer):

    def __init__(self, nb_filter, nb_row, nb_col):
        """
        2D convolutional layer. Applies ReLU activation.
        :param nb_filter: number of filters
        :param nb_row: width of each filter
        :param nb_col: height of each filter
        """

        super().__init__()

        self.nb_filter = nb_filter
        self.nb_row = nb_row
        self.nb_col = nb_col

        self.kernels = None
        self.biases = None

        self.last_preactivation = None

    def build(self, input_shape):

        if len(input_shape) != 4:

            raise ValueError("Input shape {} isn't 4-dimensional".format(input_shape))

        self.input_shape = input_shape

        # Dimensions are [images, rows, columns, filters]
        self.output_shape = (None, input_shape[1] - self.nb_row + 1, input_shape[2] - self.nb_col + 1, self.nb_filter)

        input_channels = input_shape[-1]
        kernels_shape = (self.nb_filter, self.nb_row, self.nb_col, input_channels)

        scale = np.sqrt(2 / (self.nb_row * self.nb_col * input_channels))
        self.kernels = np.random.uniform(low=-scale, high=scale, size=kernels_shape)

        self.biases = np.zeros((self.nb_filter,), dtype=np.float32)

    def forward(self, input):

        preactivation = self._get_preactivation(input)

        # Apply ReLU activation
        return self._relu(preactivation)

    def _get_preactivation(self, input):

        preactivation = np.zeros(shape=(input.shape[0],) + self.output_shape[1:])

        # Kernels with kernels count dimension rolled to last dimension
        rolled_kernels = np.rollaxis(self.kernels, 0, 4)

        # For each image
        for sample_index, sample in enumerate(input):

            for row_index in range(0, input.shape[1] - self.nb_row + 1):

                for col_index in range(0, input.shape[2] - self.nb_col + 1):

                    # Select patch over which kernels will be applied
                    input_patch = sample[row_index: row_index + self.nb_row, col_index: col_index + self.nb_col, :]

                    # input patch is 3D, but we will convolve it with a series of 3D kernels, thus a 4D entity.
                    # Thus reshape input patch to have a 4th dimension. Numpy will then broadcast input patch
                    # along that dimension to match number of kernels
                    reshaped_input_patch = input_patch.reshape(input_patch.shape + (1,))

                    # Preactivation pixel value will a 1D array that holds results of convolutions over
                    # a selected patch with all kernels, one value for each kernel
                    preactivation_pixel_values = \
                        np.sum(reshaped_input_patch * rolled_kernels, axis=(0, 1, 2)) + self.biases

                    preactivation[sample_index, row_index, col_index, :] = preactivation_pixel_values

        return preactivation

    def _relu(self, x):

        return x * (x > 0)

    def _relu_derivative(self, x):

        return 1 * (x > 0)

    def train_forward(self, input):

        self.last_input = input

        self.last_preactivation = self._get_preactivation(input)
        self.last_output = self._relu(self.last_preactivation)

        return self.last_output

    def train_backward(self, gradients, learning_rate):

        preactivation_error_gradients = gradients * self._relu_derivative(self.last_preactivation)

        # Copy old kernels, we will use original values when computing image gradients
        old_kernels = self.kernels.copy()

        self._update_kernels(preactivation_error_gradients, learning_rate)
        self._update_biases(preactivation_error_gradients, learning_rate)

        return self._get_image_gradients(preactivation_error_gradients, old_kernels)

    def _update_biases(self, preactivation_error_gradients, learning_rate):

        for kernel_index in range(len(self.kernels)):

            # Sum contributions for each image
            bias_error_gradients_sums = np.sum(preactivation_error_gradients[:, :, :, kernel_index], axis=(1, 2))

            # And take mean across contributions to different images
            mean_bias_error_gradient = np.mean(bias_error_gradients_sums)

            self.biases[kernel_index] -= learning_rate * mean_bias_error_gradient

    def _update_kernels(self, preactivation_error_gradients, learning_rate):

        for kernel_index in range(len(self.kernels)):

            for y in range(self.nb_row):

                input_row_start = y
                input_row_end = self.input_shape[1] - self.nb_row + y + 1

                for x in range(self.nb_col):

                    input_column_start = x
                    input_column_end = self.input_shape[2] - self.nb_col + x + 1

                    # Input patches have dimensions images, y, x, input channels
                    inputs_patches = self.last_input[
                                     :, input_row_start:input_row_end, input_column_start:input_column_end, :]

                    # error gradients have dimensions images, y, x, output_channels
                    error_gradients = preactivation_error_gradients[:, :, :, kernel_index]

                    # Reshape error_gradients so they have input channel dimension
                    # Numpy broadcasting will then take care of matching it with number of channels of inputs patches
                    reshaped_kernel_preactivation_error_gradients = error_gradients.reshape(error_gradients.shape + (1,))

                    total_kernel_weight_gradient = np.sum(
                        reshaped_kernel_preactivation_error_gradients * inputs_patches, axis=(0, 1, 2))

                    weight_index = (kernel_index, y, x)

                    self.kernels[weight_index] -= learning_rate * total_kernel_weight_gradient / len(inputs_patches)

    def _get_image_gradients(self, preactivation_error_gradients, kernels):

        # Tensor for storing error gradients on input image
        image_gradients = np.zeros_like(self.last_input, dtype=np.float32)

        images_count = self.last_input.shape[0]
        rows_count = self.last_input.shape[1]
        columns_count = self.last_input.shape[2]

        # For each column
        for y in range(rows_count):

            # For each row
            for x in range(columns_count):

                # Select all kernel weights this pixel was convolved with
                # kernels ordering is: (self.nb_filter, self.nb_row, self.nb_col, input_channels)

                # Get start and end indices for part of kernel tensor that pixels I[y,x] was convolved with
                subkernel_y_start = max(0, y - self.last_preactivation.shape[1] + 1)
                subkernel_x_start = max(0, x - self.last_preactivation.shape[2] + 1)

                subkernel_y_end = min(kernels.shape[1], y + 1)
                subkernel_x_end = min(kernels.shape[2], x + 1)

                # Select indices of all elements in preactivation errors that were affected by examined image pixel
                y_slice_end = None if y - subkernel_y_end == -1 else y - subkernel_y_end
                y_slice = slice(y - subkernel_y_start, y_slice_end, -1)

                x_slice_end = None if x - subkernel_x_end == -1 else x - subkernel_x_end
                x_slice = slice(x - subkernel_x_start, x_slice_end, -1)

                # Error patch has dimensions images, y, x, output_channels
                error_patch = preactivation_error_gradients[:, y_slice, x_slice, :]

                # Select part of kernels that were convolved with image region currently computed over
                # subkernel has order kernels count, y, x, input channels
                subkernel = kernels[:, subkernel_y_start:subkernel_y_end, subkernel_x_start:subkernel_x_end, :]

                # error_patch has output channel dimension in last axis, but subkernel has it in first axis, since
                # error/output channels are equal/result of number of kernels. Roll kernels count
                # axis to end so that error patch and subkernel have output channels dimension in the same location
                # subkernel will now have order: y, x, input_channels, output_channels/kernels count
                subkernel = np.rollaxis(subkernel, 0, 4)

                # Now roll input channel dimension to end
                # subkernel will now have order: y, x, output_channels, input channels
                subkernel = np.rollaxis(subkernel, 2, 4)

                # error patch is a 4D tensor with dimensions image, y, x, output channels.
                # subkernel is a 4D tensor with dimensions y, x, output_channels, input_channels.
                # To make them compatible for element wise multiplication, we need error patch to include
                # input channel dimension and subkernel to include images dimension.

                # Repeat subkernel for each image
                # subkernel now has dimensions images, y, x, output_channels, input_channels
                subkernel = np.array([subkernel] * images_count)

                # Now add input channels dimension to error patch
                # error patch now has dimensions images, y, x, output_channels, input_channels
                # Numpy will broadcast error patch along last dimension to match size of subkernel
                error_patch = error_patch.reshape(error_patch.shape + (1, ))

                pixel_gradient = np.sum(error_patch * subkernel, axis=(1, 2, 3))

                # Dimensions are images, y, x, input channels
                pixel_index = (slice(None), y, x, slice(None))

                image_gradients[pixel_index] = pixel_gradient

        return image_gradients

class Softmax:
    """
    Softmax layer
    """

    def __init__(self):

        self.input_shape = None
        self.output_shape = None

        self.last_output = None

    def build(self, input_shape):

        if len(input_shape) != 2:

            message = "Input shape must be 2D, but {}D input with shape {} was given".format(
                len(input_shape), input_shape)

            raise ValueError(message)

        if input_shape[1] == 1:

            message = "Last input shape dimension must be larger than 1, but provided input shape is: {}"\
                .format(input_shape)

            raise ValueError(message)

        self.input_shape = input_shape
        self.output_shape = input_shape

    def forward(self, x):

        if x.shape[1:] != self.input_shape[1:]:

            message = "Input shape incorrect, expected {} but {} was given".format(
                self.input_shape, x.shape)

            raise ValueError(message)

        # Clip values to sensible range for numerical stability
        clipped = np.clip(x, -50, 50)

        exponentials = np.exp(clipped)
        exponential_sums = np.sum(exponentials, axis=1).reshape((x.shape[0], 1))

        return exponentials / exponential_sums

    def train_forward(self, x):

        self.last_output = self.forward(x)
        return self.last_output

    def get_output_layer_error_gradients(self, labels):
        """
        Given correct labels for last predictions, compute error gradients
        :param labels: batch of correct labels
        :return: batch of error gradients
        """

        return self.last_output - labels

<|endoftext|>