<commit_before># -*- coding: utf8 -*-
# This file is part of PyBossa.
#
# Copyright (C) 2014 SF Isle of Man Limited
#
# PyBossa is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBossa is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa.  If not, see <http://www.gnu.org/licenses/>.

from sqlalchemy.exc import IntegrityError

from pybossa.model.blogpost import Blogpost
from pybossa.exc import WrongObjectError, DBIntegrityError

class BlogRepository(object):

    def __init__(self, db):
        self.db = db

    def get(self, id):
        return self.db.session.query(Blogpost).get(id)

    def get_by(self, **attributes):
        return self.db.session.query(Blogpost).filter_by(**attributes).first()

    def filter_by(self, limit=None, offset=0, **filters):
        query = self.db.session.query(Blogpost).filter_by(**filters)
        query = query.order_by(Blogpost.id).limit(limit).offset(offset)
        return query.all()

    def save(self, blogpost):
        self._validate_can_be('saved', blogpost)
        try:
            self.db.session.add(blogpost)
            self.db.session.commit()
        except IntegrityError as e:
            self.db.session.rollback()
            raise DBIntegrityError(e)

    def update(self, blogpost):
        self._validate_can_be('updated', blogpost)
        try:
            self.db.session.merge(blogpost)
            self.db.session.commit()
        except IntegrityError as e:
            self.db.session.rollback()
            raise DBIntegrityError(e)

    def delete(self, blogpost):
        self._validate_can_be('deleted', blogpost)
        blog = self.db.session.query(Blogpost).filter(Blogpost.id==blogpost.id).first()
        self.db.session.delete(blog)
        self.db.session.commit()

    def _validate_can_be(self, action, blogpost):
        if not isinstance(blogpost, Blogpost):
            name = blogpost.__class__.__name__
            msg = '%s cannot be %s by %s' % (name, action, self.__class__.__name__)
            raise WrongObjectError(msg)
<commit_msg>Use keyset pagination.<commit_after># -*- coding: utf8 -*-
# This file is part of PyBossa.
#
# Copyright (C) 2014 SF Isle of Man Limited
#
# PyBossa is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBossa is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa.  If not, see <http://www.gnu.org/licenses/>.

from sqlalchemy.exc import IntegrityError

from pybossa.model.blogpost import Blogpost
from pybossa.exc import WrongObjectError, DBIntegrityError

class BlogRepository(object):

    def __init__(self, db):
        self.db = db

    def get(self, id):
        return self.db.session.query(Blogpost).get(id)

    def get_by(self, **attributes):
        return self.db.session.query(Blogpost).filter_by(**attributes).first()

    def filter_by(self, limit=None, offset=0, yielded=False, last_id=None,
                  **filters):
        query = self.db.session.query(Blogpost).filter_by(**filters)
        if last_id:
            query = query.filter(Blogpost.id > last_id)
            query = query.order_by(Blogpost.id).limit(limit)
        else:
            query = query.order_by(Blogpost.id).limit(limit).offset(offset)
        if yielded:
            limit = limit or 1
            return query.yield_per(limit)
        return query.all()

    def save(self, blogpost):
        self._validate_can_be('saved', blogpost)
        try:
            self.db.session.add(blogpost)
            self.db.session.commit()
        except IntegrityError as e:
            self.db.session.rollback()
            raise DBIntegrityError(e)

    def update(self, blogpost):
        self._validate_can_be('updated', blogpost)
        try:
            self.db.session.merge(blogpost)
            self.db.session.commit()
        except IntegrityError as e:
            self.db.session.rollback()
            raise DBIntegrityError(e)

    def delete(self, blogpost):
        self._validate_can_be('deleted', blogpost)
        blog = self.db.session.query(Blogpost).filter(Blogpost.id==blogpost.id).first()
        self.db.session.delete(blog)
        self.db.session.commit()

    def _validate_can_be(self, action, blogpost):
        if not isinstance(blogpost, Blogpost):
            name = blogpost.__class__.__name__
            msg = '%s cannot be %s by %s' % (name, action, self.__class__.__name__)
            raise WrongObjectError(msg)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import logging
from os import mkdir
from os.path import join as path_join
from urllib.parse import quote_plus, urljoin
from datetime import datetime

import requests
from bs4 import BeautifulSoup

logging.basicConfig(
    format=(
        '%(asctime)s\t%(levelname)s\t'
        #'%(processName)s\t%(threadName)s\t'
        '%(name)s\t%(funcName)s:%(lineno)d\t'
        '%(message)s'
    ),
    level=logging.DEBUG
)

def _make_fake_browser():

    fake_browser = requests.Session()
    fake_browser.headers = {
        'user-agent': (
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) '
            'AppleWebKit/537.36 (KHTML, like Gecko) '
            'Chrome/54.0.2840.98 Safari/537.36'
        ),
        'accept': (
            'text/html,application/xhtml+xml,application/xml;q=0.9,'
            'image/webp,*/*;q=0.8'
        ),
        'accept-encoding': 'gzip, deflate, sdch, br',
        'accept-language': 'en-US,en;q=0.8,zh-TW;q=0.6,zh;q=0.4',
        'cookie': 'over18=1',
    }

    return fake_browser

_SHARED_FAKE_BROWSER = _make_fake_browser()
_CACHE_DIR_PATH = 'cache/'
_URL_SET_SKIPPING_CACHE = {'https://www.ptt.cc/bbs/Gossiping/index.html'}

def read_or_request(url):

    # should generate valid fname for most of the systems
    fname = quote_plus(url)
    path = path_join(_CACHE_DIR_PATH, fname)

    # try cache

    if url in _URL_SET_SKIPPING_CACHE:

        logging.info('Skip cache for {}'.format(url))

    else:

        try:
            with open(path) as f:
                logging.info('Hit {}'.format(url))
                return f.read()
        except OSError:
            logging.info('Missed {}'.format(url))

    # request

    resp = _SHARED_FAKE_BROWSER.get(url)
    text = resp.text

    while 1:

        try:
            with open(path, 'w') as f:
                f.write(text)
        except FileNotFoundError:
            mkdir(_CACHE_DIR_PATH)
            continue
        else:
            logging.info('Wrote {}'.format(url))

        # only loop once normally
        break

    return text

_ROOT = 'https://www.ptt.cc'

def parse_index_page(text):

    soup = BeautifulSoup(text, 'html.parser')

    # paging

    paging_a_tags = soup.select('.action-bar .btn-group-paging a')

    prev_a_tag = paging_a_tags[1]
    next_a_tag = paging_a_tags[2]

    if '上頁' not in prev_a_tag.string:
        raise RuntimeError('the prev button looks not right')
    if '下頁' not in next_a_tag.string:
        raise RuntimeError('the next button looks not right')

    prev_href = prev_a_tag.get('href', '')
    prev_url = prev_href and urljoin(_ROOT, prev_href)
    next_href = next_a_tag.get('href', '')
    next_url = next_href and urljoin(_ROOT, next_href)

    # entries

    entry_ds = []
    ent_tags = soup.select('.r-list-container .r-ent')
    for ent_tag in ent_tags:

        # push_score_sum

        # cases
        #
        # 1.
12

        # 2.

        # 3.
X1

        #
        push_score_sum = -65535
        span_tag = ent_tag.find(class_='nrec').find('span')

        # if case 2
        if not span_tag:

            push_score_sum = 0

        else:

            span_text = span_tag.string

            while 1:

                # if case 1
                try:
                    push_score_sum = int(span_text)
                except ValueError:
                    pass
                else:
                    break

                # if case 3
                try:
                    push_score_sum = int(span_text[1:])
                except ValueError:
                    pass
                else:
                    # TODO: is it correct?
                    push_score_sum = -(push_score_sum+10)
                    break

                # loop once normally
                break

        # title & article_url

        # cases
        #
        # 1.

        #
        #            Re: [問卦] 有沒有高雄其實比台北進步的八卦??
        #
        #

        # 2.
 (本文已被刪除) [gundam0613]

        #

        title_tag = ent_tag.find(class_='title')
        title_a_tag = title_tag.a
        if title_a_tag:
            # case 1
            title = title_a_tag.string
            article_url = urljoin(_ROOT, title_a_tag.get('href', ''))
        else:
            # case 2: deleted article
            title = title_tag.string.strip()
            article_url = ''

        # others

        ent_meta_tag = ent_tag.find(class_='meta')

        # mmdd won't contain year!
        raw_mmdd = ent_meta_tag.find(class_='date').string
        author_id = ent_meta_tag.find(class_='author').string
        # deleted article
        if not article_url and author_id == '-':
            author_id = ''

        # append

        entry_ds.append({
            'push_score_sum' : push_score_sum,
            'title'          : title,
            'article_url'    : article_url,
            'raw_mmdd'       : raw_mmdd,
            'author_id'      : author_id
        })

    return {
        'prev_url': prev_url,
        'next_url': next_url,
        'entry_ds': entry_ds
    }

_EXPECTED_ARTICLE_TAG_KEYS = ('作者', '看板', '標題', '時間')
_PUSH_TAG_STRIPPED_TEXT_SCORE_MAP = {
    '噓': -1,
    '→': 0,
    '推': 1
}

def parse_article_page(text):

    soup = BeautifulSoup(text, 'html.parser')
    main_content_tag = soup.find(id='main-content')

    # meta
    #
    #

    #     作者
    #     a77774444 (我愛ˋ台灣)
    #

    #
    for tag, key in zip(
        main_content_tag.select('.article-meta-tag'),
        _EXPECTED_ARTICLE_TAG_KEYS
    ):
        if tag.string != key:
            raise RuntimeError('the article tags may be changed')

    author_line, board_name, title, timestamp_text = (
        tag.string
        for tag in main_content_tag.select('.article-meta-value')
    )

    author_id, _, author_line_tail = author_line.partition(' (')
    author_nick = author_line_tail[:-1]

    # Tue Nov 29 05:05:03 2016
    # Fri Jul  1 21:12:25 2005
    authored_dt = datetime.strptime(timestamp_text, '%a %b %d %H:%M:%S %Y')

    # body
    # TODO: record the IPs?

    recording = False
    body_lines = []
    for i, line in enumerate(main_content_tag.stripped_strings):

        if i == 6:
            if line != '時間':
                raise RuntimeError('the article structure may be changed')
        elif i == 8:
            recording = True
        elif line.startswith('※ 發信站: 批踢踢實業坊'):
            recording = False
            break

        if recording:
            body_lines.append(line)

    else:

        raise RuntimeError('the ending text of article may be change/')

    # ... 以後台北人都要去高雄當台勞了。\n\n--'
    body = '\n'.join(body_lines).rstrip('\n-')

    # pushes

    push_ds = [
        {
            'score': _PUSH_TAG_STRIPPED_TEXT_SCORE_MAP.get(
                push_tag.find(class_='push-tag').string.strip(), -255
            ),
            'pusher_id': push_tag.find(class_='push-userid').string,
            'text': push_tag.find(class_='push-content').string,
            # won't contain year!
            'raw_mmddhhmm': (
                push_tag.find(class_='push-ipdatetime').string.strip()
            )
        }
        for push_tag in main_content_tag.select('.push')
    ]

    return {
        'board_name' : board_name,
        'author_id'  : author_id,
        'author_nick': author_nick,
        'title'      : title,
        'authored_dt': authored_dt,
        'body'       : body,
        'push_ds'    : push_ds
    }

if __name__ == '__main__':

    from pprint import pprint

    # test the index page
    pprint(parse_index_page(read_or_request(
        'https://www.ptt.cc/bbs/Gossiping/index20177.html'
    )))

    # test the article page #1
    pprint(parse_article_page(read_or_request(
        'https://www.ptt.cc/bbs/Gossiping/M.1480367106.A.A55.html'
    )))

    # test the article page #2
    pprint(parse_article_page(read_or_request(
        'https://www.ptt.cc/bbs/Gossiping/M.1480380251.A.9A4.html'
    )))
<commit_msg>Finish the crawl function<commit_after>#!/usr/bin/env python

import logging
from os import mkdir
from os.path import join as path_join
from urllib.parse import quote_plus, urljoin
from datetime import datetime
from time import sleep
from random import randint

import requests
from bs4 import BeautifulSoup

logging.basicConfig(
    format=(
        '%(asctime)s\t%(levelname)s\t'
        #'%(processName)s\t%(threadName)s\t'
        '%(name)s\t%(funcName)s:%(lineno)d\t'
        '%(message)s'
    ),
    level=logging.DEBUG
)

def _make_fake_browser():

    fake_browser = requests.Session()
    fake_browser.headers = {
        'user-agent': (
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) '
            'AppleWebKit/537.36 (KHTML, like Gecko) '
            'Chrome/54.0.2840.98 Safari/537.36'
        ),
        'accept': (
            'text/html,application/xhtml+xml,application/xml;q=0.9,'
            'image/webp,*/*;q=0.8'
        ),
        'accept-encoding': 'gzip, deflate, sdch, br',
        'accept-language': 'en-US,en;q=0.8,zh-TW;q=0.6,zh;q=0.4',
        'cookie': 'over18=1',
    }

    return fake_browser

_SHARED_FAKE_BROWSER = _make_fake_browser()
_CACHE_DIR_PATH = 'cache/'
_URL_SET_SKIPPING_CACHE = {'https://www.ptt.cc/bbs/Gossiping/index.html'}

def read_or_request(url):

    # should generate valid fname for most of the systems
    fname = quote_plus(url)
    path = path_join(_CACHE_DIR_PATH, fname)

    # try cache

    if url in _URL_SET_SKIPPING_CACHE:

        logging.info('Skip cache for {}'.format(url))

    else:

        try:
            with open(path) as f:
                logging.info('Hit {}'.format(url))
                return f.read()
        except OSError:
            logging.info('Missed {}'.format(url))

    # request

    resp = _SHARED_FAKE_BROWSER.get(url)
    text = resp.text

    while 1:

        try:
            with open(path, 'w') as f:
                f.write(text)
        except FileNotFoundError:
            mkdir(_CACHE_DIR_PATH)
            continue
        else:
            logging.info('Wrote {}'.format(url))

        # only loop once normally
        break

    return text

_ROOT = 'https://www.ptt.cc'

def parse_index_page(text):

    soup = BeautifulSoup(text, 'html.parser')

    # paging

    paging_a_tags = soup.select('.action-bar .btn-group-paging a')

    prev_a_tag = paging_a_tags[1]
    next_a_tag = paging_a_tags[2]

    if '上頁' not in prev_a_tag.string:
        raise RuntimeError('the prev button looks not right')
    if '下頁' not in next_a_tag.string:
        raise RuntimeError('the next button looks not right')

    prev_href = prev_a_tag.get('href', '')
    prev_url = prev_href and urljoin(_ROOT, prev_href)
    next_href = next_a_tag.get('href', '')
    next_url = next_href and urljoin(_ROOT, next_href)

    # entries

    entry_ds = []
    ent_tags = soup.select('.r-list-container .r-ent')
    for ent_tag in ent_tags:

        # push_score_sum

        # cases
        #
        # 1.
12

        # 2.

        # 3.
X1

        #
        push_score_sum = -65535
        span_tag = ent_tag.find(class_='nrec').find('span')

        # if case 2
        if not span_tag:

            push_score_sum = 0

        else:

            span_text = span_tag.string

            while 1:

                # if case 1
                try:
                    push_score_sum = int(span_text)
                except ValueError:
                    pass
                else:
                    break

                # if case 3
                try:
                    push_score_sum = int(span_text[1:])
                except ValueError:
                    pass
                else:
                    # TODO: is it correct?
                    push_score_sum = -(push_score_sum+10)
                    break

                # loop once normally
                break

        # title & article_url

        # cases
        #
        # 1.

        #
        #            Re: [問卦] 有沒有高雄其實比台北進步的八卦??
        #
        #

        # 2.
 (本文已被刪除) [gundam0613]

        #

        title_tag = ent_tag.find(class_='title')
        title_a_tag = title_tag.a
        if title_a_tag:
            # case 1
            title = title_a_tag.string
            article_url = urljoin(_ROOT, title_a_tag.get('href', ''))
        else:
            # case 2: deleted article
            title = title_tag.string.strip()
            article_url = ''

        # others

        ent_meta_tag = ent_tag.find(class_='meta')

        # mmdd won't contain year!
        raw_mmdd = ent_meta_tag.find(class_='date').string
        author_id = ent_meta_tag.find(class_='author').string
        # deleted article
        if not article_url and author_id == '-':
            author_id = ''

        # append

        entry_ds.append({
            'push_score_sum' : push_score_sum,
            'title'          : title,
            'article_url'    : article_url,
            'raw_mmdd'       : raw_mmdd,
            'author_id'      : author_id
        })

    return {
        'prev_url': prev_url,
        'next_url': next_url,
        'entry_ds': entry_ds
    }

_EXPECTED_ARTICLE_TAG_KEYS = ('作者', '看板', '標題', '時間')
_PUSH_TAG_STRIPPED_TEXT_SCORE_MAP = {
    '噓': -1,
    '→': 0,
    '推': 1
}

def parse_article_page(text):

    soup = BeautifulSoup(text, 'html.parser')
    main_content_tag = soup.find(id='main-content')

    # meta
    #
    #

    #     作者
    #     a77774444 (我愛ˋ台灣)
    #

    #
    for tag, key in zip(
        main_content_tag.select('.article-meta-tag'),
        _EXPECTED_ARTICLE_TAG_KEYS
    ):
        if tag.string != key:
            raise RuntimeError('the article tags may be changed')

    author_line, board_name, title, timestamp_text = (
        tag.string
        for tag in main_content_tag.select('.article-meta-value')
    )

    author_id, _, author_line_tail = author_line.partition(' (')
    author_nick = author_line_tail[:-1]

    # Tue Nov 29 05:05:03 2016
    # Fri Jul  1 21:12:25 2005
    authored_dt = datetime.strptime(timestamp_text, '%a %b %d %H:%M:%S %Y')

    # body
    # TODO: record the IPs?

    recording = False
    body_lines = []
    for i, line in enumerate(main_content_tag.stripped_strings):

        if i == 6:
            if line != '時間':
                raise RuntimeError('the article structure may be changed')
        elif i == 8:
            recording = True
        elif line.startswith('※ 發信站: 批踢踢實業坊'):
            recording = False
            break

        if recording:
            body_lines.append(line)

    else:

        raise RuntimeError('the ending text of article may be change/')

    # ... 以後台北人都要去高雄當台勞了。\n\n--'
    body = '\n'.join(body_lines).rstrip('\n-')

    # pushes

    push_ds = [
        {
            'score': _PUSH_TAG_STRIPPED_TEXT_SCORE_MAP.get(
                push_tag.find(class_='push-tag').string.strip(), -255
            ),
            'pusher_id': push_tag.find(class_='push-userid').string,
            'text': push_tag.find(class_='push-content').string,
            # won't contain year!
            'raw_mmddhhmm': (
                push_tag.find(class_='push-ipdatetime').string.strip()
            )
        }
        for push_tag in main_content_tag.select('.push')
    ]

    return {
        'board_name' : board_name,
        'author_id'  : author_id,
        'author_nick': author_nick,
        'title'      : title,
        'authored_dt': authored_dt,
        'body'       : body,
        'push_ds'    : push_ds
    }

def crawl(index_url):

    count = 0
    prev_url = index_url

    while 1:

        # crawl and parse the index page

        logging.info('Crawl the index page {} ...'.format(prev_url))
        try:
            text = read_or_request(prev_url)
        except OSError:
            # try again
            logging.info('Try again ...')
            text = read_or_request(prev_url)

        logging.info('Parse the index page {} ...'.format(prev_url))
        parsed_index_d = parse_index_page(text)

        prev_url = parsed_index_d['prev_url']

        # crawl the article_url

        for entry_d in parsed_index_d['entry_ds']:

            article_url = entry_d['article_url']
            # if deleted article
            if not article_url:
                continue

            logging.info('Crawl the article page {} ...'.format(article_url))
            try:
                read_or_request(article_url)
            except OSError:
                # try again
                logging.info('Try again ...')
                try:
                    read_or_request(article_url)
                except OSError:
                    # skip if still fail
                    logging.info('Skip')
                    continue

            count += 1

            logging.info('Sleep')
            sleep(randint(0, 100)*0.01)

        logging.info('Got {:,} articles so far'.format(count))

if __name__ == '__main__':

    from pprint import pprint

    crawl('https://www.ptt.cc/bbs/Gossiping/index.html')

    import sys
    sys.exit()

    # test the index page
    pprint(parse_index_page(read_or_request(
        'https://www.ptt.cc/bbs/Gossiping/index20177.html'
    )))

    # test the article page #1
    pprint(parse_article_page(read_or_request(
        'https://www.ptt.cc/bbs/Gossiping/M.1480367106.A.A55.html'
    )))

    # test the article page #2
    pprint(parse_article_page(read_or_request(
        'https://www.ptt.cc/bbs/Gossiping/M.1480380251.A.9A4.html'
    )))
<|endoftext|>


<commit_before># Copyright (c) 2009-2018 - Simon Conseil
# Copyright (c) 2015 - François D.
# Copyright (c) 2018 - Edwin Steele

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

# Additional copyright notice:
#
# Several lines of code concerning extraction of GPS data from EXIF tags where
# taken from a GitHub Gist by Eran Sandler at
#
#   https://gist.github.com/erans/983821
#
# and partially modified. The code in question is licensed under MIT license.

import logging
import os
import pilkit.processors
import sys
import warnings
import fractions

from copy import deepcopy
from datetime import datetime
from PIL.ExifTags import TAGS, GPSTAGS
from PIL import Image as PILImage, ImageOps, ImageFile, IptcImagePlugin
from pilkit.processors import Transpose
from pilkit.utils import save_image

from . import signals, utils
from .settings import get_thumb, Status

# Force loading of truncated files
ImageFile.LOAD_TRUNCATED_IMAGES = True

def _has_exif_tags(img):
    return hasattr(img, 'info') and 'exif' in img.info

def _read_image(file_path):
    with warnings.catch_warnings(record=True) as caught_warnings:
        im = PILImage.open(file_path)

    for warning in caught_warnings:
        if warning.category == PILImage.DecompressionBombWarning:
            logger = logging.getLogger(__name__)
            logger.info('PILImage reported a possible DecompressionBomb'
                        ' for file {}'.format(file_path))
        else:
            warnings.showwarning(warning.message, warning.category,
                                 warning.filename, warning.lineno)
    return im

def generate_image(source, outname, settings, options=None):
    """Image processor, rotate and resize the image.

    :param source: path to an image
    :param outname: output filename
    :param settings: settings dict
    :param options: dict with PIL options (quality, optimize, progressive)

    """

    logger = logging.getLogger(__name__)

    if settings['use_orig'] or source.endswith('.gif'):
        utils.copy(source, outname, symlink=settings['orig_link'])
        return

    img = _read_image(source)
    original_format = img.format

    if settings['copy_exif_data'] and settings['autorotate_images']:
        logger.warning("The 'autorotate_images' and 'copy_exif_data' settings "
                       "are not compatible because Sigal can't save the "
                       "modified Orientation tag.")

    # Preserve EXIF data
    if settings['copy_exif_data'] and _has_exif_tags(img):
        if options is not None:
            options = deepcopy(options)
        else:
            options = {}
        options['exif'] = img.info['exif']

    # Rotate the img, and catch IOError when PIL fails to read EXIF
    if settings['autorotate_images']:
        try:
            img = Transpose().process(img)
        except (IOError, IndexError):
            pass

    # Resize the image
    if settings['img_processor']:
        try:
            logger.debug('Processor: %s', settings['img_processor'])
            processor_cls = getattr(pilkit.processors,
                                    settings['img_processor'])
        except AttributeError:
            logger.error('Wrong processor name: %s', settings['img_processor'])
            sys.exit()

        width, height = settings['img_size']

        if img.size[0] < img.size[1]:
            # swap target size if image is in portrait mode
            height, width = width, height

        processor = processor_cls(width, height, upscale=False)
        img = processor.process(img)

    # signal.send() does not work here as plugins can modify the image, so we
    # iterate other the receivers to call them with the image.
    for receiver in signals.img_resized.receivers_for(img):
        img = receiver(img, settings=settings)

    outformat = img.format or original_format or 'JPEG'
    logger.debug('Save resized image to %s (%s)', outname, outformat)
    save_image(img, outname, outformat, options=options, autoconvert=True)

def generate_thumbnail(source, outname, box, fit=True, options=None,
                       thumb_fit_centering=(0.5, 0.5)):
    """Create a thumbnail image."""

    logger = logging.getLogger(__name__)
    img = _read_image(source)
    original_format = img.format

    if fit:
        img = ImageOps.fit(img, box, PILImage.ANTIALIAS,
                           centering=thumb_fit_centering)
    else:
        img.thumbnail(box, PILImage.ANTIALIAS)

    outformat = img.format or original_format or 'JPEG'
    logger.debug('Save thumnail image: %s (%s)', outname, outformat)
    save_image(img, outname, outformat, options=options, autoconvert=True)

def process_image(filepath, outpath, settings):
    """Process one image: resize, create thumbnail."""

    logger = logging.getLogger(__name__)
    logger.info('Processing %s', filepath)
    filename = os.path.split(filepath)[1]
    outname = os.path.join(outpath, filename)
    ext = os.path.splitext(filename)[1]

    if ext in ('.jpg', '.jpeg', '.JPG', '.JPEG'):
        options = settings['jpg_options']
    elif ext == '.png':
        options = {'optimize': True}
    else:
        options = {}

    try:
        generate_image(filepath, outname, settings, options=options)

        if settings['make_thumbs']:
            thumb_name = os.path.join(outpath, get_thumb(settings, filename))
            generate_thumbnail(
                outname, thumb_name, settings['thumb_size'],
                fit=settings['thumb_fit'], options=options,
                thumb_fit_centering=settings["thumb_fit_centering"])
    except Exception as e:
        logger.info('Failed to process: %r', e)
        if logger.getEffectiveLevel() == logging.DEBUG:
            raise
        else:
            return Status.FAILURE

    return Status.SUCCESS

def get_size(file_path):
    """Return image size (width and height)."""
    try:
        im = _read_image(file_path)
    except (IOError, IndexError, TypeError, AttributeError) as e:
        logger = logging.getLogger(__name__)
        logger.error("Could not read size of %s due to %r", file_path, e)
    else:
        width, height = im.size
        return {
            'width': width,
            'height': height
        }

def get_exif_data(filename):
    """Return a dict with the raw EXIF data."""

    logger = logging.getLogger(__name__)

    img = _read_image(filename)

    try:
        exif = img._getexif() or {}
    except ZeroDivisionError:
        logger.warning('Failed to read EXIF data.')
        return None

    data = {TAGS.get(tag, tag): value for tag, value in exif.items()}

    if 'GPSInfo' in data:
        try:
            data['GPSInfo'] = {GPSTAGS.get(tag, tag): value
                               for tag, value in data['GPSInfo'].items()}
        except AttributeError:
            logger = logging.getLogger(__name__)
            logger.info('Failed to get GPS Info')
            del data['GPSInfo']
    return data

def get_iptc_data(filename):
    """Return a dict with the raw IPTC data."""

    iptc_data = {}

    try:
        img = _read_image(filename)
        raw_iptc = IptcImagePlugin.getiptcinfo(img)
        # IPTC fields are catalogued in:
        # https://www.iptc.org/std/photometadata/specification/IPTC-PhotoMetadata
        # 2:05 is the IPTC title property
        if raw_iptc and (2, 5) in raw_iptc:
            iptc_data["title"] = raw_iptc[(2, 5)].decode('utf-8')

        # 2:120 is the IPTC description property
        if raw_iptc and (2, 120) in raw_iptc:
            iptc_data["description"] = raw_iptc[(2, 120)].decode('utf-8')
    except SyntaxError:
        print("IPTC Error in %s\n"%filename)
        iptc_data = {}

    return iptc_data

def dms_to_degrees(v):
    """Convert degree/minute/second to decimal degrees."""

    d = float(v[0][0]) / float(v[0][1])
    m = float(v[1][0]) / float(v[1][1])
    s = float(v[2][0]) / float(v[2][1])
    return d + (m / 60.0) + (s / 3600.0)

def get_exif_tags(data, datetime_format='%c'):
    """Make a simplified version with common tags from raw EXIF data."""

    logger = logging.getLogger(__name__)
    simple = {}

    for tag in ('Model', 'Make', 'LensModel'):
        if tag in data:
            if isinstance(data[tag], tuple):
                simple[tag] = data[tag][0].strip()
            else:
                simple[tag] = data[tag].strip()

    if 'FNumber' in data:
        fnumber = data['FNumber']
        try:
            simple['fstop'] = float(fnumber[0]) / fnumber[1]
        except Exception:
            logger.debug('Skipped invalid FNumber: %r', fnumber, exc_info=True)

    if 'FocalLength' in data:
        focal = data['FocalLength']
        try:
            simple['focal'] = round(float(focal[0]) / focal[1])
        except Exception:
            logger.debug('Skipped invalid FocalLength: %r', focal,
                         exc_info=True)

    if 'ExposureTime' in data:
        exptime = data['ExposureTime']
        if isinstance(exptime, tuple):
            try:
                simple['exposure'] = str(fractions.Fraction(exptime[0],
                                                            exptime[1]))
            except ZeroDivisionError:
                logger.info('Invalid ExposureTime: %r', exptime)
        elif isinstance(exptime, int):
            simple['exposure'] = str(exptime)
        else:
            logger.info('Unknown format for ExposureTime: %r', exptime)

    if data.get('ISOSpeedRatings'):
        simple['iso'] = data['ISOSpeedRatings']

    if 'DateTimeOriginal' in data:
        # Remove null bytes at the end if necessary
        date = data['DateTimeOriginal'].rsplit('\x00')[0]

        try:
            simple['dateobj'] = datetime.strptime(date, '%Y:%m:%d %H:%M:%S')
            simple['datetime'] = simple['dateobj'].strftime(datetime_format)
        except (ValueError, TypeError) as e:
            logger.info('Could not parse DateTimeOriginal: %s', e)

    if 'GPSInfo' in data:
        info = data['GPSInfo']
        lat_info = info.get('GPSLatitude')
        lon_info = info.get('GPSLongitude')
        lat_ref_info = info.get('GPSLatitudeRef')
        lon_ref_info = info.get('GPSLongitudeRef')

        if lat_info and lon_info and lat_ref_info and lon_ref_info:
            try:
                lat = dms_to_degrees(lat_info)
                lon = dms_to_degrees(lon_info)
            except (ZeroDivisionError, ValueError, TypeError):
                logger.info('Failed to read GPS info')
            else:
                simple['gps'] = {
                    'lat': - lat if lat_ref_info != 'N' else lat,
                    'lon': - lon if lon_ref_info != 'E' else lon,
                }

    return simple
<commit_msg>Shortens try/sxcept block to keep code cleaner<commit_after># Copyright (c) 2009-2018 - Simon Conseil
# Copyright (c) 2015 - François D.
# Copyright (c) 2018 - Edwin Steele

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

# Additional copyright notice:
#
# Several lines of code concerning extraction of GPS data from EXIF tags where
# taken from a GitHub Gist by Eran Sandler at
#
#   https://gist.github.com/erans/983821
#
# and partially modified. The code in question is licensed under MIT license.

import logging
import os
import pilkit.processors
import sys
import warnings
import fractions

from copy import deepcopy
from datetime import datetime
from PIL.ExifTags import TAGS, GPSTAGS
from PIL import Image as PILImage, ImageOps, ImageFile, IptcImagePlugin
from pilkit.processors import Transpose
from pilkit.utils import save_image

from . import signals, utils
from .settings import get_thumb, Status

# Force loading of truncated files
ImageFile.LOAD_TRUNCATED_IMAGES = True

def _has_exif_tags(img):
    return hasattr(img, 'info') and 'exif' in img.info

def _read_image(file_path):
    with warnings.catch_warnings(record=True) as caught_warnings:
        im = PILImage.open(file_path)

    for warning in caught_warnings:
        if warning.category == PILImage.DecompressionBombWarning:
            logger = logging.getLogger(__name__)
            logger.info('PILImage reported a possible DecompressionBomb'
                        ' for file {}'.format(file_path))
        else:
            warnings.showwarning(warning.message, warning.category,
                                 warning.filename, warning.lineno)
    return im

def generate_image(source, outname, settings, options=None):
    """Image processor, rotate and resize the image.

    :param source: path to an image
    :param outname: output filename
    :param settings: settings dict
    :param options: dict with PIL options (quality, optimize, progressive)

    """

    logger = logging.getLogger(__name__)

    if settings['use_orig'] or source.endswith('.gif'):
        utils.copy(source, outname, symlink=settings['orig_link'])
        return

    img = _read_image(source)
    original_format = img.format

    if settings['copy_exif_data'] and settings['autorotate_images']:
        logger.warning("The 'autorotate_images' and 'copy_exif_data' settings "
                       "are not compatible because Sigal can't save the "
                       "modified Orientation tag.")

    # Preserve EXIF data
    if settings['copy_exif_data'] and _has_exif_tags(img):
        if options is not None:
            options = deepcopy(options)
        else:
            options = {}
        options['exif'] = img.info['exif']

    # Rotate the img, and catch IOError when PIL fails to read EXIF
    if settings['autorotate_images']:
        try:
            img = Transpose().process(img)
        except (IOError, IndexError):
            pass

    # Resize the image
    if settings['img_processor']:
        try:
            logger.debug('Processor: %s', settings['img_processor'])
            processor_cls = getattr(pilkit.processors,
                                    settings['img_processor'])
        except AttributeError:
            logger.error('Wrong processor name: %s', settings['img_processor'])
            sys.exit()

        width, height = settings['img_size']

        if img.size[0] < img.size[1]:
            # swap target size if image is in portrait mode
            height, width = width, height

        processor = processor_cls(width, height, upscale=False)
        img = processor.process(img)

    # signal.send() does not work here as plugins can modify the image, so we
    # iterate other the receivers to call them with the image.
    for receiver in signals.img_resized.receivers_for(img):
        img = receiver(img, settings=settings)

    outformat = img.format or original_format or 'JPEG'
    logger.debug('Save resized image to %s (%s)', outname, outformat)
    save_image(img, outname, outformat, options=options, autoconvert=True)

def generate_thumbnail(source, outname, box, fit=True, options=None,
                       thumb_fit_centering=(0.5, 0.5)):
    """Create a thumbnail image."""

    logger = logging.getLogger(__name__)
    img = _read_image(source)
    original_format = img.format

    if fit:
        img = ImageOps.fit(img, box, PILImage.ANTIALIAS,
                           centering=thumb_fit_centering)
    else:
        img.thumbnail(box, PILImage.ANTIALIAS)

    outformat = img.format or original_format or 'JPEG'
    logger.debug('Save thumnail image: %s (%s)', outname, outformat)
    save_image(img, outname, outformat, options=options, autoconvert=True)

def process_image(filepath, outpath, settings):
    """Process one image: resize, create thumbnail."""

    logger = logging.getLogger(__name__)
    logger.info('Processing %s', filepath)
    filename = os.path.split(filepath)[1]
    outname = os.path.join(outpath, filename)
    ext = os.path.splitext(filename)[1]

    if ext in ('.jpg', '.jpeg', '.JPG', '.JPEG'):
        options = settings['jpg_options']
    elif ext == '.png':
        options = {'optimize': True}
    else:
        options = {}

    try:
        generate_image(filepath, outname, settings, options=options)

        if settings['make_thumbs']:
            thumb_name = os.path.join(outpath, get_thumb(settings, filename))
            generate_thumbnail(
                outname, thumb_name, settings['thumb_size'],
                fit=settings['thumb_fit'], options=options,
                thumb_fit_centering=settings["thumb_fit_centering"])
    except Exception as e:
        logger.info('Failed to process: %r', e)
        if logger.getEffectiveLevel() == logging.DEBUG:
            raise
        else:
            return Status.FAILURE

    return Status.SUCCESS

def get_size(file_path):
    """Return image size (width and height)."""
    try:
        im = _read_image(file_path)
    except (IOError, IndexError, TypeError, AttributeError) as e:
        logger = logging.getLogger(__name__)
        logger.error("Could not read size of %s due to %r", file_path, e)
    else:
        width, height = im.size
        return {
            'width': width,
            'height': height
        }

def get_exif_data(filename):
    """Return a dict with the raw EXIF data."""

    logger = logging.getLogger(__name__)

    img = _read_image(filename)

    try:
        exif = img._getexif() or {}
    except ZeroDivisionError:
        logger.warning('Failed to read EXIF data.')
        return None

    data = {TAGS.get(tag, tag): value for tag, value in exif.items()}

    if 'GPSInfo' in data:
        try:
            data['GPSInfo'] = {GPSTAGS.get(tag, tag): value
                               for tag, value in data['GPSInfo'].items()}
        except AttributeError:
            logger = logging.getLogger(__name__)
            logger.info('Failed to get GPS Info')
            del data['GPSInfo']
    return data

def get_iptc_data(filename):
    """Return a dict with the raw IPTC data."""

    iptc_data = {}

    # PILs IptcImagePlugin issues a SyntaxError in certain circumstances
    # with malformed metadata, see PIL/IptcImagePlugin.py", line 71.
    # ( https://github.com/python-pillow/Pillow/blob/9dd0348be2751beb2c617e32ff9985aa2f92ae5f/src/PIL/IptcImagePlugin.py#L71 )
    try:
        img = _read_image(filename)
        raw_iptc = IptcImagePlugin.getiptcinfo(img)
    except SyntaxError:
        print("IPTC Error in %s\n"%filename)

    # IPTC fields are catalogued in:
    # https://www.iptc.org/std/photometadata/specification/IPTC-PhotoMetadata
    # 2:05 is the IPTC title property
    if raw_iptc and (2, 5) in raw_iptc:
        iptc_data["title"] = raw_iptc[(2, 5)].decode('utf-8')

    # 2:120 is the IPTC description property
    if raw_iptc and (2, 120) in raw_iptc:
        iptc_data["description"] = raw_iptc[(2, 120)].decode('utf-8')

    return iptc_data

def dms_to_degrees(v):
    """Convert degree/minute/second to decimal degrees."""

    d = float(v[0][0]) / float(v[0][1])
    m = float(v[1][0]) / float(v[1][1])
    s = float(v[2][0]) / float(v[2][1])
    return d + (m / 60.0) + (s / 3600.0)

def get_exif_tags(data, datetime_format='%c'):
    """Make a simplified version with common tags from raw EXIF data."""

    logger = logging.getLogger(__name__)
    simple = {}

    for tag in ('Model', 'Make', 'LensModel'):
        if tag in data:
            if isinstance(data[tag], tuple):
                simple[tag] = data[tag][0].strip()
            else:
                simple[tag] = data[tag].strip()

    if 'FNumber' in data:
        fnumber = data['FNumber']
        try:
            simple['fstop'] = float(fnumber[0]) / fnumber[1]
        except Exception:
            logger.debug('Skipped invalid FNumber: %r', fnumber, exc_info=True)

    if 'FocalLength' in data:
        focal = data['FocalLength']
        try:
            simple['focal'] = round(float(focal[0]) / focal[1])
        except Exception:
            logger.debug('Skipped invalid FocalLength: %r', focal,
                         exc_info=True)

    if 'ExposureTime' in data:
        exptime = data['ExposureTime']
        if isinstance(exptime, tuple):
            try:
                simple['exposure'] = str(fractions.Fraction(exptime[0],
                                                            exptime[1]))
            except ZeroDivisionError:
                logger.info('Invalid ExposureTime: %r', exptime)
        elif isinstance(exptime, int):
            simple['exposure'] = str(exptime)
        else:
            logger.info('Unknown format for ExposureTime: %r', exptime)

    if data.get('ISOSpeedRatings'):
        simple['iso'] = data['ISOSpeedRatings']

    if 'DateTimeOriginal' in data:
        # Remove null bytes at the end if necessary
        date = data['DateTimeOriginal'].rsplit('\x00')[0]

        try:
            simple['dateobj'] = datetime.strptime(date, '%Y:%m:%d %H:%M:%S')
            simple['datetime'] = simple['dateobj'].strftime(datetime_format)
        except (ValueError, TypeError) as e:
            logger.info('Could not parse DateTimeOriginal: %s', e)

    if 'GPSInfo' in data:
        info = data['GPSInfo']
        lat_info = info.get('GPSLatitude')
        lon_info = info.get('GPSLongitude')
        lat_ref_info = info.get('GPSLatitudeRef')
        lon_ref_info = info.get('GPSLongitudeRef')

        if lat_info and lon_info and lat_ref_info and lon_ref_info:
            try:
                lat = dms_to_degrees(lat_info)
                lon = dms_to_degrees(lon_info)
            except (ZeroDivisionError, ValueError, TypeError):
                logger.info('Failed to read GPS info')
            else:
                simple['gps'] = {
                    'lat': - lat if lat_ref_info != 'N' else lat,
                    'lon': - lon if lon_ref_info != 'E' else lon,
                }

    return simple
<|endoftext|>


<commit_before># Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import paddle
import numpy as np
from ... import fluid
from ...fluid import dygraph
from ...fluid import layers as F
from ...fluid.layer_helper import LayerHelper
from ...fluid.data_feeder import check_variable_and_dtype

__all__ = []

def l2_norm(x, axis, epsilon=1e-12, name=None):
    if len(x.shape) == 1:
        axis = 0
    check_variable_and_dtype(x, "X", ("float32", "float64"), "norm")

    helper = LayerHelper("l2_normalize", **locals())
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
    norm = helper.create_variable_for_type_inference(dtype=x.dtype)
    helper.append_op(
        type="norm",
        inputs={"X": x},
        outputs={"Out": out,
                 "Norm": norm},
        attrs={
            "axis": 1 if axis is None else axis,
            "epsilon": epsilon,
        })
    return paddle.squeeze(norm, axis=[axis])

def norm_except_dim(p, dim):
    shape = p.shape
    ndims = len(shape)
    if dim == -1:
        return paddle.sqrt(paddle.sum(paddle.square(p)) + 1e-12)
    elif dim == 0:
        p_matrix = paddle.reshape(p, (shape[0], -1))
        return l2_norm(p_matrix, axis=1)
    elif dim == ndims - 1:
        p_matrix = paddle.reshape(p, (-1, shape[-1]))
        return l2_norm(p_matrix, axis=0)
    else:
        perm = list(range(ndims))
        perm[0] = dim
        perm[dim] = 0
        p_transposed = paddle.transpose(p, perm)
        return norm_except_dim(p_transposed, 0)

def _weight_norm(v, g, dim):
    shape = v.shape
    ndims = len(shape)

    if dim == -1:
        v_normalized = v / (paddle.sqrt(paddle.sum(paddle.square(v))) + 1e-12)
    elif dim == 0:
        p_matrix = paddle.reshape(v, (shape[0], -1))
        v_normalized = F.l2_normalize(p_matrix, axis=1)
        v_normalized = paddle.reshape(v_normalized, shape)
    elif dim == ndims - 1:
        p_matrix = paddle.reshape(v, (-1, shape[-1]))
        v_normalized = F.l2_normalize(p_matrix, axis=0)
        v_normalized = paddle.reshape(v_normalized, shape)
    else:
        perm = list(range(ndims))
        perm[0] = dim
        perm[dim] = 0
        p_transposed = paddle.transpose(v, perm)
        transposed_shape = p_transposed.shape
        p_matrix = paddle.reshape(p_transposed, (p_transposed.shape[0], -1))
        v_normalized = F.l2_normalize(p_matrix, axis=1)
        v_normalized = paddle.reshape(v_normalized, transposed_shape)
        v_normalized = paddle.transpose(v_normalized, perm)
    weight = F.elementwise_mul(
        v_normalized, g, axis=dim if dim is not None else -1)
    return weight

class WeightNorm(object):
    def __init__(self, name, dim):
        if dim is None:
            dim = -1
        self.name = name
        self.dim = dim

    def compute_weight(self, layer):
        g = getattr(layer, self.name + '_g')
        v = getattr(layer, self.name + '_v')
        return _weight_norm(v, g, self.dim)

    @staticmethod
    def apply(layer, name, dim):
        for k, hook in layer._forward_pre_hooks.items():
            if isinstance(hook, WeightNorm) and hook.name == name:
                raise RuntimeError("Cannot register two weight_norm hooks on "
                                   "the same parameter {}".format(name))

        if dim is None:
            dim = -1

        # support dim is negative numeber, (dim = -1) == (dim = None)
        weight_dim = len(layer._parameters[name].shape)
        assert (
            dim < weight_dim and dim >= -1 * weight_dim
        ), "dim must set between [-R, R), R means the dimension of weight."
        if dim != -1:
            dim = (dim + weight_dim) % weight_dim

        fn = WeightNorm(name, dim)

        w = getattr(layer, name)
        del layer._parameters[name]

        g_var = norm_except_dim(w, dim)
        v = layer.create_parameter(w.shape, dtype=w.dtype)
        layer.add_parameter(name + "_v", v)
        g = layer.create_parameter(g_var.shape, dtype=g_var.dtype)
        layer.add_parameter(name + '_g', g)
        with paddle.no_grad():
            paddle.assign(w, v)
            paddle.assign(g_var, g)
        setattr(layer, name, fn.compute_weight(layer))

        layer.register_forward_pre_hook(fn)
        return fn

    def remove(self, layer):
        w_var = self.compute_weight(layer)
        delattr(layer, self.name)
        del layer._parameters[self.name + '_g']
        del layer._parameters[self.name + '_v']
        w = layer.create_parameter(w_var.shape, dtype=w_var.dtype)
        layer.add_parameter(self.name, w)
        with paddle.no_grad():
            paddle.assign(w_var, w)

    def __call__(self, layer, inputs):
        setattr(layer, self.name, self.compute_weight(layer))

def weight_norm(layer, name='weight', dim=0):
    r"""
    This weight_norm layer applies weight normalization to a parameter according to the
    following formula:

    .. math::

        \mathbf{w} = g \dfrac{v}{\|v\|}

    Weight normalization is a reparameterization of the weight vectors in a neural network that
    decouples the magnitude of those weight vectors from their direction. Weight normalization
    replaces the parameter specified by `name`(eg: 'weight') with two parameters: one parameter
    specifying the magnitude (eg: 'weight_g') and one parameter specifying the direction
    (eg: 'weight_v'). Weight normalization has been implemented as discussed in this paper:
    `Weight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks
    <https://arxiv.org/pdf/1602.07868.pdf>`_.

    Parameters:
        layer(Layer): Layer of paddle, which has weight.
        name(str, optional): Name of the weight parameter. Default: 'weight'.
        dim(int, optional): Dimension over which to compute the norm. Dim is a non-negative number
              which is less than the rank of weight Tensor. For Example, dim can be chosen from 0,
              1, 2, 3 for convolution whose weight shape is [cout, cin, kh, kw] and rank is 4.
              If dim is set to None, meaning that all elements will be normalized. Default: 0.

    Returns:
        Origin layer with weight norm hook.

    Examples:
        .. code-block:: python

          import numpy as np
          from paddle.nn import Conv2D
          from paddle.nn.utils import weight_norm

          x = np.array([[[[0.3, 0.4], [0.3, 0.07]], [[0.83, 0.37], [0.18, 0.93]]]]).astype('float32')
          conv = Conv2D(3, 5, 3)
          wn = weight_norm(conv)
          print(conv.weight_g.shape)
          # [5]
          print(conv.weight_v.shape)
          # [5, 3, 3, 3]
    """
    WeightNorm.apply(layer, name, dim)
    return layer

def remove_weight_norm(layer, name='weight'):
    """
    remove weight normalization from layer.

    Parameters:
        layer(Layer): Layer of paddle, which has weight.
        name(str, optional): Name of the weight parameter. Default: 'weight'.

    Returns:
        Origin layer without weight norm

    Examples:
        .. code-block:: python

          import paddle
          from paddle.nn import Conv2D
          from paddle.nn.utils import weight_norm, remove_weight_norm

          conv = Conv2D(3, 5, 3)
          wn = weight_norm(conv)
          remove_weight_norm(conv)
          print(conv.weight_g)
          # AttributeError: 'Conv2D' object has no attribute 'weight_g'
    """
    for k, hook in layer._forward_pre_hooks.items():
        if isinstance(hook, WeightNorm) and hook.name == name:
            hook.remove(layer)
            del layer._forward_pre_hooks[k]
            return layer

    raise ValueError("weight_norm of '{}' not found in {}".format(name, layer))
<commit_msg>Fix example code bugs (#42739)<commit_after># Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import paddle
import numpy as np
from ... import fluid
from ...fluid import dygraph
from ...fluid import layers as F
from ...fluid.layer_helper import LayerHelper
from ...fluid.data_feeder import check_variable_and_dtype

__all__ = []

def l2_norm(x, axis, epsilon=1e-12, name=None):
    if len(x.shape) == 1:
        axis = 0
    check_variable_and_dtype(x, "X", ("float32", "float64"), "norm")

    helper = LayerHelper("l2_normalize", **locals())
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
    norm = helper.create_variable_for_type_inference(dtype=x.dtype)
    helper.append_op(
        type="norm",
        inputs={"X": x},
        outputs={"Out": out,
                 "Norm": norm},
        attrs={
            "axis": 1 if axis is None else axis,
            "epsilon": epsilon,
        })
    return paddle.squeeze(norm, axis=[axis])

def norm_except_dim(p, dim):
    shape = p.shape
    ndims = len(shape)
    if dim == -1:
        return paddle.sqrt(paddle.sum(paddle.square(p)) + 1e-12)
    elif dim == 0:
        p_matrix = paddle.reshape(p, (shape[0], -1))
        return l2_norm(p_matrix, axis=1)
    elif dim == ndims - 1:
        p_matrix = paddle.reshape(p, (-1, shape[-1]))
        return l2_norm(p_matrix, axis=0)
    else:
        perm = list(range(ndims))
        perm[0] = dim
        perm[dim] = 0
        p_transposed = paddle.transpose(p, perm)
        return norm_except_dim(p_transposed, 0)

def _weight_norm(v, g, dim):
    shape = v.shape
    ndims = len(shape)

    if dim == -1:
        v_normalized = v / (paddle.sqrt(paddle.sum(paddle.square(v))) + 1e-12)
    elif dim == 0:
        p_matrix = paddle.reshape(v, (shape[0], -1))
        v_normalized = F.l2_normalize(p_matrix, axis=1)
        v_normalized = paddle.reshape(v_normalized, shape)
    elif dim == ndims - 1:
        p_matrix = paddle.reshape(v, (-1, shape[-1]))
        v_normalized = F.l2_normalize(p_matrix, axis=0)
        v_normalized = paddle.reshape(v_normalized, shape)
    else:
        perm = list(range(ndims))
        perm[0] = dim
        perm[dim] = 0
        p_transposed = paddle.transpose(v, perm)
        transposed_shape = p_transposed.shape
        p_matrix = paddle.reshape(p_transposed, (p_transposed.shape[0], -1))
        v_normalized = F.l2_normalize(p_matrix, axis=1)
        v_normalized = paddle.reshape(v_normalized, transposed_shape)
        v_normalized = paddle.transpose(v_normalized, perm)
    weight = F.elementwise_mul(
        v_normalized, g, axis=dim if dim is not None else -1)
    return weight

class WeightNorm(object):
    def __init__(self, name, dim):
        if dim is None:
            dim = -1
        self.name = name
        self.dim = dim

    def compute_weight(self, layer):
        g = getattr(layer, self.name + '_g')
        v = getattr(layer, self.name + '_v')
        return _weight_norm(v, g, self.dim)

    @staticmethod
    def apply(layer, name, dim):
        for k, hook in layer._forward_pre_hooks.items():
            if isinstance(hook, WeightNorm) and hook.name == name:
                raise RuntimeError("Cannot register two weight_norm hooks on "
                                   "the same parameter {}".format(name))

        if dim is None:
            dim = -1

        # support dim is negative numeber, (dim = -1) == (dim = None)
        weight_dim = len(layer._parameters[name].shape)
        assert (
            dim < weight_dim and dim >= -1 * weight_dim
        ), "dim must set between [-R, R), R means the dimension of weight."
        if dim != -1:
            dim = (dim + weight_dim) % weight_dim

        fn = WeightNorm(name, dim)

        w = getattr(layer, name)
        del layer._parameters[name]

        g_var = norm_except_dim(w, dim)
        v = layer.create_parameter(w.shape, dtype=w.dtype)
        layer.add_parameter(name + "_v", v)
        g = layer.create_parameter(g_var.shape, dtype=g_var.dtype)
        layer.add_parameter(name + '_g', g)
        with paddle.no_grad():
            paddle.assign(w, v)
            paddle.assign(g_var, g)
        setattr(layer, name, fn.compute_weight(layer))

        layer.register_forward_pre_hook(fn)
        return fn

    def remove(self, layer):
        w_var = self.compute_weight(layer)
        delattr(layer, self.name)
        del layer._parameters[self.name + '_g']
        del layer._parameters[self.name + '_v']
        w = layer.create_parameter(w_var.shape, dtype=w_var.dtype)
        layer.add_parameter(self.name, w)
        with paddle.no_grad():
            paddle.assign(w_var, w)

    def __call__(self, layer, inputs):
        setattr(layer, self.name, self.compute_weight(layer))

def weight_norm(layer, name='weight', dim=0):
    r"""
    This weight_norm layer applies weight normalization to a parameter according to the
    following formula:

    .. math::

        \mathbf{w} = g \dfrac{v}{\|v\|}

    Weight normalization is a reparameterization of the weight vectors in a neural network that
    decouples the magnitude of those weight vectors from their direction. Weight normalization
    replaces the parameter specified by `name`(eg: 'weight') with two parameters: one parameter
    specifying the magnitude (eg: 'weight_g') and one parameter specifying the direction
    (eg: 'weight_v'). Weight normalization has been implemented as discussed in this paper:
    `Weight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks
    <https://arxiv.org/pdf/1602.07868.pdf>`_.

    Parameters:
        layer(Layer): Layer of paddle, which has weight.
        name(str, optional): Name of the weight parameter. Default: 'weight'.
        dim(int, optional): Dimension over which to compute the norm. Dim is a non-negative number
              which is less than the rank of weight Tensor. For Example, dim can be chosen from 0,
              1, 2, 3 for convolution whose weight shape is [cout, cin, kh, kw] and rank is 4.
              If dim is set to None, meaning that all elements will be normalized. Default: 0.

    Returns:
        Origin layer with weight norm hook.

    Examples:
        .. code-block:: python

          import numpy as np
          from paddle.nn import Conv2D
          from paddle.nn.utils import weight_norm

          x = np.array([[[[0.3, 0.4], [0.3, 0.07]], [[0.83, 0.37], [0.18, 0.93]]]]).astype('float32')
          conv = Conv2D(3, 5, 3)
          wn = weight_norm(conv)
          print(conv.weight_g.shape)
          # [5]
          print(conv.weight_v.shape)
          # [5, 3, 3, 3]
    """
    WeightNorm.apply(layer, name, dim)
    return layer

def remove_weight_norm(layer, name='weight'):
    """
    remove weight normalization from layer.

    Parameters:
        layer(Layer): Layer of paddle, which has weight.
        name(str, optional): Name of the weight parameter. Default: 'weight'.

    Returns:
        Origin layer without weight norm

    Examples:
        .. code-block:: python

            import paddle
            from paddle.nn import Conv2D
            from paddle.nn.utils import weight_norm, remove_weight_norm

            conv = Conv2D(3, 5, 3)
            wn = weight_norm(conv)
            print(conv.weight_g)
            # Parameter containing:
            # Tensor(shape=[5], dtype=float32, place=Place(gpu:0), stop_gradient=False,
            #        [0., 0., 0., 0., 0.])
            # Conv2D(3, 5, kernel_size=[3, 3], data_format=NCHW)

            remove_weight_norm(conv)
            # print(conv.weight_g)
            # AttributeError: 'Conv2D' object has no attribute 'weight_g'
    """
    for k, hook in layer._forward_pre_hooks.items():
        if isinstance(hook, WeightNorm) and hook.name == name:
            hook.remove(layer)
            del layer._forward_pre_hooks[k]
            return layer

    raise ValueError("weight_norm of '{}' not found in {}".format(name, layer))
<|endoftext|>


<commit_before>import sys

import rethinkdb as r
from rethinkdb.errors import RqlDriverError, RqlRuntimeError

from flask import Flask

configfile = os.path.join('src', 'crweb', 'instance', 'crweb.cfg')
if len(sys.argv) > 1:
    configfile = sys.argv[1]
print("Using config %s" % configfile)

app = Flask("createdb")
if app.config.from_pyfile(configfile):
    host = app.config['DBHOST']
    database = app.config['DATABASE']
    auth_key = app.config['DBAUTHKEY']
    conn = r.connect(host, 28015, auth_key=auth_key).repl()

    r.db_create(database).run(conn)
    r.db(database).table_create('monitors').run(conn)
    r.db(database).table_create('reactions').run(conn)
    r.db(database).table_create('users').run(conn)
    r.db(database).table_create('history').run(conn)
    r.db(database).table_create('events').run(conn)
    r.db(database).table_create('subscription_history').run(conn)
    r.db(database).table_create('dc1queue').run(conn)
    r.db(database).table_create('dc2queue').run(conn)
    print "Done!"

#######################
### Database Tables ###
#######################

"""
monitors - Stores monitors
reactions - Stores reactions
users - Stores user data
history - Stores historical tracking for monitors and reactions
events - Stores unique events for monitors
subscription_history - Stores subscription history for tracking signups
dc1queue - Queue for monitors and reactions in datacenter 1
dc2queue - Queue for monitors and reactions in datacenter 2
"""
<commit_msg>fixed import error in create_db.py<commit_after>import sys
import os

import rethinkdb as r
from rethinkdb.errors import RqlDriverError, RqlRuntimeError

from flask import Flask

configfile = os.path.join('src', 'crweb', 'instance', 'crweb.cfg')
if len(sys.argv) > 1:
    configfile = sys.argv[1]
print("Using config %s" % configfile)

app = Flask("createdb")
if app.config.from_pyfile(configfile):
    host = app.config['DBHOST']
    database = app.config['DATABASE']
    auth_key = app.config['DBAUTHKEY']
    conn = r.connect(host, 28015, auth_key=auth_key).repl()

    r.db_create(database).run(conn)
    r.db(database).table_create('monitors').run(conn)
    r.db(database).table_create('reactions').run(conn)
    r.db(database).table_create('users').run(conn)
    r.db(database).table_create('history').run(conn)
    r.db(database).table_create('events').run(conn)
    r.db(database).table_create('subscription_history').run(conn)
    r.db(database).table_create('dc1queue').run(conn)
    r.db(database).table_create('dc2queue').run(conn)
    print "Done!"

#######################
### Database Tables ###
#######################

"""
monitors - Stores monitors
reactions - Stores reactions
users - Stores user data
history - Stores historical tracking for monitors and reactions
events - Stores unique events for monitors
subscription_history - Stores subscription history for tracking signups
dc1queue - Queue for monitors and reactions in datacenter 1
dc2queue - Queue for monitors and reactions in datacenter 2
"""
<|endoftext|>


<commit_before>import numpy as np
import individual
import copy

class GeneticAlgorithm(object):
    ''''
            Genetic Algorithm

            A metaheurisic algorithm inspired by the process of natural
            selection that mimics biological evolution to solve constrained
            and unconstrained optimization problems.

            At each step, the genetic algorithm randomly selects individuals
            from the current population and uses them as parents to produce
            the offspring for the next generation. Over successive generations,
            the population "evolves" toward an optimal solution.

            INPUT:
                - Number of generations
                - Number of solutions in initial population
                - Number of polygons in solution
                - Height of Image
                - Width of Image
                - Target Image

     '''

    def __init__(self, generations, population, size, height, width , target):

        # Set hyperparameters

        self.generations = generations
        self.population = population
        self.size = size
        self.height = height
        self.width = width
        self.target = target

        self.cur_gen = []
        for i in range(population):
            self.cur_gen.append(individual.IndividualGen(self.size,self.height,self.width,0.1))

    def run ( self , best_k , mutation_rate ):

        # Performs generations update

        for gen in range(self.generations):

            # Sort individuals by Fitness
            scores = []
            for i in range(self.population):
                scores.append((self.cur_gen[i].fitness(self.target), self.cur_gen[i]))

            scores.sort(key=lambda x: x[0])

            # Report best individual per generation
            if gen % 2 == 0:
                scores[0][1].write("SolutionGA_Error_" + str(gen) + "_" + str(scores[0][0]) + ".jpg")

            # Create a new generation
            next_gen = []

            # Keep Top K Ranked individuals for next generation ( Ellitism )
            for i in range(best_k):
                next_gen.append(copy.deepcopy(scores[i][1]))

            # Apply genetic operations to produce the next generation
            while len(next_gen) < self.population:

                # Selection based on Fitness ( Roulette Wheel Sampling )

                order = np.random.permutation(self.population)
                rank = np.random.uniform(scores[0][0], scores[self.population-1][0])
                parent1_index = -1
                for i in range(self.population):
                    if ( scores[order[i]][0] <= rank ):
                        parent1_index = order[i]
                        break

                parent1 = copy.deepcopy( scores[parent1_index][1] )

                order = np.random.permutation(self.population)
                rank = np.random.uniform(scores[0][0], scores[self.population - 1][0])
                parent2_index = -1
                for i in range(self.population):
                    if (scores[order[i]][0] <= rank):
                        parent2_index = order[i]
                        break

                parent2 = copy.deepcopy(scores[parent2_index][1])

                # Crossover
                child1 = parent1
                child2 = parent2

                index = np.random.randint(0,self.size-1)
                child1.genes[index:] = scores[parent2_index][1].genes[index:]
                child2.genes[:index] = scores[parent1_index][1].genes[:index]

                child1.write("child1.jpg")
                child2.write("child2.jpg")

                # Mutation
                if np.random.random() < mutation_rate: child1.mutate()
                if np.random.random() < mutation_rate: child2.mutate()

                next_gen.append(child1)
                next_gen.append(child2)

            # Update new generation
            self.cur_gen = copy.deepcopy(next_gen)
<commit_msg>Upload Files<commit_after>import numpy as np
import individual
import copy

class GeneticAlgorithm(object):
    ''''
            Genetic Algorithm

            A metaheurisic algorithm inspired by the process of natural
            selection that mimics biological evolution to solve constrained
            and unconstrained optimization problems.

            At each step, the genetic algorithm randomly selects individuals
            from the current population and uses them as parents to produce
            the offspring for the next generation. Over successive generations,
            the population "evolves" toward an optimal solution.

            INPUT:
                - Number of generations
                - Number of solutions in initial population
                - Number of polygons in solution
                - Height of Image
                - Width of Image
                - Target Image

     '''

    def __init__(self, generations, population, size, height, width , target):

        # Set hyperparameters

        self.generations = generations
        self.population = population
        self.size = size
        self.height = height
        self.width = width
        self.target = target

        self.cur_gen = []
        for i in range(population):
            self.cur_gen.append(individual.IndividualGen(self.size,self.height,self.width,0.1))

    def run ( self , best_k , mutation_rate ):

        # Performs generations update

        for gen in range(self.generations):

            # Sort individuals by Fitness
            scores = []
            for i in range(self.population):
                scores.append((self.cur_gen[i].fitness(self.target), self.cur_gen[i]))

            scores.sort(key=lambda x: x[0])

            # Report best individual per generation
            if gen % 2 == 0:
                scores[0][1].write("SolutionGA_Error_" + str(gen) + "_" + str(scores[0][0]) + ".jpg")

            # Create a new generation
            next_gen = []

            # Keep Top K Ranked individuals for next generation ( Ellitism )
            for i in range(best_k):
                next_gen.append(copy.deepcopy(scores[i][1]))

            # Apply genetic operations to produce the next generation
            while len(next_gen) < self.population:

                # Selection based on Fitness ( Roulette Wheel Sampling )

                order = np.random.permutation(self.population)
                rank = np.random.uniform(scores[0][0], scores[self.population-1][0])
                parent1_index = -1
                for i in range(self.population):
                    if ( scores[order[i]][0] <= rank ):
                        parent1_index = order[i]
                        break

                parent1 = copy.deepcopy( scores[parent1_index][1] )

                order = np.random.permutation(self.population)
                rank = np.random.uniform(scores[0][0], scores[self.population - 1][0])
                parent2_index = -1
                for i in range(self.population):
                    if (scores[order[i]][0] <= rank):
                        parent2_index = order[i]
                        break

                parent2 = copy.deepcopy(scores[parent2_index][1])

                # Crossover
                child1 = parent1
                child2 = parent2

                index = np.random.randint(0,self.size-1)
                child1.genes[:index] = scores[parent2_index][1].genes[:index]
                child2.genes[:index] = scores[parent1_index][1].genes[:index]

                # Mutation
                if np.random.random() < mutation_rate: child1.mutate()
                if np.random.random() < mutation_rate: child2.mutate()

                next_gen.append(child1)
                next_gen.append(child2)

            # Update new generation
            self.cur_gen = copy.deepcopy(next_gen)
<|endoftext|>


<commit_before><commit_msg>removed disabled imports<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf8

import os

from . import KeyValueStore

class FilesystemStore(KeyValueStore):
    """Store data in files on the filesystem.

    The *FilesystemStore* stores every value as its own file on the filesystem,
    all under a common directory.
    """
    def __init__(self, root, **kwargs):
        """Initialize new FilesystemStore

        :param root: the base directory for the store
        """
        super(FilesystemStore, self).__init__(**kwargs)
        self.root = root
        self.bufsize = 1024 * 1024  # 1m

    def _build_filename(self, key):
        return os.path.join(self.root, key)

    def _delete(self, key):
        try:
            os.unlink(self._build_filename(key))
        except OSError, e:
            if not e.errno == 2:
                raise

    def _open(self, key):
        try:
            f = open(self._build_filename(key), 'rb')
            return f
        except IOError, e:
            if 2 == e.errno:
                raise KeyError(key)
            else:
                raise

    def _put(self, key, data):
        with file(self._build_filename(key), 'wb') as f:
            f.write(data)

        return key

    def _put_file(self, key, file):
        bufsize = self.bufsize
        with open(self._build_filename(key), 'wb') as f:
            while True:
                buf = file.read(bufsize)
                f.write(buf)
                if len(buf) < bufsize:
                    break

        return key
<commit_msg>Use shutil.move for moving in files in filesystem store, to avoid extra copying.<commit_after>#!/usr/bin/env python
# coding=utf8

import os
import shutil

from . import KeyValueStore

class FilesystemStore(KeyValueStore):
    """Store data in files on the filesystem.

    The *FilesystemStore* stores every value as its own file on the filesystem,
    all under a common directory.
    """
    def __init__(self, root, **kwargs):
        """Initialize new FilesystemStore

        :param root: the base directory for the store
        """
        super(FilesystemStore, self).__init__(**kwargs)
        self.root = root
        self.bufsize = 1024 * 1024  # 1m

    def _build_filename(self, key):
        return os.path.join(self.root, key)

    def _delete(self, key):
        try:
            os.unlink(self._build_filename(key))
        except OSError, e:
            if not e.errno == 2:
                raise

    def _open(self, key):
        try:
            f = open(self._build_filename(key), 'rb')
            return f
        except IOError, e:
            if 2 == e.errno:
                raise KeyError(key)
            else:
                raise

    def _put(self, key, data):
        with file(self._build_filename(key), 'wb') as f:
            f.write(data)

        return key

    def _put_file(self, key, file):
        bufsize = self.bufsize
        with open(self._build_filename(key), 'wb') as f:
            while True:
                buf = file.read(bufsize)
                f.write(buf)
                if len(buf) < bufsize:
                    break

        return key

    def _put_filename(self, key, filename):
        shutil.move(filename, self._build_filename(key))
        return key
<|endoftext|>