<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from urllib import urlencode

from django.conf import settings
from django.http import HttpResponse, HttpResponseBadRequest
from django.utils.translation import ugettext as _
from django.views.generic import TemplateView, View

from redis_cache import get_redis_connection
from smithers import data_types
from smithers import redis_keys as rkeys

if settings.ENABLE_REDIS:
    redis = get_redis_connection('smithers')
else:
    redis = False

TWITTER_URL = 'https://twitter.com/share'
FB_URL = 'https://www.facebook.com/sharer/sharer.php'
COUNT_FOOTNOTE = (u''
                  u''
                  u'')

def get_tw_share_url(**kwargs):
    kwargs.setdefault('hashtags', 'firefox')
    kwargs.setdefault('dnt', 'true')
    text = kwargs.get('text')
    if text:
        kwargs['text'] = text.encode('utf8')
    return '?'.join([TWITTER_URL, urlencode(kwargs)])

def get_fb_share_url(url):
    return '?'.join([FB_URL, urlencode({'u': url})])

class GlowView(TemplateView):
    template_name = 'base.html'

    def get_context_data(self, **kwargs):
        if redis:
            timestamp = int(redis.get(rkeys.LATEST_TIMESTAMP) or 0)
        else:
            timestamp = 0
        context = super(GlowView, self).get_context_data(**kwargs)
        context.update({
            'data_timestamp': timestamp,
            'share_map_twitter': get_tw_share_url(
                url='http://mzl.la/1g5k6OK',
                text=_('Join millions of Firefox users around the world '
                       'who are shaping the future of the Web:'),
            ),
            'share_stats_twitter': get_tw_share_url(
                url='http://mzl.la/1n0x8lA',
                text=_('Join millions of Firefox users around the world '
                       'who are shaping the future of the Web:'),
            ),
            'share_stats_twitter_privacy': get_tw_share_url(
                url='http://mzl.la/1hq7RRe',
                text='#Firefox {}'
                     .format(_('fights for government surveillance reform and was the only major browser not targeted by the NSA scandal.')),
            ),
            'share_stats_twitter_opportunity': get_tw_share_url(
                url='http://mzl.la/1lQpeeD',
                text='#Firefox {}'
                     .format(_('is made by a global volunteer community 6,000 strong, open to participation from anyone.')),
            ),
            'share_stats_twitter_access': get_tw_share_url(
                url='http://mzl.la/1lQpudA',
                text='#Firefox {}'
                     .format(_('disrupted the mobile industry with the first Web-based OS to help bring the next billion people online')),
            ),
            'share_stats_twitter_freedom': get_tw_share_url(
                url='http://mzl.la/1qGWpl2',
                text='#Firefox {}'
                     .format(_('protested against SOPA and PIPA, dangerous copyright legislation that threatened the freedom of the Web.')),
            ),
            'share_stats_twitter_learning': get_tw_share_url(
                url='http://mzl.la/1hysQTb',
                text='#Firefox {}'
                     .format(_('teaches digital skills to millions of people to help them move from using the Web to actively making it.')),
            ),
            'share_stats_twitter_control': get_tw_share_url(
                url='http://mzl.la/PXXz0j',
                text='#Firefox {}'
                     .format(_('leads the way in giving users greater control online. They pioneered features like Do Not Track and Lightbeam.')),
            ),
            'share_map_facebook': get_fb_share_url('http://mzl.la/1oKbBCb'),
            'share_stats_facebook': get_fb_share_url('http://mzl.la/1sxET6z'),
            'share_stats_facebook_privacy': get_fb_share_url('http://mzl.la/1n14u6H'),
            'share_stats_facebook_opportunity': get_fb_share_url('http://mzl.la/1hGzq48'),
            'share_stats_facebook_access': get_fb_share_url('http://mzl.la/1qo3icx'),
            'share_stats_facebook_freedom': get_fb_share_url('http://mzl.la/OPDh7Y'),
            'share_stats_facebook_learning': get_fb_share_url('http://mzl.la/1n14T8X'),
            'share_stats_facebook_control': get_fb_share_url('http://mzl.la/R9IaL7'),
            'count_footnote': COUNT_FOOTNOTE.format(_('What does this number mean?')),
        })
        return context

class ShareView(View):
    def post(self, request):
        issue = request.POST.get('issue')
        if issue not in data_types.name_to_id:
            return HttpResponseBadRequest('invalid issue id',
                                          content_type='text/plain')

        client_ip = request.META.get('HTTP_X_CLUSTER_CLIENT_IP',
                                     request.META.get('REMOTE_ADDR'))
        if redis and client_ip:
            redis.lpush(rkeys.IPLOGS, '{},{}'.format(data_types.name_to_id[issue],
                                                     client_ip))

        return HttpResponse(status=204)  # 204 no content

class StringsView(TemplateView):
    content_type = 'text/plain'
    template_name = 'strings.html'
<commit_msg>Fix unicode issue in the main view.<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals

from urllib import urlencode

from django.conf import settings
from django.http import HttpResponse, HttpResponseBadRequest
from django.utils.translation import ugettext as _
from django.views.generic import TemplateView, View

from redis_cache import get_redis_connection
from smithers import data_types
from smithers import redis_keys as rkeys

if settings.ENABLE_REDIS:
    redis = get_redis_connection('smithers')
else:
    redis = False

TWITTER_URL = 'https://twitter.com/share'
FB_URL = 'https://www.facebook.com/sharer/sharer.php'
COUNT_FOOTNOTE = (''
                  ''
                  '')

def get_tw_share_url(**kwargs):
    kwargs.setdefault('hashtags', 'firefox')
    kwargs.setdefault('dnt', 'true')
    text = kwargs.get('text')
    if text:
        kwargs['text'] = text.encode('utf8')
    return '?'.join([TWITTER_URL, urlencode(kwargs)])

def get_fb_share_url(url):
    return '?'.join([FB_URL, urlencode({'u': url})])

class GlowView(TemplateView):
    template_name = 'base.html'

    def get_context_data(self, **kwargs):
        if redis:
            timestamp = int(redis.get(rkeys.LATEST_TIMESTAMP) or 0)
        else:
            timestamp = 0
        context = super(GlowView, self).get_context_data(**kwargs)
        context.update({
            'data_timestamp': timestamp,
            'share_map_twitter': get_tw_share_url(
                url='http://mzl.la/1g5k6OK',
                text=_('Join millions of Firefox users around the world '
                       'who are shaping the future of the Web:'),
            ),
            'share_stats_twitter': get_tw_share_url(
                url='http://mzl.la/1n0x8lA',
                text=_('Join millions of Firefox users around the world '
                       'who are shaping the future of the Web:'),
            ),
            'share_stats_twitter_privacy': get_tw_share_url(
                url='http://mzl.la/1hq7RRe',
                text='#Firefox {}'
                     .format(_('fights for government surveillance reform '
                               'and was the only major browser not targeted by the NSA scandal.')),
            ),
            'share_stats_twitter_opportunity': get_tw_share_url(
                url='http://mzl.la/1lQpeeD',
                text='#Firefox {}'
                     .format(_('is made by a global volunteer community 6,000 strong, '
                               'open to participation from anyone.')),
            ),
            'share_stats_twitter_access': get_tw_share_url(
                url='http://mzl.la/1lQpudA',
                text='#Firefox {}'
                     .format(_('disrupted the mobile industry with the first Web-based OS to '
                               'help bring the next billion people online')),
            ),
            'share_stats_twitter_freedom': get_tw_share_url(
                url='http://mzl.la/1qGWpl2',
                text='#Firefox {}'
                     .format(_('protested against SOPA and PIPA, dangerous copyright legislation '
                               'that threatened the freedom of the Web.')),
            ),
            'share_stats_twitter_learning': get_tw_share_url(
                url='http://mzl.la/1hysQTb',
                text='#Firefox {}'
                     .format(_('teaches digital skills to millions of people to help them move '
                               'from using the Web to actively making it.')),
            ),
            'share_stats_twitter_control': get_tw_share_url(
                url='http://mzl.la/PXXz0j',
                text='#Firefox {}'
                     .format(_('leads the way in giving users greater control online. '
                               'They pioneered features like Do Not Track and Lightbeam.')),
            ),
            'share_map_facebook': get_fb_share_url('http://mzl.la/1oKbBCb'),
            'share_stats_facebook': get_fb_share_url('http://mzl.la/1sxET6z'),
            'share_stats_facebook_privacy': get_fb_share_url('http://mzl.la/1n14u6H'),
            'share_stats_facebook_opportunity': get_fb_share_url('http://mzl.la/1hGzq48'),
            'share_stats_facebook_access': get_fb_share_url('http://mzl.la/1qo3icx'),
            'share_stats_facebook_freedom': get_fb_share_url('http://mzl.la/OPDh7Y'),
            'share_stats_facebook_learning': get_fb_share_url('http://mzl.la/1n14T8X'),
            'share_stats_facebook_control': get_fb_share_url('http://mzl.la/R9IaL7'),
            'count_footnote': COUNT_FOOTNOTE.format(_('What does this number mean?')),
        })
        return context

class ShareView(View):
    def post(self, request):
        issue = request.POST.get('issue')
        if issue not in data_types.name_to_id:
            return HttpResponseBadRequest('invalid issue id',
                                          content_type='text/plain')

        client_ip = request.META.get('HTTP_X_CLUSTER_CLIENT_IP',
                                     request.META.get('REMOTE_ADDR'))
        if redis and client_ip:
            redis.lpush(rkeys.IPLOGS, '{},{}'.format(data_types.name_to_id[issue],
                                                     client_ip))

        return HttpResponse(status=204)  # 204 no content

class StringsView(TemplateView):
    content_type = 'text/plain'
    template_name = 'strings.html'
<|endoftext|>


<commit_before>#!/usr/bin/env python

import logging
from os import mkdir
from os.path import join as path_join
from urllib.parse import quote_plus, urljoin
from datetime import datetime
from time import sleep
from random import randint

import requests
from bs4 import BeautifulSoup

logging.basicConfig(
    format=(
        '%(asctime)s\t%(levelname)s\t'
        #'%(processName)s\t%(threadName)s\t'
        '%(name)s\t%(funcName)s:%(lineno)d\t'
        '%(message)s'
    ),
    level=logging.DEBUG
)

def _make_fake_browser():

    fake_browser = requests.Session()
    fake_browser.headers = {
        'user-agent': (
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) '
            'AppleWebKit/537.36 (KHTML, like Gecko) '
            'Chrome/54.0.2840.98 Safari/537.36'
        ),
        'accept': (
            'text/html,application/xhtml+xml,application/xml;q=0.9,'
            'image/webp,*/*;q=0.8'
        ),
        'accept-encoding': 'gzip, deflate, sdch, br',
        'accept-language': 'en-US,en;q=0.8,zh-TW;q=0.6,zh;q=0.4',
        'cookie': 'over18=1',
    }

    return fake_browser

_SHARED_FAKE_BROWSER = _make_fake_browser()
_CACHE_DIR_PATH = 'cache/'
_URL_SET_SKIPPING_CACHE = {'https://www.ptt.cc/bbs/Gossiping/index.html'}

def read_or_request(url):

    # should generate valid fname for most of the systems
    fname = quote_plus(url)
    path = path_join(_CACHE_DIR_PATH, fname)

    # try cache

    if url in _URL_SET_SKIPPING_CACHE:

        logging.info('Skip cache for {}'.format(url))

    else:

        try:
            with open(path) as f:
                logging.info('Hit {}'.format(url))
                return f.read()
        except OSError:
            logging.info('Missed {}'.format(url))

    # request

    resp = _SHARED_FAKE_BROWSER.get(url)
    text = resp.text

    while 1:

        try:
            with open(path, 'w') as f:
                f.write(text)
        except FileNotFoundError:
            mkdir(_CACHE_DIR_PATH)
            continue
        else:
            logging.info('Wrote {}'.format(url))

        # only loop once normally
        break

    return text

_ROOT = 'https://www.ptt.cc'

def parse_index_page(text):

    soup = BeautifulSoup(text, 'html.parser')

    # paging

    paging_a_tags = soup.select('.action-bar .btn-group-paging a')

    prev_a_tag = paging_a_tags[1]
    next_a_tag = paging_a_tags[2]

    if '上頁' not in prev_a_tag.string:
        raise RuntimeError('the prev button looks not right')
    if '下頁' not in next_a_tag.string:
        raise RuntimeError('the next button looks not right')

    prev_href = prev_a_tag.get('href', '')
    prev_url = prev_href and urljoin(_ROOT, prev_href)
    next_href = next_a_tag.get('href', '')
    next_url = next_href and urljoin(_ROOT, next_href)

    # entries

    entry_ds = []
    ent_tags = soup.select('.r-list-container .r-ent')
    for ent_tag in ent_tags:

        # push_score_sum

        # cases
        #
        # 1.
12

        # 2.

        # 3.
X1

        #

        push_score_sum = -65535
        span_tag = ent_tag.find(class_='nrec').find('span')

        # if case 2
        if not span_tag:

            push_score_sum = 0

        else:

            span_text = span_tag.string

            while 1:

                # if case 1
                try:
                    push_score_sum = int(span_text)
                except ValueError:
                    pass
                else:
                    break

                # if case 3
                try:
                    push_score_sum = int(span_text[1:])
                except ValueError:
                    pass
                else:
                    # TODO: is it correct?
                    push_score_sum = -(push_score_sum+10)
                    break

                # loop once normally
                break

        # title & article_url

        # cases
        #
        # 1.

        #
        #            Re: [問卦] 有沒有高雄其實比台北進步的八卦??
        #
        #

        # 2.
 (本文已被刪除) [gundam0613]

        #

        title_tag = ent_tag.find(class_='title')
        title_a_tag = title_tag.a
        if title_a_tag:
            # case 1
            title = title_a_tag.string
            article_url = urljoin(_ROOT, title_a_tag.get('href', ''))
        else:
            # case 2: deleted article
            title = title_tag.string.strip()
            article_url = ''

        # others

        ent_meta_tag = ent_tag.find(class_='meta')

        # mmdd won't contain year!
        raw_mmdd = ent_meta_tag.find(class_='date').string
        author_id = ent_meta_tag.find(class_='author').string
        # deleted article
        if not article_url and author_id == '-':
            author_id = ''

        # append

        entry_ds.append({
            'push_score_sum' : push_score_sum,
            'title'          : title,
            'article_url'    : article_url,
            'raw_mmdd'       : raw_mmdd,
            'author_id'      : author_id
        })

    return {
        'prev_url': prev_url,
        'next_url': next_url,
        'entry_ds': entry_ds
    }

_EXPECTED_ARTICLE_TAG_KEYS = ('作者', '看板', '標題', '時間')
_PUSH_TAG_STRIPPED_TEXT_SCORE_MAP = {
    '噓': -1,
    '→': 0,
    '推': 1
}

def parse_article_page(text):

    soup = BeautifulSoup(text, 'html.parser')
    main_content_tag = soup.find(id='main-content')

    # meta
    #
    #

    #     作者
    #     a77774444 (我愛ˋ台灣)
    #

    #

    for tag, key in zip(
        main_content_tag.select('.article-meta-tag'),
        _EXPECTED_ARTICLE_TAG_KEYS
    ):
        if tag.string != key:
            raise RuntimeError('the article tags may be changed')

    author_line, board_name, title, timestamp_text = (
        tag.string
        for tag in main_content_tag.select('.article-meta-value')
    )

    author_id, _, author_line_tail = author_line.partition(' (')
    author_nick = author_line_tail[:-1]

    # Tue Nov 29 05:05:03 2016
    # Fri Jul  1 21:12:25 2005
    authored_dt = datetime.strptime(timestamp_text, '%a %b %d %H:%M:%S %Y')

    # body
    # TODO: record the IPs?

    recording = False
    body_lines = []
    for i, line in enumerate(main_content_tag.stripped_strings):

        if i == 6:
            if line != '時間':
                raise RuntimeError('the article structure may be changed')
        elif i == 8:
            recording = True
        elif line.startswith('※ 發信站: 批踢踢實業坊'):
            recording = False
            break

        if recording:
            body_lines.append(line)

    else:

        raise RuntimeError('the ending text of article may be change/')

    # ... 以後台北人都要去高雄當台勞了。\n\n--'
    body = '\n'.join(body_lines).rstrip('\n-')

    # pushes

    push_ds = [
        {
            'score': _PUSH_TAG_STRIPPED_TEXT_SCORE_MAP.get(
                push_tag.find(class_='push-tag').string.strip(), -255
            ),
            'pusher_id': push_tag.find(class_='push-userid').string or '',
            # consider topractise's push in
            # https://www.ptt.cc/bbs/Gossiping/M.1480355255.A.07C.html
            'text': push_tag.find(class_='push-content').string or '',
            # won't contain year!
            'raw_mmddhhmm': (
                push_tag.find(class_='push-ipdatetime').string.strip()
            )
        }
        for push_tag in main_content_tag.select('.push')
    ]

    return {
        'board_name' : board_name,
        'author_id'  : author_id,
        'author_nick': author_nick,
        'title'      : title,
        'authored_dt': authored_dt,
        'body'       : body,
        'push_ds'    : push_ds
    }

def crawl(index_url):

    count = 0
    prev_url = index_url

    while 1:

        # crawl and parse the index page

        logging.info('Crawl the index page {} ...'.format(prev_url))
        try:
            text = read_or_request(prev_url)
        except OSError:
            # try again
            logging.info('Try again ...')
            text = read_or_request(prev_url)

        logging.info('Parse the index page {} ...'.format(prev_url))
        parsed_index_d = parse_index_page(text)

        prev_url = parsed_index_d['prev_url']

        # crawl the article_url

        for entry_d in parsed_index_d['entry_ds']:

            article_url = entry_d['article_url']
            # if deleted article
            if not article_url:
                continue

            logging.info('Crawl the article page {} ...'.format(article_url))
            try:
                read_or_request(article_url)
            except OSError:
                # try again
                logging.info('Try again ...')
                try:
                    read_or_request(article_url)
                except OSError:
                    # skip if still fail
                    logging.info('Skip')
                    continue

            count += 1

            logging.info('Sleep')
            sleep(randint(0, 10)*0.001)

        logging.info('Got {:,} articles so far'.format(count))

if __name__ == '__main__':

    from pprint import pprint

    crawl('https://www.ptt.cc/bbs/Gossiping/index.html')

    import sys
    sys.exit()

    pprint(parse_article_page(read_or_request(
        'https://www.ptt.cc/bbs/Gossiping/M.1480355255.A.07C.html'
    )))

    import sys
    sys.exit()

    # test the index page
    pprint(parse_index_page(read_or_request(
        'https://www.ptt.cc/bbs/Gossiping/index20177.html'
    )))

    # test the article page #1
    pprint(parse_article_page(read_or_request(
        'https://www.ptt.cc/bbs/Gossiping/M.1480367106.A.A55.html'
    )))

    # test the article page #2
    pprint(parse_article_page(read_or_request(
        'https://www.ptt.cc/bbs/Gossiping/M.1480380251.A.9A4.html'
    )))
<commit_msg>Only crawl news<commit_after>#!/usr/bin/env python

import logging
from os import mkdir
from os.path import join as path_join
from urllib.parse import quote_plus, urljoin
from datetime import datetime
from time import sleep
from random import randint

import requests
from bs4 import BeautifulSoup

logging.basicConfig(
    format=(
        '%(asctime)s\t%(levelname)s\t'
        #'%(processName)s\t%(threadName)s\t'
        '%(name)s\t%(funcName)s:%(lineno)d\t'
        '%(message)s'
    ),
    level=logging.DEBUG
)

def _make_fake_browser():

    fake_browser = requests.Session()
    fake_browser.headers = {
        'user-agent': (
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) '
            'AppleWebKit/537.36 (KHTML, like Gecko) '
            'Chrome/54.0.2840.98 Safari/537.36'
        ),
        'accept': (
            'text/html,application/xhtml+xml,application/xml;q=0.9,'
            'image/webp,*/*;q=0.8'
        ),
        'accept-encoding': 'gzip, deflate, sdch, br',
        'accept-language': 'en-US,en;q=0.8,zh-TW;q=0.6,zh;q=0.4',
        'cookie': 'over18=1',
    }

    return fake_browser

_SHARED_FAKE_BROWSER = _make_fake_browser()
_CACHE_DIR_PATH = 'cache/'
_URL_SET_SKIPPING_CACHE = {'https://www.ptt.cc/bbs/Gossiping/index.html'}

def read_or_request(url):

    # should generate valid fname for most of the systems
    fname = quote_plus(url)
    path = path_join(_CACHE_DIR_PATH, fname)

    # try cache

    if url in _URL_SET_SKIPPING_CACHE:

        logging.info('Skip cache for {}'.format(url))

    else:

        try:
            with open(path) as f:
                logging.info('Hit {}'.format(url))
                return f.read()
        except OSError:
            logging.info('Missed {}'.format(url))

    # request

    resp = _SHARED_FAKE_BROWSER.get(url)
    text = resp.text

    while 1:

        try:
            with open(path, 'w') as f:
                f.write(text)
        except FileNotFoundError:
            mkdir(_CACHE_DIR_PATH)
            continue
        else:
            logging.info('Wrote {}'.format(url))

        # only loop once normally
        break

    return text

_ROOT = 'https://www.ptt.cc'

def parse_index_page(text):

    soup = BeautifulSoup(text, 'html.parser')

    # paging

    paging_a_tags = soup.select('.action-bar .btn-group-paging a')

    prev_a_tag = paging_a_tags[1]
    next_a_tag = paging_a_tags[2]

    if '上頁' not in prev_a_tag.string:
        raise RuntimeError('the prev button looks not right')
    if '下頁' not in next_a_tag.string:
        raise RuntimeError('the next button looks not right')

    prev_href = prev_a_tag.get('href', '')
    prev_url = prev_href and urljoin(_ROOT, prev_href)
    next_href = next_a_tag.get('href', '')
    next_url = next_href and urljoin(_ROOT, next_href)

    # entries

    entry_ds = []
    ent_tags = soup.select('.r-list-container .r-ent')
    for ent_tag in ent_tags:

        # push_score_sum

        # cases
        #
        # 1.
12

        # 2.

        # 3.
X1

        #

        push_score_sum = -65535
        span_tag = ent_tag.find(class_='nrec').find('span')

        # if case 2
        if not span_tag:

            push_score_sum = 0

        else:

            span_text = span_tag.string

            while 1:

                # if case 1
                try:
                    push_score_sum = int(span_text)
                except ValueError:
                    pass
                else:
                    break

                # if case 3
                try:
                    push_score_sum = int(span_text[1:])
                except ValueError:
                    pass
                else:
                    # TODO: is it correct?
                    push_score_sum = -(push_score_sum+10)
                    break

                # loop once normally
                break

        # title & article_url

        # cases
        #
        # 1.

        #
        #            Re: [問卦] 有沒有高雄其實比台北進步的八卦??
        #
        #

        # 2.
 (本文已被刪除) [gundam0613]

        #

        title_tag = ent_tag.find(class_='title')
        title_a_tag = title_tag.a
        if title_a_tag:
            # case 1
            title = title_a_tag.string
            article_url = urljoin(_ROOT, title_a_tag.get('href', ''))
        else:
            # case 2: deleted article
            title = title_tag.string.strip()
            article_url = ''

        # others

        ent_meta_tag = ent_tag.find(class_='meta')

        # mmdd won't contain year!
        raw_mmdd = ent_meta_tag.find(class_='date').string
        author_id = ent_meta_tag.find(class_='author').string
        # deleted article
        if not article_url and author_id == '-':
            author_id = ''

        # append

        entry_ds.append({
            'push_score_sum' : push_score_sum,
            'title'          : title,
            'article_url'    : article_url,
            'raw_mmdd'       : raw_mmdd,
            'author_id'      : author_id
        })

    return {
        'prev_url': prev_url,
        'next_url': next_url,
        'entry_ds': entry_ds
    }

_EXPECTED_ARTICLE_TAG_KEYS = ('作者', '看板', '標題', '時間')
_PUSH_TAG_STRIPPED_TEXT_SCORE_MAP = {
    '噓': -1,
    '→': 0,
    '推': 1
}

def parse_article_page(text):

    soup = BeautifulSoup(text, 'html.parser')
    main_content_tag = soup.find(id='main-content')

    # meta
    #
    #

    #     作者
    #     a77774444 (我愛ˋ台灣)
    #

    #

    for tag, key in zip(
        main_content_tag.select('.article-meta-tag'),
        _EXPECTED_ARTICLE_TAG_KEYS
    ):
        if tag.string != key:
            raise RuntimeError('the article tags may be changed')

    author_line, board_name, title, timestamp_text = (
        tag.string
        for tag in main_content_tag.select('.article-meta-value')
    )

    author_id, _, author_line_tail = author_line.partition(' (')
    author_nick = author_line_tail[:-1]

    # Tue Nov 29 05:05:03 2016
    # Fri Jul  1 21:12:25 2005
    authored_dt = datetime.strptime(timestamp_text, '%a %b %d %H:%M:%S %Y')

    # body
    # TODO: record the IPs?

    recording = False
    body_lines = []
    for i, line in enumerate(main_content_tag.stripped_strings):

        if i == 6:
            if line != '時間':
                raise RuntimeError('the article structure may be changed')
        elif i == 8:
            recording = True
        elif line.startswith('※ 發信站: 批踢踢實業坊'):
            recording = False
            break

        if recording:
            body_lines.append(line)

    else:

        raise RuntimeError('the ending text of article may be change/')

    # ... 以後台北人都要去高雄當台勞了。\n\n--'
    body = '\n'.join(body_lines).rstrip('\n-')

    # pushes

    push_ds = [
        {
            'score': _PUSH_TAG_STRIPPED_TEXT_SCORE_MAP.get(
                push_tag.find(class_='push-tag').string.strip(), -255
            ),
            'pusher_id': push_tag.find(class_='push-userid').string or '',
            # consider topractise's push in
            # https://www.ptt.cc/bbs/Gossiping/M.1480355255.A.07C.html
            'text': push_tag.find(class_='push-content').string or '',
            # won't contain year!
            'raw_mmddhhmm': (
                push_tag.find(class_='push-ipdatetime').string.strip()
            )
        }
        for push_tag in main_content_tag.select('.push')
    ]

    return {
        'board_name' : board_name,
        'author_id'  : author_id,
        'author_nick': author_nick,
        'title'      : title,
        'authored_dt': authored_dt,
        'body'       : body,
        'push_ds'    : push_ds
    }

def crawl(index_url):

    count = 0
    prev_url = index_url

    while 1:

        # crawl and parse the index page

        logging.info('Crawl the index page {} ...'.format(prev_url))
        try:
            text = read_or_request(prev_url)
        except OSError:
            # try again
            logging.info('Try again ...')
            text = read_or_request(prev_url)

        logging.info('Parse the index page {} ...'.format(prev_url))
        parsed_index_d = parse_index_page(text)

        prev_url = parsed_index_d['prev_url']

        # crawl the article_url

        for entry_d in parsed_index_d['entry_ds']:

            article_url = entry_d['article_url']
            # if deleted article
            if not article_url:
                continue

            # skip non-news
            if not entry_d['title'].startswith('[新聞]'):
                continue

            logging.info('Crawl the article page {} ...'.format(article_url))
            try:
                read_or_request(article_url)
            except OSError:
                # try again
                logging.info('Try again ...')
                try:
                    read_or_request(article_url)
                except OSError:
                    # skip if still fail
                    logging.info('Skip')
                    continue

            count += 1

            logging.info('Sleep')
            sleep(randint(0, 10)*0.001)

        logging.info('Got {:,} articles so far'.format(count))

if __name__ == '__main__':

    from pprint import pprint

    crawl('https://www.ptt.cc/bbs/Gossiping/index.html')

    import sys
    sys.exit()

    pprint(parse_article_page(read_or_request(
        'https://www.ptt.cc/bbs/Gossiping/M.1480355255.A.07C.html'
    )))

    import sys
    sys.exit()

    # test the index page
    pprint(parse_index_page(read_or_request(
        'https://www.ptt.cc/bbs/Gossiping/index20177.html'
    )))

    # test the article page #1
    pprint(parse_article_page(read_or_request(
        'https://www.ptt.cc/bbs/Gossiping/M.1480367106.A.A55.html'
    )))

    # test the article page #2
    pprint(parse_article_page(read_or_request(
        'https://www.ptt.cc/bbs/Gossiping/M.1480380251.A.9A4.html'
    )))
<|endoftext|>


<commit_before># Copyright (c) 2018, Djaodjin Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""
PEP 386-compliant version number for the multitier django app.
"""

__version__ = '0.1.4'
<commit_msg>releases version 0.1.5 to pypi<commit_after># Copyright (c) 2018, Djaodjin Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""
PEP 386-compliant version number for the multitier django app.
"""

__version__ = '0.1.5'
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8
###############################################################################
import os
import sys

import numpy as np

from cro import CRO

def test_croCreation():
    """
    Test that the REEF has a proper shape
    """
    L = 8
    N = 2
    M = 2
    fitness_coral = lambda coral: 1 # Dummy fitness
    cro = CRO(Ngen=10, N=N, M=M, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=L)
    (REEF, REEFpob) = cro.reefinitialization()
    assert REEFpob.shape == (N*M, L)

def test_croInit():
    """
    Test that the number of corals in the reef is greater than 0
    """
    L = 8
    N = 2
    M = 2
    fitness_coral = lambda coral: 1 # Dummy fitness
    cro = CRO(Ngen=10, N=N, M=M, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=L)
    (REEF, REEFpob) = cro.reefinitialization()
    assert len(np.where(REEF!=0)[0]) > 0

def test_reefinitializationDisc():
    """
    Test that reefinitialization function works in a discrete mode option, any coral lower than required min value and
    any higher than max
    """
    L = 8
    N = 2
    M = 2
    grid = {'x': [2, 10]}      # Discrete values between 2 and 10
    fitness_coral = lambda coral: 1 # Dummy fitness

    cro = CRO(Ngen=10, N=N, M=M, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=L, mode='disc', param_grid=grid)
    (REEF, REEFpob) = cro.reefinitialization()
    p = sum(REEFpob[np.where(REEFpob!=0)]<grid['x'][0]) + sum(REEFpob[np.where(REEFpob!=0)]>grid['x'][1])
    assert p == 0

def test_larvaesettling_emptyreef():
    """
    """
    REEFpob = np.array([[0,0,0,0],
                        [0,0,0,0],
                        [0,0,0,0],
                        [0,0,0,0]])

    REEF = np.array([0,0,0,0])
    REEFfitness = np.array([0,0,0,0])

    larvae = np.array([[1,0,0,0],
                       [0,1,0,0],
                       [0,0,1,0],
                       [0,0,0,1]])

    larvaefitness = np.array([1,1,1,1])

    N, L = REEFpob.shape
    M = 1
    fitness_coral = lambda coral: 1 # Dummy fitness
    cro = CRO(Ngen=10, N=2, M=2, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=L)

    REEF_res, REEFpob_res, REEFfitness_res = cro.larvaesettling(REEF, REEFpob, REEFfitness,
                                                                larvae, larvaefitness)

    np.testing.assert_almost_equal(REEF_res, np.array([1,1,1,1]))
    np.testing.assert_almost_equal(REEFpob_res, larvae)
    np.testing.assert_almost_equal(REEFfitness_res, larvaefitness)

def test_larvaesettling_nonemptyreef():
    """
    """
    REEFpob = np.array([[0,0,0,0],
                        [0,0,0,1],
                        [0,0,1,0],
                        [1,0,1,1]])

    REEF = np.array([0,1,1,1])
    REEFfitness = -np.array([0,1,2,11])

    larvae = np.array([[1,0,0,0],
                       [0,1,1,0],
                       [0,1,0,0],
                       [1,0,0,1]])

    larvaefitness = -np.array([8,6,4,9])

    N, L = REEFpob.shape
    M = 1
    fitness_coral = lambda coral: 1 # Dummy fitness
    cro = CRO(Ngen=10, N=2, M=2, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=L, seed=0)

    REEF_res, REEFpob_res, REEFfitness_res = cro.larvaesettling(REEF, REEFpob, REEFfitness,
                                                                larvae, larvaefitness)
    """
    Due to the passed seed,
    [1,0,0,0] will be placed in the empty coral (index 0)

    Then, larva [0,1,1,0] will try to settle in indices [0,3,1], settling in the third try (index 1)
    Larva [0,1,0,0] will try in indices [0,3,3], being discarded
    Larva [1,0,0,1] will try in indices [3,3,1], settling in the third try (index 1)

    Thus, the whole REEF will be populated after the settling, with a population:
    [[1,0,0,0], and a fitness [[8,
     [1,0,0,1],                 9,
     [0,0,1,0],                 2,
     [1,0,1,1]]                 11]]
    """
    REEFpob_exp = np.array([[1,0,0,0],
                            [1,0,0,1],
                            [0,0,1,0],
                            [1,0,1,1]])
    REEFfitness_exp = -np.array([8,9,2,11])

    np.testing.assert_almost_equal(REEF_res, np.array([1,1,1,1]))
    np.testing.assert_almost_equal(REEFpob_res, REEFpob_exp)
    np.testing.assert_almost_equal(REEFfitness_res, REEFfitness_exp)

def test_larvaemutattion():
    """
    Test mutated larvae in a given position, discrete mode
    """

    larvae = np.array([[2, 4, 4, 9, 10, 8, 3, 9],
                       [2, 9, 6, 7, 6, 5, 8, 3],
                       [3, 7, 8, 10, 6, 7, 8, 8]])

    pos = np.array([[0, 3, 5]])

    fitness_coral = lambda coral: 1 # Dummy fitness
    cro = CRO(Ngen=10, N=2, M=2, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=10, seed=0)

    larvaemutated = cro._larvaemutation(larvae, pos)
    goodsol = np.array([[3, 4, 4, 9, 10, 8, 3, 9],
                        [2, 9, 6, 6, 6, 5, 8, 3],
                        [3, 7, 8, 10, 6, 6, 8, 8]])

    np.testing.assert_almost_equal(larvaemutated, goodsol)

<commit_msg>update not grid and mode in test.py<commit_after>#!/usr/bin/env python
# coding=utf-8
###############################################################################
import os
import sys

import numpy as np

from cro import CRO

def test_croCreation():
    """
    Test that the REEF has a proper shape
    """
    L = 8
    N = 2
    M = 2
    fitness_coral = lambda coral: 1 # Dummy fitness
    cro = CRO(Ngen=10, N=N, M=M, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=L)
    (REEF, REEFpob) = cro.reefinitialization()
    assert REEFpob.shape == (N*M, L)

def test_croInit():
    """
    Test that the number of corals in the reef is greater than 0
    """
    L = 8
    N = 2
    M = 2
    fitness_coral = lambda coral: 1 # Dummy fitness
    cro = CRO(Ngen=10, N=N, M=M, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=L)
    (REEF, REEFpob) = cro.reefinitialization()
    assert len(np.where(REEF!=0)[0]) > 0

def test_reefinitializationDisc():
    """
    Test that reefinitialization function works in a discrete mode option, any coral lower than required min value and
    any higher than max
    """
    L = 8
    N = 2
    M = 2
    grid = {'x': [2, 10]}      # Discrete values between 2 and 10
    fitness_coral = lambda coral: 1 # Dummy fitness

    cro = CRO(Ngen=10, N=N, M=M, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=L, mode='disc', param_grid=grid)
    (REEF, REEFpob) = cro.reefinitialization()
    p = sum(REEFpob[np.where(REEFpob!=0)]<grid['x'][0]) + sum(REEFpob[np.where(REEFpob!=0)]>grid['x'][1])
    assert p == 0

def test_larvaesettling_emptyreef():
    """
    """
    REEFpob = np.array([[0,0,0,0],
                        [0,0,0,0],
                        [0,0,0,0],
                        [0,0,0,0]])

    REEF = np.array([0,0,0,0])
    REEFfitness = np.array([0,0,0,0])

    larvae = np.array([[1,0,0,0],
                       [0,1,0,0],
                       [0,0,1,0],
                       [0,0,0,1]])

    larvaefitness = np.array([1,1,1,1])

    N, L = REEFpob.shape
    M = 1
    fitness_coral = lambda coral: 1 # Dummy fitness
    cro = CRO(Ngen=10, N=2, M=2, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=L)

    REEF_res, REEFpob_res, REEFfitness_res = cro.larvaesettling(REEF, REEFpob, REEFfitness,
                                                                larvae, larvaefitness)

    np.testing.assert_almost_equal(REEF_res, np.array([1,1,1,1]))
    np.testing.assert_almost_equal(REEFpob_res, larvae)
    np.testing.assert_almost_equal(REEFfitness_res, larvaefitness)

def test_larvaesettling_nonemptyreef():
    """
    """
    REEFpob = np.array([[0,0,0,0],
                        [0,0,0,1],
                        [0,0,1,0],
                        [1,0,1,1]])

    REEF = np.array([0,1,1,1])
    REEFfitness = -np.array([0,1,2,11])

    larvae = np.array([[1,0,0,0],
                       [0,1,1,0],
                       [0,1,0,0],
                       [1,0,0,1]])

    larvaefitness = -np.array([8,6,4,9])

    N, L = REEFpob.shape
    M = 1
    fitness_coral = lambda coral: 1 # Dummy fitness
    cro = CRO(Ngen=10, N=2, M=2, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=L, seed=0)

    REEF_res, REEFpob_res, REEFfitness_res = cro.larvaesettling(REEF, REEFpob, REEFfitness,
                                                                larvae, larvaefitness)
    """
    Due to the passed seed,
    [1,0,0,0] will be placed in the empty coral (index 0)

    Then, larva [0,1,1,0] will try to settle in indices [0,3,1], settling in the third try (index 1)
    Larva [0,1,0,0] will try in indices [0,3,3], being discarded
    Larva [1,0,0,1] will try in indices [3,3,1], settling in the third try (index 1)

    Thus, the whole REEF will be populated after the settling, with a population:
    [[1,0,0,0], and a fitness [[8,
     [1,0,0,1],                 9,
     [0,0,1,0],                 2,
     [1,0,1,1]]                 11]]
    """
    REEFpob_exp = np.array([[1,0,0,0],
                            [1,0,0,1],
                            [0,0,1,0],
                            [1,0,1,1]])
    REEFfitness_exp = -np.array([8,9,2,11])

    np.testing.assert_almost_equal(REEF_res, np.array([1,1,1,1]))
    np.testing.assert_almost_equal(REEFpob_res, REEFpob_exp)
    np.testing.assert_almost_equal(REEFfitness_res, REEFfitness_exp)

def test_larvaemutattion():
    """
    Test mutated larvae in a given position, discrete mode
    """

    larvae = np.array([[2, 4, 4, 9, 10, 8, 3, 9],
                       [2, 9, 6, 7, 6, 5, 8, 3],
                       [3, 7, 8, 10, 6, 7, 8, 8]])

    pos = np.array([[0, 3, 5]])
    mode = 'disc'
    grid = {'x': [2, 10]}      # Discrete values between 2 and 10

    fitness_coral = lambda coral: 1 # Dummy fitness
    cro = CRO(Ngen=10, N=2, M=2, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=10, seed=0, mode=mode, param_grid=grid)

    larvaemutated = cro._larvaemutation(larvae, pos)
    goodsol = np.array([[3, 4, 4, 9, 10, 8, 3, 9],
                        [2, 9, 6, 6, 6, 5, 8, 3],
                        [3, 7, 8, 10, 6, 6, 8, 8]])

    np.testing.assert_almost_equal(larvaemutated, goodsol)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2017 Mycroft AI Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from dateutil.tz import tzlocal
from difflib import SequenceMatcher
import mycroft.util.time

from mycroft.util.lang.parse_en import *
from mycroft.util.lang.parse_pt import *
from mycroft.util.lang.parse_es import *
from mycroft.util.lang.parse_it import *
from mycroft.util.lang.parse_sv import *
from mycroft.util.lang.parse_de import extractnumber_de
from mycroft.util.lang.parse_de import extract_datetime_de
from mycroft.util.lang.parse_de import normalize_de
from mycroft.util.lang.parse_fr import extractnumber_fr
from mycroft.util.lang.parse_fr import extract_datetime_fr
from mycroft.util.lang.parse_fr import normalize_fr

from mycroft.util.lang.parse_common import *

def fuzzy_match(x, against):
    """Perform a 'fuzzy' comparison between two strings.
    Returns:
        float: match percentage -- 1.0 for perfect match,
               down to 0.0 for no match at all.
    """
    return SequenceMatcher(None, x, against).ratio()

def match_one(query, choices):
    """
        Find best match from a list or dictionary given an input

        Arguments:
            query:   string to test
            choices: list or dictionary of choices

        Returns: tuple with best match, score
    """
    if isinstance(choices, dict):
        _choices = list(choices.keys())
    elif isinstance(choices, list):
        _choices = choices
    else:
        raise ValueError('a list or dict of choices must be provided')

    best = (_choices[0], fuzzy_match(query, _choices[0]))
    for c in _choices[1:]:
        score = fuzzy_match(query, c)
        if score > best[1]:
            best = (c, score)

    if isinstance(choices, dict):
        return (choices[best[0]], best[1])
    else:
        return best

# TODO:18.08
def extractnumber(text, short_scale=True, ordinals=False, lang="en-us"):
    """ Depreciated, replaced by extract_number. Will be removed
    in the 18.08b release.

    """
    return extract_number(text, short_scale, ordinals, lang)

def extract_number(text, short_scale=True, ordinals=False, lang="en-us"):
    """Takes in a string and extracts a number.

    Args:
        text (str): the string to extract a number from
        short_scale (bool): Use "short scale" or "long scale" for large
            numbers -- over a million.  The default is short scale, which
            is now common in most English speaking countries.
            See https://en.wikipedia.org/wiki/Names_of_large_numbers
        ordinals (bool): consider ordinal numbers, e.g. third=3 instead of 1/3
        lang (str): the BCP-47 code for the language to use
    Returns:
        (int, float or False): The number extracted or False if the input
                               text contains no numbers
    """
    lang_lower = str(lang).lower()
    if lang_lower.startswith("en"):
        return extractnumber_en(text, short_scale=short_scale,
                                ordinals=ordinals)
    elif lang_lower.startswith("pt"):
        return extractnumber_pt(text)
    elif lang_lower.startswith("it"):
        return extractnumber_it(text)
    elif lang_lower.startswith("fr"):
        return extractnumber_fr(text)
    elif lang_lower.startswith("sv"):
        return extractnumber_sv(text)
    elif lang_lower.startswith("de"):
        return extractnumber_de(text)
    # TODO: extractnumber for other languages
    return text

def extract_datetime(text, anchorDate=None, lang="en-us"):
    """
    Extracts date and time information from a sentence.  Parses many of the
    common ways that humans express dates and times, including relative dates
    like "5 days from today", "tomorrow', and "Tuesday".

    Vague terminology are given arbitrary values, like:
        - morning = 8 AM
        - afternoon = 3 PM
        - evening = 7 PM

    If a time isn't supplied or implied, the function defaults to 12 AM

    Args:
        text (str): the text to be interpreted
        anchorDate (:obj:`datetime`, optional): the date to be used for
            relative dating (for example, what does "tomorrow" mean?).
            Defaults to the current local date/time.
        lang (string): the BCP-47 code for the language to use

    Returns:
        [:obj:`datetime`, :obj:`str`]: 'datetime' is the extracted date
            as a datetime object in the user's local timezone.
            'leftover_string' is the original phrase with all date and time
            related keywords stripped out. See examples for further
            clarification

            Returns 'None' if the input string is empty.

    Examples:

        >>> extract_datetime(
        ... "What is the weather like the day after tomorrow?",
        ... datetime(2017, 06, 30, 00, 00)
        ... )
        [datetime.datetime(2017, 7, 2, 0, 0), 'what is weather like']

        >>> extract_datetime(
        ... "Set up an appointment 2 weeks from Sunday at 5 pm",
        ... datetime(2016, 02, 19, 00, 00)
        ... )
        [datetime.datetime(2016, 3, 6, 17, 0), 'set up appointment']
    """

    lang_lower = str(lang).lower()

    if not anchorDate:
        anchorDate = now_local()

    if lang_lower.startswith("en"):
        return extract_datetime_en(text, anchorDate)
    elif lang_lower.startswith("pt"):
        return extract_datetime_pt(text, anchorDate)
    elif lang_lower.startswith("it"):
        return extract_datetime_it(text, anchorDate)
    elif lang_lower.startswith("fr"):
        return extract_datetime_fr(text, anchorDate)
    elif lang_lower.startswith("sv"):
        return extract_datetime_sv(text, anchorDate)
    elif lang_lower.startswith("de"):
        return extract_datetime_de(text, anchorDate)
    # TODO: extract_datetime for other languages
    return text
# ==============================================================

def normalize(text, lang="en-us", remove_articles=True):
    """Prepare a string for parsing

    This function prepares the given text for parsing by making
    numbers consistent, getting rid of contractions, etc.
    Args:
        text (str): the string to normalize
        lang (str): the code for the language text is in
        remove_articles (bool): whether to remove articles (like 'a', or
                                'the'). True by default.
    Returns:
        (str): The normalized string.
    """

    lang_lower = str(lang).lower()
    if lang_lower.startswith("en"):
        return normalize_en(text, remove_articles)
    elif lang_lower.startswith("es"):
        return normalize_es(text, remove_articles)
    elif lang_lower.startswith("pt"):
        return normalize_pt(text, remove_articles)
    elif lang_lower.startswith("it"):
        return normalize_it(text, remove_articles)
    elif lang_lower.startswith("fr"):
        return normalize_fr(text, remove_articles)
    elif lang_lower.startswith("sv"):
        return normalize_sv(text, remove_articles)
    elif lang_lower.startswith("de"):
        return normalize_de(text, remove_articles)
    # TODO: Normalization for other languages
    return text

def get_gender(word, input_string="", lang="en-us"):
    '''
    guess gender of word, optionally use raw input text for context
    returns "m" if the word is male, "f" if female, False if unknown
    '''
    if "pt" in lang or "es" in lang:
        # spanish follows same rules
        return get_gender_pt(word, input_string)
    elif "it" in lang:
        return get_gender_it(word, input_string)
    return False
<commit_msg>Silence Codacy complaints<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2017 Mycroft AI Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from difflib import SequenceMatcher
from mycroft.util.time import now_local

from mycroft.util.lang.parse_en import *
from mycroft.util.lang.parse_pt import *
from mycroft.util.lang.parse_es import *
from mycroft.util.lang.parse_it import *
from mycroft.util.lang.parse_sv import *
from mycroft.util.lang.parse_de import extractnumber_de
from mycroft.util.lang.parse_de import extract_datetime_de
from mycroft.util.lang.parse_de import normalize_de
from mycroft.util.lang.parse_fr import extractnumber_fr
from mycroft.util.lang.parse_fr import extract_datetime_fr
from mycroft.util.lang.parse_fr import normalize_fr

from mycroft.util.lang.parse_common import *

def fuzzy_match(x, against):
    """Perform a 'fuzzy' comparison between two strings.
    Returns:
        float: match percentage -- 1.0 for perfect match,
               down to 0.0 for no match at all.
    """
    return SequenceMatcher(None, x, against).ratio()

def match_one(query, choices):
    """
        Find best match from a list or dictionary given an input

        Arguments:
            query:   string to test
            choices: list or dictionary of choices

        Returns: tuple with best match, score
    """
    if isinstance(choices, dict):
        _choices = list(choices.keys())
    elif isinstance(choices, list):
        _choices = choices
    else:
        raise ValueError('a list or dict of choices must be provided')

    best = (_choices[0], fuzzy_match(query, _choices[0]))
    for c in _choices[1:]:
        score = fuzzy_match(query, c)
        if score > best[1]:
            best = (c, score)

    if isinstance(choices, dict):
        return (choices[best[0]], best[1])
    else:
        return best

# TODO:18.08
def extractnumber(text, short_scale=True, ordinals=False, lang="en-us"):
    """ Depreciated, replaced by extract_number. Will be removed
    in the 18.08b release.

    """
    return extract_number(text, short_scale, ordinals, lang)

def extract_number(text, short_scale=True, ordinals=False, lang="en-us"):
    """Takes in a string and extracts a number.

    Args:
        text (str): the string to extract a number from
        short_scale (bool): Use "short scale" or "long scale" for large
            numbers -- over a million.  The default is short scale, which
            is now common in most English speaking countries.
            See https://en.wikipedia.org/wiki/Names_of_large_numbers
        ordinals (bool): consider ordinal numbers, e.g. third=3 instead of 1/3
        lang (str): the BCP-47 code for the language to use
    Returns:
        (int, float or False): The number extracted or False if the input
                               text contains no numbers
    """
    lang_lower = str(lang).lower()
    if lang_lower.startswith("en"):
        return extractnumber_en(text, short_scale=short_scale,
                                ordinals=ordinals)
    elif lang_lower.startswith("pt"):
        return extractnumber_pt(text)
    elif lang_lower.startswith("it"):
        return extractnumber_it(text)
    elif lang_lower.startswith("fr"):
        return extractnumber_fr(text)
    elif lang_lower.startswith("sv"):
        return extractnumber_sv(text)
    elif lang_lower.startswith("de"):
        return extractnumber_de(text)
    # TODO: extractnumber for other languages
    return text

def extract_datetime(text, anchorDate=None, lang="en-us"):
    """
    Extracts date and time information from a sentence.  Parses many of the
    common ways that humans express dates and times, including relative dates
    like "5 days from today", "tomorrow', and "Tuesday".

    Vague terminology are given arbitrary values, like:
        - morning = 8 AM
        - afternoon = 3 PM
        - evening = 7 PM

    If a time isn't supplied or implied, the function defaults to 12 AM

    Args:
        text (str): the text to be interpreted
        anchorDate (:obj:`datetime`, optional): the date to be used for
            relative dating (for example, what does "tomorrow" mean?).
            Defaults to the current local date/time.
        lang (string): the BCP-47 code for the language to use

    Returns:
        [:obj:`datetime`, :obj:`str`]: 'datetime' is the extracted date
            as a datetime object in the user's local timezone.
            'leftover_string' is the original phrase with all date and time
            related keywords stripped out. See examples for further
            clarification

            Returns 'None' if the input string is empty.

    Examples:

        >>> extract_datetime(
        ... "What is the weather like the day after tomorrow?",
        ... datetime(2017, 06, 30, 00, 00)
        ... )
        [datetime.datetime(2017, 7, 2, 0, 0), 'what is weather like']

        >>> extract_datetime(
        ... "Set up an appointment 2 weeks from Sunday at 5 pm",
        ... datetime(2016, 02, 19, 00, 00)
        ... )
        [datetime.datetime(2016, 3, 6, 17, 0), 'set up appointment']
    """

    lang_lower = str(lang).lower()

    if not anchorDate:
        anchorDate = now_local()

    if lang_lower.startswith("en"):
        return extract_datetime_en(text, anchorDate)
    elif lang_lower.startswith("pt"):
        return extract_datetime_pt(text, anchorDate)
    elif lang_lower.startswith("it"):
        return extract_datetime_it(text, anchorDate)
    elif lang_lower.startswith("fr"):
        return extract_datetime_fr(text, anchorDate)
    elif lang_lower.startswith("sv"):
        return extract_datetime_sv(text, anchorDate)
    elif lang_lower.startswith("de"):
        return extract_datetime_de(text, anchorDate)
    # TODO: extract_datetime for other languages
    return text
# ==============================================================

def normalize(text, lang="en-us", remove_articles=True):
    """Prepare a string for parsing

    This function prepares the given text for parsing by making
    numbers consistent, getting rid of contractions, etc.
    Args:
        text (str): the string to normalize
        lang (str): the code for the language text is in
        remove_articles (bool): whether to remove articles (like 'a', or
                                'the'). True by default.
    Returns:
        (str): The normalized string.
    """

    lang_lower = str(lang).lower()
    if lang_lower.startswith("en"):
        return normalize_en(text, remove_articles)
    elif lang_lower.startswith("es"):
        return normalize_es(text, remove_articles)
    elif lang_lower.startswith("pt"):
        return normalize_pt(text, remove_articles)
    elif lang_lower.startswith("it"):
        return normalize_it(text, remove_articles)
    elif lang_lower.startswith("fr"):
        return normalize_fr(text, remove_articles)
    elif lang_lower.startswith("sv"):
        return normalize_sv(text, remove_articles)
    elif lang_lower.startswith("de"):
        return normalize_de(text, remove_articles)
    # TODO: Normalization for other languages
    return text

def get_gender(word, input_string="", lang="en-us"):
    '''
    guess gender of word, optionally use raw input text for context
    returns "m" if the word is male, "f" if female, False if unknown
    '''
    if "pt" in lang or "es" in lang:
        # spanish follows same rules
        return get_gender_pt(word, input_string)
    elif "it" in lang:
        return get_gender_it(word, input_string)
    return False
<|endoftext|>


<commit_before>#
# This file is part of my.gpodder.org.
#
# my.gpodder.org is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# my.gpodder.org is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with my.gpodder.org. If not, see <http://www.gnu.org/licenses/>.
#

from mygpo.api.basic_auth import require_valid_user
from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, Http404, HttpResponseNotAllowed
from mygpo.api.models import Device, Podcast, SubscriptionAction, Episode, EpisodeAction, SUBSCRIBE_ACTION, UNSUBSCRIBE_ACTION, EPISODE_ACTION_TYPES, DEVICE_TYPES
from mygpo.api.httpresponse import JsonResponse
from django.core import serializers
from time import mktime
from datetime import datetime
import dateutil.parser

try:
    #try to import the JSON module (if we are on Python 2.6)
    import json
except ImportError:
    # No JSON module available - fallback to simplejson (Python < 2.6)
    print "No JSON module available - fallback to simplejson (Python < 2.6)"
    import simplejson as json

print dir(json)

@require_valid_user()
def subscriptions(request, username, device_uid):

    if request.user.username != username:
        return HttpResponseForbidden()

    now = mktime(datetime.now().timetuple())

    if request.method == 'GET':
        try:
            d = Device.objects.get(user=request.user, uid=device_uid)
        except Device.DoesNotExist:
            return Http404('device %s does not exist' % device_uid)

        try:
            since = request.GET['since']
        except KeyError:
            return HttpResponseBadRequest('parameter since missing')

        changes = get_subscription_changes(user, device, since, now)

        return JsonResponse(changes)

    elif request.method == 'POST':
        d = Device.objects.get_or_create(user=request.user, uid=device_uid)

        actions = json.loads(request.POST['data'])
        add = actions['add'] if 'add' in actions else []
        rem = actions['remove'] if 'remove' in actions else []

        update_subscriptions(request.user, d, add, rem)

        return JsonResponse({'timestamp': now})

    else:
        return HttpResponseNotAllowed(['GET', 'POST'])

def update_subscriptions(user, device, add, remove):
    for a in add:
        p = Podcast.objects.get_or_create(url=a)
        s = SubscriptionAction.objects.create(podcast=p,device=device,action=SUBSCRIBE_ACTION)

    for r in remove:
        p = Podcast.objects.get_or_create(url=r)
        s = SubscriptionAction.objects.create(podcast=p,device=device,action=UNSUBSCRIBE_ACTION)

def get_subscription_changes(user, device, since, until):
    actions = {}
    for a in SubscriptionAction.objects.filter(device=device, timestamp__gt=since, timestamp__lt=until).order_by('timestamp'):
        #ordered by ascending date; newer entries overwriter older ones
        actions[a.podcast] = a

    add = []
    remove = []

    for a in actions:
        if a.action == SUBSCRIBE_ACTION:
            add.append(a.podcast.url)
        elif a.action == UNSUBSCRIBE_ACTION:
            remove.append(a.podcast.url)

    return {'add': add, 'remove': remove, 'timestamp': until}

@require_valid_user()
def episodes(request, username):

    if request.user.username != username:
        return HttpResponseForbidden()

    now = mktime(datetime.now().timetuple())

    if request.method == 'POST':
        try:
            actions = json.loads(request.POST['data'])
        except KeyError:
            return HttpResponseBadRequest()

        update_episodes(request.user, actions)
        return HttpResponse()

    elif request.method == 'GET':
        podcast_id = request.GET.get('podcast', None)
        device_id  = request.GET.get('device', None)
        since      = request.GET.get('since', None)

        try:
            podcast = Podcast.objects.get(pk=podcast_id) if podcast_id else None
            device  = Device.objects.get(pk=device_id) if device_id else None
        except:
            return Http404

        return JsonRequest(get_episode_changes(request.user, podcast, device, since, now))

    else:
        return HttpResponseNotAllowed(['POST', 'GET'])

def get_episode_changes(user, podcast, device, since, until):
    actions = []
    for a in EpisodeAction.objects.filter(user=user, podcast=podcast, device=device, timestamp__gt=since, timestamp__lt=until):
        action = {
            'podcast': a.episode.podcast.url,
            'episode': a.episode.url,
            'action':  a.action,
            'timestamp': a.timestamp
        }

        if a.action == 'play': action['time'] = a.playmark

        actions.append(action)

    return {'timestamp': since, 'actions': actions}

def update_episodes(user, actions):
    for e in actions:
        try:
            podcast = Podcast.objects.get(url=e['podcast'])
            episode = Episode.objects.get(podcast=podcast, url=e['episode'])
            action  = e['action']
            if not action in EPISODE_ACTION_TYPES:
                return HttpResponseBadRequest('invalid action %s' % action)
        except:
            return HttpResponseBadRequest('not all required fields (podcast, episode, action) given')

        device = Device.objects.get_or_create(user=user, uid=e['device'], defaults={'name': 'Unknown', 'type': 'other'}) if 'device' in e else None
        timestamp = dateutil.parser.parse(e['timestamp']) if 'timestamp' in e else None
        position = datetime.strptime(e['position'], '%H:%M:%S').time() if 'position' in e else None

        if position and action != 'play':
            return HttpResponseBadRequest('parameter position can only be used with action play')

        EpisodeAction.objects.create(user=user, episode=episode, device=device, action=action, timestamp=timestamp, playmark=position)

@require_valid_user()
def device(request, username, device_uid):

    if request.user.username != username:
        return HttpResponseForbidden()

    if request.method == 'POST':
        d = Device.objects.get_or_create(user=request.user, uid=device_uid)

        data = json.loads(request.POST['data'])

        if 'caption' in data:
            d.name = data['caption']

        if 'type' in data:
            if not data['type'] in DEVICE_TYPES:
                return HttpResponseBadRequest('invalid device type %s' % data['type'])
            d.type = data['type']

        d.save()

        return HttpResponse()

    else:
        return HttpResponseNotAllowed(['POST'])

@require_valid_user()
def devices(request, username):

    if request.user.username != username:
        return HttpResponseForbidden()

    if request.method == 'GET':
        devices = []
        for d in Device.objects.filter(user=request.user):
            devices.append({
                'id': d.uid,
                'caption': d.name,
                'type': d.type,
                'subscriptions': Subscription.objects.filter(device=d).count()
            })

        return JsonResponse(devices)

    else:
        return HttpResponseNotAllowed(['GET'])

<commit_msg>bugfixing advanced api<commit_after>#
# This file is part of my.gpodder.org.
#
# my.gpodder.org is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# my.gpodder.org is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with my.gpodder.org. If not, see <http://www.gnu.org/licenses/>.
#

from mygpo.api.basic_auth import require_valid_user
from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, Http404, HttpResponseNotAllowed
from mygpo.api.models import Device, Podcast, SubscriptionAction, Episode, EpisodeAction, SUBSCRIBE_ACTION, UNSUBSCRIBE_ACTION, EPISODE_ACTION_TYPES, DEVICE_TYPES
from mygpo.api.httpresponse import JsonResponse
from django.core import serializers
from time import mktime
from datetime import datetime
import dateutil.parser

try:
    #try to import the JSON module (if we are on Python 2.6)
    import json
except ImportError:
    # No JSON module available - fallback to simplejson (Python < 2.6)
    print "No JSON module available - fallback to simplejson (Python < 2.6)"
    import simplejson as json

print dir(json)

@require_valid_user()
def subscriptions(request, username, device_uid):

    if request.user.username != username:
        return HttpResponseForbidden()

    now = mktime(datetime.now().timetuple())

    if request.method == 'GET':
        try:
            d = Device.objects.get(user=request.user, uid=device_uid)
        except Device.DoesNotExist:
            return Http404('device %s does not exist' % device_uid)

        try:
            since = request.GET['since']
        except KeyError:
            return HttpResponseBadRequest('parameter since missing')

        changes = get_subscription_changes(user, device, since, now)

        return JsonResponse(changes)

    elif request.method == 'POST':
        d, created = Device.objects.get_or_create(user=request.user, uid=device_uid)

        actions = json.loads(request.POST['data'])
        add = actions['add'] if 'add' in actions else []
        rem = actions['remove'] if 'remove' in actions else []

        for a in add:
            if a in rem:
                return HttpResponseBadRequest('can not add and remove %s at the same time' % a)

        update_subscriptions(request.user, d, add, rem)

        return JsonResponse({'timestamp': now})

    else:
        return HttpResponseNotAllowed(['GET', 'POST'])

def update_subscriptions(user, device, add, remove):
    for a in add:
        p, p_created = Podcast.objects.get_or_create(url=a)
        s = SubscriptionAction.objects.create(podcast=p,device=device,action=SUBSCRIBE_ACTION)

    for r in remove:
        p, p_created = Podcast.objects.get_or_create(url=r)
        s = SubscriptionAction.objects.create(podcast=p,device=device,action=UNSUBSCRIBE_ACTION)

def get_subscription_changes(user, device, since, until):
    actions = {}
    for a in SubscriptionAction.objects.filter(device=device, timestamp__gt=since, timestamp__lt=until).order_by('timestamp'):
        #ordered by ascending date; newer entries overwriter older ones
        actions[a.podcast] = a

    add = []
    remove = []

    for a in actions:
        if a.action == SUBSCRIBE_ACTION:
            add.append(a.podcast.url)
        elif a.action == UNSUBSCRIBE_ACTION:
            remove.append(a.podcast.url)

    return {'add': add, 'remove': remove, 'timestamp': until}

@require_valid_user()
def episodes(request, username):

    if request.user.username != username:
        return HttpResponseForbidden()

    now = mktime(datetime.now().timetuple())

    if request.method == 'POST':
        try:
            actions = json.loads(request.POST['data'])
        except KeyError:
            return HttpResponseBadRequest()

        update_episodes(request.user, actions)
        return HttpResponse()

    elif request.method == 'GET':
        podcast_id = request.GET.get('podcast', None)
        device_id  = request.GET.get('device', None)
        since      = request.GET.get('since', None)

        try:
            podcast = Podcast.objects.get(pk=podcast_id) if podcast_id else None
            device  = Device.objects.get(pk=device_id) if device_id else None
        except:
            return Http404

        return JsonRequest(get_episode_changes(request.user, podcast, device, since, now))

    else:
        return HttpResponseNotAllowed(['POST', 'GET'])

def get_episode_changes(user, podcast, device, since, until):
    actions = []
    for a in EpisodeAction.objects.filter(user=user, podcast=podcast, device=device, timestamp__gt=since, timestamp__lt=until):
        action = {
            'podcast': a.episode.podcast.url,
            'episode': a.episode.url,
            'action':  a.action,
            'timestamp': a.timestamp
        }

        if a.action == 'play': action['time'] = a.playmark

        actions.append(action)

    return {'timestamp': since, 'actions': actions}

def update_episodes(user, actions):
    for e in actions:
        try:
            podcast = Podcast.objects.get(url=e['podcast'])
            episode = Episode.objects.get(podcast=podcast, url=e['episode'])
            action  = e['action']
            if not action in EPISODE_ACTION_TYPES:
                return HttpResponseBadRequest('invalid action %s' % action)
        except:
            return HttpResponseBadRequest('not all required fields (podcast, episode, action) given')

        device, created = Device.objects.get_or_create(user=user, uid=e['device'], defaults={'name': 'Unknown', 'type': 'other'}) if 'device' in e else None
        timestamp = dateutil.parser.parse(e['timestamp']) if 'timestamp' in e else None
        position = datetime.strptime(e['position'], '%H:%M:%S').time() if 'position' in e else None

        if position and action != 'play':
            return HttpResponseBadRequest('parameter position can only be used with action play')

        EpisodeAction.objects.create(user=user, episode=episode, device=device, action=action, timestamp=timestamp, playmark=position)

@require_valid_user()
def device(request, username, device_uid):

    if request.user.username != username:
        return HttpResponseForbidden()

    if request.method == 'POST':
        d, created = Device.objects.get_or_create(user=request.user, uid=device_uid)

        data = json.loads(request.POST['data'])

        if 'caption' in data:
            d.name = data['caption']

        if 'type' in data:
            if not data['type'] in DEVICE_TYPES:
                return HttpResponseBadRequest('invalid device type %s' % data['type'])
            d.type = data['type']

        d.save()

        return HttpResponse()

    else:
        return HttpResponseNotAllowed(['POST'])

@require_valid_user()
def devices(request, username):

    if request.user.username != username:
        return HttpResponseForbidden()

    if request.method == 'GET':
        devices = []
        for d in Device.objects.filter(user=request.user):
            devices.append({
                'id': d.uid,
                'caption': d.name,
                'type': d.type,
                'subscriptions': Subscription.objects.filter(device=d).count()
            })

        return JsonResponse(devices)

    else:
        return HttpResponseNotAllowed(['GET'])

<|endoftext|>


<commit_before>from django.core.exceptions import PermissionDenied
from django.http import HttpResponseRedirect
from django.shortcuts import render, get_object_or_404
from django.urls import reverse
from registration.forms import User
from .forms import IdeasForm, IdeasFormUpdate
from .models import Ideas

def home(request):
    queryset = Ideas.objects.all()
    context = {
        'ideas': queryset,
    }

    return render(request, 'index.html', context)

def idea_details(request, slug):
    ideas = get_object_or_404(Ideas, slug=slug)

    context = {
        'ideas': ideas,
    }

    return render(request, 'ideas_details.html', context)

def idea_create(request):
    form = IdeasForm(request.POST)
    if form.is_valid():
        idea = form.save(commit=False)
        idea.user = request.user
        idea.save()
        form.save_m2m()
        return HttpResponseRedirect('/')

    else:
        form = IdeasForm()

    context = {
        'form': form,
        'form_media': form.media,
    }

    return render(request, 'idea_form.html', context)

def idea_update(request, slug=None):
    instance = get_object_or_404(Ideas, slug=slug)
    form = IdeasFormUpdate(request.POST or None, instance=instance)
    if request.user == instance.user:
        if form.is_valid():
            form.save()
            return HttpResponseRedirect(instance.get_absolute_url())

        context = {
            'form': form,
        }
    else:
        raise PermissionDenied

    return render(request, 'update.html', context)

def idea_delete(request, slug=None):
    instance = get_object_or_404(Ideas, slug=slug)
    if request.user == instance.user:
        instance.delete()
        return HttpResponseRedirect(reverse('profile', args=(request.user.username,)))
    else:
        raise PermissionDenied

def by_tags(request, tags):
    queryset = Ideas.objects.filter(tags=tags)

    context = {
        'queryset': queryset
    }

    return render(request, 'by_tags.html', context)

def profile(request, username):
    user = get_object_or_404(User.objects, username=username)
    queryset = Ideas.objects.filter(user=user)

    context = {
        'username': user,
        'ideas': queryset,
    }

    return render(request, 'profile.html', context)
<commit_msg>refactoring  views<commit_after>from django.core.exceptions import PermissionDenied
from django.http import HttpResponseRedirect
from django.shortcuts import render, get_object_or_404
from django.urls import reverse
from registration.forms import User
from .forms import IdeasForm, IdeasFormUpdate
from .models import Ideas

def home(request):
    queryset = Ideas.objects.all()
    context = {
        'ideas': queryset,
    }

    return render(request, 'index.html', context)

def idea_details(request, slug):
    ideas = get_object_or_404(Ideas, slug=slug)

    context = {
        'ideas': ideas,
    }

    return render(request, 'ideas_details.html', context)

def idea_create(request):
    form = IdeasForm(request.POST)
    if form.is_valid():
        idea = form.save(commit=False)
        idea.user = request.user
        idea.save()
        form.save_m2m()
        return HttpResponseRedirect('/')

    else:
        form = IdeasForm()

    context = {
        'form': form,
        'form_media': form.media,
    }

    return render(request, 'idea_form.html', context)

def idea_update(request, slug=None):
    instance = get_object_or_404(Ideas, slug=slug)
    form = IdeasFormUpdate(request.POST or None, instance=instance)
    if request.user == instance.user:
        if form.is_valid():
            form.save()
            return HttpResponseRedirect(instance.get_absolute_url())

        context = {
            'form': form,
        }
    else:
        raise PermissionDenied

    return render(request, 'update.html', context)

def idea_delete(request, slug=None):
    instance = get_object_or_404(Ideas, slug=slug)
    if request.user == instance.user:
        instance.delete()
        return HttpResponseRedirect(reverse(
            'profile', args=(request.user.username,)
        ))
    else:
        raise PermissionDenied

def by_tags(request, tags):
    queryset = Ideas.objects.filter(tags=tags)

    context = {
        'queryset': queryset
    }

    return render(request, 'by_tags.html', context)

def profile(request, username):
    user = get_object_or_404(User.objects, username=username)
    queryset = Ideas.objects.filter(user=user)

    context = {
        'username': user,
        'ideas': queryset,
    }

    return render(request, 'profile.html', context)
<|endoftext|>