<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
rss_parser finds all information in an rss item
"""
# @author chengdujin
# @contact chengdujin@gmail.com
# @created Jan 2, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from BeautifulSoup import BeautifulStoneSoup
import calendar
import chardet
from config.settings import hparser
from config.settings import logger
from processor import illustrator
from datetime import datetime, timedelta
import warnings
warnings.filterwarnings(
    action="ignore", category=DeprecationWarning, module="feedparser")
import feedparser
import html2text
import re
import requests
import socket
socket.setdefaulttimeout(10)  # 10 seconds
import time
import urllib2

# CONSTANTS
from config.settings import HEADERS
from config.settings import LANGUAGES
from config.settings import DATABASE_REMOVAL_DAYS
# prefix should not end with a slash
HIDDEN_LINKS = {'http://news.goo.ne.jp':
                ('div', {'class': 'lead fs16 bold'}), 'http://news.nifty.com': ('li', {'class': 'headnews'})}
AD_LINKS = 'http://rss.rssad.jp/rss/ad/'

def _get_actual_link(prefix, link):
    """
    find the actual news link
    """
    if not prefix or not link:
        logger.error(
            'Method malformed! Prefix:[%s], Link:[%s]' % (prefix, link))

    try:
        actual_link = None
        raw_data = urllib2.urlopen(link)
        data = raw_data.readlines()
        # str() is critical
        soup = BeautifulStoneSoup(str(data))
        html_tag, html_attrs = HIDDEN_LINKS[prefix]
        html_wrapper = soup.find(name=html_tag, attrs=html_attrs)
        if html_wrapper:
            actual_suffix = html_wrapper.find('a')['href']
            actual_link = str('%s%s' % (prefix, actual_suffix))
            return actual_link
        else:
            return None
    except Exception as k:
        logger.error('Cannot open %s' % k)
        return None

def _read_entry(e=None, feed_id=None, feed_title=None, language=None, categories=None):
    """
    read a specific entry item from a feed
    Note. categories are ids of category item
    """
    if not e or not feed_title or not language or not categories:
        logger.error('Method malformed!')
        return None
    if language not in LANGUAGES:
        logger.error("Language not supported for %s!" % feed_title)
        return None

    try:
        entry = {}
        entry['feed_id'] = feed_id
        entry['feed'] = feed_title.strip()
        entry['language'] = language.strip()
        entry['categories'] = categories

        # the easy part: the must-have
        entry['error'] = []

        # article original link
        if e.link:
            original_link = e.link.strip()
            if not original_link.startswith(AD_LINKS):
                # clean the URL
                original_link = urllib2.unquote(
                    hparser.unescape(original_link))
                response = requests.get(original_link, headers=HEADERS)
                original_link = response.url

                # find the redirected link
                matched_prefix = [
                    link for link in HIDDEN_LINKS if original_link.startswith(link)]
                found_prefix = matched_prefix[0] if matched_prefix else None
                if found_prefix:
                    actual_link = _get_actual_link(found_prefix, original_link)
                    if actual_link:
                        entry['link'] = actual_link
                    else:
                        logger.error(
                            'No actual link found for %s!' % original_link)
                        return None
                else:
                    entry['link'] = original_link
            else:
                logger.info('Advertising link %s' % original_link)
                return None
        else:
            logger.info('Feed malformed! No link found!')
            return None

        # article title
        if e.title_detail.type != 'text/plain':
            entry['title'] = urllib2.unquote(hparser.unescape(e.title.strip()))
        elif 'title' in e:
            entry['title'] = e.title.strip()
        else:
            entry['title'] = None
        # remove possible htmlized title
        entry['title'] = re.sub("<.*?>", " ", entry[
                                'title']) if 'title' in entry and entry['title'] else None

        # article published time
        # first try parsed time info
        try:
            entry['updated'] = calendar.timegm(e.updated_parsed)
            entry['updated_human'] = e.updated
        except AttributeError as k:
            try:
                entry['updated'] = calendar.timegm(e.published_parsed)
                entry['updated_human'] = e.published
            except AttributeError as k:
                entry['error'] = ['%s\n%s' % (
                    entry['error'], "no 'updated_parsed' or 'published_parsed'")]
                # then try unparsed time info
                # this is rarely possible.
                try:
                    updated = e.updated if 'updated' in e else e.published
                    if updated:
                        # get time zone
                        offset = int(updated[-5:])
                        delta = timedelta(hours=int(offset) / 100)
                        format = "%a, %d %b %Y %H:%M:%S"
                        if updated[-8:-5] != 'UTC':
                            updated = datetime.strptime(updated[:-6], format)
                        else:
                            updated = datetime.strptime(updated[:-9], format)
                        updated -= delta
                        entry['updated'] = time.mktime(updated.timetuple())
                    else:
                        logger.info(
                            "Attribute updated/published has no value")
                        return None
                except ValueError as k:
                    logger.info(str(k))
                    entry['error'].append('%s\n%s' % (entry['error'], k))
                    return None
                except AttributeError as k:
                    logger.info(str(k))
                    entry['error'].append('no update or published\n')
                    return None

        # article's summary
        try:
            # its possible summary is html-based
            summary = urllib2.unquote(hparser.unescape(e.summary))
            if isinstance(summary, str):
                summary_encoding = chardet.detect(summary)['encoding']
                summary = summary.decode(summary_encoding, 'ignore')
            # a <div, for example, and a </div
            is_html = True if len(
                re.findall(u'</?a|</?p|</?strong|</?img|</?html|</?div', summary)) > 1 else False
            if is_html:
                h = html2text.HTML2Text()
                h.ignore_images = True
                h.ignore_links = True
                h.ignore_emphasis = True
                paragraphs = (h.handle(summary)).strip().strip(
                    '#').strip().split('\n\n')
                paragraphs_above_limit = []
                # remove paragraphs that contain less than x number of words
                for paragraph in paragraphs:
                    if entry['language'].startswith('zh') or entry['language'] == 'ja':
                        if len(paragraph) > 18:
                            paragraphs_above_limit.append(paragraph)
                    else:
                        words = paragraph.split()
                        if len(words) > 12:
                            paragraphs_above_limit.append(paragraph)
                entry['summary'] = '\n\n'.join(paragraphs_above_limit)
            else:
                entry['summary'] = summary
        except AttributeError as k:
            entry['summary'] = None
        entry['summary'] = None if not entry['summary'] else entry['summary']

        # article's images
        # e.g. [{'url':'http://image.com/test.jpg, 'width': u'130', 'height':
        # u'86'}]
        entry['images'] = []
        try:
            images, media_content_new = illustrator.find_images(
                e.media_content, entry['link'])
            if images:
                entry['images'].extend(images)
        except AttributeError as k:
            pass
        try:
            images, media_content_new = illustrator.find_images(
                e.media_thumbnail, entry['link'])
            if images:
                entry['images'].extend(images)
        except AttributeError as k:
            pass
        for attribute in e:
            if 'thumbnail' in attribute:
                # currently set thumbnail to None if its a dictionary
                image = e[attribute] if isinstance(e[attribute], str) else None
                image = illustrator.find_image(image, entry['link'])
                if image:
                    entry['images'].append(image)
        try:
            links = e.links
            for link in links:
                if 'type' in link and 'image' in link.type:
                    if 'href' in link:
                        image = illustrator.find_image(
                            link.href, entry['link'])
                        if image:
                            entry['images'].append(image)
        except AttributeError as k:
            pass

        if entry.has_key('summary') and entry['summary']:
            images, entry['summary'] = illustrator.find_images(
                entry['summary'], entry['link'])
            if images:
                entry['images'].extend(images)
        # dedup images is processed at rss.py

        # article's author
        # e.g. Yuan Jin
        try:
            # i guess this could be a string or a list
            entry['author'] = e.author
        except AttributeError as k:
            entry['author'] = None

        # article's source
        # e.g. {'href': u'http://www.reuters.com/', 'title': u'Reuters'}
        try:
            entry['source'] = e.source
        except AttributeError as k:
            entry['source'] = None

        # article's tags
        # e.g. [{'term': u'Campus Party', 'scheme': None, 'label': None}]
        # term is usually combined with scheme to form a url; label is
        # the name of term
        try:
            entry['tags'] = e.tag
        except AttributeError as k:
            entry['tags'] = None

        # the FINAL return
        return entry
    except Exception as k:
        logger.error(str(k))
        return None

# TODO: boundary checkers
# TODO: update parsing info to feed database
def parse(feed_link=None, feed_id=None, feed_title=None, language=None, categories=None, etag=None, modified=None):
    """
    read rss/atom data from a given feed
    feed_id is the feed ObjectId in MongoDB
    Etag and Modified are used to save rss http server's bandwidth
    Note: category should be added to feed table/database
    """
    if not feed_link or not feed_id or not language or not categories:
        logger.error("Method malformed!")
        return None, None, feed_title, etag, modified, "Method malformed!"
    if language not in LANGUAGES:
        logger.error("Language not supported for %s!" % feed_link)
        return None, None, feed_title, etag, modified, "Language not supported for %s!" % feed_link

    def _validate_time(entry):
        """
        see if the entry's updated time is earlier than needed
        """
        deadline = datetime.utcfromtimestamp(
            entry['updated']) + timedelta(days=DATABASE_REMOVAL_DAYS)
        return True if deadline > datetime.now() else False

    try:
        # variables d and e follow feedparser tradition
        feedparser.USER_AGENT = "newsman"
        d = feedparser.parse(feed_link, etag=etag, modified=modified)
        if d:
            # http://pythonhosted.org/feedparser/reference-status.html
            # http://pythonhosted.org/feedparser/http-etag.html#http-etag
            status = d.status if 'status' in d else None

            if status == 301:
                logger.critical(
                    '%s has been permantently moved to a %s!' % (feed_link, d.href))
                return None, status, feed_title, etag, modified, '%s has been permantently moved to a %s!' % (feed_link, d.href)
            elif status == 304:
                logger.warning(
                    '%s server has not updated its feeds' % feed_link)
                return None, status, feed_title, etag, modified, '%s server has not updated its feeds' % feed_link
            elif status == 410:
                logger.critical(
                    '%s is gone! Admin should check the feed availability!' % feed_link)
                return None, status, feed_title, etag, modified, '%s is gone! Admin should check the feed availability!' % feed_link
            elif status == 200 or status == 302:
                # no need to worry.
                if status == 302:
                    logger.info(
                        '%s url has been temp moved to a new place' % feed_link)

                if not feed_title:
                    # if title were not found in feed, an AttributeError would
                    # be raised.
                    feed_title = urllib2.unquote(
                        hparser.unescape(d.feed.title)).strip()
                else:
                    feed_title = feed_title.strip()
                    if 'title' in d.feed:
                        feed_title_latest = urllib2.unquote(
                            hparser.unescape(d.feed.title)).strip()
                        if feed_title != feed_title_latest:
                            # change feed title
                            logger.info(
                                '%s title changed! Please update feed table/database' % feed_link)
                            logger.info('old title: %s' % feed_title)
                            logger.info('new title: %s' % feed_title_latest)
                            #feed_title = feed_title_latest
                    else:
                        logger.info(
                            '%s[%s] has no title in its latest RSS' % (feed_title, feed_link))

                # update etag/modified
                etag = None
                modified = None
                try:
                    etag = d.etag
                except AttributeError:
                    try:
                        modified = d.modified
                    except AttributeError:
                        pass

                if 'entries' in d:
                    language = language if 'language' not in d else d.language
                    # an Exception might be raised from _read_entry
                    entries = []
                    logger.error('%s begins processing' % feed_title)
                    for i, e in enumerate(d.entries):
                        if e:
                            entry = _read_entry(
                                e, feed_id, feed_title, language, categories)
                            if entry:
                                entries.append(entry)
                            else:
                                logger.info('Cannot parse %s' % e['link'])
                                continue
                        else:
                            logger.info(
                                'No infomation found for %s-th entry' % i)
                            continue

                    if entries:
                        # the FINAL return
                        # the last one indicates nothing wrong happended in
                        # parsing
                        return filter(_validate_time, entries), status, feed_title, etag, modified, 'OK'
                    else:
                        logger.info('Feed parsing goes wrong!')
                        return None, status, feed_title, etag, modified, 'Feed parsing goes wrong!'
                else:
                    logger.info("Feed %s has no items!" % feed_id)
                    return None, status, feed_title, etag, modified, 'Feed %s has no items!' % feed_id
            else:
                logger.info(
                    'HTTP Error Code [%s] for %s' % (status, feed_link))
                return None, status, feed_title, etag, modified, 'HTTP Error Code [%s] for %s' % (status, feed_link)
        else:
            logger.info("Cannot parse %s correctly!" % feed_id)
            return None, None, feed_title, etag, modified, "Cannot parse %s correctly!" % feed_id
    except Exception as k:
        logger.exception('%s for %s' % (str(k), feed_id))
        return None, None, feed_title, etag, modified, '%s for %s' % (str(k), feed_id)
<commit_msg>Add _find_redirected_link to get the final link from redirection<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
rss_parser finds all information in an rss item
"""
# @author chengdujin
# @contact chengdujin@gmail.com
# @created Jan 2, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from BeautifulSoup import BeautifulStoneSoup
import calendar
import chardet
from config.settings import hparser
from config.settings import logger
from processor import illustrator
from datetime import datetime, timedelta
import warnings
warnings.filterwarnings(
    action="ignore", category=DeprecationWarning, module="feedparser")
import feedparser
import html2text
import re
import requests
import socket
socket.setdefaulttimeout(10)  # 10 seconds
import time
import urllib2

# CONSTANTS
from config.settings import HEADERS
from config.settings import LANGUAGES
from config.settings import DATABASE_REMOVAL_DAYS
# prefix should not end with a slash
HIDDEN_LINKS = {'http://news.goo.ne.jp':
                ('div', {'class': 'lead fs16 bold'}), 'http://news.nifty.com': ('li', {'class': 'headnews'})}
AD_LINKS = 'http://rss.rssad.jp/rss/ad/'

def _find_redirected_link(url=None):
    """
    find the real link from redirection
    """
    if not url:
        logger.error('Link [%s] is not valid!' % url)
        return None

    IS_REDIRECTED = True
    counter = 0
    nurl = url

    while IS_REDIRECTED and counter < 10:
        resp = requests.get(nurl, headers=HEADERS)
        purl = nurl
        nurl = resp.url

        if nurl == purl:
            IS_REDIRECTED = False
        else:
            counter = counter + 1

    if not IS_REDIRECTED:
        return nurl
    else:
        return None

def _get_actual_link(prefix=None, link=None):
    """
    find the actual news link
    """
    if not prefix or not link:
        logger.error(
            'Method malformed! Prefix:[%s], Link:[%s]' % (prefix, link))

    try:
        actual_link = None
        raw_data = urllib2.urlopen(link)
        data = raw_data.readlines()
        # str() is critical
        soup = BeautifulStoneSoup(str(data))
        html_tag, html_attrs = HIDDEN_LINKS[prefix]
        html_wrapper = soup.find(name=html_tag, attrs=html_attrs)
        if html_wrapper:
            actual_suffix = html_wrapper.find('a')['href']
            actual_link = str('%s%s' % (prefix, actual_suffix))
            return actual_link
        else:
            return None
    except Exception as k:
        logger.error('Cannot open %s' % k)
        return None

def _read_entry(e=None, feed_id=None, feed_title=None, language=None, categories=None):
    """
    read a specific entry item from a feed
    Note. categories are ids of category item
    """
    if not e or not feed_title or not language or not categories:
        logger.error('Method malformed!')
        return None
    if language not in LANGUAGES:
        logger.error("Language not supported for %s!" % feed_title)
        return None

    try:
        entry = {}
        entry['feed_id'] = feed_id
        entry['feed'] = feed_title.strip()
        entry['language'] = language.strip()
        entry['categories'] = categories

        # the easy part: the must-have
        entry['error'] = []

        # article original link
        if e.link:
            original_link = e.link.strip()
            if not original_link.startswith(AD_LINKS):
                # clean the URL
                original_link = urllib2.unquote(
                    hparser.unescape(original_link))
                # find the real link from redirection
                original_link = _find_redirected_link(original_link)

                # find the redirected link
                matched_prefix = [
                    link for link in HIDDEN_LINKS if original_link.startswith(link)]
                found_prefix = matched_prefix[0] if matched_prefix else None
                if found_prefix:
                    actual_link = _get_actual_link(found_prefix, original_link)
                    if actual_link:
                        entry['link'] = actual_link
                    else:
                        logger.error(
                            'No actual link found for %s!' % original_link)
                        return None
                else:
                    entry['link'] = original_link
            else:
                logger.info('Advertising link %s' % original_link)
                return None
        else:
            logger.info('Feed malformed! No link found!')
            return None

        # article title
        if e.title_detail.type != 'text/plain':
            entry['title'] = urllib2.unquote(hparser.unescape(e.title.strip()))
        elif 'title' in e:
            entry['title'] = e.title.strip()
        else:
            entry['title'] = None
        # remove possible htmlized title
        entry['title'] = re.sub("<.*?>", " ", entry[
                                'title']) if 'title' in entry and entry['title'] else None

        # article published time
        # first try parsed time info
        try:
            entry['updated'] = calendar.timegm(e.updated_parsed)
            entry['updated_human'] = e.updated
        except AttributeError as k:
            try:
                entry['updated'] = calendar.timegm(e.published_parsed)
                entry['updated_human'] = e.published
            except AttributeError as k:
                entry['error'] = ['%s\n%s' % (
                    entry['error'], "no 'updated_parsed' or 'published_parsed'")]
                # then try unparsed time info
                # this is rarely possible.
                try:
                    updated = e.updated if 'updated' in e else e.published
                    if updated:
                        # get time zone
                        offset = int(updated[-5:])
                        delta = timedelta(hours=int(offset) / 100)
                        format = "%a, %d %b %Y %H:%M:%S"
                        if updated[-8:-5] != 'UTC':
                            updated = datetime.strptime(updated[:-6], format)
                        else:
                            updated = datetime.strptime(updated[:-9], format)
                        updated -= delta
                        entry['updated'] = time.mktime(updated.timetuple())
                    else:
                        logger.info(
                            "Attribute updated/published has no value")
                        return None
                except ValueError as k:
                    logger.info(str(k))
                    entry['error'].append('%s\n%s' % (entry['error'], k))
                    return None
                except AttributeError as k:
                    logger.info(str(k))
                    entry['error'].append('no update or published\n')
                    return None

        # article's summary
        try:
            # its possible summary is html-based
            summary = urllib2.unquote(hparser.unescape(e.summary))
            if isinstance(summary, str):
                summary_encoding = chardet.detect(summary)['encoding']
                summary = summary.decode(summary_encoding, 'ignore')
            # a <div, for example, and a </div
            is_html = True if len(
                re.findall(u'</?a|</?p|</?strong|</?img|</?html|</?div', summary)) > 1 else False
            if is_html:
                h = html2text.HTML2Text()
                h.ignore_images = True
                h.ignore_links = True
                h.ignore_emphasis = True
                paragraphs = (h.handle(summary)).strip().strip(
                    '#').strip().split('\n\n')
                paragraphs_above_limit = []
                # remove paragraphs that contain less than x number of words
                for paragraph in paragraphs:
                    if entry['language'].startswith('zh') or entry['language'] == 'ja':
                        if len(paragraph) > 18:
                            paragraphs_above_limit.append(paragraph)
                    else:
                        words = paragraph.split()
                        if len(words) > 12:
                            paragraphs_above_limit.append(paragraph)
                entry['summary'] = '\n\n'.join(paragraphs_above_limit)
            else:
                entry['summary'] = summary
        except AttributeError as k:
            entry['summary'] = None
        entry['summary'] = None if not entry['summary'] else entry['summary']

        # article's images
        # e.g. [{'url':'http://image.com/test.jpg, 'width': u'130', 'height':
        # u'86'}]
        entry['images'] = []
        try:
            images, media_content_new = illustrator.find_images(
                e.media_content, entry['link'])
            if images:
                entry['images'].extend(images)
        except AttributeError as k:
            pass
        try:
            images, media_content_new = illustrator.find_images(
                e.media_thumbnail, entry['link'])
            if images:
                entry['images'].extend(images)
        except AttributeError as k:
            pass
        for attribute in e:
            if 'thumbnail' in attribute:
                # currently set thumbnail to None if its a dictionary
                image = e[attribute] if isinstance(e[attribute], str) else None
                image = illustrator.find_image(image, entry['link'])
                if image:
                    entry['images'].append(image)
        try:
            links = e.links
            for link in links:
                if 'type' in link and 'image' in link.type:
                    if 'href' in link:
                        image = illustrator.find_image(
                            link.href, entry['link'])
                        if image:
                            entry['images'].append(image)
        except AttributeError as k:
            pass

        if entry.has_key('summary') and entry['summary']:
            images, entry['summary'] = illustrator.find_images(
                entry['summary'], entry['link'])
            if images:
                entry['images'].extend(images)
        # dedup images is processed at rss.py

        # article's author
        # e.g. Yuan Jin
        try:
            # i guess this could be a string or a list
            entry['author'] = e.author
        except AttributeError as k:
            entry['author'] = None

        # article's source
        # e.g. {'href': u'http://www.reuters.com/', 'title': u'Reuters'}
        try:
            entry['source'] = e.source
        except AttributeError as k:
            entry['source'] = None

        # article's tags
        # e.g. [{'term': u'Campus Party', 'scheme': None, 'label': None}]
        # term is usually combined with scheme to form a url; label is
        # the name of term
        try:
            entry['tags'] = e.tag
        except AttributeError as k:
            entry['tags'] = None

        # the FINAL return
        return entry
    except Exception as k:
        logger.error(str(k))
        return None

# TODO: boundary checkers
# TODO: update parsing info to feed database
def parse(feed_link=None, feed_id=None, feed_title=None, language=None, categories=None, etag=None, modified=None):
    """
    read rss/atom data from a given feed
    feed_id is the feed ObjectId in MongoDB
    Etag and Modified are used to save rss http server's bandwidth
    Note: category should be added to feed table/database
    """
    if not feed_link or not feed_id or not language or not categories:
        logger.error("Method malformed!")
        return None, None, feed_title, etag, modified, "Method malformed!"
    if language not in LANGUAGES:
        logger.error("Language not supported for %s!" % feed_link)
        return None, None, feed_title, etag, modified, "Language not supported for %s!" % feed_link

    def _validate_time(entry):
        """
        see if the entry's updated time is earlier than needed
        """
        deadline = datetime.utcfromtimestamp(
            entry['updated']) + timedelta(days=DATABASE_REMOVAL_DAYS)
        return True if deadline > datetime.now() else False

    try:
        # variables d and e follow feedparser tradition
        feedparser.USER_AGENT = "newsman"
        d = feedparser.parse(feed_link, etag=etag, modified=modified)
        if d:
            # http://pythonhosted.org/feedparser/reference-status.html
            # http://pythonhosted.org/feedparser/http-etag.html#http-etag
            status = d.status if 'status' in d else None

            if status == 301:
                logger.critical(
                    '%s has been permantently moved to a %s!' % (feed_link, d.href))
                return None, status, feed_title, etag, modified, '%s has been permantently moved to a %s!' % (feed_link, d.href)
            elif status == 304:
                logger.warning(
                    '%s server has not updated its feeds' % feed_link)
                return None, status, feed_title, etag, modified, '%s server has not updated its feeds' % feed_link
            elif status == 410:
                logger.critical(
                    '%s is gone! Admin should check the feed availability!' % feed_link)
                return None, status, feed_title, etag, modified, '%s is gone! Admin should check the feed availability!' % feed_link
            elif status == 200 or status == 302:
                # no need to worry.
                if status == 302:
                    logger.info(
                        '%s url has been temp moved to a new place' % feed_link)

                if not feed_title:
                    # if title were not found in feed, an AttributeError would
                    # be raised.
                    feed_title = urllib2.unquote(
                        hparser.unescape(d.feed.title)).strip()
                else:
                    feed_title = feed_title.strip()
                    if 'title' in d.feed:
                        feed_title_latest = urllib2.unquote(
                            hparser.unescape(d.feed.title)).strip()
                        if feed_title != feed_title_latest:
                            # change feed title
                            logger.info(
                                '%s title changed! Please update feed table/database' % feed_link)
                            logger.info('old title: %s' % feed_title)
                            logger.info('new title: %s' % feed_title_latest)
                            #feed_title = feed_title_latest
                    else:
                        logger.info(
                            '%s[%s] has no title in its latest RSS' % (feed_title, feed_link))

                # update etag/modified
                etag = None
                modified = None
                try:
                    etag = d.etag
                except AttributeError:
                    try:
                        modified = d.modified
                    except AttributeError:
                        pass

                if 'entries' in d:
                    language = language if 'language' not in d else d.language
                    # an Exception might be raised from _read_entry
                    entries = []
                    logger.error('%s begins processing' % feed_title)
                    for i, e in enumerate(d.entries):
                        if e:
                            entry = _read_entry(
                                e, feed_id, feed_title, language, categories)
                            if entry:
                                entries.append(entry)
                            else:
                                logger.info('Cannot parse %s' % e['link'])
                                continue
                        else:
                            logger.info(
                                'No infomation found for %s-th entry' % i)
                            continue

                    if entries:
                        # the FINAL return
                        # the last one indicates nothing wrong happended in
                        # parsing
                        return filter(_validate_time, entries), status, feed_title, etag, modified, 'OK'
                    else:
                        logger.info('Feed parsing goes wrong!')
                        return None, status, feed_title, etag, modified, 'Feed parsing goes wrong!'
                else:
                    logger.info("Feed %s has no items!" % feed_id)
                    return None, status, feed_title, etag, modified, 'Feed %s has no items!' % feed_id
            else:
                logger.info(
                    'HTTP Error Code [%s] for %s' % (status, feed_link))
                return None, status, feed_title, etag, modified, 'HTTP Error Code [%s] for %s' % (status, feed_link)
        else:
            logger.info("Cannot parse %s correctly!" % feed_id)
            return None, None, feed_title, etag, modified, "Cannot parse %s correctly!" % feed_id
    except Exception as k:
        logger.exception('%s for %s' % (str(k), feed_id))
        return None, None, feed_title, etag, modified, '%s for %s' % (str(k), feed_id)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright © 2015-2016 Juanjo Conti and others.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Render the author pages and feeds."""

from __future__ import unicode_literals
import os
import natsort
try:
    from urlparse import urljoin
except ImportError:
    from urllib.parse import urljoin  # NOQA
from collections import defaultdict

from blinker import signal

from nikola.plugin_categories import Task
from nikola import utils

class RenderAuthors(Task):
    """Render the author pages and feeds."""

    name = "render_authors"
    posts_per_author = None

    def set_site(self, site):
        """Set Nikola site."""
        self.generate_author_pages = False
        if site.config["ENABLE_AUTHOR_PAGES"]:
            site.register_path_handler('author_index', self.author_index_path)
            site.register_path_handler('author', self.author_path)
            site.register_path_handler('author_atom', self.author_atom_path)
            site.register_path_handler('author_rss', self.author_rss_path)
            signal('scanned').connect(self.posts_scanned)
        return super(RenderAuthors, self).set_site(site)

    def posts_scanned(self, event):
        """Called after posts are scanned via signal."""
        self.generate_author_pages = self.site.config["ENABLE_AUTHOR_PAGES"] and len(self._posts_per_author()) > 1
        self.site.GLOBAL_CONTEXT["author_pages_generated"] = self.generate_author_pages

    def gen_tasks(self):
        """Render the author pages and feeds."""
        kw = {
            "translations": self.site.config["TRANSLATIONS"],
            "blog_title": self.site.config["BLOG_TITLE"],
            "site_url": self.site.config["SITE_URL"],
            "base_url": self.site.config["BASE_URL"],
            "messages": self.site.MESSAGES,
            "output_folder": self.site.config['OUTPUT_FOLDER'],
            "filters": self.site.config['FILTERS'],
            'author_path': self.site.config['AUTHOR_PATH'],
            "author_pages_are_indexes": self.site.config['AUTHOR_PAGES_ARE_INDEXES'],
            "generate_rss": self.site.config['GENERATE_RSS'],
            "feed_teasers": self.site.config["FEED_TEASERS"],
            "feed_plain": self.site.config["FEED_PLAIN"],
            "feed_link_append_query": self.site.config["FEED_LINKS_APPEND_QUERY"],
            "show_untranslated_posts": self.site.config['SHOW_UNTRANSLATED_POSTS'],
            "feed_length": self.site.config['FEED_LENGTH'],
            "tzinfo": self.site.tzinfo,
            "pretty_urls": self.site.config['PRETTY_URLS'],
            "strip_indexes": self.site.config['STRIP_INDEXES'],
            "index_file": self.site.config['INDEX_FILE'],
        }

        self.site.scan_posts()
        yield self.group_task()

        if self.generate_author_pages:
            yield self.list_authors_page(kw)

            if not self._posts_per_author():  # this may be self.site.posts_per_author
                return

            author_list = list(self._posts_per_author().items())

            def render_lists(author, posts):
                """Render author pages as RSS files and lists/indexes."""
                post_list = sorted(posts, key=lambda a: a.date)
                post_list.reverse()
                for lang in kw["translations"]:
                    if kw["show_untranslated_posts"]:
                        filtered_posts = post_list
                    else:
                        filtered_posts = [x for x in post_list if x.is_translation_available(lang)]
                    if kw["generate_rss"]:
                        yield self.author_rss(author, lang, filtered_posts, kw)
                    # Render HTML
                    if kw['author_pages_are_indexes']:
                        yield self.author_page_as_index(author, lang, filtered_posts, kw)
                    else:
                        yield self.author_page_as_list(author, lang, filtered_posts, kw)

            for author, posts in author_list:
                for task in render_lists(author, posts):
                    yield task

    def _create_authors_page(self, kw):
        """Create a global "all authors" page for each language."""
        template_name = "authors.tmpl"
        kw = kw.copy()
        for lang in kw["translations"]:
            authors = natsort.natsorted([author for author in self._posts_per_author().keys()],
                                        alg=natsort.ns.F | natsort.ns.IC)
            has_authors = (authors != [])
            kw['authors'] = authors
            output_name = os.path.join(
                kw['output_folder'], self.site.path('author_index', None, lang))
            context = {}
            if has_authors:
                context["title"] = kw["messages"][lang]["Authors"]
                context["items"] = [(author, self.site.link("author", author, lang)) for author
                                    in authors]
                context["description"] = context["title"]
            else:
                context["items"] = None
            context["permalink"] = self.site.link("author_index", None, lang)
            context["pagekind"] = ["list", "authors_page"]
            task = self.site.generic_post_list_renderer(
                lang,
                [],
                output_name,
                template_name,
                kw['filters'],
                context,
            )
            task['uptodate'] = task['uptodate'] + [utils.config_changed(kw, 'nikola.plugins.task.authors:page')]
            task['basename'] = str(self.name)
            yield task

    def list_authors_page(self, kw):
        """Create a global "all authors" page for each language."""
        yield self._create_authors_page(kw)

    def _get_title(self, author):
        return author

    def _get_description(self, author, lang):
        descriptions = self.site.config['AUTHOR_PAGES_DESCRIPTIONS']
        return descriptions[lang][author] if lang in descriptions and author in descriptions[lang] else None

    def author_page_as_index(self, author, lang, post_list, kw):
        """Render a sort of index page collection using only this author's posts."""
        kind = "author"

        def page_link(i, displayed_i, num_pages, force_addition, extension=None):
            feed = "_atom" if extension == ".atom" else ""
            return utils.adjust_name_for_index_link(self.site.link(kind + feed, author, lang), i, displayed_i, lang, self.site, force_addition, extension)

        def page_path(i, displayed_i, num_pages, force_addition, extension=None):
            feed = "_atom" if extension == ".atom" else ""
            return utils.adjust_name_for_index_path(self.site.path(kind + feed, author, lang), i, displayed_i, lang, self.site, force_addition, extension)

        context_source = {}
        title = self._get_title(author)
        if kw["generate_rss"]:
            # On a author page, the feeds include the author's feeds
            rss_link = ("""<link rel="alternate" type="application/rss+xml" """
                        """title="RSS for author """
                        """{0} ({1})" href="{2}">""".format(
                            title, lang, self.site.link(kind + "_rss", author, lang)))
            context_source['rss_link'] = rss_link
        context_source["author"] = title
        indexes_title = kw["messages"][lang]["Posts by %s"] % title
        context_source["description"] = self._get_description(author, lang)
        context_source["pagekind"] = ["index", "author_page"]
        template_name = "authorindex.tmpl"

        yield self.site.generic_index_renderer(lang, post_list, indexes_title, template_name, context_source, kw, str(self.name), page_link, page_path)

    def author_page_as_list(self, author, lang, post_list, kw):
        """Render a single flat link list with this author's posts."""
        kind = "author"
        template_name = "author.tmpl"
        output_name = os.path.join(kw['output_folder'], self.site.path(
            kind, author, lang))
        context = {}
        context["lang"] = lang
        title = self._get_title(author)
        context["author"] = title
        context["title"] = kw["messages"][lang]["Posts by %s"] % title
        context["posts"] = post_list
        context["permalink"] = self.site.link(kind, author, lang)
        context["kind"] = kind
        context["description"] = self._get_description(author, lang)
        context["pagekind"] = ["list", "author_page"]
        task = self.site.generic_post_list_renderer(
            lang,
            post_list,
            output_name,
            template_name,
            kw['filters'],
            context,
        )
        task['uptodate'] = task['uptodate'] + [utils.config_changed(kw, 'nikola.plugins.task.authors:list')]
        task['basename'] = str(self.name)
        yield task

    def author_rss(self, author, lang, posts, kw):
        """Create a RSS feed for a single author in a given language."""
        kind = "author"
        # Render RSS
        output_name = os.path.normpath(
            os.path.join(kw['output_folder'],
                         self.site.path(kind + "_rss", author, lang)))
        feed_url = urljoin(self.site.config['BASE_URL'], self.site.link(kind + "_rss", author, lang).lstrip('/'))
        deps = []
        deps_uptodate = []
        post_list = sorted(posts, key=lambda a: a.date)
        post_list.reverse()
        for post in post_list:
            deps += post.deps(lang)
            deps_uptodate += post.deps_uptodate(lang)
        task = {
            'basename': str(self.name),
            'name': output_name,
            'file_dep': deps,
            'targets': [output_name],
            'actions': [(utils.generic_rss_renderer,
                        (lang, "{0} ({1})".format(kw["blog_title"](lang), self._get_title(author)),
                         kw["site_url"], None, post_list,
                         output_name, kw["feed_teasers"], kw["feed_plain"], kw['feed_length'],
                         feed_url, None, kw["feed_link_append_query"]))],
            'clean': True,
            'uptodate': [utils.config_changed(kw, 'nikola.plugins.task.authors:rss')] + deps_uptodate,
            'task_dep': ['render_posts'],
        }
        return utils.apply_filters(task, kw['filters'])

    def slugify_author_name(self, name, lang=None):
        """Slugify an author name."""
        if lang is None:  # TODO: remove in v8
            utils.LOGGER.warn("RenderAuthors.slugify_author_name() called without language!")
            lang = ''
        if self.site.config['SLUG_AUTHOR_PATH']:
            name = utils.slugify(name, lang)
        return name

    def author_index_path(self, name, lang):
        """Link to the author's index.

        Example:

        link://authors/ => /authors/index.html
        """
        return [_f for _f in [self.site.config['TRANSLATIONS'][lang],
                              self.site.config['AUTHOR_PATH'],
                              self.site.config['INDEX_FILE']] if _f]

    def author_path(self, name, lang):
        """Link to an author's page.

        Example:

        link://author/joe => /authors/joe.html
        """
        if self.site.config['PRETTY_URLS']:
            return [_f for _f in [
                self.site.config['TRANSLATIONS'][lang],
                self.site.config['AUTHOR_PATH'],
                self.slugify_author_name(name, lang),
                self.site.config['INDEX_FILE']] if _f]
        else:
            return [_f for _f in [
                self.site.config['TRANSLATIONS'][lang],
                self.site.config['AUTHOR_PATH'],
                self.slugify_author_name(name, lang) + ".html"] if _f]

    def author_atom_path(self, name, lang):
        """Link to an author's Atom feed.

        Example:

        link://author_atom/joe => /authors/joe.atom
        """
        return [_f for _f in [self.site.config['TRANSLATIONS'][lang],
                              self.site.config['AUTHOR_PATH'], self.slugify_author_name(name, lang) + ".atom"] if
                _f]

    def author_rss_path(self, name, lang):
        """Link to an author's RSS feed.

        Example:

        link://author_rss/joe => /authors/joe.rss
        """
        return [_f for _f in [self.site.config['TRANSLATIONS'][lang],
                              self.site.config['AUTHOR_PATH'], self.slugify_author_name(name, lang) + ".xml"] if
                _f]

    def _add_extension(self, path, extension):
        path[-1] += extension
        return path

    def _posts_per_author(self):
        """Return a dict of posts per author."""
        if self.posts_per_author is None:
            self.posts_per_author = defaultdict(list)
            for post in self.site.timeline:
                if post.is_post:
                    self.posts_per_author[post.author()].append(post)
        return self.posts_per_author
<commit_msg>Rewrote authors plugin as a Taxonomy plugin.<commit_after># -*- coding: utf-8 -*-

# Copyright © 2015-2016 Juanjo Conti and others.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Render the author pages and feeds."""

from __future__ import unicode_literals

from nikola.plugin_categories import Taxonomy
from nikola import utils

class ClassifyAuthors(Taxonomy):
    """Render the author pages and feeds."""

    name = "classify_authors"

    classification_name = "author"
    metadata_name = "author"
    overview_page_variable_name = "authors"
    more_than_one_classifications_per_post = False
    has_hierarchy = False
    template_for_classification_overview = "authors.tmpl"
    apply_to_posts = True
    apply_to_pages = False
    minimum_post_count_per_classification_in_overview = 1
    omit_empty_classifications = False
    also_create_classifications_from_other_languages = False

    def set_site(self, site):
        """Set Nikola site."""
        self.show_list_as_index = site.config['AUTHOR_PAGES_ARE_INDEXES']
        self.template_for_list_of_one_classification = "authorindex.tmpl" if self.show_list_as_index else "author.tmpl"
        return super(ClassifyAuthors, self).set_site(site)

    def is_enabled(self):
        """Return True if this taxonomy is enabled, or False otherwise."""
        return self.site.config["ENABLE_AUTHOR_PAGES"]

    def classify(self, post, lang):
        """Classify the given post for the given language."""
        return [post.author()]

    def get_list_path(self, lang):
        """A path handler for the list of all classifications."""
        return [self.site.config['AUTHOR_PATH']], True

    def get_path(self, author, lang):
        """A path handler for the given classification."""
        if self.site.config['SLUG_AUTHOR_PATH']:
            slug = utils.slugify(author, lang)
        else:
            slug = author
        return [self.site.config['AUTHOR_PATH'], slug], False

    def provide_list_context_and_uptodate(self, lang):
        """Provide data for the context and the uptodate list for the list of all classifiations."""
        kw = {
            "messages": self.site.MESSAGES,
        }
        context = {
            "title": kw["messages"][lang]["Authors"],
            "description": kw["messages"][lang]["Authors"],
            "permalink": self.site.link("author_index", None, lang),
            "pagekind": ["list", "authors_page"],
        }
        kw.update(context)
        return context, kw

    def provide_context_and_uptodate(self, author, lang):
        """Provide data for the context and the uptodate list for the list of the given classifiation."""
        descriptions = self.site.config['AUTHOR_PAGES_DESCRIPTIONS']
        kw = {
            "messages": self.site.MESSAGES,
        }
        context = {
            "title": kw["messages"][lang]["Posts by %s"] % author,
            "classification_title": author,
            "description": descriptions[lang][author] if lang in descriptions and author in descriptions[lang] else None,
            "pagekind": ["index" if self.show_list_as_index else "list", "author_page"],
        }
        kw.update(context)
        return context, kw

    def postprocess_posts_per_classification(self, posts_per_author_per_language, flat_hierarchy_per_lang=None, hierarchy_lookup_per_lang=None):
        """Rearrange, modify or otherwise use the list of posts per classification and per language."""
        more_than_one = False
        for lang, posts_per_author in posts_per_author_per_language.item():
            if len(posts_per_author.keys()) > 1:
                more_than_one = True
        self.generate_author_pages = self.site.config["ENABLE_AUTHOR_PAGES"] and more_than_one
        self.site.GLOBAL_CONTEXT["author_pages_generated"] = self.generate_author_pages
<|endoftext|>


<commit_before>"""
Assortment of helper utility methods
"""

from django.templatetags.static import static
from django.core.urlresolvers import reverse

def get_template_path(template_name):
    """
    returns a full URL path to our template directory
    """

    return static(
        'edx_notifications/templates/{template_name}'.format(
            template_name=template_name
        )
    )

def get_audio_path(audio_name):
    """
    returns a full URL path to audio directory
    """

    return static(
        'edx_notifications/audio/{audio_name}'.format(
            audio_name=audio_name
        )
    )

def get_notifications_widget_context(override_context=None):
    """
    As a convenience method, this will return all required
    context properties that the notifications_widget.html Django template needs
    """

    context = {
        'endpoints': {
            'unread_notification_count': (
                '{base_url}?read=False&unread=True'
            ). format(base_url=reverse('edx_notifications.consumer.notifications.count')),
            'mark_all_user_notifications_read': (
                '{base_url}'
            ). format(base_url=reverse('edx_notifications.consumer.notifications.mark_notifications')),
            'user_notifications_unread_only': (
                '{base_url}?read=False&unread=True'
            ). format(base_url=reverse('edx_notifications.consumer.notifications')),
            'user_notifications_all': (
                '{base_url}?read=True&unread=True'
            ). format(base_url=reverse('edx_notifications.consumer.notifications')),
            'renderer_templates_urls': reverse('edx_notifications.consumer.renderers.templates'),
        },
        'global_variables': {
            'app_name': 'Your App Name Here',
        },
        'refresh_watchers': {
            'name': 'none',
            'args': {},
        },
        'view_audios': {
            'notification_alert': get_audio_path('notification_alert.mp3'),
        },
        'add_notification_divs': True,
    }

    if override_context:
        context.update(override_context)

    return context
<commit_msg>remove unused context variable<commit_after>"""
Assortment of helper utility methods
"""

from django.templatetags.static import static
from django.core.urlresolvers import reverse

def get_template_path(template_name):
    """
    returns a full URL path to our template directory
    """

    return static(
        'edx_notifications/templates/{template_name}'.format(
            template_name=template_name
        )
    )

def get_audio_path(audio_name):
    """
    returns a full URL path to audio directory
    """

    return static(
        'edx_notifications/audio/{audio_name}'.format(
            audio_name=audio_name
        )
    )

def get_notifications_widget_context(override_context=None):
    """
    As a convenience method, this will return all required
    context properties that the notifications_widget.html Django template needs
    """

    context = {
        'endpoints': {
            'unread_notification_count': (
                '{base_url}?read=False&unread=True'
            ). format(base_url=reverse('edx_notifications.consumer.notifications.count')),
            'mark_all_user_notifications_read': (
                '{base_url}'
            ). format(base_url=reverse('edx_notifications.consumer.notifications.mark_notifications')),
            'user_notifications_unread_only': (
                '{base_url}?read=False&unread=True'
            ). format(base_url=reverse('edx_notifications.consumer.notifications')),
            'user_notifications_all': (
                '{base_url}?read=True&unread=True'
            ). format(base_url=reverse('edx_notifications.consumer.notifications')),
            'renderer_templates_urls': reverse('edx_notifications.consumer.renderers.templates'),
        },
        'global_variables': {
            'app_name': 'Your App Name Here',
        },
        'refresh_watchers': {
            'name': 'none',
            'args': {},
        },
        'view_audios': {
            'notification_alert': get_audio_path('notification_alert.mp3'),
        }
    }

    if override_context:
        context.update(override_context)

    return context
<|endoftext|>


<commit_before>import os
from django.shortcuts import render_to_response

def home(request):
    return render_to_response('foodcheck/home.html')

# vim:expandtab tabstop=8 shiftwidth=4 ts=8 sw=4 softtabstop=4
<commit_msg>Trying to pull in the RequestContext<commit_after>import os
from django.shortcuts import render

def home(request):
    return render(request, 'foodcheck/home.html',
                  {},
                 )

# vim:expandtab tabstop=8 shiftwidth=4 ts=8 sw=4 softtabstop=4
<|endoftext|>


<commit_before><commit_msg>add genotyping script to repo<commit_after><|endoftext|>


<commit_before>'''
gff2fasta.py - output sequences from genomic features
=====================================================

:Author: Andreas Heger
:Release: $Id$
:Date: |today|
:Tags: Genomics Intervals Sequences GFF Fasta Transformation

Purpose
-------

This script outputs the genomic sequences for intervals within
a :term:`gff` formatted file.

The ouput can be optionall masked and filtered.

Usage
-----

Example::

   python gff2fasta.py --genome-file=hg19 < features.gff > features.fasta

Type::

   python gff2fasta.py --help

for command line help.

Command line options
--------------------

'''
import sys
import string
import re
import optparse
import CGAT.Experiment as E
import CGAT.GTF as GTF
import CGAT.Genomics as Genomics
import CGAT.AGP as AGP
import CGAT.IndexedFasta as IndexedFasta
import CGAT.Intervals as Intervals
import CGAT.Masker as Masker
import bx.intervals.io
import bx.intervals.intersection

import CGAT.Experiment as E

def main(argv=None):
    """script main.

    parses command line options in sys.argv, unless *argv* is given.
    """

    if argv is None:
        argv = sys.argv

    parser = E.OptionParser(
        version="%prog version: $Id: gff2fasta.py 2861 2010-02-23 17:36:32Z andreas $", usage=globals()["__doc__"])

    parser.add_option("--is-gtf", dest="is_gtf", action="store_true",
                      help="input is gtf instead of gff.")

    parser.add_option("-g", "--genome-file", dest="genome_file", type="string",
                      help="filename with genome [default=%default].")

    parser.add_option("-m", "--merge", dest="merge", action="store_true",
                      help="merge adjacent intervals with the same attributes. "
                      "[default=%default]")

    parser.add_option("-e", "--feature", dest="feature", type="string",
                      help="filter by a feature, for example 'exon', 'CDS'. If "
                      "set to the empty string, all entries are output [%default].")

    parser.add_option("-f", "--filename-masks", dest="filename_masks", type="string",
                      metavar="gff",
                      help="mask sequences with regions given in gff file [%default].")

    parser.add_option("--remove-masked-regions", dest="remove_masked_regions", action="store_true",
                      help="remove regions instead of masking [%default].")

    parser.add_option("--min-length", dest="min_length", type="int",
                      help="set minimum length for sequences output [%default]")

    parser.add_option("--max-length", dest="max_length", type="int",
                      help="set maximum length for sequences output [%default]")

    parser.add_option("--extend-at", dest="extend_at", type="choice",
                      choices=("none", "3", "5", "both", "3only", "5only"),
                      help="extend at no end, 3', 5' or both ends. If "
                      "3only or 5only are set, only the added sequence "
                      "is returned [default=%default]")

    parser.add_option("--extend-by", dest="extend_by", type="int",
                      help="extend by # bases [default=%default]")

    parser.add_option("--masker", dest="masker", type="choice",
                      choices=("dust", "dustmasker", "softmask", "none"),
                      help="apply masker [%default].")

    parser.set_defaults(
        is_gtf=False,
        genome_file=None,
        merge=False,
        feature=None,
        filename_masks=None,
        remove_masked_regions=False,
        min_length=0,
        max_length=0,
        extend_at=None,
        extend_by=100,
        masker=None
    )

    (options, args) = E.Start(parser)

    if options.genome_file:
        fasta = IndexedFasta.IndexedFasta(options.genome_file)
        contigs = fasta.getContigSizes()

    if options.is_gtf:
        iterator = GTF.transcript_iterator(GTF.iterator(options.stdin))
    else:
        gffs = GTF.iterator(options.stdin)
        if options.merge:
            iterator = GTF.joined_iterator(gffs)
        else:
            iterator = GTF.chunk_iterator(gffs)

    masks = None
    if options.filename_masks:
        masks = {}
        with open(options.filename_masks, "r") as infile:
            e = GTF.readAsIntervals(GFF.iterator(infile))

        # convert intervals to intersectors
        for contig in e.keys():
            intersector = bx.intervals.intersection.Intersecter()
            for start, end in e[contig]:
                intersector.add_interval(bx.intervals.Interval(start, end))
            masks[contig] = intersector

    ninput, noutput, nmasked, nskipped_masked = 0, 0, 0, 0
    nskipped_length = 0
    nskipped_noexons = 0

    feature = options.feature

#    for item in iterator:
# print len(item) # 3, 2
#	for i in item:
# print len(i) # 9, 9, 9, 9, 9
#	   print i.contig
#	   print i.strand
#	   print i.transcript_id

    # iterator is a list containing groups (lists) of features.
    # Each group of features have in common the same transcript ID, in case of
    # GTF files.
    for ichunk in iterator:

        ninput += 1

        if feature:
            chunk = filter(lambda x: x.feature == feature, ichunk)
        else:
            chunk = ichunk

        if len(chunk) == 0:
            nskipped_noexons += 1
            E.info("no features in entry from %s:%i..%i - %s" % (ichunk[0].contig,
                                                                 ichunk[
                                                                     0].start,
                                                                 ichunk[0].end,
                                                                 str(ichunk[0])))
            continue

        contig, strand = chunk[0].contig, chunk[0].strand
        if options.is_gtf:
            name = chunk[0].transcript_id
        else:
            name = str(chunk[0].attributes)

        lcontig = contigs[contig]
        positive = Genomics.IsPositiveStrand(strand)
        intervals = [(x.start, x.end) for x in chunk]
        intervals.sort()

        if masks:
            if contig in masks:
                masked_regions = []
                for start, end in intervals:
                    masked_regions += [(x.start, x.end)
                                       for x in masks[contig].find(start, end)]

                masked_regions = Intervals.combine(masked_regions)
                if len(masked_regions):
                    nmasked += 1

                if options.remove_masked_regions:
                    intervals = Intervals.truncate(intervals, masked_regions)
                else:
                    raise "unimplemented"

                if len(intervals) == 0:
                    nskipped_masked += 1
                    if options.loglevel >= 1:
                        options.stdlog.write("# skipped because fully masked: %s: regions=%s masks=%s\n" %
                                             (name, str([(x.start, x.end) for x in chunk]), masked_regions))
                    continue

        out = intervals

        if options.extend_at:
            if options.extend_at == "5only":
                intervals = [
                    (max(0, intervals[0][0] - options.extend_by), intervals[0][0])]
            elif options.extend_at == "3only":
                intervals = [
                    (intervals[-1][1], min(lcontig, intervals[-1][1] + options.extend_by))]
            else:
                if options.extend_at in ("5", "both"):
                    intervals[0] = (
                        max(0, intervals[0][0] - options.extend_by), intervals[0][1])
                if options.extend_at in ("3", "both"):
                    intervals[-1] = (intervals[-1][0],
                                     min(lcontig, intervals[-1][1] + options.extend_by))

        if not positive:
            intervals = [(lcontig - x[1], lcontig - x[0])
                         for x in intervals[::-1]]
            out.reverse()

        s = [fasta.getSequence(contig, strand, start, end)
             for start, end in intervals]
        # IMS: allow for masking of sequences
        s = Masker.maskSequences(s, options.masker)
        l = sum([len(x) for x in s])
        if l < options.min_length or (options.max_length and l > options.max_length):
            nskipped_length += 1
            if options.loglevel >= 1:
                options.stdlog.write("# skipped because length out of bounds %s: regions=%s len=%i\n" %
                                     (name, str(intervals), l))
            continue

        options.stdout.write(">%s %s:%s:%s\n%s\n" % (name,
                                                     contig,
                                                     strand,
                                                     ";".join(
                                                         ["%i-%i" % x for x in out]),
                                                     "\n".join(s)))

        noutput += 1

    E.info("ninput=%i, noutput=%i, nmasked=%i, nskipped_noexons=%i, nskipped_masked=%i, nskipped_length=%i" %
           (ninput, noutput, nmasked, nskipped_noexons, nskipped_masked, nskipped_length))

    E.Stop()

if __name__ == "__main__":
    sys.exit(main(sys.argv))
<commit_msg> pep8 corrections to gff2fasta.py<commit_after>'''
gff2fasta.py - output sequences from genomic features
=====================================================

:Author: Andreas Heger
:Release: $Id$
:Date: |today|
:Tags: Genomics Intervals Sequences GFF Fasta Transformation

Purpose
-------

This script outputs the genomic sequences for intervals within
a :term:`gff` formatted file.

The ouput can be optionally masked and filtered.

Usage
-----

Example::

   python gff2fasta.py --genome-file=hg19 < features.gff > features.fasta

Type::

   python gff2fasta.py --help

for command line help.

Command line options
--------------------
'''

import sys
import CGAT.Experiment as E
import CGAT.GTF as GTF
import CGAT.Genomics as Genomics
import CGAT.IndexedFasta as IndexedFasta
import CGAT.Intervals as Intervals
import CGAT.Masker as Masker
import bx.intervals.io
import bx.intervals.intersection

def main(argv=None):
    """script main.

    parses command line options in sys.argv, unless *argv* is given.
    """

    if argv is None:
        argv = sys.argv

    parser = E.OptionParser(version="%prog version: "
                            "$Id: gff2fasta.py 2861 2010-02-23 17:36:32Z "
                            "andreas $",
                            usage=globals()["__doc__"])

    parser.add_option("--is-gtf", dest="is_gtf", action="store_true",
                      help="input is gtf instead of gff.")

    parser.add_option("-g", "--genome-file", dest="genome_file", type="string",
                      help="filename with genome [default=%default].")

    parser.add_option("-m", "--merge", dest="merge", action="store_true",
                      help="merge adjacent intervals with the same attributes."
                      " [default=%default]")

    parser.add_option("-e", "--feature", dest="feature", type="string",
                      help="filter by a feature, for example 'exon', 'CDS'."
                      " If set to the empty string, all entries are output "
                      "[%default].")

    parser.add_option("-f", "--filename-masks", dest="filename_masks",
                      type="string", metavar="gff",
                      help="mask sequences with regions given in gff file "
                      "[%default].")

    parser.add_option("--remove-masked-regions", dest="remove_masked_regions",
                      action="store_true",
                      help="remove regions instead of masking [%default].")

    parser.add_option("--min-length", dest="min_length", type="int",
                      help="set minimum length for sequences output "
                      "[%default]")

    parser.add_option("--max-length", dest="max_length", type="int",
                      help="set maximum length for sequences output "
                      "[%default]")

    parser.add_option("--extend-at", dest="extend_at", type="choice",
                      choices=("none", "3", "5", "both", "3only", "5only"),
                      help="extend at no end, 3', 5' or both ends. If "
                      "3only or 5only are set, only the added sequence "
                      "is returned [default=%default]")

    parser.add_option("--extend-by", dest="extend_by", type="int",
                      help="extend by # bases [default=%default]")

    parser.add_option("--masker", dest="masker", type="choice",
                      choices=("dust", "dustmasker", "softmask", "none"),
                      help="apply masker [%default].")

    parser.set_defaults(
        is_gtf=False,
        genome_file=None,
        merge=False,
        feature=None,
        filename_masks=None,
        remove_masked_regions=False,
        min_length=0,
        max_length=0,
        extend_at=None,
        extend_by=100,
        masker=None
    )

    (options, args) = E.Start(parser)

    if options.genome_file:
        fasta = IndexedFasta.IndexedFasta(options.genome_file)
        contigs = fasta.getContigSizes()

    if options.is_gtf:
        iterator = GTF.transcript_iterator(GTF.iterator(options.stdin))
    else:
        gffs = GTF.iterator(options.stdin)
        if options.merge:
            iterator = GTF.joined_iterator(gffs)
        else:
            iterator = GTF.chunk_iterator(gffs)

    masks = None
    if options.filename_masks:
        masks = {}
        with open(options.filename_masks, "r") as infile:
            e = GTF.readAsIntervals(GFF.iterator(infile))

        # convert intervals to intersectors
        for contig in e.keys():
            intersector = bx.intervals.intersection.Intersecter()
            for start, end in e[contig]:
                intersector.add_interval(bx.intervals.Interval(start, end))
            masks[contig] = intersector

    ninput, noutput, nmasked, nskipped_masked = 0, 0, 0, 0
    nskipped_length = 0
    nskipped_noexons = 0

    feature = options.feature

#    for item in iterator:
# print len(item) # 3, 2
#	for i in item:
# print len(i) # 9, 9, 9, 9, 9
#	   print i.contig
#	   print i.strand
#	   print i.transcript_id

    # iterator is a list containing groups (lists) of features.
    # Each group of features have in common the same transcript ID, in case of
    # GTF files.
    for ichunk in iterator:

        ninput += 1

        if feature:
            chunk = filter(lambda x: x.feature == feature, ichunk)
        else:
            chunk = ichunk

        if len(chunk) == 0:
            nskipped_noexons += 1
            E.info("no features in entry from "
                   "%s:%i..%i - %s" % (ichunk[0].contig,
                                       ichunk[0].start,
                                       ichunk[0].end,
                                       str(ichunk[0])))
            continue

        contig, strand = chunk[0].contig, chunk[0].strand
        if options.is_gtf:
            name = chunk[0].transcript_id
        else:
            name = str(chunk[0].attributes)

        lcontig = contigs[contig]
        positive = Genomics.IsPositiveStrand(strand)
        intervals = [(x.start, x.end) for x in chunk]
        intervals.sort()

        if masks:
            if contig in masks:
                masked_regions = []
                for start, end in intervals:
                    masked_regions += [(x.start, x.end)
                                       for x in masks[contig].find(start, end)]

                masked_regions = Intervals.combine(masked_regions)
                if len(masked_regions):
                    nmasked += 1

                if options.remove_masked_regions:
                    intervals = Intervals.truncate(intervals, masked_regions)
                else:
                    raise "unimplemented"

                if len(intervals) == 0:
                    nskipped_masked += 1
                    if options.loglevel >= 1:
                        options.stdlog.write("# skipped because fully masked: "
                                             "%s: regions=%s masks=%s\n" %
                                             (name,
                                              str([(x.start,
                                                    x.end) for x in chunk]),
                                              masked_regions))
                    continue

        out = intervals

        if options.extend_at:
            if options.extend_at == "5only":
                intervals = [(max(0, intervals[0][0] - options.extend_by),
                             intervals[0][0])]
            elif options.extend_at == "3only":
                intervals = [(intervals[-1][1],
                              min(lcontig,
                                  intervals[-1][1] + options.extend_by))]
            else:
                if options.extend_at in ("5", "both"):
                    intervals[0] = (max(0,
                                        intervals[0][0] - options.extend_by),
                                    intervals[0][1])
                if options.extend_at in ("3", "both"):
                    intervals[-1] = (intervals[-1][0],
                                     min(lcontig,
                                         intervals[-1][1] + options.extend_by))

        if not positive:
            intervals = [(lcontig - x[1], lcontig - x[0])
                         for x in intervals[::-1]]
            out.reverse()

        s = [fasta.getSequence(contig, strand, start, end)
             for start, end in intervals]
        # IMS: allow for masking of sequences
        s = Masker.maskSequences(s, options.masker)
        l = sum([len(x) for x in s])
        if (l < options.min_length or
           (options.max_length and l > options.max_length)):
            nskipped_length += 1
            if options.loglevel >= 1:
                options.stdlog.write("# skipped because length out of bounds "
                                     "%s: regions=%s len=%i\n" %
                                     (name, str(intervals), l))
                continue

        options.stdout.write(">%s %s:%s:%s\n%s\n" % (name,
                                                     contig,
                                                     strand,
                                                     ";".join(
                                                         ["%i-%i" %
                                                          x for x in out]),
                                                     "\n".join(s)))

        noutput += 1

    E.info("ninput=%i, noutput=%i, nmasked=%i, nskipped_noexons=%i, "
           "nskipped_masked=%i, nskipped_length=%i" %
           (ninput, noutput, nmasked, nskipped_noexons,
            nskipped_masked, nskipped_length))

    E.Stop()

if __name__ == "__main__":
    sys.exit(main(sys.argv))
<|endoftext|>


<commit_before>from django.db import models

from nodeconductor.structure.managers import StructureQueryset

class InstanceQueryset(StructureQueryset):
    """ Hack that allow to filter iaas instances based on service_project_links and services """

    def filter(self, *args, **kwargs):
        kwargs = kwargs.copy()
        for key in kwargs:
            if 'service_project_link' in key:
                kwargs[key.replace('service_project_link', 'cloud_project_membership')] = kwargs.pop(key)
        for key in kwargs:
            if 'service' in key:
                kwargs[key.replace('service', 'cloud')] = kwargs.pop(key)
        return super(InstanceQueryset, self).filter(*args, **kwargs)

    def order_by(self, *keys):
        keys = [key.replace('service_project_link', 'cloud_project_membership') for key in keys]
        keys = [key.replace('service', 'cloud') for key in keys]
        return super(InstanceQueryset, self).order_by(*keys)

InstanceManager = models.Manager.from_queryset(InstanceQueryset)
<commit_msg>Do not iterate 2 times over kwargs<commit_after>from django.db import models

from nodeconductor.structure.managers import StructureQueryset

class InstanceQueryset(StructureQueryset):
    """ Hack that allow to filter iaas instances based on service_project_links and services """

    def filter(self, *args, **kwargs):
        kwargs = kwargs.copy()
        for key in kwargs:
            new_key = key.replace('service_project_link', 'cloud_project_membership')
            new_key = new_key.replace('service', 'cloud')
            kwargs[new_key] = kwargs.pop(key)
        return super(InstanceQueryset, self).filter(*args, **kwargs)

    def order_by(self, *keys):
        keys = [key.replace('service_project_link', 'cloud_project_membership') for key in keys]
        keys = [key.replace('service', 'cloud') for key in keys]
        return super(InstanceQueryset, self).order_by(*keys)

InstanceManager = models.Manager.from_queryset(InstanceQueryset)
<|endoftext|>