<commit_before>import logging
import re
import string
import types
from datetime import tzinfo, timedelta, datetime
from feed import Entry

__pychecker__ = 'unusednames=parser'

log = logging.getLogger('series')

class SerieParser:
    qualities = ['1080p', '1080', '720p', '720', 'hr', 'dvd', 'dvdrip', 'hdtv', 'pdtv', 'dsr', 'dsrip', 'unknown']

    def __init__(self):
        # name of the serie
        self.name = None
        # data to parse
        self.data = None

        self.ep_regexps = ['s(\d+)e(\d+)', 's(\d+)ep(\d+)', '(\d+)x(\d+)']
        self.id_regexps = ['(\d\d\d\d).(\d+).(\d+)', '(\d+).(\d+).(\d\d\d\d)']
        self.name_regexps = []
        # parse produces these
        self.season = None
        self.episode = None
        self.id = None

        self.quality = 'unknown'
        # false if item does not match serie
        self.valid = False
        # optional for storing entry from which this instance is made from
        self.entry = None

    def parse(self):
        if not self.name or not self.data:
            raise Exception('SerieParser missing either name or data')
        if not isinstance(self.name, basestring):
            raise Exception('SerieParser name is not a string, got %s' % repr(self.name))
        if not isinstance(self.data, basestring):
            raise Exception('SerieParser data is not a string, got %s' % repr(self.data))
        def clean(s):
            res = s
            r = [('.', ' '), ('_', ' '),
                 ('[', ' '), (']', ' ')]
            for p in r:
                res = res.replace(*p)
            while (res.find('  ')!=-1):
                res = res.replace('  ', ' ')
            return res.lower()

        name = clean(self.name)
        data = clean(self.data)

        #log.debug('name: %s data: %s' % (name, data))

        name_parts = name.split(' ')
        data_parts = data.split(' ')

        # regexp name matching
        if self.name_regexps:
            name_matches = False
            # use all specified regexps to this data
            for name_re in self.name_regexps:
                m = re.search(name_re, self.data, re.IGNORECASE|re.UNICODE)
                if m:
                    name_matches = True
                    break
            if not name_matches:
                # leave this invalid
                #log.debug('FAIL: name regexps do not match')
                return
        else:
            # try to use given name old fashion way
            for part in name_parts:
                if part in data_parts:
                    data_parts.remove(part)
                else:
                    #log.debug('FAIL: part %s not found from %s' % (part, data_parts))
                    # leave this invalid
                    return

        # seach quality
        for part in data_parts:
            # search for quality
            if part in self.qualities:
                if self.qualities.index(part) < self.qualities.index(self.quality):
                    log.debug('%s storing quality %s' % (self.name, part))
                    self.quality = part
                else:
                    log.debug('%s ignoring quality %s because found better %s' % (self.name, part, self.quality))

        # search for season and episode number
        for ep_re in self.ep_regexps:
            m = re.search(ep_re, self.data, re.IGNORECASE|re.UNICODE)
            if m:
                log.debug('found episode number with regexp %s' % ep_re)
                season, episode = m.groups()
                self.season = int(season)
                self.episode = int(episode)
                self.valid = True
                self.id = "S%sE%s" % (self.season, self.episode)
                return

        # search for id
        for id_re in self.id_regexps:
            m = re.search(id_re, self.data, re.IGNORECASE|re.UNICODE)
            if m:
                log.debug('found id with regexp %s' % id_re)
                self.id = string.join(m.groups(), '-')
                self.valid = True
                return

        log.debug('FAIL: unable to find any id')

    def identifier(self):
        """Return identifier for parsed episode"""
        if not self.valid: raise Exception('Serie flagged invalid')
        return self.id

    def __str__(self):
        valid = 'INVALID'
        if self.valid:
            valid = 'OK'
        return 'serie: %s, id: %s season: %s episode: %s quality: %s status: %s' % (str(self.name), str(self.id), str(self.season), str(self.episode), str(self.quality), valid)

class FilterSeries:

    """
        Intelligent filter for tv-series. This solves duplicate downloads
        problem that occurs when using patterns (regexp) matching since same
        episode is often released by multiple groups.

        Example configuration:

        series:
          - some series
          - another series

        If "some serie" and "another serie" have understandable episode
        numbering any given episode is downloaded only once.

        So if we get same episode twice:

        Some.Series.S2E10.More.Text
        Some.Series.S2E10.Something.Else

        Only first file is downloaded.

        If different qualities come available at the same moment,
        flexget will always download the best one (up to 720p by default).

        Supports default settings trough settings block in configuration file.

        Advanced usage with regexps:
        ----------------------------

        The standard name matching is not perfect, if you're used to working with regexps you can
        specify regexp that is used to test if entry is a defined series.

        You can also give regexps to episode number matching or unique id matching if it doesn't have
        normal episode numbering scheme (season, episode).

        Example:

        series:
          - some serie:
              name_patterns: ^some.serie
              ep_patterns: (\d\d)-(\d\d\d)  # must return TWO groups
              id_patterns: (\d\d\d)         # can return any number of groups

        Timeframe:
        ----------

        Series filter allows you to specify a timeframe for each series in which
        flexget waits better quality.

        Example configuration:

        series:
          - some series:
              timeframe:
                hours: 4
                enough: 720p
          - another series
          - third series

        In this example when a epsisode of 'some serie' appears, flexget will wait
        for 4 hours in case and then proceeds to download best quality available.

        The enough parameter will tell the quality that you find good enough to start
        downloading without waiting whole timeframe. If qualities meeting enough parameter
        and above are available, flexget will prefer the enough. Ie. if enough value is set
        to 'hdtv' and qualities dsk, hdtv and 720p are available, hdtv will be chosen.
        If we take hdtv off from list, 720p would be downloaded.

        Enough has default value of 720p.

        Possible values for enough (in order): 1080p, 1080, 720p, 720, hr, dvd, hdtv, dsr, dsrip

        Custom path:
        ------------

        Specify download path for series.

        Example:

        series:
          - some series:
              path: ~/download/some_series/
          - another series
          - third series

        Example with timeframe:

        series:
          - some series:
              timeframe:
                hours: 4
              path: ~/download/some_series/
          - another series
          - third series

    """

    def register(self, manager, parser):
        manager.register('series')

    def validate(self, config):
        """Validate configuration format for this module"""
        from validator import ListValidator
        serie = ListValidator()
        # just plain names
        serie.accept(str)
        # or "bundles" with serie name as key ..
        bundle = serie.accept(dict)
        # prevent invalid indentation level
        bundle.reject_keys(['path', 'timeframe', 'name_patterns', 'ep_patterns', 'id_patterns'])
        # accept serie name, which can be anything ...
        options = bundle.accept_any_key(dict)
        options.accept('path', str)
        # these patterns can be given in as a single string ..
        options.accept('name_patterns', str)
        options.accept('ep_patterns', str)
        options.accept('id_patterns', str)
        # .. or as list containing strings
        options.accept('name_patterns', list).accept(str)
        options.accept('ep_patterns', list).accept(str)
        options.accept('id_patterns', list).accept(str)
        # timeframe dict
        timeframe = options.accept('timeframe', dict)
        timeframe.accept('hours', int)
        timeframe.accept('enough', SerieParser.qualities)
        serie.validate(config)
        return serie.errors.messages

    def feed_input(self, feed):
        """Retrieve stored series from cache, incase they've been expired from feed while waiting"""
        for name in feed.config.get('series', []):
            if type(name) == types.DictType:
                name = name.items()[0][0]
            serie = feed.cache.get(name)
            if not serie: continue
            for identifier in serie.keys():
                for quality in SerieParser.qualities:
                    if quality=='info': continue # a hack, info dict is not quality
                    entry = serie[identifier].get(quality)
                    if not entry: continue
                    # check if episode is still in feed, if not then add it
                    exists = False
                    for feed_entry in feed.entries:
                        if feed_entry['title'] == entry['title'] and feed_entry['url'] == entry['url']:
                            exists = True
                    if not exists:
                        log.debug('restoring entry %s from cache' % entry['title'])
                        # TODO: temp fix, do better
                        e = Entry()
                        e['title'] = entry['title']
                        e['url'] = entry['url']
                        feed.entries.append(e)

    def cmp_serie_quality(self, s1, s2):
        return self.cmp_quality(s1.quality, s2.quality)

    def cmp_quality(self, q1, q2):
        return cmp(SerieParser.qualities.index(q1), SerieParser.qualities.index(q2))

    def feed_filter(self, feed):
        """Filter series"""
        for name in feed.config.get('series', []):
            # start with default settings
            conf = feed.manager.get_settings('series', {})
            if type(name) == types.DictType:
                name, conf = name.items()[0]
                # merge with default settings
                conf = feed.manager.get_settings('series', conf)

            def get_as_array(conf, key):
                v = conf.get(key, [])
                if type(v) in types.StringTypes:
                    return [v]
                return v

            ep_patterns = get_as_array(conf, 'ep_patterns')
            id_patterns = get_as_array(conf, 'id_patterns')
            name_patterns = get_as_array(conf, 'name_patterns')

            series = {} # ie. S1E2: [Serie, Serie, ..]
            for entry in feed.entries:
                serie = SerieParser()
                serie.name = name
                serie.data = entry['title']
                serie.ep_regexps.extend(ep_patterns)
                serie.id_regexps.extend(id_patterns)
                serie.name_regexps.extend(name_patterns)
                try:
                    serie.parse()
                except Exception, e:
                    feed.fail(entry)
                    log.error(e)
                # serie is not valid if it does not match given name / regexp or fails with exception
                if not serie.valid:
                    log.debug('%s is not serie %s' % (entry['title'], name))
                    continue
                # set custom download path
                if conf.has_key('path'):
                    log.debug('setting %s custom path to %s' % (entry['title'], conf.get('path')))
                    entry['path'] = conf.get('path')
                serie.entry = entry
                self.store(feed, serie, entry)
                # add this episode into list of available episodes
                eps = series.setdefault(serie.identifier(), [])
                eps.append(serie)

            # choose episode from available qualities
            for identifier, eps in series.iteritems():
                if not eps: continue
                eps.sort(self.cmp_serie_quality)
                best = eps[0]

                # episode (with this id) has been downloaded
                if self.downloaded(feed, best):
                    log.debug('Rejecting all instances of %s' % identifier)
                    for ep in eps:
                        feed.reject(ep.entry)
                    continue

                # timeframe present
                if conf.has_key('timeframe'):
                    tconf = conf.get('timeframe')
                    hours = tconf.get('hours', 0)
                    enough = tconf.get('enough', '720p')

                    if not enough in SerieParser.qualities:
                        log.error('Parameter enough has unknown value: %s' % enough)

                    # scan for enough, starting from worst quality (reverse)
                    eps.reverse()
                    found_enough = False
                    for ep in eps:
                        if self.cmp_quality(enough, ep.quality) >= 0: # 1=greater, 0=equal, -1=does not meet
                            log.debug('Episode %s meets quality %s' % (ep.entry['title'], enough))
                            self.accept_serie(feed, ep)
                            found_enough = True
                            break
                    if found_enough:
                        continue

                    # timeframe
                    diff = datetime.today() - self.get_first_seen(feed, best)
                    age_hours = divmod(diff.seconds, 60*60)[0]
                    log.debug('age_hours %i - %s ' % (age_hours, best))
                    log.debug('best ep in %i hours is %s' % (hours, best))
                    if age_hours >= hours:
                        self.accept_serie(feed, best)
                    else:
                        log.debug('Timeframe ignoring %s' % (best.entry['title']))
                else:
                    # no timeframe, just choose best
                    self.accept_serie(feed, best)

        # filter ALL entries, only previously accepted will remain
        # other modules may still accept entries
        for entry in feed.entries:
            feed.filter(entry)

    def accept_serie(self, feed, serie):
        """Helper method for accepting serie"""
        log.debug('Accepting %s' % serie.entry)
        feed.accept(serie.entry)
        # store serie instance to entry for later use
        serie.entry['serie_parser'] = serie
        # remove entry instance from serie instance, not needed any more (save memory, circular reference?)
        serie.entry = None

    def reject_eps(self, feed, eps):
        for ep in eps:
            feed.reject(ep.entry)

    def get_first_seen(self, feed, serie):
        """Return datetime when this episode of serie was first seen"""
        fs = feed.cache.get(serie.name)[serie.identifier()]['info']['first_seen']
        return datetime(*fs)

    def downloaded(self, feed, serie):
        """Return true if this episode of serie is downloaded"""
        cache = feed.cache.get(serie.name)
        return cache[serie.identifier()]['info']['downloaded']

    def store(self, feed, serie, entry):
        """Stores serie into cache"""
        # serie_name:
        #   S1E2:
        #     info:
        #       first_seen: <time>
        #       downloaded: <boolean>
        #     720p: <entry>
        #     dsr: <entry>
        cache = feed.cache.storedefault(serie.name, {}, 30)
        episode = cache.setdefault(serie.identifier(), {})
        info = episode.setdefault('info', {})
        # store and make first seen time
        info.setdefault('first_seen', list(datetime.today().timetuple())[:-4])
        info.setdefault('downloaded', False)
        ec = {}
        ec['title'] = entry['title']
        ec['url'] = entry['url']
        episode.setdefault(serie.quality, ec)

    def mark_downloaded(self, feed, serie):
        log.debug('marking %s as downloaded' % serie.identifier())
        cache = feed.cache.get(serie.name)
        cache[serie.identifier()]['info']['downloaded'] = True

    def feed_exit(self, feed):
        """Learn succeeded episodes"""
        for entry in feed.entries:
            serie = entry.get('serie_parser')
            if serie:
                self.mark_downloaded(feed, serie)
            else:
                log.debug('Entry %s is not valid serie' % entry['title'])
<commit_msg>Small fix to FilterSeries, allows integer as a serie name (ie. 24)<commit_after>import logging
import re
import string
import types
from datetime import tzinfo, timedelta, datetime
from feed import Entry

__pychecker__ = 'unusednames=parser'

log = logging.getLogger('series')

class SerieParser:
    qualities = ['1080p', '1080', '720p', '720', 'hr', 'dvd', 'dvdrip', 'hdtv', 'pdtv', 'dsr', 'dsrip', 'unknown']

    def __init__(self):
        # name of the serie
        self.name = None
        # data to parse
        self.data = None

        self.ep_regexps = ['s(\d+)e(\d+)', 's(\d+)ep(\d+)', '(\d+)x(\d+)']
        self.id_regexps = ['(\d\d\d\d).(\d+).(\d+)', '(\d+).(\d+).(\d\d\d\d)']
        self.name_regexps = []
        # parse produces these
        self.season = None
        self.episode = None
        self.id = None

        self.quality = 'unknown'
        # false if item does not match serie
        self.valid = False
        # optional for storing entry from which this instance is made from
        self.entry = None

    def parse(self):
        if not self.name or not self.data:
            raise Exception('SerieParser missing either name or data')
        if not isinstance(self.name, basestring):
            raise Exception('SerieParser name is not a string, got %s' % repr(self.name))
        if not isinstance(self.data, basestring):
            raise Exception('SerieParser data is not a string, got %s' % repr(self.data))
        def clean(s):
            res = s
            r = [('.', ' '), ('_', ' '),
                 ('[', ' '), (']', ' ')]
            for p in r:
                res = res.replace(*p)
            while (res.find('  ')!=-1):
                res = res.replace('  ', ' ')
            return res.lower()

        name = clean(self.name)
        data = clean(self.data)

        #log.debug('name: %s data: %s' % (name, data))

        name_parts = name.split(' ')
        data_parts = data.split(' ')

        # regexp name matching
        if self.name_regexps:
            name_matches = False
            # use all specified regexps to this data
            for name_re in self.name_regexps:
                m = re.search(name_re, self.data, re.IGNORECASE|re.UNICODE)
                if m:
                    name_matches = True
                    break
            if not name_matches:
                # leave this invalid
                #log.debug('FAIL: name regexps do not match')
                return
        else:
            # try to use given name old fashion way
            for part in name_parts:
                if part in data_parts:
                    data_parts.remove(part)
                else:
                    #log.debug('FAIL: part %s not found from %s' % (part, data_parts))
                    # leave this invalid
                    return

        # seach quality
        for part in data_parts:
            # search for quality
            if part in self.qualities:
                if self.qualities.index(part) < self.qualities.index(self.quality):
                    log.debug('%s storing quality %s' % (self.name, part))
                    self.quality = part
                else:
                    log.debug('%s ignoring quality %s because found better %s' % (self.name, part, self.quality))

        # search for season and episode number
        for ep_re in self.ep_regexps:
            m = re.search(ep_re, self.data, re.IGNORECASE|re.UNICODE)
            if m:
                log.debug('found episode number with regexp %s' % ep_re)
                season, episode = m.groups()
                self.season = int(season)
                self.episode = int(episode)
                self.valid = True
                self.id = "S%sE%s" % (self.season, self.episode)
                return

        # search for id
        for id_re in self.id_regexps:
            m = re.search(id_re, self.data, re.IGNORECASE|re.UNICODE)
            if m:
                log.debug('found id with regexp %s' % id_re)
                self.id = string.join(m.groups(), '-')
                self.valid = True
                return

        log.debug('FAIL: unable to find any id')

    def identifier(self):
        """Return identifier for parsed episode"""
        if not self.valid: raise Exception('Serie flagged invalid')
        return self.id

    def __str__(self):
        valid = 'INVALID'
        if self.valid:
            valid = 'OK'
        return 'serie: %s, id: %s season: %s episode: %s quality: %s status: %s' % (str(self.name), str(self.id), str(self.season), str(self.episode), str(self.quality), valid)

class FilterSeries:

    """
        Intelligent filter for tv-series. This solves duplicate downloads
        problem that occurs when using patterns (regexp) matching since same
        episode is often released by multiple groups.

        Example configuration:

        series:
          - some series
          - another series

        If "some serie" and "another serie" have understandable episode
        numbering any given episode is downloaded only once.

        So if we get same episode twice:

        Some.Series.S2E10.More.Text
        Some.Series.S2E10.Something.Else

        Only first file is downloaded.

        If different qualities come available at the same moment,
        flexget will always download the best one (up to 720p by default).

        Supports default settings trough settings block in configuration file.

        Advanced usage with regexps:
        ----------------------------

        The standard name matching is not perfect, if you're used to working with regexps you can
        specify regexp that is used to test if entry is a defined series.

        You can also give regexps to episode number matching or unique id matching if it doesn't have
        normal episode numbering scheme (season, episode).

        Example:

        series:
          - some serie:
              name_patterns: ^some.serie
              ep_patterns: (\d\d)-(\d\d\d)  # must return TWO groups
              id_patterns: (\d\d\d)         # can return any number of groups

        Timeframe:
        ----------

        Series filter allows you to specify a timeframe for each series in which
        flexget waits better quality.

        Example configuration:

        series:
          - some series:
              timeframe:
                hours: 4
                enough: 720p
          - another series
          - third series

        In this example when a epsisode of 'some serie' appears, flexget will wait
        for 4 hours in case and then proceeds to download best quality available.

        The enough parameter will tell the quality that you find good enough to start
        downloading without waiting whole timeframe. If qualities meeting enough parameter
        and above are available, flexget will prefer the enough. Ie. if enough value is set
        to 'hdtv' and qualities dsk, hdtv and 720p are available, hdtv will be chosen.
        If we take hdtv off from list, 720p would be downloaded.

        Enough has default value of 720p.

        Possible values for enough (in order): 1080p, 1080, 720p, 720, hr, dvd, hdtv, dsr, dsrip

        Custom path:
        ------------

        Specify download path for series.

        Example:

        series:
          - some series:
              path: ~/download/some_series/
          - another series
          - third series

        Example with timeframe:

        series:
          - some series:
              timeframe:
                hours: 4
              path: ~/download/some_series/
          - another series
          - third series

    """

    def register(self, manager, parser):
        manager.register('series')

    def validate(self, config):
        """Validate configuration format for this module"""
        from validator import ListValidator
        serie = ListValidator()
        # just plain names
        serie.accept(str)
        # or "bundles" with serie name as key ..
        bundle = serie.accept(dict)
        # prevent invalid indentation level
        bundle.reject_keys(['path', 'timeframe', 'name_patterns', 'ep_patterns', 'id_patterns'])
        # accept serie name, which can be anything ...
        options = bundle.accept_any_key(dict)
        options.accept('path', str)
        # these patterns can be given in as a single string ..
        options.accept('name_patterns', str)
        options.accept('ep_patterns', str)
        options.accept('id_patterns', str)
        # .. or as list containing strings
        options.accept('name_patterns', list).accept(str)
        options.accept('ep_patterns', list).accept(str)
        options.accept('id_patterns', list).accept(str)
        # timeframe dict
        timeframe = options.accept('timeframe', dict)
        timeframe.accept('hours', int)
        timeframe.accept('enough', SerieParser.qualities)
        serie.validate(config)
        return serie.errors.messages

    def feed_input(self, feed):
        """Retrieve stored series from cache, incase they've been expired from feed while waiting"""
        for name in feed.config.get('series', []):
            if type(name) == types.DictType:
                name = name.items()[0][0]
            serie = feed.cache.get(name)
            if not serie: continue
            for identifier in serie.keys():
                for quality in SerieParser.qualities:
                    if quality=='info': continue # a hack, info dict is not quality
                    entry = serie[identifier].get(quality)
                    if not entry: continue
                    # check if episode is still in feed, if not then add it
                    exists = False
                    for feed_entry in feed.entries:
                        if feed_entry['title'] == entry['title'] and feed_entry['url'] == entry['url']:
                            exists = True
                    if not exists:
                        log.debug('restoring entry %s from cache' % entry['title'])
                        # TODO: temp fix, do better
                        e = Entry()
                        e['title'] = entry['title']
                        e['url'] = entry['url']
                        feed.entries.append(e)

    def cmp_serie_quality(self, s1, s2):
        return self.cmp_quality(s1.quality, s2.quality)

    def cmp_quality(self, q1, q2):
        return cmp(SerieParser.qualities.index(q1), SerieParser.qualities.index(q2))

    def feed_filter(self, feed):
        """Filter series"""
        for name in feed.config.get('series', []):
            # force name to a string
            name = str(name)

            # start with default settings
            conf = feed.manager.get_settings('series', {})
            if type(name) == types.DictType:
                name, conf = name.items()[0]
                # merge with default settings
                conf = feed.manager.get_settings('series', conf)

            def get_as_array(conf, key):
                v = conf.get(key, [])
                if type(v) in types.StringTypes:
                    return [v]
                return v

            ep_patterns = get_as_array(conf, 'ep_patterns')
            id_patterns = get_as_array(conf, 'id_patterns')
            name_patterns = get_as_array(conf, 'name_patterns')

            series = {} # ie. S1E2: [Serie, Serie, ..]
            for entry in feed.entries:
                serie = SerieParser()
                serie.name = name
                serie.data = entry['title']
                serie.ep_regexps.extend(ep_patterns)
                serie.id_regexps.extend(id_patterns)
                serie.name_regexps.extend(name_patterns)
                try:
                    serie.parse()
                except Exception, e:
                    feed.fail(entry)
                    log.error(e)
                # serie is not valid if it does not match given name / regexp or fails with exception
                if not serie.valid:
                    log.debug('%s is not serie %s' % (entry['title'], name))
                    continue
                # set custom download path
                if conf.has_key('path'):
                    log.debug('setting %s custom path to %s' % (entry['title'], conf.get('path')))
                    entry['path'] = conf.get('path')
                serie.entry = entry
                self.store(feed, serie, entry)
                # add this episode into list of available episodes
                eps = series.setdefault(serie.identifier(), [])
                eps.append(serie)

            # choose episode from available qualities
            for identifier, eps in series.iteritems():
                if not eps: continue
                eps.sort(self.cmp_serie_quality)
                best = eps[0]

                # episode (with this id) has been downloaded
                if self.downloaded(feed, best):
                    log.debug('Rejecting all instances of %s' % identifier)
                    for ep in eps:
                        feed.reject(ep.entry)
                    continue

                # timeframe present
                if conf.has_key('timeframe'):
                    tconf = conf.get('timeframe')
                    hours = tconf.get('hours', 0)
                    enough = tconf.get('enough', '720p')

                    if not enough in SerieParser.qualities:
                        log.error('Parameter enough has unknown value: %s' % enough)

                    # scan for enough, starting from worst quality (reverse)
                    eps.reverse()
                    found_enough = False
                    for ep in eps:
                        if self.cmp_quality(enough, ep.quality) >= 0: # 1=greater, 0=equal, -1=does not meet
                            log.debug('Episode %s meets quality %s' % (ep.entry['title'], enough))
                            self.accept_serie(feed, ep)
                            found_enough = True
                            break
                    if found_enough:
                        continue

                    # timeframe
                    diff = datetime.today() - self.get_first_seen(feed, best)
                    age_hours = divmod(diff.seconds, 60*60)[0]
                    log.debug('age_hours %i - %s ' % (age_hours, best))
                    log.debug('best ep in %i hours is %s' % (hours, best))
                    if age_hours >= hours:
                        self.accept_serie(feed, best)
                    else:
                        log.debug('Timeframe ignoring %s' % (best.entry['title']))
                else:
                    # no timeframe, just choose best
                    self.accept_serie(feed, best)

        # filter ALL entries, only previously accepted will remain
        # other modules may still accept entries
        for entry in feed.entries:
            feed.filter(entry)

    def accept_serie(self, feed, serie):
        """Helper method for accepting serie"""
        log.debug('Accepting %s' % serie.entry)
        feed.accept(serie.entry)
        # store serie instance to entry for later use
        serie.entry['serie_parser'] = serie
        # remove entry instance from serie instance, not needed any more (save memory, circular reference?)
        serie.entry = None

    def reject_eps(self, feed, eps):
        for ep in eps:
            feed.reject(ep.entry)

    def get_first_seen(self, feed, serie):
        """Return datetime when this episode of serie was first seen"""
        fs = feed.cache.get(serie.name)[serie.identifier()]['info']['first_seen']
        return datetime(*fs)

    def downloaded(self, feed, serie):
        """Return true if this episode of serie is downloaded"""
        cache = feed.cache.get(serie.name)
        return cache[serie.identifier()]['info']['downloaded']

    def store(self, feed, serie, entry):
        """Stores serie into cache"""
        # serie_name:
        #   S1E2:
        #     info:
        #       first_seen: <time>
        #       downloaded: <boolean>
        #     720p: <entry>
        #     dsr: <entry>
        cache = feed.cache.storedefault(serie.name, {}, 30)
        episode = cache.setdefault(serie.identifier(), {})
        info = episode.setdefault('info', {})
        # store and make first seen time
        info.setdefault('first_seen', list(datetime.today().timetuple())[:-4])
        info.setdefault('downloaded', False)
        ec = {}
        ec['title'] = entry['title']
        ec['url'] = entry['url']
        episode.setdefault(serie.quality, ec)

    def mark_downloaded(self, feed, serie):
        log.debug('marking %s as downloaded' % serie.identifier())
        cache = feed.cache.get(serie.name)
        cache[serie.identifier()]['info']['downloaded'] = True

    def feed_exit(self, feed):
        """Learn succeeded episodes"""
        for entry in feed.entries:
            serie = entry.get('serie_parser')
            if serie:
                self.mark_downloaded(feed, serie)
            else:
                log.debug('Entry %s is not valid serie' % entry['title'])
<|endoftext|>


<commit_before>''' SmartAPI Configuration '''

from copy import deepcopy

from biothings.web.settings.default import QUERY_KWARGS, ANNOTATION_KWARGS

from config_key import *

# *****************************************************************************
# Credentials
# *****************************************************************************
# Define in <project_folder>/config_key.py:
#   COOKIE_SECRET = '<Any Random String>'
#   GITHUB_CLIENT_ID = '<your Github application Client ID>'
#   GITHUB_CLIENT_SECRET = '<your Github application Client Secret>'

# *****************************************************************************
# User Input Control
# *****************************************************************************
QUERY_KWARGS = deepcopy(QUERY_KWARGS)
QUERY_KWARGS['*']['authors'] = {'type': list, 'group': 'esqb'}
QUERY_KWARGS['*']['tags'] = {'type': list, 'group': 'esqb'}

ANNOTATION_KWARGS = deepcopy(ANNOTATION_KWARGS)
# all biothings annotation common keywords are consolidated into
# the GET endpoint because we only use this method to read data
# other methods will be used to modify the records in SmartAPI
ANNOTATION_KWARGS['GET'].update(ANNOTATION_KWARGS.pop('*'))
ANNOTATION_KWARGS['POST'].clear()
# when the id is not present, the annotation endpoint will act like
# a get_all endpoint to scroll over all results through pagination
ANNOTATION_KWARGS['GET']['id']['required'] = False
# because SmartAPI documents can be large, set a small default
# return size for get_all operation and limit the maximum to 10,
# otherwise the default is 1000 in traditional biothings applications.
ANNOTATION_KWARGS['GET']['size']['default'] = 5
ANNOTATION_KWARGS['GET']['size']['max'] = 10
# the from keyword is used with size for pagination in scrolling operation
# add from_ as an alias to be compatible with the earlier version design
ANNOTATION_KWARGS['GET']['from'] = deepcopy(QUERY_KWARGS['GET']['from'])
ANNOTATION_KWARGS['GET']['from']['default'] = 0
ANNOTATION_KWARGS['GET']['from']['alias'] = ('skip', 'from_')
# traditionally, raw parameter takes two boolean values, for compatibility
# reason, we additionally support one more level to return the metadata.
# now raw can take 0 (without metadata), 1 (with metadata), and 2 (ES raw).
# additionally, since some operations are performed in the transform stage,
# it's necessary to cc this option to transform group. moreover, since raw=1
# is used to return metadata, and we have supported meta=1 in similar apps
# like discovery app, make meta an alias to raw for convenience.
ANNOTATION_KWARGS['GET']['raw']['type'] = int
ANNOTATION_KWARGS['GET']['raw']['default'] = 0
ANNOTATION_KWARGS['GET']['raw']['group'] = ('control', 'transform')
ANNOTATION_KWARGS['GET']['raw']['alias'] = 'meta'
# since there's internal field ordering for OpenAPI and Swagger documents, by default
# turn off the alphabetical ordering of keys defaulted in most biothings applictaions.
ANNOTATION_KWARGS['GET']['_sorted']['default'] = False

# *****************************************************************************
# Elasticsearch
# *****************************************************************************
ES_INDICES = {'metadata': 'smartapi_docs'}

# *****************************************************************************
# Tornado URL Patterns
# *****************************************************************************
APP_LIST = [
    (r'/api/?', 'handlers.SmartAPIHandler', {"biothing_type": "metadata"}),  # TO REMOVE
    (r'/api/query/?', 'biothings.web.handlers.QueryHandler', {"biothing_type": "metadata"}),
    (r'/api/validate/?', 'handlers.ValidateHandler'),
    (r'/api/metadata/?', 'handlers.SmartAPIHandler', {"biothing_type": "metadata"}),
    (r'/api/metadata/(.+)/?', 'handlers.SmartAPIHandler', {"biothing_type": "metadata"}),
    (r'/api/suggestion/?', 'handlers.ValueSuggestionHandler'),
]

# biothings web tester will read this
API_VERSION = ''
API_PREFIX = 'api'

# *****************************************************************************
# Biothings SDK Settings
# *****************************************************************************
ACCESS_CONTROL_ALLOW_METHODS = 'HEAD,GET,POST,DELETE,PUT,OPTIONS'
ANNOTATION_DEFAULT_SCOPES = ['_id', '_meta.slug']
ES_QUERY_BUILDER = "pipeline.SmartAPIQueryBuilder"
ES_RESULT_TRANSFORM = "pipeline.SmartAPIResultTransform"
DISABLE_CACHING = True
<commit_msg>remove unnecessary route<commit_after>''' SmartAPI Configuration '''

from copy import deepcopy

from biothings.web.settings.default import QUERY_KWARGS, ANNOTATION_KWARGS

from config_key import *

# *****************************************************************************
# Credentials
# *****************************************************************************
# Define in <project_folder>/config_key.py:
#   COOKIE_SECRET = '<Any Random String>'
#   GITHUB_CLIENT_ID = '<your Github application Client ID>'
#   GITHUB_CLIENT_SECRET = '<your Github application Client Secret>'

# *****************************************************************************
# User Input Control
# *****************************************************************************
QUERY_KWARGS = deepcopy(QUERY_KWARGS)
QUERY_KWARGS['*']['authors'] = {'type': list, 'group': 'esqb'}
QUERY_KWARGS['*']['tags'] = {'type': list, 'group': 'esqb'}

ANNOTATION_KWARGS = deepcopy(ANNOTATION_KWARGS)
# all biothings annotation common keywords are consolidated into
# the GET endpoint because we only use this method to read data
# other methods will be used to modify the records in SmartAPI
ANNOTATION_KWARGS['GET'].update(ANNOTATION_KWARGS.pop('*'))
ANNOTATION_KWARGS['POST'].clear()
# when the id is not present, the annotation endpoint will act like
# a get_all endpoint to scroll over all results through pagination
ANNOTATION_KWARGS['GET']['id']['required'] = False
# because SmartAPI documents can be large, set a small default
# return size for get_all operation and limit the maximum to 10,
# otherwise the default is 1000 in traditional biothings applications.
ANNOTATION_KWARGS['GET']['size']['default'] = 5
ANNOTATION_KWARGS['GET']['size']['max'] = 10
# the from keyword is used with size for pagination in scrolling operation
# add from_ as an alias to be compatible with the earlier version design
ANNOTATION_KWARGS['GET']['from'] = deepcopy(QUERY_KWARGS['GET']['from'])
ANNOTATION_KWARGS['GET']['from']['default'] = 0
ANNOTATION_KWARGS['GET']['from']['alias'] = ('skip', 'from_')
# traditionally, raw parameter takes two boolean values, for compatibility
# reason, we additionally support one more level to return the metadata.
# now raw can take 0 (without metadata), 1 (with metadata), and 2 (ES raw).
# additionally, since some operations are performed in the transform stage,
# it's necessary to cc this option to transform group. moreover, since raw=1
# is used to return metadata, and we have supported meta=1 in similar apps
# like discovery app, make meta an alias to raw for convenience.
ANNOTATION_KWARGS['GET']['raw']['type'] = int
ANNOTATION_KWARGS['GET']['raw']['default'] = 0
ANNOTATION_KWARGS['GET']['raw']['group'] = ('control', 'transform')
ANNOTATION_KWARGS['GET']['raw']['alias'] = 'meta'
# since there's internal field ordering for OpenAPI and Swagger documents, by default
# turn off the alphabetical ordering of keys defaulted in most biothings applictaions.
ANNOTATION_KWARGS['GET']['_sorted']['default'] = False

# *****************************************************************************
# Elasticsearch
# *****************************************************************************
ES_INDICES = {'metadata': 'smartapi_docs'}

# *****************************************************************************
# Tornado URL Patterns
# *****************************************************************************
APP_LIST = [
    (r'/api/query/?', 'biothings.web.handlers.QueryHandler', {"biothing_type": "metadata"}),
    (r'/api/validate/?', 'handlers.ValidateHandler'),
    (r'/api/metadata/?', 'handlers.SmartAPIHandler', {"biothing_type": "metadata"}),
    (r'/api/metadata/(.+)/?', 'handlers.SmartAPIHandler', {"biothing_type": "metadata"}),
    (r'/api/suggestion/?', 'handlers.ValueSuggestionHandler'),
]

# biothings web tester will read this
API_VERSION = ''
API_PREFIX = 'api'

# *****************************************************************************
# Biothings SDK Settings
# *****************************************************************************
ACCESS_CONTROL_ALLOW_METHODS = 'HEAD,GET,POST,DELETE,PUT,OPTIONS'
ANNOTATION_DEFAULT_SCOPES = ['_id', '_meta.slug']
ES_QUERY_BUILDER = "pipeline.SmartAPIQueryBuilder"
ES_RESULT_TRANSFORM = "pipeline.SmartAPIResultTransform"
DISABLE_CACHING = True
<|endoftext|>


<commit_before>from utils.exceptions import *
from utils.scraper import Scraper
from bs4 import BeautifulSoup
import os
import math
import pdfkit

class HowStuffWorks(Scraper):

    def __init__(self, base_dir, url_header, log_file):
        super().__init__(log_file)
        self._base_dir = base_dir
        self._completed_urls = []
        self._failed_urls = []
        self._completed_articles_csv = os.path.join(self._base_dir, "articles.csv")
        self._failed_articles_csv = os.path.join(self._base_dir, "articles_failed.csv")

        # Make sure file exists before we try and read form it
        open(self.create_dir(self._completed_articles_csv), 'a').close()
        # Fill self._completed_urls from articles.csv
        for line in open(self._completed_articles_csv, 'r'):
            self._completed_urls.append(line.split(',')[0])

        # Make sure file exists before we try and read form it
        open(self.create_dir(self._failed_articles_csv), 'a').close()
        # Fill self._failed_urls from articles.csv
        for line in open(self._failed_articles_csv, 'r'):
            self._failed_urls.append(line)

        # Use mobile header, the mobile site it much simpler to parse
        #   Reasons:
        #     - Both the Top10 and normal articles have the same html structure
        #     - The Top10 page on the desktop version has broken html and does not parse correctly
        self._url_header = {'User-Agent': 'Mozilla/5.0 (Linux; Android 4.4.4; en-us; Nexus 4 Build/JOP40D) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2307.2 Mobile Safari/537.36'}

    def get_latest(self):
        """
        Will always be 0 because we the parent loop to only run once
        All processing is done below in parse()
        :return: id of the newest item
        """
        self.cprint("##\tGetting newest upload id...\n", log=True)
        return 0

    def parse(self, id_):
        """
        Using BeautifulSoup, parse the page for links then will dig into found links
        :param id_: id of the first page (always 1 in this case)
        :return:
        """
        # Get number of pages to loop through
        url = "http://www.howstuffworks.com/big.htm?page=1"
        # get the html from the url
        try:
            soup = self.get_site(url, self._url_header)
        except RequestsError as e:
            return

        num_pages = int(soup.find("div", {"class": "content"}).h3.get_text().split(' ')[-1].replace(',', ''))
        num_pages = math.ceil(num_pages/500)  # There are 500 results per page

        # Now loop through each page to start parsing the links
        for i in range(1, num_pages + 1):
            self.cprint("Processing link page: " + str(i), log=True)
            url = "http://www.howstuffworks.com/big.htm?page=" + str(i)
            # get the html from the url
            try:
                soup = self.get_site(url, self._url_header)
            except RequestsError as e:
                return

            links = soup.find("ol").find_all("li")
            # Loop through each link on the page
            for link in links:
                url = link.a["href"]

                # Check if url is a quiz (we do not want these, they are interactive flash objects)
                if str(url).endswith('quiz.htm'):
                    continue

                # Check if we already have this article
                if str(url) in self._completed_urls or str(url) in self._failed_urls:
                    continue

                self.cprint("Processing: " + url.split('/')[-1], log=True)

                whole_article = self.parse_article(url)
                # Something went wrong, so skip it
                if whole_article is False:
                    self.log("Something Failed: " + url)
                    self.add_failed(url)
                    continue

                self.cprint("Saving assets: " + whole_article['id'], log=True)
                # Download/save images
                whole_article = self.save_article_images(whole_article)

                # Parse page content for links to make local
                #   - External links will be saved in pdf form
                #   - How Stuff Works links will be changed into a local path to the article
                whole_article = self.process_content_links(whole_article)

                # Save json data in case we need to rebuild the article
                self.save_props(whole_article)

                # Create html file from article
                self.html_template(whole_article)

                # Add article to completed list
                self.add_completed(url, whole_article['abs_path'], whole_article['title'])

        # Everything was successful
        return True

    def add_completed(self, url, article_path, title):
        self._completed_urls.append(url)
        with open(self._completed_articles_csv, 'a') as f:
            f.write(url + "," + article_path + "," + title + "\n")

    def add_failed(self, url):
        self._completed_urls.append(url)
        with open(self._failed_articles_csv, 'a') as f:
            f.write(url + "\n")

    def save_article_images(self, article):
        """
        Saves main image in header of page
        :param article: dict of all article data
        :return: update article dict
        """
        for idx, page in enumerate(article['content']):
            if 'image_orig' in page:
                article['content'][idx]['image_abs'] = self.download_image(article, page['image_orig'])

        return article

    def download_image(self, article, image_url):
        """
        :return: absolute path for image
        """
        abs_image_path = article['abs_path'] + "assets/" + image_url.split('/')[-1]
        self.download(image_url, self._base_dir + abs_image_path, self._url_header)
        return abs_image_path

    def process_content_links(self, article):
        """
        Parse page content for links to make local
          - External links will be saved in pdf form
          - How Stuff Works links will be changed into a local path to the article
        :param article: dict of all article data
        :return: update article dict
        """
        for idx_page, page in enumerate(article['content']):
            page_soup = BeautifulSoup(page['page_content'])
            # Download images and replace src to local file
            images = page_soup.find_all("img")
            if len(images) > 0:
                for image in images:
                    base_img_url = "http://s.hswstatic.com/"

                    img_dl_link = image['src']
                    if not image['src'].startswith(base_img_url):
                        img_dl_link = base_img_url + image['src']

                    abs_src = self.download_image(article, img_dl_link)
                    # Replace the src of downloaded image
                    article['content'][idx_page]['page_content'] = article['content'][idx_page]['page_content'].replace(image['src'], abs_src)

            # Replace HSW links with local path to file, create pdf of external sites and link locally
            # TODO: catch pdfkit errors/timeouts and try a few times before giving up
            # links = page_soup.find_all("a")
            # if len(links) > 0:
            #     for link in links:
            #         if re.match('.*howstuffworks\.com.*', link['href']):
            #             abs_path, full_path = self.get_save_path(link['href'])
            #             new_link = abs_path
            #         else:
            #             pdf_path = article['abs_path'] + "assets/" + link.get_text().replace(' ', '_') + ".pdf"
            #             new_link = pdf_path
            #             # Create pdf of external page
            #             try:
            #                 pdf_file = self._base_dir + pdf_path
            #                 if not os.path.isfile(pdf_file):
            #                     pdfkit.from_url(link['href'], pdf_file)
            #             except Exception as e:
            #                 # If it did not work, put the link back
            #                 new_link = link['href']

            #         # Convert to web safe path
            #         abs_path = urllib.request.pathname2url(new_link)
            #         # Replace link with link to article or a pdf
            #         article['content'][idx_page]['page_content'] = article['content'][idx_page]['page_content'].replace(link['href'], new_link)

        return article

    def get_save_path(self, url, crumbs=None):
        """
        Pass in the url of the article and crumbs list if you have them
        :param url: url of the article
        :param curmbs: list of bread crumbs, if not passed, a request will be sent to parse them
        :return: relative path & save path as a string
        """
        if crumbs is None:
            crumbs = self.get_crumbs(url)

        abs_path = '/articles'
        # Add crumbs to path
        for crumb in crumbs:
            crumb = self.sanitize(crumb.replace(' ', '_'))
            abs_path = os.path.join(abs_path, crumb)

        # Add article title to path
        title = url.split('/')[-1].split('.')[0]
        abs_path = os.path.join(abs_path, title)
        abs_path += '/'
        abs_path = os.path.normcase(abs_path)
        # Create full path
        full_save_path = os.path.normpath(os.path.join(self._base_dir, '.' + abs_path)) + '/'
        full_save_path = os.path.normcase(full_save_path)
        return (abs_path, full_save_path)

    def get_crumbs(self, url=None, soup=None):
        """
        Use soup if you have it, that way les requests are made to the site.
        :param url: url of article
        :param soup: soup of the page
        :return: list of crumbs
        """
        # Only request site if needed
        if soup is not None:
            page_soup = soup
        elif url is not None:
            try:
                page_soup = self.get_site(url, self._url_header)
            except RequestsError as e:
                return
        else:
            return []

        crumbs = []
        for crumb in page_soup.find("div", {"class": "breadcrumb"}).find_all("a"):
            crumbs.append(crumb.get_text().strip())

        return crumbs

    def parse_article(self, url):
        """
        Using BeautifulSoup, parse the article for its data
        :param url: url of the article
        :return:
        """
        try:
            page_soup = self.get_site(url, self._url_header)
        except RequestsError as e:
            return

        # Get links for each page in the article
        links = []
        page_links = page_soup.find("select", {"id": "pageSelector"})

        # Check if article has multipal pages
        if page_links:
            page_links = page_links.find_all("option")
            for page_link in page_links:
                page_number = int(page_link.get_text().split(' ')[0])-1
                page_url = page_link['value']
                links.insert(page_number, page_url)
        else:
            # If article is a single page
            links.append(url)

        try:
            article = {}

            # Get bread crumbs
            article['bread_crumbs'] = self.get_crumbs(soup=page_soup)

            # Get save path
            article['abs_path'], article['save_path'] = self.get_save_path(url, article['bread_crumbs'])

            # Get title & author
            header_soup = page_soup.find("div", {"id": "content-header"})
            article['title'] = header_soup.find("h1").get_text().strip()
            # I need to have a ['id']
            article['id'] = article['title'] if len(article['title']) > 1 else url.split('/')[-1]
            try:
                author = header_soup.find("span", {"class": "content-author"}).a.get_text()
                article['author'] = " ".join(author.split())
            except AttributeError:
                article['author'] = ''

            article['content'] = []  # List of content on each page
            # Parse each page in the article
            for idx, url in enumerate(links):
                self.cprint("Processing page: " + str(idx+1) + " of " + str(len(links)) + " - " + article['title'])
                if idx is not 0:  # We do not need the first page because we got that soup above
                    try:
                        page_soup = self.get_site(url, self._url_header)
                    except RequestsError as e:
                        continue
                    header_soup = page_soup.find("div", {"id": "content-header"})

                # Parse current page
                page_content = {}
                # If we are on the last page, parse a bit different
                if idx + 1 == len(links) and len(links) > 1:
                    # last page, skip it
                    continue
                # If we are not on the last page
                else:
                    try:
                        page_content['title'] = header_soup.find("h2").get_text().strip()
                    except AttributeError:
                        page_content['title'] = ''

                    media_soup = page_soup.find("div", {"class": "lead-image"})
                    if media_soup:  # if there is an image
                        page_content['image_orig'] = media_soup.find("img")['src']
                        try:
                            page_content['image_caption'] = media_soup.find("div", {"class": "media-body"}).get_text().strip()
                        except AttributeError:
                            page_content['image_caption'] = ''
                        try:
                            page_content['image_credit'] = media_soup.find("div", {"class": "media-sub"}).get_text().strip()
                        except AttributeError:
                            page_content['image_credit'] = ''

                    content = str(page_soup.find("div", {"class": "editorial-body"}))
                    # Remove the parent tag
                    content = content.replace('
', '')
                    # Remove the parents div (last div in the string)
                    content = self.rreplace(content, '
', '', 1)
                    page_content['page_content'] = content.strip()
                article['content'].insert(idx, page_content)
        except Exception as e:
            # Something went wrong scraping the page
            self.log("Exception [" + url + "]: " + str(e))
            return False

        return article

    def html_template(self, article):
        """
        """
        html = '<title>' + article['title'] + '</title>'
        html += '<link rel="stylesheet" type="text/css" href="/assets/style.css">'
        html += ''
        html += '
' + ' | '.join(article['bread_crumbs']) + '
'
        html += '<div id="title"' + article['title'] + '
'
        html += '<div id="author"' + article['author'] + '
'
        html += '
'
        for idx, page in enumerate(article['content']):
            html += '
'
            html += '
' + page['title'] + '
'
            if 'image_abs' in page:
                html += '
'
                html += ''
                html += '
' + page['image_caption'] + '
'
                html += '
' + page['image_credit'] + '
'
                html += '
'
            html += page['page_content']
            html += '
'

        html += '
'
        html += ''

        try:
            with open(article['save_path'] + "index.html", 'w') as f:
                f.write(html)
        except UnicodeEncodeError:
            self.log("UnicodeEncodeError: " + article['save_path'])
            with open(article['save_path'] + "index.html", 'w') as f:
                f.write("
Failed to create html file. ERROR: UnicodeEncodeError
")
<commit_msg>HSW is now threaded and creates pdf's of links<commit_after>from utils.exceptions import *
from utils.scraper import Scraper
from bs4 import BeautifulSoup
import urllib.request
import os
import re
import math
import pdfkit

class HowStuffWorks(Scraper):

    def __init__(self, base_dir, url_header, log_file):
        super().__init__(log_file)
        self._base_dir = base_dir
        self._completed_urls = []
        self._failed_urls = []
        self._completed_articles_csv = os.path.join(self._base_dir, "articles.csv")
        self._failed_articles_csv = os.path.join(self._base_dir, "articles_failed.csv")

        # Make sure file exists before we try and read form it
        open(self.create_dir(self._completed_articles_csv), 'a').close()
        # Fill self._completed_urls from articles.csv
        for line in open(self._completed_articles_csv, 'r'):
            self._completed_urls.append(line.split(',')[0])

        # Make sure file exists before we try and read form it
        open(self.create_dir(self._failed_articles_csv), 'a').close()
        # Fill self._failed_urls from articles.csv
        for line in open(self._failed_articles_csv, 'r'):
            self._failed_urls.append(line)

        # Use mobile header, the mobile site it much simpler to parse
        #   Reasons:
        #     - Both the Top10 and normal articles have the same html structure
        #     - The Top10 page on the desktop version has broken html and does not parse correctly
        self._url_header = {'User-Agent': 'Mozilla/5.0 (Linux; Android 4.4.4; en-us; Nexus 4 Build/JOP40D) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2307.2 Mobile Safari/537.36'}

    def get_latest(self):
        """
        If number of lines in self._completed_articles_csv is less then total articles - 500
            then run through all pages, other wise return 1
        :return: id of the newest item
        """
        self.cprint("##\tGetting newest upload id...\n", log=True)
        articles_per_page = 500

        # Get number of lines in articles.csv
        downloaded_articles = 0
        with open(self._completed_articles_csv) as f:
            for i, l in enumerate(f):
                downloaded_articles += 1

        url = "http://www.howstuffworks.com/big.htm?page=1"
        try:
            soup = self.get_site(url, self._url_header)
        except RequestsError as e:
            return 1
        num_articles = int(soup.find("div", {"class": "content"}).h3.get_text().split(' ')[-1].replace(',', ''))
        # Check if we need to scan more then the first page
        if abs(downloaded_articles - num_articles) <= 500:
            num_pages = 1
        else:
            num_pages = math.ceil(num_articles / articles_per_page)

        self.cprint("##\tGo to page: " + str(num_pages) + "\n", log=True)
        return num_pages

    def parse(self, id_):
        """
        Using BeautifulSoup, parse the page for links then will dig into found links
        :param id_: id of the page to parse
        :return:
        """
        # There is no 0 page
        if id_ == 0:
            return
        self.cprint("Processing links page: " + str(id_), log=True)
        url = "http://www.howstuffworks.com/big.htm?page=" + str(id_)
        # get the html from the url
        try:
            soup = self.get_site(url, self._url_header)
        except RequestsError as e:
            return

        links = soup.find("ol").find_all("li")
        # Loop through each link on the page
        for link in links:
            url = link.a["href"]

            # Check if url is a quiz (we do not want these, they are interactive flash objects)
            if str(url).endswith('quiz.htm'):
                continue

            # Check if we already have this article
            if str(url) in self._completed_urls or str(url) in self._failed_urls:
                continue

            self.cprint("Processing: " + url.split('/')[-1], log=True)

            whole_article = self.parse_article(url)
            # Something went wrong, so skip it
            if whole_article is False:
                self.log("Something Failed: " + url)
                self.add_failed(url)
                continue

            self.cprint("Saving assets: " + whole_article['id'], log=True)
            # Download/save images
            whole_article = self.save_article_images(whole_article)

            # Parse page content for links to make local
            #   - External links will be saved in pdf form
            #   - How Stuff Works links will be changed into a local path to the article
            whole_article = self.process_content_links(whole_article)

            # Save json data in case we need to rebuild the article
            self.save_props(whole_article)

            # Create html file from article
            self.html_template(whole_article)

            # Add article to completed list
            self.add_completed(url, whole_article['abs_path'], whole_article['title'])

        # Everything was successful
        return True

    def add_completed(self, url, article_path, title):
        self._completed_urls.append(url)
        with open(self._completed_articles_csv, 'a') as f:
            f.write(url + "," + article_path + "," + title + "\n")

    def add_failed(self, url):
        self._completed_urls.append(url)
        with open(self._failed_articles_csv, 'a') as f:
            f.write(url + "\n")

    def save_article_images(self, article):
        """
        Saves main image in header of page
        :param article: dict of all article data
        :return: update article dict
        """
        for idx, page in enumerate(article['content']):
            if 'image_orig' in page:
                article['content'][idx]['image_abs'] = self.download_image(article, page['image_orig'])

        return article

    def download_image(self, article, image_url):
        """
        :return: absolute path for image
        """
        abs_image_path = article['abs_path'] + "assets/" + image_url.split('/')[-1]
        self.download(image_url, self._base_dir + abs_image_path, self._url_header)
        return abs_image_path

    def process_content_links(self, article):
        """
        Parse page content for links to make local
          - External links will be saved in pdf form
          - How Stuff Works links will be changed into a local path to the article
        :param article: dict of all article data
        :return: update article dict
        """
        for idx_page, page in enumerate(article['content']):
            page_soup = BeautifulSoup(page['page_content'])
            # Download images and replace src to local file
            images = page_soup.find_all("img")
            if len(images) > 0:
                for image in images:
                    base_img_url = "http://s.hswstatic.com/"

                    img_dl_link = image['src']
                    if not image['src'].startswith(base_img_url):
                        img_dl_link = base_img_url + image['src']
                    self.cprint("Saving Image: " + image['src'], log=True)
                    abs_src = self.download_image(article, img_dl_link)
                    # Replace the src of downloaded image
                    article['content'][idx_page]['page_content'] = article['content'][idx_page]['page_content'].replace(image['src'], abs_src)

            # Replace HSW links with local path to file, create pdf of external sites and link locally
            # TODO: catch pdfkit errors/timeouts and try a few times before giving up
            pdfkit_options = {
                                'quiet': ''
                                }

            links = page_soup.find_all("a")
            if len(links) > 0:
                for link in links:
                    if re.match('.*howstuffworks\.com.*', link['href']):
                        abs_path, full_path = self.get_save_path(link['href'])
                        new_link = abs_path
                    else:
                        pdf_path = article['abs_path'] + "assets/" + link.get_text().replace(' ', '_') + ".pdf"
                        new_link = pdf_path

                        # Create pdf of external page
                        pdf_file = self._base_dir + pdf_path
                        if not os.path.isfile(pdf_file):
                            self.cprint("Creating pdf of: " + link['href'], log=True)
                            try:
                                pdfkit.from_url(link['href'], pdf_file, options=pdfkit_options)
                            except IOError as e:
                                self.log("pdfkit IOError: " + str(e), level='warning')
                            except Exception as e:
                                self.log("pdfkit Exception: " + str(e), level='warning')
                            else:
                                # If successful
                                # Convert to web safe path
                                abs_path = urllib.request.pathname2url(new_link)
                                # Replace link with link to article or a pdf
                                article['content'][idx_page]['page_content'] = article['content'][idx_page]['page_content'].replace(link['href'], new_link)

        return article

    def get_save_path(self, url, crumbs=None):
        """
        Pass in the url of the article and crumbs list if you have them
        :param url: url of the article
        :param curmbs: list of bread crumbs, if not passed, a request will be sent to parse them
        :return: relative path & save path as a string
        """
        if crumbs is None:
            crumbs = self.get_crumbs(url)

        abs_path = '/articles'
        # Add crumbs to path
        for crumb in crumbs:
            crumb = self.sanitize(crumb.replace(' ', '_'))
            abs_path = os.path.join(abs_path, crumb)

        # Add article title to path
        title = url.split('/')[-1].split('.')[0]
        abs_path = os.path.join(abs_path, title)
        abs_path += '/'
        abs_path = os.path.normcase(abs_path)
        # Create full path
        full_save_path = os.path.normpath(os.path.join(self._base_dir, '.' + abs_path)) + '/'
        full_save_path = os.path.normcase(full_save_path)
        return (abs_path, full_save_path)

    def get_crumbs(self, url=None, soup=None):
        """
        Use soup if you have it, that way les requests are made to the site.
        :param url: url of article
        :param soup: soup of the page
        :return: list of crumbs
        """
        # Only request site if needed
        if soup is not None:
            page_soup = soup
        elif url is not None:
            try:
                page_soup = self.get_site(url, self._url_header)
            except RequestsError as e:
                return
        else:
            return []

        crumbs = []
        for crumb in page_soup.find("div", {"class": "breadcrumb"}).find_all("a"):
            crumbs.append(crumb.get_text().strip())

        return crumbs

    def parse_article(self, url):
        """
        Using BeautifulSoup, parse the article for its data
        :param url: url of the article
        :return:
        """
        try:
            page_soup = self.get_site(url, self._url_header)
        except RequestsError as e:
            return

        # Get links for each page in the article
        links = []
        page_links = page_soup.find("select", {"id": "pageSelector"})

        # Check if article has multipal pages
        if page_links:
            page_links = page_links.find_all("option")
            for page_link in page_links:
                page_number = int(page_link.get_text().split(' ')[0])-1
                page_url = page_link['value']
                links.insert(page_number, page_url)
        else:
            # If article is a single page
            links.append(url)

        try:
            article = {}

            # Get bread crumbs
            article['bread_crumbs'] = self.get_crumbs(soup=page_soup)

            # Get save path
            article['abs_path'], article['save_path'] = self.get_save_path(url, article['bread_crumbs'])

            # Get title & author
            header_soup = page_soup.find("div", {"id": "content-header"})
            article['title'] = header_soup.find("h1").get_text().strip()

            # I need to have a ['id'], if there are non-ascii chars in title, just use the end of the url
            if len(article['title']) > 1 and type(article['title']) is not 'unicode':
                article['id'] = article['title']
            else:
                url.split('/')[-1]

            try:
                author = header_soup.find("span", {"class": "content-author"}).a.get_text()
                article['author'] = " ".join(author.split())
            except AttributeError:
                article['author'] = ''

            article['content'] = []  # List of content on each page
            # Parse each page in the article
            for idx, url in enumerate(links):
                self.cprint("Processing page: " + str(idx+1) + " of " + str(len(links)) + " - " + article['title'])
                if idx is not 0:  # We do not need the first page because we got that soup above
                    try:
                        page_soup = self.get_site(url, self._url_header)
                    except RequestsError as e:
                        continue
                    header_soup = page_soup.find("div", {"id": "content-header"})

                # Parse current page
                page_content = {}
                # If we are on the last page, parse a bit different
                if idx + 1 == len(links) and len(links) > 1:
                    # last page, skip it
                    continue
                # If we are not on the last page
                else:
                    try:
                        page_content['title'] = header_soup.find("h2").get_text().strip()
                    except AttributeError:
                        page_content['title'] = ''

                    media_soup = page_soup.find("div", {"class": "lead-image"})
                    if media_soup:  # if there is an image
                        page_content['image_orig'] = media_soup.find("img")['src']
                        try:
                            page_content['image_caption'] = media_soup.find("div", {"class": "media-body"}).get_text().strip()
                        except AttributeError:
                            page_content['image_caption'] = ''
                        try:
                            page_content['image_credit'] = media_soup.find("div", {"class": "media-sub"}).get_text().strip()
                        except AttributeError:
                            page_content['image_credit'] = ''

                    content = str(page_soup.find("div", {"class": "editorial-body"}))
                    # Remove the parent tag
                    content = content.replace('
', '')
                    # Remove the parents div (last div in the string)
                    content = self.rreplace(content, '
', '', 1)
                    page_content['page_content'] = content.strip()
                article['content'].insert(idx, page_content)
        except Exception as e:
            # Something went wrong scraping the page
            self.log("Exception [" + url + "]: " + str(e))
            return False

        return article

    def html_template(self, article):
        """
        """
        html = '<title>' + article['title'] + '</title>'
        html += '<link rel="stylesheet" type="text/css" href="/assets/style.css">'
        html += ''
        html += '
' + ' | '.join(article['bread_crumbs']) + '
'
        html += '<div id="title"' + article['title'] + '
'
        html += '<div id="author"' + article['author'] + '
'
        html += '
'
        for idx, page in enumerate(article['content']):
            html += '
'
            html += '
' + page['title'] + '
'
            if 'image_abs' in page:
                html += '
'
                html += ''
                html += '
' + page['image_caption'] + '
'
                html += '
' + page['image_credit'] + '
'
                html += '
'
            html += page['page_content']
            html += '
'

        html += '
'
        html += ''

        try:
            with open(article['save_path'] + "index.html", 'w') as f:
                f.write(html)
        except UnicodeEncodeError:
            self.log("UnicodeEncodeError: " + article['save_path'])
            with open(article['save_path'] + "index.html", 'w') as f:
                f.write("
Failed to create html file. ERROR: UnicodeEncodeError
")
<|endoftext|>


<commit_before>from random import shuffle

from insertion_sort import insertion_sort as in_sort

def test_insertion_sort():
    expected = range(20)
    unsorted = expected[:]
    shuffle(unsorted)
    in_sort(unsorted)
    actual = unsorted
    assert expected is actual

<commit_msg>Add another test<commit_after>from random import shuffle
import pytest

from insertion_sort import insertion_sort as in_sort

def test_insertion_sort():
    expected = range(20)
    unsorted = expected[:]
    shuffle(unsorted)
    in_sort(unsorted)
    actual = unsorted
    assert expected == actual

def test_insertion_sort_wrong_type():
    with pytest.raises(TypeError):
        in_sort('some string')

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Sat Mar 18 18:09:33 2017

@author: Aeolitus
"""
import Fertigkeiten
from Wolke import Wolke
import UI.CharakterFreieFert
from PyQt5 import QtWidgets, QtCore, QtGui
import logging
from Hilfsmethoden import Hilfsmethoden
import CharakterFreieFertigkeitenPickerWrapper

class CharakterFreieFertWrapper(QtCore.QObject):
    modified = QtCore.pyqtSignal()

    def __init__(self):
        super().__init__()
        logging.debug("Initializing FreieFertWrapper...")
        self.formFert = QtWidgets.QWidget()
        self.uiFert = UI.CharakterFreieFert.Ui_Form()
        self.uiFert.setupUi(self.formFert)

        kosten = [str(Wolke.DB.einstellungen["FreieFertigkeiten: Kosten Stufe1"].toInt()),
                  str(Wolke.DB.einstellungen["FreieFertigkeiten: Kosten Stufe2"].toInt()),
                  str(Wolke.DB.einstellungen["FreieFertigkeiten: Kosten Stufe3"].toInt())]
        self.uiFert.labelRegeln.setText(self.uiFert.labelRegeln.text() +
                                        " Sie kosten jeweils " + kosten[0] + "/" + kosten[1] + "/" + kosten[2] + " EP.")

        self.ffCount = 0

        self.editFF = []
        self.comboFF = []
        self.buttonFF = []
        font = QtGui.QFont("Times", QtWidgets.QApplication.instance().font().pointSize()) # serif font for the roman letters
        for row in range(0,7):
            for column in range(0,4):
                self.ffCount +=1
                ffLayout = QtWidgets.QHBoxLayout()
                ffEdit = QtWidgets.QLineEdit()
                ffEdit.editingFinished.connect(self.update)
                self.editFF.append(ffEdit)
                ffLayout.addWidget(ffEdit)
                ffCombo = QtWidgets.QComboBox()

                ffCombo.addItem("I")
                ffCombo.addItem("II")
                ffCombo.addItem("III")
                ffCombo.setFont(font)
                ffCombo.currentIndexChanged.connect(self.update)
                if self.ffCount <= Wolke.Char.freieFertigkeitenNumKostenlos:
                    ffCombo.setEnabled(False)
                self.comboFF.append(ffCombo)
                ffLayout.addWidget(ffCombo)

                ffButton = QtWidgets.QPushButton()
                self.buttonFF.append(ffButton)
                ffButton.setText("+")
                ffButton.setMaximumSize(QtCore.QSize(25, 20))
                ffButton.clicked.connect(lambda state, edit=ffEdit: self.ffButtonClicked(edit))
                ffLayout.addWidget(ffButton)

                self.uiFert.freieFertsGrid.addLayout(ffLayout, row, column)

        self.load()

    def ffButtonClicked(self, edit):
        picker = CharakterFreieFertigkeitenPickerWrapper.CharakterFreieFertigkeitenPickerWrapper(edit.text())
        if picker.fertigkeit != None:
            edit.setText(picker.fertigkeit)
            self.update()

    def load(self):
        count = 0
        for el in Wolke.Char.freieFertigkeiten:
            self.editFF[count].blockSignals(True)
            self.comboFF[count].blockSignals(True)
            self.editFF[count].setText(el.name)

            index = el.wert-1
            if count <= Wolke.Char.freieFertigkeitenNumKostenlos:
                index = 2
            self.comboFF[count].setCurrentIndex(index)
            self.editFF[count].blockSignals(False)
            self.comboFF[count].blockSignals(False)

            count += 1
            if count >= self.ffCount:
                break
        while count < self.ffCount:
            self.editFF[count].blockSignals(True)
            self.comboFF[count].blockSignals(True)
            self.editFF[count].setText("")
            index = 0
            if count < Wolke.Char.freieFertigkeitenNumKostenlos:
                index = 2
            self.comboFF[count].setCurrentIndex(index)
            self.editFF[count].blockSignals(False)
            self.comboFF[count].blockSignals(False)
            count += 1

    def update(self):
        freieNeu = []
        for count in range(0,self.ffCount):
            fert = Fertigkeiten.FreieFertigkeit()
            fert.name = self.editFF[count].text()
            fert.wert = self.comboFF[count].currentIndex()+1
            freieNeu.append(fert)

        #Preserve the position of actual elements but remove any trailing empty elements
        #This is needed for ArrayEqual later to work as intended
        for frei in reversed(freieNeu):
            if frei.name == "":
                freieNeu.pop()
            else:
                break

        if not Hilfsmethoden.ArrayEqual(freieNeu, Wolke.Char.freieFertigkeiten):
            Wolke.Char.freieFertigkeiten = freieNeu
            self.modified.emit()<commit_msg>Fix (again) freie fertigkeiten: forced value to III of one entry too many. Fix got lost during repo restructuring...<commit_after># -*- coding: utf-8 -*-
"""
Created on Sat Mar 18 18:09:33 2017

@author: Aeolitus
"""
import Fertigkeiten
from Wolke import Wolke
import UI.CharakterFreieFert
from PyQt5 import QtWidgets, QtCore, QtGui
import logging
from Hilfsmethoden import Hilfsmethoden
import CharakterFreieFertigkeitenPickerWrapper

class CharakterFreieFertWrapper(QtCore.QObject):
    modified = QtCore.pyqtSignal()

    def __init__(self):
        super().__init__()
        logging.debug("Initializing FreieFertWrapper...")
        self.formFert = QtWidgets.QWidget()
        self.uiFert = UI.CharakterFreieFert.Ui_Form()
        self.uiFert.setupUi(self.formFert)

        kosten = [str(Wolke.DB.einstellungen["FreieFertigkeiten: Kosten Stufe1"].toInt()),
                  str(Wolke.DB.einstellungen["FreieFertigkeiten: Kosten Stufe2"].toInt()),
                  str(Wolke.DB.einstellungen["FreieFertigkeiten: Kosten Stufe3"].toInt())]
        self.uiFert.labelRegeln.setText(self.uiFert.labelRegeln.text() +
                                        " Sie kosten jeweils " + kosten[0] + "/" + kosten[1] + "/" + kosten[2] + " EP.")

        self.ffCount = 0

        self.editFF = []
        self.comboFF = []
        self.buttonFF = []
        font = QtGui.QFont("Times", QtWidgets.QApplication.instance().font().pointSize()) # serif font for the roman letters
        for row in range(0,7):
            for column in range(0,4):
                self.ffCount +=1
                ffLayout = QtWidgets.QHBoxLayout()
                ffEdit = QtWidgets.QLineEdit()
                ffEdit.editingFinished.connect(self.update)
                self.editFF.append(ffEdit)
                ffLayout.addWidget(ffEdit)
                ffCombo = QtWidgets.QComboBox()

                ffCombo.addItem("I")
                ffCombo.addItem("II")
                ffCombo.addItem("III")
                ffCombo.setFont(font)
                ffCombo.currentIndexChanged.connect(self.update)
                if self.ffCount <= Wolke.Char.freieFertigkeitenNumKostenlos:
                    ffCombo.setEnabled(False)
                self.comboFF.append(ffCombo)
                ffLayout.addWidget(ffCombo)

                ffButton = QtWidgets.QPushButton()
                self.buttonFF.append(ffButton)
                ffButton.setText("+")
                ffButton.setMaximumSize(QtCore.QSize(25, 20))
                ffButton.clicked.connect(lambda state, edit=ffEdit: self.ffButtonClicked(edit))
                ffLayout.addWidget(ffButton)

                self.uiFert.freieFertsGrid.addLayout(ffLayout, row, column)

        self.load()

    def ffButtonClicked(self, edit):
        picker = CharakterFreieFertigkeitenPickerWrapper.CharakterFreieFertigkeitenPickerWrapper(edit.text())
        if picker.fertigkeit != None:
            edit.setText(picker.fertigkeit)
            self.update()

    def load(self):
        count = 0
        for el in Wolke.Char.freieFertigkeiten:
            self.editFF[count].blockSignals(True)
            self.comboFF[count].blockSignals(True)
            self.editFF[count].setText(el.name)

            index = el.wert-1
            if count < Wolke.Char.freieFertigkeitenNumKostenlos:
                index = 2
            self.comboFF[count].setCurrentIndex(index)
            self.editFF[count].blockSignals(False)
            self.comboFF[count].blockSignals(False)

            count += 1
            if count >= self.ffCount:
                break
        while count < self.ffCount:
            self.editFF[count].blockSignals(True)
            self.comboFF[count].blockSignals(True)
            self.editFF[count].setText("")
            index = 0
            if count < Wolke.Char.freieFertigkeitenNumKostenlos:
                index = 2
            self.comboFF[count].setCurrentIndex(index)
            self.editFF[count].blockSignals(False)
            self.comboFF[count].blockSignals(False)
            count += 1

    def update(self):
        freieNeu = []
        for count in range(0,self.ffCount):
            fert = Fertigkeiten.FreieFertigkeit()
            fert.name = self.editFF[count].text()
            fert.wert = self.comboFF[count].currentIndex()+1
            freieNeu.append(fert)

        #Preserve the position of actual elements but remove any trailing empty elements
        #This is needed for ArrayEqual later to work as intended
        for frei in reversed(freieNeu):
            if frei.name == "":
                freieNeu.pop()
            else:
                break

        if not Hilfsmethoden.ArrayEqual(freieNeu, Wolke.Char.freieFertigkeiten):
            Wolke.Char.freieFertigkeiten = freieNeu
            self.modified.emit()<|endoftext|>