<commit_before>#!/usr/bin/env python
"""Dump CPU affinity information as a tree."""

import sys
import collections
import re
import json

ORDER = {"physical id": 0, "core id": 1, "processor": 2}
PATTERN = "^(" + "|".join(ORDER.keys()) + ")"
FILTER_RE = re.compile(PATTERN)
ISLEAF = 0b001
ISFIRST = 0b010
ISLAST = 0b100

class DictTree(object):
    """Tree stucture implemented using dict. No need for a Node() class, but
    each key must be hashable, and sibling of the same level must be unique.
    The order of siblings is lost.
    """

    def __init__(self):
        """Create root node."""
        self._root = {}
        return None

    def add(self, inputlist):
        """Add the nodes specified in inputlist to the tree.The nth element of
        inputlist is added in the nth level.
        """
        _add(self._root, inputlist)
        return None

    def dumpjson(self):
        """Return a string dump in JSON format."""
        return json.dumps(self._root, sort_keys=True)

    def dumpascii(self, *args, **kwargs):
        """Return a string dump of an ascii drawing of the tree."""
        return _dumpascii(_tokenize(self._root, *args, **kwargs))

class CpuTree(DictTree):
    """Tree representing the CPUs found in /proc/cpuinfo."""

    def __init__(self):
        super(CpuTree, self).__init__()
        for cpu in _part3(_filter_cpuinfo()):
            self.add(cpu)
        self._sortkey = lambda x: int(x[0].rsplit(":", 1)[1].strip())
        return None

    def dumpascii(self):
        return super(CpuTree, self).dumpascii(self._sortkey)

def _add(node, inputlist):
    """Add elements in inputlist to the tree, rooted at given node.  Leaf nodes
    have an empty dictionary as child.  The nth element of inputlist is added
    to the nth level descending from node.
    """
    try:
        head = inputlist[0]
    except IndexError:
        return None
    tail = inputlist[1:]
    nextnode = node.setdefault(head, {})
    _add(nextnode, tail)

def _tokenize(node, sortkey=None):
    """Generator that traverses a tree in pre-order from node.
    Each yielded value is a tuple of (nodekey, depth, flags, symbol).
    flags is a bitwise-or'ed mask of following bits:
        0b001: node is a leaf
        0b010: node is first among siblings
        0b100: node is last among siblings (can be set with 0b010).
    """
    trackstack = []
    this = node
    depth = 0
    yield "*", 0, (ISFIRST | ISLAST)  # root
    while trackstack or this:
        children = this.items()
        if children:
            children.sort(key=sortkey, reverse=True)
            numchi = len(children)
            chiflags = [0] * numchi
            chiflags[0] |= ISLAST
            chiflags[-1] |= ISFIRST
            depth += 1
            tmp = children[-1]  # first child
            thisflag = chiflags[-1]
            tail = children[:-1]
            trackstack.extend(zip(tail,
                                  [depth] * (numchi - 1),
                                  chiflags[:-1]))
        else:
            tmp, depth, thisflag = trackstack.pop()  # Load depth and flags.
        yield tmp[0], depth, thisflag | (ISLEAF if not tmp[1] else 0)
        this = tmp[1]   # Advance to next.

def _symbol(label, flags, taillen=1):
    """Generate drawing symbol using a token's label and flags."""
    tail = "\n" if (flags & ISLEAF) else "-" * taillen
    isonlychild = (flags & ISFIRST) and (flags & ISLAST)
    if isonlychild:
        stem = "-"
    elif flags & ISFIRST:
        stem = "+"
    elif flags & ISLAST:
        stem = "`"
    else:
        stem = "|"
    return "%s-%s%s" % (stem, label, tail)

def _dumplines(tokenstream):
    """Draw using the input tokenstream iterator.  Return a list containing
    separate lines of the drawing.
    """
    text = []
    columnpos = collections.defaultdict(lambda: 0)
    for token in tokenstream:
        label, depth, flags = token
        symbol = _symbol(label, flags)
        if flags & ISFIRST:
            # need to set position for next column
            columnpos[depth + 1] = columnpos[depth] + len(symbol)
            indentstring = ""
        else:  # need indent
            indent = []
            for i in xrange(depth):
                indent.extend([" "] * (columnpos[i + 1] - columnpos[i]))
            indentstring = "".join(indent)
        text.append("%s%s" % (indentstring, symbol))
    return ("".join(text)).splitlines(True)

def _fudge_vertical_bars(lines):
    """Draw vertical bars to connect lines.  Return the "fudged" lines
    joined together as a string.
    """
    for lineno, line in enumerate(lines):
        vbidx = [i for (i, letter) in enumerate(line) if letter in ("|", "+")]
        nxlineno = lineno + 1
        try:    # try peeking next line
            nxline = lines[nxlineno]
        except IndexError:
            break
        if nxline[0] == " ":
            firstmarks = [x for x in [nxline.find(char) for char in ("`", "|")]
                          if x >= 0]
            if firstmarks:
                fmk = min(firstmarks)
            else:
                continue
            newnxline = [" "] * fmk
            for vbi in vbidx:
                if vbi < fmk:
                    newnxline[vbi] = "|"
            newnxline.append(nxline[fmk:])
            lines[nxlineno] = "".join(newnxline)
    return "".join(lines)

def _dumpascii(tokenstream):
    """Convert tokenstream to ascii drawing."""
    return _fudge_vertical_bars(_dumplines(tokenstream))

def _filter_cpuinfo():
    """Return a list of tuples, each of which looks like ("processor", 0), etc.
    """
    tmplist = []
    with open("/proc/cpuinfo", "r") as cpuinfo:
        for line in cpuinfo:
            if FILTER_RE.match(line):
                key, sep, val = (x.strip() for x in line.partition(":"))
                del sep
                tmplist.append((key, int(val)))
    return tmplist

def _part3(inputlist):
    """Return a list of id-lists.

    An id-list is a representation of a particular CPU's id.  The id-list is a
    list of ids in ascending order of each id's level. The returned list itself
    is not necessarily sorted in any particular order.
    """
    length = len(inputlist)
    tmplist = []
    for i in xrange(length / 3):
        tmptriplet = inputlist[i * 3:(i + 1) * 3]
        tmptriplet.sort(key=lambda t: ORDER[t[0]])
        tmplist.append(["%s: %s" % x for x in tmptriplet])
    return tmplist

def test():
    """Self-test"""
    cpus = CpuTree()
    sys.stdout.write(cpus.dumpascii())
    return None

if __name__ == "__main__":
    test()
<commit_msg>Simplify indent width reckoning<commit_after>#!/usr/bin/env python
"""Dump CPU affinity information as a tree."""

import sys
import collections
import re
import json

ORDER = {"physical id": 0, "core id": 1, "processor": 2}
PATTERN = "^(" + "|".join(ORDER.keys()) + ")"
FILTER_RE = re.compile(PATTERN)
ISLEAF = 0b001
ISFIRST = 0b010
ISLAST = 0b100

class DictTree(object):
    """Tree stucture implemented using dict. No need for a Node() class, but
    each key must be hashable, and sibling of the same level must be unique.
    The order of siblings is lost.
    """

    def __init__(self):
        """Create root node."""
        self._root = {}
        return None

    def add(self, inputlist):
        """Add the nodes specified in inputlist to the tree.The nth element of
        inputlist is added in the nth level.
        """
        _add(self._root, inputlist)
        return None

    def dumpjson(self):
        """Return a string dump in JSON format."""
        return json.dumps(self._root, sort_keys=True)

    def dumpascii(self, *args, **kwargs):
        """Return a string dump of an ascii drawing of the tree."""
        return _dumpascii(_tokenize(self._root, *args, **kwargs))

class CpuTree(DictTree):
    """Tree representing the CPUs found in /proc/cpuinfo."""

    def __init__(self):
        super(CpuTree, self).__init__()
        for cpu in _part3(_filter_cpuinfo()):
            self.add(cpu)
        self._sortkey = lambda x: int(x[0].rsplit(":", 1)[1].strip())
        return None

    def dumpascii(self):
        return super(CpuTree, self).dumpascii(self._sortkey)

def _add(node, inputlist):
    """Add elements in inputlist to the tree, rooted at given node.  Leaf nodes
    have an empty dictionary as child.  The nth element of inputlist is added
    to the nth level descending from node.
    """
    try:
        head = inputlist[0]
    except IndexError:
        return None
    tail = inputlist[1:]
    nextnode = node.setdefault(head, {})
    _add(nextnode, tail)

def _tokenize(node, sortkey=None):
    """Generator that traverses a tree in pre-order from node.
    Each yielded value is a tuple of (nodekey, depth, flags, symbol).
    flags is a bitwise-or'ed mask of following bits:
        0b001: node is a leaf
        0b010: node is first among siblings
        0b100: node is last among siblings (can be set with 0b010).
    """
    trackstack = []
    this = node
    depth = 0
    yield "*", 0, (ISFIRST | ISLAST)  # root
    while trackstack or this:
        children = this.items()
        if children:
            children.sort(key=sortkey, reverse=True)
            numchi = len(children)
            chiflags = [0] * numchi
            chiflags[0] |= ISLAST
            chiflags[-1] |= ISFIRST
            depth += 1
            tmp = children[-1]  # first child
            thisflag = chiflags[-1]
            tail = children[:-1]
            trackstack.extend(zip(tail,
                                  [depth] * (numchi - 1),
                                  chiflags[:-1]))
        else:
            tmp, depth, thisflag = trackstack.pop()  # Load depth and flags.
        yield tmp[0], depth, thisflag | (ISLEAF if not tmp[1] else 0)
        this = tmp[1]   # Advance to next.

def _symbol(label, flags, taillen=1):
    """Generate drawing symbol using a token's label and flags."""
    tail = "\n" if (flags & ISLEAF) else "-" * taillen
    isonlychild = (flags & ISFIRST) and (flags & ISLAST)
    if isonlychild:
        stem = "-"
    elif flags & ISFIRST:
        stem = "+"
    elif flags & ISLAST:
        stem = "`"
    else:
        stem = "|"
    return "%s-%s%s" % (stem, label, tail)

def _dumplines(tokenstream):
    """Draw using the input tokenstream iterator.  Return a list containing
    separate lines of the drawing.
    """
    text = []
    columnwidths = collections.defaultdict(lambda: 0)
    for token in tokenstream:
        label, depth, flags = token
        symbol = _symbol(label, flags)
        if flags & ISFIRST:
            # need to set position for next column
            columnwidths[depth] = len(symbol)
            indentstring = ""
        else:  # need indent
            indent = []
            for i in xrange(depth):
                indent.extend([" "] * columnwidths[i])
            indentstring = "".join(indent)
        text.append("%s%s" % (indentstring, symbol))
    return ("".join(text)).splitlines(True)

def _fudge_vertical_bars(lines):
    """Draw vertical bars to connect lines.  Return the "fudged" lines
    joined together as a string.
    """
    for lineno, line in enumerate(lines):
        vbidx = [i for (i, letter) in enumerate(line) if letter in ("|", "+")]
        nxlineno = lineno + 1
        try:    # try peeking next line
            nxline = lines[nxlineno]
        except IndexError:
            break
        if nxline[0] == " ":
            firstmarks = [x for x in [nxline.find(char) for char in ("`", "|")]
                          if x >= 0]
            if firstmarks:
                fmk = min(firstmarks)
            else:
                continue
            newnxline = [" "] * fmk
            for vbi in vbidx:
                if vbi < fmk:
                    newnxline[vbi] = "|"
            newnxline.append(nxline[fmk:])
            lines[nxlineno] = "".join(newnxline)
    return "".join(lines)

def _dumpascii(tokenstream):
    """Convert tokenstream to ascii drawing."""
    return _fudge_vertical_bars(_dumplines(tokenstream))

def _filter_cpuinfo():
    """Return a list of tuples, each of which looks like ("processor", 0), etc.
    """
    tmplist = []
    with open("/proc/cpuinfo", "r") as cpuinfo:
        for line in cpuinfo:
            if FILTER_RE.match(line):
                key, sep, val = (x.strip() for x in line.partition(":"))
                del sep
                tmplist.append((key, int(val)))
    return tmplist

def _part3(inputlist):
    """Return a list of id-lists.

    An id-list is a representation of a particular CPU's id.  The id-list is a
    list of ids in ascending order of each id's level. The returned list itself
    is not necessarily sorted in any particular order.
    """
    length = len(inputlist)
    tmplist = []
    for i in xrange(length / 3):
        tmptriplet = inputlist[i * 3:(i + 1) * 3]
        tmptriplet.sort(key=lambda t: ORDER[t[0]])
        tmplist.append(["%s: %s" % x for x in tmptriplet])
    return tmplist

def test():
    """Self-test"""
    cpus = CpuTree()
    sys.stdout.write(cpus.dumpascii())
    return None

if __name__ == "__main__":
    test()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# encoding:utf-8

import sys
reload(sys)
sys.setdefaultencoding("utf-8")

import gevent.monkey
gevent.monkey.patch_all()

import requests
from lxml import html
from db import Zhihu_User_Profile
from red_filter import check_url, re_crawl_url
from requests.packages.urllib3.exceptions import InsecureRequestWarning

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

class Zhihu_Crawler():

    '''
    basic crawler

    '''

    def __init__(self, url, option="print_data_out"):
        '''
        initialize the crawler

        '''

        self.option = option
        self.url = url
        self.header = {}
        self.header[
            "User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:43.0) Gecko/20100101 Firefox/43.0"
#        self.header["Host"]="www.zhihu.com"
        self.header["Referer"] = "www.zhihu.com"

        # cookie
        self.cookies = {"z_c0": '"QUZDQUp3czV3QWtYQUFBQVlRSlZUZkx3TVZmaFBMWlp2em04Ym1PN01BMldtRktscHRMOVVBPT0=|1460298735|34e9183179a0555057f1cfcc2c8f63660a2f4fc5"',
                        "unlock_ticket": 'QUZDQUp3czV3QWtYQUFBQVlRSlZUZnBxQ2xmSWNXX3NuVXo3SVJleUM5Uy1BLUpEdXJEcEpBPT0',
                        "login": '"ZjliNTRhNzViMDQ2NDMzY2FmZTczNjNjZDA4N2U0NGU=|1460298735|b1048ba322e44c391aa15306198503eab8b28f26"',
                        "n_c": "1",
                        "q_c1": "a15d5ad71c734d5b9ab4b1eddacea368|1460298703000|1460298703000",
                        "l_cap_id": '"YjMzMGNjMTUxMWIzNGZiMWI2OWI2ZGI1ZDM5NTAzZTQ=|1460298703|dd2d5dec11620d64a65ea057bd852f10124a283f"',
                        "d_c0": '"AJAAgETzqAmPTgxl_8gbpkFvESCkSwIZMoU=|1458736882"',
                        "cap_id": '"MmI2OTJiNWVkZGFmNGNmZDk0NDY2YTBlODI1ZjgyMWQ=|1460298703|b185f46c6887417393049379e47d961708cfdac7"'}

    def send_request(self):
        '''
        send a request to get HTML source

        '''
        added_followee_url = self.url + "/followees"
        try:
            r = requests.get(added_followee_url, cookies=self.cookies,
                             headers=self.header, verify=False)
        except:
            re_crawl_url(self.url)
            return

        content = r.text

        if r.status_code == 200:
            self.parse_user_profile(content)

    def process_xpath_source(self, source):
        if source:
            return source[0]
        else:
            return ''

    def parse_user_profile(self, html_source):
        '''
        parse the user's profile to mongo
        '''

        # initialize variances

        self.user_name = ''
        self.fuser_gender = ''
        self.user_location = ''
        self.user_followees = ''
        self.user_followers = ''
        self.user_be_agreed = ''
        self.user_be_thanked = ''
        self.user_education_school = ''
        self.user_education_subject = ''
        self.user_employment = ''
        self.user_employment_extra = ''
        self.user_info = ''
        self.user_intro = ''

        tree = html.fromstring(html_source)

        # parse the html via lxml
        self.user_name = self.process_xpath_source(
            tree.xpath("//a[@class='name']/text()"))
        self.user_location = self.process_xpath_source(
            tree.xpath("//span[@class='location item']/@title"))
        self.user_gender = self.process_xpath_source(
            tree.xpath("//span[@class='item gender']/i/@class"))
        if "female" in self.user_gender and self.user_gender:
            self.user_gender = "female"
        else:
            self.user_gender = "male"
        self.user_employment = self.process_xpath_source(
            tree.xpath("//span[@class='employment item']/@title"))
        self.user_employment_extra = self.process_xpath_source(
            tree.xpath("//span[@class='position item']/@title"))
        self.user_education_school = self.process_xpath_source(
            tree.xpath("//span[@class='education item']/@title"))
        self.user_education_subject = self.process_xpath_source(
            tree.xpath("//span[@class='education-extra item']/@title"))
        try:
            self.user_followees = tree.xpath(
                "//div[@class='zu-main-sidebar']//strong")[0].text
            self.user_followers = tree.xpath(
                "//div[@class='zu-main-sidebar']//strong")[1].text
        except:
            return

        self.user_be_agreed = self.process_xpath_source(tree.xpath(
            "//span[@class='zm-profile-header-user-agree']/strong/text()"))
        self.user_be_thanked = self.process_xpath_source(tree.xpath(
            "//span[@class='zm-profile-header-user-thanks']/strong/text()"))
        self.user_info = self.process_xpath_source(
            tree.xpath("//span[@class='bio']/@title"))
        self.user_intro = self.process_xpath_source(
            tree.xpath("//span[@class='content']/text()"))

        if self.option == "print_data_out":
            self.print_data_out()
        else:
            self.store_data_to_mongo()

        # find the follower's url
        url_list = tree.xpath("//h2[@class='zm-list-content-title']/a/@href")
        for target_url in url_list:
            target_url = target_url.replace("https", "http")
            check_url(target_url)

    def print_data_out(self):
        '''
        print out the user data
        '''

        print "*" * 60
        print '用户名:%s\n' % self.user_name
        print "用户性别:%s\n" % self.user_gender
        print '用户地址:%s\n' % self.user_location
        print "被同意:%s\n" % self.user_be_agreed
        print "被感谢:%s\n" % self.user_be_thanked
        print "被关注:%s\n" % self.user_followers
        print "关注了:%s\n" % self.user_followees
        print "工作:%s/%s" % (self.user_employment, self.user_employment_extra)
        print "教育:%s/%s" % (self.user_education_school, self.user_education_subject)
        print "用户信息:%s" % self.user_info
        print "*" * 60

    def store_data_to_mongo(self):
        '''
        store the data in mongo
        '''
        new_profile = Zhihu_User_Profile(
            user_name=self.user_name,
            user_be_agreed=self.user_be_agreed,
            user_be_thanked=self.user_be_thanked,
            user_followees=self.user_followees,
            user_followers=self.user_followers,
            user_education_school=self.user_education_school,
            user_education_subject=self.user_education_subject,
            user_employment=self.user_employment,
            user_employment_extra=self.user_employment_extra,
            user_location=self.user_location,
            user_gender=self.user_gender,
            user_info=self.user_info,
            user_intro=self.user_intro,
            user_url=self.url
        )
        new_profile.save()
        print "saved:%s \n" % self.user_name
<commit_msg>~~~~'<commit_after># -*- coding: utf-8 -*-
# encoding:utf-8

import sys
reload(sys)
sys.setdefaultencoding("utf-8")

import gevent.monkey
gevent.monkey.patch_socket()
gevent.monkey.patch_ssl()

import requests
from lxml import html
from db import Zhihu_User_Profile
from red_filter import check_url, re_crawl_url
from requests.packages.urllib3.exceptions import InsecureRequestWarning

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

class Zhihu_Crawler():

    '''
    basic crawler

    '''

    def __init__(self, url, option="print_data_out"):
        '''
        initialize the crawler

        '''

        self.option = option
        self.url = url
        self.header = {}
        self.header[
            "User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:43.0) Gecko/20100101 Firefox/43.0"
#        self.header["Host"]="www.zhihu.com"
        self.header["Referer"] = "www.zhihu.com"

        # cookie
        self.cookies = {"z_c0": '"QUZDQUp3czV3QWtYQUFBQVlRSlZUZkx3TVZmaFBMWlp2em04Ym1PN01BMldtRktscHRMOVVBPT0=|1460298735|34e9183179a0555057f1cfcc2c8f63660a2f4fc5"',
                        "unlock_ticket": 'QUZDQUp3czV3QWtYQUFBQVlRSlZUZnBxQ2xmSWNXX3NuVXo3SVJleUM5Uy1BLUpEdXJEcEpBPT0',
                        "login": '"ZjliNTRhNzViMDQ2NDMzY2FmZTczNjNjZDA4N2U0NGU=|1460298735|b1048ba322e44c391aa15306198503eab8b28f26"',
                        "n_c": "1",
                        "q_c1": "a15d5ad71c734d5b9ab4b1eddacea368|1460298703000|1460298703000",
                        "l_cap_id": '"YjMzMGNjMTUxMWIzNGZiMWI2OWI2ZGI1ZDM5NTAzZTQ=|1460298703|dd2d5dec11620d64a65ea057bd852f10124a283f"',
                        "d_c0": '"AJAAgETzqAmPTgxl_8gbpkFvESCkSwIZMoU=|1458736882"',
                        "cap_id": '"MmI2OTJiNWVkZGFmNGNmZDk0NDY2YTBlODI1ZjgyMWQ=|1460298703|b185f46c6887417393049379e47d961708cfdac7"'}

    def send_request(self):
        '''
        send a request to get HTML source

        '''
        added_followee_url = self.url + "/followees"
        try:
            r = requests.get(added_followee_url, cookies=self.cookies,
                             headers=self.header, verify=False)
        except:
            re_crawl_url(self.url)
            return

        content = r.text

        if r.status_code == 200:
            self.parse_user_profile(content)

    def process_xpath_source(self, source):
        if source:
            return source[0]
        else:
            return ''

    def parse_user_profile(self, html_source):
        '''
        parse the user's profile to mongo
        '''

        # initialize variances

        self.user_name = ''
        self.fuser_gender = ''
        self.user_location = ''
        self.user_followees = ''
        self.user_followers = ''
        self.user_be_agreed = ''
        self.user_be_thanked = ''
        self.user_education_school = ''
        self.user_education_subject = ''
        self.user_employment = ''
        self.user_employment_extra = ''
        self.user_info = ''
        self.user_intro = ''

        tree = html.fromstring(html_source)

        # parse the html via lxml
        self.user_name = self.process_xpath_source(
            tree.xpath("//a[@class='name']/text()"))
        self.user_location = self.process_xpath_source(
            tree.xpath("//span[@class='location item']/@title"))
        self.user_gender = self.process_xpath_source(
            tree.xpath("//span[@class='item gender']/i/@class"))
        if "female" in self.user_gender and self.user_gender:
            self.user_gender = "female"
        else:
            self.user_gender = "male"
        self.user_employment = self.process_xpath_source(
            tree.xpath("//span[@class='employment item']/@title"))
        self.user_employment_extra = self.process_xpath_source(
            tree.xpath("//span[@class='position item']/@title"))
        self.user_education_school = self.process_xpath_source(
            tree.xpath("//span[@class='education item']/@title"))
        self.user_education_subject = self.process_xpath_source(
            tree.xpath("//span[@class='education-extra item']/@title"))
        try:
            self.user_followees = tree.xpath(
                "//div[@class='zu-main-sidebar']//strong")[0].text
            self.user_followers = tree.xpath(
                "//div[@class='zu-main-sidebar']//strong")[1].text
        except:
            return

        self.user_be_agreed = self.process_xpath_source(tree.xpath(
            "//span[@class='zm-profile-header-user-agree']/strong/text()"))
        self.user_be_thanked = self.process_xpath_source(tree.xpath(
            "//span[@class='zm-profile-header-user-thanks']/strong/text()"))
        self.user_info = self.process_xpath_source(
            tree.xpath("//span[@class='bio']/@title"))
        self.user_intro = self.process_xpath_source(
            tree.xpath("//span[@class='content']/text()"))

        if self.option == "print_data_out":
            self.print_data_out()
        else:
            self.store_data_to_mongo()

        # find the follower's url
        url_list = tree.xpath("//h2[@class='zm-list-content-title']/a/@href")
        for target_url in url_list:
            target_url = target_url.replace("https", "http")
            check_url(target_url)

    def print_data_out(self):
        '''
        print out the user data
        '''

        print "*" * 60
        print '用户名:%s\n' % self.user_name
        print "用户性别:%s\n" % self.user_gender
        print '用户地址:%s\n' % self.user_location
        print "被同意:%s\n" % self.user_be_agreed
        print "被感谢:%s\n" % self.user_be_thanked
        print "被关注:%s\n" % self.user_followers
        print "关注了:%s\n" % self.user_followees
        print "工作:%s/%s" % (self.user_employment, self.user_employment_extra)
        print "教育:%s/%s" % (self.user_education_school, self.user_education_subject)
        print "用户信息:%s" % self.user_info
        print "*" * 60

    def store_data_to_mongo(self):
        '''
        store the data in mongo
        '''
        new_profile = Zhihu_User_Profile(
            user_name=self.user_name,
            user_be_agreed=self.user_be_agreed,
            user_be_thanked=self.user_be_thanked,
            user_followees=self.user_followees,
            user_followers=self.user_followers,
            user_education_school=self.user_education_school,
            user_education_subject=self.user_education_subject,
            user_employment=self.user_employment,
            user_employment_extra=self.user_employment_extra,
            user_location=self.user_location,
            user_gender=self.user_gender,
            user_info=self.user_info,
            user_intro=self.user_intro,
            user_url=self.url
        )
        new_profile.save()
        print "saved:%s \n" % self.user_name
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Main function
"""
import os
import re
import random
import sys

import aiohttp
import asyncio
import requests

from local_settings import talk_id

class Crawler(object):
    """
    crawler for download 755 photos
    """
    url = 'http://7gogo.jp/api/talk/post/list'
    img_path = 'images'

    def __init__(self):
        self.session = requests.session()
        pass

    def _download(self, url, filename):
        """
        Download photo

        Args:
            (String) url
            (String) file name (output)
        """
        # urllib.urlretrieve(url, filename, self._report_hook)

        req = self.session.get(url, stream=True)
        if req.status_code == 200:
            total_length = req.headers.get('content-length')
            dl_progress = 0

            output_path = "{}{}{}".format(self.img_path, os.sep, filename)
            if not os.path.exists(output_path):
                with open(output_path, 'wb') as o_file:
                    for chunk in req.iter_content(1024):
                        dl_progress += len(chunk)
                        o_file.write(chunk)

                        # Download progress report
                        percent = 100.0 * dl_progress / int(total_length)
                        sys.stdout.write("\r%2d%%" % percent)
                        sys.stdout.flush()

                print('')
            else:
                print('File exist')
        else:
            print('Visit website fail')

    async def run(self, client, talk_id):
        page_limit = 100

        payload = {
            'direction': 'PREV',
            'limit': page_limit,
            'postId': 6000,  # test 6000
            'talkId': talk_id,
        }

        r = self.session.get(self.url, params=payload)
        if r.status_code != 200:
            # handle connection fail
            sys.exit()
        else:
            raw_data = r.json()

            # handle no post
            if not raw_data['posts']:
                sys.exit()

            if not os.path.isdir(self.img_path):
                os.makedirs(self.img_path)

            count = 0
            for i in range(100):
                url = raw_data['posts'][i]['body'][0].get('image', '')
                if url:
                    count += 1
                    file_date = url.split('/')[4]
                    self._download(url, "{}_{}.jpg".format(file_date, count))

if __name__ == '__main__':
    my_cwawler = Crawler()

    loop = asyncio.get_event_loop()
    client = aiohttp.ClientSession(loop=loop)
    loop.run_until_complete(my_cwawler.run(client, talk_id))
    client.close()
<commit_msg>Add: stop time for download option<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Main function
"""
import os
import re
import random
import sys

import aiohttp
import asyncio
import requests

from local_settings import talk_id

class Crawler(object):
    """
    crawler for download 755 photos
    """
    url = 'http://7gogo.jp/api/talk/post/list'
    img_path = 'images'

    def __init__(self):
        self.session = requests.session()
        pass

    def _download(self, url, filename):
        """
        Download photo

        Args:
            (String) url
            (String) file name (output)
        """
        # urllib.urlretrieve(url, filename, self._report_hook)

        req = self.session.get(url, stream=True)
        if req.status_code == 200:
            total_length = req.headers.get('content-length')
            dl_progress = 0

            output_path = "{}{}{}".format(self.img_path, os.sep, filename)
            if not os.path.exists(output_path):
                with open(output_path, 'wb') as o_file:
                    for chunk in req.iter_content(1024):
                        dl_progress += len(chunk)
                        o_file.write(chunk)

                        # Download progress report
                        percent = 100.0 * dl_progress / int(total_length)
                        sys.stdout.write("\r%2d%%" % percent)
                        sys.stdout.flush()

                print('')
            else:
                print('File exist')
        else:
            print('Visit website fail')

    async def run(self, client, talk_id, stop_time=0):
        page_limit = 100

        payload = {
            'direction': 'PREV',
            'limit': page_limit,
            'postId': 6000,  # test 6000
            'talkId': talk_id,
        }

        r = self.session.get(self.url, params=payload)
        if r.status_code != 200:
            # handle connection fail
            sys.exit()
        else:
            raw_data = r.json()

            # handle no post
            if not raw_data['posts']:
                sys.exit()

            if not os.path.isdir(self.img_path):
                os.makedirs(self.img_path)

            count = 0
            for i in range(100):
                # if msg time too old, stop download
                if int(raw_data['posts'][i]['time']) < stop_time:
                    break

                url = raw_data['posts'][i]['body'][0].get('image', '')
                if url:
                    count += 1
                    file_date = url.split('/')[4]
                    self._download(url, "{}_{}.jpg".format(file_date, count))

if __name__ == '__main__':
    my_cwawler = Crawler()
    stop_time = 1445687490

    loop = asyncio.get_event_loop()
    client = aiohttp.ClientSession(loop=loop)
    loop.run_until_complete(my_cwawler.run(client, talk_id, stop_time))
    client.close()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from lxml import html
import requests, json, re, itertools, sys

naverComicUrl = 'http://comic.naver.com'
naverComicListUrl = naverComicUrl + '/webtoon/creation.nhn?view=image'

def init_thumbnails():
	response = requests.get(naverComicListUrl)
	parsedBody = html.fromstring(response.text)
	thumbnailDivisions = parsedBody.xpath("//div[@class='thumb']")
	thumbnails = [{'title': td.xpath('./a/@title')[0], 'href': td.xpath('./a/@href')[0], 'imgsrc': td.xpath('./a/img/@src')[0], 'id': re.search('[0-9]+', td.xpath('./a/@href')[0]).group(0)} for td in thumbnailDivisions]

	for thumbnail in thumbnails:
		print 'Getting Stars from ' + thumbnail['title'].encode('utf-8')
		thumbnail['stars'] = makeStarList(thumbnail['href'])
		thumbnail['author'] = getAuthor(thumbnail['href'])

	with open('static/thumbnails.json', 'w') as fp:
		json.dump(thumbnails, fp)

def endPageNum(href):
	return int(html.fromstring(requests.get(naverComicUrl + href + '&page=1000').text).xpath("//span[@class='current']")[0].text)

def makeStarList(href):
	pages = [naverComicUrl + href + '&page=' + str(num) for num in xrange(1, endPageNum(href) + 1)]
	stars = list(itertools.chain(*[[float(star.text)
				for star
				in [x.xpath('./strong')[0]
					for x
					in html.fromstring(requests.get(page).text).xpath("//div[@class='rating_type']")]]
			for page
			in pages]))
	return stars[::-1]

def getAuthor(href):
	author = html.fromstring(requests.get(naverComicUrl + href).text).xpath("//span[@class='wrt_nm']")[0].text
	return author

def missing_pages(tn):
	pages = [naverComicUrl + '/webtoon/detail.nhn?titleId=' + tn['id'] + '&no=' + str(num) for num in xrange(1, len(tn['stars']) + 1)]
	main_title = html.fromstring(requests.get('http://comic.naver.com/main.nhn').text).xpath('//head/title')[0].text
	mpgs = [i+1 for i, page in enumerate(pages) if html.fromstring(requests.get(page).text).xpath('//head/title')[0].text == main_title]
	print tn['title'].encode('utf-8'), mpgs
	return mpgs

def validation(start):
	with open('static/thumbnails.json', 'r') as fp:
		thumbnails = json.load(fp)
	for i in xrange(int(start), len(thumbnails)):
		t = thumbnails[i]
		thumbnail = {
			'author' : t['author'],
			'title' : t['title'],
			'href' : t['href'],
			'stars' : t['stars'],
			'imgsrc' : t['imgsrc'],
			'id' : t['id'],
			'missing' : [str(pn) for pn in missing_pages(t)]
		}
		thumbnails[i] = thumbnail
		with open('static/thumbnails.json', 'w') as fp:
			fp.write(thumbnails)
		print i, 'complete!'

if __name__ == '__main__':
	#init_thumbnails()
	validation(sys.argv[1])<commit_msg>fix validation<commit_after># -*- coding: utf-8 -*-

from lxml import html
import requests, json, re, itertools, sys

naverComicUrl = 'http://comic.naver.com'
naverComicListUrl = naverComicUrl + '/webtoon/creation.nhn?view=image'

def init_thumbnails():
	response = requests.get(naverComicListUrl)
	parsedBody = html.fromstring(response.text)
	thumbnailDivisions = parsedBody.xpath("//div[@class='thumb']")
	thumbnails = [{'title': td.xpath('./a/@title')[0], 'href': td.xpath('./a/@href')[0], 'imgsrc': td.xpath('./a/img/@src')[0], 'id': re.search('[0-9]+', td.xpath('./a/@href')[0]).group(0)} for td in thumbnailDivisions]

	for thumbnail in thumbnails:
		print 'Getting Stars from ' + thumbnail['title'].encode('utf-8')
		thumbnail['stars'] = makeStarList(thumbnail['href'])
		thumbnail['author'] = getAuthor(thumbnail['href'])

	with open('static/thumbnails.json', 'w') as fp:
		json.dump(thumbnails, fp)

def endPageNum(href):
	return int(html.fromstring(requests.get(naverComicUrl + href + '&page=1000').text).xpath("//span[@class='current']")[0].text)

def makeStarList(href):
	pages = [naverComicUrl + href + '&page=' + str(num) for num in xrange(1, endPageNum(href) + 1)]
	stars = list(itertools.chain(*[[float(star.text)
				for star
				in [x.xpath('./strong')[0]
					for x
					in html.fromstring(requests.get(page).text).xpath("//div[@class='rating_type']")]]
			for page
			in pages]))
	return stars[::-1]

def getAuthor(href):
	author = html.fromstring(requests.get(naverComicUrl + href).text).xpath("//span[@class='wrt_nm']")[0].text
	return author

def missing_pages(tn):
	pages = [naverComicUrl + '/webtoon/detail.nhn?titleId=' + tn['id'] + '&no=' + str(num) for num in xrange(1, len(tn['stars']) + 1)]
	main_title = html.fromstring(requests.get('http://comic.naver.com/main.nhn').text).xpath('//head/title')[0].text
	mpgs = [i+1 for i, page in enumerate(pages) if html.fromstring(requests.get(page).text).xpath('//head/title')[0].text == main_title]
	print tn['title'].encode('utf-8'), mpgs
	return mpgs

def validation(start):
	with open('static/thumbnails.json', 'r') as fp:
		thumbnails = json.load(fp)
	for i in xrange(int(start), len(thumbnails)):
		t = thumbnails[i]
		thumbnail = {
			'author' : t['author'],
			'title' : t['title'],
			'href' : t['href'],
			'stars' : t['stars'],
			'imgsrc' : t['imgsrc'],
			'id' : t['id'],
			'missing' : [str(pn) for pn in missing_pages(t)]
		}
		thumbnails[i] = thumbnail
		with open('static/thumbnails.json', 'w') as fp:
			json.dump(thumbnails, fp)
		print i, 'complete!'

if __name__ == '__main__':
	#init_thumbnails()
	validation(sys.argv[1])<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from proj_root import *
import os
import sys
sys.path.append(PROJ_ROOT)
os.environ['DJANGO_SETTINGS_MODULE'] = 'cronos.settings'
from BeautifulSoup import BeautifulSoup
from cronos.announcements.models import *
from django.conf import settings
import MySQLdb
import pycurl
import re
import StringIO
import urllib
import urlparse

conn = pycurl.Curl()
p = re.compile(r'<[^<]*?/?>')

### www.teilar.gr ###

for cid in xrange(30):
	b = StringIO.StringIO()
	conn.setopt(pycurl.URL, 'http://www.teilar.gr/tmimatanews.php?cid=' + str(cid))
	conn.setopt(pycurl.WRITEFUNCTION, b.write)
	conn.perform()
	output = unicode(b.getvalue(), 'utf-8', 'ignore')
	soup = BeautifulSoup(output)
	try:
		school = str(soup.findAll('td', 'BlueTextBold')[0])
		school = p.sub(' ', school)
	except:
		pass
	if (cid == 0):
		school = 'Κεντρική Σελίδα ΤΕΙ Λάρισας'
	if (len(school) > 5):
		depart = Id(
			urlid = 'cid' + str(cid),
			name = school.strip(),
			department = '',
			email = '',
		)
	try:
		depart.save()
	except:
		pass

# extra sites #
dest = [
	['noc', 'Κέντρο Διαχείρισης Δικτύου'],
	['career', 'Γραφείο Διασύνδεσης'],
	['linuxteam', 'Linux Team'],
	['dionysos', 'Πρόγραμμα Γραμματείας'],
	['library', 'Κεντρική Βιβλιοθήκη'],
	['pr', 'Γραφείο Δημοσίων και Διεθνών Σχέσεων'],
	]

for i in xrange(len(dest[:][:])):
	dest[i][0] = Id(
		urlid = 'cid' + str(50+i),
		name = str(dest[i][1]) +  ' ΤΕΙ Λάρισας',
		department = '',
		email = '',
	)

	try:
		dest[i][0].save()
	except:
		pass

### www.teilar.gr/profannews.php ###

for pid in xrange(400):
	b = StringIO.StringIO()
	conn.setopt(pycurl.URL, 'http://www.teilar.gr/person.php?pid=' + str(pid))
	conn.setopt(pycurl.WRITEFUNCTION, b.write)
	conn.perform()
	output = unicode(b.getvalue(), 'utf-8', 'ignore')
	soup = BeautifulSoup(output)
	teacher = ''
	email = ''
	depart = ''
	try:
		teacher = str(BeautifulSoup(str(soup.findAll('td', 'BlackText11Bold')[1].contents[0])))
		email = str(BeautifulSoup(str(soup.findAll('td', 'BlackText11')[5])).a.contents[0])
		depart = str(BeautifulSoup(str(soup.findAll('td', 'BlackText11')[2])))
		depart = p.sub('',depart)
	except:
		pass
	if (len(teacher) > 1):
		teachers = Id(
			urlid = 'pid' + str(pid),
			name = teacher,
			department = depart,
			email = email,
		)
		try:
			teachers.save()
		except:
			pass

### e-class.teilar.gr ###

b = StringIO.StringIO()
cookie_file_name = os.tempnam('/tmp', 'eclass')
login_form_seq = [
	('uname', settings.ECLASS_USER),
	('pass', settings.ECLASS_PASSWORD),
	('submit', 'E%95%CE%AF%CF%83%CE%BF%CE%B4%CE%BF%CF%82')
]
login_form_data = urllib.urlencode(login_form_seq)
conn.setopt(pycurl.FOLLOWLOCATION, 1)
conn.setopt(pycurl.COOKIEFILE, cookie_file_name)
conn.setopt(pycurl.COOKIEJAR, cookie_file_name)
conn.setopt(pycurl.URL, 'http://openclass.teilar.gr/index.php')
conn.setopt(pycurl.POST, 1)
conn.setopt(pycurl.POSTFIELDS, login_form_data)
conn.setopt(pycurl.WRITEFUNCTION, b.write)
conn.perform()
output = unicode(b.getvalue(), 'utf-8', 'ignore')
soup = BeautifulSoup(output).find('table', 'FormData')

i = 0

for item in soup.findAll('a'):
	if (i%2 == 0):
		cid = str(soup.findAll('a')[i].contents[0]).split('-')[0]

		lesson = ''
		for j in xrange(len((soup.findAll('a')[i].contents[0]).split('-')) - 1):
			lesson += str(soup.findAll('a')[i].contents[0]).split('-')[j+1].strip() + ' '

		eclass = Id(
			urlid = str(cid),
			name = lesson.strip(),
			department = '',
			email = '',
		)

		try:
			eclass.save()
		except:
			pass

	i += 1
<commit_msg>Improve log system (catch only specific exceptions, and define alternatives where possible), add a message when something is added in the DB, use mkstemp instead of tempnam<commit_after># -*- coding: utf-8 -*-

from proj_root import *
import os
import sys
sys.path.append(PROJECT_ROOT)
os.environ['DJANGO_SETTINGS_MODULE'] = 'cronos.settings'
from BeautifulSoup import BeautifulSoup
from cronos.announcements.models import *
from django.conf import settings
from django.db.utils import IntegrityError
import MySQLdb
import pycurl
import re
import StringIO
import tempfile
import urllib
import urlparse

conn = pycurl.Curl()
p = re.compile(r'<[^<]*?/?>')

### www.teilar.gr ###

for cid in xrange(30):
	b = StringIO.StringIO()
	conn.setopt(pycurl.URL, 'http://www.teilar.gr/tmimatanews.php?cid=' + str(cid))
	conn.setopt(pycurl.WRITEFUNCTION, b.write)
	conn.perform()
	output = unicode(b.getvalue(), 'utf-8', 'ignore')
	soup = BeautifulSoup(output)

	school = str(soup.findAll('td', 'BlueTextBold')[0])
	school = p.sub(' ', school)

	if (cid == 0):
		school = 'Κεντρική Σελίδα ΤΕΙ Λάρισας'
	if (cid == 2):
		school = 'Τμήμα Τεχνολογίας Πληροφορικής και Τηλεπικοινωνιών'
	if (len(school) > 5):
		depart = Id(
			urlid = 'cid' + str(cid),
			name = school.strip(),
			department = '',
			email = '',
		)
		try:
			depart.save()
			print 'ADDED ' + school
		except IntegrityError:
			pass

# extra sites #

dest = [
	['noc', 'Κέντρο Διαχείρισης Δικτύου'],
	['career', 'Γραφείο Διασύνδεσης'],
	['linuxteam', 'Linux Team'],
	['dionysos', 'Πρόγραμμα Γραμματείας'],
	['library', 'Κεντρική Βιβλιοθήκη'],
	['pr', 'Γραφείο Δημοσίων και Διεθνών Σχέσεων'],
]

for i in xrange(len(dest[:][:])):
	dest[i][0] = Id(
		urlid = 'cid' + str(50+i),
		name = str(dest[i][1]) +  ' ΤΕΙ Λάρισας',
		department = '',
		email = '',
	)
	try:
		dest[i][0].save()
		print 'ADDED ' + str(dest[i][1])
	except IntegrityError:
		pass

### www.teilar.gr/profannews.php ###

for pid in xrange(400):
	b = StringIO.StringIO()
	conn.setopt(pycurl.URL, 'http://www.teilar.gr/person.php?pid=' + str(pid))
	conn.setopt(pycurl.WRITEFUNCTION, b.write)
	conn.perform()
	output = unicode(b.getvalue(), 'utf-8', 'ignore')
	soup = BeautifulSoup(output)
	teacher = ''
	email = ''
	depart = ''

	try:
		teacher = str(BeautifulSoup(str(soup.findAll('td', 'BlackText11Bold')[1].contents[0]))).strip()
	except IndexError:
		pass
	try:
		email = str(BeautifulSoup(str(soup.findAll('td', 'BlackText11')[5])).a.contents[0])
	except AttributeError:
		try:
			email = str(BeautifulSoup(str(soup.findAll('td', 'BlackText11')[5].contents[0])))
		except IndexError:
			pass
	except IndexError:
		pass
	try:
		depart = str(BeautifulSoup(str(soup.findAll('td', 'BlackText11')[2]))).strip()
		depart = p.sub('',depart)
		if depart.split(' ')[1] == 'Τεχν.':
			depart = 'Τμήμα Τεχνολογίας ' + ' '.join(depart.split(' ')[2:])
	except IndexError:
		pass

	if (len(teacher) > 1):
		teachers = Id(
			urlid = 'pid' + str(pid),
			name = teacher,
			department = depart,
			email = email,
		)
		try:
			teachers.save()
			print 'ADDED ' + teacher
		except IntegrityError:
			pass

### e-class.teilar.gr ###

b = StringIO.StringIO()
fd, cookie_path = tempfile.mkstemp(prefix='eclass_', dir='/tmp')
print fd
print cookie_path
login_form_seq = [
	('uname', settings.ECLASS_USER),
	('pass', settings.ECLASS_PASSWORD),
	('submit', 'E%95%CE%AF%CF%83%CE%BF%CE%B4%CE%BF%CF%82')
]
login_form_data = urllib.urlencode(login_form_seq)
conn.setopt(pycurl.FOLLOWLOCATION, 1)
conn.setopt(pycurl.COOKIEFILE, cookie_path)
conn.setopt(pycurl.COOKIEJAR, cookie_path)
conn.setopt(pycurl.URL, 'http://openclass.teilar.gr/index.php')
conn.setopt(pycurl.POST, 1)
conn.setopt(pycurl.POSTFIELDS, login_form_data)
conn.setopt(pycurl.WRITEFUNCTION, b.write)
conn.perform()
output = unicode(b.getvalue(), 'utf-8', 'ignore')
soup = BeautifulSoup(output).find('table', 'FormData')

i = 0

lessonslist = soup.findAll('a')
for item in lessonslist:
	if (i%2 == 0):
		cid = str(soup.findAll('a')[i].contents[0]).split('-')[0]
		lesson = ''
		for j in xrange(len((soup.findAll('a')[i].contents[0]).split('-')) - 1):
			lesson += str(soup.findAll('a')[i].contents[0]).split('-')[j+1].strip() + ' '

		eclass = Id(
			urlid = str(cid),
			name = lesson.strip(),
			department = '',
			email = '',
		)
		try:
			eclass.save()
			print 'ADDED ' + lesson.strip()
		except IntegrityError:
			pass
		except MySQLdb.Warning, e:
			print 'ADDED ' + lesson.strip()
			print e
			pass

	i += 1
os.close(fd)
os.remove(cookie_path)
print "Probably finished successfully"
<|endoftext|>


<commit_before>import datetime

from math import ceil
from enum import Enum
from collections import namedtuple

class Bounds(namedtuple('Bound', ("min", "max")), Enum):
    minute = (0, 59)
    hour = (0, 23)
    dom = (1, 31)
    month = (1, 12)
    dow = (1, 7)

    def range(self, step=1):
        return Range(self.min, self.max, step)

    def range_set(self, step=1):
        return Set((self.range(),))

class Job:

    """
    Represents a cron job. This handles parsing of the job to get timing
    information, as well as determining when the job will be run next.
    >>> import parser
    >>> job = parser.parse_job("* * * * * true")
    >>> (job.mins, job.hours, job.doms, job.months, job.dows)
    (0-59/1, 0-23/1, 1-31/1, 1-12/1, 1-7/1)
    >>> parser.parse_job("* * * * * true").job
    'true'
    >>> parser.parse_job("* * * * * echo test").job
    'echo test'
    """

    def __init__(self, times, job):
        (self.mins, self.hours, self.doms, self.months, self.dows) = times
        self.job = job

    def next_value(self, dt):
        """
        Returns the next time this cron job will run.
        >>> import parser
        >>> parser.parse_job("* * * * * true").next_value(
        ...     datetime.datetime(2014, 11, 15, 17, 4, 49))
        datetime.datetime(2014, 11, 15, 17, 5)
        >>> parser.parse_job("* * 1 * * true").next_value(
        ...     datetime.datetime(2014, 11, 15, 17, 4, 49))
        datetime.datetime(2014, 12, 1, 0, 0)
        >>> parser.parse_job("* * * * 3 true").next_value(
        ...     datetime.datetime(2014, 11, 15, 17, 4, 49))
        datetime.datetime(2014, 11, 19, 0, 0)
        >>> parser.parse_job("* * * * 3 true").next_value(
        ...     datetime.datetime(2014, 11, 29, 17, 4, 49))
        datetime.datetime(2014, 12, 3, 0, 0)
        >>> parser.parse_job("* * * 11 3 true").next_value(
        ...     datetime.datetime(2014, 11, 29, 17, 4, 49))
        datetime.datetime(2015, 11, 4, 0, 0)
        >>> end_of_year = datetime.datetime(2014, 12, 31, 23, 59)
        >>> parser.parse_job("* * * * * true").next_value(end_of_year)
        datetime.datetime(2015, 1, 1, 0, 0)
        >>> parser.parse_job("* * 29 2 * true").next_value(end_of_year)
        datetime.datetime(2016, 2, 29, 0, 0)
        >>> parser.parse_job("* * 29 2 * true").next_value(
        ...     datetime.datetime(2196, 2, 29, 23, 59))
        datetime.datetime(2204, 2, 29, 0, 0)
        """
        #dt.minute, dt.hour, dt.day, dt.month, dt.isoweekday(), dt.year)
        dt = dt.replace(second=0)

        def find_minute_hour(dt, minute_carry=True):
            minute_carry, next_minute = self.mins.next_value(
                dt.minute, minute_carry)
            hour_carry, next_hour = self.hours.next_value(
                dt.hour, minute_carry)
            if hour_carry:
                _, next_minute = self.mins.next_value(0, carry=False)
            try:
                return (hour_carry,
                        dt.replace(hour=next_hour, minute=next_minute))
            except:
                raise ValueError(
                    "Couldn't find a valid time for the cron job") from None

        def find_dom_month_year(dt, hour_carry):
            dom = dt.day
            month = dt.month
            year = dt.year
            for i in range(100):
                dom_carry, dom = self.doms.next_value(dom, hour_carry)
                month_carry, month = self.months.next_value(month, dom_carry)
                year += month_carry
                if month_carry:
                    dom_carry, dom = self.doms.next_value(0, carry=False)
                try:
                    return (dom_carry or month_carry,
                            dt.replace(year=year, month=month, day=dom))
                except ValueError:
                    continue  # There should be something here
            raise ValueError("Couldn't find a valid time for the cron job")

        def find_dow_month_year(dt, hour_carry):
            old_dow = dt.isoweekday() % 7
            month = dt.month
            dow_carry, dow = self.dows.next_value(
                old_dow, carry=hour_carry)
            delta = _dows_to_timedelta(old_dow, dow)
            month_carry, next_month = self.months.next_value(
                month, carry=False)
            if (dt + delta).month == month and next_month == month:
                return dow_carry, dt + delta

            month_carry, next_month = self.months.next_value(month, carry=True)
            dt = dt.replace(year=dt.year+month_carry,
                            month=next_month, day=1)

            tmp_dow = dt.isoweekday() % 7
            _, dow = self.dows.next_value(tmp_dow, carry=False)
            return True, dt + _dows_to_timedelta(tmp_dow, dow)

        hour_carry, time = find_minute_hour(dt)
        dom_dt, dow_dt = (find_minute_hour(
            dt.replace(hour=0, minute=0), minute_carry=False)[1]
            if high_carry else dt
            for high_carry, dt in (find_dom_month_year(time, hour_carry),
                                   find_dow_month_year(time, hour_carry)))

        if self.dow_specified and self.dom_specified:
            return min(dom_dt, dow_dt)
        elif self.dow_specified and not self.dom_specified:
            return dow_dt
        else:
            return dom_dt

    def __eq__(self, other):
        return (self.mins == other.mins and
                self.hours == other.hours and
                self.doms == other.doms and
                self.months == other.months and
                self.dows == other.dows and
                self.job == other.job)

def _dows_to_timedelta(current, next):
    return datetime.timedelta(
        days=next-current, weeks=1 if next < current else 0)

class Set:

    """
    Represents a set of ranges in one particular field of one particular
    job.
    >>> Set((Range(1, 5, 4), Range(34, 57, 59), Range(59,59),
    ...      Range(0, 59, 30)))
    1-5/4,34-57/59,59-59/1,0-59/30
    """

    def __init__(self, ranges):
        self.ranges = ranges

    def __repr__(self):
        return ','.join(str(r) for r in self.ranges)

    def next_value(self, current, carry=True):
        """
        Finds the next occurring time for a set of ranges. This should
        be the first of the next_values of all of the ranges in the set.
        >>> Set((Range(1, 5, 4), Range(34, 57, 59), Range(59,59),
        ...      Range(0, 59, 30))).next_value(23)
        (False, 30)
        >>> Set((Range(1, 4), Range(3, 7))).next_value(8)
        (True, 1)
        """
        return min(r.next_value(current, carry) for r in self.ranges)

    def __eq__(self, other):
        return self.ranges == other.ranges

class Range:

    """
    Represents a range of times, plus an optional step value.
    """

    def __init__(self, min, max, step=1):
        self.min = min
        self.max = max
        self.step = step

    def __repr__(self):
        return '{}-{}/{}'.format(self.min, self.max, self.step)

    def next_value(self, current, carry=True):
        """
        Returns the following value that this particular range will be
        triggered on, as well as a bool indicating if the range had to
        wrap around to reach this value.
        >>> Bounds.minute.range().next_value(10)
        (False, 11)
        >>> Range(5, 20, 11).next_value(17)
        (True, 5)
        >>> Range(5, 20, 11).next_value(11)
        (False, 16)
        >>> Range(59, 60, 29).next_value(3)
        (False, 59)
        >>> Range(29, 60, 29).next_value(3)
        (False, 29)
        """
        distance = ceil((current + carry - self.min)/self.step)
        next = self.min + self.step*distance
        if current+carry <= self.min:
            return (False, self.min)
        elif next > self.max:
            return (True, self.min)
        else:
            return (False, next)

    def __eq__(self, other):
        if type(other) == tuple:
            print(other)
            raise ValueError
        return (type(self) == type(other) and
                self.min == other.min and
                self.max == other.max and
                self.step == other.step)

if __name__ == "__main__":
    import doctest
    doctest.testmod(optionflags=doctest.ELLIPSIS)
<commit_msg>Put space in doctest for PEP 8 compliance<commit_after>import datetime

from math import ceil
from enum import Enum
from collections import namedtuple

class Bounds(namedtuple('Bound', ("min", "max")), Enum):
    minute = (0, 59)
    hour = (0, 23)
    dom = (1, 31)
    month = (1, 12)
    dow = (1, 7)

    def range(self, step=1):
        return Range(self.min, self.max, step)

    def range_set(self, step=1):
        return Set((self.range(),))

class Job:

    """
    Represents a cron job. This handles parsing of the job to get timing
    information, as well as determining when the job will be run next.
    >>> import parser
    >>> job = parser.parse_job("* * * * * true")
    >>> (job.mins, job.hours, job.doms, job.months, job.dows)
    (0-59/1, 0-23/1, 1-31/1, 1-12/1, 1-7/1)
    >>> parser.parse_job("* * * * * true").job
    'true'
    >>> parser.parse_job("* * * * * echo test").job
    'echo test'
    """

    def __init__(self, times, job):
        (self.mins, self.hours, self.doms, self.months, self.dows) = times
        self.job = job

    def next_value(self, dt):
        """
        Returns the next time this cron job will run.
        >>> import parser
        >>> parser.parse_job("* * * * * true").next_value(
        ...     datetime.datetime(2014, 11, 15, 17, 4, 49))
        datetime.datetime(2014, 11, 15, 17, 5)
        >>> parser.parse_job("* * 1 * * true").next_value(
        ...     datetime.datetime(2014, 11, 15, 17, 4, 49))
        datetime.datetime(2014, 12, 1, 0, 0)
        >>> parser.parse_job("* * * * 3 true").next_value(
        ...     datetime.datetime(2014, 11, 15, 17, 4, 49))
        datetime.datetime(2014, 11, 19, 0, 0)
        >>> parser.parse_job("* * * * 3 true").next_value(
        ...     datetime.datetime(2014, 11, 29, 17, 4, 49))
        datetime.datetime(2014, 12, 3, 0, 0)
        >>> parser.parse_job("* * * 11 3 true").next_value(
        ...     datetime.datetime(2014, 11, 29, 17, 4, 49))
        datetime.datetime(2015, 11, 4, 0, 0)
        >>> end_of_year = datetime.datetime(2014, 12, 31, 23, 59)
        >>> parser.parse_job("* * * * * true").next_value(end_of_year)
        datetime.datetime(2015, 1, 1, 0, 0)
        >>> parser.parse_job("* * 29 2 * true").next_value(end_of_year)
        datetime.datetime(2016, 2, 29, 0, 0)
        >>> parser.parse_job("* * 29 2 * true").next_value(
        ...     datetime.datetime(2196, 2, 29, 23, 59))
        datetime.datetime(2204, 2, 29, 0, 0)
        """
        #dt.minute, dt.hour, dt.day, dt.month, dt.isoweekday(), dt.year)
        dt = dt.replace(second=0)

        def find_minute_hour(dt, minute_carry=True):
            minute_carry, next_minute = self.mins.next_value(
                dt.minute, minute_carry)
            hour_carry, next_hour = self.hours.next_value(
                dt.hour, minute_carry)
            if hour_carry:
                _, next_minute = self.mins.next_value(0, carry=False)
            try:
                return (hour_carry,
                        dt.replace(hour=next_hour, minute=next_minute))
            except:
                raise ValueError(
                    "Couldn't find a valid time for the cron job") from None

        def find_dom_month_year(dt, hour_carry):
            dom = dt.day
            month = dt.month
            year = dt.year
            for i in range(100):
                dom_carry, dom = self.doms.next_value(dom, hour_carry)
                month_carry, month = self.months.next_value(month, dom_carry)
                year += month_carry
                if month_carry:
                    dom_carry, dom = self.doms.next_value(0, carry=False)
                try:
                    return (dom_carry or month_carry,
                            dt.replace(year=year, month=month, day=dom))
                except ValueError:
                    continue  # There should be something here
            raise ValueError("Couldn't find a valid time for the cron job")

        def find_dow_month_year(dt, hour_carry):
            old_dow = dt.isoweekday() % 7
            month = dt.month
            dow_carry, dow = self.dows.next_value(
                old_dow, carry=hour_carry)
            delta = _dows_to_timedelta(old_dow, dow)
            month_carry, next_month = self.months.next_value(
                month, carry=False)
            if (dt + delta).month == month and next_month == month:
                return dow_carry, dt + delta

            month_carry, next_month = self.months.next_value(month, carry=True)
            dt = dt.replace(year=dt.year+month_carry,
                            month=next_month, day=1)

            tmp_dow = dt.isoweekday() % 7
            _, dow = self.dows.next_value(tmp_dow, carry=False)
            return True, dt + _dows_to_timedelta(tmp_dow, dow)

        hour_carry, time = find_minute_hour(dt)
        dom_dt, dow_dt = (find_minute_hour(
            dt.replace(hour=0, minute=0), minute_carry=False)[1]
            if high_carry else dt
            for high_carry, dt in (find_dom_month_year(time, hour_carry),
                                   find_dow_month_year(time, hour_carry)))

        if self.dow_specified and self.dom_specified:
            return min(dom_dt, dow_dt)
        elif self.dow_specified and not self.dom_specified:
            return dow_dt
        else:
            return dom_dt

    def __eq__(self, other):
        return (self.mins == other.mins and
                self.hours == other.hours and
                self.doms == other.doms and
                self.months == other.months and
                self.dows == other.dows and
                self.job == other.job)

def _dows_to_timedelta(current, next):
    return datetime.timedelta(
        days=next-current, weeks=1 if next < current else 0)

class Set:

    """
    Represents a set of ranges in one particular field of one particular
    job.
    >>> Set((Range(1, 5, 4), Range(34, 57, 59), Range(59, 59),
    ...      Range(0, 59, 30)))
    1-5/4,34-57/59,59-59/1,0-59/30
    """

    def __init__(self, ranges):
        self.ranges = ranges

    def __repr__(self):
        return ','.join(str(r) for r in self.ranges)

    def next_value(self, current, carry=True):
        """
        Finds the next occurring time for a set of ranges. This should
        be the first of the next_values of all of the ranges in the set.
        >>> Set((Range(1, 5, 4), Range(34, 57, 59), Range(59,59),
        ...      Range(0, 59, 30))).next_value(23)
        (False, 30)
        >>> Set((Range(1, 4), Range(3, 7))).next_value(8)
        (True, 1)
        """
        return min(r.next_value(current, carry) for r in self.ranges)

    def __eq__(self, other):
        return self.ranges == other.ranges

class Range:

    """
    Represents a range of times, plus an optional step value.
    """

    def __init__(self, min, max, step=1):
        self.min = min
        self.max = max
        self.step = step

    def __repr__(self):
        return '{}-{}/{}'.format(self.min, self.max, self.step)

    def next_value(self, current, carry=True):
        """
        Returns the following value that this particular range will be
        triggered on, as well as a bool indicating if the range had to
        wrap around to reach this value.
        >>> Bounds.minute.range().next_value(10)
        (False, 11)
        >>> Range(5, 20, 11).next_value(17)
        (True, 5)
        >>> Range(5, 20, 11).next_value(11)
        (False, 16)
        >>> Range(59, 60, 29).next_value(3)
        (False, 59)
        >>> Range(29, 60, 29).next_value(3)
        (False, 29)
        """
        distance = ceil((current + carry - self.min)/self.step)
        next = self.min + self.step*distance
        if current+carry <= self.min:
            return (False, self.min)
        elif next > self.max:
            return (True, self.min)
        else:
            return (False, next)

    def __eq__(self, other):
        if type(other) == tuple:
            print(other)
            raise ValueError
        return (type(self) == type(other) and
                self.min == other.min and
                self.max == other.max and
                self.step == other.step)

if __name__ == "__main__":
    import doctest
    doctest.testmod(optionflags=doctest.ELLIPSIS)
<|endoftext|>


<commit_before>import requests, json, config, calendar, sys
from datetime import datetime

# config
startChannelId = "UC5xDht2blPNWdVtl9PkDmgA" # SailLife
maxLevels = 3
sailingTerms = ["sail", "skipper", "circumnavigate", "yacht"]

if len(sys.argv) == 2:
	dataFile = sys.argv[1]
else:
	dataFile = "data.json"

# members
channels = {}

# READ VIDEOS PAGE
def readVideosPage(channelId, pageToken = None):

	url = "https://www.googleapis.com/youtube/v3/search?part=snippet&channelId=" + channelId + "&maxResults=50&regionCode=us&key=" + config.apiKey()

	if pageToken != None:
		url += "&pageToken=" + pageToken

	# fetch videos of channel
	r = requests.get(url)
	vids = r.json()

	videos = []
	for v in vids["items"]:

		# ignore playlists
		if v["id"]["kind"] != "youtube#video":
			continue

		d = datetime.strptime(v["snippet"]["publishedAt"], "%Y-%m-%dT%H:%M:%S.000Z")

		videos.append({
			"id": v["id"]["videoId"],
			"title": v["snippet"]["title"],
			"description": v["snippet"]["description"],
			"publishedAt": calendar.timegm(d.utctimetuple())
		})

	# is there a next page?
	if vids.has_key("nextPageToken"):
		return vids["nextPageToken"], videos
	else:
		return None, videos

# READ VIDEOS
def readVideos(channelId):

	nextPage = None
	nextPageNew = None
	videos = []

	while True:
		nextPageNew, vids = readVideosPage(channelId, nextPage)

		# extend video list with new page ones
		videos.extend(vids);

		if nextPageNew == None:
			break

		nextPage = nextPageNew

	return videos

# READ STATISTICS
def readStatistics(channelId):

	r = requests.get("https://www.googleapis.com/youtube/v3/channels?part=statistics&id=" + channelId + "&key=" + config.apiKey())
	stats = r.json()
	return stats["items"][0]["statistics"]

# READ SUBSCRIPTIONS PAGE
def readSubscriptionsPage(channelId, pageToken = None, level = 1):

	url = "https://www.googleapis.com/youtube/v3/subscriptions?part=snippet&maxResults=50&channelId=" + channelId + "&key=" + config.apiKey()

	if pageToken != None:
		url += "&pageToken=" + pageToken

	# fetch subscriptions of channel
	r = requests.get(url)
	subs = r.json()

	# error? ignore!
	if r.status_code != 200:
		print "error", r.status_code, channelId, level
		return None

	print len(subs["items"]), "items"

	# loop channel items in result set
	for i in subs["items"]:

		if i["snippet"]["resourceId"]["kind"] != "youtube#channel":
			print i["snippet"]["resourceId"]["kind"]
			continue

		subChannelId = i["snippet"]["resourceId"]["channelId"]

		# store this channel
		if not channels.has_key(subChannelId):

			stats = readStatistics(subChannelId)
			hasSailingTerm = False

			# check if one of the sailing terms is available
			for term in sailingTerms:
				if (term in i["snippet"]["title"].lower() or term in i["snippet"]["description"].lower()):
					hasSailingTerm = True
					break

			if int(stats["videoCount"]) > 0 and hasSailingTerm:

				channels[subChannelId] = {
					"id": subChannelId,
					"title": i["snippet"]["title"],
					"description": i["snippet"]["description"],
					"thumbnail": i["snippet"]["thumbnails"]["default"]["url"],
					"subscribers": int(stats["subscriberCount"]),
					"views": int(stats["viewCount"]),
					"subscribersHidden": bool(stats["hiddenSubscriberCount"]),
					"videos": readVideos(subChannelId)
				}

				# read sub level subscriptions
				subLevel = level + 1
				readSubscriptions(subChannelId, subLevel)

	# is there a next page?
	if subs.has_key("nextPageToken"):
		return subs["nextPageToken"]
	else:
		return None

# READ SUBSCRIPTIONS
def readSubscriptions(channelId, level = 1):

	# we reached the maximum level of recursion
	if level >= maxLevels:
		return None

	print channelId, level

	nextPage = None
	nextPageNew = None

	while True:
		nextPageNew = readSubscriptionsPage(channelId, nextPage, level)

		if nextPageNew == None:
			break

		nextPage = nextPageNew

# WRITE SUBSCRIPTIONS
def writeSubscriptions():

	print len(channels.keys()), " channels found"

	# order them by subscribers

	with open(dataFile, "w") as dataFile:
		dataFile.write(json.dumps(channels.values(), indent=4, sort_keys=True))

readSubscriptions(startChannelId, 1)
writeSubscriptions()
<commit_msg>dataFile<commit_after>import requests, json, config, calendar, sys
from datetime import datetime

# config
startChannelId = "UC5xDht2blPNWdVtl9PkDmgA" # SailLife
maxLevels = 3
sailingTerms = ["sail", "skipper", "circumnavigate", "yacht"]

dataFile = "data.json"
if len(sys.argv) == 2:
	dataFile = sys.argv[1]

print "data file path", dataFile

# members
channels = {}

# READ VIDEOS PAGE
def readVideosPage(channelId, pageToken = None):

	url = "https://www.googleapis.com/youtube/v3/search?part=snippet&channelId=" + channelId + "&maxResults=50&regionCode=us&key=" + config.apiKey()

	if pageToken != None:
		url += "&pageToken=" + pageToken

	# fetch videos of channel
	r = requests.get(url)
	vids = r.json()

	videos = []
	for v in vids["items"]:

		# ignore playlists
		if v["id"]["kind"] != "youtube#video":
			continue

		d = datetime.strptime(v["snippet"]["publishedAt"], "%Y-%m-%dT%H:%M:%S.000Z")

		videos.append({
			"id": v["id"]["videoId"],
			"title": v["snippet"]["title"],
			"description": v["snippet"]["description"],
			"publishedAt": calendar.timegm(d.utctimetuple())
		})

	# is there a next page?
	if vids.has_key("nextPageToken"):
		return vids["nextPageToken"], videos
	else:
		return None, videos

# READ VIDEOS
def readVideos(channelId):

	nextPage = None
	nextPageNew = None
	videos = []

	while True:
		nextPageNew, vids = readVideosPage(channelId, nextPage)

		# extend video list with new page ones
		videos.extend(vids);

		if nextPageNew == None:
			break

		nextPage = nextPageNew

	return videos

# READ STATISTICS
def readStatistics(channelId):

	r = requests.get("https://www.googleapis.com/youtube/v3/channels?part=statistics&id=" + channelId + "&key=" + config.apiKey())
	stats = r.json()
	return stats["items"][0]["statistics"]

# READ SUBSCRIPTIONS PAGE
def readSubscriptionsPage(channelId, pageToken = None, level = 1):

	url = "https://www.googleapis.com/youtube/v3/subscriptions?part=snippet&maxResults=50&channelId=" + channelId + "&key=" + config.apiKey()

	if pageToken != None:
		url += "&pageToken=" + pageToken

	# fetch subscriptions of channel
	r = requests.get(url)
	subs = r.json()

	# error? ignore!
	if r.status_code != 200:
		print "error", r.status_code, channelId, level
		return None

	print len(subs["items"]), "items"

	# loop channel items in result set
	for i in subs["items"]:

		if i["snippet"]["resourceId"]["kind"] != "youtube#channel":
			print i["snippet"]["resourceId"]["kind"]
			continue

		subChannelId = i["snippet"]["resourceId"]["channelId"]

		# store this channel
		if not channels.has_key(subChannelId):

			stats = readStatistics(subChannelId)
			hasSailingTerm = False

			# check if one of the sailing terms is available
			for term in sailingTerms:
				if (term in i["snippet"]["title"].lower() or term in i["snippet"]["description"].lower()):
					hasSailingTerm = True
					break

			if int(stats["videoCount"]) > 0 and hasSailingTerm:

				channels[subChannelId] = {
					"id": subChannelId,
					"title": i["snippet"]["title"],
					"description": i["snippet"]["description"],
					"thumbnail": i["snippet"]["thumbnails"]["default"]["url"],
					"subscribers": int(stats["subscriberCount"]),
					"views": int(stats["viewCount"]),
					"subscribersHidden": bool(stats["hiddenSubscriberCount"]),
					"videos": readVideos(subChannelId)
				}

				# read sub level subscriptions
				subLevel = level + 1
				readSubscriptions(subChannelId, subLevel)

	# is there a next page?
	if subs.has_key("nextPageToken"):
		return subs["nextPageToken"]
	else:
		return None

# READ SUBSCRIPTIONS
def readSubscriptions(channelId, level = 1):

	# we reached the maximum level of recursion
	if level >= maxLevels:
		return None

	print channelId, level

	nextPage = None
	nextPageNew = None

	while True:
		nextPageNew = readSubscriptionsPage(channelId, nextPage, level)

		if nextPageNew == None:
			break

		nextPage = nextPageNew

# WRITE SUBSCRIPTIONS
def writeSubscriptions():

	print len(channels.keys()), " channels found"

	# order them by subscribers
	with open(dataFile, "w") as dataFile:
		dataFile.write(json.dumps(channels.values(), indent=4, sort_keys=True))

readSubscriptions(startChannelId, 1)
writeSubscriptions()
<|endoftext|>


<commit_before># coding=utf-8
import logging
import json
import scrapy

from scrapy.crawler import CrawlerProcess
from scrapy.settings import Settings
from scrapy.utils.log import log_scrapy_info
from scrapy.utils.ossignal import install_shutdown_handlers
from twisted.internet import defer
from twisted.internet.error import ConnectionDone

from conf import AMQP_PARAM, EXCHANGE_PARAMS, TASK_Q_PARAMS, TASK_BIND_PARAMS
from utils.mypika import PooledConn

__author__ = 'zephor'

logger = logging.getLogger('crawler')
scrapy.optional_features.discard('boto')

class CrawlerDaemon(CrawlerProcess):

    def __init__(self):
        import crawl_conf as p_settings

        settings = Settings()
        settings.setmodule(p_settings)
        super(CrawlerProcess, self).__init__(settings)  # 跳过CrawlerProcess的初始日志配置，由init.py处理
        install_shutdown_handlers(self._signal_shutdown)
        log_scrapy_info(self.settings)

        self.__task_queue = None
        self._pconn = PooledConn(AMQP_PARAM)
        self._set_up()

    def _set_up(self, _=None):
        d = self._pconn.acquire()
        d.addCallbacks(self._on_conn, self._on_err_conn)
        d.addErrback(self._on_err)

    @defer.inlineCallbacks
    def _on_conn(self, conn):
        # in case the connection is lost; mostly closed by the mq server
        conn.ready.addCallback(self.__clear)
        conn.ready.addCallback(self._set_up)
        self._conn = conn
        channel = self._channel = yield conn.channel()
        # do some setup
        yield channel.exchange_declare(**EXCHANGE_PARAMS)
        yield channel.queue_declare(**TASK_Q_PARAMS)
        yield channel.queue_bind(**TASK_BIND_PARAMS)

        self.__task_queue, consumer_tag = yield channel.basic_consume(queue=TASK_Q_PARAMS['queue'], no_ack=False)
        yield self._on_get()

    @staticmethod
    def _on_err_conn(err):
        logger.fatal(err)

    @staticmethod
    def _on_err(err):
        if err.type is ConnectionDone:
            logger.info(err)
        else:
            logger.error(err)

    @defer.inlineCallbacks
    def _on_get(self):
        ch, method, properties, body = yield self.__task_queue.get()
        d = self._on_msg(body)
        yield ch.basic_ack(delivery_tag=method.delivery_tag)
        if isinstance(d, defer.Deferred):
            self._channel.close()
            self._pconn.release(self._conn)
            d.addCallback(self._set_up)
        else:
            d = self._on_get()
        yield d

    def _on_msg(self, body):
        logger.info('_on_msg %s' % body)
        try:
            msg = json.loads(body)
            self.settings.set('COOKIES_ENABLED', msg['is_login'], 'spider')
            d = self.crawl(msg['spider'], task_id=msg['id'], task_name=msg['name'], parser=msg['parser'])
            d.addErrback(lambda err: logger.error(err))
        except Exception as e:
            logger.error(repr(e))
        if len(self._active) > 1:
            return self.join()

    def __clear(self, _=None):
        if self.__task_queue is not None:
            self.__task_queue.close(ConnectionDone('done'))

    def crawl(self, spider_name, *args, **kwargs):
        crawler = self._create_crawler(spider_name)

        self.crawlers.add(crawler)
        d = crawler.crawl(*args, **kwargs)
        self._active.add(d)

        def _done(result):
            self.crawlers.discard(crawler)
            self._active.discard(d)
            # parser may hold large memory, release it manually
            try:
                del crawler.spider.parser
            except AttributeError:
                pass    # spider may be None in case Failure
            return result

        return d.addBoth(_done)

if __name__ == '__main__':
    import init
    init.init('crawler')
    if init.done:
        p = CrawlerDaemon()
        p.start(stop_after_crawl=False)
<commit_msg>change handle to connection lost<commit_after># coding=utf-8
import logging
import json
import scrapy

from scrapy.crawler import CrawlerProcess
from scrapy.settings import Settings
from scrapy.utils.log import log_scrapy_info
from scrapy.utils.ossignal import install_shutdown_handlers
from twisted.internet import defer
from twisted.internet.error import ConnectionDone

from conf import AMQP_PARAM, EXCHANGE_PARAMS, TASK_Q_PARAMS, TASK_BIND_PARAMS
from utils.mypika import PooledConn

__author__ = 'zephor'

logger = logging.getLogger('crawler')
scrapy.optional_features.discard('boto')

class CrawlerDaemon(CrawlerProcess):

    def __init__(self):
        import crawl_conf as p_settings

        settings = Settings()
        settings.setmodule(p_settings)
        super(CrawlerProcess, self).__init__(settings)  # 跳过CrawlerProcess的初始日志配置，由init.py处理
        install_shutdown_handlers(self._signal_shutdown)
        log_scrapy_info(self.settings)

        self.__task_queue = None
        self._pconn = PooledConn(AMQP_PARAM)
        self._set_up()

    def _set_up(self, _=None):
        d = self._pconn.acquire()
        d.addCallbacks(self._on_conn, self._on_err_conn)
        d.addErrback(self._on_err)

    @defer.inlineCallbacks
    def _on_conn(self, conn):
        # in case the connection is lost; mostly closed by the mq server
        conn.ready.addErrback(self.__clear)
        conn.ready.addCallback(self._set_up)
        self._conn = conn
        channel = self._channel = yield conn.channel()
        # do some setup
        yield channel.exchange_declare(**EXCHANGE_PARAMS)
        yield channel.queue_declare(**TASK_Q_PARAMS)
        yield channel.queue_bind(**TASK_BIND_PARAMS)

        self.__task_queue, consumer_tag = yield channel.basic_consume(queue=TASK_Q_PARAMS['queue'], no_ack=False)
        yield self._on_get()

    @staticmethod
    def _on_err_conn(err):
        logger.fatal(err)

    @staticmethod
    def _on_err(err):
        if err.type is ConnectionDone:
            logger.info(err)
        else:
            logger.error(err)

    @defer.inlineCallbacks
    def _on_get(self):
        ch, method, properties, body = yield self.__task_queue.get()
        d = self._on_msg(body)
        yield ch.basic_ack(delivery_tag=method.delivery_tag)
        if isinstance(d, defer.Deferred):
            self._channel.close()
            self._pconn.release(self._conn)
            d.addCallback(self._set_up)
        else:
            d = self._on_get()
        yield d

    def _on_msg(self, body):
        logger.info('_on_msg %s' % body)
        try:
            msg = json.loads(body)
            self.settings.set('COOKIES_ENABLED', msg['is_login'], 'spider')
            d = self.crawl(msg['spider'], task_id=msg['id'], task_name=msg['name'], parser=msg['parser'])
            d.addErrback(lambda err: logger.error(err))
        except Exception as e:
            logger.error(repr(e))
        if len(self._active) > 1:
            return self.join()

    def __clear(self, _=None):
        if self.__task_queue is not None:
            self.__task_queue.close(ConnectionDone('done'))

    def crawl(self, spider_name, *args, **kwargs):
        crawler = self._create_crawler(spider_name)

        self.crawlers.add(crawler)
        d = crawler.crawl(*args, **kwargs)
        self._active.add(d)

        def _done(result):
            self.crawlers.discard(crawler)
            self._active.discard(d)
            # parser may hold large memory, release it manually
            try:
                del crawler.spider.parser
            except AttributeError:
                pass    # spider may be None in case Failure
            return result

        return d.addBoth(_done)

if __name__ == '__main__':
    import init
    init.init('crawler')
    if init.done:
        p = CrawlerDaemon()
        p.start(stop_after_crawl=False)
<|endoftext|>


<commit_before>from BeautifulSoup import BeautifulSoup
import urllib2, urlparse
import re
import time
import httplib

# TODO: Do not apply wait time to external links

class Crawler(object):
	def __init__(self, init_url):
		# A list of to be crawled urls, where the second element
		# in the tuple is the URL via which we found this URL
		self.pages = [(init_url, None)]

		# Urls we have already visited
		self.found = set()

		# List of deadlinks for each URL we have,
		# i.e. url1: [deadlink1, deadlink2]
		self.deadlinks = {}

		# Regular expression for URLs we are interested in (our internal
		# URLs)
		self.url_match = None

		# Timeout in seconds to wait, so that we do not kill our server
		self.wait_time = 0

	def set_url_restrict(self, regexp):
		self.url_match = re.compile(regexp)

	def set_wait_time(self, seconds):
		self.wait_time = seconds

	def crawl(self):
		while len(self.pages) > 0:
			time.sleep(self.wait_time)

			next_url = self.pages.pop()

			try:
				self.visit_url(next_url)
			except urllib2.URLError:
				continue

		print("DEADLINKS")
		print(self.deadlinks)

	def visit_url(self, url_tuple):
		if self.url_match.search(url_tuple[0]):
			self.visit_url_internal(url_tuple[0])
		else:
			self.visit_url_external(url_tuple[0], url_tuple[1])

	def visit_url_internal(self, url):
		print("Crawling internal: %s" % url)

		html = urllib2.urlopen(url)

		try:
			for page in self.extract_urls(html):
				page = urlparse.urljoin(url, page)

				if not page in self.found:
					self.pages.append((page, url))
					self.found.add(page)
		except UnicodeEncodeError:
			pass

	def visit_url_external(self, url, found_via):
		print("Trying external: %s" % url)

		request = urllib2.Request(url)

		try:
			response = urllib2.urlopen(request, None, 10)
		except (urllib2.HTTPError, httplib.BadStatusLine):
			# We receive an exception in case of 404
			self.add_to_deadlinks(url, found_via)
			return

		status = response.getcode()
		if status != None and status >= 400:
			self.add_to_deadlinks(url, found_via)

	def add_to_deadlinks(self, url, found_via):
		self.deadlinks.setdefault(found_via, [])
		self.deadlinks[found_via].append(url)

		print("Found new deadlink %s on %s" % (url, found_via))

	def extract_urls(self, page):
		soup = BeautifulSoup(page)
		return [link.get('href') for link in soup.findAll('a')]

c = Crawler("http://stefan-koch.name/")
c.set_url_restrict("http://stefan-koch.name/.*")
c.set_wait_time(1)
c.crawl()
<commit_msg>also check own domain for deadlinks<commit_after>from BeautifulSoup import BeautifulSoup
import urllib2, urlparse
import re
import time
import httplib

# TODO: Do not apply wait time to external links

class Crawler(object):
	def __init__(self, init_url):
		# A list of to be crawled urls, where the second element
		# in the tuple is the URL via which we found this URL
		self.pages = [(init_url, None)]

		# Urls we have already visited
		self.found = set()

		# List of deadlinks for each URL we have,
		# i.e. url1: [deadlink1, deadlink2]
		self.deadlinks = {}

		# Regular expression for URLs we are interested in (our internal
		# URLs)
		self.url_match = None

		# Timeout in seconds to wait, so that we do not kill our server
		self.wait_time = 0

	def set_url_restrict(self, regexp):
		self.url_match = re.compile(regexp)

	def set_wait_time(self, seconds):
		self.wait_time = seconds

	def crawl(self):
		while len(self.pages) > 0:
			time.sleep(self.wait_time)

			next_url = self.pages.pop()

			try:
				self.visit_url(next_url[0], next_url[1])
			except urllib2.URLError:
				continue

		print("DEADLINKS")
		print(self.deadlinks)

	def visit_url(self, url, found_via):
		response = self.check_url(url, found_via)

		if response != None and self.url_match.search(url):
			self.collect_new_urls(url, response.read())

	def collect_new_urls(self, url, html):
		print("Fetching new URLs from: %s" % url)

		try:
			for page in self.extract_urls(html):
				page = urlparse.urljoin(url, page)

				if not page in self.found:
					self.pages.append((page, url))
					self.found.add(page)
		except UnicodeEncodeError:
			pass

	def check_url(self, url, found_via):
		print("Trying URL: %s" % url)

		request = urllib2.Request(url)

		try:
			response = urllib2.urlopen(request, None, 10)
		except (urllib2.HTTPError, httplib.BadStatusLine):
			# We receive an exception in case of 404
			self.add_to_deadlinks(url, found_via)
			return None

		status = response.getcode()
		if status != None and status >= 400:
			self.add_to_deadlinks(url, found_via)

		return response

	def add_to_deadlinks(self, url, found_via):
		self.deadlinks.setdefault(found_via, [])
		self.deadlinks[found_via].append(url)

		print("Found new deadlink %s on %s" % (url, found_via))

	def extract_urls(self, page):
		soup = BeautifulSoup(page)
		return [link.get('href') for link in soup.findAll('a')]

c = Crawler("http://stefan-koch.name/")
c.set_url_restrict("http://stefan-koch.name/.*")
c.set_wait_time(1)
c.crawl()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Crawler that sends notifications as soon as servers
on Kimsufi/OVH become available for purchase"""

import json
import sys
import logging
import importlib
import tornado.ioloop
import tornado.web
from tornado.httpclient import AsyncHTTPClient
from tornado.httpclient import HTTPError
from tornado.gen import coroutine

logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
_logger = logging.getLogger(__name__)

URL = "https://ws.ovh.com/dedicated/r2/ws.dispatcher/getAvailability2"

NOTIFIERS = {
    'email': 'notifiers.email_notifier.EmailNotifier',
    'osx': 'notifiers.osx_notifier.OSXNotifier',
    'popup': 'notifiers.popup_notifier.PopupNotifier',
    'smsapi': 'notifiers.smsapi_notifier.SmsApiNotifier',
}

SERVER_TYPES = {
    '150sk10': 'KS-1',
    '150sk20': 'KS-2a',
    '150sk21': 'KS-2b',
    '150sk22': 'KS-2c',
    '150sk30': 'KS-3',
    '150sk31': 'KS-3',
    '150sk40': 'KS-4',
    '150sk41': 'KS-4',
    '150sk42': 'KS-4',
    '150sk50': 'KS-5',
    '150sk60': 'KS-6',
}

DATACENTERS = {
    'bhs': 'Beauharnois, Canada (Americas)',
    'gra': 'Gravelines, France',
    'rbx': 'Roubaix, France (Western Europe)',
    'sbg': 'Strasbourg, France (Central Europe)',
    'par': 'Paris, France',
}

STATES = {}

def update_state(state, value, message=False):
    """Update state of particular event"""
    # if new value is True and last saved was False
    if state not in STATES:
        STATES[state] = False
    if value is not STATES[state]:
        _logger.info("State change - %s: %s", state, value)
    if value and not STATES[state]:
        notifier.notify(**message)
    STATES[state] = value

@coroutine
def run_crawler():
    """Run a crawler iteration"""
    http_client = AsyncHTTPClient()
    # request OVH availablility API asynchronously
    try:
        response = yield http_client.fetch(URL)
    except HTTPError as e:
        _logger.info("HTTP Error: {0}".format(e))
        return
    response_json = json.loads(response.body.decode('utf-8'))
    if not response_json or not response_json['answer']:
        return
    availability = response_json['answer']['availability']
    for item in availability:
        # look for servers of required types in OVH availability list
        if SERVER_TYPES.get(item['reference']) in config['servers']:
            # make a flat list of zones where servers are available
            available_zones = [e['zone'] for e in item['zones']
                               if e['availability'] not in ['unavailable',
                                                            'unknown']]
            # iterate over all tacked zones and set availability states
            for zone in config['zones']:
                server = SERVER_TYPES[item['reference']]
                state_id = '%s_available_in_%s' % (server, zone)
                # update state for each tracked zone
                text = "Server %s is available in %s" % (server, zone)
                message = {
                    'title': "Server %s available" % server,
                    'text': text,
                    'url': "http://www.kimsufi.com/fr/index.xml"
                }
                update_state(state_id, zone in available_zones, message)

if __name__ == "__main__":
    CONFIG_NAME = sys.argv[1] if len(sys.argv) == 2 else 'config.json'
    with open(CONFIG_NAME, 'r') as configfile:
        try:
            config = json.loads(configfile.read())
        except ValueError:
            _logger.error("Parsing JSON settings in config.json has failed. "
                          "Check syntax with a validator (i.e. jsonlint.com)")
            sys.exit(1)

    # Select notifier, 'email' by default
    if 'notifier' not in config:
        _logger.warning("No notifier selected in config, 'email' will be used")
        config['notifier'] = 'email'
    # Instantiate notifier class dynamically
    try:
        n_path = NOTIFIERS[config['notifier']]
        n_file, n_classname = n_path.rsplit('.', 1)
        n_module = importlib.import_module(n_file)
        notifier = getattr(n_module, n_classname)(config)
    except:
        _logger.exception("Notifier loading failed,"
                          "check configuration for errors")
        sys.exit(1)

    loop = tornado.ioloop.IOLoop.instance()
    tornado.ioloop.PeriodicCallback(run_crawler, 30000).start()
    _logger.info("Starting IO loop")

    try:
        loop.start()
    except KeyboardInterrupt:
        _logger.info("Terminated by user. Bye.")
        sys.exit(0)
<commit_msg>Add notifications for Game Server from soyoustart<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Crawler that sends notifications as soon as servers
on Kimsufi/OVH become available for purchase"""

import json
import sys
import logging
import importlib
import tornado.ioloop
import tornado.web
from tornado.httpclient import AsyncHTTPClient
from tornado.httpclient import HTTPError
from tornado.gen import coroutine

logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
_logger = logging.getLogger(__name__)

URL = "https://ws.ovh.com/dedicated/r2/ws.dispatcher/getAvailability2"

NOTIFIERS = {
    'email': 'notifiers.email_notifier.EmailNotifier',
    'osx': 'notifiers.osx_notifier.OSXNotifier',
    'popup': 'notifiers.popup_notifier.PopupNotifier',
    'smsapi': 'notifiers.smsapi_notifier.SmsApiNotifier',
}

SERVER_TYPES = {
    '150sk10': 'KS-1',
    '150sk20': 'KS-2a',
    '150sk21': 'KS-2b',
    '150sk22': 'KS-2c',
    '150sk30': 'KS-3',
    '150sk31': 'KS-3',
    '150sk40': 'KS-4',
    '150sk41': 'KS-4',
    '150sk42': 'KS-4',
    '150sk50': 'KS-5',
    '150sk60': 'KS-6',
    '141game1': 'GAME-1',
    '141game2': 'GAME-2',
    '141game3': 'GAME-3',

}

DATACENTERS = {
    'bhs': 'Beauharnois, Canada (Americas)',
    'gra': 'Gravelines, France',
    'rbx': 'Roubaix, France (Western Europe)',
    'sbg': 'Strasbourg, France (Central Europe)',
    'par': 'Paris, France',
}

STATES = {}

def update_state(state, value, message=False):
    """Update state of particular event"""
    # if new value is True and last saved was False
    if state not in STATES:
        STATES[state] = False
    if value is not STATES[state]:
        _logger.info("State change - %s: %s", state, value)
    if value and not STATES[state]:
        notifier.notify(**message)
    STATES[state] = value

@coroutine
def run_crawler():
    """Run a crawler iteration"""
    http_client = AsyncHTTPClient()
    # request OVH availablility API asynchronously
    try:
        response = yield http_client.fetch(URL)
    except HTTPError as e:
        _logger.info("HTTP Error: {0}".format(e))
        return
    response_json = json.loads(response.body.decode('utf-8'))
    if not response_json or not response_json['answer']:
        return
    availability = response_json['answer']['availability']
    for item in availability:
        # look for servers of required types in OVH availability list
        if SERVER_TYPES.get(item['reference']) in config['servers']:
            # make a flat list of zones where servers are available
            available_zones = [e['zone'] for e in item['zones']
                               if e['availability'] not in ['unavailable',
                                                            'unknown']]
            # iterate over all tacked zones and set availability states
            for zone in config['zones']:
                server = SERVER_TYPES[item['reference']]
                state_id = '%s_available_in_%s' % (server, zone)
                # update state for each tracked zone
                text = "Server %s is available in %s" % (server, zone)
                message = {
                    'title': "Server %s available" % server,
                    'text': text,
                    'url': "http://www.kimsufi.com/fr/index.xml"
                }
                update_state(state_id, zone in available_zones, message)

if __name__ == "__main__":
    CONFIG_NAME = sys.argv[1] if len(sys.argv) == 2 else 'config.json'
    with open(CONFIG_NAME, 'r') as configfile:
        try:
            config = json.loads(configfile.read())
        except ValueError:
            _logger.error("Parsing JSON settings in config.json has failed. "
                          "Check syntax with a validator (i.e. jsonlint.com)")
            sys.exit(1)

    # Select notifier, 'email' by default
    if 'notifier' not in config:
        _logger.warning("No notifier selected in config, 'email' will be used")
        config['notifier'] = 'email'
    # Instantiate notifier class dynamically
    try:
        n_path = NOTIFIERS[config['notifier']]
        n_file, n_classname = n_path.rsplit('.', 1)
        n_module = importlib.import_module(n_file)
        notifier = getattr(n_module, n_classname)(config)
    except:
        _logger.exception("Notifier loading failed,"
                          "check configuration for errors")
        sys.exit(1)

    loop = tornado.ioloop.IOLoop.instance()
    tornado.ioloop.PeriodicCallback(run_crawler, 30000).start()
    _logger.info("Starting IO loop")

    try:
        loop.start()
    except KeyboardInterrupt:
        _logger.info("Terminated by user. Bye.")
        sys.exit(0)
<|endoftext|>