<commit_before>import os
import urllib

from abc import ABCMeta
from luigi import Task, Parameter, WrapperTask, LocalTarget
from collections import OrderedDict
from tasks.util import DownloadUnzipTask, shell, TableTask, TempTableTask, classpath
from tasks.meta import current_session
from tasks.ca.statcan.geo import (
    GEO_CT, GEO_PR, GEO_CD, GEO_CSD, GEO_CMA,
    GEOGRAPHY_CODES, GEOGRAPHIES, GeographyColumns)
from tasks.ca.statcan.util import StatCanParser
from tasks.ca.statcan.cols_census import CensusColumns
from tasks.ca.statcan.cols_nhs import NHSColumns

SURVEY_CEN = 'census'
SURVEY_NHS = 'nhs'

SURVEYS = (
    SURVEY_CEN,
    SURVEY_NHS,
)

SURVEY_CODES = {
    SURVEY_CEN: '98-316-XWE2011001',
    SURVEY_NHS: '99-004-XWE2011001',
}

SURVEY_URLS = {
    SURVEY_CEN: 'census-recensement',
    SURVEY_NHS: 'nhs-enm',
}

URL = 'http://www12.statcan.gc.ca/{survey_url}/2011/dp-pd/prof/details/download-telecharger/comprehensive/comp_download.cfm?CTLG={survey_code}&FMT=CSV{geo_code}'

class BaseParams:
    __metaclass__ = ABCMeta

    resolution = Parameter(default=GEO_PR)
    survey = Parameter(default=SURVEY_CEN)

class DownloadData(BaseParams, DownloadUnzipTask):
    def download(self):
        urllib.urlretrieve(url=URL.format(
                           survey_url=SURVEY_URLS[self.survey],
                           survey_code=SURVEY_CODES[self.survey],
                           geo_code=GEOGRAPHY_CODES[self.resolution],
                           ),
                           filename=self.output().path + '.zip')

class SplitAndTransposeData(BaseParams, Task):
    def requires(self):
        return DownloadData(resolution=self.resolution, survey=self.survey)

    def run(self):
        infiles = shell('ls {input}/{survey_code}-{geo_code}*.[cC][sS][vV]'.format(
            input=self.input().path,
            survey_code=SURVEY_CODES[self.survey],
            geo_code=GEOGRAPHY_CODES[self.resolution]
        ))
        in_csv_files = infiles.strip().split('\n')
        os.makedirs(self.output().path)
        StatCanParser().parse_csv_to_files(in_csv_files, self.output().path)

    def output(self):
        return LocalTarget(os.path.join('tmp', classpath(self), self.task_id))

class CopyDataToTable(BaseParams, TempTableTask):

    table = Parameter()

    def requires(self):
        return SplitAndTransposeData(resolution=self.resolution, survey=self.survey)

    def run(self):
        infile = os.path.join(self.input().path, self.table + '.csv')
        headers = shell('head -n 1 {csv}'.format(csv=infile))
        cols = ['{} NUMERIC'.format(h) for h in headers.split(',')[1:]]

        session = current_session()
        session.execute('CREATE TABLE {output} (Geo_Code TEXT, {cols})'.format(
            output=self.output().table,
            cols=', '.join(cols)
        ))
        session.commit()
        shell("cat '{infile}' | psql -c 'COPY {output} FROM STDIN WITH CSV HEADER'".format(
            output=self.output().table,
            infile=infile,
        ))
        session.execute('ALTER TABLE {output} ADD PRIMARY KEY (geo_code)'.format(
            output=self.output().table
        ))

class ImportData(BaseParams, Task):
    def requires(self):
        return SplitAndTransposeData(resolution=self.resolution, survey=self.survey)

    def run(self):
        infiles = shell('ls {input}/*.csv'.format(
            input=self.input().path))
        fhandle = self.output().open('w')
        for infile in infiles.strip().split('\n'):
            table = os.path.split(infile)[-1].split('.csv')[0]
            data = yield CopyDataToTable(resolution=self.resolution, survey=self.survey, table=table)
            fhandle.write('{table}\n'.format(table=data.table))
        fhandle.close()

    def output(self):
        return LocalTarget(os.path.join('tmp', classpath(self), self.task_id))

class ImportAllResolutions(WrapperTask):
    survey = Parameter(default=SURVEY_CEN)

    def requires(self):
        for resolution in GEOGRAPHIES:
            yield ImportData(resolution=resolution, survey=self.survey)

class ImportAllSurveys(WrapperTask):
    resolution = Parameter(default=GEO_PR)

    def requires(self):
        for survey in SURVEYS:
            yield ImportData(resolution=self.resolution, survey=survey)

class ImportAll(WrapperTask):
    def requires(self):
        for survey in SURVEYS:
            for resolution in GEOGRAPHIES:
                yield ImportData(resolution=resolution, survey=survey)

class Survey(BaseParams, TableTask):
    def requires(self):
        '''
        Subclasses must override this.
        '''
        raise NotImplementedError('Survey must define requires()')

    def timespan(self):
        '''
        Subclasses must override this.
        '''
        raise NotImplementedError('Survey must define timespan()')

    def columns(self):
        cols = OrderedDict()
        input_ = self.input()
        cols['Geo_Code'] = input_['geom_columns']['geom_id']
        cols.update(input_['data_columns'])
        return cols

    def populate(self):
        session = current_session()
        cols = self.columns()
        inserted = False
        for line in self.input()['data'].open():
            intable = line.strip()
            table = intable.split('_')[1]
            cols_for_table = OrderedDict([
                (n, t,) for n, t in cols.iteritems() if table in t._id
            ])
            out_colnames = cols_for_table.keys()
            in_colnames = [t._id.split('.')[-1] for t in cols_for_table.values()]
            assert len(out_colnames) == len(in_colnames)
            if not out_colnames:
                continue
            if not inserted:
                stmt =  'INSERT INTO {output} (geo_code, {out_colnames}) ' \
                        'SELECT geo_code, {in_colnames} ' \
                        'FROM {input} '.format(
                            output=self.output().table,
                            out_colnames=', '.join(out_colnames),
                            in_colnames=', '.join(in_colnames),
                            input=intable)
                session.execute(stmt)
                inserted = True
            else:
                stmt =  'UPDATE {output} out ' \
                        'SET {set} ' \
                        'FROM {input} intable ' \
                        'WHERE intable.geo_code = out.geo_code ' \
                        .format(
                            set=', '.join([
                                '{} = {}'.format(a, b) for a, b in zip(
                                    out_colnames, in_colnames)
                            ]),
                            output=self.output().table,
                            input=intable)
                session.execute(stmt)

class Census(Survey):
    def requires(self):
        return {
            'data': ImportData(resolution=self.resolution, survey=SURVEY_CEN),
            'geom_columns': GeographyColumns(resolution=self.resolution),
            'data_columns': CensusColumns(),
        }

    def timespan(self):
        return 2011

class NHS(Survey):
    def requires(self):
        return {
            'data': ImportData(resolution=self.resolution, survey=SURVEY_NHS),
            'geom_columns': GeographyColumns(resolution=self.resolution),
            'data_columns': NHSColumns(),
        }

    def timespan(self):
        return 2011
<commit_msg>[sg-7] add topic table as params, refactor a little<commit_after>import os
import urllib

from abc import ABCMeta
from luigi import Task, Parameter, WrapperTask, LocalTarget
from collections import OrderedDict
from tasks.util import DownloadUnzipTask, shell, TableTask, TempTableTask, classpath
from tasks.meta import current_session
from tasks.ca.statcan.geo import (
    GEO_CT, GEO_PR, GEO_CD, GEO_CSD, GEO_CMA,
    GEOGRAPHY_CODES, GEOGRAPHIES, GeographyColumns)
from tasks.ca.statcan.util import StatCanParser
from tasks.ca.statcan.cols_census import CensusColumns
from tasks.ca.statcan.cols_nhs import NHSColumns

SURVEY_CEN = 'census'
SURVEY_NHS = 'nhs'

SURVEYS = (
    SURVEY_CEN,
    SURVEY_NHS,
)

SURVEY_CODES = {
    SURVEY_CEN: '98-316-XWE2011001',
    SURVEY_NHS: '99-004-XWE2011001',
}

SURVEY_URLS = {
    SURVEY_CEN: 'census-recensement',
    SURVEY_NHS: 'nhs-enm',
}

URL = 'http://www12.statcan.gc.ca/{survey_url}/2011/dp-pd/prof/details/download-telecharger/comprehensive/comp_download.cfm?CTLG={survey_code}&FMT=CSV{geo_code}'

class BaseParams:
    __metaclass__ = ABCMeta

    resolution = Parameter(default=GEO_PR)
    survey = Parameter(default=SURVEY_CEN)

class DownloadData(BaseParams, DownloadUnzipTask):
    def download(self):
        urllib.urlretrieve(url=URL.format(
                           survey_url=SURVEY_URLS[self.survey],
                           survey_code=SURVEY_CODES[self.survey],
                           geo_code=GEOGRAPHY_CODES[self.resolution],
                           ),
                           filename=self.output().path + '.zip')

class SplitAndTransposeData(BaseParams, Task):
    def requires(self):
        return DownloadData(resolution=self.resolution, survey=self.survey)

    def run(self):
        infiles = shell('ls {input}/{survey_code}-{geo_code}*.[cC][sS][vV]'.format(
            input=self.input().path,
            survey_code=SURVEY_CODES[self.survey],
            geo_code=GEOGRAPHY_CODES[self.resolution]
        ))
        in_csv_files = infiles.strip().split('\n')
        os.makedirs(self.output().path)
        StatCanParser().parse_csv_to_files(in_csv_files, self.output().path)

    def output(self):
        return LocalTarget(os.path.join('tmp', classpath(self), self.task_id))

#####################################
# IMPORT TO TEMP TABLES
#####################################
class CopyDataToTable(BaseParams, TempTableTask):

    table = Parameter()

    def requires(self):
        return SplitAndTransposeData(resolution=self.resolution, survey=self.survey)

    def run(self):
        infile = os.path.join(self.input().path, self.table + '.csv')
        headers = shell('head -n 1 {csv}'.format(csv=infile))
        cols = ['{} NUMERIC'.format(h) for h in headers.split(',')[1:]]

        session = current_session()
        session.execute('CREATE TABLE {output} (Geo_Code TEXT, {cols})'.format(
            output=self.output().table,
            cols=', '.join(cols)
        ))
        session.commit()
        shell("cat '{infile}' | psql -c 'COPY {output} FROM STDIN WITH CSV HEADER'".format(
            output=self.output().table,
            infile=infile,
        ))
        session.execute('ALTER TABLE {output} ADD PRIMARY KEY (geo_code)'.format(
            output=self.output().table
        ))

class ImportData(BaseParams, Task):
    def requires(self):
        return SplitAndTransposeData(resolution=self.resolution, survey=self.survey)

    def run(self):
        infiles = shell('ls {input}/*.csv'.format(
            input=self.input().path))
        fhandle = self.output().open('w')
        for infile in infiles.strip().split('\n'):
            table = os.path.split(infile)[-1].split('.csv')[0]
            data = yield CopyDataToTable(resolution=self.resolution, survey=self.survey, table=table)
            fhandle.write('{table}\n'.format(table=data.table))
        fhandle.close()

    def output(self):
        return LocalTarget(os.path.join('tmp', classpath(self), self.task_id))

class ImportAllResolutions(WrapperTask):
    survey = Parameter(default=SURVEY_CEN)

    def requires(self):
        for resolution in GEOGRAPHIES:
            yield ImportData(resolution=resolution, survey=self.survey)

class ImportAllSurveys(WrapperTask):
    resolution = Parameter(default=GEO_PR)

    def requires(self):
        for survey in SURVEYS:
            yield ImportData(resolution=self.resolution, survey=survey)

class ImportAll(WrapperTask):
    def requires(self):
        for survey in SURVEYS:
            for resolution in GEOGRAPHIES:
                yield ImportData(resolution=resolution, survey=survey)

#####################################
# COPY TO OBSERVATORY
#####################################
class Survey(BaseParams, TableTask):

    topic = Parameter(default='t001')

    def requires(self):
        '''
        Subclasses must override this.
        '''
        raise NotImplementedError('Survey must define requires()')

    def timespan(self):
        '''
        Subclasses must override this.
        '''
        raise NotImplementedError('Survey must define timespan()')

    def columns(self):
        cols = OrderedDict()
        input_ = self.input()
        cols['Geo_Code'] = input_['geometa']['geom_id']
        for colname, coltarget in input_['meta'].iteritems():
            if coltarget._id.split('.')[-1].lower().startswith(self.topic.lower()):
                cols[colname] = coltarget
        return cols

    def populate(self):
        session = current_session()
        columns = self.columns()
        out_colnames = columns.keys()
        in_table = self.input()['data']
        in_colnames = [ct._id.split('.')[-1] for ct in columns.values()]
        in_colnames[0] = 'geom_id'
        for i, in_c in enumerate(in_colnames):
            cmd =   "SELECT 'exists' FROM information_schema.columns " \
                    "WHERE table_schema = '{schema}' " \
                    "  AND table_name = '{tablename}' " \
                    "  AND column_name = '{colname}' " \
                    "  LIMIT 1".format(
                        schema=in_table.schema,
                        tablename=in_table.tablename.lower(),
                        colname=in_c.lower())
            # remove columns that aren't in input table
            if session.execute(cmd).fetchone() is None:
                in_colnames[i] = None
                out_colnames[i] = None
        in_colnames = [
            "CASE {ic}::TEXT WHEN '-6' THEN NULL ELSE {ic} END".format(ic=ic) for ic in in_colnames if ic is not None]
        out_colnames = [oc for oc in out_colnames if oc is not None]

        cmd =   'INSERT INTO {output} ({out_colnames}) ' \
                'SELECT {in_colnames} FROM {input} '.format(
                    output=self.output().table,
                    input=in_table.table,
                    in_colnames=', '.join(in_colnames),
                    out_colnames=', '.join(out_colnames))
        session.execute(cmd)

class Census(Survey):
    def requires(self):
        return {
            'data': ImportData(resolution=self.resolution, survey=SURVEY_CEN),
            'geometa': GeographyColumns(resolution=self.resolution),
            'meta': CensusColumns(),
        }

    def timespan(self):
        return 2011

class AllCensusTopics(BaseParams, WrapperTask):
    def requires(self):
        topic_range = range(1, 11)   # 1-10

        for count in topic_range:
            topic = 't{:03d}'.format(count)
            yield Census(resolution=self.resolution, topic=topic)

class NHS(Survey):
    def requires(self):
        return {
            'data': ImportData(resolution=self.resolution, survey=SURVEY_NHS),
            'geometa': GeographyColumns(resolution=self.resolution),
            'meta': NHSColumns(),
        }

    def timespan(self):
        return 2011
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
New Discussions -- Provides a list of new discussions within a WikiProject's scope
Copyright (C) 2015 James Hare, 2016 Ben Kurtovic
Licensed under MIT License: http://mitlicense.org
"""

from collections import namedtuple
from datetime import datetime
import re

from reportsbot.task import Task
from reportsbot.util import join_full_title

import mwparserfromhell
from pywikibot.data.api import Request

__all__ = ["NewDiscussions"]

_Section = namedtuple("_Section", ["name", "timestamp"])
_Discussion = namedtuple("_Discussion", ["title", "name", "timestamp"])

class NewDiscussions(Task):
    """Updates a list of new discussions within a WikiProject's scope."""
    DISCUSSION_TEMPLATE = "WPX new discussion"
    DISCUSSIONS_PER_PAGE = 15
    DISCUSSIONS_BEFORE_FOLD = 4

    @staticmethod
    def _parse_timestamp(text):
        """Return a datetime for the given timestamp string, or ValueError."""
        return datetime.strptime(text, "%H:%M, %d %B %Y (UTC)")

    def _extract_sections(self, text):
        """Return a list of section tuples for the given page."""
        code = mwparserfromhell.parse(text)
        sections = []

        for section in code.get_sections(levels=[2]):
            clean = section.strip_code()
            match = re.search(r"\d\d:\d\d,\s\d\d?\s\w+\s\d{4}\s\(UTC\)", clean)
            if not match:
                continue
            try:
                timestamp = self._parse_timestamp(match.group(0))
            except ValueError:
                continue

            name = str(section.get(0).title)
            sections.append(_Section(name, timestamp))

        return sections

    def _load_pages(self, revids):
        """Load a chunk of pages from the API by revision."""
        req = Request(self._bot.site, parameters={
            "action": "query", "prop": "revisions", "rvprop": "content",
            "revids": "|".join(str(revid) for revid in revids)
        })

        data = req.submit()
        return [(page["title"], page["revisions"][0]["*"])
                for page in data["query"]["pages"].values()
                if "*" in page["revisions"][0]]

    def _get_updated_discussions(self, start, end):
        """Return a dict mapping talk page titles to lists of section tuples.

        The only pages included in the dict are those that have been updated
        in the given time range.
        """
        query = """SELECT MAX(rc_this_oldid)
            FROM recentchanges
            WHERE rc_timestamp >= ? AND rc_timestamp < ?
            AND rc_namespace % 2 = 1 AND rc_namespace != 3
            AND (rc_type = 0 OR rc_type = 1) AND rc_bot = 0
            GROUP BY rc_namespace, rc_title"""
        # TODO: generate events for pages that have been moved/deleted

        startts = start.strftime("%Y%m%d%H%M%S")
        endts = end.strftime("%Y%m%d%H%M%S")
        self._logger.info("Fetching discussions updated between %s and %s",
                          startts, endts)

        with self._bot.wikidb as cursor:
            cursor.execute(query, (startts, endts))
            revids = [revid for (revid,) in cursor.fetchall()]

        self._logger.debug("Fetching sections for %s pages", len(revids))

        sections = {}
        chunksize = 50
        for start in range(0, len(revids), chunksize):
            chunk = revids[start:start+chunksize]
            pages = self._load_pages(chunk)
            sections.update({title: self._extract_sections(text)
                             for title, text in pages})

        return sections

    def _get_current_discussions(self, title):
        """Return a dict mapping talk page titles to lists of section tuples.

        Given a WikiProject new discussions page, return all discussions
        currently listed.
        """
        text = self._bot.get_page(title).text
        code = mwparserfromhell.parse(text)
        discussions = {}

        for tmpl in code.filter_templates():
            if tmpl.title != self.DISCUSSION_TEMPLATE:
                continue
            if not (tmpl.has("title") and tmpl.has("section") and
                    tmpl.has("timestamp")):
                continue

            try:
                timestamp = self._parse_timestamp(tmpl.get("timestamp").value)
            except ValueError:
                continue
            title = str(tmpl.get("title").value)
            section = _Section(tmpl.get("section").value, timestamp)
            if title in discussions:
                discussions[title].append(section)
            else:
                discussions[title] = [section]

        return discussions

    def _process_discussions(self, pages, current, updated):
        """Return a sorted list of the most recent discussion tuples."""
        sections = {}

        for page in pages:
            title = join_full_title(self._bot.site, page.ns + 1, page.title)
            if title in updated:
                sections[title] = updated[title]
            elif title in current:
                sections[title] = current[title]

        discussions = [_Discussion(title, section.name, section.timestamp)
                       for title in sections for section in sections[title]]
        discussions.sort(key=lambda disc: disc.timestamp, reverse=True)
        return discussions[:self.DISCUSSIONS_PER_PAGE]

    def _save_discussions(self, project, title, discussions):
        """Save the given list of discussions to the given page title."""
        text = """<noinclude>
{{Clickable button 2|%(projname)s|Return to WikiProject|class=mw-ui-neutral}}
</noinclude>
{{WPX action box|color={{{2|#086}}}|title=Have a question?|content=
{{Clickable button 2|url={{fullurl:%(projtalk)s|action=edit&section=new}}|Ask the WikiProject|class=mw-ui-progressive mw-ui-block}}

{{Clickable button 2|%(projtalk)s|View Other Discussions|class=mw-ui-block}}
}}
{{WPX list start|intro={{WPX last updated|%(title)s}}}}
%(discussions)s
{{WPX list end|more=%(title)s}}
        """
        template = "{{WPX new discussion|color={{{1|#37f}}}|title=%(title)s|section=%(name)s|timestamp=%(timestamp)s}}"

        discitems = [
            template % {
                "title": disc.title,
                "name": disc.name,
                "timestamp": disc.timestamp.strftime("%H:%M, %d %B %Y (UTC)")
            }
            for disc in discussions]

        fold = self.DISCUSSIONS_BEFORE_FOLD
        if len(discitems) > fold:
            before = "\n".join(discitems[:fold])
            after = "\n".join(discitems[fold:])
            disclist = before + "<noinclude>\n" + after + "</noinclude>"
        else:
            disclist = "\n".join(discitems)

        projtalk = self._bot.get_page(project.name).toggleTalkPage().title()

        page = self._bot.get_page(title)
        page.text = text % {
            "title": title,
            "projname": project.name,
            "projtalk": projtalk,
            "discussions": disclist
        }
        page.save("Updating new discussions", minor=False)

    def _process(self, project, updated):
        """Process new discussions for the given project."""
        self._logger.info("Updating new discussions for %s", project.name)
        title = project.name + "/Discussions"

        pages = project.get_members()
        current = self._get_current_discussions(title)
        discussions = self._process_discussions(pages, current, updated)
        self._save_discussions(project, title, discussions)

    def run(self):
        start = self._bot.get_last_updated("new_discussions")
        end = datetime.utcnow()
        updated = self._get_updated_discussions(start, end)

        for project in self._bot.get_configured_projects():
            if project.config.get("new_discussions"):
                self._process(project, updated)

        self._bot.set_last_updated("new_discussions", end)
<commit_msg>new_discussions: Strip code in section name.<commit_after># -*- coding: utf-8 -*-

"""
New Discussions -- Provides a list of new discussions within a WikiProject's scope
Copyright (C) 2015 James Hare, 2016 Ben Kurtovic
Licensed under MIT License: http://mitlicense.org
"""

from collections import namedtuple
from datetime import datetime
import re

from reportsbot.task import Task
from reportsbot.util import join_full_title

import mwparserfromhell
from pywikibot.data.api import Request

__all__ = ["NewDiscussions"]

_Section = namedtuple("_Section", ["name", "timestamp"])
_Discussion = namedtuple("_Discussion", ["title", "name", "timestamp"])

class NewDiscussions(Task):
    """Updates a list of new discussions within a WikiProject's scope."""
    DISCUSSION_TEMPLATE = "WPX new discussion"
    DISCUSSIONS_PER_PAGE = 15
    DISCUSSIONS_BEFORE_FOLD = 4

    @staticmethod
    def _parse_timestamp(text):
        """Return a datetime for the given timestamp string, or ValueError."""
        return datetime.strptime(text, "%H:%M, %d %B %Y (UTC)")

    def _extract_sections(self, text):
        """Return a list of section tuples for the given page."""
        code = mwparserfromhell.parse(text)
        sections = []

        for section in code.get_sections(levels=[2]):
            clean = section.strip_code()
            match = re.search(r"\d\d:\d\d,\s\d\d?\s\w+\s\d{4}\s\(UTC\)", clean)
            if not match:
                continue
            try:
                timestamp = self._parse_timestamp(match.group(0))
            except ValueError:
                continue

            name = str(section.get(0).title.strip_code()).strip()
            sections.append(_Section(name, timestamp))

        return sections

    def _load_pages(self, revids):
        """Load a chunk of pages from the API by revision."""
        req = Request(self._bot.site, parameters={
            "action": "query", "prop": "revisions", "rvprop": "content",
            "revids": "|".join(str(revid) for revid in revids)
        })

        data = req.submit()
        return [(page["title"], page["revisions"][0]["*"])
                for page in data["query"]["pages"].values()
                if "*" in page["revisions"][0]]

    def _get_updated_discussions(self, start, end):
        """Return a dict mapping talk page titles to lists of section tuples.

        The only pages included in the dict are those that have been updated
        in the given time range.
        """
        query = """SELECT MAX(rc_this_oldid)
            FROM recentchanges
            WHERE rc_timestamp >= ? AND rc_timestamp < ?
            AND rc_namespace % 2 = 1 AND rc_namespace != 3
            AND (rc_type = 0 OR rc_type = 1) AND rc_bot = 0
            GROUP BY rc_namespace, rc_title"""
        # TODO: generate events for pages that have been moved/deleted

        startts = start.strftime("%Y%m%d%H%M%S")
        endts = end.strftime("%Y%m%d%H%M%S")
        self._logger.info("Fetching discussions updated between %s and %s",
                          startts, endts)

        with self._bot.wikidb as cursor:
            cursor.execute(query, (startts, endts))
            revids = [revid for (revid,) in cursor.fetchall()]

        self._logger.debug("Fetching sections for %s pages", len(revids))

        sections = {}
        chunksize = 50
        for start in range(0, len(revids), chunksize):
            chunk = revids[start:start+chunksize]
            pages = self._load_pages(chunk)
            sections.update({title: self._extract_sections(text)
                             for title, text in pages})

        return sections

    def _get_current_discussions(self, title):
        """Return a dict mapping talk page titles to lists of section tuples.

        Given a WikiProject new discussions page, return all discussions
        currently listed.
        """
        text = self._bot.get_page(title).text
        code = mwparserfromhell.parse(text)
        discussions = {}

        for tmpl in code.filter_templates():
            if tmpl.title != self.DISCUSSION_TEMPLATE:
                continue
            if not (tmpl.has("title") and tmpl.has("section") and
                    tmpl.has("timestamp")):
                continue

            try:
                timestamp = self._parse_timestamp(tmpl.get("timestamp").value)
            except ValueError:
                continue
            title = str(tmpl.get("title").value)
            section = _Section(tmpl.get("section").value, timestamp)
            if title in discussions:
                discussions[title].append(section)
            else:
                discussions[title] = [section]

        return discussions

    def _process_discussions(self, pages, current, updated):
        """Return a sorted list of the most recent discussion tuples."""
        sections = {}

        for page in pages:
            title = join_full_title(self._bot.site, page.ns + 1, page.title)
            if title in updated:
                sections[title] = updated[title]
            elif title in current:
                sections[title] = current[title]

        discussions = [_Discussion(title, section.name, section.timestamp)
                       for title in sections for section in sections[title]]
        discussions.sort(key=lambda disc: disc.timestamp, reverse=True)
        return discussions[:self.DISCUSSIONS_PER_PAGE]

    def _save_discussions(self, project, title, discussions):
        """Save the given list of discussions to the given page title."""
        text = """<noinclude>
{{Clickable button 2|%(projname)s|Return to WikiProject|class=mw-ui-neutral}}
</noinclude>
{{WPX action box|color={{{2|#086}}}|title=Have a question?|content=
{{Clickable button 2|url={{fullurl:%(projtalk)s|action=edit&section=new}}|Ask the WikiProject|class=mw-ui-progressive mw-ui-block}}

{{Clickable button 2|%(projtalk)s|View Other Discussions|class=mw-ui-block}}
}}
{{WPX list start|intro={{WPX last updated|%(title)s}}}}
%(discussions)s
{{WPX list end|more=%(title)s}}
        """
        template = "{{WPX new discussion|color={{{1|#37f}}}|title=%(title)s|section=%(name)s|timestamp=%(timestamp)s}}"

        discitems = [
            template % {
                "title": disc.title,
                "name": disc.name,
                "timestamp": disc.timestamp.strftime("%H:%M, %d %B %Y (UTC)")
            }
            for disc in discussions]

        fold = self.DISCUSSIONS_BEFORE_FOLD
        if len(discitems) > fold:
            before = "\n".join(discitems[:fold])
            after = "\n".join(discitems[fold:])
            disclist = before + "<noinclude>\n" + after + "</noinclude>"
        else:
            disclist = "\n".join(discitems)

        projtalk = self._bot.get_page(project.name).toggleTalkPage().title()

        page = self._bot.get_page(title)
        page.text = text % {
            "title": title,
            "projname": project.name,
            "projtalk": projtalk,
            "discussions": disclist
        }
        page.save("Updating new discussions", minor=False)

    def _process(self, project, updated):
        """Process new discussions for the given project."""
        self._logger.info("Updating new discussions for %s", project.name)
        title = project.name + "/Discussions"

        pages = project.get_members()
        current = self._get_current_discussions(title)
        discussions = self._process_discussions(pages, current, updated)
        self._save_discussions(project, title, discussions)

    def run(self):
        start = self._bot.get_last_updated("new_discussions")
        end = datetime.utcnow()
        updated = self._get_updated_discussions(start, end)

        for project in self._bot.get_configured_projects():
            if project.config.get("new_discussions"):
                self._process(project, updated)

        self._bot.set_last_updated("new_discussions", end)
<|endoftext|>


<commit_before>import time

from .worker import DNSResolver
from exaproxy.network.functions import isip

class ResolverManager (object):
	resolverFactory = DNSResolver

	def __init__ (self, poller, configuration, max_workers):
		self.poller = poller
		self.configuration = configuration

		self.resolver_factory = self.resolverFactory(configuration)

		# The actual work is done in the worker
		self.worker = self.resolver_factory.createUDPClient()

		# All currently active clients (one UDP and many TCP)
		self.workers = {}
		self.workers[self.worker.socket] = self.worker
		self.poller.addReadSocket('read_resolver', self.worker.socket)

		# Track the clients currently expecting results
		self.clients = {}         #   client_id : identifier

		# Key should be the hostname rather than the request ID?
		self.resolving = {}       #   identifier, worker_id :

		# TCP workers that have not yet sent a complete request
		self.sending = {}         #   sock :

		# track the current queries and when they were started
		self.active = []

		self.cache = {}
		self.cached = []

		self.max_workers = max_workers
		self.worker_count = len(self.workers) # the UDP client

		self.waiting = []

	def cacheDestination (self, hostname, ip):
		if hostname not in self.cache:
			expire_time = time.time() + self.configuration.dns.ttl
			expire_time = expire_time - expire_time % self.configuration.dns.resolution
			latest_time, latest_hosts = self.cached[-1] if self.cached else (-1, None)

			if expire_time > latest_time:
				hosts = []
				self.cached.append((expire_time, hosts))
			else:
				hosts = latest_hosts

			self.cache[hostname] = ip
			hosts.append(hostname)

	def expireCache (self):
		# expire only one set of cache entries at a time
		if self.cached:
			current_time = time.time()
			expire_time, hosts = self.cached[0]

			if current_time >= expire_time:
				(expire_time, hosts), self.cached = self.cached[0], self.cached[1:]

				for hostname in hosts:
					self.cache.pop(hostname, None)

	def cleanup(self):
		cutoff = time.time() - self.configuration.dns.timeout
		count = 0

		for timestamp, client_id, sock in self.active:
			if timestamp > cutoff:
				break

			count += 1
			cli_data = self.clients.pop(client_id, None)
			worker = self.workers.get(sock)

			if cli_data is not None:
				w_id, identifier, active_time  = cli_data
				data = self.resolving.pop((w_id, identifier), None)
				if not data:
					data = self.sending.pop(sock, None)

				if data:
					client_id, original, hostname, command, decision = data
					yield client_id, 'rewrite', '\0'.join(('503', 'dns.html', '', '', '', hostname, 'peer'))

			if worker is not None:
				if worker is not self.worker:
					worker.close()
					self.workers.pop(sock)

		if count:
			self.active = self.active[count:]

	def resolves(self, command, decision):
		if command in ('download', 'connect'):
			hostname = decision.split('\0')[0]
			if isip(hostname):
				res = False
			else:
				res = True
		else:
			res = False

		if res is True:

		return res

	def extractHostname(self, command, decision):
		data = decision.split('\0')

		if command == 'download':
			hostname = data[0]

		elif command == 'connect':
			hostname = decision.split('\0')[0]

		else:
			hostname = None

		return hostname

	def resolveDecision(self, command, decision, ip):
		if command in ('download', 'connect'):
			hostname, args = decision.split('\0', 1)
			newdecision = '\0'.join((ip, args))
		else:
			newdecision = None

		return newdecision

	def startResolving(self, client_id, command, decision):
		hostname = self.extractHostname(command, decision)

		if hostname:
			if hostname in self.cache:
				identifier = None
				ip = self.cache[hostname]

				if ip is not None:
					resolved = self.resolveDecision(command, decision, ip)
					response = client_id, command, resolved

				else:
					newdecision = '\0'.join(('503', 'dns.html', 'http', '', '', hostname, 'peer'))
					response = client_id, 'rewrite', newdecision

			else:
				jumbo = max(len(p) for p in hostname.split('.')) > 255
				if jumbo:
					identifier = None
					newdecision = '\0'.join(('503', 'dns.html', 'http', '', '', hostname, 'peer'))
					response = client_id, 'rewrite', newdecision

				else:
					identifier, _ = self.worker.resolveHost(hostname)
					response = None
					active_time = time.time()

					self.resolving[(self.worker.w_id, identifier)] = client_id, hostname, hostname, command, decision
					self.clients[client_id] = (self.worker.w_id, identifier, active_time)
					self.active.append((active_time, client_id, self.worker.socket))
		else:
			identifier = None
			response = None

		return identifier, response

	def beginResolvingTCP (self, client_id, command, decision):
		if self.worker_count < self.max_workers:
			identifier = self.newTCPResolver(client_id, command, decision)
			self.worker_count += 1
		else:
			self.waiting.append((client_id, command, decision))
			identifier = None

		return identifier

	def notifyClose (self):
		paused = self.worker_count >= self.max_workers
		self.worker_count -= 1

		if paused and self.worker_count < self.max_workers:
			for _ in range(self.worker_count, self.max_workers):
				if self.waiting:
					data, self.waiting = self.waiting[0], self.waiting[1:]
					client_id, command, decision = data

					identifier = self.newTCPResolver(client_id, command, decision)
					self.worker_count += 1

	def newTCPResolver (self, client_id, command, decision):
		hostname = self.extractHostname(command, decision)

		if hostname:
			worker = self.resolver_factory.createTCPClient()
			self.workers[worker.socket] = worker

			identifier, all_sent = worker.resolveHost(hostname)
			active_time = time.time()
			self.resolving[(worker.w_id, identifier)] = client_id, hostname, hostname, command, decision
			self.clients[client_id] = (worker.w_id, identifier, active_time)
			self.active.append((active_time, client_id, self.worker.socket))

			if all_sent:
				self.poller.addReadSocket('read_resolver', worker.socket)
				self.resolving[(worker.w_id, identifier)] = client_id, hostname, hostname, command, decision
			else:
				self.poller.addWriteSocket('write_resolver', worker.socket)
				self.sending[worker.socket] = client_id, hostname, hostname, command, decision

		else:
			identifier = None

		return identifier

	def getResponse(self, sock):
		worker = self.workers.get(sock)

		if worker:
			result = worker.getResponse()

			if result:
				identifier, forhost, ip, completed, newidentifier, newhost, newcomplete = result
				data = self.resolving.pop((worker.w_id, identifier), None)

			else:
				# unable to parse response
				data = None

			if data:
				client_id, original, hostname, command, decision = data
				clidata = self.clients.pop(client_id, None)

				if completed:
					if clidata is not None:
						key = clidata[2], client_id, worker.socket
						if key in self.active:
							self.active.remove(key)

				# check to see if we received an incomplete response
				if not completed:
					newidentifier = self.beginResolvingTCP(client_id, command, decision)
					newhost = hostname
					response = None

				# check to see if the worker started a new request
				if newidentifier:
					if completed:
						active_time = time.time()
						self.resolving[(worker.w_id, newidentifier)] = client_id, original, newhost, command, decision
						self.clients[client_id] = (worker.w_id, newidentifier, active_time)
						self.active.append((active_time, client_id, worker.socket))

					response = None

					if completed and newcomplete:
						self.poller.addReadSocket('read_resolver', worker.socket)
					elif completed and not newcomplete:
						self.poller.addWriteSocket('write_resolver', worker.socket)
						self.sending[worker.socket] = client_id, original, hostname, command, decision

				# we just started a new (TCP) request and have not yet completely sent it
				elif not completed:
					response = None

				# maybe we read the wrong response?
				elif forhost != hostname:
					active_time = time.time()
					self.resolving[(worker.w_id, identifier)] = client_id, original, hostname, command, decision
					self.clients[client_id] = (worker.w_id, identifier, active_time)
					self.active.append((active_time, client_id, worker.socket))
					response = None

				# success
				elif ip is not None:
					resolved = self.resolveDecision(command, decision, ip)
					response = client_id, command, resolved
					self.cacheDestination(original, ip)

				# not found
				else:
					newdecision = '\0'.join(('503', 'dns.html', 'http', '', '', hostname, 'peer'))
					response = client_id, 'rewrite', newdecision
					#self.cacheDestination(original, ip)
			else:
				response = None

			if response or result is None:
				if worker is not self.worker:
					self.poller.removeReadSocket('read_resolver', sock)
					self.poller.removeWriteSocket('write_resolver', sock)
					worker.close()
					self.workers.pop(sock)
					self.notifyClose()

		else:
			response = None

		return response

	def continueSending(self, sock):
		"""Continue sending data over the connected TCP socket"""
		data = self.sending.get(sock)
		if data:
			client_id, original, hostname, command, decision = data
			worker = self.workers[sock]
			w_id, identifier, active_time = self.clients[client_id]

			res = worker.continueSending()

			if res is False: # we've sent all we need to send
				tmp = self.sending.pop(sock)
				self.resolving[(w_id, identifier)] = tmp

				self.poller.removeWriteSocket('write_resolver', sock)
				self.poller.addReadSocket('read_resolver', sock)
<commit_msg>Remove line that made it into the last revision<commit_after>import time

from .worker import DNSResolver
from exaproxy.network.functions import isip

class ResolverManager (object):
	resolverFactory = DNSResolver

	def __init__ (self, poller, configuration, max_workers):
		self.poller = poller
		self.configuration = configuration

		self.resolver_factory = self.resolverFactory(configuration)

		# The actual work is done in the worker
		self.worker = self.resolver_factory.createUDPClient()

		# All currently active clients (one UDP and many TCP)
		self.workers = {}
		self.workers[self.worker.socket] = self.worker
		self.poller.addReadSocket('read_resolver', self.worker.socket)

		# Track the clients currently expecting results
		self.clients = {}         #   client_id : identifier

		# Key should be the hostname rather than the request ID?
		self.resolving = {}       #   identifier, worker_id :

		# TCP workers that have not yet sent a complete request
		self.sending = {}         #   sock :

		# track the current queries and when they were started
		self.active = []

		self.cache = {}
		self.cached = []

		self.max_workers = max_workers
		self.worker_count = len(self.workers) # the UDP client

		self.waiting = []

	def cacheDestination (self, hostname, ip):
		if hostname not in self.cache:
			expire_time = time.time() + self.configuration.dns.ttl
			expire_time = expire_time - expire_time % self.configuration.dns.resolution
			latest_time, latest_hosts = self.cached[-1] if self.cached else (-1, None)

			if expire_time > latest_time:
				hosts = []
				self.cached.append((expire_time, hosts))
			else:
				hosts = latest_hosts

			self.cache[hostname] = ip
			hosts.append(hostname)

	def expireCache (self):
		# expire only one set of cache entries at a time
		if self.cached:
			current_time = time.time()
			expire_time, hosts = self.cached[0]

			if current_time >= expire_time:
				(expire_time, hosts), self.cached = self.cached[0], self.cached[1:]

				for hostname in hosts:
					self.cache.pop(hostname, None)

	def cleanup(self):
		cutoff = time.time() - self.configuration.dns.timeout
		count = 0

		for timestamp, client_id, sock in self.active:
			if timestamp > cutoff:
				break

			count += 1
			cli_data = self.clients.pop(client_id, None)
			worker = self.workers.get(sock)

			if cli_data is not None:
				w_id, identifier, active_time  = cli_data
				data = self.resolving.pop((w_id, identifier), None)
				if not data:
					data = self.sending.pop(sock, None)

				if data:
					client_id, original, hostname, command, decision = data
					yield client_id, 'rewrite', '\0'.join(('503', 'dns.html', '', '', '', hostname, 'peer'))

			if worker is not None:
				if worker is not self.worker:
					worker.close()
					self.workers.pop(sock)

		if count:
			self.active = self.active[count:]

	def resolves(self, command, decision):
		if command in ('download', 'connect'):
			hostname = decision.split('\0')[0]
			if isip(hostname):
				res = False
			else:
				res = True
		else:
			res = False

		return res

	def extractHostname(self, command, decision):
		data = decision.split('\0')

		if command == 'download':
			hostname = data[0]

		elif command == 'connect':
			hostname = decision.split('\0')[0]

		else:
			hostname = None

		return hostname

	def resolveDecision(self, command, decision, ip):
		if command in ('download', 'connect'):
			hostname, args = decision.split('\0', 1)
			newdecision = '\0'.join((ip, args))
		else:
			newdecision = None

		return newdecision

	def startResolving(self, client_id, command, decision):
		hostname = self.extractHostname(command, decision)

		if hostname:
			if hostname in self.cache:
				identifier = None
				ip = self.cache[hostname]

				if ip is not None:
					resolved = self.resolveDecision(command, decision, ip)
					response = client_id, command, resolved

				else:
					newdecision = '\0'.join(('503', 'dns.html', 'http', '', '', hostname, 'peer'))
					response = client_id, 'rewrite', newdecision

			else:
				jumbo = max(len(p) for p in hostname.split('.')) > 255
				if jumbo:
					identifier = None
					newdecision = '\0'.join(('503', 'dns.html', 'http', '', '', hostname, 'peer'))
					response = client_id, 'rewrite', newdecision

				else:
					identifier, _ = self.worker.resolveHost(hostname)
					response = None
					active_time = time.time()

					self.resolving[(self.worker.w_id, identifier)] = client_id, hostname, hostname, command, decision
					self.clients[client_id] = (self.worker.w_id, identifier, active_time)
					self.active.append((active_time, client_id, self.worker.socket))
		else:
			identifier = None
			response = None

		return identifier, response

	def beginResolvingTCP (self, client_id, command, decision):
		if self.worker_count < self.max_workers:
			identifier = self.newTCPResolver(client_id, command, decision)
			self.worker_count += 1
		else:
			self.waiting.append((client_id, command, decision))
			identifier = None

		return identifier

	def notifyClose (self):
		paused = self.worker_count >= self.max_workers
		self.worker_count -= 1

		if paused and self.worker_count < self.max_workers:
			for _ in range(self.worker_count, self.max_workers):
				if self.waiting:
					data, self.waiting = self.waiting[0], self.waiting[1:]
					client_id, command, decision = data

					identifier = self.newTCPResolver(client_id, command, decision)
					self.worker_count += 1

	def newTCPResolver (self, client_id, command, decision):
		hostname = self.extractHostname(command, decision)

		if hostname:
			worker = self.resolver_factory.createTCPClient()
			self.workers[worker.socket] = worker

			identifier, all_sent = worker.resolveHost(hostname)
			active_time = time.time()
			self.resolving[(worker.w_id, identifier)] = client_id, hostname, hostname, command, decision
			self.clients[client_id] = (worker.w_id, identifier, active_time)
			self.active.append((active_time, client_id, self.worker.socket))

			if all_sent:
				self.poller.addReadSocket('read_resolver', worker.socket)
				self.resolving[(worker.w_id, identifier)] = client_id, hostname, hostname, command, decision
			else:
				self.poller.addWriteSocket('write_resolver', worker.socket)
				self.sending[worker.socket] = client_id, hostname, hostname, command, decision

		else:
			identifier = None

		return identifier

	def getResponse(self, sock):
		worker = self.workers.get(sock)

		if worker:
			result = worker.getResponse()

			if result:
				identifier, forhost, ip, completed, newidentifier, newhost, newcomplete = result
				data = self.resolving.pop((worker.w_id, identifier), None)

			else:
				# unable to parse response
				data = None

			if data:
				client_id, original, hostname, command, decision = data
				clidata = self.clients.pop(client_id, None)

				if completed:
					if clidata is not None:
						key = clidata[2], client_id, worker.socket
						if key in self.active:
							self.active.remove(key)

				# check to see if we received an incomplete response
				if not completed:
					newidentifier = self.beginResolvingTCP(client_id, command, decision)
					newhost = hostname
					response = None

				# check to see if the worker started a new request
				if newidentifier:
					if completed:
						active_time = time.time()
						self.resolving[(worker.w_id, newidentifier)] = client_id, original, newhost, command, decision
						self.clients[client_id] = (worker.w_id, newidentifier, active_time)
						self.active.append((active_time, client_id, worker.socket))

					response = None

					if completed and newcomplete:
						self.poller.addReadSocket('read_resolver', worker.socket)
					elif completed and not newcomplete:
						self.poller.addWriteSocket('write_resolver', worker.socket)
						self.sending[worker.socket] = client_id, original, hostname, command, decision

				# we just started a new (TCP) request and have not yet completely sent it
				elif not completed:
					response = None

				# maybe we read the wrong response?
				elif forhost != hostname:
					active_time = time.time()
					self.resolving[(worker.w_id, identifier)] = client_id, original, hostname, command, decision
					self.clients[client_id] = (worker.w_id, identifier, active_time)
					self.active.append((active_time, client_id, worker.socket))
					response = None

				# success
				elif ip is not None:
					resolved = self.resolveDecision(command, decision, ip)
					response = client_id, command, resolved
					self.cacheDestination(original, ip)

				# not found
				else:
					newdecision = '\0'.join(('503', 'dns.html', 'http', '', '', hostname, 'peer'))
					response = client_id, 'rewrite', newdecision
					#self.cacheDestination(original, ip)
			else:
				response = None

			if response or result is None:
				if worker is not self.worker:
					self.poller.removeReadSocket('read_resolver', sock)
					self.poller.removeWriteSocket('write_resolver', sock)
					worker.close()
					self.workers.pop(sock)
					self.notifyClose()

		else:
			response = None

		return response

	def continueSending(self, sock):
		"""Continue sending data over the connected TCP socket"""
		data = self.sending.get(sock)
		if data:
			client_id, original, hostname, command, decision = data
			worker = self.workers[sock]
			w_id, identifier, active_time = self.clients[client_id]

			res = worker.continueSending()

			if res is False: # we've sent all we need to send
				tmp = self.sending.pop(sock)
				self.resolving[(w_id, identifier)] = tmp

				self.poller.removeWriteSocket('write_resolver', sock)
				self.poller.addReadSocket('read_resolver', sock)
<|endoftext|>


<commit_before>import time

from .worker import DNSResolver
from exaproxy.network.functions import isip

class ResolverManager (object):
	resolverFactory = DNSResolver

	def __init__ (self, poller, configuration, max_workers):
		self.poller = poller
		self.configuration = configuration

		self.resolver_factory = self.resolverFactory(configuration)

		# The actual work is done in the worker
		self.worker = self.resolver_factory.createUDPClient()

		# All currently active clients (one UDP and many TCP)
		self.workers = {}
		self.workers[self.worker.socket] = self.worker
		self.poller.addReadSocket('read_resolver', self.worker.socket)

		# Track the clients currently expecting results
		self.clients = {}  # client_id : identifier

		# Key should be the hostname rather than the request ID?
		self.resolving = {}  # identifier, worker_id :

		# TCP workers that have not yet sent a complete request
		self.sending = {}  # sock :

		# track the current queries and when they were started
		self.active = []

		self.cache = {}
		self.cached = []

		self.max_workers = max_workers
		self.worker_count = len(self.workers)  # the UDP client

		self.waiting = []

	def cacheDestination (self, hostname, ip):
		if hostname not in self.cache:
			expire_time = time.time() + self.configuration.dns.ttl
			expire_time = expire_time - expire_time % self.configuration.dns.resolution
			latest_time, latest_hosts = self.cached[-1] if self.cached else (-1, None)

			if expire_time > latest_time:
				hosts = []
				self.cached.append((expire_time, hosts))
			else:
				hosts = latest_hosts

			self.cache[hostname] = ip
			hosts.append(hostname)

	def expireCache (self):
		# expire only one set of cache entries at a time
		if self.cached:
			current_time = time.time()
			expire_time, hosts = self.cached[0]

			if current_time >= expire_time:
				(expire_time, hosts), self.cached = self.cached[0], self.cached[1:]

				for hostname in hosts:
					self.cache.pop(hostname, None)

	def cleanup(self):
		cutoff = time.time() - self.configuration.dns.timeout
		count = 0

		for timestamp, client_id, sock in self.active:
			if timestamp > cutoff:
				break

			count += 1
			cli_data = self.clients.pop(client_id, None)
			worker = self.workers.get(sock)

			if cli_data is not None:
				w_id, identifier, active_time  = cli_data
				data = self.resolving.pop((w_id, identifier), None)
				if not data:
					data = self.sending.pop(sock, None)

				if data:
					client_id, original, hostname, command, decision = data
					yield client_id, 'rewrite', '\0'.join(('503', 'dns.html', '', '', '', hostname, 'peer'))

			if worker is not None:
				if worker is not self.worker:
					worker.close()
					self.workers.pop(sock)

		if count:
			self.active = self.active[count:]

	def resolves(self, command, decision):
		if command in ('download', 'connect'):
			hostname = decision.split('\0')[0]
			if isip(hostname):
				res = False
			else:
				res = True
		else:
			res = False

		return res

	def extractHostname(self, command, decision):
		data = decision.split('\0')

		if command == 'download':
			hostname = data[0]

		elif command == 'connect':
			hostname = decision.split('\0')[0]

		else:
			hostname = None

		return hostname

	def resolveDecision(self, command, decision, ip):
		if command in ('download', 'connect'):
			hostname, args = decision.split('\0', 1)
			newdecision = '\0'.join((ip, args))
		else:
			newdecision = None

		return newdecision

	def startResolving(self, client_id, command, decision):
		hostname = self.extractHostname(command, decision)

		if hostname:
			if hostname in self.cache:
				identifier = None
				ip = self.cache[hostname]

				if ip is not None:
					resolved = self.resolveDecision(command, decision, ip)
					response = client_id, command, resolved

				else:
					newdecision = '\0'.join(('503', 'dns.html', 'http', '', '', hostname, 'peer'))
					response = client_id, 'rewrite', newdecision

			elif self.configuration.dns.fqdn and '.' not in hostname:
				identifier = 'not-found'
				response = None

			else:
				jumbo = max(len(p) for p in hostname.split('.')) > 255
				if jumbo:
					identifier = None
					newdecision = '\0'.join(('503', 'dns.html', 'http', '', '', hostname, 'peer'))
					response = client_id, 'rewrite', newdecision

				else:
					identifier, _ = self.worker.resolveHost(hostname)
					response = None
					active_time = time.time()

					self.resolving[(self.worker.w_id, identifier)] = client_id, hostname, hostname, command, decision
					self.clients[client_id] = (self.worker.w_id, identifier, active_time)
					self.active.append((active_time, client_id, self.worker.socket))
		else:
			identifier = None
			response = None

		return identifier, response

	def beginResolvingTCP (self, client_id, command, decision):
		if self.worker_count < self.max_workers:
			identifier = self.newTCPResolver(client_id, command, decision)
			self.worker_count += 1
		else:
			self.waiting.append((client_id, command, decision))
			identifier = None

		return identifier

	def notifyClose (self):
		paused = self.worker_count >= self.max_workers
		self.worker_count -= 1

		if paused and self.worker_count < self.max_workers:
			for _ in range(self.worker_count, self.max_workers):
				if self.waiting:
					data, self.waiting = self.waiting[0], self.waiting[1:]
					client_id, command, decision = data

					identifier = self.newTCPResolver(client_id, command, decision)
					self.worker_count += 1

	def newTCPResolver (self, client_id, command, decision):
		hostname = self.extractHostname(command, decision)

		if hostname:
			worker = self.resolver_factory.createTCPClient()
			self.workers[worker.socket] = worker

			identifier, all_sent = worker.resolveHost(hostname)
			active_time = time.time()
			self.resolving[(worker.w_id, identifier)] = client_id, hostname, hostname, command, decision
			self.clients[client_id] = (worker.w_id, identifier, active_time)
			self.active.append((active_time, client_id, self.worker.socket))

			if all_sent:
				self.poller.addReadSocket('read_resolver', worker.socket)
				self.resolving[(worker.w_id, identifier)] = client_id, hostname, hostname, command, decision
			else:
				self.poller.addWriteSocket('write_resolver', worker.socket)
				self.sending[worker.socket] = client_id, hostname, hostname, command, decision

		else:
			identifier = None

		return identifier

	def getResponse(self, sock):
		worker = self.workers.get(sock)

		if worker:
			result = worker.getResponse()

			if result:
				identifier, forhost, ip, completed, newidentifier, newhost, newcomplete = result
				data = self.resolving.pop((worker.w_id, identifier), None)

			else:
				# unable to parse response
				data = None

			if data:
				client_id, original, hostname, command, decision = data
				clidata = self.clients.pop(client_id, None)

				if completed:
					if clidata is not None:
						key = clidata[2], client_id, worker.socket
						if key in self.active:
							self.active.remove(key)

				# check to see if we received an incomplete response
				if not completed:
					newidentifier = self.beginResolvingTCP(client_id, command, decision)
					newhost = hostname
					response = None

				# check to see if the worker started a new request
				if newidentifier:
					if completed:
						active_time = time.time()
						self.resolving[(worker.w_id, newidentifier)] = client_id, original, newhost, command, decision
						self.clients[client_id] = (worker.w_id, newidentifier, active_time)
						self.active.append((active_time, client_id, worker.socket))

					response = None

					if completed and newcomplete:
						self.poller.addReadSocket('read_resolver', worker.socket)
					elif completed and not newcomplete:
						self.poller.addWriteSocket('write_resolver', worker.socket)
						self.sending[worker.socket] = client_id, original, hostname, command, decision

				# we just started a new (TCP) request and have not yet completely sent it
				elif not completed:
					response = None

				# maybe we read the wrong response?
				elif forhost != hostname:
					active_time = time.time()
					self.resolving[(worker.w_id, identifier)] = client_id, original, hostname, command, decision
					self.clients[client_id] = (worker.w_id, identifier, active_time)
					self.active.append((active_time, client_id, worker.socket))
					response = None

				# success
				elif ip is not None:
					resolved = self.resolveDecision(command, decision, ip)
					response = client_id, command, resolved
					self.cacheDestination(original, ip)

				# not found
				else:
					newdecision = '\0'.join(('503', 'dns.html', 'http', '', '', hostname, 'peer'))
					response = client_id, 'rewrite', newdecision
					#self.cacheDestination(original, ip)
			else:
				response = None

			if response or result is None:
				if worker is not self.worker:
					self.poller.removeReadSocket('read_resolver', sock)
					self.poller.removeWriteSocket('write_resolver', sock)
					worker.close()
					self.workers.pop(sock)
					self.notifyClose()

		else:
			response = None

		return response

	def continueSending(self, sock):
		"""Continue sending data over the connected TCP socket"""
		data = self.sending.get(sock)
		if data:
			client_id, original, hostname, command, decision = data
			worker = self.workers[sock]
			w_id, identifier, active_time = self.clients[client_id]

			res = worker.continueSending()

			if res is False: # we've sent all we need to send
				tmp = self.sending.pop(sock)
				self.resolving[(w_id, identifier)] = tmp

				self.poller.removeWriteSocket('write_resolver', sock)
				self.poller.addReadSocket('read_resolver', sock)
<commit_msg>tidy last patch<commit_after>import time

from .worker import DNSResolver
from exaproxy.network.functions import isip

class ResolverManager (object):
	resolverFactory = DNSResolver

	def __init__ (self, poller, configuration, max_workers):
		self.poller = poller
		self.configuration = configuration

		self.resolver_factory = self.resolverFactory(configuration)

		# The actual work is done in the worker
		self.worker = self.resolver_factory.createUDPClient()

		# All currently active clients (one UDP and many TCP)
		self.workers = {}
		self.workers[self.worker.socket] = self.worker
		self.poller.addReadSocket('read_resolver', self.worker.socket)

		# Track the clients currently expecting results
		self.clients = {}  # client_id : identifier

		# Key should be the hostname rather than the request ID?
		self.resolving = {}  # identifier, worker_id :

		# TCP workers that have not yet sent a complete request
		self.sending = {}  # sock :

		# track the current queries and when they were started
		self.active = []

		self.cache = {}
		self.cached = []

		self.max_workers = max_workers
		self.worker_count = len(self.workers)  # the UDP client

		self.waiting = []

	def cacheDestination (self, hostname, ip):
		if hostname not in self.cache:
			expire_time = time.time() + self.configuration.dns.ttl
			expire_time = expire_time - expire_time % self.configuration.dns.resolution
			latest_time, latest_hosts = self.cached[-1] if self.cached else (-1, None)

			if expire_time > latest_time:
				hosts = []
				self.cached.append((expire_time, hosts))
			else:
				hosts = latest_hosts

			self.cache[hostname] = ip
			hosts.append(hostname)

	def expireCache (self):
		# expire only one set of cache entries at a time
		if self.cached:
			current_time = time.time()
			expire_time, hosts = self.cached[0]

			if current_time >= expire_time:
				(expire_time, hosts), self.cached = self.cached[0], self.cached[1:]

				for hostname in hosts:
					self.cache.pop(hostname, None)

	def cleanup(self):
		cutoff = time.time() - self.configuration.dns.timeout
		count = 0

		for timestamp, client_id, sock in self.active:
			if timestamp > cutoff:
				break

			count += 1
			cli_data = self.clients.pop(client_id, None)
			worker = self.workers.get(sock)

			if cli_data is not None:
				w_id, identifier, active_time  = cli_data
				data = self.resolving.pop((w_id, identifier), None)
				if not data:
					data = self.sending.pop(sock, None)

				if data:
					client_id, original, hostname, command, decision = data
					yield client_id, 'rewrite', '\0'.join(('503', 'dns.html', '', '', '', hostname, 'peer'))

			if worker is not None:
				if worker is not self.worker:
					worker.close()
					self.workers.pop(sock)

		if count:
			self.active = self.active[count:]

	def resolves(self, command, decision):
		if command in ('download', 'connect'):
			hostname = decision.split('\0')[0]
			if isip(hostname):
				res = False
			else:
				res = True
		else:
			res = False

		return res

	def extractHostname(self, command, decision):
		data = decision.split('\0')

		if command == 'download':
			hostname = data[0]

		elif command == 'connect':
			hostname = decision.split('\0')[0]

		else:
			hostname = None

		return hostname

	def resolveDecision(self, command, decision, ip):
		if command in ('download', 'connect'):
			hostname, args = decision.split('\0', 1)
			newdecision = '\0'.join((ip, args))
		else:
			newdecision = None

		return newdecision

	def startResolving(self, client_id, command, decision):
		hostname = self.extractHostname(command, decision)

		if hostname:
			# Resolution is already in our cache
			if hostname in self.cache:
				identifier = None
				ip = self.cache[hostname]

				if ip is not None:
					resolved = self.resolveDecision(command, decision, ip)
					response = client_id, command, resolved

				else:
					newdecision = '\0'.join(('503', 'dns.html', 'http', '', '', hostname, 'peer'))
					response = client_id, 'rewrite', newdecision
			# do not try to resolve domains which are not FQDN
			elif self.configuration.dns.fqdn and '.' not in hostname:
				identifier = 'not-found'
				response = None
			# each DNS part (between the dots) must be under 256 chars
			elif max(len(p) for p in hostname.split('.')) > 255:
				identifier = None
				newdecision = '\0'.join(('503', 'dns.html', 'http', '', '', hostname, 'peer'))
				response = client_id, 'rewrite', newdecision
			# Lookup that DNS name
			else:
				identifier, _ = self.worker.resolveHost(hostname)
				response = None
				active_time = time.time()

				self.resolving[(self.worker.w_id, identifier)] = client_id, hostname, hostname, command, decision
				self.clients[client_id] = (self.worker.w_id, identifier, active_time)
				self.active.append((active_time, client_id, self.worker.socket))
		else:
			identifier = None
			response = None

		return identifier, response

	def beginResolvingTCP (self, client_id, command, decision):
		if self.worker_count < self.max_workers:
			identifier = self.newTCPResolver(client_id, command, decision)
			self.worker_count += 1
		else:
			self.waiting.append((client_id, command, decision))
			identifier = None

		return identifier

	def notifyClose (self):
		paused = self.worker_count >= self.max_workers
		self.worker_count -= 1

		if paused and self.worker_count < self.max_workers:
			for _ in range(self.worker_count, self.max_workers):
				if self.waiting:
					data, self.waiting = self.waiting[0], self.waiting[1:]
					client_id, command, decision = data

					identifier = self.newTCPResolver(client_id, command, decision)
					self.worker_count += 1

	def newTCPResolver (self, client_id, command, decision):
		hostname = self.extractHostname(command, decision)

		if hostname:
			worker = self.resolver_factory.createTCPClient()
			self.workers[worker.socket] = worker

			identifier, all_sent = worker.resolveHost(hostname)
			active_time = time.time()
			self.resolving[(worker.w_id, identifier)] = client_id, hostname, hostname, command, decision
			self.clients[client_id] = (worker.w_id, identifier, active_time)
			self.active.append((active_time, client_id, self.worker.socket))

			if all_sent:
				self.poller.addReadSocket('read_resolver', worker.socket)
				self.resolving[(worker.w_id, identifier)] = client_id, hostname, hostname, command, decision
			else:
				self.poller.addWriteSocket('write_resolver', worker.socket)
				self.sending[worker.socket] = client_id, hostname, hostname, command, decision

		else:
			identifier = None

		return identifier

	def getResponse(self, sock):
		worker = self.workers.get(sock)

		if worker:
			result = worker.getResponse()

			if result:
				identifier, forhost, ip, completed, newidentifier, newhost, newcomplete = result
				data = self.resolving.pop((worker.w_id, identifier), None)

			else:
				# unable to parse response
				data = None

			if data:
				client_id, original, hostname, command, decision = data
				clidata = self.clients.pop(client_id, None)

				if completed:
					if clidata is not None:
						key = clidata[2], client_id, worker.socket
						if key in self.active:
							self.active.remove(key)

				# check to see if we received an incomplete response
				if not completed:
					newidentifier = self.beginResolvingTCP(client_id, command, decision)
					newhost = hostname
					response = None

				# check to see if the worker started a new request
				if newidentifier:
					if completed:
						active_time = time.time()
						self.resolving[(worker.w_id, newidentifier)] = client_id, original, newhost, command, decision
						self.clients[client_id] = (worker.w_id, newidentifier, active_time)
						self.active.append((active_time, client_id, worker.socket))

					response = None

					if completed and newcomplete:
						self.poller.addReadSocket('read_resolver', worker.socket)
					elif completed and not newcomplete:
						self.poller.addWriteSocket('write_resolver', worker.socket)
						self.sending[worker.socket] = client_id, original, hostname, command, decision

				# we just started a new (TCP) request and have not yet completely sent it
				elif not completed:
					response = None

				# maybe we read the wrong response?
				elif forhost != hostname:
					active_time = time.time()
					self.resolving[(worker.w_id, identifier)] = client_id, original, hostname, command, decision
					self.clients[client_id] = (worker.w_id, identifier, active_time)
					self.active.append((active_time, client_id, worker.socket))
					response = None

				# success
				elif ip is not None:
					resolved = self.resolveDecision(command, decision, ip)
					response = client_id, command, resolved
					self.cacheDestination(original, ip)

				# not found
				else:
					newdecision = '\0'.join(('503', 'dns.html', 'http', '', '', hostname, 'peer'))
					response = client_id, 'rewrite', newdecision
					#self.cacheDestination(original, ip)
			else:
				response = None

			if response or result is None:
				if worker is not self.worker:
					self.poller.removeReadSocket('read_resolver', sock)
					self.poller.removeWriteSocket('write_resolver', sock)
					worker.close()
					self.workers.pop(sock)
					self.notifyClose()

		else:
			response = None

		return response

	def continueSending(self, sock):
		"""Continue sending data over the connected TCP socket"""
		data = self.sending.get(sock)
		if data:
			client_id, original, hostname, command, decision = data
			worker = self.workers[sock]
			w_id, identifier, active_time = self.clients[client_id]

			res = worker.continueSending()

			if res is False: # we've sent all we need to send
				tmp = self.sending.pop(sock)
				self.resolving[(w_id, identifier)] = tmp

				self.poller.removeWriteSocket('write_resolver', sock)
				self.poller.addReadSocket('read_resolver', sock)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

""" 檔案系統監視模組 """

import os
import pyinotify

from filewatcher import componentprop
from filewatcher import filewatchconfig
from filewatcher import watcher

def _get_relpath(path, start):
	relpath = os.path.relpath(path, start)
	if '.' == relpath:
		return ''
	else:
		return relpath
# ### def _get_relpath

_cached_module_prop_instance = componentprop.MonitorProp('inotify')
def get_module_prop():
	""" 取得監視器各項特性/屬性

	參數:
		(無)
	回傳值:
		傳回 componentprop.OperatorProp 物件
	"""

	return _cached_module_prop_instance
# ### def get_module_prop

_ignorance_checker = None
def set_ignorance_checker(checker):
	""" 設定忽略路徑與檔案檢查器

	參數:
		checker - 進行路徑與檔案名稱檢查的函式，函數原型: (relpath=None, filename=None) 回傳 True 表要忽略所檢查的項目
	"""
	global _ignorance_checker

	if isinstance(checker, str):
		_ignorance_checker = filewatchconfig.lookup_ignorance_checker(checker)
	else:
		_ignorance_checker = checker
# ### def set_ignorance_checker

_revise_period = None
def set_revise_period(revise_period=None):
	""" 設定重新檢驗週期
	在給定的週期時間後會重新執行 ignorance-checker 檢查所有註冊的 watcher 是不是要取消

	參數:
		revise_period=None - 要設定的週期時間，單位是秒
	"""
	global _revise_period

	if _ignorance_checker is None:
		return

	if revise_period is not None:
		try:
			_revise_period = int(revise_period)
			if _revise_period < 200:
				_revise_period = 200
		except:
			_revise_period = None
		return

	_revise_period = None
# ### def set_revise_period

def _periodical_folder_reviser(arg):
	if _ignorance_checker is None:
		return

	target_directory = arg

	for root, dirs, files in os.walk(target_directory):
		#print "linux_inotify: revising %r" % (root)
		no_further_scan = []
		wd_to_drop = []

		for d in dirs:
			dirpath = os.path.abspath(os.path.join(root, d))
			wd = _watchmanager.get_wd(dirpath)
			if wd is None:
				no_further_scan.append(d)
				#print "linux_inotify: no further revise %r" % (d,)
			elif _ignorance_checker(dirpath, None):
				wd_to_drop.append(wd)
				no_further_scan.append(d)
				print "linux_inotify: removing %r (wd=%r)" % (d, wd,)

		for d in no_further_scan:
			dirs.remove(d)
		if len(wd_to_drop) > 0:
			_watchmanager.rm_watch(wd_to_drop, rec=True)
# ### def _periodical_folder_reviser

class _ExcludeFilter:
	def __init__(self, target_directory, recursive_watch=False):
		self.target_directory = target_directory
		self.recursive_watch = recursive_watch
	# ### def __init__

	def do_exclude_filting(self, filepath):
		filepath = os.path.abspath(filepath)
		if os.path.isdir(filepath):
			path = filepath
			name = None
		else:
			path, name, = os.path.split(filepath)

		if (False == self.recursive_watch) and (path != self.target_directory):
			return True

		if _ignorance_checker is not None:
			relpath = _get_relpath(path, self.target_directory)
			r = _ignorance_checker(relpath, name)
			if r:
				return True

		return False
	# ### def do_exclude_filting

	def __call__(self, path):
		return self.do_exclude_filting(path)
	# ### def __call__
# ### class _ExcludeFilter

def monitor_configure(config, metastorage):
	""" 設定監視器組態

	參數:
		config - 帶有參數的字典
		metastorage - 中介資訊資料庫物件
	回傳值:
		(無)
	"""

	# 載入 ignorance checker
	if 'ignorance-checker' in config:
		set_ignorance_checker(str(config['ignorance-checker']))

	set_revise_period(config.get('revise-interval'))
# ### def monitor_configure

class _EventHandler(pyinotify.ProcessEvent):
	def __init__(self, watcher_instance, target_directory):
		pyinotify.ProcessEvent.__init__(self)

		self.watcher_instance = watcher_instance
		self.target_directory = target_directory
	# ### def __init__

	def trigger_operation(self, pathname, watcher_eventcode):
		path, name, = os.path.split(os.path.abspath(pathname))
		relpath = _get_relpath(path, self.target_directory)
		self.watcher_instance.discover_file_change(name, relpath, watcher_eventcode)
	# ### def perform_modified_operation

	def process_IN_CLOSE_WRITE(self, event):
		print "inotify::IN_CLOSE_WRITE: %r" % (event.pathname,)
		self.trigger_operation(event.pathname, watcher.FEVENT_MODIFIED)
	# ### def process_IN_CLOSE_WRITE

	def process_IN_MOVED_TO(self, event):
		print "inotify::IN_MOVED_TO: %r" % (event.pathname,)
		self.trigger_operation(event.pathname, watcher.FEVENT_MODIFIED)
	# ### def process.IN_MOVED_TO

	def process_IN_DELETE(self, event):
		print "inotify::IN_DELETE: %r" % (event.pathname,)
		self.trigger_operation(event.pathname, watcher.FEVENT_DELETED)
	# ### def process_IN_DELETE
# ### class EventHandler

_watchmanager = None
def monitor_start(watcher_instance, target_directory, recursive_watch=False):
	""" 開始監控目錄作業

	參數:
		watcher_instance - watcher.WatcherEngine 物件實體
		target_directory - 監測目標資料夾
		recursive_watch - 是否要遞迴監測子資料夾
	回傳值:
		(無)
	"""

	global _watchmanager

	mask = pyinotify.IN_DELETE | pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO	# watched events

	if _watchmanager is not None:
		print "ERR: linux_inotify: designed to watch one directory only, unspecified behavior with multiple monitor_start."

	_watchmanager = pyinotify.WatchManager(exclude_filter=_ExcludeFilter(target_directory, recursive_watch))
	handler = _EventHandler(watcher_instance, target_directory)
	notifier = pyinotify.AsyncNotifier(_watchmanager, handler, channel_map=watcher_instance.process_driver.async_map)

	auto_add_folder = False if (not recursive_watch) else True
	wdd = _watchmanager.add_watch(target_directory, mask, rec=True, auto_add=auto_add_folder)

	if _revise_period is not None:
		watcher_instance.process_driver.append_periodical_call(_periodical_folder_reviser, None, _revise_period)
# ### def monitor_start

def monitor_stop():
	""" 停止作業，準備結束

	參數:
		(無)
	回傳值:
		(無)
	"""

	pass
# ### def monitor_stop

# vim: ts=4 sw=4 ai nowrap
<commit_msg>add callback register function for queue overflow event<commit_after># -*- coding: utf-8 -*-

""" 檔案系統監視模組 """

import os
import pyinotify

from filewatcher import componentprop
from filewatcher import filewatchconfig
from filewatcher import watcher

def _get_relpath(path, start):
	relpath = os.path.relpath(path, start)
	if '.' == relpath:
		return ''
	else:
		return relpath
# ### def _get_relpath

_cached_module_prop_instance = componentprop.MonitorProp('inotify')
def get_module_prop():
	""" 取得監視器各項特性/屬性

	參數:
		(無)
	回傳值:
		傳回 componentprop.OperatorProp 物件
	"""

	return _cached_module_prop_instance
# ### def get_module_prop

_queue_overflow_event_callback = None
def set_queue_overflow_callback(callback_func):
	global _queue_overflow_event_callback
	_queue_overflow_event_callback = callback_func
# ### def set_queue_overflow_callback

_ignorance_checker = None
def set_ignorance_checker(checker):
	""" 設定忽略路徑與檔案檢查器

	參數:
		checker - 進行路徑與檔案名稱檢查的函式，函數原型: (relpath=None, filename=None) 回傳 True 表要忽略所檢查的項目
	"""
	global _ignorance_checker

	if isinstance(checker, str):
		_ignorance_checker = filewatchconfig.lookup_ignorance_checker(checker)
	else:
		_ignorance_checker = checker
# ### def set_ignorance_checker

_revise_period = None
def set_revise_period(revise_period=None):
	""" 設定重新檢驗週期
	在給定的週期時間後會重新執行 ignorance-checker 檢查所有註冊的 watcher 是不是要取消

	參數:
		revise_period=None - 要設定的週期時間，單位是秒
	"""
	global _revise_period

	if _ignorance_checker is None:
		return

	if revise_period is not None:
		try:
			_revise_period = int(revise_period)
			if _revise_period < 200:
				_revise_period = 200
		except:
			_revise_period = None
		return

	_revise_period = None
# ### def set_revise_period

def _periodical_folder_reviser(arg):
	if _ignorance_checker is None:
		return

	target_directory = arg

	for root, dirs, files in os.walk(target_directory):
		#print "linux_inotify: revising %r" % (root)
		no_further_scan = []
		wd_to_drop = []

		for d in dirs:
			dirpath = os.path.abspath(os.path.join(root, d))
			wd = _watchmanager.get_wd(dirpath)
			if wd is None:
				no_further_scan.append(d)
				#print "linux_inotify: no further revise %r" % (d,)
			elif _ignorance_checker(dirpath, None):
				wd_to_drop.append(wd)
				no_further_scan.append(d)
				print "linux_inotify: removing %r (wd=%r)" % (d, wd,)

		for d in no_further_scan:
			dirs.remove(d)
		if len(wd_to_drop) > 0:
			_watchmanager.rm_watch(wd_to_drop, rec=True)
# ### def _periodical_folder_reviser

class _ExcludeFilter:
	def __init__(self, target_directory, recursive_watch=False):
		self.target_directory = target_directory
		self.recursive_watch = recursive_watch
	# ### def __init__

	def do_exclude_filting(self, filepath):
		filepath = os.path.abspath(filepath)
		if os.path.isdir(filepath):
			path = filepath
			name = None
		else:
			path, name, = os.path.split(filepath)

		if (False == self.recursive_watch) and (path != self.target_directory):
			return True

		if _ignorance_checker is not None:
			relpath = _get_relpath(path, self.target_directory)
			r = _ignorance_checker(relpath, name)
			if r:
				return True

		return False
	# ### def do_exclude_filting

	def __call__(self, path):
		return self.do_exclude_filting(path)
	# ### def __call__
# ### class _ExcludeFilter

def monitor_configure(config, metastorage):
	""" 設定監視器組態

	參數:
		config - 帶有參數的字典
		metastorage - 中介資訊資料庫物件
	回傳值:
		(無)
	"""

	# 載入 ignorance checker
	if 'ignorance-checker' in config:
		set_ignorance_checker(str(config['ignorance-checker']))

	set_revise_period(config.get('revise-interval'))
# ### def monitor_configure

class _EventHandler(pyinotify.ProcessEvent):
	def __init__(self, watcher_instance, target_directory):
		pyinotify.ProcessEvent.__init__(self)

		self.watcher_instance = watcher_instance
		self.target_directory = target_directory
	# ### def __init__

	def trigger_operation(self, pathname, watcher_eventcode):
		path, name, = os.path.split(os.path.abspath(pathname))
		relpath = _get_relpath(path, self.target_directory)
		self.watcher_instance.discover_file_change(name, relpath, watcher_eventcode)
	# ### def perform_modified_operation

	def process_IN_CLOSE_WRITE(self, event):
		print "inotify::IN_CLOSE_WRITE: %r" % (event.pathname,)
		self.trigger_operation(event.pathname, watcher.FEVENT_MODIFIED)
	# ### def process_IN_CLOSE_WRITE

	def process_IN_MOVED_TO(self, event):
		print "inotify::IN_MOVED_TO: %r" % (event.pathname,)
		self.trigger_operation(event.pathname, watcher.FEVENT_MODIFIED)
	# ### def process.IN_MOVED_TO

	def process_IN_DELETE(self, event):
		print "inotify::IN_DELETE: %r" % (event.pathname,)
		self.trigger_operation(event.pathname, watcher.FEVENT_DELETED)
	# ### def process_IN_DELETE

	def process_IN_Q_OVERFLOW(self, event):
		print "inotify::IN_Q_OVERFLOW"
		if _queue_overflow_event_callback is not None:
			_queue_overflow_event_callback()
	# ### def process_IN_Q_OVERFLOW
# ### class EventHandler

_watchmanager = None
def monitor_start(watcher_instance, target_directory, recursive_watch=False):
	""" 開始監控目錄作業

	參數:
		watcher_instance - watcher.WatcherEngine 物件實體
		target_directory - 監測目標資料夾
		recursive_watch - 是否要遞迴監測子資料夾
	回傳值:
		(無)
	"""

	global _watchmanager

	mask = pyinotify.IN_DELETE | pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO	# watched events
	if _queue_overflow_event_callback is not None:
		mask = mask | pyinotify.IN_Q_OVERFLOW

	if _watchmanager is not None:
		print "ERR: linux_inotify: designed to watch one directory only, unspecified behavior with multiple monitor_start."

	_watchmanager = pyinotify.WatchManager(exclude_filter=_ExcludeFilter(target_directory, recursive_watch))
	handler = _EventHandler(watcher_instance, target_directory)
	notifier = pyinotify.AsyncNotifier(_watchmanager, handler, channel_map=watcher_instance.process_driver.async_map)

	auto_add_folder = False if (not recursive_watch) else True
	wdd = _watchmanager.add_watch(target_directory, mask, rec=True, auto_add=auto_add_folder)

	if _revise_period is not None:
		watcher_instance.process_driver.append_periodical_call(_periodical_folder_reviser, None, _revise_period)
# ### def monitor_start

def monitor_stop():
	""" 停止作業，準備結束

	參數:
		(無)
	回傳值:
		(無)
	"""

	pass
# ### def monitor_stop

# vim: ts=4 sw=4 ai nowrap
<|endoftext|>


<commit_before>import os, sys
from os.path import join, abspath, dirname

# PATH vars

here = lambda *x: join(abspath(dirname(__file__)), *x)
PROJECT_ROOT = here("..")
root = lambda *x: join(abspath(PROJECT_ROOT), *x)
repo_root = lambda *x: join(abspath(here("../..")), *x)

sys.path.insert(0, root('apps'))

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = ()

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.contrib.gis.db.backends.postgis',
        'NAME': 'polling_stations',
        'USER': 'postgres',
        'PASSWORD': '',
        'HOST': '',                      # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP.
        'PORT': '',                      # Set to empty string for default.
    }
}

import dj_database_url
DATABASES['default'] =  dj_database_url.config()
DATABASES['default']['ENGINE'] = 'django.contrib.gis.db.backends.postgis'

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'Europe/London'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-gb'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = False

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
MEDIA_ROOT = root('assets', 'uploads')

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = '/media/'

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = root('static_root')
# print(STATIC_ROOT)

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    root('assets'),
    root('static'),
)

# TODO find a way to move these in to the DC theme app?
STATIC_PRECOMPILER_ROOT =root('static')
import os
import dc_theme
root_path = os.path.dirname(dc_theme.__file__)
STATIC_PRECOMPILER_COMPILERS = (
    ('static_precompiler.compilers.scss.SCSS', {
        "sourcemap_enabled": True,
        # "output_style": "compressed",
        "load_paths": [
            root_path + '/static/dc_theme/bower_components/foundation-sites/assets',
            root_path + '/static/dc_theme/bower_components/foundation-sites/scss',
            root_path + '/static/dc_theme/bower_components/motion-ui/src',
            root_path + '/static/dc_theme/scss/',
        ],
    }),
)

# Use write-through cache for sessions
SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db'

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'asdasdasdasdasdasdasd'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    # 'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'data_finder.middleware.UTMTrackerModdleware',
    'whitelabel.middleware.WhiteLabelModdleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

TEMPLATE_CONTEXT_PROCESSORS = (
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.static",
    'django.core.context_processors.request',
    "django.core.context_processors.tz",
    "django.contrib.messages.context_processors.messages",
    "django.contrib.auth.context_processors.auth",
    "whitelabel.context_processors.base_template",
    'dc_theme.context_processors.dc_theme_context'
)

ROOT_URLCONF = 'polling_stations.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'polling_stations.wsgi.application'

TEMPLATE_DIRS = (
    root('templates'),
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',
    'django.contrib.gis',
    'rest_framework',
    'django_extensions',
)

PROJECT_APPS = (
    'pollingstations',
    'councils',
    'data_finder',
    'data_collection',
    'whitelabel',
    'nus_wales',
    'dc_theme',
    'static_precompiler',
)

INSTALLED_APPS += PROJECT_APPS

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

"""
Map config:
-----------

Set a shell environment variable TILE_LAYER
to configure which tile layer is used by leaflet.

Supported values are:
'MapQuestOpen' (default)
'MapQuestSDK'
'OpenStreetMap'
"""
TILE_LAYER = os.environ.get('TILE_LAYER', 'MapQuestOpen')
"""
Set a shell environment variable MQ_KEY
to specify MapQuestSDK API key.
"""
MQ_KEY = os.environ.get('MQ_KEY', None)

from django.utils.translation import ugettext_lazy as _
LANGUAGE_CODE = 'en'
LANGUAGES = [
  ('en', 'English'),
  ('cy-gb', 'Welsh'),
]
USE_I18N = True,
USE_L10N = True,
LOCALE_PATHS = (
    repo_root('locale'),
)

# API Settings

REST_FRAMEWORK = {
    # Use Django's standard `django.contrib.auth` permissions,
    # or allow read-only access for unauthenticated users.
    'DEFAULT_PERMISSION_CLASSES': [
        'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly'
    ]
}

EMBED_PREFIXES = (
    'embed',
)

WHITELABEL_PREFIXES = (
    'nus_wales',
)

INTERNAL_IPS = ('127.0.0.1')
SITE_TITLE = "Where Do I Vote?"

# .local.py overrides all the common settings.
try:
    from .local import *
except ImportError:
    pass

TEST_RUNNER = 'django.test.runner.DiscoverRunner'

# importing test settings file if necessary (TODO chould be done better)
if len(sys.argv) > 1 and sys.argv[1] in ['test', 'harvest']:
    from .testing import *

<commit_msg>Use signed cookies to store sessions<commit_after>import os, sys
from os.path import join, abspath, dirname

# PATH vars

here = lambda *x: join(abspath(dirname(__file__)), *x)
PROJECT_ROOT = here("..")
root = lambda *x: join(abspath(PROJECT_ROOT), *x)
repo_root = lambda *x: join(abspath(here("../..")), *x)

sys.path.insert(0, root('apps'))

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = ()

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.contrib.gis.db.backends.postgis',
        'NAME': 'polling_stations',
        'USER': 'postgres',
        'PASSWORD': '',
        'HOST': '',                      # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP.
        'PORT': '',                      # Set to empty string for default.
    }
}

import dj_database_url
DATABASES['default'] =  dj_database_url.config()
DATABASES['default']['ENGINE'] = 'django.contrib.gis.db.backends.postgis'

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'Europe/London'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-gb'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = False

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
MEDIA_ROOT = root('assets', 'uploads')

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = '/media/'

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = root('static_root')
# print(STATIC_ROOT)

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    root('assets'),
    root('static'),
)

# TODO find a way to move these in to the DC theme app?
STATIC_PRECOMPILER_ROOT =root('static')
import os
import dc_theme
root_path = os.path.dirname(dc_theme.__file__)
STATIC_PRECOMPILER_COMPILERS = (
    ('static_precompiler.compilers.scss.SCSS', {
        "sourcemap_enabled": True,
        # "output_style": "compressed",
        "load_paths": [
            root_path + '/static/dc_theme/bower_components/foundation-sites/assets',
            root_path + '/static/dc_theme/bower_components/foundation-sites/scss',
            root_path + '/static/dc_theme/bower_components/motion-ui/src',
            root_path + '/static/dc_theme/scss/',
        ],
    }),
)

# Use write-through cache for sessions
SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db'

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'asdasdasdasdasdasdasd'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    # 'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'data_finder.middleware.UTMTrackerModdleware',
    'whitelabel.middleware.WhiteLabelModdleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

TEMPLATE_CONTEXT_PROCESSORS = (
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.static",
    'django.core.context_processors.request',
    "django.core.context_processors.tz",
    "django.contrib.messages.context_processors.messages",
    "django.contrib.auth.context_processors.auth",
    "whitelabel.context_processors.base_template",
    'dc_theme.context_processors.dc_theme_context'
)

ROOT_URLCONF = 'polling_stations.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'polling_stations.wsgi.application'

TEMPLATE_DIRS = (
    root('templates'),
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',
    'django.contrib.gis',
    'rest_framework',
    'django_extensions',
)

PROJECT_APPS = (
    'pollingstations',
    'councils',
    'data_finder',
    'data_collection',
    'whitelabel',
    'nus_wales',
    'dc_theme',
    'static_precompiler',
)

INSTALLED_APPS += PROJECT_APPS

SESSION_ENGINE = "django.contrib.sessions.backends.signed_cookies"

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

"""
Map config:
-----------

Set a shell environment variable TILE_LAYER
to configure which tile layer is used by leaflet.

Supported values are:
'MapQuestOpen' (default)
'MapQuestSDK'
'OpenStreetMap'
"""
TILE_LAYER = os.environ.get('TILE_LAYER', 'MapQuestOpen')
"""
Set a shell environment variable MQ_KEY
to specify MapQuestSDK API key.
"""
MQ_KEY = os.environ.get('MQ_KEY', None)

from django.utils.translation import ugettext_lazy as _
LANGUAGE_CODE = 'en'
LANGUAGES = [
  ('en', 'English'),
  ('cy-gb', 'Welsh'),
]
USE_I18N = True,
USE_L10N = True,
LOCALE_PATHS = (
    repo_root('locale'),
)

# API Settings

REST_FRAMEWORK = {
    # Use Django's standard `django.contrib.auth` permissions,
    # or allow read-only access for unauthenticated users.
    'DEFAULT_PERMISSION_CLASSES': [
        'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly'
    ]
}

EMBED_PREFIXES = (
    'embed',
)

WHITELABEL_PREFIXES = (
    'nus_wales',
)

INTERNAL_IPS = ('127.0.0.1')
SITE_TITLE = "Where Do I Vote?"

# .local.py overrides all the common settings.
try:
    from .local import *
except ImportError:
    pass

TEST_RUNNER = 'django.test.runner.DiscoverRunner'

# importing test settings file if necessary (TODO chould be done better)
if len(sys.argv) > 1 and sys.argv[1] in ['test', 'harvest']:
    from .testing import *

<|endoftext|>