<commit_before>from citeproc.source.bibtex import BibTeX as cpBibTeX
import citeproc as cp

from collections import defaultdict
import os
from os.path import dirname, exists
import pickle
import requests
import tempfile
from six import PY2, itervalues, iteritems
import warnings

from .config import CACHE_DIR, DUECREDIT_FILE
from .entries import BibTeX, Doi
from .log import lgr

def get_doi_cache_file(doi):
    return os.path.join(CACHE_DIR, doi)

def import_doi(doi):
    cached = get_doi_cache_file(doi)

    if exists(cached):
        with open(cached) as f:
            doi = f.read()
            if PY2:
                return doi.decode('utf-8')
            return doi

    # else -- fetch it
    headers = {'Accept': 'text/bibliography; style=bibtex'}
    url = 'http://dx.doi.org/' + doi
    r = requests.get(url, headers=headers)
    r.encoding = 'UTF-8'
    bibtex = r.text.strip()
    if not bibtex.startswith('@'):
        raise ValueError('wrong doi specified')
    if not exists(cached):
        cache_dir = dirname(cached)
        if not exists(cache_dir):
            os.makedirs(cache_dir)
        with open(cached, 'w') as f:
            if PY2:
                f.write(bibtex.encode('utf-8'))
            else:
                f.write(bibtex)
    return bibtex

class TextOutput(object):  # TODO some parent class to do what...?
    def __init__(self, fd, collector, style=None):
        self.fd = fd
        self.collector = collector
        # TODO: check that CLS style actually exists
        self.style = style
        if 'DUECREDIT_STYLE' in os.environ.keys():
            self.style = os.environ['DUECREDIT_STYLE']
        else:
            self.style = 'harvard1'

    # TODO: refactor name to sth more intuitive
    def _model_citations(self, tags=None):
        if not tags:
            tags = os.environ.get('DUECREDIT_REPORT_TAGS', 'reference,implementation').split(',')
        tags = set(tags)

        citations = self.collector.citations
        if tags != {'*'}:
            # Filter out citations
            citations = dict((k, c)
                             for k, c in iteritems(citations)
                             if tags.intersection(c.tags))

        packages = {}
        modules = {}
        objects = {}

        for key in ('citations', 'entry_keys'):
            packages[key] = defaultdict(list)
            modules[key] = defaultdict(list)
            objects[key] = defaultdict(list)

        # for each path store both a list of entry keys and of citations
        for (path, entry_key), citation in iteritems(citations):
            if ':' in path:
                target_dict = objects
            elif '.' in path:
                target_dict = modules
            else:
                target_dict = packages
            target_dict['citations'][path].append(citation)
            target_dict['entry_keys'][path].append(entry_key)
        return packages, modules, objects

    def dump(self, tags=None):
        # get 'model' of citations
        packages, modules, objects = self._model_citations(tags)

        citations_ordered = []
        # set up view
        keys2refnr = defaultdict(int)  # mapping key -> ref nr
        refnr = 1

        # package level
        for package in sorted(packages['entry_keys']):
            for entry_key in packages['entry_keys'][package]:
                if entry_key not in keys2refnr:
                    keys2refnr[entry_key] = refnr
                    refnr += 1
            citations_ordered.append(package)
            # module level
            for module in sorted(filter(lambda x: package in x, modules['entry_keys'])):
                for entry_key_mod in modules['entry_keys'][module]:
                    if entry_key_mod not in keys2refnr:
                        keys2refnr[entry_key_mod] = refnr
                        refnr += 1
                citations_ordered.append(module)
            # object level
            for obj in sorted(filter(lambda x: package in x, objects['entry_keys'])):
                for entry_key_obj in objects['entry_keys'][obj]:
                    if entry_key_obj not in keys2refnr:
                        keys2refnr[entry_key_obj] = refnr
                        refnr += 1
                citations_ordered.append(obj)

        # Now we can "render" different views of our "model"
        # Here for now just text BUT that is where we can "split" the logic and provide
        # different renderings given the model -- text, rest, md, tex+latex, whatever
        self.fd.write('DueCredit Report:\n')

        for path in citations_ordered:
            if ':' in path:
                self.fd.write('  ')
                citations = objects['citations'][path]
                entry_keys = objects['entry_keys'][path]
            elif '.' in path:
                self.fd.write('  ')
                citations = modules['citations'][path]
                entry_keys = modules['entry_keys'][path]
            else:
                citations = packages['citations'][path]
                entry_keys = packages['entry_keys'][path]
            versions = sorted(map(str, set(str(r.version) for r in citations)))
            refnrs = sorted([str(keys2refnr[entry_key]) for entry_key in entry_keys])
            self.fd.write('- {0} (v {1}) [{2}]\n'.format(path, ' '.join(versions), ', '.join(refnrs)))

        # Print out some stats
        obj_names = ('packages', 'modules', 'functions')
        n_citations = map(len, (packages['citations'], modules['citations'], objects['citations']))
        for citation_type, n in zip(obj_names, n_citations):
            self.fd.write('\n{0} {1} cited'.format(n, citation_type))

        if keys2refnr:
            citations_fromentrykey = self.collector.citations_fromentrykey()
            self.fd.write('\n\nReferences\n' + '-' * 10 + '\n')
            # collect all the entries used
            refnr2keys = sorted([(nr, keys) for keys, nr in iteritems(keys2refnr)])
            for nr, key in refnr2keys:
                self.fd.write('\n[{0}] '.format(nr))
                self.fd.write(get_text_rendering(citations_fromentrykey[key], style=self.style))
            self.fd.write('\n')

def get_text_rendering(citation, style='harvard1'):
    entry = citation.entry
    if isinstance(entry, Doi):
        bibtex_rendering = get_bibtex_rendering(entry)
        return get_text_rendering(bibtex_rendering)
    elif isinstance(entry, BibTeX):
        return format_bibtex(entry, style=style)
    else:
        return str(entry)

def get_bibtex_rendering(entry):
    if isinstance(entry, Doi):
        return BibTeX(import_doi(entry.doi))
    elif isinstance(entry, BibTeX):
        return entry
    else:
        raise ValueError("Have no clue how to get bibtex out of %s" % entry)

def format_bibtex(bibtex_entry, style='harvard1'):
    key = bibtex_entry.get_key()
    # need to save it temporarily to use citeproc-py
    fname = tempfile.mktemp(suffix='.bib')
    try:
        with open(fname, 'wt') as f:
            bibtex = bibtex_entry.rawentry
            bibtex = bibtex.replace(u'\u2013', '--') + "\n"
            # TODO: manage to save/use UTF-8
            if PY2:
                bibtex = bibtex.encode('ascii', 'ignore')
            f.write(bibtex)
        # We need to avoid cpBibTex spitting out warnings
        old_filters = warnings.filters[:]  # store a copy of filters
        warnings.simplefilter('ignore', UserWarning)
        try:
            bib_source = cpBibTeX(fname)
        except Exception as e:
            lgr.error("Failed to process BibTeX file %s" % fname)
            return "ERRORED: %s" % str(e)
        finally:
            # return warnings back
            warnings.filters = old_filters
        bib_style = cp.CitationStylesStyle(style, validate=False)
        # TODO: specify which tags of formatter we want
        bibliography = cp.CitationStylesBibliography(bib_style, bib_source,
                                                     cp.formatter.plain)
        citation = cp.Citation([cp.CitationItem(key)])
        bibliography.register(citation)
    finally:
        os.unlink(fname)

    biblio_out = bibliography.bibliography()
    assert(len(biblio_out) == 1)
    biblio_out = ''.join(biblio_out[0])
    return biblio_out # if biblio_out else str(bibtex_entry)

# TODO: harmonize order of arguments
class PickleOutput(object):
    def __init__(self, collector, fn=DUECREDIT_FILE):
        self.collector = collector
        self.fn = fn

    def dump(self):
        with open(self.fn, 'wb') as f:
            pickle.dump(self.collector, f)

    @classmethod
    def load(cls, filename=DUECREDIT_FILE):
        with open(filename, 'rb') as f:
            return pickle.load(f)

class BibTeXOutput(object):  # TODO some parent class to do what...?
    def __init__(self, fd, collector):
        self.fd = fd
        self.collector = collector

    def dump(self):
        for citation in self.collector.citations.values():
            try:
                bibtex = get_bibtex_rendering(citation.entry)
            except:
                lgr.warning("Failed to generate bibtex for %s" % citation.entry)
                continue
            self.fd.write(bibtex.rawentry + "\n")

def load_due(filename):
    return PickleOutput.load(filename)
<commit_msg>Make code more pythonic<commit_after>from citeproc.source.bibtex import BibTeX as cpBibTeX
import citeproc as cp

from collections import defaultdict
import os
from os.path import dirname, exists
import pickle
import requests
import tempfile
from six import PY2, itervalues, iteritems
import warnings

from .config import CACHE_DIR, DUECREDIT_FILE
from .entries import BibTeX, Doi
from .log import lgr

def get_doi_cache_file(doi):
    return os.path.join(CACHE_DIR, doi)

def import_doi(doi):
    cached = get_doi_cache_file(doi)

    if exists(cached):
        with open(cached) as f:
            doi = f.read()
            if PY2:
                return doi.decode('utf-8')
            return doi

    # else -- fetch it
    headers = {'Accept': 'text/bibliography; style=bibtex'}
    url = 'http://dx.doi.org/' + doi
    r = requests.get(url, headers=headers)
    r.encoding = 'UTF-8'
    bibtex = r.text.strip()
    if not bibtex.startswith('@'):
        raise ValueError('wrong doi specified')
    if not exists(cached):
        cache_dir = dirname(cached)
        if not exists(cache_dir):
            os.makedirs(cache_dir)
        with open(cached, 'w') as f:
            if PY2:
                f.write(bibtex.encode('utf-8'))
            else:
                f.write(bibtex)
    return bibtex

class TextOutput(object):  # TODO some parent class to do what...?
    def __init__(self, fd, collector, style=None):
        self.fd = fd
        self.collector = collector
        # TODO: check that CLS style actually exists
        self.style = style
        if 'DUECREDIT_STYLE' in os.environ.keys():
            self.style = os.environ['DUECREDIT_STYLE']
        else:
            self.style = 'harvard1'

    # TODO: refactor name to sth more intuitive
    def _model_citations(self, tags=None):
        if not tags:
            tags = os.environ.get('DUECREDIT_REPORT_TAGS', 'reference,implementation').split(',')
        tags = set(tags)

        citations = self.collector.citations
        if tags != {'*'}:
            # Filter out citations
            citations = dict((k, c)
                             for k, c in iteritems(citations)
                             if tags.intersection(c.tags))

        packages = {}
        modules = {}
        objects = {}

        for key in ('citations', 'entry_keys'):
            packages[key] = defaultdict(list)
            modules[key] = defaultdict(list)
            objects[key] = defaultdict(list)

        # for each path store both a list of entry keys and of citations
        for (path, entry_key), citation in iteritems(citations):
            if ':' in path:
                target_dict = objects
            elif '.' in path:
                target_dict = modules
            else:
                target_dict = packages
            target_dict['citations'][path].append(citation)
            target_dict['entry_keys'][path].append(entry_key)
        return packages, modules, objects

    def dump(self, tags=None):
        # get 'model' of citations
        packages, modules, objects = self._model_citations(tags)

        citations_ordered = []
        # set up view
        keys2refnr = defaultdict(int)  # mapping key -> ref nr
        refnr = 1

        # package level
        for package in sorted(packages['entry_keys']):
            for entry_key in packages['entry_keys'][package]:
                if entry_key not in keys2refnr:
                    keys2refnr[entry_key] = refnr
                    refnr += 1
            citations_ordered.append(package)
            # module level
            for module in sorted(filter(lambda x: package in x, modules['entry_keys'])):
                for entry_key_mod in modules['entry_keys'][module]:
                    if entry_key_mod not in keys2refnr:
                        keys2refnr[entry_key_mod] = refnr
                        refnr += 1
                citations_ordered.append(module)
            # object level
            for obj in sorted(filter(lambda x: package in x, objects['entry_keys'])):
                for entry_key_obj in objects['entry_keys'][obj]:
                    if entry_key_obj not in keys2refnr:
                        keys2refnr[entry_key_obj] = refnr
                        refnr += 1
                citations_ordered.append(obj)

        # Now we can "render" different views of our "model"
        # Here for now just text BUT that is where we can "split" the logic and provide
        # different renderings given the model -- text, rest, md, tex+latex, whatever
        self.fd.write('DueCredit Report:\n')

        for path in citations_ordered:
            if ':' in path:
                self.fd.write('  ')
                target_dict = objects
            elif '.' in path:
                self.fd.write('  ')
                target_dict = modules
            else:
                target_dict = packages
            citations = target_dict['citations'][path]
            entry_keys = target_dict['entry_keys'][path]
            versions = sorted(map(str, set(str(r.version) for r in citations)))
            refnrs = sorted([str(keys2refnr[entry_key]) for entry_key in entry_keys])
            self.fd.write('- {0} (v {1}) [{2}]\n'.format(path, ' '.join(versions), ', '.join(refnrs)))

        # Print out some stats
        obj_names = ('packages', 'modules', 'functions')
        n_citations = map(len, (packages['citations'], modules['citations'], objects['citations']))
        for citation_type, n in zip(obj_names, n_citations):
            self.fd.write('\n{0} {1} cited'.format(n, citation_type))

        if keys2refnr:
            citations_fromentrykey = self.collector.citations_fromentrykey()
            self.fd.write('\n\nReferences\n' + '-' * 10 + '\n')
            # collect all the entries used
            refnr2keys = sorted([(nr, keys) for keys, nr in iteritems(keys2refnr)])
            for nr, key in refnr2keys:
                self.fd.write('\n[{0}] '.format(nr))
                self.fd.write(get_text_rendering(citations_fromentrykey[key], style=self.style))
            self.fd.write('\n')

def get_text_rendering(citation, style='harvard1'):
    entry = citation.entry
    if isinstance(entry, Doi):
        bibtex_rendering = get_bibtex_rendering(entry)
        return get_text_rendering(bibtex_rendering)
    elif isinstance(entry, BibTeX):
        return format_bibtex(entry, style=style)
    else:
        return str(entry)

def get_bibtex_rendering(entry):
    if isinstance(entry, Doi):
        return BibTeX(import_doi(entry.doi))
    elif isinstance(entry, BibTeX):
        return entry
    else:
        raise ValueError("Have no clue how to get bibtex out of %s" % entry)

def format_bibtex(bibtex_entry, style='harvard1'):
    key = bibtex_entry.get_key()
    # need to save it temporarily to use citeproc-py
    fname = tempfile.mktemp(suffix='.bib')
    try:
        with open(fname, 'wt') as f:
            bibtex = bibtex_entry.rawentry
            bibtex = bibtex.replace(u'\u2013', '--') + "\n"
            # TODO: manage to save/use UTF-8
            if PY2:
                bibtex = bibtex.encode('ascii', 'ignore')
            f.write(bibtex)
        # We need to avoid cpBibTex spitting out warnings
        old_filters = warnings.filters[:]  # store a copy of filters
        warnings.simplefilter('ignore', UserWarning)
        try:
            bib_source = cpBibTeX(fname)
        except Exception as e:
            lgr.error("Failed to process BibTeX file %s" % fname)
            return "ERRORED: %s" % str(e)
        finally:
            # return warnings back
            warnings.filters = old_filters
        bib_style = cp.CitationStylesStyle(style, validate=False)
        # TODO: specify which tags of formatter we want
        bibliography = cp.CitationStylesBibliography(bib_style, bib_source,
                                                     cp.formatter.plain)
        citation = cp.Citation([cp.CitationItem(key)])
        bibliography.register(citation)
    finally:
        os.unlink(fname)

    biblio_out = bibliography.bibliography()
    assert(len(biblio_out) == 1)
    biblio_out = ''.join(biblio_out[0])
    return biblio_out # if biblio_out else str(bibtex_entry)

# TODO: harmonize order of arguments
class PickleOutput(object):
    def __init__(self, collector, fn=DUECREDIT_FILE):
        self.collector = collector
        self.fn = fn

    def dump(self):
        with open(self.fn, 'wb') as f:
            pickle.dump(self.collector, f)

    @classmethod
    def load(cls, filename=DUECREDIT_FILE):
        with open(filename, 'rb') as f:
            return pickle.load(f)

class BibTeXOutput(object):  # TODO some parent class to do what...?
    def __init__(self, fd, collector):
        self.fd = fd
        self.collector = collector

    def dump(self):
        for citation in self.collector.citations.values():
            try:
                bibtex = get_bibtex_rendering(citation.entry)
            except:
                lgr.warning("Failed to generate bibtex for %s" % citation.entry)
                continue
            self.fd.write(bibtex.rawentry + "\n")

def load_due(filename):
    return PickleOutput.load(filename)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import logging
import os
from collections import Sequence

from crabpy.client import crab_factory
from crabpy.gateway.capakey import CapakeyRestGateway
from crabpy.gateway.crab import CrabGateway
from pyramid.config import Configurator
from pyramid.settings import asbool
from zope.interface import Interface

from crabpy_pyramid.renderers.capakey import (
    json_list_renderer as capakey_json_list_renderer,
    json_item_renderer as capakey_json_item_renderer
)
from crabpy_pyramid.renderers.crab import (
    json_list_renderer as crab_json_list_renderer,
    json_item_renderer as crab_json_item_renderer
)

log = logging.getLogger(__name__)
GENERATE_ETAG_ROUTE_NAMES = set()

class ICapakey(Interface):
    pass

class ICrab(Interface):
    pass

def _parse_settings(settings):
    defaults = {
        'capakey.include': False,
        'crab.include': True,
        'cache.file.root': '/tmp/dogpile_data'
    }
    args = defaults.copy()

    # booelean settings
    for short_key_name in ('capakey.include', 'crab.include'):
        key_name = "crabpy.%s" % short_key_name
        if key_name in settings:
            args[short_key_name] = asbool(settings.get(
                key_name, defaults.get(short_key_name)
            ))

    # string setting
    for short_key_name in ('proxy.http', 'proxy.https', 'cache.file.root'):
        key_name = "crabpy.%s" % short_key_name
        if key_name in settings:
            args[short_key_name] = settings.get(key_name)

    # cache configuration
    for short_key_name in ('crab.cache_config', 'capakey.cache_config'):
        key_name = "crabpy.%s." % short_key_name
        cache_config = {}
        for skey in settings.keys():
            if skey.startswith(key_name):
                cache_config[skey[len(key_name):]] = settings.get(skey)
        if cache_config:
            args[short_key_name] = cache_config

    # crab wsdl settings
    for short_key_name in ('crab.wsdl', ):
        key_name = "crabpy.%s" % short_key_name
        if key_name in settings:
            args[short_key_name] = settings.get(key_name)

    log.debug(settings)
    log.debug(args)
    return args

def _filter_settings(settings, prefix):
    """
    Filter all settings to only return settings that start with a certain
    prefix.

    :param dict settings: A settings dictionary.
    :param str prefix: A prefix.
    """
    ret = {}
    for skey in settings.keys():
        if skey.startswith(prefix):
            key = skey[len(prefix):]
            ret[key] = settings[skey]
    return ret

def _build_capakey(registry, settings):
    capakey = registry.queryUtility(ICapakey)
    if capakey is not None:
        return capakey
    if 'cache_config' in settings:
        cache_config = settings['cache_config']
        del settings['cache_config']
    else:
        cache_config = {}
    gateway = CapakeyRestGateway(cache_config=cache_config)

    registry.registerUtility(gateway, ICapakey)
    return registry.queryUtility(ICapakey)

def _build_crab(registry, settings):
    crab = registry.queryUtility(ICrab)
    if crab is not None:
        return crab
    if 'cache_config' in settings:
        cache_config = settings['cache_config']
        del settings['cache_config']
    else:
        cache_config = {}
    factory = crab_factory(**settings)
    gateway = CrabGateway(factory, cache_config=cache_config)

    registry.registerUtility(gateway, ICrab)
    return registry.queryUtility(ICrab)

def get_capakey(registry):
    """
    Get the Capakey Gateway

    :rtype: :class:`crabpy.gateway.capakey.CapakeyRestGateway`
    """
    # argument might be a config or a request
    regis = getattr(registry, 'registry', None)
    if regis is None:
        regis = registry

    return regis.queryUtility(ICapakey)

def get_crab(registry):
    """
    Get the Crab Gateway

    :rtype: :class:`crabpy.gateway.crab.CrabGateway`
    # argument might be a config or a request
    """
    # argument might be a config or a request
    regis = getattr(registry, 'registry', None)
    if regis is None:
        regis = registry

    return regis.queryUtility(ICrab)

def _get_proxy_settings(settings):
    base_settings = {}
    http = settings.get('proxy.http', None)
    https = settings.get('proxy.https', None)
    if (http or https):
        base_settings["proxy"] = {}
        if "proxy.http" in settings:
            base_settings["proxy"]["http"] = settings["proxy.http"]
            log.info('HTTP proxy: %s' % base_settings["proxy"]["http"])
        if "proxy.https" in settings:
            base_settings["proxy"]["https"] = settings["proxy.https"]
            log.info('HTTPS proxy: %s' % base_settings["proxy"]["https"])
    return base_settings

def add_route(config, name, pattern, *args, **kwargs):
    """
    Adds a pyramid route to the config. All args and kwargs will be
    passed on to config.add_route.

    This exists so the default behaviour of including crabpy will still be to
    cache all crabpy routes.
    """
    config.add_route(name, pattern, *args, **kwargs)
    GENERATE_ETAG_ROUTE_NAMES.add(name)

def conditional_http_tween_factory(handler, registry):
    """
    Tween that adds ETag headers and tells Pyramid to enable
    conditional responses where appropriate.
    """
    settings = registry.settings if hasattr(registry, 'settings') else {}
    if 'generate_etag_for.list' in settings:
        route_names = settings.get('generate_etag_for.list').split()
        GENERATE_ETAG_ROUTE_NAMES.update(route_names)

    def conditional_http_tween(request):
        response = handler(request)

        if request.matched_route.name in GENERATE_ETAG_ROUTE_NAMES:

            # If the Last-Modified header has been set, we want to enable the
            # conditional response processing.
            if response.last_modified is not None:
                response.conditional_response = True

            # We want to only enable the conditional machinery if either we
            # were given an explicit ETag header by the view or we have a
            # buffered response and can generate the ETag header ourself.
            if response.etag is not None:
                response.conditional_response = True
            elif (isinstance(response.app_iter, Sequence) and
                          len(response.app_iter) == 1) and response.body is not None:
                response.conditional_response = True
                response.md5_etag()

        return response

    return conditional_http_tween

def includeme(config):
    """
    Include `crabpy_pyramid` in this `Pyramid` application.

    :param pyramid.config.Configurator config: A Pyramid configurator.
    """

    settings = _parse_settings(config.registry.settings)
    base_settings = _get_proxy_settings(settings)

    # http caching tween
    if not settings.get('etag_tween_disabled', False):
        config.add_tween('crabpy_pyramid.conditional_http_tween_factory')

    # create cache
    root = settings.get('cache.file.root', '/tmp/dogpile_data')
    if not os.path.exists(root):
        os.makedirs(root)

    capakey_settings = dict(_filter_settings(settings, 'capakey.'),
                            **base_settings)
    if 'include' in capakey_settings:
        log.info("The 'capakey.include' setting is deprecated. Capakey will "
                 "always be included.")
    log.info('Adding CAPAKEY Gateway.')
    config.add_renderer('capakey_listjson', capakey_json_list_renderer)
    config.add_renderer('capakey_itemjson', capakey_json_item_renderer)
    _build_capakey(config.registry, capakey_settings)
    config.add_request_method(get_capakey, 'capakey_gateway')
    config.add_directive('get_capakey', get_capakey)
    config.include('crabpy_pyramid.routes.capakey')
    config.scan('crabpy_pyramid.views.capakey')

    crab_settings = dict(_filter_settings(settings, 'crab.'), **base_settings)
    if crab_settings['include']:
        log.info('Adding CRAB Gateway.')
        del crab_settings['include']
        config.add_renderer('crab_listjson', crab_json_list_renderer)
        config.add_renderer('crab_itemjson', crab_json_item_renderer)
        _build_crab(config.registry, crab_settings)
        config.add_directive('get_crab', get_crab)
        config.add_request_method(get_crab, 'crab_gateway')
        config.include('crabpy_pyramid.routes.crab')
        config.scan('crabpy_pyramid.views.crab')

def main(global_config, **settings):
    """
     This function returns a Pyramid WSGI application.
    """
    config = Configurator(settings=settings)

    includeme(config)
    return config.make_wsgi_app()
<commit_msg>Fix etag tween error when route is not found. (#108)<commit_after># -*- coding: utf-8 -*-

import logging
import os
from collections import Sequence

from crabpy.client import crab_factory
from crabpy.gateway.capakey import CapakeyRestGateway
from crabpy.gateway.crab import CrabGateway
from pyramid.config import Configurator
from pyramid.settings import asbool
from zope.interface import Interface

from crabpy_pyramid.renderers.capakey import (
    json_list_renderer as capakey_json_list_renderer,
    json_item_renderer as capakey_json_item_renderer
)
from crabpy_pyramid.renderers.crab import (
    json_list_renderer as crab_json_list_renderer,
    json_item_renderer as crab_json_item_renderer
)

log = logging.getLogger(__name__)
GENERATE_ETAG_ROUTE_NAMES = set()

class ICapakey(Interface):
    pass

class ICrab(Interface):
    pass

def _parse_settings(settings):
    defaults = {
        'capakey.include': False,
        'crab.include': True,
        'cache.file.root': '/tmp/dogpile_data'
    }
    args = defaults.copy()

    # booelean settings
    for short_key_name in ('capakey.include', 'crab.include'):
        key_name = "crabpy.%s" % short_key_name
        if key_name in settings:
            args[short_key_name] = asbool(settings.get(
                key_name, defaults.get(short_key_name)
            ))

    # string setting
    for short_key_name in ('proxy.http', 'proxy.https', 'cache.file.root'):
        key_name = "crabpy.%s" % short_key_name
        if key_name in settings:
            args[short_key_name] = settings.get(key_name)

    # cache configuration
    for short_key_name in ('crab.cache_config', 'capakey.cache_config'):
        key_name = "crabpy.%s." % short_key_name
        cache_config = {}
        for skey in settings.keys():
            if skey.startswith(key_name):
                cache_config[skey[len(key_name):]] = settings.get(skey)
        if cache_config:
            args[short_key_name] = cache_config

    # crab wsdl settings
    for short_key_name in ('crab.wsdl', ):
        key_name = "crabpy.%s" % short_key_name
        if key_name in settings:
            args[short_key_name] = settings.get(key_name)

    log.debug(settings)
    log.debug(args)
    return args

def _filter_settings(settings, prefix):
    """
    Filter all settings to only return settings that start with a certain
    prefix.

    :param dict settings: A settings dictionary.
    :param str prefix: A prefix.
    """
    ret = {}
    for skey in settings.keys():
        if skey.startswith(prefix):
            key = skey[len(prefix):]
            ret[key] = settings[skey]
    return ret

def _build_capakey(registry, settings):
    capakey = registry.queryUtility(ICapakey)
    if capakey is not None:
        return capakey
    if 'cache_config' in settings:
        cache_config = settings['cache_config']
        del settings['cache_config']
    else:
        cache_config = {}
    gateway = CapakeyRestGateway(cache_config=cache_config)

    registry.registerUtility(gateway, ICapakey)
    return registry.queryUtility(ICapakey)

def _build_crab(registry, settings):
    crab = registry.queryUtility(ICrab)
    if crab is not None:
        return crab
    if 'cache_config' in settings:
        cache_config = settings['cache_config']
        del settings['cache_config']
    else:
        cache_config = {}
    factory = crab_factory(**settings)
    gateway = CrabGateway(factory, cache_config=cache_config)

    registry.registerUtility(gateway, ICrab)
    return registry.queryUtility(ICrab)

def get_capakey(registry):
    """
    Get the Capakey Gateway

    :rtype: :class:`crabpy.gateway.capakey.CapakeyRestGateway`
    """
    # argument might be a config or a request
    regis = getattr(registry, 'registry', None)
    if regis is None:
        regis = registry

    return regis.queryUtility(ICapakey)

def get_crab(registry):
    """
    Get the Crab Gateway

    :rtype: :class:`crabpy.gateway.crab.CrabGateway`
    # argument might be a config or a request
    """
    # argument might be a config or a request
    regis = getattr(registry, 'registry', None)
    if regis is None:
        regis = registry

    return regis.queryUtility(ICrab)

def _get_proxy_settings(settings):
    base_settings = {}
    http = settings.get('proxy.http', None)
    https = settings.get('proxy.https', None)
    if (http or https):
        base_settings["proxy"] = {}
        if "proxy.http" in settings:
            base_settings["proxy"]["http"] = settings["proxy.http"]
            log.info('HTTP proxy: %s' % base_settings["proxy"]["http"])
        if "proxy.https" in settings:
            base_settings["proxy"]["https"] = settings["proxy.https"]
            log.info('HTTPS proxy: %s' % base_settings["proxy"]["https"])
    return base_settings

def add_route(config, name, pattern, *args, **kwargs):
    """
    Adds a pyramid route to the config. All args and kwargs will be
    passed on to config.add_route.

    This exists so the default behaviour of including crabpy will still be to
    cache all crabpy routes.
    """
    config.add_route(name, pattern, *args, **kwargs)
    GENERATE_ETAG_ROUTE_NAMES.add(name)

def conditional_http_tween_factory(handler, registry):
    """
    Tween that adds ETag headers and tells Pyramid to enable
    conditional responses where appropriate.
    """
    settings = registry.settings if hasattr(registry, 'settings') else {}
    if 'generate_etag_for.list' in settings:
        route_names = settings.get('generate_etag_for.list').split()
        GENERATE_ETAG_ROUTE_NAMES.update(route_names)

    def conditional_http_tween(request):
        response = handler(request)
        if request.matched_route is None:
            return response

        if request.matched_route.name in GENERATE_ETAG_ROUTE_NAMES:

            # If the Last-Modified header has been set, we want to enable the
            # conditional response processing.
            if response.last_modified is not None:
                response.conditional_response = True

            # We want to only enable the conditional machinery if either we
            # were given an explicit ETag header by the view or we have a
            # buffered response and can generate the ETag header ourself.
            if response.etag is not None:
                response.conditional_response = True
            elif (isinstance(response.app_iter, Sequence) and
                          len(response.app_iter) == 1) and response.body is not None:
                response.conditional_response = True
                response.md5_etag()

        return response

    return conditional_http_tween

def includeme(config):
    """
    Include `crabpy_pyramid` in this `Pyramid` application.

    :param pyramid.config.Configurator config: A Pyramid configurator.
    """

    settings = _parse_settings(config.registry.settings)
    base_settings = _get_proxy_settings(settings)

    # http caching tween
    if not settings.get('etag_tween_disabled', False):
        config.add_tween('crabpy_pyramid.conditional_http_tween_factory')

    # create cache
    root = settings.get('cache.file.root', '/tmp/dogpile_data')
    if not os.path.exists(root):
        os.makedirs(root)

    capakey_settings = dict(_filter_settings(settings, 'capakey.'),
                            **base_settings)
    if 'include' in capakey_settings:
        log.info("The 'capakey.include' setting is deprecated. Capakey will "
                 "always be included.")
    log.info('Adding CAPAKEY Gateway.')
    config.add_renderer('capakey_listjson', capakey_json_list_renderer)
    config.add_renderer('capakey_itemjson', capakey_json_item_renderer)
    _build_capakey(config.registry, capakey_settings)
    config.add_request_method(get_capakey, 'capakey_gateway')
    config.add_directive('get_capakey', get_capakey)
    config.include('crabpy_pyramid.routes.capakey')
    config.scan('crabpy_pyramid.views.capakey')

    crab_settings = dict(_filter_settings(settings, 'crab.'), **base_settings)
    if crab_settings['include']:
        log.info('Adding CRAB Gateway.')
        del crab_settings['include']
        config.add_renderer('crab_listjson', crab_json_list_renderer)
        config.add_renderer('crab_itemjson', crab_json_item_renderer)
        _build_crab(config.registry, crab_settings)
        config.add_directive('get_crab', get_crab)
        config.add_request_method(get_crab, 'crab_gateway')
        config.include('crabpy_pyramid.routes.crab')
        config.scan('crabpy_pyramid.views.crab')

def main(global_config, **settings):
    """
     This function returns a Pyramid WSGI application.
    """
    config = Configurator(settings=settings)

    includeme(config)
    return config.make_wsgi_app()
<|endoftext|>


<commit_before>import itchat
import time
import os
import re
import shutil
from random import choice

from packages import core
from packages import spider, pymath, zhconvert, audio # 我的一些功能

itchat.auto_login(hotReload=True, enableCmdQR=2) # 登录用户账号以便获取昵称

USERNAME = itchat.search_friends()['NickName'] # 获取自己的昵称
GREETINGS = ('嘿','嗨，我是饺子机器人，有何吩咐？','哟','你好啊','你好呀',':)')
DONT_UNDERSTAND = ('对不起我不明白诶……','抱歉我没听懂:(','不好意思没明白','Sorry没有听懂')
HELP = '我是一个功能性机器人哦，聊天不太在行，但是可以干很多事！'
WEEKDAYS = {'0':'天','1':'一','2':'二','3':'三','4':'四','5':'五','6':'六'}

def parse_msg(msg):
    '''处理并对信息作相对应的处理。没有返回。'''
    atName = '@' + msg['ActualNickName'] + '\u2005' # @对方的的用户名
    recipient = msg['FromUserName'] # 群聊名称(消息接受方)
    content = msg['Content'].replace('@{}\u2005'.format(USERNAME),'').replace('@{} '.format(USERNAME),'').strip() # 清理消息文本
    if content[:3] == '求数列':
        inp = content[3:].strip(' :：').replace('，',',').replace(' ','')
        try:
            seq = pymath.Sequence(inp)
            itchat.send('{}公式是：An = {}'.format(atName, seq.generateFormula()), recipient)
        except Exception as e:
            core.log('ERROR', '求数列({}) -> {}'.format(inp, e))
            itchat.send('{}额……你的数列好像出了点毛病'.format(atName), recipient)
    elif content[:3] in ('转繁体', '轉繁體'):
        original = content[3:].strip()
        itchat.send('{}繁体文字是：{}'.format(atName, zhconvert.toTraditional(original)), recipient)
    elif content[:3] in ('转简体', '轉簡體'):
        original = content[3:].strip()
        itchat.send('{}简体文字是：{}'.format(atName, zhconvert.toSimplified(original)), recipient)
    elif content[:4] == '贴吧图片':
        itchat.send('{}稍等一下哈，我收集一下图片！'.format(atName), recipient)
        inp = content[5:]
        urlRe = re.compile(r'(http://)?tieba\.baidu\.com/p/[0-9]+')
        if urlRe.findall(inp):
            if inp[:7] != 'http://':
                inp = 'http://' + inp
            if '?' in inp:
                inp = urlRe.search(inp).group()
            # dirName, imgCount = spider.get_tieba_img(inp+'/')
            # for i in range(imgCount):
            #     itchat.send('@img@{}'.format('{}.jpg'.format(i)), msg['FromUserName'])
            title = spider.get_tieba_img(inp + '/')
            itchat.send('@fil@{}.zip'.format(title), recipient)
        itchat.send('{}你只能爬取第一页楼主发的图片，要爬取更多图片，请联系@熏鱼饺子\u2005进行账户升级'.format(atName), recipient)
        time.sleep(5)
        os.remove(title + '.zip')
        shutil.rmtree(title)
    elif content[:4] == '贴吧文字':
        itchat.send('{}稍等一下哈，我整合一下！'.format(atName), recipient)
        inp = content[5:]
        urlRe = re.compile(r'(http://)?tieba\.baidu\.com/p/[0-9]+')
        if urlRe.findall(inp):
            if inp[:7] != 'http://':
                inp = 'http://' + inp
            if '?' in inp:
                inp = urlRe.search(inp).group()
            title = spider.get_tieba_text(inp + '/')
            itchat.send('@fil@{}'.format(title), recipient)
        itchat.send('{}你只能爬取第一页楼主发的文字，要爬取更多内容，请联系@熏鱼饺子\u2005进行账户升级'.format(atName), recipient)
        time.sleep(5)
        os.remove(title)
    elif '帮助' in content or ('你' in content and '除了' in content and '还' in content) or ('你' in content and ('会' in content or '能' in content) and ('干' in content or '做' in content)):
        itchat.send('{}{}'.format(atName, HELP), recipient)
    elif '几点' in content or ('现在' in content and '时间' in content):
        currentTime = time.strftime('%Y{}%m{}%d{}{}{} %H:%M:%S').format('年','月','日','星期',WEEKDAYS[time.strftime('%w')])
        itchat.send('{}现在是{}'.format(atName, currentTime), recipient)
    elif '笑话' in content and ('讲' in content or '说' in content):
        itchat.send('{}你等下哈，我编一个……'.format(atName), recipient)
        itchat.send('{}{}'.format(atName, spider.get_joke()), recipient)
    elif '新闻' in content:
        itchat.send('{}好，稍等一下'.format(atName), recipient)
        itchat.send('{}\n{}'.format(atName, spider.get_news()), recipient)
    elif '热点' in content:
        itchat.send('{}好，稍等一下'.format(atName), recipient)
        itchat.send('{}\n{}'.format(atName, spider.get_hot()), recipient)
    elif '说' in content:
        words = '说'.join(content.split('说')[1:]).strip()
        itchat.send('{}'.format(words), recipient)
    else:
        for i in ('嘿','你好','嗨','hi','hello','hey','bonjour','bonsoir'):
            if i in content.lower():
                itchat.send('{}{}'.format (atName, choice(GREETINGS)), msg['FromUserName'])
                break
        else:
            itchat.send('{}{}'.format(atName, choice(DONT_UNDERSTAND)), msg['FromUserName'])

@itchat.msg_register(['Picture', 'Recording', 'Attachment', 'Video'])
def download_files(msg):
    '''处理私信收到的附件'''
    # msg['Text'](msg['FileName'])
    itchat.send('@%s@%s'%('img' if msg['Type'] == 'Picture' else 'fil', 'test.png'), msg['FromUserName'])
    return '%s received'%msg['Type']

@itchat.msg_register('Text', isGroupChat=True)
def text_reply(msg):
    '''处理群聊中收到的信息'''
    if msg.get('isAt') or '@{}'.format(USERNAME) in msg['Content'] or '@{} '.format(USERNAME) in msg['Content']: # 如果@了我
        parse_msg(msg)

@itchat.msg_register('Recording', isGroupChat=True)
def recording_reply(msg):
    '''处理群聊中收到的语音'''
    curdir = os.path.dirname(os.path.abspath(__file__))
    fileName = curdir + '/packages/tmp/' + msg['FileName']
    msg['Text'](fileName)
    genFile = audio.generate('你说的是' + audio.recognize(fileName), voice=choice([0,1,3,4]))
    itchat.send('@fil@{}'.format(genFile), msg['FromUserName'])
    os.remove(genFile)

try:
    itchat.run()
except KeyboardInterrupt:
    print('Bye')
<commit_msg>Update Dumpling-Bot to Alpha 1.2.0<commit_after>import itchat
import time
import os
import re
import shutil
from random import choice

from packages import core
from packages import spider, pymath, zhconvert, audio, users, miscellaneous # 我的一些功能

itchat.auto_login(hotReload=True, enableCmdQR=2) # 登录用户账号以便获取昵称

USERNAME = itchat.search_friends()['NickName'] # 获取自己的昵称
GREETINGS = ('嘿','嗨，我是饺子机器人，有何吩咐？','哟','你好啊','你好呀',':)')
DONT_UNDERSTAND = ('对不起我不明白诶……','抱歉我没听懂:(','不好意思没明白','Sorry没有听懂')
HELP = '我是一个功能性机器人哦，聊天不太在行，但是可以干很多事！'
WEEKDAYS = {'0':'天','1':'一','2':'二','3':'三','4':'四','5':'五','6':'六'}
PAY_URL = 'https://yu-george.github.io/bot/upgrade/'

def parse_msg(msg):
    '''处理并对信息作相对应的处理。没有返回。'''
    print(msg)
    username = msg['ActualUserName']
    if not users.exist(username):
        users.add(username)
    atName = '@' + msg['ActualNickName'] + '\u2005' # @对方的的用户名
    recipient = msg['FromUserName'] # 群聊名称(消息接受方)
    content = msg['Content'].replace('@{}\u2005'.format(USERNAME),'').replace('@{} '.format(USERNAME),'').strip() # 清理消息文本
    if content in ('@{}'.format(USERNAME), ''): # 空信息
        itchat.send('{}有啥事？'.format(atName), recipient)
    elif content[:3] == '求数列':
        inp = content[3:].strip(' :：').replace('，',',').replace(' ','')
        try:
            seq = pymath.Sequence(inp)
            itchat.send('{}公式是：An = {}'.format(atName, seq.generateFormula()), recipient)
        except Exception as e:
            core.log('ERROR', '求数列({}) -> {}'.format(inp, e))
            itchat.send('{}额……你的数列好像出了点毛病'.format(atName), recipient)
    elif content[:4] == '三角函数':
        func = content.split()[1]
        val = content.split()[2]
        answer = pymath.trig.parseAnswer(pymath.trig.trig_val(func, int(val)))
        itchat.send('{}{}({}) = {}'.format(atName, func, val, answer), recipient)
    elif content[:3] in ('转繁体', '轉繁體'):
        original = content[3:].strip()
        itchat.send('{}繁体文字是：{}'.format(atName, zhconvert.toTraditional(original)), recipient)
    elif content[:3] in ('转简体', '轉簡體'):
        original = content[3:].strip()
        itchat.send('{}简体文字是：{}'.format(atName, zhconvert.toSimplified(original)), recipient)
    elif content[:2] == '加密':
        original = ' '.join(content.split()[1:])
        itchat.send('{}加密后的文字是：{}'.format(atName, miscellaneous.encrypt(original)), recipient)
    elif content[:2] == '解密':
        original = ' '.join(content.split()[1:])
        itchat.send('{}解密后的文字是：{}'.format(atName, miscellaneous.decrypt(original)), recipient)
    elif content[:2] == '网盘':
        itchat.send('{}稍等片刻……'.format(atName), recipient)
        keyword = ''.join(content.split()[1:])
        result = ''
        for url, password in spider.search_baidupan(keyword):
            if password:
                password = ' （密码：{}）'.format(password)
            result += '- {}{}\n'.format(url, password)
        itchat.send('{}\n{}'.format(atName, result), recipient)
    elif content[:4] == '贴吧图片':
        itchat.send('{}稍等一下哈，我收集一下图片！'.format(atName), recipient)
        inp = content[5:]
        urlRe = re.compile(r'(http://)?tieba\.baidu\.com/p/[0-9]+')
        if urlRe.findall(inp):
            if inp[:7] != 'http://':
                inp = 'http://' + inp
            if '?' in inp:
                inp = urlRe.search(inp).group()
            # dirName, imgCount = spider.get_tieba_img(inp+'/')
            # for i in range(imgCount):
            #     itchat.send('@img@{}'.format('{}.jpg'.format(i)), msg['FromUserName'])
            title = spider.get_tieba_img(inp + '/')
            itchat.send('@fil@{}.zip'.format(title), recipient)
        itchat.send('{}你只能爬取第一页楼主发的图片，要爬取更多图片，请前往{}进行账户升级'.format(atName, PAY_URL), recipient)
        time.sleep(5)
        os.remove(title + '.zip')
        shutil.rmtree(title)
    elif content[:4] == '贴吧文字':
        itchat.send('{}稍等一下哈，我整合一下！'.format(atName), recipient)
        inp = content[5:]
        urlRe = re.compile(r'(http://)?tieba\.baidu\.com/p/[0-9]+')
        if urlRe.findall(inp):
            if inp[:7] != 'http://':
                inp = 'http://' + inp
            if '?' in inp:
                inp = urlRe.search(inp).group()
            title = spider.get_tieba_text(inp + '/')
            itchat.send('@fil@{}'.format(title), recipient)
        itchat.send('{}你只能爬取第一页楼主发的文字，要爬取更多内容，请前往{}进行账户升级'.format(atName, PAY_URL), recipient)
        time.sleep(5)
        os.remove(title)
    elif content[:3] == '查等级':
        itchat.send('{}你目前是{}级'.format(atName, users.get_level(username)), recipient)
    elif content[:2] == '升级':
        users.upgrade(username)
        itchat.send('{}升级完毕！你目前是{}级（内测功能）'.format(atName, users.get_level(username)), recipient)
    elif content[:2] == '降级':
        users.degrade(username)
        itchat.send('{}降级完毕！你目前是{}级（内测功能）'.format(atName, users.get_level(username)), recipient)
    elif '帮助' in content or ('你' in content and '除了' in content and '还' in content) or ('你' in content and ('会' in content or '能' in content) and ('干' in content or '做' in content)):
        itchat.send('{}{}'.format(atName, HELP), recipient)
    elif '几点' in content or ('现在' in content and '时间' in content):
        currentTime = time.strftime('%Y{}%m{}%d{}{}{} %H:%M:%S').format('年','月','日','星期',WEEKDAYS[time.strftime('%w')])
        itchat.send('{}现在是{}'.format(atName, currentTime), recipient)
    elif '笑话' in content and ('讲' in content or '说' in content):
        itchat.send('{}你等下哈，我编一个……'.format(atName), recipient)
        itchat.send('{}{}'.format(atName, spider.get_joke()), recipient)
    elif '新闻' in content:
        itchat.send('{}好，稍等一下'.format(atName), recipient)
        itchat.send('{}\n{}'.format(atName, spider.get_news()), recipient)
    elif '热点' in content:
        itchat.send('{}好，稍等一下'.format(atName), recipient)
        itchat.send('{}\n{}'.format(atName, spider.get_hot()), recipient)
    elif '说' in content:
        words = '说'.join(content.split('说')[1:]).strip()
        itchat.send('{}'.format(words), recipient)
    else:
        for i in ('嘿','你好','嗨','hi','hello','hey','bonjour','bonsoir'):
            if i in content.lower():
                itchat.send('{}{}'.format (atName, choice(GREETINGS)), msg['FromUserName'])
                break
        else:
            itchat.send('{}{}'.format(atName, choice(DONT_UNDERSTAND)), msg['FromUserName'])

@itchat.msg_register(['Picture', 'Recording', 'Attachment', 'Video'])
def download_files(msg):
    '''处理私信收到的附件'''
    # msg['Text'](msg['FileName'])
    itchat.send('@%s@%s'%('img' if msg['Type'] == 'Picture' else 'fil', 'test.png'), msg['FromUserName'])
    return '%s received'%msg['Type']

@itchat.msg_register('Text')
def text_reply(msg):
    '''处理私聊中收到的信息'''
    print(msg)

@itchat.msg_register('Text', isGroupChat=True)
def text_reply(msg):
    '''处理群聊中收到的信息'''
    if msg.get('isAt') or '@{}'.format(USERNAME) in msg['Content'] or '@{} '.format(USERNAME) in msg['Content']: # 如果@了我
        parse_msg(msg)

@itchat.msg_register('Recording', isGroupChat=True)
def recording_reply(msg):
    '''处理群聊中收到的语音'''
    # print(msg)
    curdir = os.path.dirname(os.path.abspath(__file__))
    fileName = curdir + '/tmp/' + msg['FileName']
    msg['Text'](fileName)
    genFile = audio.generate('你说的是{}'.format(audio.recognize(fileName)), voice=choice([0,1,3,4]))
    itchat.send('@fil@{}'.format(genFile), msg['FromUserName'])
    os.remove(genFile)

myself = itchat.search_friends()['UserName']
if not users.exist(myself):
    users.add(myself, 3)

itchat.run(debug=True)

users._write_data()
print('终止饺子机器人')
<|endoftext|>


<commit_before>from collections import defaultdict, namedtuple
import six
import json
import ast

import taxcalc

from helpers import (INPUTS_META, BOOL_PARAMS, is_reverse, is_wildcard,
                     make_bool, convert_val, TRUE_REGEX, FALSE_REGEX)

MetaParam = namedtuple("MetaParam", ["param_name", "param_meta"])

def benefit_switch_fixup(fields, model, name="ID_BenefitSurtax_Switch"):
    """
    Take the incoming POST, the user reform, and the TaxSaveInputs
    model and fixup the switches _0, ..., _6 to one array of
    bools. Also set the model values correctly based on incoming
    POST
    """
    # Django forms needs switches to be True/False but in the interest of
    # ensuring that reforms created from a file or the GUI interface are the
    # same (down to the data type) the reform data are set to 1.0/0.0
    # _ids = [name + '_' + str(i) for i in range(7)]
    # values_from_model = [[fields[_id][0] for _id in _ids if _id in fields]]
    # final_values = [[True if _id in fields else False for (fields, _id) in zip(values_from_model[0], _ids)]]
    # for _id, val in zip(_ids, final_values[0]):
    #     fields[_id] = [1 if val else 0]
    #     setattr(model, _id, unicode(val))
    return fields

def amt_fixup(fields, model):
    """
    Take the regular tax captial gains parameters from the user reform
    and set them as the equivalent Alternative Minimum Tax capital
    gains parameters
    """
    cap_gains_params = ["CG_rt1", "CG_brk1_0", "CG_brk1_1",
                        "CG_brk1_2", "CG_brk1_3", "CG_brk1_cpi",
                        "CG_rt2", "CG_brk2_0", "CG_brk2_1",
                        "CG_brk2_2", "CG_brk2_3", "CG_brk2_cpi",
                        "CG_rt3"]

    for cgparam in cap_gains_params:
        if cgparam in fields:
            fields['AMT_' + cgparam] = fields[cgparam]
            if cgparam.endswith("_cpi"):
                setattr(model, 'AMT_' + cgparam, fields[cgparam])
            else:
                setattr(model, 'AMT_' + cgparam, fields[cgparam][0])

    return fields

def growth_fixup(mod):
    # if mod['growth_choice']:
    #     if mod['growth_choice'] == 'factor_adjustment':
    #         del mod['factor_target']
    #     if mod['growth_choice'] == 'factor_target':
    #         del mod['factor_adjustment']
    # else:
    #     if 'factor_adjustment' in mod:
    #         del mod['factor_adjustment']
    #     if 'factor_target' in mod:
    #         del mod['factor_target']
    #
    # del mod['growth_choice']

    return mod

def switch_fixup(fields, taxbrain_model):
    growth_fixup(fields)
    benefit_names = ["ID_BenefitSurtax_Switch", "ID_BenefitCap_Switch",
                     "ID_AmountCap_Switch"]
    for benefit_name in benefit_names:
        benefit_switch_fixup(fields, taxbrain_model, name=benefit_name)

    amt_fixup(fields, taxbrain_model)

def parse_value(value, meta_param):
    """
    Parse the value to the type that the upstream package specification
    requires, but we let the upstream package deal with creating error messages

    returns: parsed value
    """
    # expecting a string
    assert isinstance(value, six.string_types)

    value = value.strip()

    # if value is wildcard or reverse operator, then return it
    if is_wildcard(value) or is_reverse(value):
        return value

    # get type requirements from model specification
    boolean_value = meta_param.param_meta["boolean_value"]
    if boolean_value:
        integer_value = False
    else:
        integer_value = meta_param.param_meta["integer_value"]
    float_value = not(boolean_value or integer_value)

    # Try to parse case-insensitive True/False strings
    if value.title() == "True":
        prepped = "True"
    elif value.title() == "False":
        prepped = "False"
    else:
        # ast.literal_eval won't parse negative numbers
        negate = 1
        if value.rfind("-") == 0:
            negate = -1
            prepped = value[1:]
        else:
            prepped = value

    # Try to parse string and except ValueError if
    # value is not an integer, float, or boolean string
    try:
        parsed = ast.literal_eval(prepped)
    except ValueError:
        return parsed

    # Use information given to us by upstream specs to convert this value
    # into desired type or fail silently
    if isinstance(parsed, bool):
        return parsed
    elif isinstance(parsed, int):
        if boolean_value:
            return True if parsed else False
        elif float_value:
            return negate * float(parsed)
        else:
            return negate * parsed
    elif isinstance(parsed, float):
        if boolean_value:
            return True if parsed else False
        elif integer_value:
            # Don't want to lose info when we cast the float down to int
            # Note: 5.0 % 1.0 == 0.0 but 5.2 % 1.0 == 0.2
            if parsed % 1.0 > 0:
                return negate * parsed
            else:
                return negate * int(parsed)
        else:
            return negate * parsed
    else:
        return parsed

def parse_fields(param_dict, default_params):
    """
    Parses the raw GUI input into the correct types and maps the names to the
    appropriate default names

    returns: dictionary with parsed data
    """
    parsed = {}
    for k, v in param_dict.items():
        # user did not specify a value for this param
        if not v:
            continue

        # get upstream package parameter name and meta info
        meta_param = get_default_policy_param(k, default_params)
        values = []
        if meta_param.param_name.endswith("cpi"):
            assert len(v.split(',')) == 1
            values = bool(ast.literal_eval(v))
            assert isinstance(values, bool)
        else:
            for item in v.split(","):
                 values.append(parse_value(item, meta_param))
        parsed[meta_param.param_name] = values

    return parsed

def get_default_policy_param(param, default_params):
    """
    Map TaxBrain field name to Tax-Calculator parameter name
    For example: STD_0 maps to _STD_single

    returns: named tuple with taxcalc param name and metadata
    """
    if '_' + param in default_params: # ex. EITC_indiv --> _EITC_indiv
        return MetaParam('_' + param, default_params['_' + param])
    param_pieces = param.split('_')
    end_piece = param_pieces[-1]
    no_suffix = '_' + '_'.join(param_pieces[:-1])
    if end_piece == 'cpi': # ex. SS_Earnings_c_cpi --> _SS_Earnings_c_cpi
        if no_suffix in default_params:
            return MetaParam('_' + param, default_params[no_suffix])
        else:
            msg = "Received unexpected parameter: {}"
            raise ValueError(msg.format(param))
    if no_suffix in default_params: # ex. STD_0 --> _STD_single
        try:
            ix = int(end_piece)
        except ValueError:
            msg = "Parsing {}: Expected integer for index but got {}"
            raise ValueError(msg.format(param, end_piece))
        num_columns = len(default_params[no_suffix]['col_label'])
        if ix < 0 or ix >= num_columns:
            msg = "Parsing {}: Index {} not in range ({}, {})"
            raise IndexError(msg.format(param, ix, 0, num_columns))
        col_label = default_params[no_suffix]['col_label'][ix]
        return MetaParam(no_suffix + '_' + col_label, default_params[no_suffix])
    msg = "Received unexpected parameter: {}"
    raise ValueError(msg.format(param))

def to_json_reform(start_year, fields):
    """
    Convert fields style dictionary to json reform style dictionary
    For example:
    start_year = 2017, cls = taxcalc.Policy
    fields = {'_CG_nodiff': [False]},
              '_FICA_ss_trt': ["*", 0.1, "*", 0.2],
              '_ID_Charity_c_cpi': True,
              '_EITC_rt_2kids': [1.0]}
    to
    reform = {'_CG_nodiff': {'2017': [False]},
              '_FICA_ss_trt': {'2020': [0.2], '2018': [0.1]},
              '_ID_Charity_c_cpi': {'2017': True},
              '_EITC_rt_2kids': {'2017': [1.0]}}

    returns: json style reform
    """
    number_reverse_operators = 1
    ignore = INPUTS_META

    reform = {}
    for param in fields:
        reform[param] = {}
        if not isinstance(fields[param], list):
            assert isinstance(fields[param], bool) and param.endswith('_cpi')
            reform[param][str(start_year)] = fields[param]
            continue
        i = 0
        while i < len(fields[param]):
            if is_wildcard(fields[param][i]):
                # may need to do something here
                pass
            elif is_reverse(fields[param][i]):
                # only the first character can be a reverse char
                # and there must be a following character
                assert len(fields[param]) > 1
                # set value for parameter in start_year - 1
                assert (isinstance(fields[param][i + 1], (int, float)) or
                        isinstance(fields[param][i + 1], bool))
                reform[param][str(start_year - 1)] = \
                    [fields[param][i + 1]]

                # realign year and parameter indices
                for op in (0, number_reverse_operators + 1):
                    fields[param].pop(0)
                continue
            else:
                assert (isinstance(fields[param][i], (int, float)) or
                        isinstance(fields[param][i], bool))
                reform[param][str(start_year + i)] = \
                    [fields[param][i]]

            i += 1

    return reform

def append_errors_warnings(errors_warnings, append_func):
    """
    Appends warning/error messages to some object, append_obj, according to
    the provided function, append_func
    """
    for action in ['warnings', 'errors']:
        for param in errors_warnings[action]:
            for year in sorted(
                errors_warnings[action][param].keys(),
                key=lambda x: int(x)
            ):
                msg = errors_warnings[action][param][year]
                append_func(param, msg)

def parse_errors_warnings(errors_warnings):
    """
    Parse error messages so that they can be mapped to Taxbrain param ID. This
    allows the messages to be displayed under the field where the value is
    entered.

    returns: dictionary 'parsed' with keys: 'errors' and 'warnings'
        parsed['errors/warnings'] = {year: {tb_param_name: 'error message'}}
    """
    parsed = {'errors': defaultdict(dict), 'warnings': defaultdict(dict)}
    for action in errors_warnings:
        msgs = errors_warnings[action]
        if len(msgs) == 0:
            continue
        for msg in msgs.split('\n'):
            if len(msg) == 0: # new line
                continue
            msg_spl = msg.split()
            msg_action = msg_spl[0]
            year = msg_spl[1]
            curr_id = msg_spl[2][1:]
            msg_parse = msg_spl[2:]
            parsed[action][curr_id][year] = ' '.join([msg_action] + msg_parse +
                                                         ['for', year])

    return parsed

def read_json_reform(reform, assumptions):
    """
    Read reform and parse errors

    returns reform and assumption dictionaries that are compatible with
            taxcalc.Policy.implement_reform
            parsed warning and error messsages to be displayed on input page
            if necessary
    """
    policy_dict = taxcalc.Calculator.read_json_param_objects(
        reform,
        assumptions,
    )
    # get errors and warnings on parameters that do not cause ValueErrors
    errors_warnings = taxcalc.tbi.reform_warnings_errors(policy_dict)
    errors_warnings = parse_errors_warnings(errors_warnings)
    # separate reform and assumptions
    reform_dict = policy_dict["policy"]
    assumptions_dict = {k: v for k, v in policy_dict.items() if k != "policy"}

    return reform_dict, assumptions_dict, errors_warnings

def get_reform_from_file(request_files, reform_text=None,
                         assumptions_text=None):
    """
    Parse files from request object and collect errors_warnings

    returns reform and assumptions dictionaries that are compatible with
            taxcalc.Policy.implement_reform
            raw reform and assumptions text
            parsed warning and error messsages to be displayed on input page
            if necessary
    """
    if "docfile" in request_files:
        inmemfile_reform = request_files['docfile']
        reform_text = inmemfile_reform.read()
    if 'assumpfile' in request_files:
        inmemfile_assumption = request_files['assumpfile']
        assumptions_text = inmemfile_assumption.read()

    (reform_dict, assumptions_dict,
        errors_warnings) = read_json_reform(reform_text,
                                            assumptions_text)

    assumptions_text = assumptions_text or ""

    return (reform_dict, assumptions_dict, reform_text, assumptions_text,
            errors_warnings)

def get_reform_from_gui(start_year, taxbrain_fields=None, behavior_fields=None,
                        stored_errors=None):
    """
    Parse request and model objects and collect reforms and warnings
    This function is also called by dynamic/views.behavior_model.  In the
    future, this could be used as a generic GUI parameter parsing function.

    returns reform and assumptions dictionaries that are compatible with
            taxcalc.Policy.implement_reform
            raw reform and assumptions text
            parsed warning and error messsages to be displayed on input page
            if necessary
    """
    if taxbrain_fields is None and behavior_fields is None:
        raise ValueError("Neither taxbrain data nor behavior data was given")

    policy_dict_json = {}
    assumptions_dict_json = {}

    # prepare taxcalc params from TaxSaveInputs model
    if taxbrain_fields is not None:
        # convert GUI input to json style taxcalc reform
        policy_dict_json = to_json_reform(int(start_year),
                                          taxbrain_fields)
    if behavior_fields is not None:
        assumptions_dict_json = to_json_reform(int(start_year),
                                               behavior_fields)

    policy_dict_json = {"policy": policy_dict_json}

    policy_dict_json = json.dumps(policy_dict_json)

    assumptions_dict_json = {"behavior": assumptions_dict_json,
                             "growdiff_response": {},
                             "consumption": {},
                             "growdiff_baseline": {}}
    assumptions_dict_json = json.dumps(assumptions_dict_json)

    (reform_dict, assumptions_dict,
        errors_warnings) = read_json_reform(policy_dict_json,
                                            assumptions_dict_json)

    return (reform_dict, assumptions_dict, "", "", errors_warnings)
<commit_msg>Turns out ast.literal_eval does work for negative numbers<commit_after>from collections import defaultdict, namedtuple
import six
import json
import ast

import taxcalc

from helpers import (INPUTS_META, BOOL_PARAMS, is_reverse, is_wildcard,
                     make_bool, convert_val, TRUE_REGEX, FALSE_REGEX)

MetaParam = namedtuple("MetaParam", ["param_name", "param_meta"])

def benefit_switch_fixup(fields, model, name="ID_BenefitSurtax_Switch"):
    """
    Take the incoming POST, the user reform, and the TaxSaveInputs
    model and fixup the switches _0, ..., _6 to one array of
    bools. Also set the model values correctly based on incoming
    POST
    """
    # Django forms needs switches to be True/False but in the interest of
    # ensuring that reforms created from a file or the GUI interface are the
    # same (down to the data type) the reform data are set to 1.0/0.0
    # _ids = [name + '_' + str(i) for i in range(7)]
    # values_from_model = [[fields[_id][0] for _id in _ids if _id in fields]]
    # final_values = [[True if _id in fields else False for (fields, _id) in zip(values_from_model[0], _ids)]]
    # for _id, val in zip(_ids, final_values[0]):
    #     fields[_id] = [1 if val else 0]
    #     setattr(model, _id, unicode(val))
    return fields

def amt_fixup(fields, model):
    """
    Take the regular tax captial gains parameters from the user reform
    and set them as the equivalent Alternative Minimum Tax capital
    gains parameters
    """
    cap_gains_params = ["CG_rt1", "CG_brk1_0", "CG_brk1_1",
                        "CG_brk1_2", "CG_brk1_3", "CG_brk1_cpi",
                        "CG_rt2", "CG_brk2_0", "CG_brk2_1",
                        "CG_brk2_2", "CG_brk2_3", "CG_brk2_cpi",
                        "CG_rt3"]

    for cgparam in cap_gains_params:
        if cgparam in fields:
            fields['AMT_' + cgparam] = fields[cgparam]
            if cgparam.endswith("_cpi"):
                setattr(model, 'AMT_' + cgparam, fields[cgparam])
            else:
                setattr(model, 'AMT_' + cgparam, fields[cgparam][0])

    return fields

def growth_fixup(mod):
    # if mod['growth_choice']:
    #     if mod['growth_choice'] == 'factor_adjustment':
    #         del mod['factor_target']
    #     if mod['growth_choice'] == 'factor_target':
    #         del mod['factor_adjustment']
    # else:
    #     if 'factor_adjustment' in mod:
    #         del mod['factor_adjustment']
    #     if 'factor_target' in mod:
    #         del mod['factor_target']
    #
    # del mod['growth_choice']

    return mod

def switch_fixup(fields, taxbrain_model):
    growth_fixup(fields)
    benefit_names = ["ID_BenefitSurtax_Switch", "ID_BenefitCap_Switch",
                     "ID_AmountCap_Switch"]
    for benefit_name in benefit_names:
        benefit_switch_fixup(fields, taxbrain_model, name=benefit_name)

    amt_fixup(fields, taxbrain_model)

def parse_value(value, meta_param):
    """
    Parse the value to the type that the upstream package specification
    requires, but we let the upstream package deal with creating error messages

    returns: parsed value
    """
    # expecting a string
    assert isinstance(value, six.string_types)

    value = value.strip()

    # if value is wildcard or reverse operator, then return it
    if is_wildcard(value) or is_reverse(value):
        return value

    # get type requirements from model specification
    boolean_value = meta_param.param_meta["boolean_value"]
    if boolean_value:
        integer_value = False
    else:
        integer_value = meta_param.param_meta["integer_value"]
    float_value = not(boolean_value or integer_value)

    # Try to parse case-insensitive True/False strings
    if value.title() == "True":
        prepped = "True"
    elif value.title() == "False":
        prepped = "False"
    else:
        prepped = value

    # Try to parse string and except ValueError if
    # value is not an integer, float, or boolean string
    try:
        parsed = ast.literal_eval(prepped)
    except ValueError:
        return parsed

    # Use information given to us by upstream specs to convert this value
    # into desired type or fail silently
    if isinstance(parsed, bool):
        return parsed
    elif isinstance(parsed, int):
        if boolean_value:
            return True if parsed else False
        elif float_value:
            return float(parsed)
        else:
            return parsed
    elif isinstance(parsed, float):
        if boolean_value:
            return True if parsed else False
        elif integer_value:
            # Don't want to lose info when we cast the float down to int
            # Note: 5.0 % 1.0 == 0.0 but 5.2 % 1.0 == 0.2
            if parsed % 1.0 > 0:
                return parsed
            else:
                return int(parsed)
        else:
            return parsed
    else:
        return parsed

def parse_fields(param_dict, default_params):
    """
    Parses the raw GUI input into the correct types and maps the names to the
    appropriate default names

    returns: dictionary with parsed data
    """
    parsed = {}
    for k, v in param_dict.items():
        # user did not specify a value for this param
        if not v:
            continue

        # get upstream package parameter name and meta info
        meta_param = get_default_policy_param(k, default_params)
        values = []
        if meta_param.param_name.endswith("cpi"):
            assert len(v.split(',')) == 1
            values = bool(ast.literal_eval(v))
            assert isinstance(values, bool)
        else:
            for item in v.split(","):
                 values.append(parse_value(item, meta_param))
        parsed[meta_param.param_name] = values

    return parsed

def get_default_policy_param(param, default_params):
    """
    Map TaxBrain field name to Tax-Calculator parameter name
    For example: STD_0 maps to _STD_single

    returns: named tuple with taxcalc param name and metadata
    """
    if '_' + param in default_params: # ex. EITC_indiv --> _EITC_indiv
        return MetaParam('_' + param, default_params['_' + param])
    param_pieces = param.split('_')
    end_piece = param_pieces[-1]
    no_suffix = '_' + '_'.join(param_pieces[:-1])
    if end_piece == 'cpi': # ex. SS_Earnings_c_cpi --> _SS_Earnings_c_cpi
        if no_suffix in default_params:
            return MetaParam('_' + param, default_params[no_suffix])
        else:
            msg = "Received unexpected parameter: {}"
            raise ValueError(msg.format(param))
    if no_suffix in default_params: # ex. STD_0 --> _STD_single
        try:
            ix = int(end_piece)
        except ValueError:
            msg = "Parsing {}: Expected integer for index but got {}"
            raise ValueError(msg.format(param, end_piece))
        num_columns = len(default_params[no_suffix]['col_label'])
        if ix < 0 or ix >= num_columns:
            msg = "Parsing {}: Index {} not in range ({}, {})"
            raise IndexError(msg.format(param, ix, 0, num_columns))
        col_label = default_params[no_suffix]['col_label'][ix]
        return MetaParam(no_suffix + '_' + col_label, default_params[no_suffix])
    msg = "Received unexpected parameter: {}"
    raise ValueError(msg.format(param))

def to_json_reform(start_year, fields):
    """
    Convert fields style dictionary to json reform style dictionary
    For example:
    start_year = 2017, cls = taxcalc.Policy
    fields = {'_CG_nodiff': [False]},
              '_FICA_ss_trt': ["*", 0.1, "*", 0.2],
              '_ID_Charity_c_cpi': True,
              '_EITC_rt_2kids': [1.0]}
    to
    reform = {'_CG_nodiff': {'2017': [False]},
              '_FICA_ss_trt': {'2020': [0.2], '2018': [0.1]},
              '_ID_Charity_c_cpi': {'2017': True},
              '_EITC_rt_2kids': {'2017': [1.0]}}

    returns: json style reform
    """
    number_reverse_operators = 1
    ignore = INPUTS_META

    reform = {}
    for param in fields:
        reform[param] = {}
        if not isinstance(fields[param], list):
            assert isinstance(fields[param], bool) and param.endswith('_cpi')
            reform[param][str(start_year)] = fields[param]
            continue
        i = 0
        while i < len(fields[param]):
            if is_wildcard(fields[param][i]):
                # may need to do something here
                pass
            elif is_reverse(fields[param][i]):
                # only the first character can be a reverse char
                # and there must be a following character
                assert len(fields[param]) > 1
                # set value for parameter in start_year - 1
                assert (isinstance(fields[param][i + 1], (int, float)) or
                        isinstance(fields[param][i + 1], bool))
                reform[param][str(start_year - 1)] = \
                    [fields[param][i + 1]]

                # realign year and parameter indices
                for op in (0, number_reverse_operators + 1):
                    fields[param].pop(0)
                continue
            else:
                assert (isinstance(fields[param][i], (int, float)) or
                        isinstance(fields[param][i], bool))
                reform[param][str(start_year + i)] = \
                    [fields[param][i]]

            i += 1

    return reform

def append_errors_warnings(errors_warnings, append_func):
    """
    Appends warning/error messages to some object, append_obj, according to
    the provided function, append_func
    """
    for action in ['warnings', 'errors']:
        for param in errors_warnings[action]:
            for year in sorted(
                errors_warnings[action][param].keys(),
                key=lambda x: int(x)
            ):
                msg = errors_warnings[action][param][year]
                append_func(param, msg)

def parse_errors_warnings(errors_warnings):
    """
    Parse error messages so that they can be mapped to Taxbrain param ID. This
    allows the messages to be displayed under the field where the value is
    entered.

    returns: dictionary 'parsed' with keys: 'errors' and 'warnings'
        parsed['errors/warnings'] = {year: {tb_param_name: 'error message'}}
    """
    parsed = {'errors': defaultdict(dict), 'warnings': defaultdict(dict)}
    for action in errors_warnings:
        msgs = errors_warnings[action]
        if len(msgs) == 0:
            continue
        for msg in msgs.split('\n'):
            if len(msg) == 0: # new line
                continue
            msg_spl = msg.split()
            msg_action = msg_spl[0]
            year = msg_spl[1]
            curr_id = msg_spl[2][1:]
            msg_parse = msg_spl[2:]
            parsed[action][curr_id][year] = ' '.join([msg_action] + msg_parse +
                                                         ['for', year])

    return parsed

def read_json_reform(reform, assumptions):
    """
    Read reform and parse errors

    returns reform and assumption dictionaries that are compatible with
            taxcalc.Policy.implement_reform
            parsed warning and error messsages to be displayed on input page
            if necessary
    """
    policy_dict = taxcalc.Calculator.read_json_param_objects(
        reform,
        assumptions,
    )
    # get errors and warnings on parameters that do not cause ValueErrors
    errors_warnings = taxcalc.tbi.reform_warnings_errors(policy_dict)
    errors_warnings = parse_errors_warnings(errors_warnings)
    # separate reform and assumptions
    reform_dict = policy_dict["policy"]
    assumptions_dict = {k: v for k, v in policy_dict.items() if k != "policy"}

    return reform_dict, assumptions_dict, errors_warnings

def get_reform_from_file(request_files, reform_text=None,
                         assumptions_text=None):
    """
    Parse files from request object and collect errors_warnings

    returns reform and assumptions dictionaries that are compatible with
            taxcalc.Policy.implement_reform
            raw reform and assumptions text
            parsed warning and error messsages to be displayed on input page
            if necessary
    """
    if "docfile" in request_files:
        inmemfile_reform = request_files['docfile']
        reform_text = inmemfile_reform.read()
    if 'assumpfile' in request_files:
        inmemfile_assumption = request_files['assumpfile']
        assumptions_text = inmemfile_assumption.read()

    (reform_dict, assumptions_dict,
        errors_warnings) = read_json_reform(reform_text,
                                            assumptions_text)

    assumptions_text = assumptions_text or ""

    return (reform_dict, assumptions_dict, reform_text, assumptions_text,
            errors_warnings)

def get_reform_from_gui(start_year, taxbrain_fields=None, behavior_fields=None,
                        stored_errors=None):
    """
    Parse request and model objects and collect reforms and warnings
    This function is also called by dynamic/views.behavior_model.  In the
    future, this could be used as a generic GUI parameter parsing function.

    returns reform and assumptions dictionaries that are compatible with
            taxcalc.Policy.implement_reform
            raw reform and assumptions text
            parsed warning and error messsages to be displayed on input page
            if necessary
    """
    if taxbrain_fields is None and behavior_fields is None:
        raise ValueError("Neither taxbrain data nor behavior data was given")

    policy_dict_json = {}
    assumptions_dict_json = {}

    # prepare taxcalc params from TaxSaveInputs model
    if taxbrain_fields is not None:
        # convert GUI input to json style taxcalc reform
        policy_dict_json = to_json_reform(int(start_year),
                                          taxbrain_fields)
    if behavior_fields is not None:
        assumptions_dict_json = to_json_reform(int(start_year),
                                               behavior_fields)

    policy_dict_json = {"policy": policy_dict_json}

    policy_dict_json = json.dumps(policy_dict_json)

    assumptions_dict_json = {"behavior": assumptions_dict_json,
                             "growdiff_response": {},
                             "consumption": {},
                             "growdiff_baseline": {}}
    assumptions_dict_json = json.dumps(assumptions_dict_json)

    (reform_dict, assumptions_dict,
        errors_warnings) = read_json_reform(policy_dict_json,
                                            assumptions_dict_json)

    return (reform_dict, assumptions_dict, "", "", errors_warnings)
<|endoftext|>


<commit_before>import urllib.request
import logging

def save_rss(feed, city, cursor):
    section, url = feed
    url = url % city

    logging.info('%s', city)
    logging.info('%s', section)
    logging.info('%s', url)

    listing = urllib.request.urlopen(url, timeout=60).read()

    cursor.execute("""INSERT INTO rss (section, url, raw, city) """
                   """VALUES (%s, %s, %s, %s) """
                   """RETURNING rss_id""",
                   (section, url, listing, city))

if __name__ == '__main__':
    import psycopg2
    import urllib.error
    import time

    try:
        from raven import Client
        from .sentry import DSN
        client = Client(DSN)
    except ImportError:
        pass

    from . import config

    logging.basicConfig(level=logging.INFO)

    db = psycopg2.connect(database="neighborhood")
    c = db.cursor()

    for city in config.cities :
        if city == "newyork" :
            feeds = config.ny_feeds
        else :
            feeds = config.std_feeds
        for feed in feeds:
            try:
                save_rss(feed, city, c)
            except urllib.error.HTTPError as e:
                logging.info(e)
                logging.info(url)
            except:
                client.captureException()
                raise

            db.commit()
            time.sleep(1)

    c.close()
    db.close()

<commit_msg>use requests library<commit_after>import requests
import logging

def save_rss(feed, city, cursor):
    section, url = feed
    url = url % city

    logging.info('%s', city)
    logging.info('%s', section)
    logging.info('%s', url)

    listing = requests.get(url).text

    cursor.execute("""INSERT INTO rss (section, url, raw, city) """
                   """VALUES (%s, %s, %s, %s) """
                   """RETURNING rss_id""",
                   (section, url, listing, city))

if __name__ == '__main__':
    import psycopg2
    import urllib.error
    import time

    try:
        from raven import Client
        from .sentry import DSN
        client = Client(DSN)
    except ImportError:
        pass

    from . import config

    logging.basicConfig(level=logging.INFO)

    db = psycopg2.connect(database="neighborhood")
    c = db.cursor()

    for city in config.cities :
        if city == "newyork" :
            feeds = config.ny_feeds
        else :
            feeds = config.std_feeds
        for feed in feeds:
            try:
                save_rss(feed, city, c)
            except requests.HTTPError as e:
                logging.info(e)
                logging.info(url)
            except:
                client.captureException()
                raise

            db.commit()
            time.sleep(1)

    c.close()
    db.close()

<|endoftext|>


<commit_before>import json

from django.contrib import messages
from django.contrib.auth.mixins import LoginRequiredMixin
from django.views.generic import ListView, FormView, View
from django.http import Http404, HttpResponseRedirect, HttpResponse, HttpResponseForbidden
from django.shortcuts import redirect
from django.db.models import Prefetch
from django.db import transaction
from django.utils.functional import cached_property
from django.core.exceptions import ValidationError
from django.contrib.auth import get_user_model
from django.conf import settings

from .models import Bug, Action, Comment
from .forms import FilterForm, PresetFilterForm
from .mutation import BuggyBugMutator
from .enums import State
from . import webhook

User = get_user_model()

class BugListView(LoginRequiredMixin, ListView):
    ORDER_FIELDS = {
        'number': 'id',
        'project': 'project__name',
        'bug': 'title',
        'modified': 'modified_at',
        'creator': 'created_by__name',
        'assigned_to': 'assigned_to__name',
        'state': 'state',
        'priority': 'priority',
    }

    queryset = Bug.objects.select_related(
        'project', 'created_by', 'assigned_to'
    ).order_by(
        '-modified_at'
    ).defer('fulltext')  # We don't use the column, so there's no need to detoast a long string.

    context_object_name = 'bugs'
    form_class = FilterForm

    def get_form_kwargs(self):
        return {
            'data': self.request.GET,
            'label_suffix': '',
        }

    def get_form(self):
        return self.form_class(**self.get_form_kwargs())

    def get(self, *args, **kwargs):
        self.form = self.get_form()
        return super().get(*args, **kwargs)

    def get_sort_links(self):
        sort_links = {}
        querydict = self.request.GET.copy()
        current_sort, desc = self.sort_type()
        for order_field in self.ORDER_FIELDS.keys():
            if 'desc' in querydict:
                del querydict['desc']
            if current_sort == order_field and not desc:
                querydict['desc'] = True
            querydict['sort'] = order_field
            sort_links[order_field] = '?{}'.format(querydict.urlencode())
        return sort_links

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['form'] = self.form
        context['preset_form'] = PresetFilterForm(label_suffix='')
        context['sort_links'] = self.get_sort_links()
        context['sort_by'], context['sort_desc'] = self.sort_type()
        return context

    def get_queryset(self):
        qs = super().get_queryset()
        if self.form.is_valid():
            qs = self.form.filter(qs)
            order_field, desc = self.sort_type()
            return qs.order_by(('-' if desc else '') + self.ORDER_FIELDS[order_field])
        else:
            return qs.none()

    def get_template_names(self):
        if self.request.META.get('HTTP_X_PJAX'):
            return ['buggy/_bug_list.html']
        else:
            return super().get_template_names()

    def sort_type(self):
        order_field = self.request.GET.get('sort')
        if order_field not in self.ORDER_FIELDS:
            return ('modified', True)
        else:
            return (order_field, bool(self.request.GET.get('desc')))

class BugMutationMixin(object):
    mutator_class = BuggyBugMutator

    @cached_property
    def state_machine(self):
        return self.mutator_class(self.request.user, self.object)

    def get_form_class(self):
        return self.state_machine.get_form_class()

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['actions'] = self.state_machine.get_actions()
        context['buggy_user_names'] = [
            user.get_short_name().lower() for user in User.objects.filter(is_active=True)
        ]
        context['buggy_open_bugs'] = [
            {
                'title': bug.title,
                'number': bug.number,
            } for bug in Bug.objects.exclude(state=State.CLOSED).defer('fulltext')
        ]
        return context

    def form_valid(self, form):
        try:
            action = self.state_machine.process_action(form.cleaned_data)
        except ValidationError as e:
            for error in e.error_list:
                form.add_error(None, e)
            return self.form_invalid(form)
        else:
            return HttpResponseRedirect(action.bug.get_absolute_url())

    def get_form_kwargs(self):
        kwargs = super().get_form_kwargs()
        kwargs['label_suffix'] = ''
        return kwargs

class BugDetailView(LoginRequiredMixin, BugMutationMixin, FormView):
    template_name = 'buggy/bug_detail.html'

    queryset = Bug.objects.all().prefetch_related(
        Prefetch('actions', queryset=Action.objects.select_related(
            'user', 'comment', 'setpriority', 'setassignment__assigned_to',
            'setstate', 'setproject', 'settitle',
        ).prefetch_related('attachments'))
    ).select_related(
        'created_by', 'assigned_to', 'project'
    )

    def get(self, request, *args, **kwargs):
        self.object = self.get_object()
        return super().get(request, *args, **kwargs)

    @transaction.atomic
    def post(self, request, *args, **kwargs):
        self.object = self.get_object()
        return super().post(request, *args, **kwargs)

    def get_object(self):
        try:
            if self.request.method == 'POST':
                # We'd like to just use select_for_update on the main queryset,
                # but the select_related does a left join. Postgres does not
                # support locking the outer side of an outer join. The SQL we
                # want is `SELECT ... FOR UPDATE OF buggy_bug`, which would
                # only lock the one table, but Django can't yet generate that
                # SQL: <https://code.djangoproject.com/ticket/28010>.
                # BBB: This extra query can be replaced with
                # select_for_update(of=('self',)) as soon as it's supported in
                # Django.
                Bug.objects.all().select_for_update().get_by_number(self.kwargs['bug_number'])
            return self.queryset.get_by_number(self.kwargs['bug_number'])
        except Bug.DoesNotExist as e:
            raise Http404(*e.args)

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['bug'] = self.object
        return context

class BugCreateView(BugMutationMixin, FormView):
    template_name = 'buggy/bug_create.html'
    object = None

class AddPresetView(LoginRequiredMixin, View):
    def post(self, request):
        data = request.POST.copy()
        data['user'] = request.user.id
        form = PresetFilterForm(data)
        if form.is_valid():
            form.save()
        else:
            messages.error(request, 'Preset names must be unique.')
        return redirect('buggy:bug_list')

class RemovePresetView(LoginRequiredMixin, View):
    def post(self, request, pk):
        request.user.presetfilter_set.filter(pk=pk).delete()
        return redirect('buggy:bug_list')

class MarkdownPreviewView(LoginRequiredMixin, View):
    def post(self, request):
        return HttpResponse(Comment(comment=request.POST.get('preview', '')).html)

class GitCommitWebhookView(View):
    def post(self, request):
        if settings.GIT_COMMIT_WEBHOOK_SECRET is None or webhook.validate_signature(
            settings.GIT_COMMIT_WEBHOOK_SECRET,
            request.body,
            request.META['HTTP_X_HUB_SIGNATURE'],
        ):
            data = json.loads(request.body.decode('utf-8'))
            for commit in data['commits']:
                webhook.process_commit(commit)
            return HttpResponse('', status=201)
        else:
            return HttpResponseForbidden('Signature does not match.')
<commit_msg>Fix pjax query param duplication<commit_after>import json

from django.contrib import messages
from django.contrib.auth.mixins import LoginRequiredMixin
from django.views.generic import ListView, FormView, View
from django.http import Http404, HttpResponseRedirect, HttpResponse, HttpResponseForbidden
from django.shortcuts import redirect
from django.db.models import Prefetch
from django.db import transaction
from django.utils.functional import cached_property
from django.core.exceptions import ValidationError
from django.contrib.auth import get_user_model
from django.conf import settings

from .models import Bug, Action, Comment
from .forms import FilterForm, PresetFilterForm
from .mutation import BuggyBugMutator
from .enums import State
from . import webhook

User = get_user_model()

class BugListView(LoginRequiredMixin, ListView):
    ORDER_FIELDS = {
        'number': 'id',
        'project': 'project__name',
        'bug': 'title',
        'modified': 'modified_at',
        'creator': 'created_by__name',
        'assigned_to': 'assigned_to__name',
        'state': 'state',
        'priority': 'priority',
    }

    queryset = Bug.objects.select_related(
        'project', 'created_by', 'assigned_to'
    ).order_by(
        '-modified_at'
    ).defer('fulltext')  # We don't use the column, so there's no need to detoast a long string.

    context_object_name = 'bugs'
    form_class = FilterForm

    def get_form_kwargs(self):
        return {
            'data': self.request.GET,
            'label_suffix': '',
        }

    def get_form(self):
        return self.form_class(**self.get_form_kwargs())

    def get(self, *args, **kwargs):
        self.form = self.get_form()
        return super().get(*args, **kwargs)

    def get_sort_links(self):
        sort_links = {}
        querydict = self.request.GET.copy()
        if '_pjax' in querydict:
            del querydict['_pjax']  # pjax adds this param for cache purposes.

        current_sort, desc = self.sort_type()
        for order_field in self.ORDER_FIELDS.keys():
            if 'desc' in querydict:
                del querydict['desc']
            if current_sort == order_field and not desc:
                querydict['desc'] = True
            querydict['sort'] = order_field
            sort_links[order_field] = '?{}'.format(querydict.urlencode())
        return sort_links

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['form'] = self.form
        context['preset_form'] = PresetFilterForm(label_suffix='')
        context['sort_links'] = self.get_sort_links()
        context['sort_by'], context['sort_desc'] = self.sort_type()
        return context

    def get_queryset(self):
        qs = super().get_queryset()
        if self.form.is_valid():
            qs = self.form.filter(qs)
            order_field, desc = self.sort_type()
            return qs.order_by(('-' if desc else '') + self.ORDER_FIELDS[order_field])
        else:
            return qs.none()

    def get_template_names(self):
        if self.request.META.get('HTTP_X_PJAX'):
            return ['buggy/_bug_list.html']
        else:
            return super().get_template_names()

    def sort_type(self):
        order_field = self.request.GET.get('sort')
        if order_field not in self.ORDER_FIELDS:
            return ('modified', True)
        else:
            return (order_field, bool(self.request.GET.get('desc')))

class BugMutationMixin(object):
    mutator_class = BuggyBugMutator

    @cached_property
    def state_machine(self):
        return self.mutator_class(self.request.user, self.object)

    def get_form_class(self):
        return self.state_machine.get_form_class()

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['actions'] = self.state_machine.get_actions()
        context['buggy_user_names'] = [
            user.get_short_name().lower() for user in User.objects.filter(is_active=True)
        ]
        context['buggy_open_bugs'] = [
            {
                'title': bug.title,
                'number': bug.number,
            } for bug in Bug.objects.exclude(state=State.CLOSED).defer('fulltext')
        ]
        return context

    def form_valid(self, form):
        try:
            action = self.state_machine.process_action(form.cleaned_data)
        except ValidationError as e:
            for error in e.error_list:
                form.add_error(None, e)
            return self.form_invalid(form)
        else:
            return HttpResponseRedirect(action.bug.get_absolute_url())

    def get_form_kwargs(self):
        kwargs = super().get_form_kwargs()
        kwargs['label_suffix'] = ''
        return kwargs

class BugDetailView(LoginRequiredMixin, BugMutationMixin, FormView):
    template_name = 'buggy/bug_detail.html'

    queryset = Bug.objects.all().prefetch_related(
        Prefetch('actions', queryset=Action.objects.select_related(
            'user', 'comment', 'setpriority', 'setassignment__assigned_to',
            'setstate', 'setproject', 'settitle',
        ).prefetch_related('attachments'))
    ).select_related(
        'created_by', 'assigned_to', 'project'
    )

    def get(self, request, *args, **kwargs):
        self.object = self.get_object()
        return super().get(request, *args, **kwargs)

    @transaction.atomic
    def post(self, request, *args, **kwargs):
        self.object = self.get_object()
        return super().post(request, *args, **kwargs)

    def get_object(self):
        try:
            if self.request.method == 'POST':
                # We'd like to just use select_for_update on the main queryset,
                # but the select_related does a left join. Postgres does not
                # support locking the outer side of an outer join. The SQL we
                # want is `SELECT ... FOR UPDATE OF buggy_bug`, which would
                # only lock the one table, but Django can't yet generate that
                # SQL: <https://code.djangoproject.com/ticket/28010>.
                # BBB: This extra query can be replaced with
                # select_for_update(of=('self',)) as soon as it's supported in
                # Django.
                Bug.objects.all().select_for_update().get_by_number(self.kwargs['bug_number'])
            return self.queryset.get_by_number(self.kwargs['bug_number'])
        except Bug.DoesNotExist as e:
            raise Http404(*e.args)

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['bug'] = self.object
        return context

class BugCreateView(BugMutationMixin, FormView):
    template_name = 'buggy/bug_create.html'
    object = None

class AddPresetView(LoginRequiredMixin, View):
    def post(self, request):
        data = request.POST.copy()
        data['user'] = request.user.id
        form = PresetFilterForm(data)
        if form.is_valid():
            form.save()
        else:
            messages.error(request, 'Preset names must be unique.')
        return redirect('buggy:bug_list')

class RemovePresetView(LoginRequiredMixin, View):
    def post(self, request, pk):
        request.user.presetfilter_set.filter(pk=pk).delete()
        return redirect('buggy:bug_list')

class MarkdownPreviewView(LoginRequiredMixin, View):
    def post(self, request):
        return HttpResponse(Comment(comment=request.POST.get('preview', '')).html)

class GitCommitWebhookView(View):
    def post(self, request):
        if settings.GIT_COMMIT_WEBHOOK_SECRET is None or webhook.validate_signature(
            settings.GIT_COMMIT_WEBHOOK_SECRET,
            request.body,
            request.META['HTTP_X_HUB_SIGNATURE'],
        ):
            data = json.loads(request.body.decode('utf-8'))
            for commit in data['commits']:
                webhook.process_commit(commit)
            return HttpResponse('', status=201)
        else:
            return HttpResponseForbidden('Signature does not match.')
<|endoftext|>