<commit_before>import re
from re import compile as rec

from tornado.escape import _unicode, xhtml_escape, url_escape
from misaka import HtmlRenderer, Markdown
import misaka as m

from bnw_web.linkshit import LinkParser, shittypes
from bnw_web.linkshit import _URL_RE, _USER_RE, _MSG_RE

#: Displaying thumbs allowed for the following hostings:
linkhostings = [
    (ur'(?i)http://rghost.ru/([0-9]+)', lambda m: 'http://rghost.ru/%s/thumb.png' % (m(1),)),
    (ur'(?i)http://imgur.com/([A-Za-z0-9]+)', lambda m: 'http://i.imgur.com/%ss.png' % (m(1),)),
    (ur'http://ompldr.org/v([A-Za-z0-9]+)(/.+)?', lambda m: 'http://ompldr.org/t%s' % (m(1),)),
    (ur'http://2-ch.ru/([a-z]+)/src/([0-9]+).(png|gif|jpg)', lambda m: 'http://2-ch.ru/%s/thumb/%ss.%s' % (m(1),m(2),m(3))),
    (ur'(?i)http://(.+.(?:png|gif|jpg|jpeg))', lambda m: 'http://fuck.blasux.ru/thumb?img=%s' % (url_escape(m(0)),)),
]
linkhostings = [(re.compile('^' + k + '$'), v, k) for (k, v) in linkhostings]

def thumbify(text, format='markdown'):
    """Format text and generate thumbs using available formatters.

    :param format: default: 'markdown'. Specify text formatter.
                   Variants: 'moinmoin', 'markdown' or None,
                   which fallbacks to markdown.

    Return: (formatted_text, thumbs)
    Thumbs: list of (original_file_url, thumb_url)
    """
    if format == 'moinmoin':
        return moinmoin(text)
    else:
        return markdown(text)

def linkify(text, format='markdown'):
    """Only do text formatting. See :py:meth:`thumbify`."""
    return thumbify(text, format=format)[0]

###
# New markdown formatter with some custom render rules.
###

def ignore_trailing_newlines(text):
    return text.rstrip('\n')

def msg_url(match):
    text = match.group(0)
    url_part = match.group(1).replace('/', '#')
    return '[{0}](/p/{1})'.format(text, url_part)

def get_thumbs(raw):
    # Don't forget about escaping
    text = xhtml_escape(raw)
    thumbs = []
    for match in _URL_RE.finditer(text):
        url = match.group(1)
        for regexp, handler, _ in linkhostings:
            m = regexp.match(url)
            if m:
                thumb = handler(m.group)
                thumbs.append((url, thumb))
                break
    return thumbs

class BnwRenderer(HtmlRenderer):
    """Wrapper around default misaka's renderer."""

    def preprocess(self, text):
        """Apply some additional BnW's rules."""
        text = _USER_RE.sub('[\g<0>](/u/\g<1>)', text)
        text = _MSG_RE.sub(msg_url, text)
        return text

    def block_quote(self, text):
        """Do some wakaba-like fixes.
        Through wakaba parses block quotes in something different
        manner (they are not stick together).
        """
        # TODO: Should we be more wakabic?
        text = ignore_trailing_newlines(text)
        text = '\n'.join('> '+s for s in text.split('\n'))
        return '
{0}
\n'.format(text)

    def header(self, text, level):
        """Fix odd newlines in default header render."""
        return '<h{0}>{1}</h{0}>'.format(level, text)

    def paragraph(self, text):
        """Use just newlines instead of paragraphs
        (becase we already in the
 tag).
        """
        return text + '\n\n'

    def image(self, link, title, alt_text):
        """Don't allow images (they could be big).
        Use simple links intead.
        """
        # Don't forget about escaping
        return '{0}'.format(xhtml_escape(link))

    def block_code(self, code, language):
        # Don't forget about escaping
        code = ignore_trailing_newlines(xhtml_escape(code))
        if language:
            language = xhtml_escape(language)
            klass = ' class="language-{0}"'.format(language)
        else:
            klass = ''
        return '
{1}
\n'.format(klass, code)

# Don't touch HTML_ESCAPE flag!
renderer = BnwRenderer(m.HTML_ESCAPE | m.HTML_SAFELINK)
markdown_parser = Markdown(
    renderer,
    m.EXT_NO_INTRA_EMPHASIS | m.EXT_AUTOLINK | m.EXT_FENCED_CODE)

def markdown(raw):
    raw = _unicode(raw)
    formatted_text = ignore_trailing_newlines(markdown_parser.render(raw))
    thumbs = get_thumbs(raw)
    return formatted_text, thumbs

###
# Simple MoinMoin-like formatter. Legacy.
###

bnwtypes = (
    ("emph", rec(ur'(?<!:)//'), lambda m: ()),
    ("strong", rec(ur'\*\*'), lambda m: ()),
    ("namedlink", rec(ur'''\[\[\s*(?P<link_target>.+?)\s*[|]\s*(?P<link_text>.+?)\s*]]'''), lambda m: (m.group('link_target'),m.group('link_text'))),
    ("source", rec(ur'''{{{(?:#!(\w+)\s+)?(.*?)}}}''', re.MULTILINE|re.DOTALL), lambda m: (m.group(1),m.group(2))),
)

formatting_tags = {
    'emph': ('', ''),
    'strong': ('', ''),
}

parser = LinkParser(types=bnwtypes + shittypes)

def moinmoin(text):
    text = _unicode(xhtml_escape(text))
    permitted_protocols = ["http", "https", "ftp", "git",
                           "gopher", "magnet", "mailto", "xmpp"]
    texta = []
    thumbs = []
    stack = []
    instrong = False
    inemph = False
    for m in parser.parse(text):
        if isinstance(m, tuple):
            if m[0] in ('url', 'namedlink'):
                # TODO: Move checking for permitted protocols
                # in linkshit module? Matching twice is bad.
                if m[0] == 'namedlink':
                    up = m[2].split(':', 1)
                    proto = up[0] if len(up) == 2 else None
                    url = m[2]
                else:
                    up = _URL_RE.match(m[2])
                    url = m[2] if up is None else up.group(1)
                    proto = None if up is None else up.group(2)
                if proto and proto not in permitted_protocols:
                    texta.append('%s' % (m[1],))
                else:
                    if not proto:
                        url = "http://" + url
                    for lh in linkhostings:
                        mn = lh[0].match(url)
                        if mn:
                            thumb = lh[1](mn.group)
                            thumbs.append((url, thumb))
                            break
                    texta.append('%s' % (url, m[3]))
            elif m[0] in formatting_tags.keys():
                tag = formatting_tags[m[0]]
                if not m[0] in stack:
                    texta.append(tag[0])
                    stack.insert(0, m[0])
                else:
                    tp = stack.index(m[0])
                    for uptag in stack[:tp + 1]:
                        texta.append(formatting_tags[uptag][1])
                    for uptag in reversed(stack[:tp]):
                        texta.append(formatting_tags[uptag][0])
                    del stack[tp]
            elif m[0] == 'msg':
                texta.append(
                    '%s' % (m[2].replace('/', '#'), m[1]))
            elif m[0] == 'user':
                texta.append('%s' % (m[2], m[1]))
            elif m[0] == 'source':
                cs = (' class="language-' + m[2] + '"') if m[2] else ''
                texta.append('
%s
' % (cs, m[3]))
            else:
                texta.append('%s' % (m[1], m[0]))
        else:
            texta.append(m)
    for i in stack:
        texta.append(formatting_tags[i][1])
    return ''.join(texta), thumbs
<commit_msg>YouTube thumbnails<commit_after>import re
from re import compile as rec

from tornado.escape import _unicode, xhtml_escape, url_escape
from misaka import HtmlRenderer, Markdown
import misaka as m

from bnw_web.linkshit import LinkParser, shittypes
from bnw_web.linkshit import _URL_RE, _USER_RE, _MSG_RE

#: Displaying thumbs allowed for the following hostings:
linkhostings = [
    (ur'(?i)http://rghost.ru/([0-9]+)', lambda m: 'http://rghost.ru/%s/thumb.png' % (m(1),)),
    (ur'(?i)http://imgur.com/([A-Za-z0-9]+)', lambda m: 'http://i.imgur.com/%ss.png' % (m(1),)),
    (ur'http://ompldr.org/v([A-Za-z0-9]+)(/.+)?', lambda m: 'http://ompldr.org/t%s' % (m(1),)),
    (ur'http://2-ch.ru/([a-z]+)/src/([0-9]+).(png|gif|jpg)', lambda m: 'http://2-ch.ru/%s/thumb/%ss.%s' % (m(1),m(2),m(3))),
    (ur'https?://(?:www\.)?youtube.com/watch\?(?:.+&)?v\=([A-Z0-9a-z_-]+)(?:&.+)?', lambda m: 'http://img.youtube.com/vi/%s/default.jpg' % (m(1),)),
    (ur'(?i)http://(.+.(?:png|gif|jpg|jpeg))', lambda m: 'http://fuck.blasux.ru/thumb?img=%s' % (url_escape(m(0)),)),
]
linkhostings = [(re.compile('^' + k + '$'), v, k) for (k, v) in linkhostings]

def thumbify(text, format='markdown'):
    """Format text and generate thumbs using available formatters.

    :param format: default: 'markdown'. Specify text formatter.
                   Variants: 'moinmoin', 'markdown' or None,
                   which fallbacks to markdown.

    Return: (formatted_text, thumbs)
    Thumbs: list of (original_file_url, thumb_url)
    """
    if format == 'moinmoin':
        return moinmoin(text)
    else:
        return markdown(text)

def linkify(text, format='markdown'):
    """Only do text formatting. See :py:meth:`thumbify`."""
    return thumbify(text, format=format)[0]

###
# New markdown formatter with some custom render rules.
###

def ignore_trailing_newlines(text):
    return text.rstrip('\n')

def msg_url(match):
    text = match.group(0)
    url_part = match.group(1).replace('/', '#')
    return '[{0}](/p/{1})'.format(text, url_part)

def get_thumbs(raw):
    # Don't forget about escaping
    text = xhtml_escape(raw)
    thumbs = []
    for match in _URL_RE.finditer(text):
        url = match.group(1)
        for regexp, handler, _ in linkhostings:
            m = regexp.match(url)
            if m:
                thumb = handler(m.group)
                thumbs.append((url, thumb))
                break
    return thumbs

class BnwRenderer(HtmlRenderer):
    """Wrapper around default misaka's renderer."""

    def preprocess(self, text):
        """Apply some additional BnW's rules."""
        text = _USER_RE.sub('[\g<0>](/u/\g<1>)', text)
        text = _MSG_RE.sub(msg_url, text)
        return text

    def block_quote(self, text):
        """Do some wakaba-like fixes.
        Through wakaba parses block quotes in something different
        manner (they are not stick together).
        """
        # TODO: Should we be more wakabic?
        text = ignore_trailing_newlines(text)
        text = '\n'.join('> '+s for s in text.split('\n'))
        return '
{0}
\n'.format(text)

    def header(self, text, level):
        """Fix odd newlines in default header render."""
        return '<h{0}>{1}</h{0}>'.format(level, text)

    def paragraph(self, text):
        """Use just newlines instead of paragraphs
        (becase we already in the
 tag).
        """
        return text + '\n\n'

    def image(self, link, title, alt_text):
        """Don't allow images (they could be big).
        Use simple links intead.
        """
        # Don't forget about escaping
        return '{0}'.format(xhtml_escape(link))

    def block_code(self, code, language):
        # Don't forget about escaping
        code = ignore_trailing_newlines(xhtml_escape(code))
        if language:
            language = xhtml_escape(language)
            klass = ' class="language-{0}"'.format(language)
        else:
            klass = ''
        return '
{1}
\n'.format(klass, code)

# Don't touch HTML_ESCAPE flag!
renderer = BnwRenderer(m.HTML_ESCAPE | m.HTML_SAFELINK)
markdown_parser = Markdown(
    renderer,
    m.EXT_NO_INTRA_EMPHASIS | m.EXT_AUTOLINK | m.EXT_FENCED_CODE)

def markdown(raw):
    raw = _unicode(raw)
    formatted_text = ignore_trailing_newlines(markdown_parser.render(raw))
    thumbs = get_thumbs(raw)
    return formatted_text, thumbs

###
# Simple MoinMoin-like formatter. Legacy.
###

bnwtypes = (
    ("emph", rec(ur'(?<!:)//'), lambda m: ()),
    ("strong", rec(ur'\*\*'), lambda m: ()),
    ("namedlink", rec(ur'''\[\[\s*(?P<link_target>.+?)\s*[|]\s*(?P<link_text>.+?)\s*]]'''), lambda m: (m.group('link_target'),m.group('link_text'))),
    ("source", rec(ur'''{{{(?:#!(\w+)\s+)?(.*?)}}}''', re.MULTILINE|re.DOTALL), lambda m: (m.group(1),m.group(2))),
)

formatting_tags = {
    'emph': ('', ''),
    'strong': ('', ''),
}

parser = LinkParser(types=bnwtypes + shittypes)

def moinmoin(text):
    text = _unicode(xhtml_escape(text))
    permitted_protocols = ["http", "https", "ftp", "git",
                           "gopher", "magnet", "mailto", "xmpp"]
    texta = []
    thumbs = []
    stack = []
    instrong = False
    inemph = False
    for m in parser.parse(text):
        if isinstance(m, tuple):
            if m[0] in ('url', 'namedlink'):
                # TODO: Move checking for permitted protocols
                # in linkshit module? Matching twice is bad.
                if m[0] == 'namedlink':
                    up = m[2].split(':', 1)
                    proto = up[0] if len(up) == 2 else None
                    url = m[2]
                else:
                    up = _URL_RE.match(m[2])
                    url = m[2] if up is None else up.group(1)
                    proto = None if up is None else up.group(2)
                if proto and proto not in permitted_protocols:
                    texta.append('%s' % (m[1],))
                else:
                    if not proto:
                        url = "http://" + url
                    for lh in linkhostings:
                        mn = lh[0].match(url)
                        if mn:
                            thumb = lh[1](mn.group)
                            thumbs.append((url, thumb))
                            break
                    texta.append('%s' % (url, m[3]))
            elif m[0] in formatting_tags.keys():
                tag = formatting_tags[m[0]]
                if not m[0] in stack:
                    texta.append(tag[0])
                    stack.insert(0, m[0])
                else:
                    tp = stack.index(m[0])
                    for uptag in stack[:tp + 1]:
                        texta.append(formatting_tags[uptag][1])
                    for uptag in reversed(stack[:tp]):
                        texta.append(formatting_tags[uptag][0])
                    del stack[tp]
            elif m[0] == 'msg':
                texta.append(
                    '%s' % (m[2].replace('/', '#'), m[1]))
            elif m[0] == 'user':
                texta.append('%s' % (m[2], m[1]))
            elif m[0] == 'source':
                cs = (' class="language-' + m[2] + '"') if m[2] else ''
                texta.append('
%s
' % (cs, m[3]))
            else:
                texta.append('%s' % (m[1], m[0]))
        else:
            texta.append(m)
    for i in stack:
        texta.append(formatting_tags[i][1])
    return ''.join(texta), thumbs
<|endoftext|>


<commit_before>import io
import pandas as pd
from operator import attrgetter

def _generate_csv_header_line(*, header_names, header_prefix='', header=True, sep=',', newline='\n'):
    """
    Helper function to generate a CSV header line depending on
    the combination of arguments provided.
    """
    if isinstance(header, str):  # user-provided header line
        header_line = header + newline
    else:
        if not (header is None or isinstance(header, bool)):
            raise ValueError(f"Invalid value for argument `header`: {header}")
        else:
            if header:
                header_line = header_prefix + sep.join(header_names) + newline
            else:
                header_line = ""
    return header_line

class ItemList:

    def __init__(self, items, N):
        self.items = items if isinstance(items, list) else list(items)
        self.N = N

    def __repr__(self):
        return f"<ItemList containing {self.N} items>"

    def __eq__(self, other):
        return self.items == other

    def __len__(self):
        return self.N

    def __getitem__(self, idx):
        return self.items[idx]

    def __iter__(self):
        return iter(self.items)

    def to_df(self, fields=None):
        if fields is None:
            return pd.DataFrame([x.to_series() for x in self.items])
        else:
            attr_getters = [(field_name, attrgetter(attr_name)) for (field_name, attr_name) in fields.items()]
            return pd.DataFrame([pd.Series({field_name: func(x) for (field_name, func) in attr_getters}) for x in self.items])

    def to_csv(self, filename=None, *, fields=None, append=False, header=True, header_prefix='', sep=',', newline='\n'):
        """
        Parameters
        ----------
        filename: str or None
            The file to which output will be written. By default, any existing content is
            overwritten. Use `append=True` to open the file in append mode instead.
            If filename is None, the generated CSV output is returned instead of written
            to a file.
        fields: dict
            Dictionary which maps output field names to attribute names of the generators.
            Example: `fields={'CSV_COL1': 'field_name_1', 'CSV_COL2': 'field_name_2'}
        append: bool
            If `True`, open the file in 'append' mode to avoid overwriting existing content.
            Default is `False`, i.e. any existing content will be overwritten.
            This argument only has an effect if `filename` is given (i.e. if output happens
            to a file instead of returning a CSV string).
        header: bool or str or None
            If `header=False` or `header=None` then no header line will be written.
            If `header` is a string then this string will be used as the header line.
            If `header=True` then a header line will be automatically generated from
            the field names of the custom generator.
        header_prefix: str
            If `header=True` then the auto-generated header line will be prefixed
            with `header_prefix` (otherwise this argument has no effect). For example,
            set `header_prefix='#'` to make the header line start with '#'. Default: ''
        sep: str
            Field separator to use in the output. Default: ','
        newline: str
            Line terminator to use in the output. Default: '\n'

        Returns
        -------
        The return value depends on the value of `filename`.
        If `filename` is given, writes the output to the file and returns `None`.
        If `filename` is `None`, returns a string containing the CSV output.
        """
        assert isinstance(append, bool)

        if fields is None:
            raise NotImplementedError("TODO: derive field names automatically from the generator which produced this item list")

        header_line = _generate_csv_header_line(header=header, header_prefix=header_prefix, header_names=fields.keys(), sep=sep, newline=newline)

        file_or_string = open(filename, 'a' if append else 'w') if (filename is not None) else io.StringIO()

        retval = None
        attr_getters = [attrgetter(attr_name) for attr_name in fields.values()]
        try:

            file_or_string.write(header_line)

            for x in self.items:
                line = sep.join([format(func(x)) for func in attr_getters]) + newline
                file_or_string.write(line)

            if filename is None:
                retval = file_or_string.getvalue()

        finally:
            file_or_string.close()

        return retval
<commit_msg>Support exporting ItemList to Postgres<commit_after>import io
import pandas as pd
from operator import attrgetter
from sqlalchemy import create_engine

def _generate_csv_header_line(*, header_names, header_prefix='', header=True, sep=',', newline='\n'):
    """
    Helper function to generate a CSV header line depending on
    the combination of arguments provided.
    """
    if isinstance(header, str):  # user-provided header line
        header_line = header + newline
    else:
        if not (header is None or isinstance(header, bool)):
            raise ValueError(f"Invalid value for argument `header`: {header}")
        else:
            if header:
                header_line = header_prefix + sep.join(header_names) + newline
            else:
                header_line = ""
    return header_line

class ItemList:

    def __init__(self, items, N):
        self.items = items if isinstance(items, list) else list(items)
        self.N = N

    def __repr__(self):
        return f"<ItemList containing {self.N} items>"

    def __eq__(self, other):
        return self.items == other

    def __len__(self):
        return self.N

    def __getitem__(self, idx):
        return self.items[idx]

    def __iter__(self):
        return iter(self.items)

    def to_df(self, fields=None):
        if fields is None:
            return pd.DataFrame([x.to_series() for x in self.items])
        else:
            attr_getters = [(field_name, attrgetter(attr_name)) for (field_name, attr_name) in fields.items()]
            return pd.DataFrame([pd.Series({field_name: func(x) for (field_name, func) in attr_getters}) for x in self.items])

    def to_csv(self, filename=None, *, fields=None, append=False, header=True, header_prefix='', sep=',', newline='\n'):
        """
        Parameters
        ----------
        filename: str or None
            The file to which output will be written. By default, any existing content is
            overwritten. Use `append=True` to open the file in append mode instead.
            If filename is None, the generated CSV output is returned instead of written
            to a file.
        fields: dict
            Dictionary which maps output field names to attribute names of the generators.
            Example: `fields={'CSV_COL1': 'field_name_1', 'CSV_COL2': 'field_name_2'}
        append: bool
            If `True`, open the file in 'append' mode to avoid overwriting existing content.
            Default is `False`, i.e. any existing content will be overwritten.
            This argument only has an effect if `filename` is given (i.e. if output happens
            to a file instead of returning a CSV string).
        header: bool or str or None
            If `header=False` or `header=None` then no header line will be written.
            If `header` is a string then this string will be used as the header line.
            If `header=True` then a header line will be automatically generated from
            the field names of the custom generator.
        header_prefix: str
            If `header=True` then the auto-generated header line will be prefixed
            with `header_prefix` (otherwise this argument has no effect). For example,
            set `header_prefix='#'` to make the header line start with '#'. Default: ''
        sep: str
            Field separator to use in the output. Default: ','
        newline: str
            Line terminator to use in the output. Default: '\n'

        Returns
        -------
        The return value depends on the value of `filename`.
        If `filename` is given, writes the output to the file and returns `None`.
        If `filename` is `None`, returns a string containing the CSV output.
        """
        assert isinstance(append, bool)

        if fields is None:
            raise NotImplementedError("TODO: derive field names automatically from the generator which produced this item list")

        header_line = _generate_csv_header_line(header=header, header_prefix=header_prefix, header_names=fields.keys(), sep=sep, newline=newline)

        file_or_string = open(filename, 'a' if append else 'w') if (filename is not None) else io.StringIO()

        retval = None
        attr_getters = [attrgetter(attr_name) for attr_name in fields.values()]
        try:

            file_or_string.write(header_line)

            for x in self.items:
                line = sep.join([format(func(x)) for func in attr_getters]) + newline
                file_or_string.write(line)

            if filename is None:
                retval = file_or_string.getvalue()

        finally:
            file_or_string.close()

        return retval

    def to_psql(self, url, table_name, *, if_exists="fail"):
        """
        Export items as rows in a PostgreSQL table.

        Parameters
        ----------

        url: string
            Connection string to connect to the database.
            Example: "postgresql://postgres@127.0.0.1:5432/testdb"

        table_name: string
            Name of the database table.

        if_exists : {'fail', 'replace', 'append'}, default 'fail'
            - fail: If table exists, raise an error.
            - replace: If table exists, drop it, recreate it, and insert data.
            - append: If table exists, insert data. Create if does not exist.
        """
        engine = create_engine(url)
        with engine.begin() as conn:
            self.to_df().to_sql(table_name, conn, index=False, if_exists=if_exists)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
@manhg/tokit

A kit for development with Tornado web framework.

See https://github.com/manhg/writekit for usage demo.
"""
version_info = (0, 1, 1, 0)
version = "0.1.1"
import os, sys, subprocess

if __name__ == '__main__':
    # TODO Create project structure, dependancies
    print("Project installation: ", os.getcwd())
    if not input("Press Enter to continue, N to exit: "):
        print("Writing: ")
    [os.makedirs(p) for p in ("static", "src/vendor")]
    subprocess.call(
        "pyvenv . ;"
        "source bin/activate;"
        "pip3 install tornado;"
        , stderr=subprocess.STDOUT, shell=True)
    sys.exit(0)

__all__ = ['Register', 'Request', 'Websocket', 'Module', 'Static', 'Config']

import re
import collections
import logging
import time
import signal
import json
import importlib
from contextlib import contextmanager

import tornado.locale
import tornado.httpserver
import tornado.web
import tornado.websocket
from tornado.ioloop import IOLoop

def to_json(obj):
    return json.dumps(obj, ensure_ascii=False, indent=4)

class Register(type):
    """ A metaclass to log information of all subclasses """
    _repo = collections.defaultdict(list)

    def __init__(cls, name, bases, nmspc, **kwarg):
        Register._repo[bases[0].__name__].append(cls)
        super(Register, cls).__init__(name, bases, nmspc)

    @classmethod
    def collect(mcs, parent_name) -> list:
        """ Get all subclass of ``parent_name``

        parent_name -- str pure class name without module prefix
        """
        return Register._repo[parent_name]

class Request(tornado.web.RequestHandler, metaclass=Register):
    """ Base class for handling request """

    ROUTE = None
    """
    Route can be pattern or (pattern, name) or an URLSpec::

        class Post(Request):
            ROUTE = r'/post/.*'
            def get(self, slug):
                pass # Add logic here
    """

    def set_default_headers(self):
        self.set_header('Server', 'Python3')

    def get_template_namespace(self):
        namespace = super(Request, self).get_template_namespace()
        namespace['json'] = to_json
        return namespace

    def js(self):
        """ List (to preserved ordering) of JS path to to used by layout file """
        return []

    def css(self):
        """ List (to preserved ordering) of CSS path to to used by layout file """
        return []

    @classmethod
    def known(cls):
        """ Get Request's subclasses """
        routes = []
        for handler in Register.collect(cls.__name__):
            if not handler.ROUTE:
                logging.debug("Missing route for handler %s.%s",
                    handler.__module__, handler.__name__)
                continue
            if isinstance(handler.ROUTE, str):
                routes.append(tornado.web.URLSpec(handler.ROUTE, handler))
            elif isinstance(handler.ROUTE, tornado.web.URLSpec):
                routes.append(handler.ROUTE)
            else:
                pattern, name = handler.ROUTE
                routes.append(tornado.web.URLSpec(pattern, handler, name=name))
        return routes

class Websocket(tornado.websocket.WebSocketHandler, metaclass=Register):
    pass

class Module(tornado.web.UIModule, metaclass=Register):
    """ Subclass this to create a UIModule
    It's available as class name::

        class Table(Module):
            def render(self):
                pass # then in template, you can use {% module Table() %}

    """

    @classmethod
    def known(cls):
        return {c.__name__: c for c in Register.collect(cls.__name__)}

class Static(tornado.web.StaticFileHandler):

    ALLOW_TYPES = 'tag', 'js', 'css', 'png', 'jpg'
    VALID_PATH = re.compile(r'.*\.({types})$'.format(types='|'.join(ALLOW_TYPES)))

    def validate_absolute_path(self, root, absolute_path):
        """ Add file types checking """
        if not self.VALID_PATH.match(absolute_path):
            raise tornado.web.HTTPError(403, 'Unallowed file type')
        return absolute_path

class Event():
    """Event handlers storage.

    Example:

    >>> def handler(**kwargs):
    ...     print("Triggered:", kwargs)
    ...
    >>> Event.get('init').attach(handler)
    >>> Event.get('init').emit(status='OK')
    Triggered: {'status': 'OK'}

    """

    _repo = {}

    def __init__(self, name):
        self._handlers = set()
        self.name = name

    def attach(self, task):
        self._handlers.add(task)

    def detach(self, task):
        self._handlers.remove(task)

    @classmethod
    def get(cls, name):
        instance = cls._repo.get(name, None)
        if not instance:
            instance = cls(name)
            cls._repo[name] = instance
        return instance

    @contextmanager
    def subscribe(self, *tasks):
        for task in tasks:
            self.attach(task)
        try:
            yield
        finally:
            for task in tasks:
                self.detach(task)

    def emit(self, *args, **kwargs):
        for handler in self._handlers:
            handler(*args, **kwargs)

class Config:
    """ Subclass this to customize runtime config """

    settings = dict(
        compress_response=True,
        static_path='.',
        static_url_prefix='/static/',
        static_handler_class=Static,
        debug=True,
        compiled_template_cache=False,
        cookie_secret='TODO'
    )
    root_path = None
    production = False
    timezone = 'UTC'
    locale = 'en'
    modules = ()
    kill_blocking = 2  # (second) time to kill process if blocking too long
    session_timeout = 60 * 24 * 3600

    def __init__(self, base_file):
        self.root_path = os.path.abspath(os.path.dirname(base_file))
        tornado.locale.load_translations('lang')

    def dev(self):
        """ Expose debug """
        import tornado.options
        tornado.options.parse_command_line()
        logging.basicConfig(level=logging.DEBUG)

    def production(self):
        """ Common config for production """
        self.production = True
        logging.basicConfig(level=logging.WARNING)
        self.cookie_secret=os.environ.get('SECRET')

class App(tornado.web.Application):
    pass

def load(config):
    """ Import declared modules from config, during this imports, routes
    and other components will be registered """
    try:
        os.environ['TZ'] = config.timezone
        time.tzset()
        tornado.locale.set_default_locale(config.locale)
        for m in config.modules:
            importlib.import_module(m)
    except SyntaxError:
        ex = sys.exc_info()
        logging.error("Broken module: ", ex[0].__name__,
                      os.path.basename(
                          sys.exc_info()[2].tb_frame.f_code.co_filename),
                      ex[2].tb_lineno)
        sys.exit(1)

def start(port, config):
    """ Entry point for application. This setups IOLoop, load classes and run HTTP server """
    load(config)
    Event.get('setting').emit(config.settings)
    config.settings['ui_modules'] = Module.known()
    app = App(**config.settings)
    app.config = config
    http_server = tornado.httpserver.HTTPServer(app, xheaders=True)
    ioloop = IOLoop.instance()
    Event.get('init').emit(app)
    app.add_handlers('.*$', Request.known())

    def _graceful():
        def _shutdown():
            pass

        ioloop.stop()
        logging.info('Shutting down...')
        http_server.stop()
        ioloop.call_later(1, _shutdown)

    def _on_term(*args):
        ioloop.add_callback_from_signal(_graceful)

    http_server.listen(port, 'localhost')
    signal.signal(signal.SIGTERM, _on_term)

    logging.info('Running PID {pid} @ localhost:{port}'.format(pid=os.getpid(), port=port))
    if config.production:
        ioloop.set_blocking_log_threshold(1)
        ioloop.set_blocking_signal_threshold(config.kill_blocking, action=None)
    try:
        ioloop.start()
    except KeyboardInterrupt:
        logging.info('Bye.')
<commit_msg>Support mixin<commit_after>#!/usr/bin/env python3
"""
@manhg/tokit

A kit for development with Tornado web framework.

See https://github.com/manhg/writekit for usage demo.
"""
version_info = (0, 1, 1, 0)
version = "0.1.1"
import os, sys, subprocess

if __name__ == '__main__':
    # TODO Create project structure, dependancies
    print("Project installation: ", os.getcwd())
    if not input("Press Enter to continue, N to exit: "):
        print("Writing: ")
    [os.makedirs(p) for p in ("static", "src/vendor")]
    subprocess.call(
        "pyvenv . ;"
        "source bin/activate;"
        "pip3 install tornado;"
        , stderr=subprocess.STDOUT, shell=True)
    sys.exit(0)

__all__ = ['Register', 'Request', 'Websocket', 'Module', 'Static', 'Config']

import re
import collections
import logging
import time
import signal
import json
import importlib
from contextlib import contextmanager

import tornado.locale
import tornado.httpserver
import tornado.web
import tornado.websocket
from tornado.ioloop import IOLoop

def to_json(obj):
    return json.dumps(obj, ensure_ascii=False, indent=4)

class Register(type):
    """ A metaclass to log information of all subclasses """
    _repo = collections.defaultdict(list)

    def __init__(cls, name, bases, nmspc, **kwarg):
        *_, base_cls = bases
        Register._repo[base_cls.__name__].append(cls)
        super(Register, cls).__init__(name, bases, nmspc)

    @classmethod
    def collect(mcs, parent_name) -> list:
        """ Get all subclass of ``parent_name``

        :param str parent_name: pure class name without module prefix
        """
        return Register._repo[parent_name]

class Request(tornado.web.RequestHandler, metaclass=Register):
    """
    Base class for handling request
    Class hierarchy is defined right to left, methods are resolved is from left to right
    http://www.ianlewis.org/en/mixins-and-python
    Therefore, mixin if exists, should be added to the left of declare::

        class Foo(BarMixin, Request):
            pass
    """

    ROUTE = None
    """
    Route can be pattern or (pattern, name) or an URLSpec::

        class Post(Request):
            ROUTE = r'/post/.*'
            def get(self, slug):
                pass # Add logic here
    """

    def set_default_headers(self):
        self.set_header('Server', 'Python3')

    def get_template_namespace(self):
        namespace = super(Request, self).get_template_namespace()
        namespace['json'] = to_json
        return namespace

    def js(self):
        """ List (to preserved ordering) of JS path to to used by layout file """
        return []

    def css(self):
        """ List (to preserved ordering) of CSS path to to used by layout file """
        return []

    @classmethod
    def known(cls):
        """ Get Request's subclasses """
        routes = []
        for handler in Register.collect(cls.__name__):
            if not handler.ROUTE:
                logging.debug("Missing route for handler %s.%s",
                    handler.__module__, handler.__name__)
                continue
            if isinstance(handler.ROUTE, str):
                routes.append(tornado.web.URLSpec(handler.ROUTE, handler))
            elif isinstance(handler.ROUTE, tornado.web.URLSpec):
                routes.append(handler.ROUTE)
            else:
                pattern, name = handler.ROUTE
                routes.append(tornado.web.URLSpec(pattern, handler, name=name))
        return routes

class Websocket(tornado.websocket.WebSocketHandler, metaclass=Register):
    pass

class Module(tornado.web.UIModule, metaclass=Register):
    """ Subclass this to create a UIModule
    It's available as class name::

        class Table(Module):
            def render(self):
                pass # then in template, you can use {% module Table() %}

    """

    @classmethod
    def known(cls):
        return {c.__name__: c for c in Register.collect(cls.__name__)}

class Static(tornado.web.StaticFileHandler):

    ALLOW_TYPES = 'tag', 'js', 'css', 'png', 'jpg'
    VALID_PATH = re.compile(r'.*\.({types})$'.format(types='|'.join(ALLOW_TYPES)))

    def validate_absolute_path(self, root, absolute_path):
        """ Add file types checking """
        if not self.VALID_PATH.match(absolute_path):
            raise tornado.web.HTTPError(403, 'Unallowed file type')
        return absolute_path

class Event():
    """Event handlers storage.

    Example:

    >>> def handler(**kwargs):
    ...     print("Triggered:", kwargs)
    ...
    >>> Event.get('init').attach(handler)
    >>> Event.get('init').emit(status='OK')
    Triggered: {'status': 'OK'}

    """

    _repo = {}

    def __init__(self, name):
        self._handlers = set()
        self.name = name

    def attach(self, task):
        self._handlers.add(task)

    def detach(self, task):
        self._handlers.remove(task)

    @classmethod
    def get(cls, name):
        instance = cls._repo.get(name, None)
        if not instance:
            instance = cls(name)
            cls._repo[name] = instance
        return instance

    @contextmanager
    def subscribe(self, *tasks):
        for task in tasks:
            self.attach(task)
        try:
            yield
        finally:
            for task in tasks:
                self.detach(task)

    def emit(self, *args, **kwargs):
        for handler in self._handlers:
            handler(*args, **kwargs)

class Config:
    """ Subclass this to customize runtime config """

    settings = dict(
        compress_response=True,
        static_path='.',
        static_url_prefix='/static/',
        static_handler_class=Static,
        debug=True,
        compiled_template_cache=False,
        cookie_secret='TODO'
    )
    root_path = None
    production = False
    timezone = 'UTC'
    locale = 'en'
    modules = ()
    kill_blocking = 2  # (second) time to kill process if blocking too long
    session_timeout = 60 * 24 * 3600

    def __init__(self, base_file):
        self.root_path = os.path.abspath(os.path.dirname(base_file))
        tornado.locale.load_translations('lang')

    def dev(self):
        """ Expose debug """
        import tornado.options
        tornado.options.parse_command_line()
        logging.basicConfig(level=logging.DEBUG)

    def production(self):
        """ Common config for production """
        self.production = True
        logging.basicConfig(level=logging.WARNING)
        self.cookie_secret=os.environ.get('SECRET')

class App(tornado.web.Application):
    pass

def load(config):
    """ Import declared modules from config, during this imports, routes
    and other components will be registered """
    try:
        os.environ['TZ'] = config.timezone
        time.tzset()
        tornado.locale.set_default_locale(config.locale)
        for m in config.modules:
            importlib.import_module(m)
    except SyntaxError:
        ex = sys.exc_info()
        logging.error("Broken module: ", ex[0].__name__,
                      os.path.basename(
                          sys.exc_info()[2].tb_frame.f_code.co_filename),
                      ex[2].tb_lineno)
        sys.exit(1)

def start(port, config):
    """ Entry point for application. This setups IOLoop, load classes and run HTTP server """
    load(config)
    Event.get('setting').emit(config.settings)
    config.settings['ui_modules'] = Module.known()
    app = App(**config.settings)
    app.config = config
    http_server = tornado.httpserver.HTTPServer(app, xheaders=True)
    ioloop = IOLoop.instance()
    Event.get('init').emit(app)
    app.add_handlers('.*$', Request.known())

    def _graceful():
        def _shutdown():
            pass

        ioloop.stop()
        logging.info('Shutting down...')
        http_server.stop()
        ioloop.call_later(1, _shutdown)

    def _on_term(*args):
        ioloop.add_callback_from_signal(_graceful)

    http_server.listen(port, 'localhost')
    signal.signal(signal.SIGTERM, _on_term)

    logging.info('Running PID {pid} @ localhost:{port}'.format(pid=os.getpid(), port=port))
    if config.production:
        ioloop.set_blocking_log_threshold(1)
        ioloop.set_blocking_signal_threshold(config.kill_blocking, action=None)
    try:
        ioloop.start()
    except KeyboardInterrupt:
        logging.info('Bye.')
<|endoftext|>


<commit_before>from django.conf import settings
from django.contrib.auth.models import User
from django.db import models
from django.utils.translation import ugettext_lazy as _

from core.models import BaseModel

class Profile(BaseModel):
    user = models.OneToOneField(User)

    # Note to coders: The '_url' fields below need to JUST be the name of the account.
    #     Examples:
    #       github_url = 'pydanny'
    #       bitbucket_url = 'pydanny'
    #       google_code_url = 'pydanny'
    github_account = models.CharField(_("Github account"), null=True, blank=True, max_length=40)
    github_url = models.CharField(_("Github account"), null=True, blank=True, max_length=100, editable=False)
    bitbucket_url = models.CharField(_("Bitbucket account"), null=True, blank=True, max_length=100)
    google_code_url = models.CharField(_("Google Code account"), null=True, blank=True, max_length=100)
    email = models.EmailField(_("Email"), null=True, blank=True)

    def __str__(self):
        if not self.github_account:
            return self.user.username
        return self.github_account

    def save(self, **kwargs):
        """ Override save to always populate email changes to auth.user model
        """
        if self.email is not None:

            email = self.email.strip()
            user_obj = User.objects.get(username=self.user.username)
            user_obj.email = email
            user_obj.save()

        super(Profile, self).save(**kwargs)

    def url_for_repo(self, repo):
        """Return the profile's URL for a given repo.

        If url doesn't exist return None.
        """
        url_mapping = {
            'Github': self.github_account,
            'BitBucket': self.bitbucket_url,
            'Google Code': self.google_code_url}
        return url_mapping.get(repo.title)

    def my_packages(self):
        """Return a list of all packages the user contributes to.

        List is sorted by package name.
        """
        from package.repos import get_repo, supported_repos

        packages = []
        for repo in supported_repos():
            repo = get_repo(repo)
            repo_packages = repo.packages_for_profile(self)
            packages.extend(repo_packages)
        packages.sort(key=lambda a, b: ((a.title > b.title) - (a.title < b.title)))
        return packages

    @models.permalink
    def get_absolute_url(self):
        return ("profile_detail", [self.github_account])

    # define permission properties as properties so we can access in templates

    @property
    def can_add_package(self):
        if getattr(settings, 'RESTRICT_PACKAGE_EDITORS', False):
            return self.user.has_perm('package.add_package')
        # anyone can add
        return True

    @property
    def can_edit_package(self):
        if getattr(settings, 'RESTRICT_PACKAGE_EDITORS', False):
            # this is inconsistent, fix later?
            return self.user.has_perm('package.change_package')
        # anyone can edit
        return True

    # Grids
    @property
    def can_edit_grid(self):
        if getattr(settings, 'RESTRICT_GRID_EDITORS', False):
            return self.user.has_perm('grid.change_grid')
        return True

    @property
    def can_add_grid(self):
        if getattr(settings, 'RESTRICT_GRID_EDITORS', False):
            return self.user.has_perm('grid.add_grid')
        return True

    # Grid Features
    @property
    def can_add_grid_feature(self):
        if getattr(settings, 'RESTRICT_GRID_EDITORS', False):
            return self.user.has_perm('grid.add_feature')
        return True

    @property
    def can_edit_grid_feature(self):
        if getattr(settings, 'RESTRICT_GRID_EDITORS', False):
            return self.user.has_perm('grid.change_feature')
        return True

    @property
    def can_delete_grid_feature(self):
        if getattr(settings, 'RESTRICT_GRID_EDITORS', False):
            return self.user.has_perm('grid.delete_feature')
        return True

    # Grid Packages
    @property
    def can_add_grid_package(self):
        if getattr(settings, 'RESTRICT_GRID_EDITORS', False):
            return self.user.has_perm('grid.add_gridpackage')
        return True

    @property
    def can_delete_grid_package(self):
        if getattr(settings, 'RESTRICT_GRID_EDITORS', False):
            return self.user.has_perm('grid.delete_gridpackage')
        return True

    # Grid Element (cells in grid)
    @property
    def can_edit_grid_element(self):
        if getattr(settings, 'RESTRICT_GRID_EDITORS', False):
            return self.user.has_perm('grid.change_element')
        return True
<commit_msg>remove auto generated comp<commit_after>from django.conf import settings
from django.contrib.auth.models import User
from django.db import models
from django.utils.translation import ugettext_lazy as _

from core.models import BaseModel

class Profile(BaseModel):
    user = models.OneToOneField(User)

    # Note to coders: The '_url' fields below need to JUST be the name of the account.
    #     Examples:
    #       github_url = 'pydanny'
    #       bitbucket_url = 'pydanny'
    #       google_code_url = 'pydanny'
    github_account = models.CharField(_("Github account"), null=True, blank=True, max_length=40)
    github_url = models.CharField(_("Github account"), null=True, blank=True, max_length=100, editable=False)
    bitbucket_url = models.CharField(_("Bitbucket account"), null=True, blank=True, max_length=100)
    google_code_url = models.CharField(_("Google Code account"), null=True, blank=True, max_length=100)
    email = models.EmailField(_("Email"), null=True, blank=True)

    def __str__(self):
        if not self.github_account:
            return self.user.username
        return self.github_account

    def save(self, **kwargs):
        """ Override save to always populate email changes to auth.user model
        """
        if self.email is not None:

            email = self.email.strip()
            user_obj = User.objects.get(username=self.user.username)
            user_obj.email = email
            user_obj.save()

        super(Profile, self).save(**kwargs)

    def url_for_repo(self, repo):
        """Return the profile's URL for a given repo.

        If url doesn't exist return None.
        """
        url_mapping = {
            'Github': self.github_account,
            'BitBucket': self.bitbucket_url,
            'Google Code': self.google_code_url}
        return url_mapping.get(repo.title)

    def my_packages(self):
        """Return a list of all packages the user contributes to.

        List is sorted by package name.
        """
        from package.repos import get_repo, supported_repos

        packages = []
        for repo in supported_repos():
            repo = get_repo(repo)
            repo_packages = repo.packages_for_profile(self)
            packages.extend(repo_packages)
        packages.sort(key=lambda a: a.title)
        return packages

    @models.permalink
    def get_absolute_url(self):
        return ("profile_detail", [self.github_account])

    # define permission properties as properties so we can access in templates

    @property
    def can_add_package(self):
        if getattr(settings, 'RESTRICT_PACKAGE_EDITORS', False):
            return self.user.has_perm('package.add_package')
        # anyone can add
        return True

    @property
    def can_edit_package(self):
        if getattr(settings, 'RESTRICT_PACKAGE_EDITORS', False):
            # this is inconsistent, fix later?
            return self.user.has_perm('package.change_package')
        # anyone can edit
        return True

    # Grids
    @property
    def can_edit_grid(self):
        if getattr(settings, 'RESTRICT_GRID_EDITORS', False):
            return self.user.has_perm('grid.change_grid')
        return True

    @property
    def can_add_grid(self):
        if getattr(settings, 'RESTRICT_GRID_EDITORS', False):
            return self.user.has_perm('grid.add_grid')
        return True

    # Grid Features
    @property
    def can_add_grid_feature(self):
        if getattr(settings, 'RESTRICT_GRID_EDITORS', False):
            return self.user.has_perm('grid.add_feature')
        return True

    @property
    def can_edit_grid_feature(self):
        if getattr(settings, 'RESTRICT_GRID_EDITORS', False):
            return self.user.has_perm('grid.change_feature')
        return True

    @property
    def can_delete_grid_feature(self):
        if getattr(settings, 'RESTRICT_GRID_EDITORS', False):
            return self.user.has_perm('grid.delete_feature')
        return True

    # Grid Packages
    @property
    def can_add_grid_package(self):
        if getattr(settings, 'RESTRICT_GRID_EDITORS', False):
            return self.user.has_perm('grid.add_gridpackage')
        return True

    @property
    def can_delete_grid_package(self):
        if getattr(settings, 'RESTRICT_GRID_EDITORS', False):
            return self.user.has_perm('grid.delete_gridpackage')
        return True

    # Grid Element (cells in grid)
    @property
    def can_edit_grid_element(self):
        if getattr(settings, 'RESTRICT_GRID_EDITORS', False):
            return self.user.has_perm('grid.change_element')
        return True
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
:author: Joseph Martinot-Lagarde

Created on Sat Jan 19 14:57:57 2013
"""

# Standard library imports
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

# Third party imports
ERR_MSG = ''
try:
    import autopep8

    try:
        FIX_LIST = [(code.strip(), description.strip())
                    for code, description in autopep8.supported_fixes()]
        DEFAULT_IGNORE = ["E711", "E712", "W6"]
    except AttributeError:
        ERR_MSG = "Please install autopep8 >= 1.0, and pep8 >= 1.4.2."
except ImportError:
    ERR_MSG = "Please install autopep8 >= 1.0, and pep8 >= 1.4.2."

from qtpy.QtWidgets import (QWidget, QVBoxLayout, QGroupBox,
                            QScrollArea, QLabel, QCheckBox)
from qtpy.QtGui import QTextCursor

from spyderlib.config.base import get_translation
from spyderlib.plugins import SpyderPluginMixin, PluginConfigPage
from spyderlib.utils.qthelpers import get_icon, create_action
from spyderlib.utils import icon_manager as ima

try:
    from spyderlib.py3compat import to_text_string
except ImportError:
    # Python 2
    to_text_string = unicode

# Local imports
from .data import images

_ = get_translation("autopep8", dirname="spyder_autopep8")

class AutoPEP8ConfigPage(PluginConfigPage):
    """
    Widget with configuration options for line profiler.
    """
    GROUPS = {
        "1": "Indentation",
        "2": "Whitespace",
        "3": "Blank line",
        "4": "Import",
        "5": "Line length",
        "6": "Deprecation",
        "7": "Statement",
        "9": "Runtime"}
    CODES = {
        # See http://pep8.readthedocs.org/en/latest/intro.html#error-codes
        "E101": "indentation contains mixed spaces and tabs",
        "E111": "indentation is not a multiple of four",
        "E112": "expected an indented block",
        "E113": "unexpected indentation",
        "E121": "continuation line indentation is not a multiple of four",
        "E122": "continuation line missing indentation or outdented",
        "E123": "closing bracket does not match indentation of opening"
        " bracket’s line",
        "E124": "closing bracket does not match visual indentation",
        "E125": "continuation line does not distinguish itself from next"
        " logical line",
        "E126": "continuation line over-indented for hanging indent",
        "E127": "continuation line over-indented for visual indent",
        "E128": "continuation line under-indented for visual indent",
        "E133": "closing bracket is missing indentation",
        "E201": "whitespace after ‘(‘",
        "E202": "whitespace before ‘)’",
        "E203": "whitespace before ‘:’",
        "E211": "whitespace before ‘(‘",
        "E221": "multiple spaces before operator",
        "E222": "multiple spaces after operator",
        "E223": "tab before operator",
        "E224": "tab after operator",
        "E225": "missing whitespace around operator",
        "E226": "missing whitespace around arithmetic operator",
        "E227": "missing whitespace around bitwise or shift operator",
        "E228": "missing whitespace around modulo operator",
        "E231": "missing whitespace after ‘,’",
        "E241": "multiple spaces after ‘,’",
        "E242": "tab after ‘,’",
        "E251": "unexpected spaces around keyword / parameter equals",
        "E261": "at least two spaces before inline comment",
        "E262": "inline comment should start with ‘# ‘",
        "E271": "multiple spaces after keyword",
        "E272": "multiple spaces before keyword",
        "E273": "tab after keyword",
        "E274": "tab before keyword",
        "E301": "expected 1 blank line, found 0",
        "E302": "expected 2 blank lines, found 0",
        "E303": "too many blank lines (3)",
        "E304": "blank lines found after function decorator",
        "E401": "multiple imports on one line",
        "E501": "line too long (82 > 79 characters)",
        "E502": "the backslash is redundant between brackets",
        "E701": "multiple statements on one line (colon)",
        "E702": "multiple statements on one line (semicolon)",
        "E703": "statement ends with a semicolon",
        "E711": "comparison to None should be ‘if cond is None:’",
        "E712": "comparison to True should be ‘if cond is True:’ or"
        " ‘if cond:’",
        "E721": "do not compare types, use ‘isinstance()’",
        "E901": "SyntaxError or IndentationError",
        "E902": "IOError",
        "W191": "indentation contains tabs",
        "W291": "trailing whitespace",
        "W292": "no newline at end of file",
        "W293": "blank line contains whitespace",
        "W391": "blank line at end of file",
        "W6": "apply, except, exec, execfile, exitfunc, has_key, idioms,"
        " import, methodattrs, ne, numliterals, operator, paren, reduce,"
        " renames, repr, standarderror, sys_exc, throw, tuple_params,"
        " types, xreadlines",
        "W601": ".has_key() is deprecated, use ‘in’",
        "W602": "deprecated form of raising exception",
        "W603": "‘<>’ is deprecated, use ‘!=’",
        "W604": "backticks are deprecated, use ‘repr()’)"}

    def setup_page(self):
        if ERR_MSG:
            label = QLabel(_("Could not load plugin:\n{0}".format(ERR_MSG)))
            layout = QVBoxLayout()
            layout.addWidget(label)
            self.setLayout(layout)
            return

        # Layout parameter
        indent = QCheckBox().sizeHint().width()

        # General options
        options_group = QGroupBox(_("Options"))
        # Hack : the spinbox widget will be added to self.spinboxes
        spinboxes_before = set(self.spinboxes)
        passes_spin = self.create_spinbox(
            _("Number of pep8 passes: "), "", 'passes',
            default=0, min_=0, max_=1000000, step=1)
        spinbox = set(self.spinboxes) - spinboxes_before
        spinbox = spinbox.pop()
        spinbox.setSpecialValueText(_("Infinite"))
        aggressive1_checkbox = self.create_checkbox(
            "Aggressivity level 1", "aggressive1", default=False)
        aggressive1_label = QLabel(_(
            "Allow possibly unsafe fixes (E711 and W6), shorten lines"
            " and remove trailing whitespace more aggressively (in"
            " docstrings and multiline strings)."))
        aggressive1_label.setWordWrap(True)
        aggressive1_label.setIndent(indent)
        font_description = aggressive1_label.font()
        font_description.setPointSizeF(font_description.pointSize() * 0.9)
        aggressive1_label.setFont(font_description)
        aggressive2_checkbox = self.create_checkbox(
            "Aggressivity level 2", "aggressive2", default=False)
        aggressive2_label = QLabel(_(
            "Allow more possibly unsafe fixes (E712) and shorten lines."))
        aggressive2_label.setWordWrap(True)
        aggressive2_label.setIndent(indent)
        aggressive2_label.setFont(font_description)

        aggressive1_checkbox.toggled.connect(
            aggressive2_checkbox.setEnabled)
        aggressive1_checkbox.toggled.connect(
            aggressive2_label.setEnabled)
        aggressive2_checkbox.setEnabled(aggressive1_checkbox.isChecked())
        aggressive2_label.setEnabled(aggressive1_checkbox.isChecked())

        # Enable/disable error codes
        fix_layout = QVBoxLayout()
        last_group = ""
        FIX_LIST.sort(key=lambda item: item[0][1])
        for code, description in FIX_LIST:
            # Create a new group if necessary
            if code[1] != last_group:
                last_group = code[1]
                group = QGroupBox(_(self.GROUPS.get(code[1], "")))
                fix_layout.addWidget(group)
                group_layout = QVBoxLayout(group)

            # Checkbox for the option
            text = code
            default = True
            if code in DEFAULT_IGNORE:
                text += _(" (UNSAFE)")
                default = False
            option = self.create_checkbox(text, code, default=default)

            # Label for description
            if code in self.CODES:
                label = QLabel("{autopep8} ({pep8}).".format(
                    autopep8=_(description).rstrip("."),
                    pep8=self.CODES[code]))
            else:
                label = QLabel(_(description))
            label.setWordWrap(True)
            label.setIndent(indent)
            label.setFont(font_description)

            # Add widgets to layout
            option_layout = QVBoxLayout()
            option_layout.setSpacing(0)
            option_layout.addWidget(option)
            option_layout.addWidget(label)
            group_layout.addLayout(option_layout)

            # Special cases
            if code in ("E711", "W6"):
                aggressive1_checkbox.toggled.connect(option.setEnabled)
                aggressive1_checkbox.toggled.connect(label.setEnabled)
                option.setEnabled(aggressive1_checkbox.isChecked())
                label.setEnabled(aggressive1_checkbox.isChecked())
            if code == "E712":
                def e712_enabled():
                    enabled = (aggressive1_checkbox.isChecked()
                               and aggressive2_checkbox.isChecked())
                    option.setEnabled(enabled)
                    label.setEnabled(enabled)
                aggressive1_checkbox.toggled.connect(e712_enabled)
                aggressive2_checkbox.toggled.connect(e712_enabled)
                e712_enabled()

        # General layout
        aggressive2_layout = QVBoxLayout()
        margins = aggressive2_layout.contentsMargins()
        margins.setLeft(indent)
        aggressive2_layout.setContentsMargins(margins)
        aggressive2_layout.addWidget(aggressive2_checkbox)
        aggressive2_layout.addWidget(aggressive2_label)

        options_layout = QVBoxLayout()
        options_layout.addWidget(passes_spin)
        options_layout.addWidget(aggressive1_checkbox)
        options_layout.addWidget(aggressive1_label)
        options_layout.addLayout(aggressive2_layout)
        options_group.setLayout(options_layout)

        widget_scroll = QWidget()
        widget_scroll.setLayout(fix_layout)
        fix_scroll = QScrollArea()
        fix_scroll.setWidget(widget_scroll)
        fix_scroll.setWidgetResizable(True)
        fig_out_layout = QVBoxLayout()
        fig_out_layout.addWidget(fix_scroll, 1)
        fix_group = QGroupBox(_("Errors/warnings to fix"))
        fix_group.setLayout(fig_out_layout)

        vlayout = QVBoxLayout()
        vlayout.addWidget(options_group)
        vlayout.addWidget(fix_group, 1)
        self.setLayout(vlayout)

class DummyDock(object):

    def close(self):
        pass

class AutoPEP8(SpyderPluginMixin):  # pylint: disable=R0904
    """Python source code automatic formatting based on autopep8.

    QObject is needed to register the action.
    """
    CONF_SECTION = "spyder.autopep8"
    CONFIGWIDGET_CLASS = AutoPEP8ConfigPage

    def __init__(self, main):
        super(AutoPEP8, self).__init__(main)
        self.dockwidget = DummyDock()

    # --- SpyderPlugin API ----------------------------------------------------
    def get_plugin_title(self):
        """Return widget title."""
        return _("Autopep8")

    def get_plugin_icon(self):
        """Return widget icon."""
        return ima.icon(self.CONF_SECTION)

    def register_plugin(self):
        """Register plugin in Spyder's main window."""
        autopep8_act = create_action(
            self.main, _("Run autopep8 code autoformatting"),
            #icon=self.get_plugin_icon(),
            triggered=self.run_autopep8)
        self.register_shortcut(autopep8_act, context="Editor",
                               name="Run autoformatting", default="Shift+F8")
        self.main.source_menu_actions += [None, autopep8_act]
        self.main.editor.pythonfile_dependent_actions += [autopep8_act]

    def apply_plugin_settings(self, options):
        """Needs to be redefined."""
        pass

    def closing_plugin(self, cancelable=False):
        """Perform actions before parent main window is closed."""
        return True

    # --- Public API ---------------------------------------------------------
    def run_autopep8(self):
        """Format code with autopep8."""
        if ERR_MSG:
            self.main.statusBar().showMessage(
                _("Unable to run: {0}".format(ERR_MSG)))
            return

        # Retrieve active fixes
        ignore = []
        for code, description in FIX_LIST:
            if not self.get_option(code, True):
                ignore.append(code)

        # Retrieve text of current opened file
        editorstack = self.main.editor.get_current_editorstack()
        index = editorstack.get_stack_index()
        finfo = editorstack.data[index]
        editor = finfo.editor
        cursor = editor.textCursor()
        cursor.beginEditBlock()  # Start cancel block
        if not cursor.hasSelection():
            position_start = 0
            cursor.select(QTextCursor.Document)  # Select all
        else:
            # Select whole lines
            position_end = cursor.selectionEnd()
            cursor.setPosition(cursor.selectionStart())
            cursor.movePosition(QTextCursor.StartOfLine)
            position_start = cursor.position()
            cursor.setPosition(position_end, QTextCursor.KeepAnchor)
            cursor.movePosition(QTextCursor.StartOfLine,
                                QTextCursor.KeepAnchor)
            position_lastline_start = cursor.position()
            if not position_end == position_lastline_start:
                cursor.movePosition(QTextCursor.EndOfLine,
                                    QTextCursor.KeepAnchor)
                # Select EOL if not on a new line
                if not position_lastline_start == cursor.position():
                    cursor.movePosition(QTextCursor.Right,
                                        QTextCursor.KeepAnchor)

            # Disable checks of newlines at end of file
            if not cursor.atEnd():
                ignore.append("W391")

        # replace(): See qt doc for QTextCursor.selectedText()
        text_before = to_text_string(
            cursor.selectedText().replace("\u2029", "\n"))

        # Run autopep8
        line_length = self.main.window().editor.get_option("edge_line_column")
        options = ["", "--ignore", ",".join(ignore),
                   "--pep8-passes", str(self.get_option("passes", 0) - 1),
                   "--max-line-length", str(line_length)]
        if self.get_option("aggressive1", False):
            options.append("--aggressive")
            if self.get_option("aggressive2", False):
                options.append("--aggressive")
        options = autopep8.parse_args(options)
        text_after = autopep8.fix_code(text_before, options)

        # Apply new text if needed
        if text_before != text_after:
            cursor.insertText(text_after)  # Change text

        cursor.endEditBlock()  # End cancel block

        # Select changed text
        position_end = cursor.position()
        cursor.setPosition(position_start, QTextCursor.MoveAnchor)
        cursor.setPosition(position_end, QTextCursor.KeepAnchor)
        editor.setTextCursor(cursor)

        self.main.statusBar().showMessage(
            _("Autopep8 finished !"))
<commit_msg>Use fixed shortcuts (for now)<commit_after># -*- coding: utf-8 -*-
"""
:author: Joseph Martinot-Lagarde

Created on Sat Jan 19 14:57:57 2013
"""

# Standard library imports
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

# Third party imports
ERR_MSG = ''
try:
    import autopep8

    try:
        FIX_LIST = [(code.strip(), description.strip())
                    for code, description in autopep8.supported_fixes()]
        DEFAULT_IGNORE = ["E711", "E712", "W6"]
    except AttributeError:
        ERR_MSG = "Please install autopep8 >= 1.0, and pep8 >= 1.4.2."
except ImportError:
    ERR_MSG = "Please install autopep8 >= 1.0, and pep8 >= 1.4.2."

from qtpy.QtWidgets import (QWidget, QVBoxLayout, QGroupBox,
                            QScrollArea, QLabel, QCheckBox)
from qtpy.QtGui import QTextCursor

from spyderlib.config.base import get_translation
from spyderlib.config.gui import fixed_shortcut
from spyderlib.plugins import SpyderPluginMixin, PluginConfigPage
from spyderlib.utils.qthelpers import get_icon, create_action
from spyderlib.utils import icon_manager as ima

try:
    from spyderlib.py3compat import to_text_string
except ImportError:
    # Python 2
    to_text_string = unicode

_ = get_translation("autopep8", dirname="spyder_autopep8")

class AutoPEP8ConfigPage(PluginConfigPage):
    """
    Widget with configuration options for line profiler.
    """
    GROUPS = {
        "1": "Indentation",
        "2": "Whitespace",
        "3": "Blank line",
        "4": "Import",
        "5": "Line length",
        "6": "Deprecation",
        "7": "Statement",
        "9": "Runtime"}
    CODES = {
        # See http://pep8.readthedocs.org/en/latest/intro.html#error-codes
        "E101": "indentation contains mixed spaces and tabs",
        "E111": "indentation is not a multiple of four",
        "E112": "expected an indented block",
        "E113": "unexpected indentation",
        "E121": "continuation line indentation is not a multiple of four",
        "E122": "continuation line missing indentation or outdented",
        "E123": "closing bracket does not match indentation of opening"
        " bracket’s line",
        "E124": "closing bracket does not match visual indentation",
        "E125": "continuation line does not distinguish itself from next"
        " logical line",
        "E126": "continuation line over-indented for hanging indent",
        "E127": "continuation line over-indented for visual indent",
        "E128": "continuation line under-indented for visual indent",
        "E133": "closing bracket is missing indentation",
        "E201": "whitespace after ‘(‘",
        "E202": "whitespace before ‘)’",
        "E203": "whitespace before ‘:’",
        "E211": "whitespace before ‘(‘",
        "E221": "multiple spaces before operator",
        "E222": "multiple spaces after operator",
        "E223": "tab before operator",
        "E224": "tab after operator",
        "E225": "missing whitespace around operator",
        "E226": "missing whitespace around arithmetic operator",
        "E227": "missing whitespace around bitwise or shift operator",
        "E228": "missing whitespace around modulo operator",
        "E231": "missing whitespace after ‘,’",
        "E241": "multiple spaces after ‘,’",
        "E242": "tab after ‘,’",
        "E251": "unexpected spaces around keyword / parameter equals",
        "E261": "at least two spaces before inline comment",
        "E262": "inline comment should start with ‘# ‘",
        "E271": "multiple spaces after keyword",
        "E272": "multiple spaces before keyword",
        "E273": "tab after keyword",
        "E274": "tab before keyword",
        "E301": "expected 1 blank line, found 0",
        "E302": "expected 2 blank lines, found 0",
        "E303": "too many blank lines (3)",
        "E304": "blank lines found after function decorator",
        "E401": "multiple imports on one line",
        "E501": "line too long (82 > 79 characters)",
        "E502": "the backslash is redundant between brackets",
        "E701": "multiple statements on one line (colon)",
        "E702": "multiple statements on one line (semicolon)",
        "E703": "statement ends with a semicolon",
        "E711": "comparison to None should be ‘if cond is None:’",
        "E712": "comparison to True should be ‘if cond is True:’ or"
        " ‘if cond:’",
        "E721": "do not compare types, use ‘isinstance()’",
        "E901": "SyntaxError or IndentationError",
        "E902": "IOError",
        "W191": "indentation contains tabs",
        "W291": "trailing whitespace",
        "W292": "no newline at end of file",
        "W293": "blank line contains whitespace",
        "W391": "blank line at end of file",
        "W6": "apply, except, exec, execfile, exitfunc, has_key, idioms,"
        " import, methodattrs, ne, numliterals, operator, paren, reduce,"
        " renames, repr, standarderror, sys_exc, throw, tuple_params,"
        " types, xreadlines",
        "W601": ".has_key() is deprecated, use ‘in’",
        "W602": "deprecated form of raising exception",
        "W603": "‘<>’ is deprecated, use ‘!=’",
        "W604": "backticks are deprecated, use ‘repr()’)"}

    def setup_page(self):
        if ERR_MSG:
            label = QLabel(_("Could not load plugin:\n{0}".format(ERR_MSG)))
            layout = QVBoxLayout()
            layout.addWidget(label)
            self.setLayout(layout)
            return

        # Layout parameter
        indent = QCheckBox().sizeHint().width()

        # General options
        options_group = QGroupBox(_("Options"))
        # Hack : the spinbox widget will be added to self.spinboxes
        spinboxes_before = set(self.spinboxes)
        passes_spin = self.create_spinbox(
            _("Number of pep8 passes: "), "", 'passes',
            default=0, min_=0, max_=1000000, step=1)
        spinbox = set(self.spinboxes) - spinboxes_before
        spinbox = spinbox.pop()
        spinbox.setSpecialValueText(_("Infinite"))
        aggressive1_checkbox = self.create_checkbox(
            "Aggressivity level 1", "aggressive1", default=False)
        aggressive1_label = QLabel(_(
            "Allow possibly unsafe fixes (E711 and W6), shorten lines"
            " and remove trailing whitespace more aggressively (in"
            " docstrings and multiline strings)."))
        aggressive1_label.setWordWrap(True)
        aggressive1_label.setIndent(indent)
        font_description = aggressive1_label.font()
        font_description.setPointSizeF(font_description.pointSize() * 0.9)
        aggressive1_label.setFont(font_description)
        aggressive2_checkbox = self.create_checkbox(
            "Aggressivity level 2", "aggressive2", default=False)
        aggressive2_label = QLabel(_(
            "Allow more possibly unsafe fixes (E712) and shorten lines."))
        aggressive2_label.setWordWrap(True)
        aggressive2_label.setIndent(indent)
        aggressive2_label.setFont(font_description)

        aggressive1_checkbox.toggled.connect(
            aggressive2_checkbox.setEnabled)
        aggressive1_checkbox.toggled.connect(
            aggressive2_label.setEnabled)
        aggressive2_checkbox.setEnabled(aggressive1_checkbox.isChecked())
        aggressive2_label.setEnabled(aggressive1_checkbox.isChecked())

        # Enable/disable error codes
        fix_layout = QVBoxLayout()
        last_group = ""
        FIX_LIST.sort(key=lambda item: item[0][1])
        for code, description in FIX_LIST:
            # Create a new group if necessary
            if code[1] != last_group:
                last_group = code[1]
                group = QGroupBox(_(self.GROUPS.get(code[1], "")))
                fix_layout.addWidget(group)
                group_layout = QVBoxLayout(group)

            # Checkbox for the option
            text = code
            default = True
            if code in DEFAULT_IGNORE:
                text += _(" (UNSAFE)")
                default = False
            option = self.create_checkbox(text, code, default=default)

            # Label for description
            if code in self.CODES:
                label = QLabel("{autopep8} ({pep8}).".format(
                    autopep8=_(description).rstrip("."),
                    pep8=self.CODES[code]))
            else:
                label = QLabel(_(description))
            label.setWordWrap(True)
            label.setIndent(indent)
            label.setFont(font_description)

            # Add widgets to layout
            option_layout = QVBoxLayout()
            option_layout.setSpacing(0)
            option_layout.addWidget(option)
            option_layout.addWidget(label)
            group_layout.addLayout(option_layout)

            # Special cases
            if code in ("E711", "W6"):
                aggressive1_checkbox.toggled.connect(option.setEnabled)
                aggressive1_checkbox.toggled.connect(label.setEnabled)
                option.setEnabled(aggressive1_checkbox.isChecked())
                label.setEnabled(aggressive1_checkbox.isChecked())
            if code == "E712":
                def e712_enabled():
                    enabled = (aggressive1_checkbox.isChecked()
                               and aggressive2_checkbox.isChecked())
                    option.setEnabled(enabled)
                    label.setEnabled(enabled)
                aggressive1_checkbox.toggled.connect(e712_enabled)
                aggressive2_checkbox.toggled.connect(e712_enabled)
                e712_enabled()

        # General layout
        aggressive2_layout = QVBoxLayout()
        margins = aggressive2_layout.contentsMargins()
        margins.setLeft(indent)
        aggressive2_layout.setContentsMargins(margins)
        aggressive2_layout.addWidget(aggressive2_checkbox)
        aggressive2_layout.addWidget(aggressive2_label)

        options_layout = QVBoxLayout()
        options_layout.addWidget(passes_spin)
        options_layout.addWidget(aggressive1_checkbox)
        options_layout.addWidget(aggressive1_label)
        options_layout.addLayout(aggressive2_layout)
        options_group.setLayout(options_layout)

        widget_scroll = QWidget()
        widget_scroll.setLayout(fix_layout)
        fix_scroll = QScrollArea()
        fix_scroll.setWidget(widget_scroll)
        fix_scroll.setWidgetResizable(True)
        fig_out_layout = QVBoxLayout()
        fig_out_layout.addWidget(fix_scroll, 1)
        fix_group = QGroupBox(_("Errors/warnings to fix"))
        fix_group.setLayout(fig_out_layout)

        vlayout = QVBoxLayout()
        vlayout.addWidget(options_group)
        vlayout.addWidget(fix_group, 1)
        self.setLayout(vlayout)

class DummyDock(object):

    def close(self):
        pass

class AutoPEP8(SpyderPluginMixin):  # pylint: disable=R0904
    """Python source code automatic formatting based on autopep8.

    QObject is needed to register the action.
    """
    CONF_SECTION = "spyder.autopep8"
    CONFIGWIDGET_CLASS = AutoPEP8ConfigPage

    def __init__(self, main):
        super(AutoPEP8, self).__init__(main)
        self.dockwidget = DummyDock()

    # --- SpyderPlugin API ----------------------------------------------------
    def get_plugin_title(self):
        """Return widget title."""
        return _("Autopep8")

    def get_plugin_icon(self):
        """Return widget icon."""
        return ima.icon(self.CONF_SECTION)

    def register_plugin(self):
        """Register plugin in Spyder's main window."""
        autopep8_act = create_action(
            self.main, _("Run autopep8 code autoformatting"),
            icon=self.get_plugin_icon(),
            triggered=self.run_autopep8)
        fixed_shortcut("Shift+F8", self.main,
                       self.run_autopep8)
        self.main.source_menu_actions += [None, autopep8_act]
        self.main.editor.pythonfile_dependent_actions += [autopep8_act]

    def apply_plugin_settings(self, options):
        """Needs to be redefined."""
        pass

    def closing_plugin(self, cancelable=False):
        """Perform actions before parent main window is closed."""
        return True

    # --- Public API ---------------------------------------------------------
    def run_autopep8(self):
        """Format code with autopep8."""
        if ERR_MSG:
            self.main.statusBar().showMessage(
                _("Unable to run: {0}".format(ERR_MSG)))
            return

        # Retrieve active fixes
        ignore = []
        for code, description in FIX_LIST:
            if not self.get_option(code, True):
                ignore.append(code)

        # Retrieve text of current opened file
        editorstack = self.main.editor.get_current_editorstack()
        index = editorstack.get_stack_index()
        finfo = editorstack.data[index]
        editor = finfo.editor
        cursor = editor.textCursor()
        cursor.beginEditBlock()  # Start cancel block
        if not cursor.hasSelection():
            position_start = 0
            cursor.select(QTextCursor.Document)  # Select all
        else:
            # Select whole lines
            position_end = cursor.selectionEnd()
            cursor.setPosition(cursor.selectionStart())
            cursor.movePosition(QTextCursor.StartOfLine)
            position_start = cursor.position()
            cursor.setPosition(position_end, QTextCursor.KeepAnchor)
            cursor.movePosition(QTextCursor.StartOfLine,
                                QTextCursor.KeepAnchor)
            position_lastline_start = cursor.position()
            if not position_end == position_lastline_start:
                cursor.movePosition(QTextCursor.EndOfLine,
                                    QTextCursor.KeepAnchor)
                # Select EOL if not on a new line
                if not position_lastline_start == cursor.position():
                    cursor.movePosition(QTextCursor.Right,
                                        QTextCursor.KeepAnchor)

            # Disable checks of newlines at end of file
            if not cursor.atEnd():
                ignore.append("W391")

        # replace(): See qt doc for QTextCursor.selectedText()
        text_before = to_text_string(
            cursor.selectedText().replace("\u2029", "\n"))

        # Run autopep8
        line_length = self.main.window().editor.get_option("edge_line_column")
        options = ["", "--ignore", ",".join(ignore),
                   "--pep8-passes", str(self.get_option("passes", 0) - 1),
                   "--max-line-length", str(line_length)]
        if self.get_option("aggressive1", False):
            options.append("--aggressive")
            if self.get_option("aggressive2", False):
                options.append("--aggressive")
        options = autopep8.parse_args(options)
        text_after = autopep8.fix_code(text_before, options)

        # Apply new text if needed
        if text_before != text_after:
            cursor.insertText(text_after)  # Change text

        cursor.endEditBlock()  # End cancel block

        # Select changed text
        position_end = cursor.position()
        cursor.setPosition(position_start, QTextCursor.MoveAnchor)
        cursor.setPosition(position_end, QTextCursor.KeepAnchor)
        editor.setTextCursor(cursor)

        self.main.statusBar().showMessage(
            _("Autopep8 finished !"))
<|endoftext|>


<commit_before># SpatialDB scons tool
#
# It builds the library for the SpatialDB C++ class library,
# and provides CPP and linker specifications for the header
# and libraries.
#
# SpatialDB depends on SqliteDB, which provides the interface to
# sqlite3. It also depends on SpatiaLite. Since SpatiaLite is
# is also needed by SQLiteDB, we let the sqlitedb tool provide
# the required SpatiaLite plumbing.

toolname = 'spatialdb'

thisDir = Dir('.').abspath
upDir   = Dir('./../').abspath

# Define tool
def spatialdb(env):

	# get options and look for SPATIALITEDIR
    options = env.GlobalVariables()
    options.AddVariables(PathVariable('SPATIALITEDIR', 'SpatiaLite installation root.', None))
    options.Update(env)

    # other tools we need
    env.Require(['qt4', 'prefixoptions', 'sqlitedb'])
    env.EnableQt4Modules(['QtCore','QtGui'])

    env.AppendUnique(CPPPATH = upDir)
    env.AppendUnique(CPPPATH = thisDir)
    env.AppendUnique(LIBPATH = thisDir)
    if env.has_key('SPATIALITEDIR'):
        env.AppendUnique(CPPPATH=[env['SPATIALITEDIR']+'/include',])
        env.AppendUnique(LIBPATH=[env['SPATIALITEDIR']+'/lib',])

    # Add libraries
    env.AppendLibrary(toolname)
    env.AppendLibrary('spatialite')
    env.AppendLibrary('geos')
    env.AppendLibrary('geos_c')
    env.AppendLibrary('proj')
    if (env['PLATFORM'] != 'posix'):
        env.AppendLibrary('iconv')
    if (env['PLATFORM'] == 'win32'):
        env.AppendLibrary('freexl')

Export(toolname)

# Build the library
env = Environment(tools = ['default', 'doxygen', toolname])

sources = Split("""
   SpatiaLiteDB.cpp
""")

headers = Split("""
   SpatiaLiteDB.h
""")

lib = env.Library(toolname, sources)
env.Default(lib)

# Create doxygen
doxref = env.Apidocs(sources + headers, DOXYFILE_DICT={'PROJECT_NAME':toolname, 'PROJECT_NUMBER':'1.0'})
<commit_msg>Save the QC data in the soundings database.<commit_after># SpatialDB scons tool
#
# It builds the library for the SpatialDB C++ class library,
# and provides CPP and linker specifications for the header
# and libraries.
#
# SpatialDB depends on SqliteDB, which provides the interface to
# sqlite3. It also depends on SpatiaLite. Since SpatiaLite is
# is also needed by SQLiteDB, we let the sqlitedb tool provide
# the required SpatiaLite plumbing.

toolname = 'spatialdb'

thisDir = Dir('.').abspath
upDir   = Dir('./../').abspath

# Define tool
def spatialdb(env):

	# get options and look for SPATIALITEDIR
    options = env.GlobalVariables()
    options.AddVariables(PathVariable('SPATIALITEDIR', 'SpatiaLite installation root.', None))
    options.Update(env)

    # other tools we need
    env.Require(['qt4', 'prefixoptions', 'sqlitedb'])
    env.EnableQt4Modules(['QtCore','QtGui'])

    env.AppendUnique(CPPPATH = upDir)
    env.AppendUnique(CPPPATH = thisDir)
    env.AppendUnique(LIBPATH = thisDir)
    if env.has_key('SPATIALITEDIR'):
        env.AppendUnique(CPPPATH=[env['SPATIALITEDIR']+'/include',])
        env.AppendUnique(LIBPATH=[env['SPATIALITEDIR']+'/lib',])

    # Add libraries
    env.AppendLibrary(toolname)
    env.AppendLibrary('spatialite')
    env.AppendLibrary('geos')
    env.AppendLibrary('geos_c')
    env.AppendLibrary('proj')
    if (env['PLATFORM'] != 'posix'):
        env.AppendLibrary('iconv')
    if (env['PLATFORM'] == 'win32'):
        env.AppendLibrary('freexl')

Export(toolname)

# Build the library
env = Environment(tools = ['default', 'doxygen', toolname])

sources = Split("""
   SpatiaLiteDB.cpp
""")

headers = Split("""
   SpatiaLiteDB.h
""")

lib = env.Library(toolname, sources)
env.Default(lib)

# Create doxygen
doxref = env.Apidocs(sources + headers, DOXYFILE_DICT={'PROJECT_NAME':toolname, 'PROJECT_NUMBER':'1.0'})

<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import shutil
import subprocess

configDir = os.path.join(os.path.split(__file__)[0],'config')

def sextract(imageFile,catFile,psfFile=None,clobber=False,full=False,
             verbose=0,**kwargs):
	if not clobber and os.path.exists(catFile):
		if verbose > 0:
			print catFile,' already exists, skipping'
		return
	cmd = ['sex','-c',os.path.join(configDir,'bok_extract.sex')]
	cmd.extend(['-CATALOG_NAME',catFile])
	if full:
		cmd.extend(['-PARAMETERS_NAME',
		            os.path.join(configDir,'bok_catalog.par')])
		cmd.extend(['-PHOT_APERTURES',"7.5,15.0,22.0"])
		cmd.extend(['-CATALOG_TYPE','FITS_1.0'])
		if psfFile is None:
			psfFile = catFile.replace('.fits','.psf')
		cmd.extend(['-PSF_NAME',psfFile])
	else:
		cmd.extend(['-PARAMETERS_NAME',
		            os.path.join(configDir,'bok_extract.par')])
		cmd.extend(['-CATALOG_TYPE','FITS_LDAC'])
	cmd.extend([
	  '-FILTER_NAME',os.path.join(configDir,'default.conv'),
	  '-STARNNW_NAME',os.path.join(configDir,'default.nnw'),
	])
	if verbose > 2:
		cmd.extend(['-VERBOSE_TYPE','FULL'])
	elif verbose > 1:
		cmd.extend(['-VERBOSE_TYPE','NORMAL'])
	elif verbose > 0:
		print 'generating wcs catalog for ',imageFile
	cmd.append(imageFile)
	if verbose > 1:
		print ' '.join(cmd)
	subprocess.call(cmd)

def run_psfex(catFile,psfFile=None,clobber=False,verbose=0,**kwargs):
	defPsfFile = catFile.replace('.fits','.psf')
	if psfFile is not None and psfFile != defPsfFile:
		rename = True
	else:
		if psfFile is None:
			psfFile = defPsfFile
		rename = False
	if not clobber and os.path.exists(psfFile):
		if verbose > 0:
			print psfFile,' already exists, skipping'
		return
	cmd = ['psfex','-c',os.path.join(configDir,'bok90.psfex'),catFile]
	if verbose > 2:
		cmd.extend(['-VERBOSE_TYPE','FULL'])
	elif verbose > 1:
		cmd.extend(['-VERBOSE_TYPE','NORMAL'])
	elif verbose > 0:
		print 'generating psf for ',catFile
	if verbose > 1:
		print ' '.join(cmd)
	subprocess.call(cmd)
	if rename:
		shutil.move(defPsfFile,psfFile)

<commit_msg>add aperture phot routines<commit_after>#!/usr/bin/env python

import os
import shutil
import subprocess
import numpy as np
from astropy.table import Table,vstack
import fitsio

from .bokastrom import read_headers,wcs_from_header

try:
	import sep
except ImportError:
	print 'sep is not installed, cannot perform aperture photometry routines'

configDir = os.path.join(os.path.split(__file__)[0],'config')

def sextract(imageFile,catFile,psfFile=None,clobber=False,full=False,
             verbose=0,**kwargs):
	if not clobber and os.path.exists(catFile):
		if verbose > 0:
			print catFile,' already exists, skipping'
		return
	cmd = ['sex','-c',os.path.join(configDir,'bok_extract.sex')]
	cmd.extend(['-CATALOG_NAME',catFile])
	if full:
		cmd.extend(['-PARAMETERS_NAME',
		            os.path.join(configDir,'bok_catalog.par')])
		cmd.extend(['-PHOT_APERTURES',"7.5,15.0,22.0"])
		cmd.extend(['-CATALOG_TYPE','FITS_1.0'])
		if psfFile is None:
			psfFile = catFile.replace('.fits','.psf')
		cmd.extend(['-PSF_NAME',psfFile])
	else:
		cmd.extend(['-PARAMETERS_NAME',
		            os.path.join(configDir,'bok_extract.par')])
		cmd.extend(['-CATALOG_TYPE','FITS_LDAC'])
	cmd.extend([
	  '-FILTER_NAME',os.path.join(configDir,'default.conv'),
	  '-STARNNW_NAME',os.path.join(configDir,'default.nnw'),
	])
	if verbose > 2:
		cmd.extend(['-VERBOSE_TYPE','FULL'])
	elif verbose > 1:
		cmd.extend(['-VERBOSE_TYPE','NORMAL'])
	elif verbose > 0:
		print 'generating wcs catalog for ',imageFile
	cmd.append(imageFile)
	if verbose > 1:
		print ' '.join(cmd)
	subprocess.call(cmd)

def run_psfex(catFile,psfFile=None,clobber=False,verbose=0,**kwargs):
	defPsfFile = catFile.replace('.fits','.psf')
	if psfFile is not None and psfFile != defPsfFile:
		rename = True
	else:
		if psfFile is None:
			psfFile = defPsfFile
		rename = False
	if not clobber and os.path.exists(psfFile):
		if verbose > 0:
			print psfFile,' already exists, skipping'
		return
	cmd = ['psfex','-c',os.path.join(configDir,'bok90.psfex'),catFile]
	if verbose > 2:
		cmd.extend(['-VERBOSE_TYPE','FULL'])
	elif verbose > 1:
		cmd.extend(['-VERBOSE_TYPE','NORMAL'])
	elif verbose > 0:
		print 'generating psf for ',catFile
	if verbose > 1:
		print ' '.join(cmd)
	subprocess.call(cmd)
	if rename:
		shutil.move(defPsfFile,psfFile)

def aper_phot(image,hdr,ra,dec,aperRad,badPixMask,edge_buf=5):
	w = wcs_from_header(hdr)
	y,x = w.wcs_world2pix(ra,dec,0,ra_dec_order=True)
	ii = np.where((x>edge_buf) & (y>edge_buf) &
	              (x<4096-edge_buf) & (y<4032-edge_buf))[0]
	#bkg = sep.Background(image)
	nObj,nAper = len(ii),len(aperRad)
	cts = np.empty((nObj,nAper),dtype=np.float32)
	ctserr = np.empty((nObj,nAper),dtype=np.float32)
	flags = np.empty((nObj,nAper),dtype=np.int32)
	for j,aper in enumerate(aperRad):
		rv = sep.sum_circle(image,x[ii],y[ii],aper,mask=badPixMask,
		                    gain=1.0,bkgann=(25.,32.))
		cts[:,j],ctserr[:,j],flags[:,j] = rv
	return x[ii],y[ii],ii,cts,ctserr,flags

def aper_phot_image(imageFile,ra,dec,aperRad,badPixMask,
	                aHeadFile=None,**kwargs):
	if aHeadFile is not None:
		hdrs = read_headers(aHeadFile)
	tabs = []
	fitsData = fitsio.FITS(imageFile)
	for i,hdu in enumerate(fitsData[1:]):
		im = hdu.read()
		if aHeadFile is None:
			hdr = hdu.read_header()
		else:
			hdr = hdrs[i]
		phot = aper_phot(im,hdr,ra,dec,aperRad,badPixMask,**kwargs)
		n = len(phot[0])
		if n==0:
			continue
		# add ccd number
		phot += (np.repeat(i+1,n),)
		t = Table(phot,
		          names=('x','y','idx','counts','countsErr','flags','ccdNum'),
		          dtype=('f4','f4','i4','f4','f4','i4','i4'))
		tabs.append(t)
	phot = vstack(tabs)
	return phot

<|endoftext|>