<commit_before># Copyright (c) 2014 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#           http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import json
import abc

import launchpadstats

LOG = logging.getLogger('launchpadstats')

# characters used as separator between items in the CSV output
CSV_SEPARATOR = '; '
# which metrics should be skipped when a sum is made of the metrics
SKIP_FROM_SUM = ['marks', 'loc']

class Table(object):
    """Base class for the table generators.
    """
    __metaclass__ = abc.ABCMeta

    # will be shown on the top left corner, change it in specific
    # implementation of the table
    header_info = "column/row"
    # if True, it will flip the rows and columns in the CSV representation
    # (transpose the matrix)
    _flip = False
    # show the total of the data at the end of the table
    _show_sum = False

    def __init__(self, people, releases, metrics, **kwargs):
        """Set filters for the queries.

        :param people: comma-separated string with list of user IDs in
            the stackalytics web page
        :param releases: comma-separated string with list of OpenStack
            releases that will be passed as the 'release' parameter to the
            query
        :param metrics: comma-separated string with list of metrics to show in
            the CSV table (and in some cases, create a sum total of them)
        """
        self.people = people.split(',')
        self.releases = releases.split(',')
        self.metrics = metrics.split(',')
        self._data = dict()
        self._data_matrix = list()

    @abc.abstractmethod
    def generate(self):
        """Do a set of queries on stackalysis and save the data.

        If `self._flip` is True, the matrix of data will be flipped
        (transposed), i.e. the the columns and rows will be swapped.

        The queries depend on what kind of table we need to generate.
        The result should be saved in `self._data`.
        """
        pass

    def json(self):
        """Return the JSON raw data, with all the metrics"""
        return self._data

    def csv(self, delimiter=CSV_SEPARATOR):
        """Return a string with a CSV representation of the data.

        The first item on the top left is going to be the `self.header_info`
        with the description of what the columns and rows are.

        :param delimiter: use to separate items
        """
        return '\n'.join([delimiter.join(line) for line in self._data_matrix])

    def html(self):
        """Return a string with an HTML representation of the data.

        The first item on the top left is going to be the `self.header_info`
        with the description of what the columns and rows are.
        """
        return _get_html_table(self._data_matrix)

    def _parse_data(self):
        self._add_metrics_sum()
        LOG.info(json.dumps(self._data, indent=4))
        # header (or first collumn if it gets flipped)
        header = self._data.keys()
        row = [self.header_info] + header
        result = [row]
        # print data
        for metric in self.metrics + ['sum']:
            if metric == 'sum' and not self._show_sum:
                # don't show the sum if it's not in the data
                continue
            row = [self._prettify_metric(metric)]
            for item in header:
                row.append(self._prettify_data(self._data[item], metric))
            result.append(row)

        if self._flip:
            # transpose the matrix
            result = zip(*result)
        self._data_matrix = result

    def _add_metrics_sum(self):
        """Add up the fields in `self.metrics`.

        Skip the items in `SKIP_FROM_SUM` even if they are given in
        `self.metrics`, e.g. the review marks.
        The result should be saved in the `self._data[item]['sum']`. It will be
        printed in the CSV representation."""
        for key, metrics in self._data.iteritems():
            total = 0
            for metric, value in metrics.iteritems():
                if metric in self.metrics and metric not in SKIP_FROM_SUM:
                    total += value
            self._data[key]['sum'] = total

    def _prettify_metric(self, metric):
        """Change the names of some metrics to something more readable.
        """
        if metric == 'reviews':
            return 'reviews (-2, -1, +1, +2, A)'
        else:
            return metric

    def _prettify_data(self, data, metric):
        """Change some data (e.g. review marks) into something more readable.
        """
        if metric == 'reviews':
            marks = data['marks']
            result = [str(marks[i]) for i in ['-2', '-1', '1', '2', 'A']]
            result = '(' + ', '.join(result) + ')'
        else:
            result = str(data[metric])
        return result

class GroupMetricsTable(Table):
    """Show the metrics of the group per release in columns.

    The 'group' is defined as the list of people passed in the 'people'
    parameter. One metric per line is shown, releases are columns. Shows a sum
    of the metrics per release.
    """
    header_info = 'metric/release'
    _show_sum = True

    def generate(self):
        for release in self.releases:
            params = {'release': release, 'user_id': ','.join(self.people)}
            stats = launchpadstats.get_stats(params)
            self._data[release] = stats['contribution']
        self._parse_data()

class UserMetricsTable(Table):
    """Show metrics of each user on a separate line.

    For each person in the 'people' parameter, display a line with their
    metrics (given in the 'metrics' param), summed up in all the releases
    specified in the 'releases' param.
    """
    header_info = "user/metric"
    _flip = True

    def generate(self):
        for person in self.people:
            params = {'release': ','.join(self.releases), 'user_id': person}
            stats = launchpadstats.get_stats(params)
            self._data[person] = stats['contribution']
        self._parse_data()

    def html(self):
        """Generate HTML representation and add links to users."""
        new_matrix = [self._data_matrix[0]]
        for row in self._data_matrix[1:]:
            row = list(row)
            user = row[0]
            row[0] = '%s' \
                     % (launchpadstats.STACKALYTICS_URL,
                        user, ','.join(self.releases), user)
            new_matrix.append(row)
        return _get_html_table(new_matrix)

def get_table(table_type, params):
    """Return the correct Table subclass based on table_type.

    :param table_type: name of one of the implemented Table subclasses, e.g.
        'group-metrics', 'user-metrics'. The 'group-metrics' type will be used
        as default, if None is passed.
    :param params: dictionary of parameters that will be passed to the init
        method of the Table subclass
    """
    if table_type == 'group-metrics' or table_type is None:
        return launchpadstats.GroupMetricsTable(**params)
    elif table_type == 'user-metrics':
        return launchpadstats.UserMetricsTable(**params)
    else:
        raise Exception("Unknown table type '%s'" % table_type)

def _get_html_table(matrix):
    result = '
\n'
    for row in matrix:
        result += '
\n    '
        result += '	\n    '.join(row)
        result += '	\n
\n'
    result += '
'
    return result
<commit_msg>Fix #1, keep order of releases or people<commit_after># Copyright (c) 2014 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#           http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import json
import abc
import collections

import launchpadstats

LOG = logging.getLogger('launchpadstats')

# characters used as separator between items in the CSV output
CSV_SEPARATOR = '; '
# which metrics should be skipped when a sum is made of the metrics
SKIP_FROM_SUM = ['marks', 'loc']

class Table(object):
    """Base class for the table generators.
    """
    __metaclass__ = abc.ABCMeta

    # will be shown on the top left corner, change it in specific
    # implementation of the table
    header_info = "column/row"
    # if True, it will flip the rows and columns in the CSV representation
    # (transpose the matrix)
    _flip = False
    # show the total of the data at the end of the table
    _show_sum = False

    def __init__(self, people, releases, metrics, **kwargs):
        """Set filters for the queries.

        :param people: comma-separated string with list of user IDs in
            the stackalytics web page
        :param releases: comma-separated string with list of OpenStack
            releases that will be passed as the 'release' parameter to the
            query
        :param metrics: comma-separated string with list of metrics to show in
            the CSV table (and in some cases, create a sum total of them)
        """
        self.people = people.split(',')
        self.releases = releases.split(',')
        self.metrics = metrics.split(',')
        self._data = collections.OrderedDict()
        self._data_matrix = list()

    @abc.abstractmethod
    def generate(self):
        """Do a set of queries on stackalysis and save the data.

        If `self._flip` is True, the matrix of data will be flipped
        (transposed), i.e. the the columns and rows will be swapped.

        The queries depend on what kind of table we need to generate.
        The result should be saved in `self._data`.
        """
        pass

    def json(self):
        """Return the JSON raw data, with all the metrics"""
        return self._data

    def csv(self, delimiter=CSV_SEPARATOR):
        """Return a string with a CSV representation of the data.

        The first item on the top left is going to be the `self.header_info`
        with the description of what the columns and rows are.

        :param delimiter: use to separate items
        """
        return '\n'.join([delimiter.join(line) for line in self._data_matrix])

    def html(self):
        """Return a string with an HTML representation of the data.

        The first item on the top left is going to be the `self.header_info`
        with the description of what the columns and rows are.
        """
        return _get_html_table(self._data_matrix)

    def _parse_data(self):
        self._add_metrics_sum()
        LOG.info(json.dumps(self._data, indent=4))
        # header (or first collumn if it gets flipped)
        header = self._data.keys()
        row = [self.header_info] + header
        result = [row]
        # print data
        for metric in self.metrics + ['sum']:
            if metric == 'sum' and not self._show_sum:
                # don't show the sum if it's not in the data
                continue
            row = [self._prettify_metric(metric)]
            for item in header:
                row.append(self._prettify_data(self._data[item], metric))
            result.append(row)

        if self._flip:
            # transpose the matrix
            result = zip(*result)
        self._data_matrix = result

    def _add_metrics_sum(self):
        """Add up the fields in `self.metrics`.

        Skip the items in `SKIP_FROM_SUM` even if they are given in
        `self.metrics`, e.g. the review marks.
        The result should be saved in the `self._data[item]['sum']`. It will be
        printed in the CSV representation."""
        for key, metrics in self._data.iteritems():
            total = 0
            for metric, value in metrics.iteritems():
                if metric in self.metrics and metric not in SKIP_FROM_SUM:
                    total += value
            self._data[key]['sum'] = total

    def _prettify_metric(self, metric):
        """Change the names of some metrics to something more readable.
        """
        if metric == 'reviews':
            return 'reviews (-2, -1, +1, +2, A)'
        else:
            return metric

    def _prettify_data(self, data, metric):
        """Change some data (e.g. review marks) into something more readable.
        """
        if metric == 'reviews':
            marks = data['marks']
            result = [str(marks[i]) for i in ['-2', '-1', '1', '2', 'A']]
            result = '(' + ', '.join(result) + ')'
        else:
            result = str(data[metric])
        return result

class GroupMetricsTable(Table):
    """Show the metrics of the group per release in columns.

    The 'group' is defined as the list of people passed in the 'people'
    parameter. One metric per line is shown, releases are columns. Shows a sum
    of the metrics per release.
    """
    header_info = 'metric/release'
    _show_sum = True

    def generate(self):
        for release in self.releases:
            params = {'release': release, 'user_id': ','.join(self.people)}
            stats = launchpadstats.get_stats(params)
            self._data[release] = stats['contribution']
        self._parse_data()

class UserMetricsTable(Table):
    """Show metrics of each user on a separate line.

    For each person in the 'people' parameter, display a line with their
    metrics (given in the 'metrics' param), summed up in all the releases
    specified in the 'releases' param.
    """
    header_info = "user/metric"
    _flip = True

    def generate(self):
        for person in self.people:
            params = {'release': ','.join(self.releases), 'user_id': person}
            stats = launchpadstats.get_stats(params)
            self._data[person] = stats['contribution']
        self._parse_data()

    def html(self):
        """Generate HTML representation and add links to users."""
        new_matrix = [self._data_matrix[0]]
        for row in self._data_matrix[1:]:
            row = list(row)
            user = row[0]
            row[0] = '%s' \
                     % (launchpadstats.STACKALYTICS_URL,
                        user, ','.join(self.releases), user)
            new_matrix.append(row)
        return _get_html_table(new_matrix)

def get_table(table_type, params):
    """Return the correct Table subclass based on table_type.

    :param table_type: name of one of the implemented Table subclasses, e.g.
        'group-metrics', 'user-metrics'. The 'group-metrics' type will be used
        as default, if None is passed.
    :param params: dictionary of parameters that will be passed to the init
        method of the Table subclass
    """
    if table_type == 'group-metrics' or table_type is None:
        return launchpadstats.GroupMetricsTable(**params)
    elif table_type == 'user-metrics':
        return launchpadstats.UserMetricsTable(**params)
    else:
        raise Exception("Unknown table type '%s'" % table_type)

def _get_html_table(matrix):
    result = '
\n'
    for row in matrix:
        result += '
\n    '
        result += '	\n    '.join(row)
        result += '	\n
\n'
    result += '
'
    return result
<|endoftext|>


<commit_before>"""
Parsing module for models.py files. Extracts information in a more reliable
way than inspect + regexes.
"""

import re
import inspect
import parser
import symbol
import token
import keyword

from django.db import models

def name_that_thing(thing):
    "Turns a symbol/token int into its name."
    for name in dir(symbol):
        if getattr(symbol, name) == thing:
            return "symbol.%s" % name
    for name in dir(token):
        if getattr(token, name) == thing:
            return "token.%s" % name
    return str(thing)

def thing_that_name(name):
    "Turns a name of a symbol/token into its integer value."
    if name in dir(symbol):
        return getattr(symbol, name)
    if name in dir(token):
        return getattr(token, name)
    raise ValueError("Cannot convert '%s'" % name)

def prettyprint(tree, indent=0, omit_singles=False):
    "Prettyprints the tree, with symbol/token names. For debugging."
    if omit_singles and isinstance(tree, tuple) and len(tree) == 2:
        return prettyprint(tree[1], indent, omit_singles)
    if isinstance(tree, tuple):
        return " (\n%s\n" % "".join([prettyprint(x, indent+1) for x in tree]) + \
            (" " * indent) + ")"
    elif isinstance(tree, int):
        return (" " * indent) + name_that_thing(tree)
    else:
        return " " + repr(tree)

class STTree(object):

    "A syntax tree wrapper class."

    def __init__(self, tree):
        self.tree = tree

    def __eq__(self, other):
        return other.tree == self.tree

    def __hash__(self):
        return hash(self.tree)

    @property
    def root(self):
        return self.tree[0]

    @property
    def value(self):
        return self.tree

    def walk(self, recursive=True):
        """
        Yields (symbol, subtree) for the entire subtree.
        Comes out with node 1, node 1's children, node 2, etc.
        """
        stack = [self.tree]
        done_outer = False
        while stack:
            atree = stack.pop()
            if isinstance(atree, tuple):
                if done_outer:
                    yield atree[0], STTree(atree)
                if recursive or not done_outer:
                    for bit in reversed(atree[1:]):
                        stack.append(bit)
                    done_outer = True

    def flatten(self):
        "Yields the tokens/symbols in the tree only, in order."
        bits = []
        for sym, subtree in self.walk():
            if sym in token_map:
                bits.append(sym)
            elif sym == token.NAME:
                bits.append(subtree.value)
            elif sym == token.STRING:
                bits.append(subtree.value)
            elif sym == token.NUMBER:
                bits.append(subtree.value)
        return bits

    def reform(self):
        "Prints how the tree's input probably looked."
        return reform(self.flatten())

    def findAllType(self, ntype, recursive=True):
        "Returns all nodes with the given type in the tree."
        for symbol, subtree in self.walk(recursive=recursive):
            if symbol == ntype:
                yield subtree

    def find(self, selector):
        """
        Searches the syntax tree with a CSS-like selector syntax.
        You can use things like 'suite simple_stmt', 'suite, simple_stmt'
        or 'suite > simple_stmt'. Not guaranteed to return in order.
        """
        # Split up the overall parts
        patterns = [x.strip() for x in selector.split(",")]
        results = []
        for pattern in patterns:
            # Split up the parts
            parts = re.split(r'(?:[\s]|(>))+', pattern)
            # Take the first part, use it for results
            if parts[0] == "^":
                subresults = [self]
            else:
                subresults = list(self.findAllType(thing_that_name(parts[0])))
            recursive = True
            # For each remaining part, do something
            for part in parts[1:]:
                if not subresults:
                    break
                if part == ">":
                    recursive = False
                elif not part:
                    pass
                else:
                    thing = thing_that_name(part)
                    newresults = [
                        list(tree.findAllType(thing, recursive))
                        for tree in subresults
                    ]
                    subresults = []
                    for stuff in newresults:
                        subresults.extend(stuff)
                    recursive = True
            results.extend(subresults)
        return results

    def __str__(self):
        return prettyprint(self.tree)
    __repr__ = __str__

def get_model_tree(model):
    # Get the source of the model's file
    source = open(inspect.getsourcefile(model)).read()
    tree = STTree(parser.suite(source).totuple())
    # Now, we have to find it
    for poss in tree.find("compound_stmt"):
        if poss.value[1][0] == symbol.classdef and \
           poss.value[1][2][1].lower() == model.__name__.lower():
            # This is the tree
            return poss

token_map = {
    token.DOT: ".",
    token.LPAR: "(",
    token.RPAR: ")",
    token.EQUAL: "=",
    token.EQEQUAL: "==",
    token.COMMA: ",",
    token.LSQB: "[",
    token.RSQB: "]",
    token.AMPER: "&",
    token.BACKQUOTE: "`",
    token.CIRCUMFLEX: "^",
    token.CIRCUMFLEXEQUAL: "^=",
    token.COLON: ":",
    token.DOUBLESLASH: "//",
    token.DOUBLESLASHEQUAL: "//=",
    token.DOUBLESTAR: "**",
    token.DOUBLESLASHEQUAL: "**=",
    token.GREATER: ">",
    token.LESS: "<",
    token.GREATEREQUAL: ">=",
    token.LESSEQUAL: "<=",
    token.LBRACE: "{",
    token.RBRACE: "}",
    token.SEMI: ";",
    token.PLUS: "+",
    token.MINUS: "-",
    token.STAR: "*",
    token.SLASH: "/",
    token.VBAR: "|",
    token.PERCENT: "%",
    token.TILDE: "~",
    token.AT: "@",
    token.NOTEQUAL: "!=",
    token.LEFTSHIFT: "<<",
    token.RIGHTSHIFT: ">>",
    token.LEFTSHIFTEQUAL: "<<=",
    token.RIGHTSHIFTEQUAL: ">>=",
    token.PLUSEQUAL: "+=",
    token.MINEQUAL: "-=",
    token.STAREQUAL: "*=",
    token.SLASHEQUAL: "/=",
    token.VBAREQUAL: "|=",
    token.PERCENTEQUAL: "%=",
    token.AMPEREQUAL: "&=",
}

def reform(bits):
    "Returns the string that the list of tokens/symbols 'bits' represents"
    output = ""
    for bit in bits:
        if bit in token_map:
            output += token_map[bit]
        elif bit[0] in [token.NAME, token.STRING, token.NUMBER]:
            if keyword.iskeyword(bit[1]):
                output += " %s " % bit[1]
            else:
                if bit[1] not in symbol.sym_name:
                    output += bit[1]
    return output

def parse_arguments(argstr):
    """
    Takes a string representing arguments and returns the positional and
    keyword argument list and dict respectively.
    All the entries in these are python source, except the dict keys.
    """
    # Get the tree
    tree = STTree(parser.suite(argstr).totuple())

    # Initialise the lists
    curr_kwd = None
    args = []
    kwds = {}

    # Walk through, assigning things
    testlists = tree.find("testlist")
    for i, testlist in enumerate(testlists):
        # BTW: A testlist is to the left or right of an =.
        items = list(testlist.walk(recursive=False))
        for j, item in enumerate(items):
            if item[0] == symbol.test:
                if curr_kwd:
                    kwds[curr_kwd] = item[1].reform()
                    curr_kwd = None
                elif j == len(items)-1 and i != len(testlists)-1:
                    # Last item in a group must be a keyword, unless it's last overall
                    curr_kwd = item[1].reform()
                else:
                    args.append(item[1].reform())
    return args, kwds

def extract_field(tree):
    # Collapses the tree and tries to parse it as a field def
    bits = tree.flatten()
    ## Check it looks right:
    # Second token should be equals
    if len(bits) < 2 or bits[1] != token.EQUAL:
        return
    ## Split into meaningful sections
    name = bits[0][1]
    declaration = bits[2:]
    # Find the first LPAR; stuff before that is the class.
    try:
        lpar_at = declaration.index(token.LPAR)
    except ValueError:
        return
    clsname = reform(declaration[:lpar_at])
    # Now, inside that, find the last RPAR, and we'll take the stuff between
    # them as the arguments
    declaration.reverse()
    rpar_at = (len(declaration) - 1) - declaration.index(token.RPAR)
    declaration.reverse()
    args = declaration[lpar_at+1:rpar_at]
    # Now, extract the arguments as a list and dict
    try:
        args, kwargs = parse_arguments(reform(args))
    except SyntaxError:
        return
    # OK, extract and reform it
    return name, clsname, args, kwargs

def get_model_fields(model):
    """
    Given a model class, will return the dict of name: field_constructor
    mappings.
    """
    tree = get_model_tree(model)
    if tree is None:
        raise ValueError("Cannot find source for model '%s'." % model)
    possible_field_defs = tree.find("^ > classdef > suite > stmt > simple_stmt > small_stmt > expr_stmt")
    field_defs = {}

    # Go through all the found defns, and try to parse them
    for pfd in possible_field_defs:
        field = extract_field(pfd)
        if field:
            field_defs[field[0]] = field[1:]

    inherited_fields = {}
    # Go through all bases (that are themselves models, but not Model)
    for base in model.__bases__:
        if base != models.Model and issubclass(base, models.Model):
            inherited_fields.update(get_model_fields(base))

    # Now, go through all the fields and try to get their definition
    fields = {}
    for field in model._meta.local_fields:
        # Get its name
        fieldname = field.name
        if isinstance(field, models.related.RelatedObject):
            continue
        # Now, try to get the defn
        if fieldname in field_defs:
            fields[fieldname] = field_defs[fieldname]
        # Try the South definition workaround?
        elif hasattr(field, 'south_field_triple'):
            fields[fieldname] = field.south_field_triple()
        elif hasattr(field, 'south_field_definition'):
            print "Your custom field %s provides the outdated south_field_definition method.\nPlease consider implementing south_field_triple too; it's more reliably evaluated." % field
            fields[fieldname] = field.south_field_definition()
        # Try a parent?
        elif fieldname in inherited_fields:
            fields[fieldname] = inherited_fields[fieldname]
        # Is it a _ptr?
        elif fieldname.endswith("_ptr"):
            fields[fieldname] = ("models.OneToOneField", [field.rel.to.__name__], {})
        # Try a default for 'id'.
        elif fieldname == "id":
            fields[fieldname] = ("models.AutoField", [], {"primary_key": "True"})
        else:
            fields[fieldname] = None

    return fields

def get_model_meta(model):
    """
    Given a model class, will return the dict representing the Meta class.
    """
    tree = get_model_tree(model)

    result = {}

    # First, try to get any unusual inherited classes
    for base in model.__bases__:
        if base is not models.Model:
            if "_bases" not in result:
                result['_bases'] = []
            result['_bases'].append(base.__module__ + "." + base.__name__)

    # Find all classes exactly two levels deep
    possible_meta_classes = set(tree.find("classdef classdef"))
    possible_meta_classes.difference(set(tree.find("classdef classdef classdef")))

    # Select only those called 'Meta', and expand all their assignments
    possible_meta_classes = [
        tree.find("^ > suite > stmt > simple_stmt > small_stmt > expr_stmt")
        for tree in possible_meta_classes
        if tree.value[2][1] == "Meta"
    ]

    if possible_meta_classes:
        # Now, for each possible definition, try it. (Only for last Meta,
        # since that's how python interprets it)
        for defn in possible_meta_classes[-1]:
            bits = defn.flatten()
            if len(bits) > 1 and bits[1] == token.EQUAL:
                result[bits[0][1]] = reform(bits[2:])

    return result or None
<commit_msg>Fix abstract model inheritance<commit_after>"""
Parsing module for models.py files. Extracts information in a more reliable
way than inspect + regexes.
"""

import re
import inspect
import parser
import symbol
import token
import keyword

from django.db import models

def name_that_thing(thing):
    "Turns a symbol/token int into its name."
    for name in dir(symbol):
        if getattr(symbol, name) == thing:
            return "symbol.%s" % name
    for name in dir(token):
        if getattr(token, name) == thing:
            return "token.%s" % name
    return str(thing)

def thing_that_name(name):
    "Turns a name of a symbol/token into its integer value."
    if name in dir(symbol):
        return getattr(symbol, name)
    if name in dir(token):
        return getattr(token, name)
    raise ValueError("Cannot convert '%s'" % name)

def prettyprint(tree, indent=0, omit_singles=False):
    "Prettyprints the tree, with symbol/token names. For debugging."
    if omit_singles and isinstance(tree, tuple) and len(tree) == 2:
        return prettyprint(tree[1], indent, omit_singles)
    if isinstance(tree, tuple):
        return " (\n%s\n" % "".join([prettyprint(x, indent+1) for x in tree]) + \
            (" " * indent) + ")"
    elif isinstance(tree, int):
        return (" " * indent) + name_that_thing(tree)
    else:
        return " " + repr(tree)

class STTree(object):

    "A syntax tree wrapper class."

    def __init__(self, tree):
        self.tree = tree

    def __eq__(self, other):
        return other.tree == self.tree

    def __hash__(self):
        return hash(self.tree)

    @property
    def root(self):
        return self.tree[0]

    @property
    def value(self):
        return self.tree

    def walk(self, recursive=True):
        """
        Yields (symbol, subtree) for the entire subtree.
        Comes out with node 1, node 1's children, node 2, etc.
        """
        stack = [self.tree]
        done_outer = False
        while stack:
            atree = stack.pop()
            if isinstance(atree, tuple):
                if done_outer:
                    yield atree[0], STTree(atree)
                if recursive or not done_outer:
                    for bit in reversed(atree[1:]):
                        stack.append(bit)
                    done_outer = True

    def flatten(self):
        "Yields the tokens/symbols in the tree only, in order."
        bits = []
        for sym, subtree in self.walk():
            if sym in token_map:
                bits.append(sym)
            elif sym == token.NAME:
                bits.append(subtree.value)
            elif sym == token.STRING:
                bits.append(subtree.value)
            elif sym == token.NUMBER:
                bits.append(subtree.value)
        return bits

    def reform(self):
        "Prints how the tree's input probably looked."
        return reform(self.flatten())

    def findAllType(self, ntype, recursive=True):
        "Returns all nodes with the given type in the tree."
        for symbol, subtree in self.walk(recursive=recursive):
            if symbol == ntype:
                yield subtree

    def find(self, selector):
        """
        Searches the syntax tree with a CSS-like selector syntax.
        You can use things like 'suite simple_stmt', 'suite, simple_stmt'
        or 'suite > simple_stmt'. Not guaranteed to return in order.
        """
        # Split up the overall parts
        patterns = [x.strip() for x in selector.split(",")]
        results = []
        for pattern in patterns:
            # Split up the parts
            parts = re.split(r'(?:[\s]|(>))+', pattern)
            # Take the first part, use it for results
            if parts[0] == "^":
                subresults = [self]
            else:
                subresults = list(self.findAllType(thing_that_name(parts[0])))
            recursive = True
            # For each remaining part, do something
            for part in parts[1:]:
                if not subresults:
                    break
                if part == ">":
                    recursive = False
                elif not part:
                    pass
                else:
                    thing = thing_that_name(part)
                    newresults = [
                        list(tree.findAllType(thing, recursive))
                        for tree in subresults
                    ]
                    subresults = []
                    for stuff in newresults:
                        subresults.extend(stuff)
                    recursive = True
            results.extend(subresults)
        return results

    def __str__(self):
        return prettyprint(self.tree)
    __repr__ = __str__

def get_model_tree(model):
    # Get the source of the model's file
    source = open(inspect.getsourcefile(model)).read()
    tree = STTree(parser.suite(source).totuple())
    # Now, we have to find it
    for poss in tree.find("compound_stmt"):
        if poss.value[1][0] == symbol.classdef and \
           poss.value[1][2][1].lower() == model.__name__.lower():
            # This is the tree
            return poss

token_map = {
    token.DOT: ".",
    token.LPAR: "(",
    token.RPAR: ")",
    token.EQUAL: "=",
    token.EQEQUAL: "==",
    token.COMMA: ",",
    token.LSQB: "[",
    token.RSQB: "]",
    token.AMPER: "&",
    token.BACKQUOTE: "`",
    token.CIRCUMFLEX: "^",
    token.CIRCUMFLEXEQUAL: "^=",
    token.COLON: ":",
    token.DOUBLESLASH: "//",
    token.DOUBLESLASHEQUAL: "//=",
    token.DOUBLESTAR: "**",
    token.DOUBLESLASHEQUAL: "**=",
    token.GREATER: ">",
    token.LESS: "<",
    token.GREATEREQUAL: ">=",
    token.LESSEQUAL: "<=",
    token.LBRACE: "{",
    token.RBRACE: "}",
    token.SEMI: ";",
    token.PLUS: "+",
    token.MINUS: "-",
    token.STAR: "*",
    token.SLASH: "/",
    token.VBAR: "|",
    token.PERCENT: "%",
    token.TILDE: "~",
    token.AT: "@",
    token.NOTEQUAL: "!=",
    token.LEFTSHIFT: "<<",
    token.RIGHTSHIFT: ">>",
    token.LEFTSHIFTEQUAL: "<<=",
    token.RIGHTSHIFTEQUAL: ">>=",
    token.PLUSEQUAL: "+=",
    token.MINEQUAL: "-=",
    token.STAREQUAL: "*=",
    token.SLASHEQUAL: "/=",
    token.VBAREQUAL: "|=",
    token.PERCENTEQUAL: "%=",
    token.AMPEREQUAL: "&=",
}

def reform(bits):
    "Returns the string that the list of tokens/symbols 'bits' represents"
    output = ""
    for bit in bits:
        if bit in token_map:
            output += token_map[bit]
        elif bit[0] in [token.NAME, token.STRING, token.NUMBER]:
            if keyword.iskeyword(bit[1]):
                output += " %s " % bit[1]
            else:
                if bit[1] not in symbol.sym_name:
                    output += bit[1]
    return output

def parse_arguments(argstr):
    """
    Takes a string representing arguments and returns the positional and
    keyword argument list and dict respectively.
    All the entries in these are python source, except the dict keys.
    """
    # Get the tree
    tree = STTree(parser.suite(argstr).totuple())

    # Initialise the lists
    curr_kwd = None
    args = []
    kwds = {}

    # Walk through, assigning things
    testlists = tree.find("testlist")
    for i, testlist in enumerate(testlists):
        # BTW: A testlist is to the left or right of an =.
        items = list(testlist.walk(recursive=False))
        for j, item in enumerate(items):
            if item[0] == symbol.test:
                if curr_kwd:
                    kwds[curr_kwd] = item[1].reform()
                    curr_kwd = None
                elif j == len(items)-1 and i != len(testlists)-1:
                    # Last item in a group must be a keyword, unless it's last overall
                    curr_kwd = item[1].reform()
                else:
                    args.append(item[1].reform())
    return args, kwds

def extract_field(tree):
    # Collapses the tree and tries to parse it as a field def
    bits = tree.flatten()
    ## Check it looks right:
    # Second token should be equals
    if len(bits) < 2 or bits[1] != token.EQUAL:
        return
    ## Split into meaningful sections
    name = bits[0][1]
    declaration = bits[2:]
    # Find the first LPAR; stuff before that is the class.
    try:
        lpar_at = declaration.index(token.LPAR)
    except ValueError:
        return
    clsname = reform(declaration[:lpar_at])
    # Now, inside that, find the last RPAR, and we'll take the stuff between
    # them as the arguments
    declaration.reverse()
    rpar_at = (len(declaration) - 1) - declaration.index(token.RPAR)
    declaration.reverse()
    args = declaration[lpar_at+1:rpar_at]
    # Now, extract the arguments as a list and dict
    try:
        args, kwargs = parse_arguments(reform(args))
    except SyntaxError:
        return
    # OK, extract and reform it
    return name, clsname, args, kwargs

def get_model_fields(model):
    """
    Given a model class, will return the dict of name: field_constructor
    mappings.
    """
    tree = get_model_tree(model)
    if tree is None:
        raise ValueError("Cannot find source for model '%s'." % model)
    possible_field_defs = tree.find("^ > classdef > suite > stmt > simple_stmt > small_stmt > expr_stmt")
    field_defs = {}

    # Go through all the found defns, and try to parse them
    for pfd in possible_field_defs:
        field = extract_field(pfd)
        if field:
            field_defs[field[0]] = field[1:]

    inherited_fields = {}
    # Go through all bases (that are themselves models, but not Model)
    for base in model.__bases__:
        if base != models.Model and issubclass(base, models.Model):
            inherited_fields.update(get_model_fields(base))

    # Now, go through all the fields and try to get their definition
    fields = {}
    for field in model._meta.local_fields:
        # Get its name
        fieldname = field.name
        if isinstance(field, models.related.RelatedObject):
            continue
        # Now, try to get the defn
        if fieldname in field_defs:
            fields[fieldname] = field_defs[fieldname]
        # Try the South definition workaround?
        elif hasattr(field, 'south_field_triple'):
            fields[fieldname] = field.south_field_triple()
        elif hasattr(field, 'south_field_definition'):
            print "Your custom field %s provides the outdated south_field_definition method.\nPlease consider implementing south_field_triple too; it's more reliably evaluated." % field
            fields[fieldname] = field.south_field_definition()
        # Try a parent?
        elif fieldname in inherited_fields:
            fields[fieldname] = inherited_fields[fieldname]
        # Is it a _ptr?
        elif fieldname.endswith("_ptr"):
            fields[fieldname] = ("models.OneToOneField", [field.rel.to.__name__], {})
        # Try a default for 'id'.
        elif fieldname == "id":
            fields[fieldname] = ("models.AutoField", [], {"primary_key": "True"})
        else:
            fields[fieldname] = None

    return fields

def get_model_meta(model):
    """
    Given a model class, will return the dict representing the Meta class.
    """
    tree = get_model_tree(model)

    result = {}

    # First, try to get any unusual inherited classes
    for base in model.__bases__:
        if base is not models.Model:
            if not base._meta.abstract:
                # Abstract models' fields are included anyway, and we don't
                # want extra dependencies
                if "_bases" not in result:
                    result['_bases'] = []
                result['_bases'].append(base.__module__ + "." + base.__name__)

    # Find all classes exactly two levels deep
    possible_meta_classes = set(tree.find("classdef classdef"))
    possible_meta_classes.difference(set(tree.find("classdef classdef classdef")))

    # Select only those called 'Meta', and expand all their assignments
    possible_meta_classes = [
        tree.find("^ > suite > stmt > simple_stmt > small_stmt > expr_stmt")
        for tree in possible_meta_classes
        if tree.value[2][1] == "Meta"
    ]

    if possible_meta_classes:
        # Now, for each possible definition, try it. (Only for last Meta,
        # since that's how python interprets it)
        for defn in possible_meta_classes[-1]:
            bits = defn.flatten()
            if len(bits) > 1 and bits[1] == token.EQUAL:
                result[bits[0][1]] = reform(bits[2:])

    return result or None
<|endoftext|>


<commit_before># Copyright 2009 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Parse the history file for just about anything."""
import glob
import operator
import os
import os.path
import re
import sys
import time
import threading

def sourceToTitle(source):
  """Convert a source tuple to a title."""
  (short_name, full_name, num_hosts) = source
  return '%s (%s)' % (full_name, num_hosts)

class ParserThread(threading.Thread):
  def __init__(self, record_type):
    threading.Thread.__init__(self)
    self.type = record_type

  def run(self):
    hp = HistoryParser()
    self.hosts = hp.ParseByType(self.type, complain=False, max_age_days=30)
    return self.hosts

class HistoryParser(object):
  """Parse the history file from files and web browsers and such."""

  MAX_NON_UNIQUE_RECORD_COUNT = 500000
  MIN_FILE_SIZE = 10000
  MIN_RECOMMENDED_RECORD_COUNT = 200
  INTERNAL_RE = re.compile('\.prod|\.corp|\.bor|internal|dmz')
  IP_RE = re.compile('^[\d\.]+$')
  TYPES = {}

  def __init__(self, status_callback=None):
    self.TYPES = {
        'alexa': ('Alexa Top 10000 Domains', None),
        'chrome': ('Google Chrome', self.GoogleChromeHistoryPath),
        'chromium': ('Chromium', self.ChromiumHistoryPath),
        'epiphany': ('Epiphany', self.EpiphanyHistoryPath),
        'opera': ('Opera', self.OperaHistoryPath),
        'safari': ('Apple Safari', self.SafariHistoryPath),
        'firefox': ('Mozilla Firefox', self.FirefoxHistoryPath),
        'internet_explorer': ('Microsoft Internet Explorer', self.InternetExplorerHistoryPath),
        'konqueror': ('Konqueror', self.KonquerorHistoryPath),
        'seamonkey': ('Mozilla Seamonkey', self.SeamonkeyHistoryPath),
        'squid': ('Squid Web Proxy', self.SquidLogPath),
    }
    self.imported_sources = {}
    self.status_callback = status_callback

  def msg(self, msg, **kwargs):
    if self.status_callback:
      self.status_callback(msg, **kwargs)
    else:
      print '- %s' % msg

  def GetTypeName(self, name):
    return self.TYPES[name][0]

  def GetTypes(self):
    """Return a tuple of type names with a description."""
    return dict([(x, self.GetTypeName(x)) for x in self.TYPES])

  def GetParsedSource(self, type):
    return self.imported_sources[type]

  def GetAvailableHistorySources(self):
    """Seek out and create a list of valid data sources.

    Returns:
      sources: A list of tuples (type, description, record count)
    """
    source_desc = self.GetTypes()
    self.imported_sources = self.ParseAllTypes()
    sources = []
    for src_type in self.imported_sources:
      sources.append(
        (src_type, source_desc[src_type], len(self.imported_sources[src_type]))
      )
    sources.sort(key=operator.itemgetter(2), reverse=True)
    # TODO(tstromberg): Don't hardcode input types.
    sources.append((None, 'Alexa Top Global Domains', 10000))
    return sources

  def GetTypeMethod(self, type):
    return self.TYPES[type][1]

  def Parse(self, path_or_type, store=False):
    if path_or_type.lower() in self.TYPES:
      return self.ParseByType(path_or_type.lower(), store=store)
    else:
      return self.ParseByFilename(path_or_type, store=store)

  def ParseByType(self, source, complain=True, store=False, max_age_days=None):
    """Given a type, parse the newest file and return a list of hosts."""

    (paths, tried) = self.FindGlobPaths(self.GetTypeMethod(source)())
    if not paths:
      if complain:
        self.msg('%s: no matches in %s' % (source, tried))
      return False
    newest = sorted(paths, key=os.path.getmtime)[-1]
    if max_age_days:
      age_days = (time.time() - os.path.getmtime(newest)) / 86400
      if age_days > max_age_days:
        self.msg('Ignoring %s from %s (%2.0f days old)' % (newest, source, age_days))
        return []

    # Do not use this with multiple threads.
    results = self.ParseByFilename(newest)
    if store:
      self.imported_sources[source] = results
    return results

  def ParseAllTypes(self):
    """For each type we know of, attempt to find and parse each of them.

    Returns:
      dict of type: list of hosts
    """
    start_time = time.time()
    results = {}
    records = 0
    threads = []

    for type in self.GetTypes():
      if not self.GetTypeMethod(type):
        continue
      thread = ParserThread(type)
      thread.start()
      threads.append(thread)

    for thread in threads:
      thread.join()
      if thread.hosts:
        records += len(thread.hosts)

      if thread.hosts and len(thread.hosts) >= self.MIN_RECOMMENDED_RECORD_COUNT:
        results[thread.type] = thread.hosts
        self.msg('%s: Found %s good records' % (self.GetTypeName(thread.type),
                                                len(thread.hosts)))
      elif thread.hosts == False:
        pass
      elif not thread.hosts:
        self.msg('%s: No records found!' % (self.GetTypeName(thread.type)))
      else:
        self.msg('%s: Too few records to consider (%s)' %
                 (self.GetTypeName(thread.type), len(thread.hosts)))

    return results

  def ParseByFilename(self, filename, store=False):
    # Only matches http://host.domain type entries (needs at least one subdom)
    parse_re = re.compile('https*://([\-\w]+\.[\-\w\.]+)')
    size_mb =  os.path.getsize(filename) / 1024.0 / 1024.0
    self.msg('Reading %s (%0.0fMB)' % (filename, size_mb))
    # binary mode is necessary for running under Windows
    start_time = time.time()
    results = parse_re.findall(open(filename, 'rb').read())
    if store:
      self.imported_sources[filename] = results
    duration = time.time() - start_time
    if duration > 5:
      self.msg('Read %s total history records in %0.2fs' %
               (len(results), duration))
    return results

  def GenerateTestDataFromInput(self, path_or_type):
    hosts = self.Parse(path_or_type)
    if len(hosts) > self.MAX_NON_UNIQUE_RECORD_COUNT:
      sorted_unique = True
    else:
      sorted_unique = False
    return self.GenerateTestData(hosts, sorted_unique=sorted_unique)

  def GenerateTestData(self, hosts, sorted_unique=False):
    """Given a set of hosts, generate test data.

    Args:
      hosts: A list of hostnames
      sorted_unique: Return a sorted unique list of tests. Useful for large
                     data sets.

    Returns:
      A list of strings representing DNS requests to test with.
    """
    history = []
    hits = {}
    last_host = None

    for host in hosts:
      if not host.endswith('.'):
        host = host + '.'

      if self.INTERNAL_RE.search(host):
        continue

      if self.IP_RE.match(host):
        continue

      if host != last_host:
        if sorted_unique:
          hits[host] = hits.get(host, 0) + 1
        else:
          history.append('A %s' % host)
        last_host = host

    if sorted_unique:
      for (hit, count) in sorted(hits.items(), key=operator.itemgetter(1),
                                 reverse=True):
        history.append('A %s # %s hits' % (hit, count))
    return history

  def _AddRoamLocalWindowsPaths(self, paths):
    """Windows path munging: Add neutered and swapped paths for Local/roaming dirs."""
    keywords = ('Local', 'Roaming')
    new_paths = list(paths)
    for path in paths:
      for keyword in keywords:
        if path[0] and keyword in path[0]:
          swapped_path = list(path)
          neutered_path = list(path)

          replacement = keywords[keywords.index(keyword)-1]

          swapped_path[0] = swapped_path[0].replace('\\%s' % keyword, '\\%s' % replacement)
          neutered_path[0] = neutered_path[0].replace('\\%s' % keyword, '')
          new_paths.extend([swapped_path, neutered_path])
    return new_paths

  def FindGlobPaths(self, paths):
    """Given a list of glob paths, return a list of matches containing data.

    Returns:
      A tuple with (file path (str), list of paths checked)
    """
    tried = []
    found = []
    if sys.platform[:3] == 'win':
      paths = self._AddRoamLocalWindowsPaths(paths)

    for path_elements in paths:
      # path contains an environment variable that we could not resolve.
      if False in path_elements:
        continue
      path = os.path.join(*path_elements)
      tried.append(path)
      for filename in glob.glob(path):
        if os.path.getsize(filename) > self.MIN_FILE_SIZE:
          found.append(filename)
        else:
          self.msg('Ignoring %s (only %s bytes)' % (filename, os.path.getsize(filename)))

    return (found, tried)

  def GoogleChromeHistoryPath(self):
    paths = (
        (os.getenv('HOME', False), 'Library', 'Application Support', 'Google',
         'Chrome', 'Default', 'History'),
        (os.getenv('HOME', False), '.config', 'google-chrome', 'Default',
         'History'),
        (os.getenv('APPDATA', False), 'Google', 'Chrome', 'User Data', 'Default',
         'History'),
        (os.getenv('USERPROFILE', False), 'Local Settings', 'Application Data',
         'Google', 'Chrome', 'User Data', 'Default', 'History'),
    )
    return paths

  def ChromiumHistoryPath(self):
    """It's like Chrome, but with the branding stripped out."""

    # TODO(tstromberg): Find a terser way to do this.
    paths = []
    for path in self.GoogleChromeHistoryPath():
      new_path = list(path)
      if 'Google' in new_path:
        new_path.remove('Google')
      for (index, part) in enumerate(new_path):
        if part == 'Chrome':
          new_path[index] = 'Chromium'
        elif part == 'chrome' or part == 'google-chrome':
          new_path[index] = 'chromium'
      paths.append(new_path)

    return paths

  def OperaHistoryPath(self):
    paths = (
        (os.getenv('HOME', False), 'Library', 'Preferences', 'Opera Preferences',
         'global_history.dat'),
        (os.getenv('HOME', False), 'Library', 'Preferences', 'Opera Preferences 10',
         'global_history.dat'),
        (os.getenv('APPDATA', False), 'Opera', 'Opera', 'global_history.dat'),
        (os.getenv('HOME', False), '.opera', 'global_history.dat'),
    )
    return paths

  def SafariHistoryPath(self):
    paths = (
        (os.getenv('HOME', False), 'Library', 'Safari', 'History.plist'),
        (os.getenv('APPDATA', False), 'Apple Computer', 'Safari',
         'History.plist')
    )
    return paths

  def FirefoxHistoryPath(self):
    paths = (
        (os.getenv('HOME', False), 'Library', 'Application Support', 'Firefox',
         'Profiles', '*', 'places.sqlite'),
        (os.getenv('HOME', False), '.mozilla', 'firefox', '*', 'places.sqlite'),
        (os.getenv('APPDATA', False), 'Mozilla', 'Firefox', 'Profiles', '*',
         'places.sqlite'),
        # firefox v2.0
        (os.getenv('HOME', False), 'Library', 'Application Support', 'Firefox',
         'Profiles', '*', 'history.dat'),
        (os.getenv('HOME', False), '.mozilla', 'firefox', '*', 'history.dat'),
        (os.getenv('APPDATA', False), 'Mozilla', 'Firefox', 'Profiles', '*',
         'history.dat')
    )
    return paths

  def SeamonkeyHistoryPath(self):
    paths = (
        (os.getenv('HOME', False), 'Library', 'Application Support', 'Seamonkey',
         'Profiles', '*', 'history.dat'),
        (os.getenv('HOME', False), '.mozilla', 'seamonkey', '*', 'history.dat'),
        (os.getenv('HOME', False), '.mozilla', 'default', '*', 'history.dat'),
        (os.getenv('APPDATA', False), 'Mozilla', 'Seamonkey', 'Profiles', '*',
         'history.dat')
    )
    return paths

  def InternetExplorerHistoryPath(self):
    paths = (
        # XP
        (os.getenv('USERPROFILE', False), 'Local Settings', 'History',
         'History.IE5', 'index.dat'),
        (os.getenv('APPDATA', False), 'Microsoft', 'Windows', 'History',
         'History.IE5', 'index.dat'),
        (os.getenv('APPDATA', False), 'Microsoft', 'Windows', 'History', 'Low',
         'History.IE5', 'index.dat'),
    )
    return paths

  def EpiphanyHistoryPath(self):
    paths = (
        (os.getenv('HOME', False), '.gnome2', 'epiphany', 'ephy-history.xml'),
    )
    return paths

  def KonquerorHistoryPath(self):
    paths = (
        (os.getenv('HOME', False), '.kde4', 'share',  'apps', 'konqueror', 'konq_history'),
        (os.getenv('HOME', False), '.kde', 'share',  'apps', 'konqueror', 'konq_history'),
    )
    return paths

  def SquidLogPath(self):
    paths = (
        ('/usr/local/squid/logs/access.log',),
        ('/var/log/squid/access_log',)
    )
    return paths

if __name__ == '__main__':
  parser = HistoryParser()
  types_str = ', '.join(parser.TYPES.keys())
  if len(sys.argv) < 2:
    print 'You must provide a filename or history file type (%s)' % types_str
    sys.exit(1)

  records = parser.Parse(sys.argv[1])
  for record in records:
    print record

<commit_msg>GetTypeName no longer raises KeyError on unknown names<commit_after># Copyright 2009 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Parse the history file for just about anything."""
import glob
import operator
import os
import os.path
import re
import sys
import time
import threading

def sourceToTitle(source):
  """Convert a source tuple to a title."""
  (short_name, full_name, num_hosts) = source
  return '%s (%s)' % (full_name, num_hosts)

class ParserThread(threading.Thread):
  def __init__(self, record_type):
    threading.Thread.__init__(self)
    self.type = record_type

  def run(self):
    hp = HistoryParser()
    self.hosts = hp.ParseByType(self.type, complain=False, max_age_days=30)
    return self.hosts

class HistoryParser(object):
  """Parse the history file from files and web browsers and such."""

  MAX_NON_UNIQUE_RECORD_COUNT = 500000
  MIN_FILE_SIZE = 10000
  MIN_RECOMMENDED_RECORD_COUNT = 200
  INTERNAL_RE = re.compile('\.prod|\.corp|\.bor|internal|dmz')
  IP_RE = re.compile('^[\d\.]+$')
  TYPES = {}

  def __init__(self, status_callback=None):
    self.TYPES = {
        'alexa': ('Alexa Top 10000 Domains', None),
        'chrome': ('Google Chrome', self.GoogleChromeHistoryPath),
        'chromium': ('Chromium', self.ChromiumHistoryPath),
        'epiphany': ('Epiphany', self.EpiphanyHistoryPath),
        'opera': ('Opera', self.OperaHistoryPath),
        'safari': ('Apple Safari', self.SafariHistoryPath),
        'firefox': ('Mozilla Firefox', self.FirefoxHistoryPath),
        'internet_explorer': ('Microsoft Internet Explorer', self.InternetExplorerHistoryPath),
        'konqueror': ('Konqueror', self.KonquerorHistoryPath),
        'seamonkey': ('Mozilla Seamonkey', self.SeamonkeyHistoryPath),
        'squid': ('Squid Web Proxy', self.SquidLogPath),
    }
    self.imported_sources = {}
    self.status_callback = status_callback

  def msg(self, msg, **kwargs):
    if self.status_callback:
      self.status_callback(msg, **kwargs)
    else:
      print '- %s' % msg

  def GetTypeName(self, name):
    if name in self.TYPES:
      return self.TYPES[name][0]
    else:
      return name

  def GetTypes(self):
    """Return a tuple of type names with a description."""
    return dict([(x, self.GetTypeName(x)) for x in self.TYPES])

  def GetParsedSource(self, type):
    return self.imported_sources[type]

  def GetAvailableHistorySources(self):
    """Seek out and create a list of valid data sources.

    Returns:
      sources: A list of tuples (type, description, record count)
    """
    source_desc = self.GetTypes()
    self.imported_sources = self.ParseAllTypes()
    sources = []
    for src_type in self.imported_sources:
      sources.append(
        (src_type, source_desc[src_type], len(self.imported_sources[src_type]))
      )
    sources.sort(key=operator.itemgetter(2), reverse=True)
    # TODO(tstromberg): Don't hardcode input types.
    sources.append((None, 'Alexa Top Global Domains', 10000))
    return sources

  def GetTypeMethod(self, type):
    return self.TYPES[type][1]

  def Parse(self, path_or_type, store=False):
    if path_or_type.lower() in self.TYPES:
      return self.ParseByType(path_or_type.lower(), store=store)
    else:
      return self.ParseByFilename(path_or_type, store=store)

  def ParseByType(self, source, complain=True, store=False, max_age_days=None):
    """Given a type, parse the newest file and return a list of hosts."""

    (paths, tried) = self.FindGlobPaths(self.GetTypeMethod(source)())
    if not paths:
      if complain:
        self.msg('%s: no matches in %s' % (source, tried))
      return False
    newest = sorted(paths, key=os.path.getmtime)[-1]
    if max_age_days:
      age_days = (time.time() - os.path.getmtime(newest)) / 86400
      if age_days > max_age_days:
        self.msg('Ignoring %s from %s (%2.0f days old)' % (newest, source, age_days))
        return []

    # Do not use this with multiple threads.
    results = self.ParseByFilename(newest)
    if store:
      self.imported_sources[source] = results
    return results

  def ParseAllTypes(self):
    """For each type we know of, attempt to find and parse each of them.

    Returns:
      dict of type: list of hosts
    """
    start_time = time.time()
    results = {}
    records = 0
    threads = []

    for type in self.GetTypes():
      if not self.GetTypeMethod(type):
        continue
      thread = ParserThread(type)
      thread.start()
      threads.append(thread)

    for thread in threads:
      thread.join()
      if thread.hosts:
        records += len(thread.hosts)

      if thread.hosts and len(thread.hosts) >= self.MIN_RECOMMENDED_RECORD_COUNT:
        results[thread.type] = thread.hosts
        self.msg('%s: Found %s good records' % (self.GetTypeName(thread.type),
                                                len(thread.hosts)))
      elif thread.hosts == False:
        pass
      elif not thread.hosts:
        self.msg('%s: No records found!' % (self.GetTypeName(thread.type)))
      else:
        self.msg('%s: Too few records to consider (%s)' %
                 (self.GetTypeName(thread.type), len(thread.hosts)))

    return results

  def ParseByFilename(self, filename, store=False):
    # Only matches http://host.domain type entries (needs at least one subdom)
    parse_re = re.compile('https*://([\-\w]+\.[\-\w\.]+)')
    size_mb =  os.path.getsize(filename) / 1024.0 / 1024.0
    self.msg('Reading %s (%0.0fMB)' % (filename, size_mb))
    # binary mode is necessary for running under Windows
    start_time = time.time()
    results = parse_re.findall(open(filename, 'rb').read())
    if store:
      self.imported_sources[filename] = results
    duration = time.time() - start_time
    if duration > 5:
      self.msg('Read %s total history records in %0.2fs' %
               (len(results), duration))
    return results

  def GenerateTestDataFromInput(self, path_or_type):
    hosts = self.Parse(path_or_type)
    if len(hosts) > self.MAX_NON_UNIQUE_RECORD_COUNT:
      sorted_unique = True
    else:
      sorted_unique = False
    return self.GenerateTestData(hosts, sorted_unique=sorted_unique)

  def GenerateTestData(self, hosts, sorted_unique=False):
    """Given a set of hosts, generate test data.

    Args:
      hosts: A list of hostnames
      sorted_unique: Return a sorted unique list of tests. Useful for large
                     data sets.

    Returns:
      A list of strings representing DNS requests to test with.
    """
    history = []
    hits = {}
    last_host = None

    for host in hosts:
      if not host.endswith('.'):
        host = host + '.'

      if self.INTERNAL_RE.search(host):
        continue

      if self.IP_RE.match(host):
        continue

      if host != last_host:
        if sorted_unique:
          hits[host] = hits.get(host, 0) + 1
        else:
          history.append('A %s' % host)
        last_host = host

    if sorted_unique:
      for (hit, count) in sorted(hits.items(), key=operator.itemgetter(1),
                                 reverse=True):
        history.append('A %s # %s hits' % (hit, count))
    return history

  def _AddRoamLocalWindowsPaths(self, paths):
    """Windows path munging: Add neutered and swapped paths for Local/roaming dirs."""
    keywords = ('Local', 'Roaming')
    new_paths = list(paths)
    for path in paths:
      for keyword in keywords:
        if path[0] and keyword in path[0]:
          swapped_path = list(path)
          neutered_path = list(path)

          replacement = keywords[keywords.index(keyword)-1]

          swapped_path[0] = swapped_path[0].replace('\\%s' % keyword, '\\%s' % replacement)
          neutered_path[0] = neutered_path[0].replace('\\%s' % keyword, '')
          new_paths.extend([swapped_path, neutered_path])
    return new_paths

  def FindGlobPaths(self, paths):
    """Given a list of glob paths, return a list of matches containing data.

    Returns:
      A tuple with (file path (str), list of paths checked)
    """
    tried = []
    found = []
    if sys.platform[:3] == 'win':
      paths = self._AddRoamLocalWindowsPaths(paths)

    for path_elements in paths:
      # path contains an environment variable that we could not resolve.
      if False in path_elements:
        continue
      path = os.path.join(*path_elements)
      tried.append(path)
      for filename in glob.glob(path):
        if os.path.getsize(filename) > self.MIN_FILE_SIZE:
          found.append(filename)
        else:
          self.msg('Ignoring %s (only %s bytes)' % (filename, os.path.getsize(filename)))

    return (found, tried)

  def GoogleChromeHistoryPath(self):
    paths = (
        (os.getenv('HOME', False), 'Library', 'Application Support', 'Google',
         'Chrome', 'Default', 'History'),
        (os.getenv('HOME', False), '.config', 'google-chrome', 'Default',
         'History'),
        (os.getenv('APPDATA', False), 'Google', 'Chrome', 'User Data', 'Default',
         'History'),
        (os.getenv('USERPROFILE', False), 'Local Settings', 'Application Data',
         'Google', 'Chrome', 'User Data', 'Default', 'History'),
    )
    return paths

  def ChromiumHistoryPath(self):
    """It's like Chrome, but with the branding stripped out."""

    # TODO(tstromberg): Find a terser way to do this.
    paths = []
    for path in self.GoogleChromeHistoryPath():
      new_path = list(path)
      if 'Google' in new_path:
        new_path.remove('Google')
      for (index, part) in enumerate(new_path):
        if part == 'Chrome':
          new_path[index] = 'Chromium'
        elif part == 'chrome' or part == 'google-chrome':
          new_path[index] = 'chromium'
      paths.append(new_path)

    return paths

  def OperaHistoryPath(self):
    paths = (
        (os.getenv('HOME', False), 'Library', 'Preferences', 'Opera Preferences',
         'global_history.dat'),
        (os.getenv('HOME', False), 'Library', 'Preferences', 'Opera Preferences 10',
         'global_history.dat'),
        (os.getenv('APPDATA', False), 'Opera', 'Opera', 'global_history.dat'),
        (os.getenv('HOME', False), '.opera', 'global_history.dat'),
    )
    return paths

  def SafariHistoryPath(self):
    paths = (
        (os.getenv('HOME', False), 'Library', 'Safari', 'History.plist'),
        (os.getenv('APPDATA', False), 'Apple Computer', 'Safari',
         'History.plist')
    )
    return paths

  def FirefoxHistoryPath(self):
    paths = (
        (os.getenv('HOME', False), 'Library', 'Application Support', 'Firefox',
         'Profiles', '*', 'places.sqlite'),
        (os.getenv('HOME', False), '.mozilla', 'firefox', '*', 'places.sqlite'),
        (os.getenv('APPDATA', False), 'Mozilla', 'Firefox', 'Profiles', '*',
         'places.sqlite'),
        # firefox v2.0
        (os.getenv('HOME', False), 'Library', 'Application Support', 'Firefox',
         'Profiles', '*', 'history.dat'),
        (os.getenv('HOME', False), '.mozilla', 'firefox', '*', 'history.dat'),
        (os.getenv('APPDATA', False), 'Mozilla', 'Firefox', 'Profiles', '*',
         'history.dat')
    )
    return paths

  def SeamonkeyHistoryPath(self):
    paths = (
        (os.getenv('HOME', False), 'Library', 'Application Support', 'Seamonkey',
         'Profiles', '*', 'history.dat'),
        (os.getenv('HOME', False), '.mozilla', 'seamonkey', '*', 'history.dat'),
        (os.getenv('HOME', False), '.mozilla', 'default', '*', 'history.dat'),
        (os.getenv('APPDATA', False), 'Mozilla', 'Seamonkey', 'Profiles', '*',
         'history.dat')
    )
    return paths

  def InternetExplorerHistoryPath(self):
    paths = (
        # XP
        (os.getenv('USERPROFILE', False), 'Local Settings', 'History',
         'History.IE5', 'index.dat'),
        (os.getenv('APPDATA', False), 'Microsoft', 'Windows', 'History',
         'History.IE5', 'index.dat'),
        (os.getenv('APPDATA', False), 'Microsoft', 'Windows', 'History', 'Low',
         'History.IE5', 'index.dat'),
    )
    return paths

  def EpiphanyHistoryPath(self):
    paths = (
        (os.getenv('HOME', False), '.gnome2', 'epiphany', 'ephy-history.xml'),
    )
    return paths

  def KonquerorHistoryPath(self):
    paths = (
        (os.getenv('HOME', False), '.kde4', 'share',  'apps', 'konqueror', 'konq_history'),
        (os.getenv('HOME', False), '.kde', 'share',  'apps', 'konqueror', 'konq_history'),
    )
    return paths

  def SquidLogPath(self):
    paths = (
        ('/usr/local/squid/logs/access.log',),
        ('/var/log/squid/access_log',)
    )
    return paths

if __name__ == '__main__':
  parser = HistoryParser()
  types_str = ', '.join(parser.TYPES.keys())
  if len(sys.argv) < 2:
    print 'You must provide a filename or history file type (%s)' % types_str
    sys.exit(1)

  records = parser.Parse(sys.argv[1])
  for record in records:
    print record

<|endoftext|>


<commit_before>"""
Parsing module for models.py files. Extracts information in a more reliable
way than inspect + regexes.
"""

import re
import inspect
import parser
import symbol
import token
import keyword

from django.db import models

def name_that_thing(thing):
    "Turns a symbol/token int into its name."
    for name in dir(symbol):
        if getattr(symbol, name) == thing:
            return "symbol.%s" % name
    for name in dir(token):
        if getattr(token, name) == thing:
            return "token.%s" % name
    return str(thing)

def thing_that_name(name):
    "Turns a name of a symbol/token into its integer value."
    if name in dir(symbol):
        return getattr(symbol, name)
    if name in dir(token):
        return getattr(token, name)
    raise ValueError("Cannot convert '%s'" % name)

def prettyprint(tree, indent=0, omit_singles=False):
    "Prettyprints the tree, with symbol/token names. For debugging."
    if omit_singles and isinstance(tree, tuple) and len(tree) == 2:
        return prettyprint(tree[1], indent, omit_singles)
    if isinstance(tree, tuple):
        return " (\n%s\n" % "".join([prettyprint(x, indent+1) for x in tree]) + \
            (" " * indent) + ")"
    elif isinstance(tree, int):
        return (" " * indent) + name_that_thing(tree)
    else:
        return " " + repr(tree)

class STTree(object):

    "A syntax tree wrapper class."

    def __init__(self, tree):
        self.tree = tree

    def __eq__(self, other):
        return other.tree == self.tree

    def __hash__(self):
        return hash(self.tree)

    @property
    def root(self):
        return self.tree[0]

    @property
    def value(self):
        return self.tree

    def walk(self, recursive=True):
        """
        Yields (symbol, subtree) for the entire subtree.
        Comes out with node 1, node 1's children, node 2, etc.
        """
        stack = [self.tree]
        done_outer = False
        while stack:
            atree = stack.pop()
            if isinstance(atree, tuple):
                if done_outer:
                    yield atree[0], STTree(atree)
                if recursive or not done_outer:
                    for bit in reversed(atree[1:]):
                        stack.append(bit)
                    done_outer = True

    def flatten(self):
        "Yields the tokens/symbols in the tree only, in order."
        bits = []
        for sym, subtree in self.walk():
            if sym in token_map:
                bits.append(sym)
            elif sym == token.NAME:
                bits.append(subtree.value)
            elif sym == token.STRING:
                bits.append(subtree.value)
            elif sym == token.NUMBER:
                bits.append(subtree.value)
        return bits

    def reform(self):
        "Prints how the tree's input probably looked."
        return reform(self.flatten())

    def findAllType(self, ntype, recursive=True):
        "Returns all nodes with the given type in the tree."
        for symbol, subtree in self.walk(recursive=recursive):
            if symbol == ntype:
                yield subtree

    def find(self, selector):
        """
        Searches the syntax tree with a CSS-like selector syntax.
        You can use things like 'suite simple_stmt', 'suite, simple_stmt'
        or 'suite > simple_stmt'. Not guaranteed to return in order.
        """
        # Split up the overall parts
        patterns = [x.strip() for x in selector.split(",")]
        results = []
        for pattern in patterns:
            # Split up the parts
            parts = re.split(r'(?:[\s]|(>))+', pattern)
            # Take the first part, use it for results
            if parts[0] == "^":
                subresults = [self]
            else:
                subresults = list(self.findAllType(thing_that_name(parts[0])))
            recursive = True
            # For each remaining part, do something
            for part in parts[1:]:
                if not subresults:
                    break
                if part == ">":
                    recursive = False
                elif not part:
                    pass
                else:
                    thing = thing_that_name(part)
                    newresults = [
                        list(tree.findAllType(thing, recursive))
                        for tree in subresults
                    ]
                    subresults = []
                    for stuff in newresults:
                        subresults.extend(stuff)
                    recursive = True
            results.extend(subresults)
        return results

    def __str__(self):
        return prettyprint(self.tree)
    __repr__ = __str__

def get_model_tree(model):
    # Get the source of the model's file
    source = open(inspect.getsourcefile(model)).read() + "\n"
    tree = STTree(parser.suite(source).totuple())
    # Now, we have to find it
    for poss in tree.find("compound_stmt"):
        if poss.value[1][0] == symbol.classdef and \
           poss.value[1][2][1].lower() == model.__name__.lower():
            # This is the tree
            return poss

token_map = {
    token.DOT: ".",
    token.LPAR: "(",
    token.RPAR: ")",
    token.EQUAL: "=",
    token.EQEQUAL: "==",
    token.COMMA: ",",
    token.LSQB: "[",
    token.RSQB: "]",
    token.AMPER: "&",
    token.BACKQUOTE: "`",
    token.CIRCUMFLEX: "^",
    token.CIRCUMFLEXEQUAL: "^=",
    token.COLON: ":",
    token.DOUBLESLASH: "//",
    token.DOUBLESLASHEQUAL: "//=",
    token.DOUBLESTAR: "**",
    token.DOUBLESLASHEQUAL: "**=",
    token.GREATER: ">",
    token.LESS: "<",
    token.GREATEREQUAL: ">=",
    token.LESSEQUAL: "<=",
    token.LBRACE: "{",
    token.RBRACE: "}",
    token.SEMI: ";",
    token.PLUS: "+",
    token.MINUS: "-",
    token.STAR: "*",
    token.SLASH: "/",
    token.VBAR: "|",
    token.PERCENT: "%",
    token.TILDE: "~",
    token.AT: "@",
    token.NOTEQUAL: "!=",
    token.LEFTSHIFT: "<<",
    token.RIGHTSHIFT: ">>",
    token.LEFTSHIFTEQUAL: "<<=",
    token.RIGHTSHIFTEQUAL: ">>=",
    token.PLUSEQUAL: "+=",
    token.MINEQUAL: "-=",
    token.STAREQUAL: "*=",
    token.SLASHEQUAL: "/=",
    token.VBAREQUAL: "|=",
    token.PERCENTEQUAL: "%=",
    token.AMPEREQUAL: "&=",
}

def reform(bits):
    "Returns the string that the list of tokens/symbols 'bits' represents"
    output = ""
    for bit in bits:
        if bit in token_map:
            output += token_map[bit]
        elif bit[0] in [token.NAME, token.STRING, token.NUMBER]:
            if keyword.iskeyword(bit[1]):
                output += " %s " % bit[1]
            else:
                if bit[1] not in symbol.sym_name:
                    output += bit[1]
    return output

def parse_arguments(argstr):
    """
    Takes a string representing arguments and returns the positional and
    keyword argument list and dict respectively.
    All the entries in these are python source, except the dict keys.
    """
    # Get the tree
    tree = STTree(parser.suite(argstr).totuple())

    # Initialise the lists
    curr_kwd = None
    args = []
    kwds = {}

    # Walk through, assigning things
    testlists = tree.find("testlist")
    for i, testlist in enumerate(testlists):
        # BTW: A testlist is to the left or right of an =.
        items = list(testlist.walk(recursive=False))
        for j, item in enumerate(items):
            if item[0] == symbol.test:
                if curr_kwd:
                    kwds[curr_kwd] = item[1].reform()
                    curr_kwd = None
                elif j == len(items)-1 and i != len(testlists)-1:
                    # Last item in a group must be a keyword, unless it's last overall
                    curr_kwd = item[1].reform()
                else:
                    args.append(item[1].reform())
    return args, kwds

def extract_field(tree):
    # Collapses the tree and tries to parse it as a field def
    bits = tree.flatten()
    ## Check it looks right:
    # Second token should be equals
    if len(bits) < 2 or bits[1] != token.EQUAL:
        return
    ## Split into meaningful sections
    name = bits[0][1]
    declaration = bits[2:]
    # Find the first LPAR; stuff before that is the class.
    try:
        lpar_at = declaration.index(token.LPAR)
    except ValueError:
        return
    clsname = reform(declaration[:lpar_at])
    # Now, inside that, find the last RPAR, and we'll take the stuff between
    # them as the arguments
    declaration.reverse()
    rpar_at = (len(declaration) - 1) - declaration.index(token.RPAR)
    declaration.reverse()
    args = declaration[lpar_at+1:rpar_at]
    # Now, extract the arguments as a list and dict
    try:
        args, kwargs = parse_arguments(reform(args))
    except SyntaxError:
        return
    # OK, extract and reform it
    return name, clsname, args, kwargs

def get_model_fields(model, m2m=False):
    """
    Given a model class, will return the dict of name: field_constructor
    mappings.
    """
    tree = get_model_tree(model)
    if tree is None:
        raise ValueError("Cannot find source for model '%s'." % model)
    possible_field_defs = tree.find("^ > classdef > suite > stmt > simple_stmt > small_stmt > expr_stmt")
    field_defs = {}

    # Go through all the found defns, and try to parse them
    for pfd in possible_field_defs:
        field = extract_field(pfd)
        if field:
            field_defs[field[0]] = field[1:]

    inherited_fields = {}
    # Go through all bases (that are themselves models, but not Model)
    for base in model.__bases__:
        if base != models.Model and issubclass(base, models.Model):
            inherited_fields.update(get_model_fields(base))

    # Now, go through all the fields and try to get their definition
    source = model._meta.local_fields[:]
    if m2m:
        source += model._meta.many_to_many
    fields = {}
    for field in source:
        # Get its name
        fieldname = field.name
        if isinstance(field, models.related.RelatedObject):
            continue
        # Now, try to get the defn
        if fieldname in field_defs:
            fields[fieldname] = field_defs[fieldname]
        # Try the South definition workaround?
        elif hasattr(field, 'south_field_triple'):
            fields[fieldname] = field.south_field_triple()
        elif hasattr(field, 'south_field_definition'):
            print "Your custom field %s provides the outdated south_field_definition method.\nPlease consider implementing south_field_triple too; it's more reliably evaluated." % field
            fields[fieldname] = field.south_field_definition()
        # Try a parent?
        elif fieldname in inherited_fields:
            fields[fieldname] = inherited_fields[fieldname]
        # Is it a _ptr?
        elif fieldname.endswith("_ptr"):
            fields[fieldname] = ("models.OneToOneField", [field.rel.to.__name__], {})
        # Try a default for 'id'.
        elif fieldname == "id":
            fields[fieldname] = ("models.AutoField", [], {"primary_key": "True"})
        else:
            fields[fieldname] = None

    return fields

def get_model_meta(model):
    """
    Given a model class, will return the dict representing the Meta class.
    """
    tree = get_model_tree(model)

    result = {}

    # First, try to get any unusual inherited classes
    for base in model.__bases__:
        if base is not models.Model:
            if not base._meta.abstract:
                # Abstract models' fields are included anyway, and we don't
                # want extra dependencies
                if "_bases" not in result:
                    result['_bases'] = []
                result['_bases'].append(base.__module__ + "." + base.__name__)

    # Find all classes exactly two levels deep
    possible_meta_classes = set(tree.find("classdef classdef"))
    possible_meta_classes.difference(set(tree.find("classdef classdef classdef")))

    # Select only those called 'Meta', and expand all their assignments
    possible_meta_classes = [
        tree.find("^ > suite > stmt > simple_stmt > small_stmt > expr_stmt")
        for tree in possible_meta_classes
        if tree.value[2][1] == "Meta"
    ]

    if possible_meta_classes:
        # Now, for each possible definition, try it. (Only for last Meta,
        # since that's how python interprets it)
        for defn in possible_meta_classes[-1]:
            bits = defn.flatten()
            if len(bits) > 1 and bits[1] == token.EQUAL:
                result[bits[0][1]] = reform(bits[2:])

    return result or None
<commit_msg>Don't throw in inherited non-local M2Ms.<commit_after>"""
Parsing module for models.py files. Extracts information in a more reliable
way than inspect + regexes.
"""

import re
import inspect
import parser
import symbol
import token
import keyword

from django.db import models

def name_that_thing(thing):
    "Turns a symbol/token int into its name."
    for name in dir(symbol):
        if getattr(symbol, name) == thing:
            return "symbol.%s" % name
    for name in dir(token):
        if getattr(token, name) == thing:
            return "token.%s" % name
    return str(thing)

def thing_that_name(name):
    "Turns a name of a symbol/token into its integer value."
    if name in dir(symbol):
        return getattr(symbol, name)
    if name in dir(token):
        return getattr(token, name)
    raise ValueError("Cannot convert '%s'" % name)

def prettyprint(tree, indent=0, omit_singles=False):
    "Prettyprints the tree, with symbol/token names. For debugging."
    if omit_singles and isinstance(tree, tuple) and len(tree) == 2:
        return prettyprint(tree[1], indent, omit_singles)
    if isinstance(tree, tuple):
        return " (\n%s\n" % "".join([prettyprint(x, indent+1) for x in tree]) + \
            (" " * indent) + ")"
    elif isinstance(tree, int):
        return (" " * indent) + name_that_thing(tree)
    else:
        return " " + repr(tree)

class STTree(object):

    "A syntax tree wrapper class."

    def __init__(self, tree):
        self.tree = tree

    def __eq__(self, other):
        return other.tree == self.tree

    def __hash__(self):
        return hash(self.tree)

    @property
    def root(self):
        return self.tree[0]

    @property
    def value(self):
        return self.tree

    def walk(self, recursive=True):
        """
        Yields (symbol, subtree) for the entire subtree.
        Comes out with node 1, node 1's children, node 2, etc.
        """
        stack = [self.tree]
        done_outer = False
        while stack:
            atree = stack.pop()
            if isinstance(atree, tuple):
                if done_outer:
                    yield atree[0], STTree(atree)
                if recursive or not done_outer:
                    for bit in reversed(atree[1:]):
                        stack.append(bit)
                    done_outer = True

    def flatten(self):
        "Yields the tokens/symbols in the tree only, in order."
        bits = []
        for sym, subtree in self.walk():
            if sym in token_map:
                bits.append(sym)
            elif sym == token.NAME:
                bits.append(subtree.value)
            elif sym == token.STRING:
                bits.append(subtree.value)
            elif sym == token.NUMBER:
                bits.append(subtree.value)
        return bits

    def reform(self):
        "Prints how the tree's input probably looked."
        return reform(self.flatten())

    def findAllType(self, ntype, recursive=True):
        "Returns all nodes with the given type in the tree."
        for symbol, subtree in self.walk(recursive=recursive):
            if symbol == ntype:
                yield subtree

    def find(self, selector):
        """
        Searches the syntax tree with a CSS-like selector syntax.
        You can use things like 'suite simple_stmt', 'suite, simple_stmt'
        or 'suite > simple_stmt'. Not guaranteed to return in order.
        """
        # Split up the overall parts
        patterns = [x.strip() for x in selector.split(",")]
        results = []
        for pattern in patterns:
            # Split up the parts
            parts = re.split(r'(?:[\s]|(>))+', pattern)
            # Take the first part, use it for results
            if parts[0] == "^":
                subresults = [self]
            else:
                subresults = list(self.findAllType(thing_that_name(parts[0])))
            recursive = True
            # For each remaining part, do something
            for part in parts[1:]:
                if not subresults:
                    break
                if part == ">":
                    recursive = False
                elif not part:
                    pass
                else:
                    thing = thing_that_name(part)
                    newresults = [
                        list(tree.findAllType(thing, recursive))
                        for tree in subresults
                    ]
                    subresults = []
                    for stuff in newresults:
                        subresults.extend(stuff)
                    recursive = True
            results.extend(subresults)
        return results

    def __str__(self):
        return prettyprint(self.tree)
    __repr__ = __str__

def get_model_tree(model):
    # Get the source of the model's file
    source = open(inspect.getsourcefile(model)).read() + "\n"
    tree = STTree(parser.suite(source).totuple())
    # Now, we have to find it
    for poss in tree.find("compound_stmt"):
        if poss.value[1][0] == symbol.classdef and \
           poss.value[1][2][1].lower() == model.__name__.lower():
            # This is the tree
            return poss

token_map = {
    token.DOT: ".",
    token.LPAR: "(",
    token.RPAR: ")",
    token.EQUAL: "=",
    token.EQEQUAL: "==",
    token.COMMA: ",",
    token.LSQB: "[",
    token.RSQB: "]",
    token.AMPER: "&",
    token.BACKQUOTE: "`",
    token.CIRCUMFLEX: "^",
    token.CIRCUMFLEXEQUAL: "^=",
    token.COLON: ":",
    token.DOUBLESLASH: "//",
    token.DOUBLESLASHEQUAL: "//=",
    token.DOUBLESTAR: "**",
    token.DOUBLESLASHEQUAL: "**=",
    token.GREATER: ">",
    token.LESS: "<",
    token.GREATEREQUAL: ">=",
    token.LESSEQUAL: "<=",
    token.LBRACE: "{",
    token.RBRACE: "}",
    token.SEMI: ";",
    token.PLUS: "+",
    token.MINUS: "-",
    token.STAR: "*",
    token.SLASH: "/",
    token.VBAR: "|",
    token.PERCENT: "%",
    token.TILDE: "~",
    token.AT: "@",
    token.NOTEQUAL: "!=",
    token.LEFTSHIFT: "<<",
    token.RIGHTSHIFT: ">>",
    token.LEFTSHIFTEQUAL: "<<=",
    token.RIGHTSHIFTEQUAL: ">>=",
    token.PLUSEQUAL: "+=",
    token.MINEQUAL: "-=",
    token.STAREQUAL: "*=",
    token.SLASHEQUAL: "/=",
    token.VBAREQUAL: "|=",
    token.PERCENTEQUAL: "%=",
    token.AMPEREQUAL: "&=",
}

def reform(bits):
    "Returns the string that the list of tokens/symbols 'bits' represents"
    output = ""
    for bit in bits:
        if bit in token_map:
            output += token_map[bit]
        elif bit[0] in [token.NAME, token.STRING, token.NUMBER]:
            if keyword.iskeyword(bit[1]):
                output += " %s " % bit[1]
            else:
                if bit[1] not in symbol.sym_name:
                    output += bit[1]
    return output

def parse_arguments(argstr):
    """
    Takes a string representing arguments and returns the positional and
    keyword argument list and dict respectively.
    All the entries in these are python source, except the dict keys.
    """
    # Get the tree
    tree = STTree(parser.suite(argstr).totuple())

    # Initialise the lists
    curr_kwd = None
    args = []
    kwds = {}

    # Walk through, assigning things
    testlists = tree.find("testlist")
    for i, testlist in enumerate(testlists):
        # BTW: A testlist is to the left or right of an =.
        items = list(testlist.walk(recursive=False))
        for j, item in enumerate(items):
            if item[0] == symbol.test:
                if curr_kwd:
                    kwds[curr_kwd] = item[1].reform()
                    curr_kwd = None
                elif j == len(items)-1 and i != len(testlists)-1:
                    # Last item in a group must be a keyword, unless it's last overall
                    curr_kwd = item[1].reform()
                else:
                    args.append(item[1].reform())
    return args, kwds

def extract_field(tree):
    # Collapses the tree and tries to parse it as a field def
    bits = tree.flatten()
    ## Check it looks right:
    # Second token should be equals
    if len(bits) < 2 or bits[1] != token.EQUAL:
        return
    ## Split into meaningful sections
    name = bits[0][1]
    declaration = bits[2:]
    # Find the first LPAR; stuff before that is the class.
    try:
        lpar_at = declaration.index(token.LPAR)
    except ValueError:
        return
    clsname = reform(declaration[:lpar_at])
    # Now, inside that, find the last RPAR, and we'll take the stuff between
    # them as the arguments
    declaration.reverse()
    rpar_at = (len(declaration) - 1) - declaration.index(token.RPAR)
    declaration.reverse()
    args = declaration[lpar_at+1:rpar_at]
    # Now, extract the arguments as a list and dict
    try:
        args, kwargs = parse_arguments(reform(args))
    except SyntaxError:
        return
    # OK, extract and reform it
    return name, clsname, args, kwargs

def get_model_fields(model, m2m=False):
    """
    Given a model class, will return the dict of name: field_constructor
    mappings.
    """
    tree = get_model_tree(model)
    if tree is None:
        raise ValueError("Cannot find source for model '%s'." % model)
    possible_field_defs = tree.find("^ > classdef > suite > stmt > simple_stmt > small_stmt > expr_stmt")
    field_defs = {}

    # Go through all the found defns, and try to parse them
    for pfd in possible_field_defs:
        field = extract_field(pfd)
        if field:
            field_defs[field[0]] = field[1:]

    inherited_fields = {}
    # Go through all bases (that are themselves models, but not Model)
    for base in model.__bases__:
        if base != models.Model and issubclass(base, models.Model):
            inherited_fields.update(get_model_fields(base))

    # Now, go through all the fields and try to get their definition
    source = model._meta.local_fields[:]
    if m2m:
        source += model._meta.local_many_to_many
    fields = {}
    for field in source:
        # Get its name
        fieldname = field.name
        if isinstance(field, models.related.RelatedObject):
            continue
        # Now, try to get the defn
        if fieldname in field_defs:
            fields[fieldname] = field_defs[fieldname]
        # Try the South definition workaround?
        elif hasattr(field, 'south_field_triple'):
            fields[fieldname] = field.south_field_triple()
        elif hasattr(field, 'south_field_definition'):
            print "Your custom field %s provides the outdated south_field_definition method.\nPlease consider implementing south_field_triple too; it's more reliably evaluated." % field
            fields[fieldname] = field.south_field_definition()
        # Try a parent?
        elif fieldname in inherited_fields:
            fields[fieldname] = inherited_fields[fieldname]
        # Is it a _ptr?
        elif fieldname.endswith("_ptr"):
            fields[fieldname] = ("models.OneToOneField", [field.rel.to.__name__], {})
        # Try a default for 'id'.
        elif fieldname == "id":
            fields[fieldname] = ("models.AutoField", [], {"primary_key": "True"})
        else:
            fields[fieldname] = None

    return fields

def get_model_meta(model):
    """
    Given a model class, will return the dict representing the Meta class.
    """
    tree = get_model_tree(model)

    result = {}

    # First, try to get any unusual inherited classes
    for base in model.__bases__:
        if base is not models.Model:
            if not base._meta.abstract:
                # Abstract models' fields are included anyway, and we don't
                # want extra dependencies
                if "_bases" not in result:
                    result['_bases'] = []
                result['_bases'].append(base.__module__ + "." + base.__name__)

    # Find all classes exactly two levels deep
    possible_meta_classes = set(tree.find("classdef classdef"))
    possible_meta_classes.difference(set(tree.find("classdef classdef classdef")))

    # Select only those called 'Meta', and expand all their assignments
    possible_meta_classes = [
        tree.find("^ > suite > stmt > simple_stmt > small_stmt > expr_stmt")
        for tree in possible_meta_classes
        if tree.value[2][1] == "Meta"
    ]

    if possible_meta_classes:
        # Now, for each possible definition, try it. (Only for last Meta,
        # since that's how python interprets it)
        for defn in possible_meta_classes[-1]:
            bits = defn.flatten()
            if len(bits) > 1 and bits[1] == token.EQUAL:
                result[bits[0][1]] = reform(bits[2:])

    return result or None
<|endoftext|>


<commit_before>from fabric.api import *
from fabric.colors import green, red, blue

### Prompt ###
### This class contains several helper functions for getting data between end-user's input and a schema.Schema object.
### It also containers several formatting functions, which are currently just a convenience wrapper around printing Fabric colors.

class Prompt():

    # Initialisation
    def __init__(self):
        pass

    ### Formatting ###

    # Prints a success message.
    def success(self, msg):
        print(green(msg))

    # Prints an error message.
    def error(self, msg):
        print(red(msg))

    # Prints a notice message.
    def notice(self, msg):
        print(blue(msg))

    ### Prompts ###

    # Returns the value from a validated bool prompt.
    def bool(self, label, default = False):
        val = prompt(label + '? (y/n)', default = self.fmt_bool(default), validate = r'^(y|n)$')
        return self.defmt_bool(val)

    ### Formatting ###

    # Takes a python bool and returns it as y|n.
    def fmt_bool(self, p):
        if p is True:
            return 'y'
        if p is False:
            return 'n'
        return ''

    # Takes y|n and returns a python bool.
    def defmt_bool(self, p):
        if p == 'y':
            return True
        if p == 'n':
            return False
        return None

<commit_msg>Updated prompt success/error messages<commit_after>from fabric.api import *
from fabric.colors import green, red, blue

### Prompt ###
### This class contains several helper functions for getting data between end-user's input and a schema.Schema object.
### It also containers several formatting functions, which are currently just a convenience wrapper around printing Fabric colors.

class Prompt():

    # Initialisation
    def __init__(self):
        pass

    ### Formatting ###

    # Prints a success message.
    def success(self, msg):
        print(green("Success: " + msg))

    # Prints an error message.
    def error(self, msg):
        print(red("Error: " + msg))

    # Prints a notice message.
    def notice(self, msg):
        print(blue(msg))

    ### Prompts ###

    # Returns the value from a validated bool prompt.
    def bool(self, label, default = False):
        val = prompt(label + '? (y/n)', default = self.fmt_bool(default), validate = r'^(y|n)$')
        return self.defmt_bool(val)

    ### Formatting ###

    # Takes a python bool and returns it as y|n.
    def fmt_bool(self, p):
        if p is True:
            return 'y'
        if p is False:
            return 'n'
        return ''

    # Takes y|n and returns a python bool.
    def defmt_bool(self, p):
        if p == 'y':
            return True
        if p == 'n':
            return False
        return None

<|endoftext|>