<commit_before>import time
import datetime
import mutagen
import magic
import wave
import logging
from analyzer import Analyzer

class MetadataAnalyzer(Analyzer):

    @staticmethod
    def analyze(filename, metadata):
        ''' Extract audio metadata from tags embedded in the file (eg. ID3 tags)

            Keyword arguments:
                filename: The path to the audio file to extract metadata from.
                metadata: A dictionary that the extracted metadata will be added to.
        '''
        if not isinstance(filename, unicode):
            raise TypeError("filename must be unicode. Was of type " + type(filename).__name__)
        if not isinstance(metadata, dict):
            raise TypeError("metadata must be a dict. Was of type " + type(metadata).__name__)

        #Airtime <= 2.5.x nonsense:
        metadata["ftype"] = "audioclip"
        #Other fields we'll want to set for Airtime:
        metadata["hidden"] = False

        # Mutagen doesn't handle WAVE files so we use a different package
        mime_check = magic.from_file(filename, mime=True)
        metadata["mime"] = mime_check
        if mime_check == 'audio/x-wav':
            return MetadataAnalyzer._analyze_wave(filename, metadata)

        #Extract metadata from an audio file using mutagen
        audio_file = mutagen.File(filename, easy=True)

        #Bail if the file couldn't be parsed. The title should stay as the filename
        #inside Airtime.
        if audio_file == None:  # Don't use "if not" here. It is wrong due to mutagen's design.
            return metadata
        # Note that audio_file can equal {} if the file is valid but there's no metadata tags.
        # We can still try to grab the info variables below.

        #Grab other file information that isn't encoded in a tag, but instead usually
        #in the file header. Mutagen breaks that out into a separate "info" object:
        info = audio_file.info
        if hasattr(info, "sample_rate"): # Mutagen is annoying and inconsistent
            metadata["sample_rate"] = info.sample_rate
        if hasattr(info, "length"):
            metadata["length_seconds"] = info.length
            #Converting the length in seconds (float) to a formatted time string
            track_length = datetime.timedelta(seconds=info.length)
            metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length)
            # Other fields for Airtime
            metadata["cueout"] = metadata["length"]

        if hasattr(info, "bitrate"):
            metadata["bit_rate"] = info.bitrate

        # Use the mutagen to get the MIME type, if it has one. This is more reliable and
        # consistent for certain types of MP3s or MPEG files than the MIMEs returned by magic.
        if audio_file.mime:
            metadata["mime"] = audio_file.mime[0]

        #Try to get the number of channels if mutagen can...
        try:
            #Special handling for getting the # of channels from MP3s. It's in the "mode" field
            #which is 0=Stereo, 1=Joint Stereo, 2=Dual Channel, 3=Mono. Part of the ID3 spec...
            if metadata["mime"] in ["audio/mpeg", 'audio/mp3']:
                if info.mode == 3:
                    metadata["channels"] = 1
                else:
                    metadata["channels"] = 2
            else:
                metadata["channels"] = info.channels
        except (AttributeError, KeyError):
            #If mutagen can't figure out the number of channels, we'll just leave it out...
            pass

        #Try to extract the number of tracks on the album if we can (the "track total")
        try:
            track_number = audio_file["tracknumber"]
            if isinstance(track_number, list): # Sometimes tracknumber is a list, ugh
                track_number = track_number[0]
            track_number_tokens = track_number
            if u'/' in track_number:
                track_number_tokens = track_number.split(u'/')
            elif u'-' in track_number:
                track_number_tokens = track_number.split(u'-')
            track_number = track_number_tokens[0]
            metadata["track_number"] = track_number
            track_total = track_number_tokens[1]
            metadata["track_total"] = track_total
        except (AttributeError, KeyError, IndexError):
            #If we couldn't figure out the track_number or track_total, just ignore it...
            pass

        #We normalize the mutagen tags slightly here, so in case mutagen changes,
        #we find the
        mutagen_to_airtime_mapping = {
            'title':        'track_title',
            'artist':       'artist_name',
            'album':        'album_title',
            'bpm':          'bpm',
            'composer':     'composer',
            'conductor':    'conductor',
            'copyright':    'copyright',
            'comment':      'comment',
            'encoded_by':   'encoder',
            'genre':        'genre',
            'isrc':         'isrc',
            'label':        'label',
            'organization': 'label',
            #'length':       'length',
            'language':     'language',
            'last_modified':'last_modified',
            'mood':         'mood',
            'bit_rate':     'bit_rate',
            'replay_gain':  'replaygain',
            #'tracknumber':  'track_number',
            #'track_total':  'track_total',
            'website':      'website',
            'date':         'year',
            #'mime_type':    'mime',
        }

        for mutagen_tag, airtime_tag in mutagen_to_airtime_mapping.iteritems():
            try:
                metadata[airtime_tag] = audio_file[mutagen_tag]

                # Some tags are returned as lists because there could be multiple values.
                # This is unusual so we're going to always just take the first item in the list.
                if isinstance(metadata[airtime_tag], list):
                    if metadata[airtime_tag]:
                        metadata[airtime_tag] = metadata[airtime_tag][0]
                    else: # Handle empty lists
                        metadata[airtime_tag] = ""

            except KeyError:
                continue

        return metadata

    @staticmethod
    def _analyze_wave(filename, metadata):
        try:
            reader = wave.open(filename, 'rb')
            metadata["mime"] = magic.from_file(filename, mime=True)
            metadata["channels"] = reader.getnchannels()
            metadata["sample_rate"] = reader.getframerate()
            length_seconds = float(reader.getnframes()) / float(metadata["sample_rate"])
            #Converting the length in seconds (float) to a formatted time string
            track_length = datetime.timedelta(seconds=length_seconds)
            metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length)
            metadata["length_seconds"] = length_seconds
            metadata["cueout"] = metadata["length"]
        except wave.Error:
            logging.error("Invalid WAVE file.")
            raise
        return metadata
<commit_msg>SAAS-596: Store file size and hash in database<commit_after>import time
import datetime
import mutagen
import magic
import wave
import logging
import os
import hashlib
from analyzer import Analyzer

class MetadataAnalyzer(Analyzer):

    @staticmethod
    def analyze(filename, metadata):
        ''' Extract audio metadata from tags embedded in the file (eg. ID3 tags)

            Keyword arguments:
                filename: The path to the audio file to extract metadata from.
                metadata: A dictionary that the extracted metadata will be added to.
        '''
        if not isinstance(filename, unicode):
            raise TypeError("filename must be unicode. Was of type " + type(filename).__name__)
        if not isinstance(metadata, dict):
            raise TypeError("metadata must be a dict. Was of type " + type(metadata).__name__)

        #Airtime <= 2.5.x nonsense:
        metadata["ftype"] = "audioclip"
        #Other fields we'll want to set for Airtime:
        metadata["hidden"] = False

        # Mutagen doesn't handle WAVE files so we use a different package
        mime_check = magic.from_file(filename, mime=True)
        metadata["mime"] = mime_check
        if mime_check == 'audio/x-wav':
            return MetadataAnalyzer._analyze_wave(filename, metadata)

        #Extract metadata from an audio file using mutagen
        audio_file = mutagen.File(filename, easy=True)

        #Bail if the file couldn't be parsed. The title should stay as the filename
        #inside Airtime.
        if audio_file == None:  # Don't use "if not" here. It is wrong due to mutagen's design.
            return metadata
        # Note that audio_file can equal {} if the file is valid but there's no metadata tags.
        # We can still try to grab the info variables below.

        #Grab other file information that isn't encoded in a tag, but instead usually
        #in the file header. Mutagen breaks that out into a separate "info" object:
        info = audio_file.info
        if hasattr(info, "sample_rate"): # Mutagen is annoying and inconsistent
            metadata["sample_rate"] = info.sample_rate
        if hasattr(info, "length"):
            metadata["length_seconds"] = info.length
            #Converting the length in seconds (float) to a formatted time string
            track_length = datetime.timedelta(seconds=info.length)
            metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length)
            # Other fields for Airtime
            metadata["cueout"] = metadata["length"]

        if hasattr(info, "bitrate"):
            metadata["bit_rate"] = info.bitrate

        # Use the mutagen to get the MIME type, if it has one. This is more reliable and
        # consistent for certain types of MP3s or MPEG files than the MIMEs returned by magic.
        if audio_file.mime:
            metadata["mime"] = audio_file.mime[0]

        #Try to get the number of channels if mutagen can...
        try:
            #Special handling for getting the # of channels from MP3s. It's in the "mode" field
            #which is 0=Stereo, 1=Joint Stereo, 2=Dual Channel, 3=Mono. Part of the ID3 spec...
            if metadata["mime"] in ["audio/mpeg", 'audio/mp3']:
                if info.mode == 3:
                    metadata["channels"] = 1
                else:
                    metadata["channels"] = 2
            else:
                metadata["channels"] = info.channels
        except (AttributeError, KeyError):
            #If mutagen can't figure out the number of channels, we'll just leave it out...
            pass

        #Try to extract the number of tracks on the album if we can (the "track total")
        try:
            track_number = audio_file["tracknumber"]
            if isinstance(track_number, list): # Sometimes tracknumber is a list, ugh
                track_number = track_number[0]
            track_number_tokens = track_number
            if u'/' in track_number:
                track_number_tokens = track_number.split(u'/')
            elif u'-' in track_number:
                track_number_tokens = track_number.split(u'-')
            track_number = track_number_tokens[0]
            metadata["track_number"] = track_number
            track_total = track_number_tokens[1]
            metadata["track_total"] = track_total
        except (AttributeError, KeyError, IndexError):
            #If we couldn't figure out the track_number or track_total, just ignore it...
            pass

        # Get file size and md5 hash of the file
        try:
            metadata["filesize"] = os.path.getsize(filename)

            with open(filename, 'rb') as fh:
                m = hashlib.md5()
                while True:
                    data = fh.read(8192)
                    if not data:
                        break
                    m.update(data)
                metadata["md5_hash"] = m.hexdigest()
        except (OSError, IOError) as e:
            raise e

        #We normalize the mutagen tags slightly here, so in case mutagen changes,
        #we find the
        mutagen_to_airtime_mapping = {
            'title':        'track_title',
            'artist':       'artist_name',
            'album':        'album_title',
            'bpm':          'bpm',
            'composer':     'composer',
            'conductor':    'conductor',
            'copyright':    'copyright',
            'comment':      'comment',
            'encoded_by':   'encoder',
            'genre':        'genre',
            'isrc':         'isrc',
            'label':        'label',
            'organization': 'label',
            #'length':       'length',
            'language':     'language',
            'last_modified':'last_modified',
            'mood':         'mood',
            'bit_rate':     'bit_rate',
            'replay_gain':  'replaygain',
            #'tracknumber':  'track_number',
            #'track_total':  'track_total',
            'website':      'website',
            'date':         'year',
            #'mime_type':    'mime',
        }

        for mutagen_tag, airtime_tag in mutagen_to_airtime_mapping.iteritems():
            try:
                metadata[airtime_tag] = audio_file[mutagen_tag]

                # Some tags are returned as lists because there could be multiple values.
                # This is unusual so we're going to always just take the first item in the list.
                if isinstance(metadata[airtime_tag], list):
                    if metadata[airtime_tag]:
                        metadata[airtime_tag] = metadata[airtime_tag][0]
                    else: # Handle empty lists
                        metadata[airtime_tag] = ""

            except KeyError:
                continue

        return metadata

    @staticmethod
    def _analyze_wave(filename, metadata):
        try:
            reader = wave.open(filename, 'rb')
            metadata["mime"] = magic.from_file(filename, mime=True)
            metadata["channels"] = reader.getnchannels()
            metadata["sample_rate"] = reader.getframerate()
            length_seconds = float(reader.getnframes()) / float(metadata["sample_rate"])
            #Converting the length in seconds (float) to a formatted time string
            track_length = datetime.timedelta(seconds=length_seconds)
            metadata["length"] = str(track_length) #time.strftime("%H:%M:%S.%f", track_length)
            metadata["length_seconds"] = length_seconds
            metadata["cueout"] = metadata["length"]
        except wave.Error:
            logging.error("Invalid WAVE file.")
            raise
        return metadata
<|endoftext|>


<commit_before>#
# A simple trace module
#
# JL Meunier - May 2004
# Copyright XRCE, 2004
#

import sys

global traceFD
traceFD = sys.stderr

def setTraceFD(fd):
    global traceFD
    traceFD = fd

def trace(*msg):
    global traceFD
    for i in msg:
        try: traceFD.write(str(i))
        except UnicodeEncodeError:sys.stderr.write(i.encode("utf-8"))

def traceln(*msg):
    global traceFD

    apply(trace, msg)
    trace("\n")
    traceFD.flush()

#SELF-TEST
if __name__=="__main__":

    trace(1)
    trace(" aut")
    trace("o")
    traceln("-test")
    trace("2 auto", "-", "test")
    trace()
    traceln()
    traceln("Done")
<commit_msg>add flush<commit_after>#
# A simple trace module
#
# JL Meunier - May 2004
# Copyright XRCE, 2004
#

import sys

global traceFD
traceFD = sys.stderr

def setTraceFD(fd):
    global traceFD
    traceFD = fd

def trace(*msg):
    global traceFD
    for i in msg:
        try: traceFD.write(str(i))
        except UnicodeEncodeError:sys.stderr.write(i.encode("utf-8"))

def traceln(*msg):
    global traceFD

    apply(trace, msg)
    trace("\n")
    traceFD.flush()

def flush():
    traceFD.flush()

#SELF-TEST
if __name__=="__main__":

    trace(1)
    trace(" aut")
    trace("o")
    traceln("-test")
    trace("2 auto", "-", "test")
    trace()
    traceln()
    traceln("Done")
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

import rapidsms
from rapidsms.models import Connection
from rapidsms.messages.outgoing import OutgoingMessage

class App (rapidsms.App):
    """This app receives outgoing messages from the WebUI (via the
       AJAX app), and relays them to the router."""

    PRIORITY = "lowest"

    def ajax_POST_send_message(self, params, form):
        '''
        Sends a message to a connection.  You can call this method
        via the ajax app by posting to the url:

            ajax/messaging/send_message

        You can also call this directly from a view by calling:

            ajax.utils.call_router("messaging", "send_message",
                       **{"connection_id": 1, "text": "hello rapidsms!" })
        '''
        connection = Connection.objects.get(pk=form["connection_id"])
        return self._send_message(connection, form["text"])

    def _send_message(self, connection, message_body):
        '''Attempts to send a message through a given connection'''
        # abort if we can't find a valid backend. Backend
        # objects SHOULD refer to a valid RapidSMS backend (via their
        # slug), but sometimes backends are removed or renamed.

        # attempt to send the message
        # TODO: what could go wrong here?
        msg = OutgoingMessage(connection, message_body)
        return msg.send()

<commit_msg>removing deprecated comment<commit_after>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

import rapidsms
from rapidsms.models import Connection
from rapidsms.messages.outgoing import OutgoingMessage

class App (rapidsms.App):
    """This app receives outgoing messages from the WebUI (via the
       AJAX app), and relays them to the router."""

    PRIORITY = "lowest"

    def ajax_POST_send_message(self, params, form):
        '''
        Sends a message to a connection.  You can call this method
        via the ajax app by posting to the url:

            ajax/messaging/send_message

        You can also call this directly from a view by calling:

            ajax.utils.call_router("messaging", "send_message",
                       **{"connection_id": 1, "text": "hello rapidsms!" })
        '''
        connection = Connection.objects.get(pk=form["connection_id"])
        return self._send_message(connection, form["text"])

    def _send_message(self, connection, message_body):
        '''Attempts to send a message through a given connection'''
        # attempt to send the message
        # TODO: what could go wrong here?
        msg = OutgoingMessage(connection, message_body)
        return msg.send()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: sw=4:ts=4:expandtab

"""
gogo
~~~~

A Python logging library with super powers

Examples:
    literal blocks::

        python example_google.py

Attributes:
    ENCODING (str): The module encoding
"""

from __future__ import (
    absolute_import, division, print_function, with_statement,
    unicode_literals)

__version__ = '0.2.1'

__title__ = 'gogo'
__author__ = 'Reuben Cummings'
__description__ = 'A Python logging library with super powers'
__email__ = 'reubano@gmail.com'
__license__ = 'MIT'
__copyright__ = 'Copyright 2015 Reuben Cummings'

ENCODING = 'utf-8'
<commit_msg>Bump to version 0.2.2<commit_after># -*- coding: utf-8 -*-
# vim: sw=4:ts=4:expandtab

"""
gogo
~~~~

A Python logging library with super powers

Examples:
    literal blocks::

        python example_google.py

Attributes:
    ENCODING (str): The module encoding
"""

from __future__ import (
    absolute_import, division, print_function, with_statement,
    unicode_literals)

__version__ = '0.2.2'

__title__ = 'gogo'
__author__ = 'Reuben Cummings'
__description__ = 'A Python logging library with super powers'
__email__ = 'reubano@gmail.com'
__license__ = 'MIT'
__copyright__ = 'Copyright 2015 Reuben Cummings'

ENCODING = 'utf-8'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""\
This is a python port of "Goose" orignialy licensed to Gravity.com
under one or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.

Python port was written by Xavier Grangier for Recrutae

Gravity.com licenses this file
to you under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

version_info = (1, 0, 15)
__version__ = ".".join(map(str, version_info))
<commit_msg>bumped version<commit_after># -*- coding: utf-8 -*-
"""\
This is a python port of "Goose" orignialy licensed to Gravity.com
under one or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.

Python port was written by Xavier Grangier for Recrutae

Gravity.com licenses this file
to you under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

version_info = (1, 0, 17)
__version__ = ".".join(map(str, version_info))
<|endoftext|>


<commit_before># This file is part of gorm, an object relational mapper for versioned graphs.
# Copyright (C) 2014 Zachary Spector.
from collections import defaultdict, deque
from .graph import (
    Graph,
    DiGraph,
    MultiGraph,
    MultiDiGraph,
)
from .query import QueryEngine
from .reify import reify

class GraphNameError(KeyError):
    pass

class ORM(object):
    """Instantiate this with the same string argument you'd use for a
    SQLAlchemy ``create_engine`` call. This will be your interface to
    gorm.

    """
    @reify
    def _graph_val_cache(self):
        assert(self.caching)
        from .window import WindowDict
        r = defaultdict(  # graph:
            lambda: defaultdict(  # key:
                lambda: defaultdict(  # branch:
                    WindowDict  # rev: value
                )
            )
        )
        for (graph, key, branch, rev, value) in self.db.graph_val_dump():
            r[graph][key][branch][rev] = value
        return r

    @reify
    def _node_val_cache(self):
        assert(self.caching)
        from .window import WindowDict
        r = defaultdict(  # graph:
            lambda: defaultdict(  # node:
                lambda: defaultdict(  # key:
                    lambda: defaultdict(  # branch:
                        WindowDict  # rev: value
                    )
                )
            )
        )
        for (graph, node, key, branch, rev, value) in self.db.node_val_dump():
            r[graph][node][key][branch][rev] = value
        return r

    @reify
    def _nodes_cache(self):
        assert(self.caching)
        from .window import WindowDict
        r = defaultdict(  # graph:
            lambda: defaultdict(  # node:
                lambda: defaultdict(  # branch:
                    WindowDict  # rev: extant
                )
            )
        )
        for (graph, node, branch, rev, extant) in self.db.nodes_dump():
            r[graph][node][branch][rev] = extant
        return r

    @reify
    def _edge_val_cache(self):
        assert(self.caching)
        from .window import WindowDict
        r = defaultdict(  # graph:
            lambda: defaultdict(  # nodeA:
                lambda: defaultdict(  # nodeB:
                    lambda: defaultdict(  # idx:
                        lambda: defaultdict(  # key:
                            lambda: defaultdict(  # branch:
                                WindowDict  # rev: value
                            )
                        )
                    )
                )
            )
        )
        for (
                graph, nodeA, nodeB, idx, key, branch, rev, value
        ) in self.db.edge_val_dump():
            r[graph][nodeA][nodeB][idx][key][branch][rev] = value
        return r

    @reify
    def _edges_cache(self):
        assert self.caching
        from .window import WindowDict
        r = defaultdict(  # graph:
            lambda: defaultdict(  # nodeA:
                lambda: defaultdict(  # nodeB:
                    lambda: defaultdict(  # idx:
                        lambda: defaultdict(  # branch:
                            WindowDict  # rev: extant
                        )
                    )
                )
            )
        )
        for (
                graph, nodeA, nodeB, idx, branch, rev, extant
        ) in self.db.edges_dump():
            r[graph][nodeA][nodeB][idx][branch][rev] = extant
        return r

    def __init__(
            self,
            dbstring,
            alchemy=True,
            connect_args={},
            query_engine_class=QueryEngine,
            json_dump=None,
            json_load=None,
            caching=True
    ):
        """Make a SQLAlchemy engine if possible, else a sqlite3 connection. In
        either case, begin a transaction.

        """
        self.db = query_engine_class(dbstring, connect_args, alchemy, json_dump, json_load)
        self._branches = {}
        self._obranch = None
        self._orev = None
        self.db.initdb()
        if caching:
            self.caching = True
            self._obranch = self.branch
            self._orev = self.rev
            self._timestream = {'master': {}}
            self._branch_start = {}
            self._branches = {'master': self._timestream['master']}
            self._branch_parents = {}
            self._active_branches_cache = []
            self.db.active_branches = self._active_branches
            todo = deque(self.db.timestream_data())
            while todo:
                (branch, parent, parent_tick) = working = todo.popleft()
                if branch == 'master':
                    continue
                if parent in self._branches:
                    assert(branch not in self._branches)
                    self._branches[parent][branch] = {}
                    self._branches[branch] = self._branches[parent][branch]
                    self._branch_parents['branch'] = parent
                    self._branch_start[branch] = parent_tick
                else:
                    todo.append(working)

    def __enter__(self):
        """Enable the use of the ``with`` keyword"""
        return self

    def __exit__(self, *args):
        """Alias for ``close``"""
        self.close()

    def _havebranch(self, b):
        """Private use. Checks that the branch is known about."""
        if self.caching and b in self._branches:
            return True
        return self.db.have_branch(b)

    def is_parent_of(self, parent, child):
        """Return whether ``child`` is a branch descended from ``parent`` at
        any remove.

        """
        # trivial cases
        if child in self._branches and self._branches[child][0] == parent:
            return True
        elif child == parent:
            return False
        # I will be recursing a lot so just cache all the branch info
        self._childbranch = {}
        self._ancestry = {}
        for (branch, parent, parent_rev) in self.db.all_branches():
            self._branches[branch] = (parent, parent_rev)
            self._childbranch[parent] = branch

        self._ancestry[child] = set([parent])
        lineage = self._ancestry[child]

        def recurse(oneparent):
            if oneparent in lineage:
                return True
            if oneparent not in self._branches:
                return False
            if self._branches[oneparent][0] in lineage:
                return True
            lineage.add(oneparent)
            return recurse(self._branches[oneparent][0])
        return recurse(child)

    @property
    def branch(self):
        """Return the global value ``branch``, or ``self._obranch`` if it's
        set

        """
        if self._obranch is not None:
            return self._obranch
        return self.db.globl['branch']

    @branch.setter
    def branch(self, v):
        """Set the global value ``branch`` and note that the branch's (parent,
        parent_rev) are the (branch, tick) set previously

        """
        curbranch = self.branch
        currev = self.rev
        if not self._havebranch(v):
            # assumes the present revision in the parent branch has
            # been finalized.
            self.db.new_branch(v, curbranch, currev)
        if v == 'master':
            return
        # make sure I'll end up within the revision range of the
        # destination branch
        if self.caching:
            if v not in self._branch_parents:
                self._branch_parents[v] = curbranch
                self._branch_start[v] = currev
            parrev = self._branch_start[v]
        else:
            parrev = self.db.parrev(v)
        if currev < parrev:
            raise ValueError(
                "Tried to jump to branch {br}, which starts at revision {rv}. "
                "Go to rev {rv} or later to use this branch.".format(
                    br=v,
                    rv=currev
                )
            )
        self.db.globl['branch'] = v
        if self.engine.caching:
            self._obranch = v

    @property
    def rev(self):
        """Return the global value ``rev``, or ``self._orev`` if that's set"""
        if self._orev is not None:
            return self._orev
        return self.db.globl['rev']

    @rev.setter
    def rev(self, v):
        """Set the global value ``rev``, first checking that it's not before
        the start of this branch. If it is, also go to the parent
        branch.

        """
        # first make sure the cursor is not before the start of this branch
        branch = self.branch
        if branch != 'master':
            if self.caching:
                parent = self._branch_parents[branch]
                parent_rev = self._branch_start[branch]
            else:
                (parent, parent_rev) = self.db.parparrev(branch)
            if v < int(parent_rev):
                raise ValueError(
                    "The revision number {revn} "
                    "occurs before the start of "
                    "the branch {brnch}".format(revn=v, brnch=branch)
                )
        self.db.globl['rev'] = v
        assert(self.rev == v)
        if self.caching:
            self._orev = v

    def commit(self):
        """Alias of ``self.db.commit``"""
        self.db.commit()

    def close(self):
        """Alias of ``self.db.close``"""
        self.db.close()

    def initdb(self):
        """Alias of ``self.db.initdb``"""
        self.db.initdb()

    def _init_graph(self, name, type_s='Graph'):
        if self.db.have_graph(name):
            raise GraphNameError("Already have a graph by that name")
        self.db.new_graph(name, type_s)

    def new_graph(self, name, data=None, **attr):
        """Return a new instance of type Graph, initialized with the given
        data if provided.

        """
        self._init_graph(name, 'Graph')
        return Graph(self, name, data, **attr)

    def new_digraph(self, name, data=None, **attr):
        """Return a new instance of type DiGraph, initialized with the given
        data if provided.

        """
        self._init_graph(name, 'DiGraph')
        return DiGraph(self, name, data, **attr)

    def new_multigraph(self, name, data=None, **attr):
        """Return a new instance of type MultiGraph, initialized with the given
        data if provided.

        """
        self._init_graph(name, 'MultiGraph')
        return MultiGraph(self, name, data, **attr)

    def new_multidigraph(self, name, data=None, **attr):
        """Return a new instance of type MultiDiGraph, initialized with the given
        data if provided.

        """
        self._init_graph(name, 'MultiDiGraph')
        return MultiDiGraph(self, name, data, **attr)

    def get_graph(self, name):
        """Return a graph previously created with ``new_graph``,
        ``new_digraph``, ``new_multigraph``, or
        ``new_multidigraph``

        """
        graphtypes = {
            'Graph': Graph,
            'DiGraph': DiGraph,
            'MultiGraph': MultiGraph,
            'MultiDiGraph': MultiDiGraph
        }
        type_s = self.db.graph_type(name)
        if type_s not in graphtypes:
            raise GraphNameError("I don't know of a graph named {}".format(name))
        return graphtypes[type_s](self, name)

    def del_graph(self, name):
        """Remove all traces of a graph's existence from the database"""
        # make sure the graph exists before deleting anything
        self.get_graph(name)
        self.db.del_graph(name)

    def _active_branches(self, branch=None, rev=None):
        """Private use. Iterate over (branch, rev) pairs, where the branch is
        a descendant of the previous (starting with whatever branch is
        presently active and ending at 'master'), and the rev is the
        latest revision in the branch that matters.

        """
        b = branch or self.branch
        r = rev or self.rev
        if self.caching:
            yield b, r
            while b in self._branch_parents:
                r = self._branch_start[b]
                b = self._branch_parents[b]
                yield b, r
            return

        for pair in self.db.active_branches(b, r):
            yield pair

    def _branch_descendants(self, branch=None):
        """Iterate over all branches immediately descended from the current
        one (or the given one, if available).

        """
        branch = branch or self.branch
        if not self.caching:
            for desc in self.db.branch_descendants(branch):
                yield desc
            return
        for b in self._branches[branch].keys():
            yield b
        for child in self._branches[branch].keys():
            for b in self._branch_descendants(child):
                yield b

__all__ = [ORM, 'alchemy', 'graph', 'query', 'reify', 'window', 'xjson']
<commit_msg>Fix a NameError where I forgot I was dealing with the engine already<commit_after># This file is part of gorm, an object relational mapper for versioned graphs.
# Copyright (C) 2014 Zachary Spector.
from collections import defaultdict, deque
from .graph import (
    Graph,
    DiGraph,
    MultiGraph,
    MultiDiGraph,
)
from .query import QueryEngine
from .reify import reify

class GraphNameError(KeyError):
    pass

class ORM(object):
    """Instantiate this with the same string argument you'd use for a
    SQLAlchemy ``create_engine`` call. This will be your interface to
    gorm.

    """
    @reify
    def _graph_val_cache(self):
        assert(self.caching)
        from .window import WindowDict
        r = defaultdict(  # graph:
            lambda: defaultdict(  # key:
                lambda: defaultdict(  # branch:
                    WindowDict  # rev: value
                )
            )
        )
        for (graph, key, branch, rev, value) in self.db.graph_val_dump():
            r[graph][key][branch][rev] = value
        return r

    @reify
    def _node_val_cache(self):
        assert(self.caching)
        from .window import WindowDict
        r = defaultdict(  # graph:
            lambda: defaultdict(  # node:
                lambda: defaultdict(  # key:
                    lambda: defaultdict(  # branch:
                        WindowDict  # rev: value
                    )
                )
            )
        )
        for (graph, node, key, branch, rev, value) in self.db.node_val_dump():
            r[graph][node][key][branch][rev] = value
        return r

    @reify
    def _nodes_cache(self):
        assert(self.caching)
        from .window import WindowDict
        r = defaultdict(  # graph:
            lambda: defaultdict(  # node:
                lambda: defaultdict(  # branch:
                    WindowDict  # rev: extant
                )
            )
        )
        for (graph, node, branch, rev, extant) in self.db.nodes_dump():
            r[graph][node][branch][rev] = extant
        return r

    @reify
    def _edge_val_cache(self):
        assert(self.caching)
        from .window import WindowDict
        r = defaultdict(  # graph:
            lambda: defaultdict(  # nodeA:
                lambda: defaultdict(  # nodeB:
                    lambda: defaultdict(  # idx:
                        lambda: defaultdict(  # key:
                            lambda: defaultdict(  # branch:
                                WindowDict  # rev: value
                            )
                        )
                    )
                )
            )
        )
        for (
                graph, nodeA, nodeB, idx, key, branch, rev, value
        ) in self.db.edge_val_dump():
            r[graph][nodeA][nodeB][idx][key][branch][rev] = value
        return r

    @reify
    def _edges_cache(self):
        assert self.caching
        from .window import WindowDict
        r = defaultdict(  # graph:
            lambda: defaultdict(  # nodeA:
                lambda: defaultdict(  # nodeB:
                    lambda: defaultdict(  # idx:
                        lambda: defaultdict(  # branch:
                            WindowDict  # rev: extant
                        )
                    )
                )
            )
        )
        for (
                graph, nodeA, nodeB, idx, branch, rev, extant
        ) in self.db.edges_dump():
            r[graph][nodeA][nodeB][idx][branch][rev] = extant
        return r

    def __init__(
            self,
            dbstring,
            alchemy=True,
            connect_args={},
            query_engine_class=QueryEngine,
            json_dump=None,
            json_load=None,
            caching=True
    ):
        """Make a SQLAlchemy engine if possible, else a sqlite3 connection. In
        either case, begin a transaction.

        """
        self.db = query_engine_class(dbstring, connect_args, alchemy, json_dump, json_load)
        self._branches = {}
        self._obranch = None
        self._orev = None
        self.db.initdb()
        if caching:
            self.caching = True
            self._obranch = self.branch
            self._orev = self.rev
            self._timestream = {'master': {}}
            self._branch_start = {}
            self._branches = {'master': self._timestream['master']}
            self._branch_parents = {}
            self._active_branches_cache = []
            self.db.active_branches = self._active_branches
            todo = deque(self.db.timestream_data())
            while todo:
                (branch, parent, parent_tick) = working = todo.popleft()
                if branch == 'master':
                    continue
                if parent in self._branches:
                    assert(branch not in self._branches)
                    self._branches[parent][branch] = {}
                    self._branches[branch] = self._branches[parent][branch]
                    self._branch_parents['branch'] = parent
                    self._branch_start[branch] = parent_tick
                else:
                    todo.append(working)

    def __enter__(self):
        """Enable the use of the ``with`` keyword"""
        return self

    def __exit__(self, *args):
        """Alias for ``close``"""
        self.close()

    def _havebranch(self, b):
        """Private use. Checks that the branch is known about."""
        if self.caching and b in self._branches:
            return True
        return self.db.have_branch(b)

    def is_parent_of(self, parent, child):
        """Return whether ``child`` is a branch descended from ``parent`` at
        any remove.

        """
        # trivial cases
        if child in self._branches and self._branches[child][0] == parent:
            return True
        elif child == parent:
            return False
        # I will be recursing a lot so just cache all the branch info
        self._childbranch = {}
        self._ancestry = {}
        for (branch, parent, parent_rev) in self.db.all_branches():
            self._branches[branch] = (parent, parent_rev)
            self._childbranch[parent] = branch

        self._ancestry[child] = set([parent])
        lineage = self._ancestry[child]

        def recurse(oneparent):
            if oneparent in lineage:
                return True
            if oneparent not in self._branches:
                return False
            if self._branches[oneparent][0] in lineage:
                return True
            lineage.add(oneparent)
            return recurse(self._branches[oneparent][0])
        return recurse(child)

    @property
    def branch(self):
        """Return the global value ``branch``, or ``self._obranch`` if it's
        set

        """
        if self._obranch is not None:
            return self._obranch
        return self.db.globl['branch']

    @branch.setter
    def branch(self, v):
        """Set the global value ``branch`` and note that the branch's (parent,
        parent_rev) are the (branch, tick) set previously

        """
        curbranch = self.branch
        currev = self.rev
        if not self._havebranch(v):
            # assumes the present revision in the parent branch has
            # been finalized.
            self.db.new_branch(v, curbranch, currev)
        if v == 'master':
            return
        # make sure I'll end up within the revision range of the
        # destination branch
        if self.caching:
            if v not in self._branch_parents:
                self._branch_parents[v] = curbranch
                self._branch_start[v] = currev
            parrev = self._branch_start[v]
        else:
            parrev = self.db.parrev(v)
        if currev < parrev:
            raise ValueError(
                "Tried to jump to branch {br}, which starts at revision {rv}. "
                "Go to rev {rv} or later to use this branch.".format(
                    br=v,
                    rv=currev
                )
            )
        self.db.globl['branch'] = v
        if self.caching:
            self._obranch = v

    @property
    def rev(self):
        """Return the global value ``rev``, or ``self._orev`` if that's set"""
        if self._orev is not None:
            return self._orev
        return self.db.globl['rev']

    @rev.setter
    def rev(self, v):
        """Set the global value ``rev``, first checking that it's not before
        the start of this branch. If it is, also go to the parent
        branch.

        """
        # first make sure the cursor is not before the start of this branch
        branch = self.branch
        if branch != 'master':
            if self.caching:
                parent = self._branch_parents[branch]
                parent_rev = self._branch_start[branch]
            else:
                (parent, parent_rev) = self.db.parparrev(branch)
            if v < int(parent_rev):
                raise ValueError(
                    "The revision number {revn} "
                    "occurs before the start of "
                    "the branch {brnch}".format(revn=v, brnch=branch)
                )
        self.db.globl['rev'] = v
        assert(self.rev == v)
        if self.caching:
            self._orev = v

    def commit(self):
        """Alias of ``self.db.commit``"""
        self.db.commit()

    def close(self):
        """Alias of ``self.db.close``"""
        self.db.close()

    def initdb(self):
        """Alias of ``self.db.initdb``"""
        self.db.initdb()

    def _init_graph(self, name, type_s='Graph'):
        if self.db.have_graph(name):
            raise GraphNameError("Already have a graph by that name")
        self.db.new_graph(name, type_s)

    def new_graph(self, name, data=None, **attr):
        """Return a new instance of type Graph, initialized with the given
        data if provided.

        """
        self._init_graph(name, 'Graph')
        return Graph(self, name, data, **attr)

    def new_digraph(self, name, data=None, **attr):
        """Return a new instance of type DiGraph, initialized with the given
        data if provided.

        """
        self._init_graph(name, 'DiGraph')
        return DiGraph(self, name, data, **attr)

    def new_multigraph(self, name, data=None, **attr):
        """Return a new instance of type MultiGraph, initialized with the given
        data if provided.

        """
        self._init_graph(name, 'MultiGraph')
        return MultiGraph(self, name, data, **attr)

    def new_multidigraph(self, name, data=None, **attr):
        """Return a new instance of type MultiDiGraph, initialized with the given
        data if provided.

        """
        self._init_graph(name, 'MultiDiGraph')
        return MultiDiGraph(self, name, data, **attr)

    def get_graph(self, name):
        """Return a graph previously created with ``new_graph``,
        ``new_digraph``, ``new_multigraph``, or
        ``new_multidigraph``

        """
        graphtypes = {
            'Graph': Graph,
            'DiGraph': DiGraph,
            'MultiGraph': MultiGraph,
            'MultiDiGraph': MultiDiGraph
        }
        type_s = self.db.graph_type(name)
        if type_s not in graphtypes:
            raise GraphNameError("I don't know of a graph named {}".format(name))
        return graphtypes[type_s](self, name)

    def del_graph(self, name):
        """Remove all traces of a graph's existence from the database"""
        # make sure the graph exists before deleting anything
        self.get_graph(name)
        self.db.del_graph(name)

    def _active_branches(self, branch=None, rev=None):
        """Private use. Iterate over (branch, rev) pairs, where the branch is
        a descendant of the previous (starting with whatever branch is
        presently active and ending at 'master'), and the rev is the
        latest revision in the branch that matters.

        """
        b = branch or self.branch
        r = rev or self.rev
        if self.caching:
            yield b, r
            while b in self._branch_parents:
                r = self._branch_start[b]
                b = self._branch_parents[b]
                yield b, r
            return

        for pair in self.db.active_branches(b, r):
            yield pair

    def _branch_descendants(self, branch=None):
        """Iterate over all branches immediately descended from the current
        one (or the given one, if available).

        """
        branch = branch or self.branch
        if not self.caching:
            for desc in self.db.branch_descendants(branch):
                yield desc
            return
        for b in self._branches[branch].keys():
            yield b
        for child in self._branches[branch].keys():
            for b in self._branch_descendants(child):
                yield b

__all__ = [ORM, 'alchemy', 'graph', 'query', 'reify', 'window', 'xjson']
<|endoftext|>


<commit_before>from django.conf import settings
from django.conf.urls import url, include
from django.conf.urls.static import static
from django.contrib import admin

from main.views import *

urlpatterns = [
    # url(r'^/', include([
    # Admin
    url(r'^admin/', admin.site.urls),

    # Index
    url(r'^$', index, name='index'),

    # Pages
    url(r'^studio/$', studio, name='about_us'),
    url(r'^about_us/$', about_us, name='about_us'),

    # Profile
    url(r'^profile/$', profile_page, name='SelfProfile'),
    url(r'^profile/(?P<_id>[0-9]+)/$', profile_page, name='Profile'),
    url(r'^profile/edit/$', update_profile),
    url(r'^profile/remove_education/$', remove_education),
    url(r'^profile/add_achievement/$', add_achievement),
    url(r'^profile/add_project/$', add_project),
    url(r'^profile/remove_achievement/$', remove_achievement),
    url(r'^profile/remove_project/$', remove_project),

    # Login system
    url(r'^logout/$', logout_wrapper),
    url(r'^login/$', login_wrapper),
    url(r'^activate/', activation),

    # Signup render
    url(r'^signup/$', signup_page),

    # Social backend
    url(r'^accounts/', include('allauth.urls')),

    # API
    url(r'^api/get_needed_skills/', get_needed_skills),
    url(r'^api/get_needed_cities/', get_needed_cities),
    url(r'^api/get_needed_schools_names/', get_needed_schools_names),
    url(r'^api/update_avatar/', update_avatar),
    url(r'^api/get_user_info/', generate_csv),
    url(r'^api/add_subscriber/', subscribe_email),

    # Bank
    url(r'^bank/shop/', shop),
    url(r'^bank/buy/', buy_good)
    # ]))
]

if settings.DEBUG:
    from django.views.static import serve

    urlpatterns.append(url(r'^favicon.ico', lambda r: serve(r, 'favicon.ico')))

if settings.DEBUG:
    # static files (images, css, javascript, etc.)
    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
<commit_msg>Add redirect from summer<commit_after>from django.conf import settings
from django.conf.urls import url, include
from django.conf.urls.static import static
from django.contrib import admin

from main.views import *

urlpatterns = [
    # url(r'^/', include([
    # Admin
    url(r'^admin/', admin.site.urls),

    # Index
    url(r'^$', index, name='index'),

    url(r'camp_summer/^$', lambda: redirect('/')),

    # Pages
    url(r'^studio/$', studio, name='about_us'),
    url(r'^about_us/$', about_us, name='about_us'),

    # Profile
    url(r'^profile/$', profile_page, name='SelfProfile'),
    url(r'^profile/(?P<_id>[0-9]+)/$', profile_page, name='Profile'),
    url(r'^profile/edit/$', update_profile),
    url(r'^profile/remove_education/$', remove_education),
    url(r'^profile/add_achievement/$', add_achievement),
    url(r'^profile/add_project/$', add_project),
    url(r'^profile/remove_achievement/$', remove_achievement),
    url(r'^profile/remove_project/$', remove_project),

    # Login system
    url(r'^logout/$', logout_wrapper),
    url(r'^login/$', login_wrapper),
    url(r'^activate/', activation),

    # Signup render
    url(r'^signup/$', signup_page),

    # Social backend
    url(r'^accounts/', include('allauth.urls')),

    # API
    url(r'^api/get_needed_skills/', get_needed_skills),
    url(r'^api/get_needed_cities/', get_needed_cities),
    url(r'^api/get_needed_schools_names/', get_needed_schools_names),
    url(r'^api/update_avatar/', update_avatar),
    url(r'^api/get_user_info/', generate_csv),
    url(r'^api/add_subscriber/', subscribe_email),

    # Bank
    url(r'^bank/shop/', shop),
    url(r'^bank/buy/', buy_good)
    # ]))
]

if settings.DEBUG:
    from django.views.static import serve

    urlpatterns.append(url(r'^favicon.ico', lambda r: serve(r, 'favicon.ico')))

if settings.DEBUG:
    # static files (images, css, javascript, etc.)
    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

from gpmcc.utils import config as cu
from gpmcc.utils import test as tu
from gpmcc import state
import numpy

# This script generates acolumn of every data type and plots inference
# in real time

numpy.random.seed(10)

# set up the data generation
n_rows = 200
view_weights = numpy.ones(1)
cluster_weights = [ numpy.array([.33, .33, .34]) ]
cctypes = ['beta_uc', 'normal','normal_uc','poisson','multinomial(k=8)',
    'vonmises', 'vonmises_uc','binomial', 'lognormal','exponential',
    'geometric']
separation = [.95]*11
cctypes, distargs = cu.parse_distargs(cctypes)

T, Zv, Zc, dims = tu.gen_data_table(n_rows, view_weights, cluster_weights,
    cctypes, distargs, separation, return_dims=True)

S = state.State(T, cctypes, distargs, seed=0)
S.transition(N=10, do_plot=True)
<commit_msg>Parametrize example.<commit_after># -*- coding: utf-8 -*-

#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

from gpmcc.utils import config as cu
from gpmcc.utils import test as tu
from gpmcc import state
import numpy

# This script generates acolumn of every data type and plots inference
# in real time

numpy.random.seed(10)

# set up the data generation
n_rows = 200
view_weights = numpy.ones(1)
cluster_weights = [ numpy.array([.33, .33, .34]) ]
cctypes = ['beta_uc', 'normal','normal_uc','poisson','multinomial(k=8)',
    'vonmises', 'vonmises_uc','binomial', 'lognormal','exponential',
    'geometric']
separation = [.95] * len(cctypes)
cctypes, distargs = cu.parse_distargs(cctypes)

T, Zv, Zc, dims = tu.gen_data_table(n_rows, view_weights, cluster_weights,
    cctypes, distargs, separation, return_dims=True)

S = state.State(T, cctypes, distargs, seed=0)
S.transition(N=10, do_plot=True)
<|endoftext|>


<commit_before>"""
The Response class is the result of network request maden with Grab instance.
"""
import re
from copy import copy
import logging
import email
from StringIO import StringIO
#from cookielib import CookieJar
from urllib2 import Request
import os
import json
from urlparse import urlsplit, parse_qs

from tools.files import hash_path

RE_XML_DECLARATION = re.compile(r'^[\r\n\t]*<\?xml[^>]+\?>', re.I)
RE_DECLARATION_ENCODING = re.compile(r'encoding\s*=\s*["\']([^"\']+)["\']')
RE_META_CHARSET = re.compile(r'<meta[^>]+content\s*=\s*[^>]+charset=([-\w]+)',
                             re.I)

class Response(object):
    """
    HTTP Response.
    """

    def __init__(self):
        self.status = None
        self.code = None
        self.head = None
        self.body = None
        self.headers =None
        self.time = None
        self.url = None
        self.cookies = {}
        #self.cookiejar = None
        self.charset = 'utf-8'
        self._unicode_body = None

    def parse(self, charset=None):
        """
        Parse headers and cookies.

        This method is called after Grab instance performes network request.
        """

        # Extract only valid lines which contain ":" character
        valid_lines = []
        for line in self.head.split('\n'):
            line = line.rstrip('\r')
            if line:
                # Each HTTP line meand the start of new response
                # self.head could contains info about multiple responses
                # For example, then 301/302 redirect was processed automatically
                # Maybe it is a bug and should be fixed
                # Anyway, we handle this issue here and save headers
                # only from last response
                if line.startswith('HTTP'):
                    self.status = line
                    valid_lines = []
                else:
                    if ':' in line:
                        valid_lines.append(line)

        self.headers = email.message_from_string('\n'.join(valid_lines))
        #self.cookiejar = CookieJar()
        #self.cookiejar._extract_cookies(self, Request(self.url))
        #for cookie in self.cookiejar:
            #self.cookies[cookie.name] = cookie.value

        if charset is None:
            self.detect_charset()
        else:
            self.charset = charset

        self._unicode_body = None

    def info(self):
        """
        This method need for using Response instance in
        ``Cookiejar.extract_cookies`` method.
        """

        return self.headers

    def detect_charset(self):
        """
        Detect charset of the response.

        Try following methods:
        * meta[name="Http-Equiv"]
        * XML declaration
        * HTTP Content-Type header

        Ignore unknown charsets.

        Use utf-8 as fallback charset.
        """

        charset = None

        # Try to extract charset from http-equiv meta tag
        if self.body:
            body_lower = self.body.lower()
            pos = body_lower.find('')
            if pos > -1:
                html_head = body_lower[:pos]
                if html_head.find('http-equiv') > -1:
                    try:
                        charset = RE_META_CHARSET.search(html_head).group(1)
                    except AttributeError:
                        pass

        # Try to process XML declaration
        if not charset:
            if self.body:
                if self.body.startswith('<?xml'):
                    match = RE_XML_DECLARATION.search(self.body)
                    if match:
                        enc_match = RE_DECLARATION_ENCODING.search(match.group(0))
                        if enc_match:
                            charset = enc_match.group(1)

        if not charset:
            if 'Content-Type' in self.headers:
                pos = self.headers['Content-Type'].find('charset=')
                if pos > -1:
                    charset = self.headers['Content-Type'][(pos + 8):]

        if charset:
            # Check that python know such charset
            try:
                u'x'.encode(charset)
            except LookupError:
                logging.error('Unknown charset found: %s' % charset)
                self.charset = 'utf-8'
            else:
                self.charset = charset

    def unicode_body(self, ignore_errors=True, strip_xml_declaration=False):
        """
        Return response body as unicode string.
        """

        if not self._unicode_body:
            if ignore_errors:
                errors = 'ignore'
            else:
                errors = 'strict'
            ubody = self.body.decode(self.charset, errors)
            if strip_xml_declaration:
                ubody = RE_XML_DECLARATION.sub('', ubody)
            self._unicode_body = ubody
        return self._unicode_body

    def copy(self):
        """
        Clone the Response object.
        """

        obj = Response()

        copy_keys = ('status', 'code', 'head', 'body', 'time',
                     'url', 'charset', '_unicode_body')
        for key in copy_keys:
            setattr(obj, key, getattr(self, key))

        obj.headers = copy(self.headers)
        obj.cookies = copy(self.cookies)
        #obj.cookiejar = copy(self.cookiejar)

        return obj

    def save(self, path, create_dirs=False):
        """
        Save response body to file.
        """

        path_dir, path_fname = os.path.split(path)
        if not os.path.exists(path_dir):
            try:
                os.makedirs(path_dir)
            except OSError:
                pass

        with open(path, 'wb') as out:
            out.write(self.body)

    def save_hash(self, location, basedir, ext=None):
        """
        Save response body into file with special path
        builded from hash. That allows to lower number of files
        per directory.

        :param location: URL of file or something else. It is
            used to build the SHA1 hash.
        :param basedir: base directory to save the file. Note that
            file will not be saved directly to this directory but to
            some sub-directory of `basedir`
        :param ext: extension which should be appended to file name. The
            dot is inserted automatically between filename and extension.
        :returns: path to saved file relative to `basedir`

        Example::

            >>> url = 'http://yandex.ru/logo.png'
            >>> g.go(url)
            >>> g.response.save_hash(url, 'some_dir', ext='png')
            'e8/dc/f2918108788296df1facadc975d32b361a6a.png'
            # the file was saved to $PWD/some_dir/e8/dc/...

        TODO: replace `basedir` with two options: root and save_to. And
        returns save_to + path
        """

        rel_path = hash_path(location, ext=ext)
        path = os.path.join(basedir, rel_path)
        if not os.path.exists(path):
            path_dir, path_fname = os.path.split(path)
            try:
                os.makedirs(path_dir)
            except OSError:
                pass
            with open(path, 'wb') as out:
                out.write(self.body)
        return rel_path

    @property
    def json(self):
        """
        Return response body deserialized into JSON object.
        """

        return json.loads(self.body)

    def url_details(self):
        """
        Return result of urlsplit function applied to response url.
        """

        return urlsplit(self.url)

    def query_param(self, key):
        """
        Return value of parameter in query string.
        """

        return parse_qs(self.url_details().query)[key][0]
<commit_msg>Add browse method to response class. This method allows to open content of current response in GUI webbrowser<commit_after>"""
The Response class is the result of network request maden with Grab instance.
"""
import re
from copy import copy
import logging
import email
from StringIO import StringIO
#from cookielib import CookieJar
from urllib2 import Request
import os
import json
from urlparse import urlsplit, parse_qs
import tempfile
import webbrowser

from tools.files import hash_path

RE_XML_DECLARATION = re.compile(r'^[\r\n\t]*<\?xml[^>]+\?>', re.I)
RE_DECLARATION_ENCODING = re.compile(r'encoding\s*=\s*["\']([^"\']+)["\']')
RE_META_CHARSET = re.compile(r'<meta[^>]+content\s*=\s*[^>]+charset=([-\w]+)',
                             re.I)

class Response(object):
    """
    HTTP Response.
    """

    def __init__(self):
        self.status = None
        self.code = None
        self.head = None
        self.body = None
        self.headers =None
        self.time = None
        self.url = None
        self.cookies = {}
        #self.cookiejar = None
        self.charset = 'utf-8'
        self._unicode_body = None

    def parse(self, charset=None):
        """
        Parse headers and cookies.

        This method is called after Grab instance performes network request.
        """

        # Extract only valid lines which contain ":" character
        valid_lines = []
        for line in self.head.split('\n'):
            line = line.rstrip('\r')
            if line:
                # Each HTTP line meand the start of new response
                # self.head could contains info about multiple responses
                # For example, then 301/302 redirect was processed automatically
                # Maybe it is a bug and should be fixed
                # Anyway, we handle this issue here and save headers
                # only from last response
                if line.startswith('HTTP'):
                    self.status = line
                    valid_lines = []
                else:
                    if ':' in line:
                        valid_lines.append(line)

        self.headers = email.message_from_string('\n'.join(valid_lines))
        #self.cookiejar = CookieJar()
        #self.cookiejar._extract_cookies(self, Request(self.url))
        #for cookie in self.cookiejar:
            #self.cookies[cookie.name] = cookie.value

        if charset is None:
            self.detect_charset()
        else:
            self.charset = charset

        self._unicode_body = None

    def info(self):
        """
        This method need for using Response instance in
        ``Cookiejar.extract_cookies`` method.
        """

        return self.headers

    def detect_charset(self):
        """
        Detect charset of the response.

        Try following methods:
        * meta[name="Http-Equiv"]
        * XML declaration
        * HTTP Content-Type header

        Ignore unknown charsets.

        Use utf-8 as fallback charset.
        """

        charset = None

        # Try to extract charset from http-equiv meta tag
        if self.body:
            body_lower = self.body.lower()
            pos = body_lower.find('')
            if pos > -1:
                html_head = body_lower[:pos]
                if html_head.find('http-equiv') > -1:
                    try:
                        charset = RE_META_CHARSET.search(html_head).group(1)
                    except AttributeError:
                        pass

        # Try to process XML declaration
        if not charset:
            if self.body:
                if self.body.startswith('<?xml'):
                    match = RE_XML_DECLARATION.search(self.body)
                    if match:
                        enc_match = RE_DECLARATION_ENCODING.search(match.group(0))
                        if enc_match:
                            charset = enc_match.group(1)

        if not charset:
            if 'Content-Type' in self.headers:
                pos = self.headers['Content-Type'].find('charset=')
                if pos > -1:
                    charset = self.headers['Content-Type'][(pos + 8):]

        if charset:
            # Check that python know such charset
            try:
                u'x'.encode(charset)
            except LookupError:
                logging.error('Unknown charset found: %s' % charset)
                self.charset = 'utf-8'
            else:
                self.charset = charset

    def unicode_body(self, ignore_errors=True, strip_xml_declaration=False):
        """
        Return response body as unicode string.
        """

        if not self._unicode_body:
            if ignore_errors:
                errors = 'ignore'
            else:
                errors = 'strict'
            ubody = self.body.decode(self.charset, errors)
            if strip_xml_declaration:
                ubody = RE_XML_DECLARATION.sub('', ubody)
            self._unicode_body = ubody
        return self._unicode_body

    def copy(self):
        """
        Clone the Response object.
        """

        obj = Response()

        copy_keys = ('status', 'code', 'head', 'body', 'time',
                     'url', 'charset', '_unicode_body')
        for key in copy_keys:
            setattr(obj, key, getattr(self, key))

        obj.headers = copy(self.headers)
        obj.cookies = copy(self.cookies)
        #obj.cookiejar = copy(self.cookiejar)

        return obj

    def save(self, path, create_dirs=False):
        """
        Save response body to file.
        """

        path_dir, path_fname = os.path.split(path)
        if not os.path.exists(path_dir):
            try:
                os.makedirs(path_dir)
            except OSError:
                pass

        with open(path, 'wb') as out:
            out.write(self.body)

    def save_hash(self, location, basedir, ext=None):
        """
        Save response body into file with special path
        builded from hash. That allows to lower number of files
        per directory.

        :param location: URL of file or something else. It is
            used to build the SHA1 hash.
        :param basedir: base directory to save the file. Note that
            file will not be saved directly to this directory but to
            some sub-directory of `basedir`
        :param ext: extension which should be appended to file name. The
            dot is inserted automatically between filename and extension.
        :returns: path to saved file relative to `basedir`

        Example::

            >>> url = 'http://yandex.ru/logo.png'
            >>> g.go(url)
            >>> g.response.save_hash(url, 'some_dir', ext='png')
            'e8/dc/f2918108788296df1facadc975d32b361a6a.png'
            # the file was saved to $PWD/some_dir/e8/dc/...

        TODO: replace `basedir` with two options: root and save_to. And
        returns save_to + path
        """

        rel_path = hash_path(location, ext=ext)
        path = os.path.join(basedir, rel_path)
        if not os.path.exists(path):
            path_dir, path_fname = os.path.split(path)
            try:
                os.makedirs(path_dir)
            except OSError:
                pass
            with open(path, 'wb') as out:
                out.write(self.body)
        return rel_path

    @property
    def json(self):
        """
        Return response body deserialized into JSON object.
        """

        return json.loads(self.body)

    def url_details(self):
        """
        Return result of urlsplit function applied to response url.
        """

        return urlsplit(self.url)

    def query_param(self, key):
        """
        Return value of parameter in query string.
        """

        return parse_qs(self.url_details().query)[key][0]

    def browse(self):
        """
        Save response in temporary file and open it in GUI browser.
        """

        fh, path = tempfile.mkstemp()
        self.save(path)
        webbrowser.open('file://' + path)
<|endoftext|>


<commit_before>from django.db import models

class Grain(models.Model):
    key = models.CharField(max_length=1024, unique=True, db_index=True)
    content_type = models.CharField(max_length=255, default='text/plain')
    value = models.TextField(blank=True)
<commit_msg>Added a unicode representation for grain instances<commit_after>from django.db import models

class Grain(models.Model):
    key = models.CharField(max_length=1024, unique=True, db_index=True)
    content_type = models.CharField(max_length=255, default='text/plain')
    value = models.TextField(blank=True)

    def __unicode__(self):
        return u'{0} ({1}): {2!r}'.format(
            self.key, self.content_type, self.value[:100])
<|endoftext|>


<commit_before>import os
import sys
import json
import string
import argparse as ap
from os import path
from glob import glob
from collections import namedtuple

import jedi

global verbose_, quiet_

def log(msg):
    if verbose_:
        sys.stderr.write(msg + '\n')

def error(msg):
    if not quiet_:
        sys.stderr.write(msg + '\n')

def graph(dir_, pretty=False, verbose=False, quiet=False, nSourceFilesTrunc=None):
    global verbose_, quiet_
    verbose_, quiet_ = verbose, quiet
    os.chdir(dir_)          # set working directory to be source directory

    source_files = get_source_files('.')
    if nSourceFilesTrunc is not None:
        source_files = source_files[:nSourceFilesTrunc]

    jedi.cache.never_clear_cache = True # never clear caches, because running in batch

    modules_and_files = [(filename_to_module_name(f), f) for f in source_files]
    for mf in modules_and_files:
        jedi.api.precache_parser(mf[1])

    defs, refs = get_defs_refs(source_files)

    # Add module/package defs
    for module, filename in modules_and_files:
        defs.append(Def(
            Path=module.replace('.', '/'),
            Kind='module',
            Name=string.split(module, '.')[-1],
            File=filename,
            DefStart=0,
            DefEnd=0,
            Exported=True,
            Docstring='',           # TODO: extract module/package-level doc
            Data=None,
        ))

    # De-duplicate definitions (local variables may be defined in more than one
    # place). Could do something smarter here, but for now, just take the first
    # definition that appears. (References also point to the first definition.)
    unique_defs = []
    unique_def_paths = set([])
    for def_ in defs:
        if not def_.Path in unique_def_paths:
            unique_defs.append(def_)
            unique_def_paths.add(def_.Path)

    # Self-references, dedup
    unique_refs = []
    unique_ref_keys = set([])
    for def_ in unique_defs:
        ref = Ref(
            DefPath=def_.Path,
            DefFile=path.abspath(def_.File),
            Def=True,
            File=def_.File,
            Start=def_.DefStart,
            End=def_.DefEnd,
            ToBuiltin=False,
        )
        ref_key = (ref.DefPath, ref.DefFile, ref.File, ref.Start, ref.End)
        if ref_key not in unique_ref_keys:
            unique_ref_keys.add(ref_key)
            unique_refs.append(ref)
    for ref in refs:
        ref_key = (ref.DefPath, ref.DefFile, ref.File, ref.Start, ref.End)
        if ref_key not in unique_ref_keys:
            unique_ref_keys.add(ref_key)
            unique_refs.append(ref)

    json_indent = 2 if pretty else None
    print json.dumps({
        'Defs': [d.__dict__ for d in unique_defs],
        'Refs': [r.__dict__ for r in unique_refs],
    }, indent=json_indent)

def get_source_files(dir_):
    source_files = []
    for dirpath, dirnames, filenames in os.walk(dir_):
        rel_dirpath = os.path.relpath(dirpath, dir_)
        for filename in filenames:
            if os.path.splitext(filename)[1] == '.py':
                source_files.append(os.path.join(rel_dirpath, filename))
    return source_files

def get_defs_refs(source_files):
    defs, refs = [], []

    evaluator = jedi.evaluate.Evaluator()
    for i, source_file in enumerate(source_files):
        parserContext = ParserContext(source_file)
        linecoler = LineColToOffConverter(parserContext.source)

        log('getting defs for source file (%d/%d) %s' % (i, len(source_files), source_file))
        try:
            for def_name in parserContext.defs():
                jedi_def = jedi.api.classes.Definition(evaluator, def_name)
                def_, err = jedi_def_to_def(jedi_def, source_file, linecoler)
                if err is None:
                    defs.append(def_)
                else:
                    error(err)
        except Exception as e:
            error('failed to get defs for source file %s: %s' % (source_file, str(e)))

        log('getting refs for source file (%d/%d) %s' % (i, len(source_files), source_file))
        try:
            for name_part, def_ in parserContext.refs():
                try:
                    full_name, err = full_name_of_def(def_, from_ref=True)
                    if err is not None or full_name == '':
                        raise Exception(err)
                    start = linecoler.convert(name_part.start_pos)
                    end = linecoler.convert(name_part.end_pos)
                    refs.append(Ref(
                        DefPath=full_name.replace('.', '/'),
                        DefFile=def_.module_path,
                        Def=False,
                        File=source_file,
                        Start=start,
                        End=end,
                        ToBuiltin=def_.in_builtin_module(),
                    ))
                except Exception as e:
                    error('failed to convert ref (%s) in source file %s: %s' % (str((name_part, def_)), source_file, str(e)))
        except Exception as e:
            error('failed to get refs for source file %s: %s' % (source_file, str(e)))

    return defs, refs

def jedi_def_to_def(def_, source_file, linecoler):
    full_name, err = full_name_of_def(def_)
    if err is not None:
        return None, err

    # If def_ is a name, then the location of the definition is the last name part
    if isinstance(def_._definition, jedi.parser.representation.Name):
        last_name = def_._definition.names[-1]
        start = linecoler.convert(last_name.start_pos)
        end = start + len(last_name._string)
    else:
        start = linecoler.convert(def_.start_pos)
        end = start_pos + len(def_.name)

    return Def(
        Path=full_name.replace('.', '/'),
        Kind=def_.type,
        Name=def_.name,
        File=source_file,
        DefStart=start,
        DefEnd=end,
        Exported=True,          # TODO: not all vars are exported
        Docstring=def_.docstring(),
        Data=None,
    ), None

def full_name_of_def(def_, from_ref=False):
    # TODO: This function
    # - currently fails for tuple assignments (e.g., 'x, y = 1, 3')
    # - doesn't distinguish between m(module).n(submodule) and m(module).n(contained-variable)

    if def_.in_builtin_module():
        return def_.full_name, None

    if def_.type == 'statement':
        # kludge for self.* definitions
        if def_.parent().type == 'function' and def_._definition.names[0]._string == u'self':
            parent = def_.parent()
            while parent.type != 'class':
                parent = parent.parent()
            full_name = ('%s.%s' % (parent.full_name, def_.name))
        else:
            full_name = ('%s.%s' % (def_.full_name, def_.name))
    elif def_.type == 'param':
        full_name = ('%s.%s' % (def_.full_name, def_.name))
    else:
        full_name = def_.full_name

    module_path = def_.module_path
    if from_ref:
        module_path, err = abs_module_path_to_relative_module_path(module_path)
        if err is not None:
            return None, err

    supermodule = supermodule_path(module_path).replace('/', '.')

    # definition definitions' full_name property contains only the promixal module, so we need to add back the parent
    # module components. Luckily, the module_path is relative in this case.
    return path.join(supermodule, full_name), None

def supermodule_path(module_path):
    if path.basename(module_path) == '__init__.py':
        return path.dirname(path.dirname(module_path))
    return path.dirname(module_path)

def abs_module_path_to_relative_module_path(module_path):
    relpath = path.relpath(module_path) # relative from pwd (which is set in main)
    if not relpath.startswith('..'):
        return relpath, None
    components = module_path.split(os.sep)
    pIdx = -1
    for i, cmpt in enumerate(components):
        if cmpt in ['site-packages', 'dist-packages']:
            pIdx = i
            break
    if pIdx != -1:
        return path.join(*components[i+1:]), None

    for i, cmpt in enumerate(components):
        if cmpt.startswith('python'):
            pIdx = i
            break
    if pIdx != -1:
        return path.join(*components[i+1:]), None
    return None, ("could not convert absolute module path %s to relative module path" % module_path)

Def = namedtuple('Def', ['Path', 'Kind', 'Name', 'File', 'DefStart', 'DefEnd', 'Exported', 'Docstring', 'Data'])
Ref = namedtuple('Ref', ['DefPath', 'DefFile', 'Def', 'File', 'Start', 'End', "ToBuiltin"])

class ParserContext(object):
    def __init__(self, source_file):
        self.source_file = source_file
        with open(source_file) as sf:
            self.source = jedi.common.source_to_unicode(sf.read(), 'utf-8')
            self.parser = jedi.parser.Parser(self.source, module_path=source_file)

    def defs(self):
        for name in self.parser.module.get_defined_names():
            for d in self.defs_(name): yield d

    def defs_(self, name):
        if isinstance(name.parent, jedi.parser.representation.Import):
            return

        yield name
        if isinstance(name.parent, jedi.parser.representation.Scope) and not isinstance(name.parent, jedi.parser.representation.Flow):
            for subname in name.parent.get_defined_names():
                for d in self.defs_(subname): yield d

    def refs(self):
        for r in self.scope_refs(self.parser.module):
            yield r

    def scope_refs(self, scope):
        for import_ in scope.imports:
            for r in self.import_refs(import_):
                yield r
        for stmt in scope.statements:
            for r in self.stmt_refs(stmt):
                yield r
        for ret in scope.returns:
            for r in self.stmt_refs(ret):
                yield r
        for subscope in scope.subscopes:
            for r in self.scope_refs(subscope):
                yield r

    def import_refs(self, import_):
        for name in import_.get_all_import_names():
            for name_part in name.names:
                defs = jedi.api.Script(
                    path=self.source_file,
                    line=name_part.start_pos[0],
                    column=name_part.start_pos[1],
                ).goto_assignments()
                for def_ in defs:
                    yield (name_part, def_)

    def stmt_refs(self, stmt):
        if isinstance(stmt, jedi.parser.representation.KeywordStatement):
            return
        if stmt is None:
            return

        if stmt.is_scope():
            for r in self.scope_refs(stmt): yield r
            return

        for token in stmt._token_list:
            if not isinstance(token, jedi.parser.representation.Name):
                continue
            for name_part in token.names:
                # Note: we call goto_definitions instead of goto_assignments,
                # because otherwise the reference will not follow imports (and
                # also generates bogus local definitions whose paths conflict
                # with those of actual definitions). This uses a modified
                # goto_definitions (resolve_variables_to_types option) that
                # *DOES NOT* follow assignment statements to resolve variables
                # to types (because that's not what we want).
                defs = jedi.api.Script(
                    path=self.source_file,
                    line=name_part.start_pos[0],
                    column=name_part.start_pos[1],
                    resolve_variables_to_types=False,
                ).goto_definitions()

                # Note(beyang): For now, only yield the first definition.
                # Otherwise, multiple references to multiple definitions will
                # yield dup references. In the future, might want to do
                # something smarter here.
                i = 0
                for def_ in defs:
                    if i > 0: break
                    yield (name_part, def_)
                    i += 1

def resolve_import_paths(scopes):
    for s in scopes.copy():
        if isinstance(s, jedi.evaluate.imports.ImportWrapper):
            scopes.remove(s)
            scopes.update(resolve_import_paths(set(s.follow())))
    return scopes

def filename_to_module_name(filename):
    if path.basename(filename) == '__init__.py':
        return path.dirname(filename).replace('/', '.')
    return path.splitext(filename)[0].replace('/', '.')

class LineColToOffConverter(object):
    def __init__(self, source):
        source_lines = source.split('\n')
        cumulative_off = [0]
        for line in source_lines:
            cumulative_off.append(cumulative_off[-1] + len(line) + 1)
        self._cumulative_off = cumulative_off

    # Converts from (line, col) position to byte offset. line is 1-indexed, col is 0-indexed
    def convert(self, linecol):
        line, col = linecol[0] - 1, linecol[1]         # convert line to 0-indexed
        if line >= len(self._cumulative_off):
            return None, 'requested line out of bounds %d > %d' % (line+1, len(self._cumulative_off)-1)
        return self._cumulative_off[line] + col

if __name__ == '__main__':
    argser = ap.ArgumentParser(description='graph.py is a command that dumps all Python definitions and references found in code rooted at a directory')
    argser.add_argument('dir', help='path to root directory of code')
    argser.add_argument('--pretty', help='pretty print JSON output', action='store_true', default=False)
    argser.add_argument('--verbose', help='verbose', action='store_true', default=False)
    argser.add_argument('--quiet', help='quiet', action='store_true', default=False)
    args = argser.parse_args()
    if args.dir == '':
        error('target directory must not be empty')
        os.exit(1)
    graph(args.dir, pretty=args.pretty, verbose=args.verbose, quiet=args.quiet)
<commit_msg>builtins have no full_name, so just use name<commit_after>import os
import sys
import json
import string
import argparse as ap
from os import path
from glob import glob
from collections import namedtuple

import jedi

global verbose_, quiet_

def log(msg):
    if verbose_:
        sys.stderr.write(msg + '\n')

def error(msg):
    if not quiet_:
        sys.stderr.write(msg + '\n')

def graph(dir_, pretty=False, verbose=False, quiet=False, nSourceFilesTrunc=None):
    global verbose_, quiet_
    verbose_, quiet_ = verbose, quiet
    os.chdir(dir_)          # set working directory to be source directory

    source_files = get_source_files('.')
    if nSourceFilesTrunc is not None:
        source_files = source_files[:nSourceFilesTrunc]

    jedi.cache.never_clear_cache = True # never clear caches, because running in batch

    modules_and_files = [(filename_to_module_name(f), f) for f in source_files]
    for mf in modules_and_files:
        jedi.api.precache_parser(mf[1])

    defs, refs = get_defs_refs(source_files)

    # Add module/package defs
    for module, filename in modules_and_files:
        defs.append(Def(
            Path=module.replace('.', '/'),
            Kind='module',
            Name=string.split(module, '.')[-1],
            File=filename,
            DefStart=0,
            DefEnd=0,
            Exported=True,
            Docstring='',           # TODO: extract module/package-level doc
            Data=None,
        ))

    # De-duplicate definitions (local variables may be defined in more than one
    # place). Could do something smarter here, but for now, just take the first
    # definition that appears. (References also point to the first definition.)
    unique_defs = []
    unique_def_paths = set([])
    for def_ in defs:
        if not def_.Path in unique_def_paths:
            unique_defs.append(def_)
            unique_def_paths.add(def_.Path)

    # Self-references, dedup
    unique_refs = []
    unique_ref_keys = set([])
    for def_ in unique_defs:
        ref = Ref(
            DefPath=def_.Path,
            DefFile=path.abspath(def_.File),
            Def=True,
            File=def_.File,
            Start=def_.DefStart,
            End=def_.DefEnd,
            ToBuiltin=False,
        )
        ref_key = (ref.DefPath, ref.DefFile, ref.File, ref.Start, ref.End)
        if ref_key not in unique_ref_keys:
            unique_ref_keys.add(ref_key)
            unique_refs.append(ref)
    for ref in refs:
        ref_key = (ref.DefPath, ref.DefFile, ref.File, ref.Start, ref.End)
        if ref_key not in unique_ref_keys:
            unique_ref_keys.add(ref_key)
            unique_refs.append(ref)

    json_indent = 2 if pretty else None
    print json.dumps({
        'Defs': [d.__dict__ for d in unique_defs],
        'Refs': [r.__dict__ for r in unique_refs],
    }, indent=json_indent)

def get_source_files(dir_):
    source_files = []
    for dirpath, dirnames, filenames in os.walk(dir_):
        rel_dirpath = os.path.relpath(dirpath, dir_)
        for filename in filenames:
            if os.path.splitext(filename)[1] == '.py':
                source_files.append(os.path.join(rel_dirpath, filename))
    return source_files

def get_defs_refs(source_files):
    defs, refs = [], []

    evaluator = jedi.evaluate.Evaluator()
    for i, source_file in enumerate(source_files):
        parserContext = ParserContext(source_file)
        linecoler = LineColToOffConverter(parserContext.source)

        log('getting defs for source file (%d/%d) %s' % (i, len(source_files), source_file))
        try:
            for def_name in parserContext.defs():
                jedi_def = jedi.api.classes.Definition(evaluator, def_name)
                def_, err = jedi_def_to_def(jedi_def, source_file, linecoler)
                if err is None:
                    defs.append(def_)
                else:
                    error(err)
        except Exception as e:
            error('failed to get defs for source file %s: %s' % (source_file, str(e)))

        log('getting refs for source file (%d/%d) %s' % (i, len(source_files), source_file))
        try:
            for name_part, def_ in parserContext.refs():
                try:
                    full_name, err = full_name_of_def(def_, from_ref=True)
                    if err is not None:
                        raise Exception(err)
                    elif full_name == '':
                        raise Exception('full_name is empty')
                    start = linecoler.convert(name_part.start_pos)
                    end = linecoler.convert(name_part.end_pos)
                    refs.append(Ref(
                        DefPath=full_name.replace('.', '/'),
                        DefFile=def_.module_path,
                        Def=False,
                        File=source_file,
                        Start=start,
                        End=end,
                        ToBuiltin=def_.in_builtin_module(),
                    ))
                except Exception as e:
                    error('failed to convert ref (%s) in source file %s: %s' % (str((name_part, def_)), source_file, str(e)))
        except Exception as e:
            error('failed to get refs for source file %s: %s' % (source_file, str(e)))

    return defs, refs

def jedi_def_to_def(def_, source_file, linecoler):
    full_name, err = full_name_of_def(def_)
    if err is not None:
        return None, err

    # If def_ is a name, then the location of the definition is the last name part
    if isinstance(def_._definition, jedi.parser.representation.Name):
        last_name = def_._definition.names[-1]
        start = linecoler.convert(last_name.start_pos)
        end = start + len(last_name._string)
    else:
        start = linecoler.convert(def_.start_pos)
        end = start_pos + len(def_.name)

    return Def(
        Path=full_name.replace('.', '/'),
        Kind=def_.type,
        Name=def_.name,
        File=source_file,
        DefStart=start,
        DefEnd=end,
        Exported=True,          # TODO: not all vars are exported
        Docstring=def_.docstring(),
        Data=None,
    ), None

def full_name_of_def(def_, from_ref=False):
    # TODO: This function
    # - currently fails for tuple assignments (e.g., 'x, y = 1, 3')
    # - doesn't distinguish between m(module).n(submodule) and m(module).n(contained-variable)

    if def_.in_builtin_module():
        return def_.name, None

    if def_.type == 'statement':
        # kludge for self.* definitions
        if def_.parent().type == 'function' and def_._definition.names[0]._string == u'self':
            parent = def_.parent()
            while parent.type != 'class':
                parent = parent.parent()
            full_name = ('%s.%s' % (parent.full_name, def_.name))
        else:
            full_name = ('%s.%s' % (def_.full_name, def_.name))
    elif def_.type == 'param':
        full_name = ('%s.%s' % (def_.full_name, def_.name))
    else:
        full_name = def_.full_name

    module_path = def_.module_path
    if from_ref:
        module_path, err = abs_module_path_to_relative_module_path(module_path)
        if err is not None:
            return None, err

    supermodule = supermodule_path(module_path).replace('/', '.')

    # definition definitions' full_name property contains only the promixal module, so we need to add back the parent
    # module components. Luckily, the module_path is relative in this case.
    return path.join(supermodule, full_name), None

def supermodule_path(module_path):
    if path.basename(module_path) == '__init__.py':
        return path.dirname(path.dirname(module_path))
    return path.dirname(module_path)

def abs_module_path_to_relative_module_path(module_path):
    relpath = path.relpath(module_path) # relative from pwd (which is set in main)
    if not relpath.startswith('..'):
        return relpath, None
    components = module_path.split(os.sep)
    pIdx = -1
    for i, cmpt in enumerate(components):
        if cmpt in ['site-packages', 'dist-packages']:
            pIdx = i
            break
    if pIdx != -1:
        return path.join(*components[i+1:]), None

    for i, cmpt in enumerate(components):
        if cmpt.startswith('python'):
            pIdx = i
            break
    if pIdx != -1:
        return path.join(*components[i+1:]), None
    return None, ("could not convert absolute module path %s to relative module path" % module_path)

Def = namedtuple('Def', ['Path', 'Kind', 'Name', 'File', 'DefStart', 'DefEnd', 'Exported', 'Docstring', 'Data'])
Ref = namedtuple('Ref', ['DefPath', 'DefFile', 'Def', 'File', 'Start', 'End', "ToBuiltin"])

class ParserContext(object):
    def __init__(self, source_file):
        self.source_file = source_file
        with open(source_file) as sf:
            self.source = jedi.common.source_to_unicode(sf.read(), 'utf-8')
            self.parser = jedi.parser.Parser(self.source, module_path=source_file)

    def defs(self):
        for name in self.parser.module.get_defined_names():
            for d in self.defs_(name): yield d

    def defs_(self, name):
        if isinstance(name.parent, jedi.parser.representation.Import):
            return

        yield name
        if isinstance(name.parent, jedi.parser.representation.Scope) and not isinstance(name.parent, jedi.parser.representation.Flow):
            for subname in name.parent.get_defined_names():
                for d in self.defs_(subname): yield d

    def refs(self):
        for r in self.scope_refs(self.parser.module):
            yield r

    def scope_refs(self, scope):
        for import_ in scope.imports:
            for r in self.import_refs(import_):
                yield r
        for stmt in scope.statements:
            for r in self.stmt_refs(stmt):
                yield r
        for ret in scope.returns:
            for r in self.stmt_refs(ret):
                yield r
        for subscope in scope.subscopes:
            for r in self.scope_refs(subscope):
                yield r

    def import_refs(self, import_):
        for name in import_.get_all_import_names():
            for name_part in name.names:
                defs = jedi.api.Script(
                    path=self.source_file,
                    line=name_part.start_pos[0],
                    column=name_part.start_pos[1],
                ).goto_assignments()
                for def_ in defs:
                    yield (name_part, def_)

    def stmt_refs(self, stmt):
        if isinstance(stmt, jedi.parser.representation.KeywordStatement):
            return
        if stmt is None:
            return

        if stmt.is_scope():
            for r in self.scope_refs(stmt): yield r
            return

        for token in stmt._token_list:
            if not isinstance(token, jedi.parser.representation.Name):
                continue
            for name_part in token.names:
                # Note: we call goto_definitions instead of goto_assignments,
                # because otherwise the reference will not follow imports (and
                # also generates bogus local definitions whose paths conflict
                # with those of actual definitions). This uses a modified
                # goto_definitions (resolve_variables_to_types option) that
                # *DOES NOT* follow assignment statements to resolve variables
                # to types (because that's not what we want).
                defs = jedi.api.Script(
                    path=self.source_file,
                    line=name_part.start_pos[0],
                    column=name_part.start_pos[1],
                    resolve_variables_to_types=False,
                ).goto_definitions()

                # Note(beyang): For now, only yield the first definition.
                # Otherwise, multiple references to multiple definitions will
                # yield dup references. In the future, might want to do
                # something smarter here.
                i = 0
                for def_ in defs:
                    if i > 0: break
                    yield (name_part, def_)
                    i += 1

def resolve_import_paths(scopes):
    for s in scopes.copy():
        if isinstance(s, jedi.evaluate.imports.ImportWrapper):
            scopes.remove(s)
            scopes.update(resolve_import_paths(set(s.follow())))
    return scopes

def filename_to_module_name(filename):
    if path.basename(filename) == '__init__.py':
        return path.dirname(filename).replace('/', '.')
    return path.splitext(filename)[0].replace('/', '.')

class LineColToOffConverter(object):
    def __init__(self, source):
        source_lines = source.split('\n')
        cumulative_off = [0]
        for line in source_lines:
            cumulative_off.append(cumulative_off[-1] + len(line) + 1)
        self._cumulative_off = cumulative_off

    # Converts from (line, col) position to byte offset. line is 1-indexed, col is 0-indexed
    def convert(self, linecol):
        line, col = linecol[0] - 1, linecol[1]         # convert line to 0-indexed
        if line >= len(self._cumulative_off):
            return None, 'requested line out of bounds %d > %d' % (line+1, len(self._cumulative_off)-1)
        return self._cumulative_off[line] + col

if __name__ == '__main__':
    argser = ap.ArgumentParser(description='graph.py is a command that dumps all Python definitions and references found in code rooted at a directory')
    argser.add_argument('dir', help='path to root directory of code')
    argser.add_argument('--pretty', help='pretty print JSON output', action='store_true', default=False)
    argser.add_argument('--verbose', help='verbose', action='store_true', default=False)
    argser.add_argument('--quiet', help='quiet', action='store_true', default=False)
    args = argser.parse_args()
    if args.dir == '':
        error('target directory must not be empty')
        os.exit(1)
    graph(args.dir, pretty=args.pretty, verbose=args.verbose, quiet=args.quiet)
<|endoftext|>