<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Kenneth Reitz's GitHub Syncer

This script uses the GitHub API to get a list of all forked, mirrored, public, and
private repos in your GitHub account. If the repo already exists locally, it will
update it via git-pull. Otherwise, it will properly clone the repo.

It will organize your repos into the following directory structure:

+ repos
  ├── forks    (public fork repos)
  ├── mirrors  (public mirror repos)
  ├── private  (private repos)
  ├── public   (public repos)
  └── watched  (public watched repos)

Requires Ask Solem's github2 (http://pypi.python.org/pypi/github2).

Inspired by Gisty (http://github.com/swdyh/gisty).
"""

import os
import sys
from commands import getoutput as cmd

from github2.client import Github

__author__ = 'Kenneth Reitz'
__license__ = 'ISC'
__copyright__ = '2011 Kenneth REitz'
__version__ = '0.2'

# GitHub configurations
GITHUB_USER = cmd('git config github.user')
GITHUB_TOKEN = cmd('git config github.token')

GHSYNC_DIR = os.environ.get('GHSYNC_DIR', '.')

def run():

    os.chdir(GHSYNC_DIR)

    # API Object
    github = Github(username=GITHUB_USER, api_token=GITHUB_TOKEN)

    # repo slots
    repos = {}

    repos['watched'] = [r for r in github.repos.watching(GITHUB_USER)]
    repos['private'] = []
    repos['mirrors'] = []
    repos['public'] = []
    repos['forks'] = []

    # Collect GitHub repos via API
    for repo in github.repos.list():

        if repo.private:
            repos['private'].append(repo)
        elif repo.fork:
            repos['forks'].append(repo)
        elif 'mirror' in repo.description.lower():
            # mirrors owned by self if mirror in description...
            repos['mirrors'].append(repo)
        else:
            repos['public'].append(repo)

    for org, repos in repos.iteritems():
        for repo in repos:

            # create org directory (safely)
            try:
                os.makedirs(org)
            except OSError:
                pass

            # enter org dir
            os.chdir(org)

            # I own the repo
            private = True if org in ('private', 'fork', 'mirror') else False

            # just `git pull` if it's already there
            if os.path.exists(repo.name):
                os.chdir(repo.name)
                print('Updating repo: %s' % (repo.name))
                os.system('git pull')
                os.chdir('..')
            else:
                if private:
                    print('Cloning private repo: %s' % (repo.name))
                    os.system('git clone git@github.com:%s/%s.git' % (repo.owner, repo.name))
                else:
                    print('Cloning repo: %s' % (repo.name))
                    os.system('git clone git://github.com/%s/%s.git' % (repo.owner, repo.name))

            # return to base
            os.chdir('..')
            print

if __name__ == '__main__':
    run()
<commit_msg>version bump<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Kenneth Reitz's GitHub Syncer

This script uses the GitHub API to get a list of all forked, mirrored, public, and
private repos in your GitHub account. If the repo already exists locally, it will
update it via git-pull. Otherwise, it will properly clone the repo.

It will organize your repos into the following directory structure:

+ repos
  ├── forks    (public fork repos)
  ├── mirrors  (public mirror repos)
  ├── private  (private repos)
  ├── public   (public repos)
  └── watched  (public watched repos)

Requires Ask Solem's github2 (http://pypi.python.org/pypi/github2).

Inspired by Gisty (http://github.com/swdyh/gisty).
"""

import os
import sys
from commands import getoutput as cmd

from github2.client import Github

__author__ = 'Kenneth Reitz'
__license__ = 'ISC'
__copyright__ = '2011 Kenneth REitz'
__version__ = '0.2.1'

# GitHub configurations
GITHUB_USER = cmd('git config github.user')
GITHUB_TOKEN = cmd('git config github.token')

GHSYNC_DIR = os.environ.get('GHSYNC_DIR', '.')

def run():

    os.chdir(GHSYNC_DIR)

    # API Object
    github = Github(username=GITHUB_USER, api_token=GITHUB_TOKEN)

    # repo slots
    repos = {}

    repos['watched'] = [r for r in github.repos.watching(GITHUB_USER)]
    repos['private'] = []
    repos['mirrors'] = []
    repos['public'] = []
    repos['forks'] = []

    # Collect GitHub repos via API
    for repo in github.repos.list():

        if repo.private:
            repos['private'].append(repo)
        elif repo.fork:
            repos['forks'].append(repo)
        elif 'mirror' in repo.description.lower():
            # mirrors owned by self if mirror in description...
            repos['mirrors'].append(repo)
        else:
            repos['public'].append(repo)

    for org, repos in repos.iteritems():
        for repo in repos:

            # create org directory (safely)
            try:
                os.makedirs(org)
            except OSError:
                pass

            # enter org dir
            os.chdir(org)

            # I own the repo
            private = True if org in ('private', 'fork', 'mirror') else False

            # just `git pull` if it's already there
            if os.path.exists(repo.name):
                os.chdir(repo.name)
                print('Updating repo: %s' % (repo.name))
                os.system('git pull')
                os.chdir('..')
            else:
                if private:
                    print('Cloning private repo: %s' % (repo.name))
                    os.system('git clone git@github.com:%s/%s.git' % (repo.owner, repo.name))
                else:
                    print('Cloning repo: %s' % (repo.name))
                    os.system('git clone git://github.com/%s/%s.git' % (repo.owner, repo.name))

            # return to base
            os.chdir('..')
            print

if __name__ == '__main__':
    run()
<|endoftext|>


<commit_before>#!/usr/bin/env python

###
# Copyright (c) 2002, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

"""
Provides several commands that go out to websites and get things.

Commands include:
  freshmeat
  stockquote
  foldoc
  gkstats
  zipcode
  weather
  slashdot
"""

from baseplugin import *

import re
import urllib
import urllib2

import utils
import debug
import privmsgs
import callbacks

class FreshmeatException(Exception):
    pass

class Http(callbacks.Privmsg):
    threaded = True
    _fmProject = re.compile('<projectname_full>([^<]+)</projectname_full>')
    _fmVersion = re.compile('<latest_version>([^<]+)</latest_version>')
    _fmVitality = re.compile('<vitality_percent>([^<]+)</vitality_percent>')
    _fmPopular=re.compile('<popularity_percent>([^<]+)</popularity_percent>')
    _fmLastUpdated = re.compile('<date_updated>([^<]+)</date_updated>')
    def freshmeat(self, irc, msg, args):
        """<project name>

        Returns Freshmeat data about a given project.
        """
        project = privmsgs.getArgs(args)
        url = 'http://www.freshmeat.net/projects-xml/%s' % project
        try:
            fd = urllib2.urlopen(url)
            text = fd.read()
            fd.close()
            if text.startswith('Error'):
                raise FreshmeatException, text
            project = self._fmProject.search(text).group(1)
            version = self._fmVersion.search(text).group(1)
            vitality = self._fmVitality.search(text).group(1)
            popularity = self._fmPopular.search(text).group(1)
            lastupdated = self._fmLastUpdated.search(text).group(1)
            irc.reply(msg,
              '%s, last updated %s, with a vitality percent of %s '\
              'and a popularity of %s, is in version %s.' % \
              (project, lastupdated, vitality, popularity, version))
        except FreshmeatException, e:
            irc.reply(msg, debug.exnToString(e))
        except Exception, e:
            debug.recoverableException()
            irc.reply(msg, debug.exnToString(e))

    def stockquote(self, irc, msg, args):
        """<company symbol>

        Gets the information about the current price and change from the
        previous day of a given compny (represented by a stock symbol).
        """
        symbol = privmsgs.getArgs(args)
        url = 'http://finance.yahoo.com/d/quotes.csv?s=%s'\
              '&f=sl1d1t1c1ohgv&e=.csv' % symbol
        try:
            fd = urllib2.urlopen(url)
            quote = fd.read()
            fd.close()
        except Exception, e:
            irc.reply(msg, debug.exnToString(e))
            return
        data = quote.split(',')
        #debug.printf(data) # debugging
        if data[1] != '0.00':
            irc.reply(msg,
                       'The current price of %s is %s, as of %s EST.  '\
                       'A change of %s from the last business day.' %\
                       (data[0][1:-1], data[1], data[3][1:-1], data[4]))
            return
        else:
            m = 'I couldn\'t find a listing for %s' % symbol
            irc.error(msg, m)
            return

    def foldoc(self, irc, msg, args):
        """<something to lookup on foldoc>

        FOLDOC is a searchable dictionary of acryonyms, jargon, programming
        languages, tools, architecture, operating systems, networking, theory,
        conventions, standards, methamatics, telecoms, electronics, history,
        in fact anything having to do with computing.  This commands searches
        that dictionary.
        """
        if not args:
            raise callbacks.ArgumentError
        search = '+'.join([urllib.quote(arg) for arg in args])
        url = 'http://foldoc.doc.ic.ac.uk/foldoc/foldoc.cgi?query=%s' % search
        try:
            fd = urllib2.urlopen(url)
            html = fd.read()
            fd.close()
        except Exception, e:
            irc.error(msg, debug.exnToString(e))
        text = html.split('
\n', 2)[1]
        text = text.replace('.\n', '.  ')
        text = text.replace('\n', ' ')
        text = utils.htmlToText(text)
        irc.reply(msg, text.strip())

    _gkrating = re.compile(r'(\d+)')
    _gkgames = re.compile(r's:  	(\d+)
')
    _gkrecord = re.compile(r'"#FFFF00">(\d+)[^"]+"#FFFF00">(\d+)[^"]+'\
        '"#FFFF00">(\d+)')
    _gkteam = re.compile(r'Team:(<.*?>)+(?P<name>.*?)')
    _gkseen = re.compile(r'(seen on GK:\s+([^[]+)\s+|.*?is hiding.*?)')
    def gkstats(self, irc, msg, args):
        """<name>

        Returns the stats Gameknot keeps on <name>.  Gameknot is an online
        website for playing chess (rather similar to correspondence chess, just
        somewhat faster) against players from all over the world.
        """
        name = privmsgs.getArgs(args)
        gkprofile = 'http://www.gameknot.com/stats.pl?%s' % name
        try:
            fd = urllib2.urlopen(gkprofile)
            profile = fd.read()
            fd.close()
            rating = self._gkrating.search(profile).group(1)
            games = self._gkgames.search(profile).group(1)
            (w, l, d) = self._gkrecord.search(profile).groups()
            seen = self._gkseen.search(utils.htmlToText(profile))
            if seen.group(0).find("is hiding") != -1:
                seen = '%s is hiding his/her online status.' % name
            elif seen.group(2).startswith('0'):
                seen = '%s is on gameknot right now.' % name
            else:
                seen = '%s was last seen on Gameknot %s.' % (name,
                seen.group(2))
            if profile.find('Team:') >= 0:
                team = self._gkteam.search(profile).group('name')
                irc.reply(msg, '%s (team: %s) is rated %s and has %s active ' \
                          'games and a record of W-%s, L-%s, D-%s.  %s' % \
                          (name, team, rating, games, w, l, d, seen))
            else:
                irc.reply(msg, '%s is rated %s and has %s active games ' \
                          'and a record of W-%s, L-%s, D-%s.  %s' % \
                          (name, rating, games, w, l, d, seen))
        except AttributeError:
            if profile.find('User %s not found!' % name) != -1:
                irc.error(msg, 'No user %s exists.')
            else:
                irc.error(msg, 'The format of the page was odd.')
        except urllib2.URLError:
            irc.error(msg, 'Couldn\'t connect to gameknot.com.')

    _zipcode = re.compile(r'Local Forecast for (.*), (.*?) ')
    def zipcode(self, irc, msg, args):
        """<US zip code>

        Returns the city and state of a given US Zip code.
        """
        zip = privmsgs.getArgs(args)
        url = "http://www.weather.com/weather/local/%s?lswe=%s" % (zip, zip)
        try:
            html = urllib2.urlopen(url).read()
            (city, state) = self._zipcode.search(html).groups()
            irc.reply(msg, '%s, %s' % (city, state))
        except AttributeError:
            irc.error(msg, 'the format of the page was odd.')
        except urllib2.URLError:
            irc.error(msg, 'Couldn\'t open search page.')

    _tempregex = re.compile('CLASS=obsTempTextA>(\d+)°F	',\
                         re.IGNORECASE)
    _cityregex = re.compile(r'Local Forecast for (.*), (.*?) ')
    _condregex = re.compile('CLASS=obsInfo2>(.*)	',\
                         re.IGNORECASE)
    def weather(self, irc, msg, args):
        """<US zip code>

        Returns the approximate weather conditions at a given US Zip code.
        """
        zip = privmsgs.getArgs(args)
        url = "http://www.weather.com/weather/local/%s?lswe=%s" % (zip, zip)
        try:
            html = urllib2.urlopen(url).read()
            city, state = self._cityregex.search(html).groups()
            temp = self._tempregex.search(html).group(1)
            conds = self._condregex.search(html).group(1)
            irc.reply(msg, 'The current temperature in %s, %s is %dF with %s'\
                           ' conditions' % (city, state, int(temp), conds))
        except AttributeError:
            irc.error(msg, 'the format of the page was odd.')
        except urllib2.URLError:
            irc.error(msg, 'Couldn\'t open the search page.')

    _geekquotere = re.compile('
(.*?)
')
    def geekquote(self, irc, msg, args):
        """[<multiline>]

        Returns a random geek quote from bash.org; the optional argument
        <multiline> specifies whether multi-line quotes (which are longer
        than other quotes, generally) are to be allowed.
        """
        multiline = privmsgs.getArgs(args, needed=0, optional=1)
        try:
            fd = urllib2.urlopen('http://bash.org/?random1')
        except urllib2.URLError:
            irc.error(msg, 'Error connecting to geekquote server.')
            return
        html = fd.read()
        fd.close()
        if multiline:
            m = self._geekquotere.search(html, re.M)
        else:
            m = self._geekquotere.search(html)
        if m is None:
            irc.error(msg, 'No quote found.')
            return
        quote = utils.htmlToText(m.group(1))
        quote = ' // '.join(quote.splitlines())
        irc.reply(msg, quote)

    _acronymre = re.compile(r']+>(?:)?([^<]+)(?:)?')
    def acronym(self, irc, msg, args):
        """<acronym>

        Displays the first 5 acronym matches from acronymfinder.com
        """
        acronym = privmsgs.getArgs(args)
        try:
            url = 'http://www.acronymfinder.com/' \
                  'af-query.asp?String=exact&Acronym=%s' % acronym
            request = urllib2.Request(url, headers={'User-agent':
              'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0)'})
            fd = urllib2.urlopen(request)
        except urllib2.URLError:
            irc.error(msg, 'Couldn\'t connect to acronymfinder.com')
            return
        html = fd.read()
        fd.close()
        defs = self._acronymre.findall(html)
        if len(defs) == 0:
            irc.reply(msg, 'No definitions found.')
        else:
            defs=[repr(x.strip()) for x in defs[1:-1]][:5]
            irc.reply(msg, '%s could be %s' % (acronym, ', or '.join(defs)))

    _netcraftre = re.compile(r'whatos text -->(.*?)<a href="/up/acc', re.S)
    def netcraft(self, irc, msg, args):
        """<hostname|ip>

        Returns Netcraft.com's determination of what operating system and
        webserver is running on the host given.
        """
        hostname = privmsgs.getArgs(args)
        url = 'http://uptime.netcraft.com/up/graph/?host=%s' % hostname
        fd = urllib2.urlopen(url)
        html = fd.read()
        fd.close()
        m = self._netcraftre.search(html)
        if m:
            html = m.group(1)
            s = utils.htmlToText(html, tagReplace='').strip('\xa0 ')
            irc.reply(msg, s[9:]) # Snip off "the site"
        elif html.find('We could not get any results') != -1:
            irc.reply(msg, 'No results found for %s.' % hostname)
        else:
            irc.error(msg, 'The format of the was odd.')

Class = Http

# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:
<commit_msg>Added title command; fixed a latent bug or two.<commit_after>#!/usr/bin/env python

###
# Copyright (c) 2002, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

"""
Provides several commands that go out to websites and get things.

Commands include:
  freshmeat
  stockquote
  foldoc
  gkstats
  zipcode
  weather
  slashdot
"""

from baseplugin import *

import re
import urllib
import urllib2

import utils
import debug
import privmsgs
import callbacks

class FreshmeatException(Exception):
    pass

class Http(callbacks.Privmsg):
    threaded = True

    _titleRe = re.compile(r'<title>(.*)</title>')
    def title(self, irc, msg, args):
        """<url>

        Returns the HTML <title>...</title> of a URL.
        """
        url = privmsgs.getArgs(args)
        try:
            fd = urllib2.urlopen(url)
            text = fd.read()
            m = self._titleRe.search(text)
            if m is not None:
                irc.reply(msg, m.group(1))
            else:
                irc.reply(msg, 'That URL appears to have no HTML title.')
        except Exception, e:
            irc.error(msg, debug.exnToString(e))

    _fmProject = re.compile('<projectname_full>([^<]+)</projectname_full>')
    _fmVersion = re.compile('<latest_version>([^<]+)</latest_version>')
    _fmVitality = re.compile('<vitality_percent>([^<]+)</vitality_percent>')
    _fmPopular=re.compile('<popularity_percent>([^<]+)</popularity_percent>')
    _fmLastUpdated = re.compile('<date_updated>([^<]+)</date_updated>')
    def freshmeat(self, irc, msg, args):
        """<project name>

        Returns Freshmeat data about a given project.
        """
        project = privmsgs.getArgs(args)
        url = 'http://www.freshmeat.net/projects-xml/%s' % project
        try:
            fd = urllib2.urlopen(url)
            text = fd.read()
            fd.close()
            if text.startswith('Error'):
                raise FreshmeatException, text
            project = self._fmProject.search(text).group(1)
            version = self._fmVersion.search(text).group(1)
            vitality = self._fmVitality.search(text).group(1)
            popularity = self._fmPopular.search(text).group(1)
            lastupdated = self._fmLastUpdated.search(text).group(1)
            irc.reply(msg,
              '%s, last updated %s, with a vitality percent of %s '\
              'and a popularity of %s, is in version %s.' % \
              (project, lastupdated, vitality, popularity, version))
        except FreshmeatException, e:
            irc.error(msg, debug.exnToString(e))
        except Exception, e:
            debug.recoverableException()
            irc.error(msg, debug.exnToString(e))

    def stockquote(self, irc, msg, args):
        """<company symbol>

        Gets the information about the current price and change from the
        previous day of a given compny (represented by a stock symbol).
        """
        symbol = privmsgs.getArgs(args)
        url = 'http://finance.yahoo.com/d/quotes.csv?s=%s'\
              '&f=sl1d1t1c1ohgv&e=.csv' % symbol
        try:
            fd = urllib2.urlopen(url)
            quote = fd.read()
            fd.close()
        except Exception, e:
            irc.error(msg, debug.exnToString(e))
            return
        data = quote.split(',')
        #debug.printf(data) # debugging
        if data[1] != '0.00':
            irc.reply(msg,
                       'The current price of %s is %s, as of %s EST.  '\
                       'A change of %s from the last business day.' %\
                       (data[0][1:-1], data[1], data[3][1:-1], data[4]))
            return
        else:
            m = 'I couldn\'t find a listing for %s' % symbol
            irc.error(msg, m)
            return

    def foldoc(self, irc, msg, args):
        """<something to lookup on foldoc>

        FOLDOC is a searchable dictionary of acryonyms, jargon, programming
        languages, tools, architecture, operating systems, networking, theory,
        conventions, standards, methamatics, telecoms, electronics, history,
        in fact anything having to do with computing.  This commands searches
        that dictionary.
        """
        if not args:
            raise callbacks.ArgumentError
        search = '+'.join([urllib.quote(arg) for arg in args])
        url = 'http://foldoc.doc.ic.ac.uk/foldoc/foldoc.cgi?query=%s' % search
        try:
            fd = urllib2.urlopen(url)
            html = fd.read()
            fd.close()
        except Exception, e:
            irc.error(msg, debug.exnToString(e))
            return
        text = html.split('
\n', 2)[1]
        text = text.replace('.\n', '.  ')
        text = text.replace('\n', ' ')
        text = utils.htmlToText(text)
        irc.reply(msg, text.strip())

    _gkrating = re.compile(r'(\d+)')
    _gkgames = re.compile(r's:  	(\d+)
')
    _gkrecord = re.compile(r'"#FFFF00">(\d+)[^"]+"#FFFF00">(\d+)[^"]+'\
        '"#FFFF00">(\d+)')
    _gkteam = re.compile(r'Team:(<.*?>)+(?P<name>.*?)')
    _gkseen = re.compile(r'(seen on GK:\s+([^[]+)\s+|.*?is hiding.*?)')
    def gkstats(self, irc, msg, args):
        """<name>

        Returns the stats Gameknot keeps on <name>.  Gameknot is an online
        website for playing chess (rather similar to correspondence chess, just
        somewhat faster) against players from all over the world.
        """
        name = privmsgs.getArgs(args)
        gkprofile = 'http://www.gameknot.com/stats.pl?%s' % name
        try:
            fd = urllib2.urlopen(gkprofile)
            profile = fd.read()
            fd.close()
            rating = self._gkrating.search(profile).group(1)
            games = self._gkgames.search(profile).group(1)
            (w, l, d) = self._gkrecord.search(profile).groups()
            seen = self._gkseen.search(utils.htmlToText(profile))
            if seen.group(0).find("is hiding") != -1:
                seen = '%s is hiding his/her online status.' % name
            elif seen.group(2).startswith('0'):
                seen = '%s is on gameknot right now.' % name
            else:
                seen = '%s was last seen on Gameknot %s.' % (name,
                seen.group(2))
            if profile.find('Team:') >= 0:
                team = self._gkteam.search(profile).group('name')
                irc.reply(msg, '%s (team: %s) is rated %s and has %s active ' \
                          'games and a record of W-%s, L-%s, D-%s.  %s' % \
                          (name, team, rating, games, w, l, d, seen))
            else:
                irc.reply(msg, '%s is rated %s and has %s active games ' \
                          'and a record of W-%s, L-%s, D-%s.  %s' % \
                          (name, rating, games, w, l, d, seen))
        except AttributeError:
            if profile.find('User %s not found!' % name) != -1:
                irc.error(msg, 'No user %s exists.')
            else:
                irc.error(msg, 'The format of the page was odd.')
        except urllib2.URLError:
            irc.error(msg, 'Couldn\'t connect to gameknot.com.')

    _zipcode = re.compile(r'Local Forecast for (.*), (.*?) ')
    def zipcode(self, irc, msg, args):
        """<US zip code>

        Returns the city and state of a given US Zip code.
        """
        zip = privmsgs.getArgs(args)
        url = "http://www.weather.com/weather/local/%s?lswe=%s" % (zip, zip)
        try:
            html = urllib2.urlopen(url).read()
            (city, state) = self._zipcode.search(html).groups()
            irc.reply(msg, '%s, %s' % (city, state))
        except AttributeError:
            irc.error(msg, 'the format of the page was odd.')
        except urllib2.URLError:
            irc.error(msg, 'Couldn\'t open search page.')

    _tempregex = re.compile('CLASS=obsTempTextA>(\d+)°F	',\
                         re.IGNORECASE)
    _cityregex = re.compile(r'Local Forecast for (.*), (.*?) ')
    _condregex = re.compile('CLASS=obsInfo2>(.*)	',\
                         re.IGNORECASE)
    def weather(self, irc, msg, args):
        """<US zip code>

        Returns the approximate weather conditions at a given US Zip code.
        """
        zip = privmsgs.getArgs(args)
        url = "http://www.weather.com/weather/local/%s?lswe=%s" % (zip, zip)
        try:
            html = urllib2.urlopen(url).read()
            city, state = self._cityregex.search(html).groups()
            temp = self._tempregex.search(html).group(1)
            conds = self._condregex.search(html).group(1)
            irc.reply(msg, 'The current temperature in %s, %s is %dF with %s'\
                           ' conditions' % (city, state, int(temp), conds))
        except AttributeError:
            irc.error(msg, 'the format of the page was odd.')
        except urllib2.URLError:
            irc.error(msg, 'Couldn\'t open the search page.')

    _geekquotere = re.compile('
(.*?)
')
    def geekquote(self, irc, msg, args):
        """[<multiline>]

        Returns a random geek quote from bash.org; the optional argument
        <multiline> specifies whether multi-line quotes (which are longer
        than other quotes, generally) are to be allowed.
        """
        multiline = privmsgs.getArgs(args, needed=0, optional=1)
        try:
            fd = urllib2.urlopen('http://bash.org/?random1')
        except urllib2.URLError:
            irc.error(msg, 'Error connecting to geekquote server.')
            return
        html = fd.read()
        fd.close()
        if multiline:
            m = self._geekquotere.search(html, re.M)
        else:
            m = self._geekquotere.search(html)
        if m is None:
            irc.error(msg, 'No quote found.')
            return
        quote = utils.htmlToText(m.group(1))
        quote = ' // '.join(quote.splitlines())
        irc.reply(msg, quote)

    _acronymre = re.compile(r']+>(?:)?([^<]+)(?:)?')
    def acronym(self, irc, msg, args):
        """<acronym>

        Displays the first 5 acronym matches from acronymfinder.com
        """
        acronym = privmsgs.getArgs(args)
        try:
            url = 'http://www.acronymfinder.com/' \
                  'af-query.asp?String=exact&Acronym=%s' % acronym
            request = urllib2.Request(url, headers={'User-agent':
              'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0)'})
            fd = urllib2.urlopen(request)
        except urllib2.URLError:
            irc.error(msg, 'Couldn\'t connect to acronymfinder.com')
            return
        html = fd.read()
        fd.close()
        defs = self._acronymre.findall(html)
        if len(defs) == 0:
            irc.reply(msg, 'No definitions found.')
        else:
            defs=[repr(x.strip()) for x in defs[1:-1]][:5]
            irc.reply(msg, '%s could be %s' % (acronym, ', or '.join(defs)))

    _netcraftre = re.compile(r'whatos text -->(.*?)<a href="/up/acc', re.S)
    def netcraft(self, irc, msg, args):
        """<hostname|ip>

        Returns Netcraft.com's determination of what operating system and
        webserver is running on the host given.
        """
        hostname = privmsgs.getArgs(args)
        url = 'http://uptime.netcraft.com/up/graph/?host=%s' % hostname
        fd = urllib2.urlopen(url)
        html = fd.read()
        fd.close()
        m = self._netcraftre.search(html)
        if m:
            html = m.group(1)
            s = utils.htmlToText(html, tagReplace='').strip('\xa0 ')
            irc.reply(msg, s[9:]) # Snip off "the site"
        elif html.find('We could not get any results') != -1:
            irc.reply(msg, 'No results found for %s.' % hostname)
        else:
            irc.error(msg, 'The format of the was odd.')

Class = Http

# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:
<|endoftext|>


<commit_before># lamosity needed to make things reliable on Windows :-(
# (borrowed from Python's test_support.py)
import os, sys, shutil

if sys.platform.startswith("win"):
    def _waitfor(func, pathname, waitall=False):
        # Perform the operation
        func(pathname)
        # Now setup the wait loop
        if waitall:
            dirname = pathname
        else:
            dirname, name = os.path.split(pathname)
            dirname = dirname or '.'
        # Check for `pathname` to be removed from the filesystem.
        # The exponential backoff of the timeout amounts to a total
        # of ~1 second after which the deletion is probably an error
        # anyway.
        # Testing on a i7@4.3GHz shows that usually only 1 iteration is
        # required when contention occurs.
        timeout = 0.001
        while timeout < 1.0: # pragma: no branch
            # Note we are only testing for the existence of the file(s) in
            # the contents of the directory regardless of any security or
            # access rights.  If we have made it this far, we have sufficient
            # permissions to do that much using Python's equivalent of the
            # Windows API FindFirstFile.
            # Other Windows APIs can fail or give incorrect results when
            # dealing with files that are pending deletion.
            L = os.listdir(dirname)
            if not (L if waitall else name in L): # pragma: no branch
                return
            # Increase the timeout and try again
            time.sleep(timeout) # pragma: no cover
            timeout *= 2 # pragma: no cover
        warnings.warn('tests may fail, delete still pending for ' + pathname, # pragma: no cover
                      RuntimeWarning, stacklevel=4)

    def _rmtree(path):
        def _rmtree_inner(path):
            for name in os.listdir(path):
                fullname = os.path.join(path, name)
                if os.path.isdir(fullname):
                    _waitfor(_rmtree_inner, fullname, waitall=True)
                    os.rmdir(fullname)
                else:
                    os.unlink(fullname)
        _waitfor(_rmtree_inner, path, waitall=True)
        _waitfor(os.rmdir, path)
else:
    _rmtree = shutil.rmtree

def rmtree(path):
    try:
        _rmtree(path)
    except OSError as e: # pragma: no cover
        # Unix returns ENOENT, Windows returns ESRCH.
        if e.errno not in (errno.ENOENT, errno.ESRCH): # pragma: no branch
            raise
<commit_msg>fix missing imports, thanks to Matthias Lehmann for pointing these out.<commit_after># lamosity needed to make things reliable on Windows :-(
# (borrowed from Python's test_support.py)
import errno, os, sys, shutil, time, warnings

if sys.platform.startswith("win"):
    def _waitfor(func, pathname, waitall=False):
        # Perform the operation
        func(pathname)
        # Now setup the wait loop
        if waitall:
            dirname = pathname
        else:
            dirname, name = os.path.split(pathname)
            dirname = dirname or '.'
        # Check for `pathname` to be removed from the filesystem.
        # The exponential backoff of the timeout amounts to a total
        # of ~1 second after which the deletion is probably an error
        # anyway.
        # Testing on a i7@4.3GHz shows that usually only 1 iteration is
        # required when contention occurs.
        timeout = 0.001
        while timeout < 1.0: # pragma: no branch
            # Note we are only testing for the existence of the file(s) in
            # the contents of the directory regardless of any security or
            # access rights.  If we have made it this far, we have sufficient
            # permissions to do that much using Python's equivalent of the
            # Windows API FindFirstFile.
            # Other Windows APIs can fail or give incorrect results when
            # dealing with files that are pending deletion.
            L = os.listdir(dirname)
            if not (L if waitall else name in L): # pragma: no branch
                return
            # Increase the timeout and try again
            time.sleep(timeout) # pragma: no cover
            timeout *= 2 # pragma: no cover
        warnings.warn('tests may fail, delete still pending for ' + pathname, # pragma: no cover
                      RuntimeWarning, stacklevel=4)

    def _rmtree(path):
        def _rmtree_inner(path):
            for name in os.listdir(path):
                fullname = os.path.join(path, name)
                if os.path.isdir(fullname):
                    _waitfor(_rmtree_inner, fullname, waitall=True)
                    os.rmdir(fullname)
                else:
                    os.unlink(fullname)
        _waitfor(_rmtree_inner, path, waitall=True)
        _waitfor(os.rmdir, path)
else:
    _rmtree = shutil.rmtree

def rmtree(path):
    try:
        _rmtree(path)
    except OSError as e: # pragma: no cover
        # Unix returns ENOENT, Windows returns ESRCH.
        if e.errno not in (errno.ENOENT, errno.ESRCH): # pragma: no branch
            raise
<|endoftext|>


<commit_before>from nalaf.learning.taggers import RelationExtractor
from nalaf.learning.taggers import StubSameSentenceRelationExtractor
from nalaf.learning.svmlight import SVMLightTreeKernels
from nalaf.structures.relation_pipelines import RelationExtractionPipeline
from loctext.features.specific import LocationWordFeatureGenerator
from loctext.features.specific import ProteinWordFeatureGenerator
from nalaf.features.relations import TokenFeatureGenerator
from nalaf.features.relations.context import LinearDistanceFeatureGenerator
from nalaf.features.relations.context import EntityOrderFeatureGenerator
from nalaf.features.relations.context import IntermediateTokensFeatureGenerator
from nalaf.features.relations.path import PathFeatureGenerator
from nalaf.features.relations.sentence import NamedEntityCountFeatureGenerator, BagOfWordsFeatureGenerator, StemmedBagOfWordsFeatureGenerator
from nalaf.features.relations.entityhead import EntityHeadTokenUpperCaseFeatureGenerator, EntityHeadTokenDigitsFeatureGenerator, EntityHeadTokenPunctuationFeatureGenerator
from nalaf.preprocessing.edges import SimpleEdgeGenerator, SentenceDistanceEdgeGenerator
from nalaf import print_verbose, print_debug

class LocTextSSmodelRelationExtractor(RelationExtractor):

    def __init__(
            self,
            entity1_class,
            entity2_class,
            rel_type,
            feature_generators=None,
            pipeline=None,
            execute_pipeline=True,
            svmlight=None,
            **svmlight_params):

        super().__init__(entity1_class, entity2_class, rel_type)

        if pipeline:
            feature_generators = pipeline.feature_generators
        elif feature_generators is not None:  # Trick: if [], this will use pipeline's default generators
            feature_generators = feature_generators
        else:
            feature_generators = self.feature_generators()

        edge_generator = SentenceDistanceEdgeGenerator(entity1_class, entity2_class, rel_type, distance=0)
        self.pipeline = pipeline if pipeline else RelationExtractionPipeline(entity1_class, entity2_class, rel_type, edge_generator=edge_generator, feature_generators=feature_generators)

        assert feature_generators == self.pipeline.feature_generators or feature_generators == [], str((feature_generators, self.pipeline.feature_generators))

        self.execute_pipeline = execute_pipeline

        # TODO this would require setting the default model_path
        self.svmlight = svmlight if svmlight else SVMLightTreeKernels(**svmlight_params)

    def annotate(self, target_corpus):
        if self.execute_pipeline:
            self.pipeline.execute(target_corpus, train=False)

        instancesfile = self.svmlight.create_input_file(target_corpus, 'predict', self.pipeline.feature_set)
        predictionsfile = self.svmlight.classify(instancesfile)
        self.svmlight.read_predictions(target_corpus, predictionsfile)

        return target_corpus

    def feature_generators(self):
        return __class__.default_feature_generators(self.entity1_class, self.entity2_class)

    @staticmethod
    def default_feature_generators(prot_e_id, loc_e_id, graphs=None):

        GRAPHS_CLOSURE_VARIABLE = {} if graphs is None else graphs

        return [
            LocationWordFeatureGenerator(
                loc_e_id,
                prefix1=2
            ),

            ProteinWordFeatureGenerator(
                GRAPHS_CLOSURE_VARIABLE,
                prefix_PWPE_bow=7,
                prefix_PWPE_bow_masked=9,
                prefix_PWPE_dep=10,
                prefix_protein_word_found=13,
                prefix_protein_not_word_found=14
            ),

            IntermediateTokensFeatureGenerator(
                prefix_fwd_pos_intermediate=34,

                prefix_bkd_bow_intermediate=35,
                prefix_bkd_bow_intermediate_masked=36,
                prefix_bkd_stem_intermediate=37,
                prefix_bkd_pos_intermediate=38,

                prefix_bow_intermediate=39,
                prefix_bow_intermediate_masked=40,
                prefix_stem_intermediate=41,
                prefix_pos_intermediate=42,
            ),

            LinearDistanceFeatureGenerator(
                distance=5,
                prefix_entity_linear_distance_greater_than=43,
                prefix_entity_linear_distance_lesser_than=44,
                # prefix_entity_linear_distance=45
            ),

            EntityOrderFeatureGenerator(
                prefix_order_entity1_entity2=46,
                prefix_order_entity2_entity1=47,
            ),

            PathFeatureGenerator(
                GRAPHS_CLOSURE_VARIABLE,

                token_feature_generator=TokenFeatureGenerator(
                    # prefix_txt=78,  # 73 in relna
                    prefix_pos=79,  # 74
                    prefix_masked_txt=77,  # 75
                    prefix_stem_masked_txt=81,  # 76
                    prefix_ann_type=80,  # 77
                ),

                prefix_45_len_tokens=73,
                prefix_46_len=None,  # None
                prefix_47_word_in_path=None,  # None
                prefix_48_dep_forward=65,
                prefix_49_dep_reverse=66,
                prefix_50_internal_pos=67,
                prefix_51_internal_masked_txt=68,
                prefix_52_internal_txt=69,
                prefix_53_internal_stem=70,
                prefix_54_internal_dep_forward=71,
                prefix_55_internal_dep_reverse=72,
                prefix_56_token_path=64,
                prefix_57_dep_style_gram=60,
                prefix_58_edge_gram=None,  # None
                prefix_59_ann_edge_gram=None,  # None
                prefix_60_edge_directions=63,
                prefix_61_dep_1=49,
                prefix_62_masked_txt_dep_0=50,
                prefix_63_pos_dep_0=51,
                prefix_64_ann_type_1=52,
                prefix_65_dep_to_1=None,
                prefix_66_masked_txt_dep_to_0=53,
                prefix_67_pos_to=54,
                prefix_68_ann_type_2=55,
                prefix_69_gov_g_text=56,
                prefix_70_gov_g_pos=57,
                prefix_71_gov_anns=58,
                prefix_72_triple=59,
            ),

            EntityHeadTokenUpperCaseFeatureGenerator(
                prefix_entity1_upper_case_middle=87.1,
                prefix_entity2_upper_case_middle=87.2,
            ),

            EntityHeadTokenDigitsFeatureGenerator(
                prefix_entity1_has_hyphenated_digits=89.1,
                prefix_entity2_has_hyphenated_digits=89.2,
            ),

            EntityHeadTokenPunctuationFeatureGenerator(
                prefix_entity1_has_hyphen=90.1,
                prefix_entity1_has_fslash=91.1,
                prefix_entity2_has_hyphen=90.2,
                prefix_entity2_has_fslash=91.2,
            ),

            BagOfWordsFeatureGenerator(
                prefix_bow_text=2,
                prefix_ne_bow_count=3,
            ),

            StemmedBagOfWordsFeatureGenerator(
                prefix_bow_stem=4
            ),

            NamedEntityCountFeatureGenerator(
                prot_e_id,
                prefix=107
            ),

            NamedEntityCountFeatureGenerator(
                loc_e_id,
                prefix=108
            )
        ]

class LocTextDSmodelRelationExtractor(RelationExtractor):

    def __init__(
            self,
            entity1_class,
            entity2_class,
            rel_type,
            feature_generators=None,
            pipeline=None,
            execute_pipeline=True,
            svmlight=None,
            **svmlight_params):

        super().__init__(entity1_class, entity2_class, rel_type)

        if pipeline:
            feature_generators = pipeline.feature_generators
        elif feature_generators is not None:  # Trick: if [], this will use pipeline's default generators
            feature_generators = feature_generators
        else:
            feature_generators = self.feature_generators()

        edge_generator = SentenceDistanceEdgeGenerator(entity1_class, entity2_class, rel_type, distance=1)
        self.pipeline = pipeline if pipeline else RelationExtractionPipeline(entity1_class, entity2_class, rel_type, edge_generator=edge_generator, feature_generators=feature_generators)

        assert feature_generators == self.pipeline.feature_generators or feature_generators == [], str((feature_generators, self.pipeline.feature_generators))

        self.execute_pipeline = execute_pipeline

        # TODO this would require setting the default model_path
        self.svmlight = svmlight if svmlight else SVMLightTreeKernels(**svmlight_params)

    def annotate(self, target_corpus):
        if self.execute_pipeline:
            self.pipeline.execute(target_corpus, train=False)

        instancesfile = self.svmlight.create_input_file(target_corpus, 'predict', self.pipeline.feature_set)
        predictionsfile = self.svmlight.classify(instancesfile)
        self.svmlight.read_predictions(target_corpus, predictionsfile)

        return target_corpus

    def feature_generators(self):
        return __class__.default_feature_generators(self.entity1_class, self.entity2_class)

    @staticmethod
    def default_feature_generators(prot_e_id, loc_e_id, graphs=None):
        from loctext.features import DS

        GRAPHS_CLOSURE_VARIABLE = {} if graphs is None else graphs

        return [
            # Comment from Shrikant:
            # TODO ...Commenting follwing two function calls increases the overall Fscore by 0.3 points...

            # TODO and, should we have features for unigram too?

            # Trigram
            DS.AnyNGramFeatureGenerator(
                n_gram=3,
                #
                prefix_pos=3
            ),

            # Bigram
            DS.AnyNGramFeatureGenerator(
                n_gram=2,
                #
                prefix_bow=5,
                prefix_bow_masked=6,
                prefix_pos=7,
                prefix_stem=8
            ),

            DS.PatternFeatureGenerator(
                e1_class=prot_e_id,
                e2_class=loc_e_id,
                #
                prefix_ProtVerbWord=39,
                prefix_LocVerbWord=41,
                prefix_ProtVerbWordLocVerbWord=43,
                prefix_WordVerbProtLocVerbWord=45,
            ),

            DS.SameWordFeatureGenerator(
                prefix_sameWordsSamePOS=48,
                prefix_sameStemSamePOS=50,
            ),

            DS.LocEntityFeatureGenerator(
                localization_class_id=loc_e_id,
                prefix_localizationVerb=51
            ),

            DS.IndividualSentencesFeatureGenerator(
                prefix_sentence_1_POS=54,
                #
                prefix_sentence_2_POS=57
            ),

            NamedEntityCountFeatureGenerator(
                prot_e_id,
                prefix=130
            ),

            NamedEntityCountFeatureGenerator(
                loc_e_id,
                prefix=131
            ),
        ]

class LocTextCombinedModelRelationExtractor(RelationExtractor):

    def __init__(
            self,
            entity1_class,
            entity2_class,
            rel_type,
            ss_model,
            ds_model):

        super().__init__(entity1_class, entity2_class, rel_type)

        self.ss_model = ss_model
        self.ds_model = ds_model
        self.submodels = [self.ss_model, self.ds_model]

    def annotate(self, target_corpus):

        for model in self.submodels:
            model.annotate(target_corpus)

        return target_corpus
<commit_msg>remove<commit_after>from nalaf.learning.taggers import RelationExtractor
from nalaf.learning.taggers import StubSameSentenceRelationExtractor
from nalaf.learning.svmlight import SVMLightTreeKernels
from nalaf.structures.relation_pipelines import RelationExtractionPipeline
from loctext.features.specific import LocationWordFeatureGenerator
from loctext.features.specific import ProteinWordFeatureGenerator
from nalaf.features.relations import TokenFeatureGenerator
from nalaf.features.relations.context import LinearDistanceFeatureGenerator
from nalaf.features.relations.context import EntityOrderFeatureGenerator
from nalaf.features.relations.context import IntermediateTokensFeatureGenerator
from nalaf.features.relations.path import PathFeatureGenerator
from nalaf.features.relations.sentence import NamedEntityCountFeatureGenerator, BagOfWordsFeatureGenerator, StemmedBagOfWordsFeatureGenerator
from nalaf.features.relations.entityhead import EntityHeadTokenUpperCaseFeatureGenerator, EntityHeadTokenDigitsFeatureGenerator, EntityHeadTokenPunctuationFeatureGenerator
from nalaf.preprocessing.edges import SimpleEdgeGenerator, SentenceDistanceEdgeGenerator
from nalaf import print_verbose, print_debug

class LocTextSSmodelRelationExtractor(RelationExtractor):

    def __init__(
            self,
            entity1_class,
            entity2_class,
            rel_type,
            feature_generators=None,
            pipeline=None,
            execute_pipeline=True,
            svmlight=None,
            **svmlight_params):

        super().__init__(entity1_class, entity2_class, rel_type)

        if pipeline:
            feature_generators = pipeline.feature_generators
        elif feature_generators is not None:  # Trick: if [], this will use pipeline's default generators
            feature_generators = feature_generators
        else:
            feature_generators = self.feature_generators()

        edge_generator = SentenceDistanceEdgeGenerator(entity1_class, entity2_class, rel_type, distance=0)
        self.pipeline = pipeline if pipeline else RelationExtractionPipeline(entity1_class, entity2_class, rel_type, edge_generator=edge_generator, feature_generators=feature_generators)

        assert feature_generators == self.pipeline.feature_generators or feature_generators == [], str((feature_generators, self.pipeline.feature_generators))

        self.execute_pipeline = execute_pipeline

        # TODO this would require setting the default model_path
        self.svmlight = svmlight if svmlight else SVMLightTreeKernels(**svmlight_params)

    def annotate(self, target_corpus):
        if self.execute_pipeline:
            self.pipeline.execute(target_corpus, train=False)

        instancesfile = self.svmlight.create_input_file(target_corpus, 'predict', self.pipeline.feature_set)
        predictionsfile = self.svmlight.classify(instancesfile)
        self.svmlight.read_predictions(target_corpus, predictionsfile)

        return target_corpus

    def feature_generators(self):
        return __class__.default_feature_generators(self.entity1_class, self.entity2_class)

    @staticmethod
    def default_feature_generators(prot_e_id, loc_e_id, graphs=None):

        GRAPHS_CLOSURE_VARIABLE = {} if graphs is None else graphs

        return [
            LocationWordFeatureGenerator(
                loc_e_id,
                prefix1=2
            ),

            ProteinWordFeatureGenerator(
                GRAPHS_CLOSURE_VARIABLE,
                prefix_PWPE_bow=7,
                prefix_PWPE_bow_masked=9,
                prefix_PWPE_dep=10,
                prefix_protein_word_found=13,
                prefix_protein_not_word_found=14
            ),

            IntermediateTokensFeatureGenerator(
                prefix_fwd_pos_intermediate=34,

                prefix_bkd_bow_intermediate=35,
                prefix_bkd_bow_intermediate_masked=36,
                prefix_bkd_stem_intermediate=37,
                prefix_bkd_pos_intermediate=38,

                prefix_bow_intermediate=39,
                prefix_bow_intermediate_masked=40,
                prefix_stem_intermediate=41,
                prefix_pos_intermediate=42,
            ),

            LinearDistanceFeatureGenerator(
                distance=5,
                prefix_entity_linear_distance_greater_than=43,
                prefix_entity_linear_distance_lesser_than=44,
                # prefix_entity_linear_distance=45
            ),

            EntityOrderFeatureGenerator(
                prefix_order_entity1_entity2=46,
                prefix_order_entity2_entity1=47,
            ),

            PathFeatureGenerator(
                GRAPHS_CLOSURE_VARIABLE,

                token_feature_generator=TokenFeatureGenerator(
                    # prefix_txt=78,  # 73 in relna
                    prefix_pos=79,  # 74
                    prefix_masked_txt=77,  # 75
                    prefix_stem_masked_txt=81,  # 76
                    prefix_ann_type=80,  # 77
                ),

                prefix_45_len_tokens=73,
                prefix_46_len=None,  # None
                prefix_47_word_in_path=None,  # None
                prefix_48_dep_forward=65,
                prefix_49_dep_reverse=66,
                prefix_50_internal_pos=67,
                prefix_51_internal_masked_txt=68,
                prefix_52_internal_txt=69,
                prefix_53_internal_stem=70,
                prefix_54_internal_dep_forward=71,
                prefix_55_internal_dep_reverse=72,
                prefix_56_token_path=64,
                prefix_57_dep_style_gram=60,
                prefix_58_edge_gram=None,  # None
                prefix_59_ann_edge_gram=None,  # None
                prefix_60_edge_directions=63,
                prefix_61_dep_1=49,
                prefix_62_masked_txt_dep_0=50,
                prefix_63_pos_dep_0=51,
                prefix_64_ann_type_1=52,
                prefix_65_dep_to_1=None,
                prefix_66_masked_txt_dep_to_0=53,
                prefix_67_pos_to=54,
                prefix_68_ann_type_2=55,
                prefix_69_gov_g_text=56,
                prefix_70_gov_g_pos=57,
                prefix_71_gov_anns=58,
                prefix_72_triple=59,
            ),

            EntityHeadTokenUpperCaseFeatureGenerator(
                prefix_entity1_upper_case_middle=87.1,
                prefix_entity2_upper_case_middle=87.2,
            ),

            EntityHeadTokenDigitsFeatureGenerator(
                prefix_entity1_has_hyphenated_digits=89.1,
                prefix_entity2_has_hyphenated_digits=89.2,
            ),

            EntityHeadTokenPunctuationFeatureGenerator(
                prefix_entity1_has_hyphen=90.1,
                prefix_entity1_has_fslash=91.1,
                prefix_entity2_has_hyphen=90.2,
                prefix_entity2_has_fslash=91.2,
            ),

            BagOfWordsFeatureGenerator(
                prefix_bow_text=2,
                prefix_ne_bow_count=3,
            ),

            StemmedBagOfWordsFeatureGenerator(
                prefix_bow_stem=4
            ),

            NamedEntityCountFeatureGenerator(
                prot_e_id,
                prefix=107
            ),

            NamedEntityCountFeatureGenerator(
                loc_e_id,
                prefix=108
            )
        ]

class LocTextDSmodelRelationExtractor(RelationExtractor):

    def __init__(
            self,
            entity1_class,
            entity2_class,
            rel_type,
            feature_generators=None,
            pipeline=None,
            execute_pipeline=True,
            svmlight=None,
            **svmlight_params):

        super().__init__(entity1_class, entity2_class, rel_type)

        if pipeline:
            feature_generators = pipeline.feature_generators
        elif feature_generators is not None:  # Trick: if [], this will use pipeline's default generators
            feature_generators = feature_generators
        else:
            feature_generators = self.feature_generators()

        edge_generator = SentenceDistanceEdgeGenerator(entity1_class, entity2_class, rel_type, distance=1)
        self.pipeline = pipeline if pipeline else RelationExtractionPipeline(entity1_class, entity2_class, rel_type, edge_generator=edge_generator, feature_generators=feature_generators)

        assert feature_generators == self.pipeline.feature_generators or feature_generators == [], str((feature_generators, self.pipeline.feature_generators))

        self.execute_pipeline = execute_pipeline

        # TODO this would require setting the default model_path
        self.svmlight = svmlight if svmlight else SVMLightTreeKernels(**svmlight_params)

    def annotate(self, target_corpus):
        if self.execute_pipeline:
            self.pipeline.execute(target_corpus, train=False)

        instancesfile = self.svmlight.create_input_file(target_corpus, 'predict', self.pipeline.feature_set)
        predictionsfile = self.svmlight.classify(instancesfile)
        self.svmlight.read_predictions(target_corpus, predictionsfile)

        return target_corpus

    def feature_generators(self):
        return __class__.default_feature_generators(self.entity1_class, self.entity2_class)

    @staticmethod
    def default_feature_generators(prot_e_id, loc_e_id, graphs=None):
        from loctext.features import DS

        GRAPHS_CLOSURE_VARIABLE = {} if graphs is None else graphs

        return [
            # Comment from Shrikant:
            # TODO ...Commenting follwing two function calls increases the overall Fscore by 0.3 points...

            # Trigram
            DS.AnyNGramFeatureGenerator(
                n_gram=3,
                #
                prefix_pos=3
            ),

            # Bigram
            DS.AnyNGramFeatureGenerator(
                n_gram=2,
                #
                prefix_bow=5,
                prefix_bow_masked=6,
                prefix_pos=7,
                prefix_stem=8
            ),

            DS.PatternFeatureGenerator(
                e1_class=prot_e_id,
                e2_class=loc_e_id,
                #
                prefix_ProtVerbWord=39,
                prefix_LocVerbWord=41,
                prefix_ProtVerbWordLocVerbWord=43,
                prefix_WordVerbProtLocVerbWord=45,
            ),

            DS.SameWordFeatureGenerator(
                prefix_sameWordsSamePOS=48,
                prefix_sameStemSamePOS=50,
            ),

            DS.LocEntityFeatureGenerator(
                localization_class_id=loc_e_id,
                prefix_localizationVerb=51
            ),

            DS.IndividualSentencesFeatureGenerator(
                prefix_sentence_1_POS=54,
                #
                prefix_sentence_2_POS=57
            ),

            NamedEntityCountFeatureGenerator(
                prot_e_id,
                prefix=130
            ),

            NamedEntityCountFeatureGenerator(
                loc_e_id,
                prefix=131
            ),
        ]

class LocTextCombinedModelRelationExtractor(RelationExtractor):

    def __init__(
            self,
            entity1_class,
            entity2_class,
            rel_type,
            ss_model,
            ds_model):

        super().__init__(entity1_class, entity2_class, rel_type)

        self.ss_model = ss_model
        self.ds_model = ds_model
        self.submodels = [self.ss_model, self.ds_model]

    def annotate(self, target_corpus):

        for model in self.submodels:
            model.annotate(target_corpus)

        return target_corpus
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

# Copyright 2017 Eddie Antonio Santos <easantos@ualberta.ca>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Access to the corpus.
"""

import os
from pathlib import Path, PurePosixPath
from typing import Any, Dict, Iterator, Set, Tuple, Union

from sqlalchemy import MetaData, create_engine, event  # type: ignore
from sqlalchemy.engine import Engine  # type: ignore
from sqlalchemy.sql import select, text  # type: ignore

from sensibility.language import SourceSummary

from ._schema import (eligible_source, failure, meta, metadata, repository,
                      repository_source, source_file, source_summary)
from .connection import get_sqlite3_path
from .models import (MockSourceFile, RepositoryID, RepositoryMetadata,
                     SourceFile, SourceFileInRepository)

class NewCorpusError(Exception):
    """
    Raised when querying an empty corpus.
    """
    pass

class FileInfo:
    """
    Contains metadata over a file, as referenced by a filehash.
    Note that one filehash maps uniquely to content, however the same content
    can be present in multiple files, within a repository, or between
    repositories. Use is_unique to determine whether this is the only copy of
    the file within the corpus.
    """
    def __init__(self, mappings: Set[SourceFileInRepository],
                 summary: SourceSummary) -> None:
        assert len(mappings) > 0
        self._mappings = mappings
        self.summary = summary

    @property
    def filehash(self) -> str:
        return self._any.filehash

    @property
    def owner(self) -> str:
        return self._any.owner

    @property
    def name(self) -> str:
        return self._any.name

    @property
    def href(self) -> str:
        return self._any.href

    @property
    def license(self) -> str:
        return self._any.license

    @property
    def path(self) -> PurePosixPath:
        return self._any.path

    @property
    def n_tokens(self) -> int:
        return self.summary.n_tokens

    @property
    def sloc(self) -> int:
        return self.summary.sloc

    @property
    def is_unique(self) -> bool:
        """
        True when there is one and only one file with the contents of this
        file. False otherwise.
        """
        return len(self._mappings) == 1

    @property
    def _any(self) -> SourceFileInRepository:
        """
        Returns any owner/repo/path containing this file.
        """
        return next(iter(self._mappings))

class Corpus:
    """
    A corpus of source code, backed by an SQLite3 database.

    Uses SQLAlchemy (as an experiment), but it's honestly not really worth it.
    """
    def __init__(self, engine=None,
                 url: str=None, path: Union[os.PathLike, str]=None,
                 writable=False) -> None:
        if engine is not None:
            self.engine = engine
        else:
            if url is None:
                if path is None:
                    path = get_sqlite3_path()

                # Ensure the containing directory exists.
                effective_path = Path(path)
                containing_dir = effective_path.parent
                if not containing_dir.is_dir():
                    containing_dir.mkdir(parents=True)

                url = f"sqlite:///{os.fspath(effective_path)}"

            self.engine = create_engine(url)

        self._initialize_sqlite3(writable)

        if self.empty:
            metadata.create_all(self.engine)

        self.conn = self.engine.connect()

    @property
    def language(self) -> str:
        """
        The main laguage of the mined sources.
        """
        query = select([meta.c.value]).\
            where(meta.c.key == 'language')
        try:
            result, = self.conn.execute(query)
        except ValueError:
            raise NewCorpusError
        else:
            return result[meta.c.value]

    @property
    def empty(self) -> bool:
        metadata = MetaData()
        metadata.reflect(self.engine)
        return 'repository' not in metadata

    @property
    def eligible_sources(self) -> Iterator[SourceFile]:
        """
        Yields source files eligible for training, validation, and testing
        (source and filehash).
        """
        query = select([source_file.c.source])\
            .select_from(source_file.join(eligible_source))
        for row in self.conn.execute(query):
            yield SourceFile(row[source_file.c.source])

    @property
    def source_summaries(self) -> Iterator[Tuple[str, SourceSummary]]:
        """
        Yields sources with computed summaries.
        """
        query = select([source_summary.c.hash, source_summary.c.n_tokens,
                        source_summary.c.sloc])
        for row in self.conn.execute(query):
            yield (row[source_summary.c.hash],
                   SourceSummary(sloc=row[source_summary.c.sloc],
                                 n_tokens=row[source_summary.c.n_tokens]))

    @property
    def sources_with_repository(self) -> Iterator[Tuple[str, str,
                                                        PurePosixPath, bytes]]:
        """
        Returns ALL sources including their repository and their repository
        path.
        """
        query = select([repository_source.c.owner, repository_source.c.name,
                        repository_source.c.path, source_file.c.source])\
            .select_from(repository_source.join(source_file))
        for owner, name, pathstr, source in self.conn.execute(query):
            yield owner, name, PurePosixPath(pathstr), source

    def __getitem__(self, filehash: str) -> bytes:
        """
        Returns a file from the corpus.
        """
        return self.get_source(filehash)

    def set_metadata(self, **kwargs: Any) -> None:
        """
        Sets the metadata table.
        """
        self.conn.execute(meta.insert(), [
            {'key': key, 'value': str(value)}
            for key, value in kwargs.items()
        ])

    def insert_repository(self, repo: RepositoryMetadata) -> None:
        """
        Insert a repository's metadata into the database.
        """
        self.conn.execute(repository.insert(),
                          owner=repo.owner, name=repo.name,
                          revision=repo.revision, license=repo.license,
                          commit_date=repo.commit_date)

    def insert_source_file_from_repo(self, entry: SourceFileInRepository) -> None:
        """
        Insert a source file from a repository into the database. If the
        source file already exists in the database, only the relation is
        inserted.
        """
        trans = self.conn.begin()
        try:
            self.conn.execute((source_file.insert()
                               .prefix_with('OR IGNORE', dialect='sqlite')),
                              source=entry.source_file.source,
                              hash=entry.filehash)
            self.conn.execute(repository_source.insert(),
                              owner=entry.owner, name=entry.name,
                              hash=entry.filehash, path=str(entry.path))
        except Exception:
            trans.rollback()
            raise
        else:
            trans.commit()

    def insert_source_summary(self, filehash: str, summary: SourceSummary) -> None:
        """
        Insert the word count into the source summary.
        """
        self.conn.execute(source_summary.insert(),
                          hash=filehash,
                          sloc=summary.sloc, n_tokens=summary.n_tokens)

    def insert_failure(self, filehash: str, reason: str=None,
                       ignore: bool=False) -> None:
        """
        Insert the word count into the source summary.

        Can add an optional reason, and choose to ignore
        """
        self.conn.execute((failure.insert().prefix_with('OR IGNORE')
                           if ignore else failure.insert()),
                          hash=filehash, reason=reason)

    def get_source(self, filehash: str) -> bytes:
        """
        Returns the source code for one file.
        """
        query = select([source_file.c.source])\
            .where(source_file.c.hash == filehash)
        result, = self.conn.execute(query)
        return result[source_file.c.source]

    def get_info(self, filehash: str) -> FileInfo:
        # Do an intense query, combining multiple tables.
        query = select([source_summary, repository_source, repository]).select_from(
            source_file.join(repository_source)
            .join(repository)
            .join(source_summary, isouter=True)
        ).where(source_file.c.hash == filehash)
        results = self.conn.execute(query).fetchall()

        mock_file = MockSourceFile(filehash)
        repos = {
            (row[repository.c.owner], row[repository.c.name]): RepositoryMetadata(
                owner=row[repository.c.owner],
                name=row[repository.c.name],
                license=row[repository.c.license],
                revision=row[repository.c.revision],
                commit_date=row[repository.c.commit_date]
            ) for row in results
        }
        mappings = set(
            SourceFileInRepository(
                repository=repos[row[repository.c.owner], row[repository.c.name]],
                source_file=mock_file,
                path=PurePosixPath(row[repository_source.c.path])
            ) for row in results
        )

        # TODO: what if it's not found?
        row, *_ = results
        summary = SourceSummary(sloc=row[source_summary.c.sloc],
                                n_tokens=row[source_summary.c.n_tokens])
        return FileInfo(mappings, summary)

    def get_repositories_with_n_tokens(self) -> Iterator[Tuple[RepositoryID, int]]:
        """
        An incredibly specific method to return repositories, along with how
        many tokens that repository contains. This number may include
        duplicates.
        """
        query = text("""
            SELECT owner, name, n_tokens
            FROM repository_source JOIN eligible_source USING (hash)
            GROUP BY owner, name
        """)
        cursor = self.conn.execute(query)
        for owner, name, n_tokens in cursor:
            yield RepositoryID(owner, name), n_tokens

    def get_eligible_hashes_in_repo(self, repo: RepositoryID, elligible=False) -> Iterator[str]:
        """
        Lists all file hashes in this repository.
        """
        query = select([repository_source.c.hash])\
            .select_from(repository_source.join(
                eligible_source,
                repository_source.c.hash == eligible_source.c.hash
            ))\
            .where(repository_source.c.owner == repo.owner)\
            .where(repository_source.c.name == repo.name)
        for row in self.conn.execute(query):
            yield row[repository_source.c.hash]

    def _initialize_sqlite3(self, writable: bool) -> None:
        """
        Set some pragmas for initially creating the SQLite3 database.
        """

        @event.listens_for(Engine, "connect")
        def set_sqlite_pragma(dbapi_connection, _connection_record):
            cur = dbapi_connection.cursor()
            cur.execute('PRAGMA encoding = "UTF-8"')
            cur.execute('PRAGMA foreign_keys = ON')
            if writable:
                cur.executescript('''
                    PRAGMA journal_mode = WAL;
                    PRAGMA synchronous = NORMAL;
                ''')
            cur.close()
<commit_msg>Address minor style issues.<commit_after>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

# Copyright 2017 Eddie Antonio Santos <easantos@ualberta.ca>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Access to the corpus.
"""

import os
from pathlib import Path, PurePosixPath
from typing import Any, Dict, Iterator, Set, Tuple, Union

from sqlalchemy import MetaData, create_engine, event  # type: ignore
from sqlalchemy.engine import Engine  # type: ignore
from sqlalchemy.sql import select, text  # type: ignore

from sensibility.language import SourceSummary

from ._schema import (eligible_source, failure, meta, metadata, repository,
                      repository_source, source_file, source_summary)
from .connection import get_sqlite3_path
from .models import (MockSourceFile, RepositoryID, RepositoryMetadata,
                     SourceFile, SourceFileInRepository)

class NewCorpusError(Exception):
    """
    Raised when querying an empty corpus.
    """
    pass

class FileInfo:
    """
    Contains metadata over a file, as referenced by a filehash.
    Note that one filehash maps uniquely to content, however the same content
    can be present in multiple files, within a repository, or between
    repositories. Use is_unique to determine whether this is the only copy of
    the file within the corpus.
    """
    def __init__(self, mappings: Set[SourceFileInRepository],
                 summary: SourceSummary) -> None:
        assert len(mappings) > 0
        self._mappings = mappings
        self.summary = summary

    @property
    def filehash(self) -> str:
        return self._any.filehash

    @property
    def owner(self) -> str:
        return self._any.owner

    @property
    def name(self) -> str:
        return self._any.name

    @property
    def href(self) -> str:
        return self._any.href

    @property
    def license(self) -> str:
        return self._any.license

    @property
    def path(self) -> PurePosixPath:
        return self._any.path

    @property
    def n_tokens(self) -> int:
        return self.summary.n_tokens

    @property
    def sloc(self) -> int:
        return self.summary.sloc

    @property
    def is_unique(self) -> bool:
        """
        True when there is one and only one file with the contents of this
        file. False otherwise.
        """
        return len(self._mappings) == 1

    @property
    def _any(self) -> SourceFileInRepository:
        """
        Returns any owner/repo/path containing this file.
        """
        return next(iter(self._mappings))

class Corpus:
    """
    A corpus of source code, backed by an SQLite3 database.

    Uses SQLAlchemy (as an experiment), but it's honestly not really worth it.
    """
    def __init__(self, engine=None,
                 url: str = None,
                 path: Union[os.PathLike, str] = None,
                 writable=False) -> None:
        if engine is not None:
            self.engine = engine
        else:
            if url is None:
                if path is None:
                    path = get_sqlite3_path()

                # Ensure the containing directory exists.
                effective_path = Path(path)
                containing_dir = effective_path.parent
                if not containing_dir.is_dir():
                    containing_dir.mkdir(parents=True)

                url = f"sqlite:///{os.fspath(effective_path)}"

            self.engine = create_engine(url)

        self._initialize_sqlite3(writable)

        if self.empty:
            metadata.create_all(self.engine)

        self.conn = self.engine.connect()

    @property
    def language(self) -> str:
        """
        The main laguage of the mined sources.
        """
        query = select([meta.c.value]).\
            where(meta.c.key == 'language')
        try:
            result, = self.conn.execute(query)
        except ValueError:
            raise NewCorpusError
        else:
            return result[meta.c.value]

    @property
    def empty(self) -> bool:
        metadata = MetaData()
        metadata.reflect(self.engine)
        return 'repository' not in metadata

    @property
    def eligible_sources(self) -> Iterator[SourceFile]:
        """
        Yields source files eligible for training, validation, and testing
        (source and filehash).
        """
        query = select([source_file.c.source])\
            .select_from(source_file.join(eligible_source))
        for row in self.conn.execute(query):
            yield SourceFile(row[source_file.c.source])

    @property
    def source_summaries(self) -> Iterator[Tuple[str, SourceSummary]]:
        """
        Yields sources with computed summaries.
        """
        query = select([source_summary.c.hash, source_summary.c.n_tokens,
                        source_summary.c.sloc])
        for row in self.conn.execute(query):
            yield (row[source_summary.c.hash],
                   SourceSummary(sloc=row[source_summary.c.sloc],
                                 n_tokens=row[source_summary.c.n_tokens]))

    @property
    def sources_with_repository(self) -> Iterator[Tuple[str, str,
                                                        PurePosixPath, bytes]]:
        """
        Returns ALL sources including their repository and their repository
        path.
        """
        query = select([repository_source.c.owner, repository_source.c.name,
                        repository_source.c.path, source_file.c.source])\
            .select_from(repository_source.join(source_file))
        for owner, name, pathstr, source in self.conn.execute(query):
            yield owner, name, PurePosixPath(pathstr), source

    def __getitem__(self, filehash: str) -> bytes:
        """
        Returns a file from the corpus.
        """
        return self.get_source(filehash)

    def set_metadata(self, **kwargs: Any) -> None:
        """
        Sets the metadata table.
        """
        self.conn.execute(meta.insert(), [
            {'key': key, 'value': str(value)}
            for key, value in kwargs.items()
        ])

    def insert_repository(self, repo: RepositoryMetadata) -> None:
        """
        Insert a repository's metadata into the database.
        """
        self.conn.execute(repository.insert(),
                          owner=repo.owner, name=repo.name,
                          revision=repo.revision, license=repo.license,
                          commit_date=repo.commit_date)

    def insert_source_file_from_repo(self, entry: SourceFileInRepository) -> None:
        """
        Insert a source file from a repository into the database. If the
        source file already exists in the database, only the relation is
        inserted.
        """
        trans = self.conn.begin()
        try:
            self.conn.execute((source_file.insert()
                               .prefix_with('OR IGNORE', dialect='sqlite')),
                              source=entry.source_file.source,
                              hash=entry.filehash)
            self.conn.execute(repository_source.insert(),
                              owner=entry.owner, name=entry.name,
                              hash=entry.filehash, path=str(entry.path))
        except Exception:
            trans.rollback()
            raise
        else:
            trans.commit()

    def insert_source_summary(self, filehash: str, summary: SourceSummary) -> None:
        """
        Insert the word count into the source summary.
        """
        self.conn.execute(source_summary.insert(),
                          hash=filehash,
                          sloc=summary.sloc, n_tokens=summary.n_tokens)

    def insert_failure(self, filehash: str, reason: str = None,
                       ignore: bool = False) -> None:
        """
        Insert the word count into the source summary.

        Can add an optional reason, and choose to ignore
        """
        self.conn.execute((failure.insert().prefix_with('OR IGNORE')
                           if ignore else failure.insert()),
                          hash=filehash, reason=reason)

    def get_source(self, filehash: str) -> bytes:
        """
        Returns the source code for one file.
        """
        query = select([source_file.c.source])\
            .where(source_file.c.hash == filehash)
        result, = self.conn.execute(query)
        return result[source_file.c.source]

    def get_info(self, filehash: str) -> FileInfo:
        # Do an intense query, combining multiple tables.
        query = select([source_summary, repository_source, repository]).select_from(
            source_file.join(repository_source)
            .join(repository)
            .join(source_summary, isouter=True)
        ).where(source_file.c.hash == filehash)
        results = self.conn.execute(query).fetchall()

        mock_file = MockSourceFile(filehash)
        repos = {
            (row[repository.c.owner], row[repository.c.name]): RepositoryMetadata(
                owner=row[repository.c.owner],
                name=row[repository.c.name],
                license=row[repository.c.license],
                revision=row[repository.c.revision],
                commit_date=row[repository.c.commit_date]
            ) for row in results
        }
        mappings = set(
            SourceFileInRepository(
                repository=repos[row[repository.c.owner], row[repository.c.name]],
                source_file=mock_file,
                path=PurePosixPath(row[repository_source.c.path])
            ) for row in results
        )

        # TODO: what if it's not found?
        row, *_ = results
        summary = SourceSummary(sloc=row[source_summary.c.sloc],
                                n_tokens=row[source_summary.c.n_tokens])
        return FileInfo(mappings, summary)

    def get_repositories_with_n_tokens(self) -> Iterator[Tuple[RepositoryID, int]]:
        """
        An incredibly specific method to return repositories, along with how
        many tokens that repository contains. This number may include
        duplicates.
        """
        query = text("""
            SELECT owner, name, n_tokens
            FROM repository_source JOIN eligible_source USING (hash)
            GROUP BY owner, name
        """)
        cursor = self.conn.execute(query)
        for owner, name, n_tokens in cursor:
            yield RepositoryID(owner, name), n_tokens

    def get_eligible_hashes_in_repo(self, repo: RepositoryID, elligible=False) -> Iterator[str]:
        """
        Lists all file hashes in this repository.
        """
        query = select([repository_source.c.hash])\
            .select_from(repository_source.join(
                eligible_source,
                repository_source.c.hash == eligible_source.c.hash
            ))\
            .where(repository_source.c.owner == repo.owner)\
            .where(repository_source.c.name == repo.name)
        for row in self.conn.execute(query):
            yield row[repository_source.c.hash]

    def _initialize_sqlite3(self, writable: bool) -> None:
        """
        Set some pragmas for initially creating the SQLite3 database.
        """

        @event.listens_for(Engine, "connect")
        def set_sqlite_pragma(dbapi_connection, _connection_record):
            cur = dbapi_connection.cursor()
            cur.execute('PRAGMA encoding = "UTF-8"')
            cur.execute('PRAGMA foreign_keys = ON')
            if writable:
                cur.executescript('''
                    PRAGMA journal_mode = WAL;
                    PRAGMA synchronous = NORMAL;
                ''')
            cur.close()
<|endoftext|>