<commit_before>#!/usr/bin/env python3
import pygame
import subprocess
import os
import configparser
import sys
import time
import calendar
from itertools import chain

config = {
    "photostorage": "Pictures",
    "chdkptp": "chdkptp",
    "fullscreen": "-w" not in sys.argv,  # -w for windowed
    "bgcolor": (0, 0, 0),
    "textcolor": (255, 255, 255),
    "displayPhotoFor": 5,
    "photosPerSet": 4,
    "countdown": 4,
    "countdownSpeed": 1
}

def configure():
    cf = configparser.SafeConfigParser()
    cf.read(['photobooth.cfg', os.path.expanduser('~/.photobooth.cfg')])
    if cf.has_section('photobooth'):
        for key in config:
            if cf.has_option('photobooth', key):
                if type(config[key]) == int:
                    config[key] = cf.getint('photobooth', key)
                else:
                    config[key] = cf.get('photobooth', key)
                    if key in ["bgcolor", "textcolor"]:
                        config[key] = config[key].split(",")
                        for color in range(len(config[key])):
                            config[key][color] = int(config[key][color].strip())
                        config[key] = tuple(config[key])
    print(config)

def waitForInput(stream, waitFor=">"):
    char = stream.read(1).decode('utf-8')
    while char != waitFor:
        print(char, end="")
        sys.stdout.flush()
        char = stream.read(1).decode('utf-8')
    print(char)

## The following code was stolen from http://www.pygame.org/wiki/TextWrapping
def truncline(text, font, maxwidth):
    real = len(text)
    stext = text
    l = font.size(text)[0]
    cut = 0
    a = 0
    done = 1
    while l > maxwidth:
        a = a+1
        n = text.rsplit(None, a)[0]
        if stext == n:
            cut += 1
            stext = n[:-cut]
        else:
            stext = n
        l = font.size(stext)[0]
        real = len(stext)
        done = 0
    return real, done, stext

def wrapline(text, font, maxwidth):
    done = 0
    wrapped = []

    while not done:
        nl, done, stext = truncline(text, font, maxwidth)
        wrapped.append(stext.strip())
        text = text[nl:]
    return wrapped

def renderText(textstr, game, fontSize=1000):
    font = pygame.font.Font(None, fontSize)
    lines = wrapline(textstr, font, game['size'][0]-100)
    height = font.size(lines[0])[1]
    for line in range(0, len(lines)):
        text = font.render(lines[line], 1, config['textcolor'])
        textpos = text.get_rect()
        textpos.centerx = game['background'].get_rect().centerx
        textpos.centery = height*0.5 + height*line
        print("Text position: %s" % textpos)
        game['background'].blit(text, textpos)
        game['screen'].blit(game['background'], (0, 0))
    pygame.display.flip()
    game['clock'].tick()

def takePhotoSet(chdkptp, game):
    stripnumber = calendar.timegm(time.gmtime())
    filename = "%s/%s" % (config['photostorage'], stripnumber)
    # Enter rsint mode
    chdkptp.stdin.write(("rsint %s-last\n" % filename).encode())
    waitForInput(chdkptp.stdout)
    photonumber = 0
    while photonumber < config['photosPerSet']:
        doCountdown(game)
        takePhoto(chdkptp)
        os.rename("%s-last.jpg" % filename, "%s-%s.jpg" % (filename, photonumber))
        displayPhoto("%s-%s.jpg" % (filename, photonumber), game)
        photonumber += 1
    chdkptp.stdin.write(b"q\n")
    waitForInput(chdkptp.stdout)

def doCountdown(game):
    countdown = config['countdown']
    while countdown >= 0:
        game['background'].fill(config['bgcolor'])
        if countdown > 0:
            renderText(str(countdown), game, fontSize=1500)
        elif countdown == 0:
            renderText("smile", game)
        countdown -= 1
        time.sleep(config['countdownSpeed'])

def takePhoto(chdkptp):
    chdkptp.stdin.write(b"s\n")
    waitForInput(chdkptp.stdout)

def displayPhoto(filename, game):
    img = pygame.image.load(filename)
    imgposition = img.get_rect()
    imgposition.centerx = game['background'].get_rect().centerx
    imgposition.centery = game['background'].get_rect().centery
    game['background'].blit(img, imgposition)
    game['screen'].blit(game['background'], (0, 0))
    pygame.display.flip()
    game['clock'].tick()
    time.sleep(config['displayPhotoFor'])

def waitForTrigger(game):
    triggered = False
    game['background'].fill(config['bgcolor'])
    renderText("Press the #BigRedButton to begin", game, fontSize=200)
    while not triggered:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                sys.exit()
            if event.type == pygame.KEYDOWN:
                if event.key == 32:  # spacebar
                    triggered = True
                elif event.key in [113, 27]:  # q, esc
                    sys.exit()
        time.sleep(0.05)
    print("triggered!!")

def main():
    configure()

    pygame.init()

    displayinfo = pygame.display.Info()  # Find some info out about the display
    if config['fullscreen']:
        size = displayinfo.current_w, displayinfo.current_h
    else:
        size = 1800, 800

    screen = pygame.display.set_mode(size)

    if config['fullscreen']:
        pygame.display.toggle_fullscreen()
    c = pygame.time.Clock()

    background = pygame.Surface(screen.get_size())

    chdkptp = subprocess.Popen(config['chdkptp'],
                               stdin=subprocess.PIPE,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               bufsize=0)
    waitForInput(chdkptp.stdout)
    chdkptp.stdin.write(b"c\n")
    print("Connecting to camera....")
    waitForInput(chdkptp.stdout)
    chdkptp.stdin.write(b"rec\n")
    print("Entering shooting mode...")
    waitForInput(chdkptp.stdout)
    print("Annnd into the loop we go!")

    game = {
        "screen": screen,
        "background": background,
        "clock": c,
        "size": size
    }

    while True:
        waitForTrigger(game)
        takePhotoSet(chdkptp, game)

if __name__ == "__main__":
    main()
<commit_msg>remove extra debug text<commit_after>#!/usr/bin/env python3
import pygame
import subprocess
import os
import configparser
import sys
import time
import calendar
from itertools import chain

config = {
    "photostorage": "Pictures",
    "chdkptp": "chdkptp",
    "fullscreen": "-w" not in sys.argv,  # -w for windowed
    "bgcolor": (0, 0, 0),
    "textcolor": (255, 255, 255),
    "displayPhotoFor": 5,
    "photosPerSet": 4,
    "countdown": 4,
    "countdownSpeed": 1
}

def configure():
    cf = configparser.SafeConfigParser()
    cf.read(['photobooth.cfg', os.path.expanduser('~/.photobooth.cfg')])
    if cf.has_section('photobooth'):
        for key in config:
            if cf.has_option('photobooth', key):
                if type(config[key]) == int:
                    config[key] = cf.getint('photobooth', key)
                else:
                    config[key] = cf.get('photobooth', key)
                    if key in ["bgcolor", "textcolor"]:
                        config[key] = config[key].split(",")
                        for color in range(len(config[key])):
                            config[key][color] = int(config[key][color].strip())
                        config[key] = tuple(config[key])
    print(config)

def waitForInput(stream, waitFor=">"):
    char = stream.read(1).decode('utf-8')
    while char != waitFor:
        print(char, end="")
        sys.stdout.flush()
        char = stream.read(1).decode('utf-8')
    print(char)

## The following code was stolen from http://www.pygame.org/wiki/TextWrapping
def truncline(text, font, maxwidth):
    real = len(text)
    stext = text
    l = font.size(text)[0]
    cut = 0
    a = 0
    done = 1
    while l > maxwidth:
        a = a+1
        n = text.rsplit(None, a)[0]
        if stext == n:
            cut += 1
            stext = n[:-cut]
        else:
            stext = n
        l = font.size(stext)[0]
        real = len(stext)
        done = 0
    return real, done, stext

def wrapline(text, font, maxwidth):
    done = 0
    wrapped = []

    while not done:
        nl, done, stext = truncline(text, font, maxwidth)
        wrapped.append(stext.strip())
        text = text[nl:]
    return wrapped

def renderText(textstr, game, fontSize=1000):
    font = pygame.font.Font(None, fontSize)
    lines = wrapline(textstr, font, game['size'][0]-100)
    height = font.size(lines[0])[1]
    for line in range(0, len(lines)):
        text = font.render(lines[line], 1, config['textcolor'])
        textpos = text.get_rect()
        textpos.centerx = game['background'].get_rect().centerx
        textpos.centery = height*0.5 + height*line
        game['background'].blit(text, textpos)
        game['screen'].blit(game['background'], (0, 0))
    pygame.display.flip()
    game['clock'].tick()

def takePhotoSet(chdkptp, game):
    stripnumber = calendar.timegm(time.gmtime())
    filename = "%s/%s" % (config['photostorage'], stripnumber)
    # Enter rsint mode
    chdkptp.stdin.write(("rsint %s-last\n" % filename).encode())
    waitForInput(chdkptp.stdout)
    photonumber = 0
    while photonumber < config['photosPerSet']:
        doCountdown(game)
        takePhoto(chdkptp)
        os.rename("%s-last.jpg" % filename, "%s-%s.jpg" % (filename, photonumber))
        displayPhoto("%s-%s.jpg" % (filename, photonumber), game)
        photonumber += 1
    chdkptp.stdin.write(b"q\n")
    waitForInput(chdkptp.stdout)

def doCountdown(game):
    countdown = config['countdown']
    while countdown >= 0:
        game['background'].fill(config['bgcolor'])
        if countdown > 0:
            renderText(str(countdown), game, fontSize=1500)
        elif countdown == 0:
            renderText("smile", game)
        countdown -= 1
        time.sleep(config['countdownSpeed'])

def takePhoto(chdkptp):
    chdkptp.stdin.write(b"s\n")
    waitForInput(chdkptp.stdout)

def displayPhoto(filename, game):
    img = pygame.image.load(filename)
    imgposition = img.get_rect()
    imgposition.centerx = game['background'].get_rect().centerx
    imgposition.centery = game['background'].get_rect().centery
    game['background'].blit(img, imgposition)
    game['screen'].blit(game['background'], (0, 0))
    pygame.display.flip()
    game['clock'].tick()
    time.sleep(config['displayPhotoFor'])

def waitForTrigger(game):
    triggered = False
    game['background'].fill(config['bgcolor'])
    renderText("Press the #BigRedButton to begin", game, fontSize=200)
    while not triggered:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                sys.exit()
            if event.type == pygame.KEYDOWN:
                if event.key == 32:  # spacebar
                    triggered = True
                elif event.key in [113, 27]:  # q, esc
                    sys.exit()
        time.sleep(0.05)
    print("triggered!!")

def main():
    configure()

    pygame.init()

    displayinfo = pygame.display.Info()  # Find some info out about the display
    if config['fullscreen']:
        size = displayinfo.current_w, displayinfo.current_h
    else:
        size = 1800, 800

    screen = pygame.display.set_mode(size)

    if config['fullscreen']:
        pygame.display.toggle_fullscreen()
    c = pygame.time.Clock()

    background = pygame.Surface(screen.get_size())

    chdkptp = subprocess.Popen(config['chdkptp'],
                               stdin=subprocess.PIPE,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               bufsize=0)
    waitForInput(chdkptp.stdout)
    chdkptp.stdin.write(b"c\n")
    print("Connecting to camera....")
    waitForInput(chdkptp.stdout)
    chdkptp.stdin.write(b"rec\n")
    print("Entering shooting mode...")
    waitForInput(chdkptp.stdout)
    print("Annnd into the loop we go!")

    game = {
        "screen": screen,
        "background": background,
        "clock": c,
        "size": size
    }

    while True:
        waitForTrigger(game)
        takePhotoSet(chdkptp, game)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import sys

class Cube(object):
    def __init__(self, levels):
        self.levels = levels

    @property
    def height(self):
        return len(self.levels)

    def __str__(self):
        result = []
        counter = 0
        for level in self.levels:
            counter += 1
            result.append("Level %i" % counter)
            for row in level: result.append(''.join(row))
            result.append("\n")
        return "\n".join(result)

    def _find_empty_cell_in_border(self, level):
        # Top row.
        for i in xrange(0, self.height):
            if self.levels[level][0][i] == ' ': return 0, i
        # Bottom row.
        for i in xrange(0, self.height):
            if self.levels[level][self.height-1][i] == ' ': return self.height-1, i
        # Sides.
        for i in xrange(1, self.height-1):
            if self.levels[level][i][0] == ' ': return i, 0
            if self.levels[level][i][self.height-1] == ' ': return i, self.height-1
        # Should never reach here if there is an empty cell in the border.
        return None

    @property
    def entrance(self):
        return self._find_empty_cell_in_border(0)

    @property
    def exit(self):
        return self._find_empty_cell_in_border(self.height-1)

def main():
    test_cases = open(sys.argv[1], 'r')
    for test in test_cases:
        test = test.strip()
        if len(test) == 0:
            continue

        height = int(test.split(';')[0])
        levels = zip(*[iter(zip(*[iter(test.split(';')[1])]*height))]*height)

        cube = Cube(levels)

        print "============ %i" % cube.height
        print cube.entrance
        print cube.exit

        # TODO finish this...

    test_cases.close()

if __name__ == '__main__':
    main()<commit_msg>WIP: cube problem; naming update.<commit_after>import sys

class Cube(object):
    def __init__(self, levels):
        self.levels = levels

    @property
    def length(self):
        return len(self.levels)

    def __str__(self):
        result = []
        counter = 0
        for level in self.levels:
            counter += 1
            result.append("Level %i" % counter)
            for row in level: result.append(''.join(row))
            result.append("\n")
        return "\n".join(result)

    def _find_empty_cell_in_border(self, level):
        # Top row.
        for i in xrange(0, self.length):
            if self.levels[level][0][i] == ' ': return 0, i
        # Bottom row.
        for i in xrange(0, self.length):
            if self.levels[level][self.length-1][i] == ' ': return self.length-1, i
        # Sides.
        for i in xrange(1, self.length-1):
            if self.levels[level][i][0] == ' ': return i, 0
            if self.levels[level][i][self.length-1] == ' ': return i, self.length-1
        # Should never reach here if there is an empty cell in the border.
        return None

    @property
    def entrance(self):
        return self._find_empty_cell_in_border(0)

    @property
    def exit(self):
        return self._find_empty_cell_in_border(self.length-1)

def main():
    test_cases = open(sys.argv[1], 'r')
    for test in test_cases:
        test = test.strip()
        if len(test) == 0:
            continue

        length = int(test.split(';')[0])
        levels = zip(*[iter(zip(*[iter(test.split(';')[1])]*length))]*length)

        cube = Cube(levels)

        print "============ %i" % cube.length
        print cube.entrance
        print cube.exit

        # TODO finish this...

    test_cases.close()

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>from django.test import RequestFactory, TestCase
from django.test.client import Client

from gem.middleware import GemMoloGoogleAnalyticsMiddleware
from gem.models import GemSettings

from mock import patch

from molo.core.models import SiteSettings
from molo.core.tests.base import MoloTestCaseMixin

class TestCustomGemMiddleware(TestCase, MoloTestCaseMixin):

    submit_tracking_method = (
        "gem.middleware.GemMoloGoogleAnalyticsMiddleware.submit_tracking"
    )

    def setUp(self):
        self.client = Client()
        self.mk_main()

        self.site_settings = SiteSettings.for_site(self.site)
        self.site_settings.local_ga_tracking_code = 'local_ga_tracking_code'
        self.site_settings.save()

        GemSettings.objects.create(
            site_id=self.site.id,
            bbm_ga_account_subdomain='bbm',
            bbm_ga_tracking_code='bbm_tracking_code',
        )

        self.response = self.client.get('/')

    @patch(submit_tracking_method)
    def test_submit_to_additional_ga_account(self, mock_submit_tracking):
        '''
        Given that bbm_ga_account_subdomain and bbm_ga_tracking_code
        are set in Gem Settings, and the URL contains the
        bbm_ga_account_subdomain, info should be sent to
        the additional GA account.
        '''

        request = RequestFactory().get('/', HTTP_HOST='bbm.example.com')
        request.site = self.site

        middleware = GemMoloGoogleAnalyticsMiddleware()
        middleware.submit_to_local_account(
            request, self.response, self.site_settings)

        mock_submit_tracking.assert_called_once_with(
            'bbm_tracking_code', request, self.response)

    @patch(submit_tracking_method)
    def test_submit_to_bbm_analytics_if_cookie_set(self, mock_submit_tracking):
        '''
        Given that bbm_ga_account_subdomain and bbm_ga_tracking_code
        are set in Gem Settings, and the BBM cookie is set, info
        should be sent to the additional GA account.
        '''

        request = RequestFactory().get('/', HTTP_HOST='example.com')
        request.COOKIES['bbm'] = 'true'
        request.site = self.site

        middleware = GemMoloGoogleAnalyticsMiddleware()
        middleware.submit_to_local_account(
            request, self.response, self.site_settings)

        mock_submit_tracking.assert_called_once_with(
            'bbm_tracking_code', request, self.response)

    @patch(submit_tracking_method)
    def test_submit_to_local_ga_account(self, mock_submit_tracking):
        '''
        Given that bbm_ga_account_subdomain and bbm_ga_tracking_code
        are set in Gem Settings, and the URL does not contain the
        bbm_ga_account_subdomain, info should be sent to
        the local GA account, not the additional GA account.
        '''

        request = RequestFactory().get('/', HTTP_HOST='example.com')
        request.site = self.site

        middleware = GemMoloGoogleAnalyticsMiddleware()
        middleware.submit_to_local_account(
            request, self.response, self.site_settings)

        mock_submit_tracking.assert_called_once_with(
            'local_ga_tracking_code', request, self.response)
<commit_msg>Fix test hostnames to match settings.ALLOWED_HOSTS<commit_after>from django.test import RequestFactory, TestCase
from django.test.client import Client

from gem.middleware import GemMoloGoogleAnalyticsMiddleware
from gem.models import GemSettings

from mock import patch

from molo.core.models import SiteSettings
from molo.core.tests.base import MoloTestCaseMixin

class TestCustomGemMiddleware(TestCase, MoloTestCaseMixin):

    submit_tracking_method = (
        "gem.middleware.GemMoloGoogleAnalyticsMiddleware.submit_tracking"
    )

    def setUp(self):
        self.client = Client()
        self.mk_main()

        self.site_settings = SiteSettings.for_site(self.site)
        self.site_settings.local_ga_tracking_code = 'local_ga_tracking_code'
        self.site_settings.save()

        GemSettings.objects.create(
            site_id=self.site.id,
            bbm_ga_account_subdomain='bbm',
            bbm_ga_tracking_code='bbm_tracking_code',
        )

        self.response = self.client.get('/')

    @patch(submit_tracking_method)
    def test_submit_to_additional_ga_account(self, mock_submit_tracking):
        '''
        Given that bbm_ga_account_subdomain and bbm_ga_tracking_code
        are set in Gem Settings, and the URL contains the
        bbm_ga_account_subdomain, info should be sent to
        the additional GA account.
        '''

        request = RequestFactory().get('/', HTTP_HOST='bbm.localhost')
        request.site = self.site

        middleware = GemMoloGoogleAnalyticsMiddleware()
        middleware.submit_to_local_account(
            request, self.response, self.site_settings)

        mock_submit_tracking.assert_called_once_with(
            'bbm_tracking_code', request, self.response)

    @patch(submit_tracking_method)
    def test_submit_to_bbm_analytics_if_cookie_set(self, mock_submit_tracking):
        '''
        Given that bbm_ga_account_subdomain and bbm_ga_tracking_code
        are set in Gem Settings, and the BBM cookie is set, info
        should be sent to the additional GA account.
        '''

        request = RequestFactory().get('/', HTTP_HOST='localhost')
        request.COOKIES['bbm'] = 'true'
        request.site = self.site

        middleware = GemMoloGoogleAnalyticsMiddleware()
        middleware.submit_to_local_account(
            request, self.response, self.site_settings)

        mock_submit_tracking.assert_called_once_with(
            'bbm_tracking_code', request, self.response)

    @patch(submit_tracking_method)
    def test_submit_to_local_ga_account(self, mock_submit_tracking):
        '''
        Given that bbm_ga_account_subdomain and bbm_ga_tracking_code
        are set in Gem Settings, and the URL does not contain the
        bbm_ga_account_subdomain, info should be sent to
        the local GA account, not the additional GA account.
        '''

        request = RequestFactory().get('/', HTTP_HOST='localhost')
        request.site = self.site

        middleware = GemMoloGoogleAnalyticsMiddleware()
        middleware.submit_to_local_account(
            request, self.response, self.site_settings)

        mock_submit_tracking.assert_called_once_with(
            'local_ga_tracking_code', request, self.response)
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
Takes a phylogeny and a taxonomy and labels nodes representing higher-order
taxa in the phylogeny.

Both the phylogeny and the taxonomy should be trees in a format supported by
BioPython.

Usage:

    python phylolabel.py tree_file taxonomy_file
                         [tree_format] [taxonomy_format] [output_format]

Output will be printed to stdout.
'''
import Bio.Phylo as bp
from time import time
import sys

def convert_labels(t):
    '''Replace underscores with spaces in taxonomy labels for consistency.'''
    for x in t.find_elements():
        if x.name:
            x.name = x.name.replace('_', ' ')

def label_tree(tree, taxonomy):
    '''Add taxonomic labels to phylogeny. Operates on the phylogeny in-place.'''

    # standardize labels
    convert_labels(tree)
    convert_labels(taxonomy)

    # cache labels
    tree.cache_labels()
    taxonomy.cache_labels()

    # get all named terminal nodes from phylogeny
    tree_species = [sp for sp in tree.get_terminals() if sp.name]

    # create two dictionaries, mapping the nodes from the phylogeny to the taxonomy
    # and vice versa
    tax_to_tree = {}
    tree_to_tax = {}
    for sp in tree_species:
        x = taxonomy.find_any(sp.name)
        if x:
            tree_to_tax[sp] = x
            tax_to_tree[x] = sp

    # walk through species of phylogeny, marking the common ancestors of each
    # taxonomic grouping they're a member of
    done = set()
    for sp in tree_species:
        if not sp in tree_to_tax: continue
        if sp.name in done: continue

        tax_sp = tree_to_tax[sp]
        tax_parents = tax_sp.get_parents(False)
        for parent in tax_parents:
            if parent.name in done: continue

            fellows = parent.find_elements()
            tree_fellows = (tax_to_tree[x] for x in fellows if x in tax_to_tree)
            group_root = tree.common_ancestor(tree_fellows)
            if not group_root.name:
                group_root.name = parent.name
            else:
                # TODO: if the node is already labeled, split it into two nodes
                pass

            done.add(parent.name)

        done.add(sp.name)

if __name__ == '__main__':
    tree_file, tax_file = sys.argv[1:3]
    try: tree_format = sys.argv[3]
    except: tree_format = 'newick'
    try: tax_format = sys.argv[4]
    except: tax_format = 'newick'
    try: output_format = sys.argv[5]
    except: output_format = 'newick'

    # read in the tree and taxonomy
    tree = bp.read(tree_file, tree_format)
    taxonomy = bp.read(tax_file, tax_format)

    label_tree(tree, taxonomy)

    # write output to stdout
    print tree.format(output_format)
<commit_msg>When a single node can have multiple taxonomic labels (e.g. for monotypic genera), add a new node directly above with branch length 0.<commit_after>#!/usr/bin/env python
'''
Takes a phylogeny and a taxonomy and labels nodes representing higher-order
taxa in the phylogeny.

Both the phylogeny and the taxonomy should be trees in a format supported by
BioPython.

Usage:

    python phylolabel.py tree_file taxonomy_file
                         [tree_format] [taxonomy_format] [output_format]

Output will be printed to stdout.
'''
import Bio.Phylo as bp
from time import time
import sys

def convert_labels(t):
    '''Replace underscores with spaces in taxonomy labels for consistency.'''
    for x in t.find_elements():
        if x.name:
            x.name = x.name.replace('_', ' ')

def label_tree(tree, taxonomy):
    '''Add taxonomic labels to phylogeny. Operates on the phylogeny in-place.'''

    # standardize labels
    convert_labels(tree)
    convert_labels(taxonomy)

    # cache labels
    tree.cache_labels()
    taxonomy.cache_labels()

    # get all named terminal nodes from phylogeny
    tree_species = [sp for sp in tree.get_terminals() if sp.name]

    # create two dictionaries, mapping the nodes from the phylogeny to the taxonomy
    # and vice versa
    tax_to_tree = {}
    tree_to_tax = {}
    for sp in tree_species:
        x = taxonomy.find_any(sp.name)
        if x:
            tree_to_tax[sp] = x
            tax_to_tree[x] = sp

    # walk through species of phylogeny, marking the common ancestors of each
    # taxonomic grouping they're a member of
    done = set()
    for sp in tree_species:
        if not sp in tree_to_tax: continue
        if sp.name in done: continue

        tax_sp = tree_to_tax[sp]
        tax_parents = tax_sp.get_parents(False)
        for parent in tax_parents:
            if parent.name in done: continue

            fellows = parent.find_elements()
            tree_fellows = (tax_to_tree[x] for x in fellows if x in tax_to_tree)
            group_root = tree.common_ancestor(tree_fellows)
            if not group_root.name:
                group_root.name = parent.name
            else:
                new_clade = bp.BaseTree.Clade(name=parent.name, branch_length=0)

                if group_root is tree.root:
                    tree.root = new_clade
                else:
                    old_parent = group_root._parent
                    old_parent.clades.remove(group_root)
                    old_parent.clades.append(new_clade)

                new_clade.clades.append(group_root)

            done.add(parent.name)

        done.add(sp.name)

if __name__ == '__main__':
    tree_file, tax_file = sys.argv[1:3]
    try: tree_format = sys.argv[3]
    except: tree_format = 'newick'
    try: tax_format = sys.argv[4]
    except: tax_format = 'newick'
    try: output_format = sys.argv[5]
    except: output_format = 'newick'

    # read in the tree and taxonomy
    tree = bp.read(tree_file, tree_format)
    taxonomy = bp.read(tax_file, tax_format)

    label_tree(tree, taxonomy)

    # write output to stdout
    print tree.format(output_format)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Location database."""

import logging
import os

import arrow

from sqlalchemy import (
    Column,
    MetaData,
    Table,
    create_engine,
    select,
)
from sqlalchemy.types import (
    DateTime,
    Float,
    String,
)
from xdg import BaseDirectory

logger = logging.getLogger(__name__)

class Database(object):

    """Generic database object.

    This class is subclassed to provide additional functionality specific to
    artifacts and/or documents.

    :param db_filename: Path to the sqlite database file
    :type db_filename: str

    """

    def __init__(self, db_filename):
        """Connect to database and create session object."""
        self.db_filename = db_filename
        self.engine = create_engine(
            'sqlite:///{}'.format(db_filename),
            connect_args={'check_same_thread': False},
        )
        self.connection = None
        self.metadata = MetaData(bind=self.engine)

    def connect(self):
        """Create connection."""
        logger.debug('Connecting to SQLite database: %r', self.db_filename)
        self.connection = self.engine.connect()

    def disconnect(self):
        """Close connection."""
        assert not self.connection.closed
        logger.debug(
            'Disconnecting from SQLite database: %r', self.db_filename)
        self.connection.close()

    def __enter__(self):
        """Connect on entering context."""
        self.connect()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        """Disconnect on exiting context."""
        self.disconnect()

    def __getitem__(self, table_name):
        """Get table object in database.

        :param table_name: Name of the table
        :type table_name: str
        :return: Table object that can be used in queries
        :rtype: sqlalchemy.schema.Table

        """
        if not isinstance(table_name, basestring):
            raise TypeError('Unexpected table name: {}'.format(table_name))
        table = self.metadata.tables.get(table_name)
        if table is None:
            table = Table(table_name, self.metadata, autoload=True)
        return table

class LocationDB(Database):

    """Location database.

    Store location information from picture files into a sqlite database.

    """

    def __init__(self):
        """Create database if needed."""
        base_directory = BaseDirectory.save_data_path('pic2map')
        db_filename = os.path.join(base_directory, 'location.db')
        Database.__init__(self, db_filename)

        if os.path.isfile(db_filename):
            self.location_table = self['location']
        else:
            logger.debug('Creating location database %r...', db_filename)

            self.location_table = Table(
                'location',
                self.metadata,
                Column('filename', String),
                Column('latitude', Float),
                Column('longitude', Float),
                Column('datetime', DateTime),
            )
            self.location_table.create()

    def insert(self, rows):
        """Insert rows in location table.

        :param rows: Rows to be inserted in the database
        :type rows: list(dict(str))

        """
        insert_query = self.location_table.insert()
        result = self.connection.execute(insert_query, rows)
        logger.debug('%d rows inserted', result.rowcount)

    def select_all(self):
        """Get all rows from the location table.

        :returns: Location information rows
        :rtype: sqlalchemy.engine.result.ResultProxy

        """
        select_query = select([self.location_table])
        result = self.connection.execute(select_query)
        return result

    def delete(self, directory):
        """Delete rows with a filename under a given directory.

        :param directory: Directory to be used as prefix in the delete query
        :type directory: str

        """
        table = self.location_table
        delete_query = (
            table.delete()
            .where(table.c.filename.startswith(directory))
        )
        result = self.connection.execute(delete_query)
        logger.debug('%d rows deleted', result.rowcount)
        return result

    def count(self):
        """Get number of rows in the database.

        :return: Number of rows
        :rtype: int

        """
        select_query = self.location_table.count()
        result = self.connection.execute(select_query)
        return result.scalar()

def transform_metadata_to_row(metadata):
    """Transform GPS metadata in database rows.

    :param metadata: GPS metadata as returned by exiftool
    :type metadata: dict(str)
    :returns: Database row
    :rtype: dict(str)

    """
    new_metadata = {
        'filename': metadata['SourceFile'],
        'latitude': metadata['EXIF:GPSLatitude'],
        'longitude': metadata['EXIF:GPSLongitude'],
    }

    if metadata['EXIF:GPSLatitudeRef'] == u'S':
        new_metadata['latitude'] *= -1
    if metadata['EXIF:GPSLongitudeRef'] == u'W':
        new_metadata['longitude'] *= -1

    if ('EXIF:GPSDateStamp' in metadata and
            'EXIF:GPSTimeStamp' in metadata):
        datetime_str = u'{} {}'.format(
            metadata['EXIF:GPSDateStamp'],
            metadata['EXIF:GPSTimeStamp'],
        )

        new_metadata['datetime'] = arrow.get(
            datetime_str,
            ['YYYY:MM:DD HH:mm:ss.SSS', 'YYYY:MM:DD HH:mm:ss'],
        ).datetime

    return new_metadata
<commit_msg>Add unique constraint to filename<commit_after># -*- coding: utf-8 -*-
"""Location database."""

import logging
import os

import arrow

from sqlalchemy import (
    Column,
    MetaData,
    Table,
    create_engine,
    select,
)
from sqlalchemy.types import (
    DateTime,
    Float,
    String,
)
from xdg import BaseDirectory

logger = logging.getLogger(__name__)

class Database(object):

    """Generic database object.

    This class is subclassed to provide additional functionality specific to
    artifacts and/or documents.

    :param db_filename: Path to the sqlite database file
    :type db_filename: str

    """

    def __init__(self, db_filename):
        """Connect to database and create session object."""
        self.db_filename = db_filename
        self.engine = create_engine(
            'sqlite:///{}'.format(db_filename),
            connect_args={'check_same_thread': False},
        )
        self.connection = None
        self.metadata = MetaData(bind=self.engine)

    def connect(self):
        """Create connection."""
        logger.debug('Connecting to SQLite database: %r', self.db_filename)
        self.connection = self.engine.connect()

    def disconnect(self):
        """Close connection."""
        assert not self.connection.closed
        logger.debug(
            'Disconnecting from SQLite database: %r', self.db_filename)
        self.connection.close()

    def __enter__(self):
        """Connect on entering context."""
        self.connect()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        """Disconnect on exiting context."""
        self.disconnect()

    def __getitem__(self, table_name):
        """Get table object in database.

        :param table_name: Name of the table
        :type table_name: str
        :return: Table object that can be used in queries
        :rtype: sqlalchemy.schema.Table

        """
        if not isinstance(table_name, basestring):
            raise TypeError('Unexpected table name: {}'.format(table_name))
        table = self.metadata.tables.get(table_name)
        if table is None:
            table = Table(table_name, self.metadata, autoload=True)
        return table

class LocationDB(Database):

    """Location database.

    Store location information from picture files into a sqlite database.

    """

    def __init__(self):
        """Create database if needed."""
        base_directory = BaseDirectory.save_data_path('pic2map')
        db_filename = os.path.join(base_directory, 'location.db')
        Database.__init__(self, db_filename)

        if os.path.isfile(db_filename):
            self.location_table = self['location']
        else:
            logger.debug('Creating location database %r...', db_filename)

            self.location_table = Table(
                'location',
                self.metadata,
                Column('filename', String, unique=True),
                Column('latitude', Float),
                Column('longitude', Float),
                Column('datetime', DateTime),
            )
            self.location_table.create()

    def insert(self, rows):
        """Insert rows in location table.

        :param rows: Rows to be inserted in the database
        :type rows: list(dict(str))

        """
        insert_query = self.location_table.insert()
        result = self.connection.execute(insert_query, rows)
        logger.debug('%d rows inserted', result.rowcount)

    def select_all(self):
        """Get all rows from the location table.

        :returns: Location information rows
        :rtype: sqlalchemy.engine.result.ResultProxy

        """
        select_query = select([self.location_table])
        result = self.connection.execute(select_query)
        return result

    def delete(self, directory):
        """Delete rows with a filename under a given directory.

        :param directory: Directory to be used as prefix in the delete query
        :type directory: str

        """
        table = self.location_table
        delete_query = (
            table.delete()
            .where(table.c.filename.startswith(directory))
        )
        result = self.connection.execute(delete_query)
        logger.debug('%d rows deleted', result.rowcount)
        return result

    def count(self):
        """Get number of rows in the database.

        :return: Number of rows
        :rtype: int

        """
        select_query = self.location_table.count()
        result = self.connection.execute(select_query)
        return result.scalar()

def transform_metadata_to_row(metadata):
    """Transform GPS metadata in database rows.

    :param metadata: GPS metadata as returned by exiftool
    :type metadata: dict(str)
    :returns: Database row
    :rtype: dict(str)

    """
    new_metadata = {
        'filename': metadata['SourceFile'],
        'latitude': metadata['EXIF:GPSLatitude'],
        'longitude': metadata['EXIF:GPSLongitude'],
    }

    if metadata['EXIF:GPSLatitudeRef'] == u'S':
        new_metadata['latitude'] *= -1
    if metadata['EXIF:GPSLongitudeRef'] == u'W':
        new_metadata['longitude'] *= -1

    if ('EXIF:GPSDateStamp' in metadata and
            'EXIF:GPSTimeStamp' in metadata):
        datetime_str = u'{} {}'.format(
            metadata['EXIF:GPSDateStamp'],
            metadata['EXIF:GPSTimeStamp'],
        )

        new_metadata['datetime'] = arrow.get(
            datetime_str,
            ['YYYY:MM:DD HH:mm:ss.SSS', 'YYYY:MM:DD HH:mm:ss'],
        ).datetime

    return new_metadata
<|endoftext|>


<commit_before>__version__ = "0.0.24"
<commit_msg>bump version<commit_after>__version__ = "0.0.25"
<|endoftext|>


<commit_before># coding: utf-8

# maposmatic, the web front-end of the MapOSMatic city map generation system
# Copyright (C) 2009  David Decotigny
# Copyright (C) 2009  Frédéric Lehobey
# Copyright (C) 2009  David Mentré
# Copyright (C) 2009  Maxime Petazzoni
# Copyright (C) 2009  Thomas Petazzoni
# Copyright (C) 2009  Gaël Utard

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Nominatim parsing + json export
# Note: we query nominatim in XML format because otherwise we cannot
# access the osm_id tag. Then we format it as json back to the
# javascript routines

"""
Simple API to query http://nominatim.openstreetmap.org
"""

import www.settings
import psycopg2
from urllib import urlencode
import urllib2
from xml.etree.ElementTree import parse as XMLTree

NOMINATIM_BASE_URL = "http://nominatim.openstreetmap.org/search/"

def query(query_text, with_polygons = False):
    """
    Query the nominatim service for the given city query and return a
    (python) list of entries for the given squery (eg. "Paris"). Each
    entry is a dictionary key -> value (value is always a
    string). When possible, we also try to uncover the OSM database
    IDs associated with the entries; in that case, an
    "ocitysmap_params" key is provided, which maps to a dictionary
    containing:
      - key "table": when "line" -> refers to table "planet_osm_line";
        when "polygon" -> "planet_osm_polygon"
      - key "id": ID of the OSM database entry
      - key "admin_level": The value stored in the OSM table for admin_level
    """
    entries = _fetch_entries(query_text, with_polygons)
    return _canonicalize_data(_retrieve_missing_data_from_GIS(entries))

def _fetch_entries(query_text, with_polygons):
    """
    Query the nominatim service for the given city query and return a
    (python) list of entries for the given squery (eg. "Paris"). Each
    entry is a dictionary key -> value (value is always a
    string).
    """
    # For some reason, the "xml" nominatim output is ALWAYS used, even
    # though we will later (in views.py) transform this into
    # json. This is because we know that this xml output is correct
    # and complete (at least the "osm_id" field is missing from the
    # json output)
    query_tags = dict(q=query_text.encode("UTF-8"),
                      format='xml', addressdetails=1)
    if with_polygons:
        query_tags['polygon']=1

    qdata = urlencode(query_tags)
    f = urllib2.urlopen(url="%s?%s" % (NOMINATIM_BASE_URL, qdata))

    result = []
    for place in XMLTree(f).getroot().getchildren():
        attribs = dict(place.attrib)
        for elt in place.getchildren():
            attribs[elt.tag] = elt.text
        result.append(attribs)

    return result

def _canonicalize_data(data):
    """
    Take a structure containing strings (dict, list, scalars, ...)
    and convert it into the same structure with the proper conversions
    to float or integers, etc.
    """
    if type(data) is tuple:
        return tuple(_canonicalize_data(x) for x in data)
    elif type(data) is list:
        return [_canonicalize_data(x) for x in data]
    elif type(data) is dict:
        return dict([(_canonicalize_data(k),
                      _canonicalize_data(v)) for k,v in data.iteritems()])
    try:
        return int(data)
    except ValueError:
        try:
            return float(data)
        except ValueError:
            pass
    return data

def _retrieve_missing_data_from_GIS(entries):
    """
    Try to retrieve additional OSM information for the given nominatim
    entries. Among the information, we try to determine the real ID in
    an OSM table for each of these entries. All these additional data
    are stored in the "ocitysmap_params" key of the entry, which maps
    to a dictionary containing:
      - key "table": when "line" -> refers to table "planet_osm_line";
        when "polygon" -> "planet_osm_polygon"
      - key "id": ID of the OSM database entry
      - key "admin_level": The value stored in the OSM table for admin_level
    """
    if not www.settings.has_gis_database():
        return entries

    try:
        conn = psycopg2.connect("dbname='%s' user='%s' host='%s' password='%s'" %
                                (www.settings.GIS_DATABASE_NAME,
                                 www.settings.DATABASE_USER,
                                 www.settings.DATABASE_HOST,
                                 www.settings.DATABASE_PASSWORD))
    except psycopg2.OperationalError:
        return entries

    # Nominatim returns a field "osm_id" for each result
    # entry. Depending on the type of the entry, it can point to
    # various database entries. For admin boundaries, osm_id is
    # supposed to point to either the 'polygon' or the 'line'
    # table. Usually, the database entry ID in the table is derived by
    # the "relation" items by osm2pgsql, which assigns to that ID the
    # opposite of osm_id... But we still consider that it could be the
    # real osm_id (not its opposite). Let's have fun...

    # Will sort the entries so that the admin boundaries appear
    # fdirst, then cities, towns, etc.
    unsorted_entries = []
    admin_boundary_names = set()
    PLACE_RANKS = { 'city': 20, 'town': 30, 'municipality': 40,
                    'village': 50, 'hamlet': 60, 'suburb': 70,
                    'island': 80, 'islet': 90, 'locality': 100 }
    ADMIN_LEVEL_RANKS = { '8': 0, '7': 1, '6': 2, '5':3 } # level 8 is best !
    try:
        cursor = conn.cursor()
        for entry in entries:
            # Should we try to lookup the id in the OSM DB ?
            lookup_OSM = False

            # Highest rank = last in the output
            entry_rank = 1000

            # Try to determine the order in which this entry should appear
            if entry.get("class") == "boundary":
                if entry.get("type") == "administrative":
                    entry_rank = 10
                    admin_boundary_names.add(entry.get("display_name", 42))
                    lookup_OSM = True
                else:
                    # Just don't try to lookup any additional
                    # information from OSM when the nominatim entry is
                    # not an administrative boundary
                    continue
            elif entry.get("class") == "place":
                try:
                    entry_rank = PLACE_RANKS[entry.get("type")]
                except KeyError:
                    # Will ignore all the other place tags
                    continue
            else:
                # We ignore all the other classes
                continue

            # Try to lookup in the OSM DB, when needed and when it
            # makes sense (ie. the data is coming from a relation)
            if lookup_OSM and (entry.get('osm_type') == "relation"):
                for table_name in ("polygon", "line"):
                    # Lookup the polygon/line table for both osm_id and
                    # the opposite of osm_id
                    cursor.execute("""select osm_id, admin_level
                                      from planet_osm_%s
                                      where osm_id = -%s""" \
                                       % (table_name,entry["osm_id"]))
                    result = tuple(set(cursor.fetchall()))
                    if len(result) == 1:
                        osm_id, admin_level = result[0]
                        entry["ocitysmap_params"] \
                            = dict(table=table_name, id=osm_id,
                                   admin_level=admin_level)
                        # Make these first in list, priviledging level 8
                        entry_rank = ADMIN_LEVEL_RANKS.get(admin_level,9)
                        break

            # Register this entry for the results
            unsorted_entries.append((entry_rank, entry))

        # Some cleanup
        cursor.close()
    finally:
        conn.close()

    # Sort the entries according to their rank
    sorted_entries = [entry for rank,entry in sorted(unsorted_entries,
                                                     key=lambda kv: kv[0])]

    # Remove those non-admin-boundaries having the same name as an
    # admin boundary
    retval = []
    for e in sorted_entries:
        if e.get("class") != "boundary" or e.get("type") != "administrative":
            if e.get("display_name") in admin_boundary_names:
                continue
        retval.append(e)

    return retval

if __name__ == "__main__":
    import pprint, sys
    pp = pprint.PrettyPrinter(indent=4)

    for city in sys.argv[1:]:
        print "###### %s:" % city
        pp.pprint(query(city))
<commit_msg>for similar admin_level, sort according to the city area<commit_after># coding: utf-8

# maposmatic, the web front-end of the MapOSMatic city map generation system
# Copyright (C) 2009  David Decotigny
# Copyright (C) 2009  Frédéric Lehobey
# Copyright (C) 2009  David Mentré
# Copyright (C) 2009  Maxime Petazzoni
# Copyright (C) 2009  Thomas Petazzoni
# Copyright (C) 2009  Gaël Utard

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Nominatim parsing + json export
# Note: we query nominatim in XML format because otherwise we cannot
# access the osm_id tag. Then we format it as json back to the
# javascript routines

"""
Simple API to query http://nominatim.openstreetmap.org
"""

import www.settings
import psycopg2
from urllib import urlencode
import urllib2
from xml.etree.ElementTree import parse as XMLTree

NOMINATIM_BASE_URL = "http://nominatim.openstreetmap.org/search/"

def query(query_text, with_polygons = False):
    """
    Query the nominatim service for the given city query and return a
    (python) list of entries for the given squery (eg. "Paris"). Each
    entry is a dictionary key -> value (value is always a
    string). When possible, we also try to uncover the OSM database
    IDs associated with the entries; in that case, an
    "ocitysmap_params" key is provided, which maps to a dictionary
    containing:
      - key "table": when "line" -> refers to table "planet_osm_line";
        when "polygon" -> "planet_osm_polygon"
      - key "id": ID of the OSM database entry
      - key "admin_level": The value stored in the OSM table for admin_level
    """
    entries = _fetch_entries(query_text, with_polygons)
    return _canonicalize_data(_retrieve_missing_data_from_GIS(entries))

def _fetch_entries(query_text, with_polygons):
    """
    Query the nominatim service for the given city query and return a
    (python) list of entries for the given squery (eg. "Paris"). Each
    entry is a dictionary key -> value (value is always a
    string).
    """
    # For some reason, the "xml" nominatim output is ALWAYS used, even
    # though we will later (in views.py) transform this into
    # json. This is because we know that this xml output is correct
    # and complete (at least the "osm_id" field is missing from the
    # json output)
    query_tags = dict(q=query_text.encode("UTF-8"),
                      format='xml', addressdetails=1)
    if with_polygons:
        query_tags['polygon']=1

    qdata = urlencode(query_tags)
    f = urllib2.urlopen(url="%s?%s" % (NOMINATIM_BASE_URL, qdata))

    result = []
    for place in XMLTree(f).getroot().getchildren():
        attribs = dict(place.attrib)
        for elt in place.getchildren():
            attribs[elt.tag] = elt.text
        result.append(attribs)

    return result

def _canonicalize_data(data):
    """
    Take a structure containing strings (dict, list, scalars, ...)
    and convert it into the same structure with the proper conversions
    to float or integers, etc.
    """
    if type(data) is tuple:
        return tuple(_canonicalize_data(x) for x in data)
    elif type(data) is list:
        return [_canonicalize_data(x) for x in data]
    elif type(data) is dict:
        return dict([(_canonicalize_data(k),
                      _canonicalize_data(v)) for k,v in data.iteritems()])
    try:
        return int(data)
    except ValueError:
        try:
            return float(data)
        except ValueError:
            pass
    return data

def _retrieve_missing_data_from_GIS(entries):
    """
    Try to retrieve additional OSM information for the given nominatim
    entries. Among the information, we try to determine the real ID in
    an OSM table for each of these entries. All these additional data
    are stored in the "ocitysmap_params" key of the entry, which maps
    to a dictionary containing:
      - key "table": when "line" -> refers to table "planet_osm_line";
        when "polygon" -> "planet_osm_polygon"
      - key "id": ID of the OSM database entry
      - key "admin_level": The value stored in the OSM table for admin_level
    """
    if not www.settings.has_gis_database():
        return entries

    try:
        conn = psycopg2.connect("dbname='%s' user='%s' host='%s' password='%s'" %
                                (www.settings.GIS_DATABASE_NAME,
                                 www.settings.DATABASE_USER,
                                 www.settings.DATABASE_HOST,
                                 www.settings.DATABASE_PASSWORD))
    except psycopg2.OperationalError:
        return entries

    # Nominatim returns a field "osm_id" for each result
    # entry. Depending on the type of the entry, it can point to
    # various database entries. For admin boundaries, osm_id is
    # supposed to point to either the 'polygon' or the 'line'
    # table. Usually, the database entry ID in the table is derived by
    # the "relation" items by osm2pgsql, which assigns to that ID the
    # opposite of osm_id... But we still consider that it could be the
    # real osm_id (not its opposite). Let's have fun...

    # Will sort the entries so that the admin boundaries appear first,
    # then cities, towns, etc. Second order: larger cities
    # (ie. greater way_area) are listed first
    unsorted_entries = []
    admin_boundary_names = set()
    PLACE_RANKS = { 'city': 20, 'town': 30, 'municipality': 40,
                    'village': 50, 'hamlet': 60, 'suburb': 70,
                    'island': 80, 'islet': 90, 'locality': 100 }
    ADMIN_LEVEL_RANKS = { '8': 0, '7': 1, '6': 2, '5':3 } # level 8 is best !
    try:
        cursor = conn.cursor()
        for entry in entries:
            # Should we try to lookup the id in the OSM DB ?
            lookup_OSM = False

            # Highest rank = last in the output
            entry_rank = (1000,0) # tuple (sort rank, -area)

            # Try to determine the order in which this entry should appear
            if entry.get("class") == "boundary":
                if entry.get("type") == "administrative":
                    entry_rank = (10,0)
                    admin_boundary_names.add(entry.get("display_name", 42))
                    lookup_OSM = True
                else:
                    # Just don't try to lookup any additional
                    # information from OSM when the nominatim entry is
                    # not an administrative boundary
                    continue
            elif entry.get("class") == "place":
                try:
                    entry_rank = (PLACE_RANKS[entry.get("type")],0)
                except KeyError:
                    # Will ignore all the other place tags
                    continue
            else:
                # We ignore all the other classes
                continue

            # Try to lookup in the OSM DB, when needed and when it
            # makes sense (ie. the data is coming from a relation)
            if lookup_OSM and (entry.get('osm_type') == "relation"):
                for table_name in ("polygon", "line"):
                    # Lookup the polygon/line table for both osm_id and
                    # the opposite of osm_id
                    cursor.execute("""select osm_id, admin_level, way_area
                                      from planet_osm_%s
                                      where osm_id = -%s""" \
                                       % (table_name,entry["osm_id"]))
                    result = tuple(set(cursor.fetchall()))
                    if len(result) == 1:
                        osm_id, admin_level, way_area = result[0]
                        entry["ocitysmap_params"] \
                            = dict(table=table_name, id=osm_id,
                                   admin_level=admin_level,
                                   way_area=way_area)
                        # Make these first in list, priviledging level 8
                        entry_rank = (ADMIN_LEVEL_RANKS.get(admin_level,9),-way_area)
                        break

            # Register this entry for the results
            unsorted_entries.append((entry_rank, entry))

        # Some cleanup
        cursor.close()
    finally:
        conn.close()

    # Sort the entries according to their rank
    sorted_entries = [entry for rank,entry in sorted(unsorted_entries,
                                                     key=lambda kv: kv[0])]

    # Remove those non-admin-boundaries having the same name as an
    # admin boundary
    retval = []
    for e in sorted_entries:
        if e.get("class") != "boundary" or e.get("type") != "administrative":
            if e.get("display_name") in admin_boundary_names:
                continue
        retval.append(e)

    return retval

if __name__ == "__main__":
    import pprint, sys
    pp = pprint.PrettyPrinter(indent=4)

    for city in sys.argv[1:]:
        print "###### %s:" % city
        pp.pprint(query(city))
<|endoftext|>


<commit_before>import sys
import urllib2
from BeautifulSoup import BeautifulSoup
import os.path
import shutil
import tarfile
import bson
import json
from glob import glob
import re
import shelve
from mpi4py import MPI
from array import array
import time
import socket

class Progress(object):
    def __init__(self, message, total_size):
        self.message = message
        self.total_size = total_size
        self.previous_percent = -1

    def show_progress(self, current_size):
        percent = int(current_size * 100. / self.total_size)
        if percent != self.previous_percent:
            self.print_progress('{:3d}%'.format(percent))
            self.previous_percent = percent

    def print_progress(self, text):
        status = '{} [{}]'.format(self.message, text)
        if MPI.COMM_WORLD.size > 1:
            status = '#{}. {}\n'.format(MPI.COMM_WORLD.rank, status)
        else:
            # Clear previous message and progress
            clear = chr(8) * (len(self.message) + 7)
            if self.previous_percent != -1:
                sys.stdout.write(clear)

        sys.stdout.write(status)
        sys.stdout.flush()

    def finish(self):
        self.print_progress("finished")
        sys.stdout.write('\n')

class ProgressFile(file, Progress):
    def __init__(self, *a, **kw):
        message = kw.pop('message', None)

        file.__init__(self, *a, **kw)

        if message is None:
            message = 'Reading file {}'.format(self.name)

        Progress.__init__(self, message, os.path.getsize(self.name))

    def read(self, size):
        self.show_progress(self.tell())
        return file.read(self, size)

    def close(self):
        if not self.closed:
            self.finish()
        file.close(self)

class Preprocessor(object):
    DOWNLOADS_URL = "http://ghtorrent.org/downloads/"
    BSON_FILE_DIR = "dump/github/"

    def __init__(self):
        self.dataset = ''
        self.bson_file = ''
        self.path = ''
        self.keep_fields = []

    def preprocess(self):
        self.get_bson()
        self.convert_bson()

    def get_bson(self):
        if not os.path.isfile(self.path + self.dataset + '.tar.gz'):
            self.download(self.path + self.dataset + '.tar.gz')
        if not os.path.isfile(self.bson_file + '.bson'):
            self.extract()

    def download(self, target):
        message = 'Downloading "{}" dataset'.format(self.dataset)
        self.download_file(self.DOWNLOADS_URL + self.dataset + '.tar.gz', target, message=message)

    def download_file(self, url, target, message=None):
        stream = urllib2.urlopen(url)
        file = open(target, 'wb')
        file_size = int(stream.info().getheaders('Content-Length')[0])
        downloaded_size = 0
        block_size = 8192

        if message is None:
            message = 'Downloading "{}"'.format(url)

        progress = Progress(message, file_size)

        while True:
            buffer = stream.read(block_size)
            if not buffer:
                break

            downloaded_size += len(buffer)
            file.write(buffer)
            progress.show_progress(downloaded_size)

        progress.finish()
        file.close()

    def extract(self):
        message = 'Untarring "{}" dataset'.format(self.dataset)
        file = ProgressFile(self.path + self.dataset + '.tar.gz', message=message)
        tar = tarfile.open(fileobj=file)
        if self.path != "":
            tar.extractall(self.path)
        else:
            tar.extractall()
        tar.close()
        file.close()

    def convert_bson(self):
        raise NotImplementedError("Cannot call convert_bson on the base class: a subclass must implement this method instead")

class Commit_Comments_Preprocessor(Preprocessor):
    def __init__(self, group):
        super(Commit_Comments_Preprocessor, self).__init__()
        self.dataset = 'commit_comments-dump.2015-01-29'
        self.bson_file = self.BSON_FILE_DIR + 'commit_comments.bson'
        self.group = group
        self.keep_fields = ['id', 'body']
        if group not in self.keep_fields:
            self.keep_fields.append(group)

    def is_latin(self, string):
        try:
            string.encode('ascii')
        except UnicodeEncodeError:
            return False

        return True

    def merge_shelves(self):
        files = sorted(glob('languages-*.shelf'))
        count = len(files)
        message = 'Merging partial shelves'
        progress = Progress(message, count)

        merged = shelve.open('languages.shelf')
        for index in range(count):
            progress.show_progress(index)
            shelf = shelve.open(files[index])
            merged.update(shelf)
            shelf.close()
            os.remove(files[index])
        merged.close()
        progress.finish()

    def convert_bson(self):
        output = open(self.dataset + '.json', 'wb')
        message = 'Converting BSON "{}" and removing unused fields'.format(self.dataset)
        bson_file = ProgressFile(self.bson_file, 'rb', message=message)

        if os.path.isfile('languages-1.shelf'):
            # We have at least one partial shelve waiting to be merged.
            self.merge_shelves()

        if os.path.isfile('languages.shelf'):
            languages = shelve.open('languages.shelf', writeback=True)
        else:
            languages = {}

        # Read every BSON object as an iterator to save memory.
        for raw_json in bson.decode_file_iter(bson_file):
            if not self.is_latin(raw_json['body']):
                continue

            preprocessed_json = {}
            repository = str(re.search(r"repos/([^/]+/[^/]+)(/|$)", raw_json['url']).group(1))
            raw_json['language'] = ''
            if repository in languages:
                raw_json['language'] = languages[repository]
            for item in self.keep_fields:
                preprocessed_json[item] = raw_json[item]

            json.dump(preprocessed_json, output)
            output.write('\n')

        output.close()
        bson_file.close()
        os.remove(self.bson_file)
        os.removedirs(self.BSON_FILE_DIR)

class Repos_Preprocessor(Preprocessor):
    def __init__(self, process_id, date, path=""):
        super(Repos_Preprocessor, self).__init__()
        if path != "" and path[-1] != "/":
            path = path + "/"

        self.path = path + str(process_id) + '/'
        if not os.path.exists(self.path):
            os.makedirs(self.path, 0700)

        self.dataset = 'repos-dump.' + date
        self.bson_file = self.path + self.BSON_FILE_DIR + 'repos.bson'
        self.process_id = str(process_id)

    def convert_bson(self):
        message = 'Converting BSON "{}" to language shelf #{}'.format(self.dataset, self.process_id)
        bson_file = ProgressFile(self.bson_file, 'rb', message=message)
        shelf_name = 'languages-' + self.process_id + '.shelf'
        shelf_path = self.path + shelf_name # Local running path
        languages = shelve.open(shelf_path, writeback=True)

        # Read every BSON object as an iterator to save memory.
        for raw_json in bson.decode_file_iter(bson_file):
            repository = raw_json['full_name'].encode('utf-8')
            language = raw_json['language'].encode('utf-8') if raw_json['language'] is not None else ''
            languages[repository] = language

        languages.close()
        bson_file.close()
        os.remove(self.bson_file)
        os.removedirs(self.path + self.BSON_FILE_DIR)
        if self.path != "":
            print('#{}. Moving shelf "{}" to shared directory...'.format(MPI.COMM_WORLD.rank, shelf_name))
            if os.path.exists(shelf_name):
                os.remove(shelf_name)
            shutil.move(shelf_name, '.')

class Process(object):
    def __init__(self, path):
        self.path = path
        self.comm = MPI.COMM_WORLD
        self.process_id = self.comm.rank
        self.num_processes = self.comm.size
        # MPI buffer
        self.ready = array('c', '\0')

    def execute(self):
        if self.process_id == 0:
            if self.num_processes == 1:
                self.run_sequential()
            else:
                self.run_master()
        else:
            self.run_process()

    def run_sequential(self):
        # Fetch the repo dumps from the GHTorrent website and
        # perform sequentially
        dates = self.get_downloads('repos-dump')
        for tag in xrange(len(dates)):
            preprocessor = Repos_Preprocessor(tag, dates[tag]['date'], self.path)
            preprocessor.preprocess()

    def run_master(self):
        print('MASTER on node {}'.format(socket.gethostname()))

        # Fetch the repo dumps from the GHTorrent website and
        # process them in parallel on the other processes.
        # Start with the largest data sets so that we have better balancing
        dates = self.get_downloads('repos-dump')
        dates = sorted(dates, key=lambda v: v['size'], reverse=True)

        print('MASTER: Waiting to distribute {} jobs'.format(len(dates)))
        # Automatically balance the jobs across the processes by sending jobs
        # to processes that tell us they are free.
        for tag in xrange(len(dates)):
            ready = '\0'
            while ready != '\1':
                (ready, status) = self.wait_ready()

            pid = status.Get_source()

            # A process is ready to receive, so send a job
            print('MASTER: Process {} receives job {}'.format(pid, tag))
            self.comm.send(dates[tag]['date'], dest=pid, tag=tag)

        print('MASTER: Done distributing jobs, starting finish run')
        self.finish_master()
        print('MASTER: Done')

    def finish_master(self):
        # We run another cycle through all the other processes to let them know
        # they are done.
        for i in xrange(self.num_processes - 1):
            ready = '\0'
            while ready != '\1':
                (ready, status) = self.wait_ready()

            # We are done sending jobs, so tell the process that it is done
            pid = status.Get_source()
            print('MASTER: Process {} is done ({} of {})'.format(pid, i+1, self.num_processes))
            self.comm.send("", dest=pid, tag=i)

    def run_process(self):
        print('PROCESS {} on node {}'.format(self.process_id, socket.gethostname()))

        # Process repo dumps as jobs that we receive from the master.
        # Keep on running until the master lets us know we are done.
        while True:
            # Let the master know that this process is ready
            self.comm.Send(array('c', '\1'), dest=0, tag=self.process_id)
            print('PROCESS {}: Sent ready message'.format(self.process_id))

            # Execute only the preprocessor for this particular job
            # if it received the signal to run.
            status = MPI.Status()
            date = self.comm.recv(source=0, tag=MPI.ANY_TAG, status=status)
            tag = status.Get_tag()
            if date == "":
                print('PROCESS {}: Done'.format(self.process_id))
                break

            print('PROCESS {}: Received job {} with date {} for preprocessing'.format(self.process_id, tag, date))
            preprocessor = Repos_Preprocessor(tag, date, self.path)
            preprocessor.preprocess()

    def wait_ready(self):
        # Wait for processes to be ready. We poll a receive of a message from
        # any process. In order to reduce CPU usage of the idle master, we use
        # our own sleep loop.
        status = MPI.Status()
        req = self.comm.Irecv(self.ready, source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG)
        while not req.Test(status=status):
            time.sleep(1)

        # Finish the request
        req.Wait()
        return self.ready.tostring(), status

    def get_downloads(self, prefix):
        dates = []
        html_page = urllib2.urlopen(Preprocessor.DOWNLOADS_URL)
        soup = BeautifulSoup(html_page)
        for link in soup.findAll('a'):
            href = link.get('href')
            if href.startswith(prefix):
                date = href[len(prefix)+1:-7]
                attrs = re.split('\s\s+', link.nextSibling)
                dates.append({'date': date, 'size': int(attrs[2])})

        return dates

def main(argv):
    task = argv[0] if len(argv) > 0 else "commit_comments"
    group = argv[1] if len(argv) > 1 else "id"
    path = argv[2] if len(argv) > 2 else ""

    # Buffers of MPI
    date = ""
    ready = array('c', '\0')

    if task == "repos":
        if group == "language" and not os.path.isfile('languages.shelf'):
            process = Process(path)
            process.execute()
        else:
            print('Nothing to be done for task {}, group {}.'.format(task, group))
    elif task == "commit_comments":
        commit_comments = Commit_Comments_Preprocessor(group)
        commit_comments.preprocess()
    else:
        print("Unrecognized value for 'task': '{}'".format(task))
        print("Must be either 'repos' or 'commit_comments'")
        print("Usage: [mpiexec -n <num_processes>] python preprocess.py <task> [group]")

if __name__ == "__main__":
    main(sys.argv[1:])
<commit_msg>preprocess: Fixed move<commit_after>import sys
import urllib2
from BeautifulSoup import BeautifulSoup
import os.path
import shutil
import tarfile
import bson
import json
from glob import glob
import re
import shelve
from mpi4py import MPI
from array import array
import time
import socket

class Progress(object):
    def __init__(self, message, total_size):
        self.message = message
        self.total_size = total_size
        self.previous_percent = -1

    def show_progress(self, current_size):
        percent = int(current_size * 100. / self.total_size)
        if percent != self.previous_percent:
            self.print_progress('{:3d}%'.format(percent))
            self.previous_percent = percent

    def print_progress(self, text):
        status = '{} [{}]'.format(self.message, text)
        if MPI.COMM_WORLD.size > 1:
            status = '#{}. {}\n'.format(MPI.COMM_WORLD.rank, status)
        else:
            # Clear previous message and progress
            clear = chr(8) * (len(self.message) + 7)
            if self.previous_percent != -1:
                sys.stdout.write(clear)

        sys.stdout.write(status)
        sys.stdout.flush()

    def finish(self):
        self.print_progress("finished")
        sys.stdout.write('\n')

class ProgressFile(file, Progress):
    def __init__(self, *a, **kw):
        message = kw.pop('message', None)

        file.__init__(self, *a, **kw)

        if message is None:
            message = 'Reading file {}'.format(self.name)

        Progress.__init__(self, message, os.path.getsize(self.name))

    def read(self, size):
        self.show_progress(self.tell())
        return file.read(self, size)

    def close(self):
        if not self.closed:
            self.finish()
        file.close(self)

class Preprocessor(object):
    DOWNLOADS_URL = "http://ghtorrent.org/downloads/"
    BSON_FILE_DIR = "dump/github/"

    def __init__(self):
        self.dataset = ''
        self.bson_file = ''
        self.path = ''
        self.keep_fields = []

    def preprocess(self):
        self.get_bson()
        self.convert_bson()

    def get_bson(self):
        if not os.path.isfile(self.path + self.dataset + '.tar.gz'):
            self.download(self.path + self.dataset + '.tar.gz')
        if not os.path.isfile(self.bson_file + '.bson'):
            self.extract()

    def download(self, target):
        message = 'Downloading "{}" dataset'.format(self.dataset)
        self.download_file(self.DOWNLOADS_URL + self.dataset + '.tar.gz', target, message=message)

    def download_file(self, url, target, message=None):
        stream = urllib2.urlopen(url)
        file = open(target, 'wb')
        file_size = int(stream.info().getheaders('Content-Length')[0])
        downloaded_size = 0
        block_size = 8192

        if message is None:
            message = 'Downloading "{}"'.format(url)

        progress = Progress(message, file_size)

        while True:
            buffer = stream.read(block_size)
            if not buffer:
                break

            downloaded_size += len(buffer)
            file.write(buffer)
            progress.show_progress(downloaded_size)

        progress.finish()
        file.close()

    def extract(self):
        message = 'Untarring "{}" dataset'.format(self.dataset)
        file = ProgressFile(self.path + self.dataset + '.tar.gz', message=message)
        tar = tarfile.open(fileobj=file)
        if self.path != "":
            tar.extractall(self.path)
        else:
            tar.extractall()
        tar.close()
        file.close()

    def convert_bson(self):
        raise NotImplementedError("Cannot call convert_bson on the base class: a subclass must implement this method instead")

class Commit_Comments_Preprocessor(Preprocessor):
    def __init__(self, group):
        super(Commit_Comments_Preprocessor, self).__init__()
        self.dataset = 'commit_comments-dump.2015-01-29'
        self.bson_file = self.BSON_FILE_DIR + 'commit_comments.bson'
        self.group = group
        self.keep_fields = ['id', 'body']
        if group not in self.keep_fields:
            self.keep_fields.append(group)

    def is_latin(self, string):
        try:
            string.encode('ascii')
        except UnicodeEncodeError:
            return False

        return True

    def merge_shelves(self):
        files = sorted(glob('languages-*.shelf'))
        count = len(files)
        message = 'Merging partial shelves'
        progress = Progress(message, count)

        merged = shelve.open('languages.shelf')
        for index in range(count):
            progress.show_progress(index)
            shelf = shelve.open(files[index])
            merged.update(shelf)
            shelf.close()
            os.remove(files[index])
        merged.close()
        progress.finish()

    def convert_bson(self):
        output = open(self.dataset + '.json', 'wb')
        message = 'Converting BSON "{}" and removing unused fields'.format(self.dataset)
        bson_file = ProgressFile(self.bson_file, 'rb', message=message)

        if os.path.isfile('languages-1.shelf'):
            # We have at least one partial shelve waiting to be merged.
            self.merge_shelves()

        if os.path.isfile('languages.shelf'):
            languages = shelve.open('languages.shelf', writeback=True)
        else:
            languages = {}

        # Read every BSON object as an iterator to save memory.
        for raw_json in bson.decode_file_iter(bson_file):
            if not self.is_latin(raw_json['body']):
                continue

            preprocessed_json = {}
            repository = str(re.search(r"repos/([^/]+/[^/]+)(/|$)", raw_json['url']).group(1))
            raw_json['language'] = ''
            if repository in languages:
                raw_json['language'] = languages[repository]
            for item in self.keep_fields:
                preprocessed_json[item] = raw_json[item]

            json.dump(preprocessed_json, output)
            output.write('\n')

        output.close()
        bson_file.close()
        os.remove(self.bson_file)
        os.removedirs(self.BSON_FILE_DIR)

class Repos_Preprocessor(Preprocessor):
    def __init__(self, process_id, date, path=""):
        super(Repos_Preprocessor, self).__init__()
        if path != "" and path[-1] != "/":
            path = path + "/"

        self.path = path + str(process_id) + '/'
        if not os.path.exists(self.path):
            os.makedirs(self.path, 0700)

        self.dataset = 'repos-dump.' + date
        self.bson_file = self.path + self.BSON_FILE_DIR + 'repos.bson'
        self.process_id = str(process_id)

    def convert_bson(self):
        message = 'Converting BSON "{}" to language shelf #{}'.format(self.dataset, self.process_id)
        bson_file = ProgressFile(self.bson_file, 'rb', message=message)
        shelf_name = 'languages-' + self.process_id + '.shelf'
        shelf_path = self.path + shelf_name # Local running path
        languages = shelve.open(shelf_path, writeback=True)

        # Read every BSON object as an iterator to save memory.
        for raw_json in bson.decode_file_iter(bson_file):
            repository = raw_json['full_name'].encode('utf-8')
            language = raw_json['language'].encode('utf-8') if raw_json['language'] is not None else ''
            languages[repository] = language

        languages.close()
        bson_file.close()
        os.remove(self.bson_file)
        os.removedirs(self.path + self.BSON_FILE_DIR)
        if self.path != "":
            print('#{}. Moving shelf "{}" to shared directory...'.format(MPI.COMM_WORLD.rank, shelf_name))
            if os.path.exists(shelf_name):
                os.remove(shelf_name)
            shutil.move(shelf_path, '.')

class Process(object):
    def __init__(self, path):
        self.path = path
        self.comm = MPI.COMM_WORLD
        self.process_id = self.comm.rank
        self.num_processes = self.comm.size
        # MPI buffer
        self.ready = array('c', '\0')

    def execute(self):
        if self.process_id == 0:
            if self.num_processes == 1:
                self.run_sequential()
            else:
                self.run_master()
        else:
            self.run_process()

    def run_sequential(self):
        # Fetch the repo dumps from the GHTorrent website and
        # perform sequentially
        dates = self.get_downloads('repos-dump')
        for tag in xrange(len(dates)):
            preprocessor = Repos_Preprocessor(tag, dates[tag]['date'], self.path)
            preprocessor.preprocess()

    def run_master(self):
        print('MASTER on node {}'.format(socket.gethostname()))

        # Fetch the repo dumps from the GHTorrent website and
        # process them in parallel on the other processes.
        # Start with the largest data sets so that we have better balancing
        dates = self.get_downloads('repos-dump')
        dates = sorted(dates, key=lambda v: v['size'], reverse=True)

        print('MASTER: Waiting to distribute {} jobs'.format(len(dates)))
        # Automatically balance the jobs across the processes by sending jobs
        # to processes that tell us they are free.
        for tag in xrange(len(dates)):
            ready = '\0'
            while ready != '\1':
                (ready, status) = self.wait_ready()

            pid = status.Get_source()

            # A process is ready to receive, so send a job
            print('MASTER: Process {} receives job {}'.format(pid, tag))
            self.comm.send(dates[tag]['date'], dest=pid, tag=tag)

        print('MASTER: Done distributing jobs, starting finish run')
        self.finish_master()
        print('MASTER: Done')

    def finish_master(self):
        # We run another cycle through all the other processes to let them know
        # they are done.
        for i in xrange(self.num_processes - 1):
            ready = '\0'
            while ready != '\1':
                (ready, status) = self.wait_ready()

            # We are done sending jobs, so tell the process that it is done
            pid = status.Get_source()
            print('MASTER: Process {} is done ({} of {})'.format(pid, i+1, self.num_processes))
            self.comm.send("", dest=pid, tag=i)

    def run_process(self):
        print('PROCESS {} on node {}'.format(self.process_id, socket.gethostname()))

        # Process repo dumps as jobs that we receive from the master.
        # Keep on running until the master lets us know we are done.
        while True:
            # Let the master know that this process is ready
            self.comm.Send(array('c', '\1'), dest=0, tag=self.process_id)
            print('PROCESS {}: Sent ready message'.format(self.process_id))

            # Execute only the preprocessor for this particular job
            # if it received the signal to run.
            status = MPI.Status()
            date = self.comm.recv(source=0, tag=MPI.ANY_TAG, status=status)
            tag = status.Get_tag()
            if date == "":
                print('PROCESS {}: Done'.format(self.process_id))
                break

            print('PROCESS {}: Received job {} with date {} for preprocessing'.format(self.process_id, tag, date))
            preprocessor = Repos_Preprocessor(tag, date, self.path)
            preprocessor.preprocess()

    def wait_ready(self):
        # Wait for processes to be ready. We poll a receive of a message from
        # any process. In order to reduce CPU usage of the idle master, we use
        # our own sleep loop.
        status = MPI.Status()
        req = self.comm.Irecv(self.ready, source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG)
        while not req.Test(status=status):
            time.sleep(1)

        # Finish the request
        req.Wait()
        return self.ready.tostring(), status

    def get_downloads(self, prefix):
        dates = []
        html_page = urllib2.urlopen(Preprocessor.DOWNLOADS_URL)
        soup = BeautifulSoup(html_page)
        for link in soup.findAll('a'):
            href = link.get('href')
            if href.startswith(prefix):
                date = href[len(prefix)+1:-7]
                attrs = re.split('\s\s+', link.nextSibling)
                dates.append({'date': date, 'size': int(attrs[2])})

        return dates

def main(argv):
    task = argv[0] if len(argv) > 0 else "commit_comments"
    group = argv[1] if len(argv) > 1 else "id"
    path = argv[2] if len(argv) > 2 else ""

    # Buffers of MPI
    date = ""
    ready = array('c', '\0')

    if task == "repos":
        if group == "language" and not os.path.isfile('languages.shelf'):
            process = Process(path)
            process.execute()
        else:
            print('Nothing to be done for task {}, group {}.'.format(task, group))
    elif task == "commit_comments":
        commit_comments = Commit_Comments_Preprocessor(group)
        commit_comments.preprocess()
    else:
        print("Unrecognized value for 'task': '{}'".format(task))
        print("Must be either 'repos' or 'commit_comments'")
        print("Usage: [mpiexec -n <num_processes>] python preprocess.py <task> [group]")

if __name__ == "__main__":
    main(sys.argv[1:])
<|endoftext|>