<commit_before>#!/usr/bin/env python

from signal import signal, SIGPIPE, SIG_DFL
signal(SIGPIPE, SIG_DFL)

import argparse
import csv
from collections import defaultdict
import concurrent.futures
import operator
import networkx as nx
import sys
import pymorphy2
from functools import lru_cache

parser = argparse.ArgumentParser()
parser.add_argument('--freq', required=True, type=argparse.FileType('r'))
parser.add_argument('-n', nargs='?', type=int, default=300)
parser.add_argument('-k', nargs='?', type=int, default=3)
parser.add_argument('--skip', nargs='?', type=int, default=0)
parser.add_argument('--no-inflection', dest='inflection', action='store_false')
parser.add_argument('path', nargs='+')
args = parser.parse_args()

def sanitize(s):
    return s.lower().replace(' ', '_')

with args.freq as f:
    reader = csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE)
    freq = {sanitize(row['Lemma']): float(row['Freq']) for row in reader if row['PoS'] == 's'}

def top(data, n, skip=0, reverse=False):
    head = sorted(data.items(), key=operator.itemgetter(1), reverse=reverse)
    return [word for i, (word, _) in enumerate(head) if i < skip + n and i >= skip]

lexicon = top(freq, args.n, args.skip, reverse=True)

def isas(path):
    G = nx.DiGraph()

    with open(path, newline='') as f:
        reader = csv.reader(f, delimiter='\t')

        for row in reader:
            hyponym, hypernym = sanitize(row[0]), sanitize(row[1])

            if hyponym in freq and hyponym != hypernym:
                if not hyponym in G:
                    G.add_node(hyponym)

                if hypernym not in G[hyponym]:
                    G[hyponym][hypernym] = len(G[hyponym]) + 1

    return G

with concurrent.futures.ProcessPoolExecutor() as executor:
    resources = {path: G for path, G in zip(args.path, executor.map(isas, args.path))}

morph = pymorphy2.MorphAnalyzer()

@lru_cache(maxsize=None)
def inflect(word):
    if not word:
        return word

    suffix = ',' if word[-1] in {',', ')'} else ''
    word = word.rstrip(suffix)

    parses = morph.parse(word)

    if not parses:
        return word + suffix

    candidates = [p for p in parses if {'NOUN'} in p.tag]

    if not candidates:
        candidates = parses

    match = max(parses, key=lambda p: p.score)

    inflection = match.inflect({'gent'})

    return (inflection.word if inflection else word) + suffix

writer = csv.writer(sys.stdout, delimiter='\t')
writer.writerow(('resource', 'hyponym', 'found', 'hypernym', 'genitive', 'freq', 'n'))

for n, hyponym in enumerate(lexicon):
    for resource, G in resources.items():
        hypernyms = top(G[hyponym], args.k) if hyponym in G else []

        while len(hypernyms) < args.k:
            hypernyms.append(None)

        for hypernym in hypernyms:
            genitive = '_'.join([inflect(word) for word in hypernym.split('_')]) if args.inflection and hypernym else hypernym
            writer.writerow((resource, hyponym, int(not not hypernym), hypernym, genitive, freq[hyponym], n))
<commit_msg>Better handling of suffixes<commit_after>#!/usr/bin/env python

from signal import signal, SIGPIPE, SIG_DFL
signal(SIGPIPE, SIG_DFL)

import argparse
import csv
from collections import defaultdict
import concurrent.futures
import operator
import networkx as nx
import sys
import pymorphy2
from functools import lru_cache

parser = argparse.ArgumentParser()
parser.add_argument('--freq', required=True, type=argparse.FileType('r'))
parser.add_argument('-n', nargs='?', type=int, default=300)
parser.add_argument('-k', nargs='?', type=int, default=3)
parser.add_argument('--skip', nargs='?', type=int, default=0)
parser.add_argument('--no-inflection', dest='inflection', action='store_false')
parser.add_argument('path', nargs='+')
args = parser.parse_args()

def sanitize(s):
    return s.lower().replace(' ', '_')

with args.freq as f:
    reader = csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE)
    freq = {sanitize(row['Lemma']): float(row['Freq']) for row in reader if row['PoS'] == 's'}

def top(data, n, skip=0, reverse=False):
    head = sorted(data.items(), key=operator.itemgetter(1), reverse=reverse)
    return [word for i, (word, _) in enumerate(head) if i < skip + n and i >= skip]

lexicon = top(freq, args.n, args.skip, reverse=True)

def isas(path):
    G = nx.DiGraph()

    with open(path, newline='') as f:
        reader = csv.reader(f, delimiter='\t')

        for row in reader:
            hyponym, hypernym = sanitize(row[0]), sanitize(row[1])

            if hyponym in freq and hyponym != hypernym:
                if not hyponym in G:
                    G.add_node(hyponym)

                if hypernym not in G[hyponym]:
                    G[hyponym][hypernym] = len(G[hyponym]) + 1

    return G

with concurrent.futures.ProcessPoolExecutor() as executor:
    resources = {path: G for path, G in zip(args.path, executor.map(isas, args.path))}

morph = pymorphy2.MorphAnalyzer()

@lru_cache(maxsize=None)
def inflect(word):
    if not word:
        return word

    suffix = word[-1] if word[-1] in {',', ')'} else ''
    word = word.rstrip(suffix)

    parses = morph.parse(word)

    if not parses:
        return word + suffix

    candidates = [p for p in parses if {'NOUN'} in p.tag]

    if not candidates:
        candidates = parses

    match = max(parses, key=lambda p: p.score)

    inflection = match.inflect({'gent'})

    return (inflection.word if inflection else word) + suffix

writer = csv.writer(sys.stdout, delimiter='\t')
writer.writerow(('resource', 'hyponym', 'found', 'hypernym', 'genitive', 'freq', 'n'))

for n, hyponym in enumerate(lexicon):
    for resource, G in resources.items():
        hypernyms = top(G[hyponym], args.k) if hyponym in G else []

        while len(hypernyms) < args.k:
            hypernyms.append(None)

        for hypernym in hypernyms:
            genitive = '_'.join([inflect(word) for word in hypernym.split('_')]) if args.inflection and hypernym else hypernym
            writer.writerow((resource, hyponym, int(not not hypernym), hypernym, genitive, freq[hyponym], n))
<|endoftext|>


<commit_before>import telebot
from datetime import datetime, date, time

from keys import SUBSCRIBER_TOKEN
from database import db as database
from prototype import weekday_schedule

bot = telebot.TeleBot(SUBSCRIBER_TOKEN)

groups = database.get_groups()

day = datetime.weekday(datetime.now()) + 1 if 0 <= datetime.weekday(datetime.now()) <= 4 else 0 if datetime.weekday(datetime.now()) == 6 else 6

if day != 6:
	for group in groups:
		schedule = weekday_schedule(group['id'], day, True)
		subscribers = database.get_group_subscribers(group['id'])
		for subscriber in subscribers:
			bot.send_message(subscriber['id'], schedule)
<commit_msg>bug fix<commit_after>import telebot
from datetime import datetime, date, time

from keys import SUBSCRIBER_TOKEN
from database import db as database
from subscribers_bot import weekday_schedule

bot = telebot.TeleBot(SUBSCRIBER_TOKEN)

groups = database.get_groups()

day = datetime.weekday(datetime.now()) + 1 if 0 <= datetime.weekday(datetime.now()) <= 4 else 0 if datetime.weekday(datetime.now()) == 6 else 6

if day != 6:
	for group in groups:
		schedule = weekday_schedule(group['id'], day, True)
		subscribers = database.get_group_subscribers(group['id'])
		for subscriber in subscribers:
			bot.send_message(subscriber['id'], schedule)
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Copyright 2010 Anso Labs, LLC
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
"""
Fake LDAP server for test harnesses.

This class does very little error checking, and knows nothing about ldap
class definitions. It implements the minimum emulation of the python ldap
library to work with nova.
"""

import json

from nova import datastore

SCOPE_SUBTREE  = 2
MOD_ADD = 0
MOD_DELETE = 1

class NO_SUCH_OBJECT(Exception):
    pass

class OBJECT_CLASS_VIOLATION(Exception):
    pass

def initialize(uri):
    return FakeLDAP()

def _match_query(query, attrs):
    """Match an ldap query to an attribute dictionary.

    &, |, and ! are supported in the query. No syntax checking is performed,
    so malformed querys will not work correctly.

    """
    # cut off the parentheses
    inner = query[1:-1]
    if inner.startswith('&'):
        # cut off the &
        l, r = _paren_groups(inner[1:])
        return _match_query(l, attrs) and _match_query(r, attrs)
    if inner.startswith('|'):
        # cut off the |
        l, r = _paren_groups(inner[1:])
        return _match_query(l, attrs) or _match_query(r, attrs)
    if inner.startswith('!'):
        # cut off the ! and the nested parentheses
        return not _match_query(query[2:-1], attrs)

    (k, sep, v) = inner.partition('=')
    return _match(k, v, attrs)

def _paren_groups(source):
    """Split a string into parenthesized groups."""
    count = 0
    start = 0
    result = []
    for pos in xrange(len(source)):
        if source[pos] == '(':
            if count == 0:
                start = pos
            count += 1
        if source[pos] == ')':
            count -= 1
            if count == 0:
                result.append(source[start:pos+1])
    return result

def _match(k, v, attrs):
    """Match a given key and value against an attribute list."""
    if k not in attrs:
        return False
    if k != "objectclass":
        return v in attrs[k]
    # it is an objectclass check, so check subclasses
    values = _subs(v)
    for value in values:
        if value in attrs[k]:
            return True
    return False

def _subs(value):
    """Returns a list of subclass strings.

    The strings represent the ldap objectclass plus any subclasses that
    inherit from it. Fakeldap doesn't know about the ldap object structure,
    so subclasses need to be defined manually in the dictionary below.

    """
    subs = {'groupOfNames': ['novaProject']}
    if value in subs:
        return [value] + subs[value]
    return [value]

class FakeLDAP(object):
    #TODO(vish): refactor this class to use a wrapper instead of accessing
    #            redis directly

    def simple_bind_s(self, dn, password):
        """This method is ignored, but provided for compatibility."""
        pass

    def unbind_s(self):
        """This method is ignored, but provided for compatibility."""
        pass

    def add_s(self, dn, attr):
        """Add an object with the specified attributes at dn."""
        key = "%s%s" % (self.__redis_prefix, dn)

        value_dict = dict([(k, self.__to_json(v)) for k, v in attr])
        datastore.Redis.instance().hmset(key, value_dict)

    def delete_s(self, dn):
        """Remove the ldap object at specified dn."""
        datastore.Redis.instance().delete("%s%s" % (self.__redis_prefix, dn))

    def modify_s(self, dn, attrs):
        """Modify the object at dn using the attribute list.

        Args:
        dn -- a dn
        attrs -- a list of tuples in the following form:
            ([MOD_ADD | MOD_DELETE], attribute, value)

        """
        redis = datastore.Redis.instance()
        key = "%s%s" % (self.__redis_prefix, dn)

        for cmd, k, v in attrs:
            values = self.__from_json(redis.hget(key, k))
            if cmd == MOD_ADD:
                values.append(v)
            else:
                values.remove(v)
            values = redis.hset(key, k, self.__to_json(values))

    def search_s(self, dn, scope, query=None, fields=None):
        """Search for all matching objects under dn using the query.

        Args:
        dn -- dn to search under
        scope -- only SCOPE_SUBTREE is supported
        query -- query to filter objects by
        fields -- fields to return. Returns all fields if not specified

        """
        if scope != SCOPE_SUBTREE:
            raise NotImplementedError(str(scope))
        redis = datastore.Redis.instance()
        keys = redis.keys("%s*%s" % (self.__redis_prefix, dn))
        objects = []
        for key in keys:
            # get the attributes from redis
            attrs = redis.hgetall(key)
            # turn the values from redis into lists
            attrs = dict([(k, self.__from_json(v))
                          for k, v in attrs.iteritems()])
            # filter the objects by query
            if not query or _match_query(query, attrs):
                # filter the attributes by fields
                attrs = dict([(k, v) for k, v in attrs.iteritems()
                              if not fields or k in fields])
                objects.append((key[len(self.__redis_prefix):], attrs))
        if objects == []:
            raise NO_SUCH_OBJECT()
        return objects

    @property
    def __redis_prefix(self):
        return 'ldap:'

    def __from_json(self, encoded):
        """Convert attribute values from json representation.

        Args:
        encoded -- a json encoded string

        Returns a list of strings

        """
        return [str(x) for x in json.loads(encoded)]

    def __to_json(self, unencoded):
        """Convert attribute values into json representation.

        Args:
        unencoded -- an unencoded string or list of strings.  If it
            is a single string, it will be converted into a list.

        Returns a json string

        """
        return json.dumps(list(unencoded))
<commit_msg>review reformat<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Copyright 2010 Anso Labs, LLC
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
"""
Fake LDAP server for test harnesses.

This class does very little error checking, and knows nothing about ldap
class definitions. It implements the minimum emulation of the python ldap
library to work with nova.
"""

import json

from nova import datastore

SCOPE_SUBTREE  = 2
MOD_ADD = 0
MOD_DELETE = 1

class NO_SUCH_OBJECT(Exception):
    pass

class OBJECT_CLASS_VIOLATION(Exception):
    pass

def initialize(uri):
    return FakeLDAP()

def _match_query(query, attrs):
    """Match an ldap query to an attribute dictionary.

    &, |, and ! are supported in the query. No syntax checking is performed,
    so malformed querys will not work correctly.

    """
    # cut off the parentheses
    inner = query[1:-1]
    if inner.startswith('&'):
        # cut off the &
        l, r = _paren_groups(inner[1:])
        return _match_query(l, attrs) and _match_query(r, attrs)
    if inner.startswith('|'):
        # cut off the |
        l, r = _paren_groups(inner[1:])
        return _match_query(l, attrs) or _match_query(r, attrs)
    if inner.startswith('!'):
        # cut off the ! and the nested parentheses
        return not _match_query(query[2:-1], attrs)

    (k, sep, v) = inner.partition('=')
    return _match(k, v, attrs)

def _paren_groups(source):
    """Split a string into parenthesized groups."""
    count = 0
    start = 0
    result = []
    for pos in xrange(len(source)):
        if source[pos] == '(':
            if count == 0:
                start = pos
            count += 1
        if source[pos] == ')':
            count -= 1
            if count == 0:
                result.append(source[start:pos+1])
    return result

def _match(k, v, attrs):
    """Match a given key and value against an attribute list."""
    if k not in attrs:
        return False
    if k != "objectclass":
        return v in attrs[k]
    # it is an objectclass check, so check subclasses
    values = _subs(v)
    for value in values:
        if value in attrs[k]:
            return True
    return False

def _subs(value):
    """Returns a list of subclass strings.

    The strings represent the ldap objectclass plus any subclasses that
    inherit from it. Fakeldap doesn't know about the ldap object structure,
    so subclasses need to be defined manually in the dictionary below.

    """
    subs = {'groupOfNames': ['novaProject']}
    if value in subs:
        return [value] + subs[value]
    return [value]

def _from_json(encoded):
    """Convert attribute values from json representation.

    Args:
    encoded -- a json encoded string

    Returns a list of strings

    """
    return [str(x) for x in json.loads(encoded)]

def _to_json(unencoded):
    """Convert attribute values into json representation.

    Args:
    unencoded -- an unencoded string or list of strings.  If it
        is a single string, it will be converted into a list.

    Returns a json string

    """
    return json.dumps(list(unencoded))

class FakeLDAP(object):
    #TODO(vish): refactor this class to use a wrapper instead of accessing
    #            redis directly

    def simple_bind_s(self, dn, password):
        """This method is ignored, but provided for compatibility."""
        pass

    def unbind_s(self):
        """This method is ignored, but provided for compatibility."""
        pass

    def add_s(self, dn, attr):
        """Add an object with the specified attributes at dn."""
        key = "%s%s" % (self.__redis_prefix, dn)

        value_dict = dict([(k, _to_json(v)) for k, v in attr])
        datastore.Redis.instance().hmset(key, value_dict)

    def delete_s(self, dn):
        """Remove the ldap object at specified dn."""
        datastore.Redis.instance().delete("%s%s" % (self.__redis_prefix, dn))

    def modify_s(self, dn, attrs):
        """Modify the object at dn using the attribute list.

        Args:
        dn -- a dn
        attrs -- a list of tuples in the following form:
            ([MOD_ADD | MOD_DELETE], attribute, value)

        """
        redis = datastore.Redis.instance()
        key = "%s%s" % (self.__redis_prefix, dn)

        for cmd, k, v in attrs:
            values = _from_json(redis.hget(key, k))
            if cmd == MOD_ADD:
                values.append(v)
            else:
                values.remove(v)
            values = redis.hset(key, k, _to_json(values))

    def search_s(self, dn, scope, query=None, fields=None):
        """Search for all matching objects under dn using the query.

        Args:
        dn -- dn to search under
        scope -- only SCOPE_SUBTREE is supported
        query -- query to filter objects by
        fields -- fields to return. Returns all fields if not specified

        """
        if scope != SCOPE_SUBTREE:
            raise NotImplementedError(str(scope))
        redis = datastore.Redis.instance()
        keys = redis.keys("%s*%s" % (self.__redis_prefix, dn))
        objects = []
        for key in keys:
            # get the attributes from redis
            attrs = redis.hgetall(key)
            # turn the values from redis into lists
            attrs = dict([(k, _from_json(v))
                          for k, v in attrs.iteritems()])
            # filter the objects by query
            if not query or _match_query(query, attrs):
                # filter the attributes by fields
                attrs = dict([(k, v) for k, v in attrs.iteritems()
                              if not fields or k in fields])
                objects.append((key[len(self.__redis_prefix):], attrs))
        if objects == []:
            raise NO_SUCH_OBJECT()
        return objects

    @property
    def __redis_prefix(self):
        return 'ldap:'

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import csv
import re

rule_token_re = re.compile(u'''
    (?P<prefix>[全單雙連至])?
    (?P<number>.+?)
    (?P<unit>[巷弄號樓]|附號全)
    (?P<suffix>全|以下|以上|含附號全?|含附號以下|及以上附號)?
''', re.X)

addr_token_re = re.compile(u'''
    (?P<name>.+?)
    (?P<unit>[縣市鄉鎮市區村里路街巷弄號樓])
''', re.X)

fail_counter = 0

with open('zipcodetw-20140131.csv') as f:

    next(f)

    for row in csv.reader(f):

        zipcode = row[0]
        triple_addr = row[1:-1]
        rule_str = row[-1].decode('utf-8').replace(u' ', u'').replace(u'　', u'')
        rule_tokens = rule_token_re.findall(rule_str)
        if len(rule_tokens) != (
            sum(rule_str.count(k) for k in u'巷弄號樓')
            -(u'附號' in rule_str)
            +(u'附號全' in rule_str)
            -(u'含附號全' in rule_str)
        ):
            print rule_str
            print rule_tokens
            print len(rule_tokens)
            fail_counter += 1

print fail_counter

addr_str = u'臺北市信義區市府路1之23號'

print addr_str
print addr_token_re.findall(addr_str)
<commit_msg>added blank lines<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import csv
import re

rule_token_re = re.compile(u'''
    (?P<prefix>[全單雙連至])?
    (?P<number>.+?)
    (?P<unit>[巷弄號樓]|附號全)
    (?P<suffix>全|以下|以上|含附號全?|含附號以下|及以上附號)?
''', re.X)

addr_token_re = re.compile(u'''
    (?P<name>.+?)
    (?P<unit>[縣市鄉鎮市區村里路街巷弄號樓])
''', re.X)

fail_counter = 0

with open('zipcodetw-20140131.csv') as f:

    next(f)

    for row in csv.reader(f):

        zipcode = row[0]
        triple_addr = row[1:-1]
        rule_str = row[-1].decode('utf-8').replace(u' ', u'').replace(u'　', u'')

        rule_tokens = rule_token_re.findall(rule_str)

        if len(rule_tokens) != (
            sum(rule_str.count(k) for k in u'巷弄號樓')
            -(u'附號' in rule_str)
            +(u'附號全' in rule_str)
            -(u'含附號全' in rule_str)
        ):

            print rule_str
            print rule_tokens
            print len(rule_tokens)
            fail_counter += 1

print fail_counter

addr_str = u'臺北市信義區市府路1之23號'

print addr_str
print addr_token_re.findall(addr_str)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import csv
import re

addr_token_re = re.compile(u'''
    (?P<name>.+?)
    (?P<unit>[縣市鄉鎮市區村里路街巷弄號樓])
''', re.X)

rule_token_re = re.compile(u'''
    (?P<special>[單雙連至]|全|以下|以上|含附號全?|含附號以下|及以上附號)
    |
    (?P<number>[\d之]+?)
    (?P<unit>[巷弄號樓]|附號全)
''', re.X)

#test_cases = [
#]
#
#for test_case in test_cases:
#    print test_case
#    print rule_token_re.findall(test_case)
#
#import sys; sys.exit()

triple_addr_rules_zip = {}

with open('zipcodetw-20140131.csv') as f:

    # skip the title line
    next(f)

    for row in csv.reader(f):

        row = [item.decode('utf-8') for item in row]

        zipcode = row[0]

        triple_addr = tuple(addr_token_re.findall(''.join(row[1:-1])))

        rule_str = row[-1].replace(u' ', u'').replace(u'　', u'')
        rule_tokens = rule_token_re.findall(rule_str)

        if triple_addr not in triple_addr_rules_zip:
            triple_addr_rules_zip[triple_addr] = ([rule_tokens], [zipcode])
        else:
            triple_addr_rules_zip[triple_addr][0].append(rule_tokens)
            triple_addr_rules_zip[triple_addr][1].append(zipcode)

# test addr_token_re

import uniout
from pprint import pprint

addr_str = u'臺北市信義區市府路1之23號'
print addr_str
addr_tokens = tuple(addr_token_re.findall(addr_str))
print addr_tokens
print addr_tokens[:3]
pprint(triple_addr_rules_zip[addr_tokens[:3]])
<commit_msg>picked most representative test cases<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import csv
import re

addr_token_re = re.compile(u'''
    (?P<name>.+?)
    (?P<unit>[縣市鄉鎮市區村里路街巷弄號樓])
''', re.X)

rule_token_re = re.compile(u'''
    (?P<special>[單雙連至]|全|以下|以上|含附號全?|含附號以下|及以上附號)
    |
    (?P<number>[\d之]+?)
    (?P<unit>[巷弄號樓]|附號全)
''', re.X)

#test_cases = [
#    u'全', u'單全',
#    u'雙48號以下', u'雙50號以上',
#    u'7號',
#    u'連2之4號以上',
#    u'14號含附號', u'14巷全',
#    u'47號', u'47附號全',
#    u'雙68巷至70號含附號全',
#    u'連49號含附號以下',
#    u'1之3號及以上附號',
#]
#
#for test_case in test_cases:
#    print test_case
#    print rule_token_re.findall(test_case)
#
#import sys; sys.exit()

triple_addr_rules_zip = {}

with open('zipcodetw-20140131.csv') as f:

    # skip the title line
    next(f)

    for row in csv.reader(f):

        row = [item.decode('utf-8') for item in row]

        zipcode = row[0]

        triple_addr = tuple(addr_token_re.findall(''.join(row[1:-1])))

        rule_str = row[-1].replace(u' ', u'').replace(u'　', u'')
        rule_tokens = rule_token_re.findall(rule_str)

        if triple_addr not in triple_addr_rules_zip:
            triple_addr_rules_zip[triple_addr] = ([rule_tokens], [zipcode])
        else:
            triple_addr_rules_zip[triple_addr][0].append(rule_tokens)
            triple_addr_rules_zip[triple_addr][1].append(zipcode)

# test addr_token_re

import uniout
from pprint import pprint

addr_str = u'臺北市信義區市府路1之23號'
print addr_str
addr_tokens = tuple(addr_token_re.findall(addr_str))
print addr_tokens
print addr_tokens[:3]
pprint(triple_addr_rules_zip[addr_tokens[:3]])
<|endoftext|>


<commit_before>import inspect
import logging
import os
import shlex
import subprocess
import threading
import time
from contextlib import contextmanager
from functools import wraps

import parsl
from parsl.version import VERSION

logger = logging.getLogger(__name__)

def get_version():
    version = parsl.__version__
    work_tree = os.path.dirname(os.path.dirname(__file__))
    git_dir = os.path.join(work_tree, '.git')
    if os.path.exists(git_dir):
        env = {'GIT_WORK_TREE': work_tree, 'GIT_DIR': git_dir}
        try:
            cmd = shlex.split('git rev-parse --short HEAD')
            head = subprocess.check_output(cmd, env=env).strip().decode('utf-8')
            diff = subprocess.check_output(shlex.split('git diff HEAD'), env=env)
            status = 'dirty' if diff else 'clean'
            version = '{v}-{head}-{status}'.format(v=VERSION, head=head, status=status)
        except Exception:
            pass

    return version

def get_all_checkpoints(rundir="runinfo"):
    """Finds the checkpoints from all last runs.

    Note that checkpoints are incremental, and this helper will not find
    previous checkpoints from earlier than the most recent run. It probably
    should be made to do so.

    Kwargs:
       - rundir(str) : Path to the runinfo directory

    Returns:
       - a list suitable for the checkpointFiles parameter of DataFlowKernel
         constructor

    """

    if(not os.path.isdir(rundir)):
        return []

    dirs = sorted(os.listdir(rundir))

    checkpoints = []

    for runid in dirs:

        checkpoint = os.path.abspath('{}/{}/checkpoint'.format(rundir, runid))

        if os.path.isdir(checkpoint):
            checkpoints.append(checkpoint)

    return checkpoints

def get_last_checkpoint(rundir="runinfo"):
    """Find the checkpoint from the last run, if one exists.

    Note that checkpoints are incremental, and this helper will not find
    previous checkpoints from earlier than the most recent run. It probably
    should be made to do so.

    Kwargs:
       - rundir(str) : Path to the runinfo directory

    Returns:
     - a list suitable for checkpointFiles parameter of DataFlowKernel
       constructor, with 0 or 1 elements

    """
    if not os.path.isdir(rundir):
        return []

    dirs = sorted(os.listdir(rundir))

    if len(dirs) == 0:
        return []

    last_runid = dirs[-1]
    last_checkpoint = os.path.abspath('{}/{}/checkpoint'.format(rundir, last_runid))

    if(not(os.path.isdir(last_checkpoint))):
        return []

    return [last_checkpoint]

def timeout(seconds=None):
    def decorator(func, *args, **kwargs):
        @wraps(func)
        def wrapper(*args, **kwargs):
            t = threading.Thread(target=func, args=args, kwargs=kwargs)
            t.start()
            result = t.join(seconds)
            if t.is_alive():
                raise RuntimeError('timed out in {}'.format(func))
            return result
        return wrapper
    return decorator

@contextmanager
def wait_for_file(path, seconds=10):
    for i in range(0, int(seconds * 100)):
        time.sleep(seconds / 100.)
        if os.path.exists(path):
            break
    yield

@contextmanager
def time_limited_open(path, mode, seconds=1):
    wait_for_file(path, seconds)

    f = open(path, mode)
    yield f
    f.close()

def wtime_to_minutes(time_string):
    ''' wtime_to_minutes

    Convert standard wallclock time string to minutes.

    Args:
        - Time_string in HH:MM:SS format

    Returns:
        (int) minutes

    '''
    hours, mins, seconds = time_string.split(':')
    return int(hours) * 60 + int(mins) + 1

class RepresentationMixin(object):
    """A mixin class for adding a __repr__ method.

    The __repr__ method will return a string equivalent to the code used to instantiate
    the child class, with any defaults included explicitly. The __max_width__ class variable
    controls the maximum width of the representation string. If this width is exceeded,
    the representation string will be split up, with one argument or keyword argument per line.

    Any arguments or keyword arguments in the constructor must be defined as attributes, or
    an AttributeError will be raised.

    Examples
    --------
    >>> from parsl.utils import RepresentationMixin
    >>> class Foo(RepresentationMixin):
            def __init__(self, first, second, third='three', fourth='fourth'):
                self.first = first
                self.second = second
                self.third = third
                self.fourth = fourth
    >>> bar = Foo(1, 'two', fourth='baz')
    >>> bar
    Foo(1, 'two', third='three', fourth='baz')
    """
    __max_width__ = 80

    def __repr__(self):
        argspec = inspect.getfullargspec(self.__init__)
        if len(argspec.args) > 1:
            defaults = dict(zip(reversed(argspec.args), reversed(argspec.defaults)))
        else:
            defaults = []

        for arg in argspec.args[1:]:
            if not hasattr(self, arg):
                template = 'class {} uses {} in the constructor, but does not define it as an attribute'
                raise AttributeError(template.format(self.__class__.__name__, arg))

        args = [getattr(self, a) for a in argspec.args[1:-len(defaults)]]
        kwargs = {key: getattr(self, key) for key in defaults}

        def assemble_multiline(args, kwargs):
            def indent(text):
                lines = text.splitlines()
                if len(lines) <= 1:
                    return text
                return "\n".join("    " + l for l in lines).strip()
            args = ["\n    {},".format(indent(repr(a))) for a in args]
            kwargs = ["\n    {}={}".format(k, indent(repr(v)))
                      for k, v in sorted(kwargs.items())]

            info = "".join(args) + ", ".join(kwargs)
            return self.__class__.__name__ + "({}\n)".format(info)

        def assemble_line(args, kwargs):
            kwargs = ['{}={}'.format(k, repr(v)) for k, v in sorted(kwargs.items())]

            info = ", ".join([repr(a) for a in args] + kwargs)
            return self.__class__.__name__ + "({})".format(info)

        if len(assemble_line(args, kwargs)) <= self.__class__.__max_width__:
            return assemble_line(args, kwargs)
        else:
            return assemble_multiline(args, kwargs)
<commit_msg>Parse walltime strings to actual number of minutes<commit_after>import inspect
import logging
import os
import shlex
import subprocess
import threading
import time
from contextlib import contextmanager
from functools import wraps

import parsl
from parsl.version import VERSION

logger = logging.getLogger(__name__)

def get_version():
    version = parsl.__version__
    work_tree = os.path.dirname(os.path.dirname(__file__))
    git_dir = os.path.join(work_tree, '.git')
    if os.path.exists(git_dir):
        env = {'GIT_WORK_TREE': work_tree, 'GIT_DIR': git_dir}
        try:
            cmd = shlex.split('git rev-parse --short HEAD')
            head = subprocess.check_output(cmd, env=env).strip().decode('utf-8')
            diff = subprocess.check_output(shlex.split('git diff HEAD'), env=env)
            status = 'dirty' if diff else 'clean'
            version = '{v}-{head}-{status}'.format(v=VERSION, head=head, status=status)
        except Exception:
            pass

    return version

def get_all_checkpoints(rundir="runinfo"):
    """Finds the checkpoints from all last runs.

    Note that checkpoints are incremental, and this helper will not find
    previous checkpoints from earlier than the most recent run. It probably
    should be made to do so.

    Kwargs:
       - rundir(str) : Path to the runinfo directory

    Returns:
       - a list suitable for the checkpointFiles parameter of DataFlowKernel
         constructor

    """

    if(not os.path.isdir(rundir)):
        return []

    dirs = sorted(os.listdir(rundir))

    checkpoints = []

    for runid in dirs:

        checkpoint = os.path.abspath('{}/{}/checkpoint'.format(rundir, runid))

        if os.path.isdir(checkpoint):
            checkpoints.append(checkpoint)

    return checkpoints

def get_last_checkpoint(rundir="runinfo"):
    """Find the checkpoint from the last run, if one exists.

    Note that checkpoints are incremental, and this helper will not find
    previous checkpoints from earlier than the most recent run. It probably
    should be made to do so.

    Kwargs:
       - rundir(str) : Path to the runinfo directory

    Returns:
     - a list suitable for checkpointFiles parameter of DataFlowKernel
       constructor, with 0 or 1 elements

    """
    if not os.path.isdir(rundir):
        return []

    dirs = sorted(os.listdir(rundir))

    if len(dirs) == 0:
        return []

    last_runid = dirs[-1]
    last_checkpoint = os.path.abspath('{}/{}/checkpoint'.format(rundir, last_runid))

    if(not(os.path.isdir(last_checkpoint))):
        return []

    return [last_checkpoint]

def timeout(seconds=None):
    def decorator(func, *args, **kwargs):
        @wraps(func)
        def wrapper(*args, **kwargs):
            t = threading.Thread(target=func, args=args, kwargs=kwargs)
            t.start()
            result = t.join(seconds)
            if t.is_alive():
                raise RuntimeError('timed out in {}'.format(func))
            return result
        return wrapper
    return decorator

@contextmanager
def wait_for_file(path, seconds=10):
    for i in range(0, int(seconds * 100)):
        time.sleep(seconds / 100.)
        if os.path.exists(path):
            break
    yield

@contextmanager
def time_limited_open(path, mode, seconds=1):
    wait_for_file(path, seconds)

    f = open(path, mode)
    yield f
    f.close()

def wtime_to_minutes(time_string):
    ''' wtime_to_minutes

    Convert standard wallclock time string to minutes.

    Args:
        - Time_string in HH:MM:SS format

    Returns:
        (int) minutes

    '''
    hours, mins, seconds = time_string.split(':')
    total_mins = int(hours) * 60 + int(mins)
    if total_mins < 1:
        logger.warning("Time string '{}' parsed to {} minutes, less than 1".format(time_string, total_mins))
    return total_mins

class RepresentationMixin(object):
    """A mixin class for adding a __repr__ method.

    The __repr__ method will return a string equivalent to the code used to instantiate
    the child class, with any defaults included explicitly. The __max_width__ class variable
    controls the maximum width of the representation string. If this width is exceeded,
    the representation string will be split up, with one argument or keyword argument per line.

    Any arguments or keyword arguments in the constructor must be defined as attributes, or
    an AttributeError will be raised.

    Examples
    --------
    >>> from parsl.utils import RepresentationMixin
    >>> class Foo(RepresentationMixin):
            def __init__(self, first, second, third='three', fourth='fourth'):
                self.first = first
                self.second = second
                self.third = third
                self.fourth = fourth
    >>> bar = Foo(1, 'two', fourth='baz')
    >>> bar
    Foo(1, 'two', third='three', fourth='baz')
    """
    __max_width__ = 80

    def __repr__(self):
        argspec = inspect.getfullargspec(self.__init__)
        if len(argspec.args) > 1:
            defaults = dict(zip(reversed(argspec.args), reversed(argspec.defaults)))
        else:
            defaults = []

        for arg in argspec.args[1:]:
            if not hasattr(self, arg):
                template = 'class {} uses {} in the constructor, but does not define it as an attribute'
                raise AttributeError(template.format(self.__class__.__name__, arg))

        args = [getattr(self, a) for a in argspec.args[1:-len(defaults)]]
        kwargs = {key: getattr(self, key) for key in defaults}

        def assemble_multiline(args, kwargs):
            def indent(text):
                lines = text.splitlines()
                if len(lines) <= 1:
                    return text
                return "\n".join("    " + l for l in lines).strip()
            args = ["\n    {},".format(indent(repr(a))) for a in args]
            kwargs = ["\n    {}={}".format(k, indent(repr(v)))
                      for k, v in sorted(kwargs.items())]

            info = "".join(args) + ", ".join(kwargs)
            return self.__class__.__name__ + "({}\n)".format(info)

        def assemble_line(args, kwargs):
            kwargs = ['{}={}'.format(k, repr(v)) for k, v in sorted(kwargs.items())]

            info = ", ".join([repr(a) for a in args] + kwargs)
            return self.__class__.__name__ + "({})".format(info)

        if len(assemble_line(args, kwargs)) <= self.__class__.__max_width__:
            return assemble_line(args, kwargs)
        else:
            return assemble_multiline(args, kwargs)
<|endoftext|>


<commit_before>import numpy

def pca(data):
    """Perform PCA on a dataset.

    Arguments:
      data: shape (n, m) array, where n is the number of data points and m is
        the dimensionality of each data point.

    Returns: (mean, pcs, norm_pcs, variances, positions, norm_positions)
      mean: array of shape m, representing the mean of the input data.
      pcs: array of shape (p, m) containing the p principal components. Each
        component is normalized to euclidian distance of 1. The number of
        components p is min(n, m).
      norm_pcs: same as the array pcs, except that each component is scaled by
      the standard deviation along each component.
      variances: length-p array containing the amount of variance explained by
        each component.
      positions: the position of each data point in the basis of the principal
        components; shape = (n, p).
      norm_positions: the position of each data point in the basis of the
        normalized principal components. Positions in this basis are in terms
        of standard deviations away from the mean, rather than at the original
        scale of the data points; shape = (n, p).

    Example:
      mean, pcs, norm_pcs, variances, positions, norm_positions = pca(data)
      synthetic_point = mean + 3*norm_pcs[0]

      Here, 'synthetic_point' is three standard deviations away from the mean
      in one direction along the first principle component.
    """
    data = numpy.asarray(data)
    mean = data.mean(axis=0)
    centered = data - mean
    # could use _pca_svd below, but that appears empirically slower...
    pcs, variances, stds, positions, norm_positions = _pca_eig(centered)
    norm_pcs = pcs * stds[:, numpy.newaxis]
    return mean, pcs, norm_pcs, variances, positions, norm_positions

def pca_dimensionality_reduce(data, required_variance_explained):
    """Use PCA to reduce the dimensionality of a dataset by retaining only a
    subset of the principal components.

    Arguments:
      data: shape (n, m) array, where n is the number of data points and m is
        the dimensionality of each data point.
      required_variance_explained: fraction of the total variance which must be
        explained by the retained principal components. Must be in the range
        (0,1]

    Returns: (mean, pcs, norm_pcs, variances, total_variance, positions, norm_positions)
      mean: array of shape m, representing the mean of the input data.
      pcs: array of shape (p, m) containing the p principal components. Each
        component is normalized to euclidian distance of 1. The number of
        components p is <= min(n, m).
      norm_pcs: same as the array pcs, except that each component is scaled by
      the standard deviation along each component.
      variances: length-p array containing the amount of variance explained by
        each component.
      total_variance: the total variance in the original dataset, of which
        variances.sum() is explained by the given principal components.
      positions: the position of each data point in the basis of the principal
        components; shape = (n, p).
      norm_positions: the position of each data point in the basis of the
        normalized principal components. Positions in this basis are in terms
        of standard deviations away from the mean, rather than at the original
        scale of the data points; shape = (n, p).
    """
    mean, pcs, norm_pcs, variances, positions, norm_positions = pca(data)
    total_variance = numpy.add.accumulate(variances / numpy.sum(variances))
    num = numpy.searchsorted(total_variance, required_variance_explained) + 1
    return mean, pcs[:num], norm_pcs[:num], variances[:num], numpy.sum(variances), positions[:,:num], norm_positions[:,:num]

def pca_decompose(data, pcs, mean):
    """Return the positions of new data points in the space spanned by a set of
    principle components.

    Arguments:
      data: shape (n, m) array, where n is the number of data points and m is
        the dimensionality of each data point.
      pcs: shape (p, m) array of p principal components. These may either be
        the unit-length 'pcs' or standard-deviation-scaled 'norm_pcs'.
      mean: the mean of the original data points from which the pcs were
        obtained.

    Returns: shape (n, p) projection of the data into the principal components
      basis.
    """
    if numpy.allclose((pcs**2).sum(axis=1), numpy.ones(len(pcs))):
        # pcs are of unit length, so we assume that they form an orthonormal basis
        inv_pcs = pcs.T
    else:
        # assume pcs is invertible (e.g. we're using norm_pcs)
        inv_pcs = numpy.linalg.inv(pcs)
    projection = numpy.dot(data - mean, inv_pcs)
    return projection

def pca_reconstruct(positions, pcs, mean):
    """Given positions in some PCA basis, return data points in the original
    data space.

    Arguments:
      positions: shape (n, p) array containing the projection of n data points
      into the basis of p principal components.
      pcs: shape (p, m) array of p principal components. These may either be
        the unit-length 'pcs' or standard-deviation-scaled 'norm_pcs'.
      mean: the mean of the original data points from which the pcs were
        obtained.

    Returns: shape (n, m) array of data points in the original m-dimensional
      space.
    """
    return mean + numpy.dot(positions, pcs)

def _pca_eig(data):
    """Perform PCA on a dataset using a symmetric eigenvalue decomposition."""
    values, vectors = _symm_eig(data)
    pcs = vectors.T
    variances = values / len(data)
    stds = numpy.sqrt(variances)
    positions = numpy.dot(data, vectors)
    with numpy.errstate(divide='ignore', invalid='ignore'):
        norm_positions = positions / stds
    norm_positions[~numpy.isfinite(norm_positions)] = 0
    return pcs, variances, stds, positions, norm_positions

def _symm_eig(a):
    """Return the eigenvectors and eigenvalues of the symmetric matrix a'a. If
    a has more columns than rows, then that matrix will be rank-deficient,
    and the non-zero eigenvalues and eigenvectors can be more easily extracted
    from the matrix aa'. From the properties of the SVD:
        if matrix a of shape (m,n) has SVD u*s*v', then:
            a'a = v*s's*v'
            aa' = u*ss'*u'
        let s_hat, an array of shape (m,n), be such that s * s_hat = I(m,m)
        and s_hat * s = I(n,n). Thus, we can solve for u or v in terms of the other:
            v = a'*u*s_hat'
            u = a*v*s_hat
    """
    m, n = a.shape
    if m >= n:
        # just return the eigenvalues and eigenvectors of a'a
        vecs, vals = _eigh(numpy.dot(a.T, a))
        # if elements are < 0 due to numerical instabilities, set to 0
        vecs[vecs < 0] = 0
        return vecs, vals
    else:
        # figure out the eigenvalues and vectors based on aa', which is smaller
        sst_diag, u = _eigh(numpy.dot(a, a.T))
        # if elements are < 0 due to numerical instabilities, set to 0
        sst_diag[sst_diag < 0] = 0
        # now get the inverse square root of the diagonal, which will form the
        # main diagonal of s_hat
        with numpy.errstate(divide='ignore', invalid='ignore'):
            s_hat_diag = 1/numpy.sqrt(sst_diag)
        s_hat_diag[~numpy.isfinite(s_hat_diag)] = 0
        # s_hat_diag is a list of length m, a'u is shape (n, m), so we can just use
        # numpy's broadcasting instead of matrix multiplication, and only create
        # the upper m x m block of a'u, since that's all we'll use anyway...
        v = numpy.dot(a.T, u[:,:m]) * s_hat_diag
        return sst_diag, v

def _eigh(m):
    """Return eigenvalues and eigenvectors of hermetian matrix m, sorted in
    by largest eigenvalue first."""
    values, vectors = numpy.linalg.eigh(m)
    order = values.argsort()[::-1]
    return values[order], vectors[:,order]

def _pca_svd(data):
    """Perform PCA on a dataset using the singular value decomposition."""
    u, s, vt = numpy.linalg.svd(data, full_matrices=0)
    pcs = vt
    data_count = len(data)
    variances = s**2 / data_count
    root_data_count = numpy.sqrt(data_count)
    stds = s / root_data_count
    positions = u * s
    norm_positions = u * root_data_count
    return pcs, variances, stds, positions, norm_positions
<commit_msg>fix for rank-deficient PCA with more dimensions than data points<commit_after>import numpy

def pca(data):
    """Perform PCA on a dataset.

    Arguments:
      data: shape (n, m) array, where n is the number of data points and m is
        the dimensionality of each data point.

    Returns: (mean, pcs, norm_pcs, variances, positions, norm_positions)
      mean: array of shape m, representing the mean of the input data.
      pcs: array of shape (p, m) containing the p principal components. Each
        component is normalized to euclidian distance of 1. The number of
        components p is min(n - 1, m).
      norm_pcs: same as the array pcs, except that each component is scaled by
      the standard deviation along each component.
      variances: length-p array containing the amount of variance explained by
        each component.
      positions: the position of each data point in the basis of the principal
        components; shape = (n, p).
      norm_positions: the position of each data point in the basis of the
        normalized principal components. Positions in this basis are in terms
        of standard deviations away from the mean, rather than at the original
        scale of the data points; shape = (n, p).

    Example:
      mean, pcs, norm_pcs, variances, positions, norm_positions = pca(data)
      synthetic_point = mean + 3*norm_pcs[0]

      Here, 'synthetic_point' is three standard deviations away from the mean
      in one direction along the first principle component.
    """
    data = numpy.asarray(data)
    mean = data.mean(axis=0)
    centered = data - mean
    # could use _pca_svd below, but that appears empirically slower...
    pcs, variances, stds, positions, norm_positions = _pca_eig(centered)
    norm_pcs = pcs * stds[:, numpy.newaxis]
    return mean, pcs, norm_pcs, variances, positions, norm_positions

def pca_dimensionality_reduce(data, required_variance_explained):
    """Use PCA to reduce the dimensionality of a dataset by retaining only a
    subset of the principal components.

    Arguments:
      data: shape (n, m) array, where n is the number of data points and m is
        the dimensionality of each data point.
      required_variance_explained: fraction of the total variance which must be
        explained by the retained principal components. Must be in the range
        (0,1]

    Returns: (mean, pcs, norm_pcs, variances, total_variance, positions, norm_positions)
      mean: array of shape m, representing the mean of the input data.
      pcs: array of shape (p, m) containing the p principal components. Each
        component is normalized to euclidian distance of 1. The number of
        components p is <= min(n, m).
      norm_pcs: same as the array pcs, except that each component is scaled by
      the standard deviation along each component.
      variances: length-p array containing the amount of variance explained by
        each component.
      total_variance: the total variance in the original dataset, of which
        variances.sum() is explained by the given principal components.
      positions: the position of each data point in the basis of the principal
        components; shape = (n, p).
      norm_positions: the position of each data point in the basis of the
        normalized principal components. Positions in this basis are in terms
        of standard deviations away from the mean, rather than at the original
        scale of the data points; shape = (n, p).
    """
    mean, pcs, norm_pcs, variances, positions, norm_positions = pca(data)
    total_variance = numpy.add.accumulate(variances / numpy.sum(variances))
    num = numpy.searchsorted(total_variance, required_variance_explained) + 1
    return mean, pcs[:num], norm_pcs[:num], variances[:num], numpy.sum(variances), positions[:,:num], norm_positions[:,:num]

def pca_decompose(data, pcs, mean):
    """Return the positions of new data points in the space spanned by a set of
    principle components.

    Arguments:
      data: shape (n, m) array, where n is the number of data points and m is
        the dimensionality of each data point.
      pcs: shape (p, m) array of p principal components. These may either be
        the unit-length 'pcs' or standard-deviation-scaled 'norm_pcs'.
      mean: the mean of the original data points from which the pcs were
        obtained.

    Returns: shape (n, p) projection of the data into the principal components
      basis.
    """
    projection = numpy.dot(data - mean, pcs.T)
    pc_lens = (pcs**2).sum(axis=1)
    if not numpy.allclose(pc_lens, numpy.ones(len(pcs))):
        # pcs are not of unit length; we must be using the normalized versions and
        # thus desire the normalized positions.
        # The normalized position is numpy.dot(data - mean, non_norm_pcs.T) / standard_deviations
        # and norm_pcs = non_norm_pcs * standard_deviations, so
        # we want numpy.dot(data - mean, norm_pcs.T) / standard_deviations**2
        # Note that the squared length of the norm_pcs is exactly standard_deviations**2
        projection /= pc_lens
    return projection

def pca_reconstruct(positions, pcs, mean):
    """Given positions in some PCA basis, return data points in the original
    data space.

    Arguments:
      positions: shape (n, p) array containing the projection of n data points
      into the basis of p principal components.
      pcs: shape (p, m) array of p principal components. These may either be
        the unit-length 'pcs' or standard-deviation-scaled 'norm_pcs'.
      mean: the mean of the original data points from which the pcs were
        obtained.

    Returns: shape (n, m) array of data points in the original m-dimensional
      space.
    """
    return mean + numpy.dot(positions, pcs)

def _pca_eig(data):
    """Perform PCA on a dataset using a symmetric eigenvalue decomposition."""
    values, vectors = _symm_eig(data)
    n, m = data.shape
    if n <= m:
        # in this case we can have at most n-1 PCs, because n points can span
        # at most an n-1-dimensional subspace. So we need to ditch the last PC,
        # which will just be noise.
        values = values[:-1]
        vectors = vectors[:, :-1]
    pcs = vectors.T
    variances = values / len(data)
    stds = numpy.sqrt(variances)
    positions = numpy.dot(data, vectors)
    with numpy.errstate(divide='ignore', invalid='ignore'):
        norm_positions = positions / stds
    norm_positions[~numpy.isfinite(norm_positions)] = 0
    return pcs, variances, stds, positions, norm_positions

def _symm_eig(a):
    """Return the eigenvectors and eigenvalues of the symmetric matrix a'a. If
    a has more columns than rows, then that matrix will be rank-deficient,
    and the non-zero eigenvalues and eigenvectors can be more easily extracted
    from the matrix aa'. From the properties of the SVD:
        if matrix a of shape (m,n) has SVD u*s*v', then:
            a'a = v*s's*v'
            aa' = u*ss'*u'
        let s_hat, an array of shape (m,n), be such that s * s_hat = I(m,m)
        and s_hat * s = I(n,n). Thus, we can solve for u or v in terms of the other:
            v = a'*u*s_hat'
            u = a*v*s_hat
    """
    m, n = a.shape
    if m >= n:
        # just return the eigenvalues and eigenvectors of a'a
        vecs, vals = _eigh(numpy.dot(a.T, a))
        # if elements are < 0 due to numerical instabilities, set to 0
        vecs[vecs < 0] = 0
        return vecs, vals
    else:
        # figure out the eigenvalues and vectors based on aa', which is smaller
        sst_diag, u = _eigh(numpy.dot(a, a.T))
        # if elements are < 0 due to numerical instabilities, set to 0
        sst_diag[sst_diag < 0] = 0
        # now get the inverse square root of the diagonal, which will form the
        # main diagonal of s_hat
        with numpy.errstate(divide='ignore', invalid='ignore'):
            s_hat_diag = 1/numpy.sqrt(sst_diag)
        s_hat_diag[~numpy.isfinite(s_hat_diag)] = 0
        # s_hat_diag is a list of length m, a'u is shape (n, m), so we can just use
        # numpy's broadcasting instead of matrix multiplication, and only create
        # the upper m x m block of a'u, since that's all we'll use anyway...
        v = numpy.dot(a.T, u[:,:m]) * s_hat_diag
        return sst_diag, v

def _eigh(m):
    """Return eigenvalues and eigenvectors of hermetian matrix m, sorted in
    by largest eigenvalue first."""
    values, vectors = numpy.linalg.eigh(m)
    order = values.argsort()[::-1]
    return values[order], vectors[:,order]

def _pca_svd(data):
    """Perform PCA on a dataset using the singular value decomposition."""
    u, s, vt = numpy.linalg.svd(data, full_matrices=0)
    n, m = data.shape
    if n <= m:
        # in this case we can have at most n-1 PCs, because n points can span
        # at most an n-1-dimensional subspace. So we need to ditch the last PC,
        # which will just be noise.
        u = u[:-1, :-1]
        s = s[:-1]
        vt = vt[:-1]
    pcs = vt
    data_count = len(data)
    variances = s**2 / data_count
    root_data_count = numpy.sqrt(data_count)
    stds = s / root_data_count
    positions = u * s
    norm_positions = u * root_data_count
    return pcs, variances, stds, positions, norm_positions
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np
from scipy.ndimage import map_coordinates

def calculate_speeds(IM, n):
    # This section is to get the speed distribution.
    # The original matlab version used an analytical formula to get the speed distribution directly
    # from the basis coefficients. But, the C version of BASEX uses a numerical method similar to
    # the one implemented here. The difference between the two methods is negligable.
    """ Generating the speed distribution """

    nx,ny = np.shape(IM)
    xi = np.linspace(-100, 100, nx)
    yi = np.linspace(-100, 100, ny)
    X,Y = np.meshgrid(xi, yi)

    polarIM, ri, thetai = reproject_image_into_polar(IM)

    speeds = np.sum(polarIM, axis=1)
    speeds = speeds[:n//2] #Clip off the corners

    return speeds

def center_image(data, center, n, ndim=2):
    """ This centers the image at the given center and makes it of size n by n"""

    Nh,Nw = data.shape
    n_2 = n//2
    if ndim == 1:
        cx = int(center)
        im = np.zeros((1, 2*n))
        im[0, n-cx:n-cx+Nw] = data
        im = im[:, n_2:n+n_2]
        # This is really not efficient
        # Processing 2D image with identical rows while we just want a
        # 1D slice
        im = np.repeat(im, n, axis=0)

    elif ndim == 2:
        cx, cy = np.asarray(center, dtype='int')

        #make an array of zeros that is large enough for cropping or padding:
        sz = 2*np.round(n + np.max((Nw, Nh)))
        im = np.zeros((sz, sz))
        im[sz//2-cy:sz//2-cy+Nh, sz//2-cx:sz//2-cx+Nw] = data

        if n%2==0:
            # If the image size is even, use lower left of the central 4 pixels as center
            im = im[ sz//2-n_2-1:n_2+sz//2, sz//2-n_2-1:n_2+sz//2]
        if n%2==1:
            # If the image size is odd, set the central pixel to be exactly center
            im = im[ sz//2-n_2:n_2+sz//2+1, sz//2-n_2:n_2+sz//2+1]

        #print(np.shape(im))
    else:
        raise ValueError

    return im

# I got these next two functions from a stackoverflow page and slightly modified them.
# http://stackoverflow.com/questions/3798333/image-information-along-a-polar-coordinate-system
# It is possible that there is a faster way to get the speed distribution.
# If you figure it out, pease let me know! (danhickstein@gmail.com)

def reproject_image_into_polar(data, origin=None):
    """Reprojects a 2D numpy array ("data") into a polar coordinate system.
    "origin" is a tuple of (x0, y0) and defaults to the center of the image.
    """
    ny, nx = data.shape[:2]
    if origin is None:
        origin = (nx//2, ny//2)

    # Determine that the min and max r and theta coords will be...
    x, y = index_coords(data, origin=origin)
    r, theta = cart2polar(x, y)

    nr = r.max()
    nt = ny//2

    # Make a regular (in polar space) grid based on the min and max r & theta
    r_i = np.linspace(r.min(), r.max(), nr)
    theta_i = np.linspace(theta.min(), theta.max(), nt)
    theta_grid, r_grid = np.meshgrid(theta_i, r_i)

    # Project the r and theta grid back into pixel coordinates
    X, Y = polar2cart(r_grid, theta_grid)
    X += origin[0] # We need to shift the origin
    Y += origin[1] # back to the lower-left corner...
    xi, yi = X.flatten(), Y.flatten()
    coords = np.vstack((xi, yi)) # (map_coordinates requires a 2xn array)

    zi = map_coordinates(data, coords)
    output = zi.reshape((nr, nt))
    return output, r_i, theta_i

def index_coords(data, origin=None):
    """Creates x & y coords for the indicies in a numpy array "data".
    "origin" defaults to the center of the image. Specify origin=(0,0)
    to set the origin to the lower left corner of the image.
    """
    ny, nx = data.shape[:2]
    if origin is None:
        origin_x, origin_y = nx // 2, ny // 2
    else:
        origin_x, origin_y = origin
    x, y = np.meshgrid(np.arange(nx), np.arange(ny))
    x -= origin_x
    y -= origin_y
    return x, y

def cart2polar(x, y):
    """
    Transform carthesian coordinates to polar
    """
    r = np.sqrt(x**2 + y**2)
    theta = np.arctan2(y, x)
    return r, theta

def polar2cart(r, theta):
    """
    Transform polar coordinates to carthesian
    """
    x = r * np.sin(theta)
    y = r * np.cos(theta)
    return x, y
<commit_msg>Made center one line<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np
from scipy.ndimage import map_coordinates

def calculate_speeds(IM, n):
    # This section is to get the speed distribution.
    # The original matlab version used an analytical formula to get the speed distribution directly
    # from the basis coefficients. But, the C version of BASEX uses a numerical method similar to
    # the one implemented here. The difference between the two methods is negligable.
    """ Generating the speed distribution """

    nx,ny = np.shape(IM)
    xi = np.linspace(-100, 100, nx)
    yi = np.linspace(-100, 100, ny)
    X,Y = np.meshgrid(xi, yi)

    polarIM, ri, thetai = reproject_image_into_polar(IM)

    speeds = np.sum(polarIM, axis=1)
    speeds = speeds[:n//2] #Clip off the corners

    return speeds

def center_image(data, center, n, ndim=2):
    """ This centers the image at the given center and makes it of size n by n"""

    Nh,Nw = data.shape
    n_2 = n//2
    if ndim == 1:
        cx = int(center)
        im = np.zeros((1, 2*n))
        im[0, n-cx:n-cx+Nw] = data
        im = im[:, n_2:n+n_2]
        # This is really not efficient
        # Processing 2D image with identical rows while we just want a
        # 1D slice
        im = np.repeat(im, n, axis=0)

    elif ndim == 2:
        cx, cy = np.asarray(center, dtype='int')

        #make an array of zeros that is large enough for cropping or padding:
        sz = 2*np.round(n + np.max((Nw, Nh)))
        im = np.zeros((sz, sz))
        im[sz//2-cy:sz//2-cy+Nh, sz//2-cx:sz//2-cx+Nw] = data

        im = im[ sz//2-n_2:n_2+sz//2+1, sz//2-n_2:n_2+sz//2+1]

        #print(np.shape(im))
    else:
        raise ValueError

    return im

# I got these next two functions from a stackoverflow page and slightly modified them.
# http://stackoverflow.com/questions/3798333/image-information-along-a-polar-coordinate-system
# It is possible that there is a faster way to get the speed distribution.
# If you figure it out, pease let me know! (danhickstein@gmail.com)

def reproject_image_into_polar(data, origin=None):
    """Reprojects a 2D numpy array ("data") into a polar coordinate system.
    "origin" is a tuple of (x0, y0) and defaults to the center of the image.
    """
    ny, nx = data.shape[:2]
    if origin is None:
        origin = (nx//2, ny//2)

    # Determine that the min and max r and theta coords will be...
    x, y = index_coords(data, origin=origin)
    r, theta = cart2polar(x, y)

    nr = r.max()
    nt = ny//2

    # Make a regular (in polar space) grid based on the min and max r & theta
    r_i = np.linspace(r.min(), r.max(), nr)
    theta_i = np.linspace(theta.min(), theta.max(), nt)
    theta_grid, r_grid = np.meshgrid(theta_i, r_i)

    # Project the r and theta grid back into pixel coordinates
    X, Y = polar2cart(r_grid, theta_grid)
    X += origin[0] # We need to shift the origin
    Y += origin[1] # back to the lower-left corner...
    xi, yi = X.flatten(), Y.flatten()
    coords = np.vstack((xi, yi)) # (map_coordinates requires a 2xn array)

    zi = map_coordinates(data, coords)
    output = zi.reshape((nr, nt))
    return output, r_i, theta_i

def index_coords(data, origin=None):
    """Creates x & y coords for the indicies in a numpy array "data".
    "origin" defaults to the center of the image. Specify origin=(0,0)
    to set the origin to the lower left corner of the image.
    """
    ny, nx = data.shape[:2]
    if origin is None:
        origin_x, origin_y = nx // 2, ny // 2
    else:
        origin_x, origin_y = origin
    x, y = np.meshgrid(np.arange(nx), np.arange(ny))
    x -= origin_x
    y -= origin_y
    return x, y

def cart2polar(x, y):
    """
    Transform carthesian coordinates to polar
    """
    r = np.sqrt(x**2 + y**2)
    theta = np.arctan2(y, x)
    return r, theta

def polar2cart(r, theta):
    """
    Transform polar coordinates to carthesian
    """
    x = r * np.sin(theta)
    y = r * np.cos(theta)
    return x, y
<|endoftext|>


<commit_before>from __future__ import absolute_import
__author__ = 'Aneil Mallavarapu (http://github.com/aneilbaboo)'

from datetime import datetime

import six
import dateutil.parser

from .errors import InvalidTypeException

def default_timestamp_parser(s):
    try:
        if dateutil.parser.parse(s):
            return True
        else:
            return False
    except:
        return False

def schema_from_record(record, timestamp_parser=default_timestamp_parser):
    """Generate a BigQuery schema given an example of a record that is to be
    inserted into BigQuery.

    Parameters
    ----------
    record : dict
        Example of a record that is to be inserted into BigQuery
    timestamp_parser : function, optional
        Unary function taking a ``str`` and returning and ``bool`` that is
        True if the string represents a date

    Returns
    -------
    Schema: list
    """
    return [describe_field(k, v, timestamp_parser=timestamp_parser)
            for k, v in list(record.items())]

def describe_field(k, v, timestamp_parser=default_timestamp_parser):
    """Given a key representing a column name and value representing the value
    stored in the column, return a representation of the BigQuery schema
    element describing that field. Raise errors if invalid value types are
    provided.

    Parameters
    ----------
    k : Union[str, unicode]
        Key representing the column
    v : Union[str, unicode, int, float, datetime, object]
        Value mapped to by `k`

    Returns
    -------
    object
        Describing the field

    Raises
    ------
    Exception
        If invalid value types are provided.

    Examples
    --------
    >>> describe_field("username", "Bob")
    {"name": "username", "type": "string", "mode": "nullable"}
    >>> describe_field("users", [{"username": "Bob"}])
    {"name": "users", "type": "record", "mode": "repeated",
     "fields": [{"name":"username","type":"string","mode":"nullable"}]}
    """

    def bq_schema_field(name, bq_type, mode):
        return {"name": name, "type": bq_type, "mode": mode}

    if isinstance(v, list):
        if len(v) == 0:
            raise Exception(
                "Can't describe schema because of empty list {0}:[]".format(k))
        v = v[0]
        mode = "repeated"
    else:
        mode = "nullable"

    bq_type = bigquery_type(v, timestamp_parser=timestamp_parser)
    if not bq_type:
        raise InvalidTypeException(k, v)

    field = bq_schema_field(k, bq_type, mode)
    if bq_type == "record":
        try:
            field['fields'] = schema_from_record(v, timestamp_parser)
        except InvalidTypeException as e:
            # recursively construct the key causing the error
            raise InvalidTypeException("%s.%s" % (k, e.key), e.value)

    return field

def bigquery_type(o, timestamp_parser=default_timestamp_parser):
    """Given a value, return the matching BigQuery type of that value. Must be
    one of str/unicode/int/float/datetime/record, where record is a dict
    containing value which have matching BigQuery types.

    Parameters
    ----------
    o : object
        A Python object
    time_stamp_parser : function, optional
        Unary function taking a ``str`` and returning and ``bool`` that is
        True if the string represents a date

    Returns
    -------
    Union[str, None]
        Name of the corresponding BigQuery type for `o`, or None if no type
        could be found

    Examples
    --------
    >>> bigquery_type("abc")
    "string"
    >>> bigquery_type(123)
    "integer"
    """

    t = type(o)
    if t == int:
        return "integer"
    elif (t == six.binary_type and six.PY2) or t == six.text_type:
        if timestamp_parser and timestamp_parser(o):
            return "timestamp"
        else:
            return "string"
    elif t == float:
        return "float"
    elif t == bool:
        return "boolean"
    elif t == dict:
        return "record"
    elif t == datetime:
        return "timestamp"
    else:
        return None  # failed to find a type
<commit_msg>Add support for long in schema_builder<commit_after>from __future__ import absolute_import
__author__ = 'Aneil Mallavarapu (http://github.com/aneilbaboo)'

from datetime import datetime

import six
import dateutil.parser

from .errors import InvalidTypeException

def default_timestamp_parser(s):
    try:
        if dateutil.parser.parse(s):
            return True
        else:
            return False
    except:
        return False

def schema_from_record(record, timestamp_parser=default_timestamp_parser):
    """Generate a BigQuery schema given an example of a record that is to be
    inserted into BigQuery.

    Parameters
    ----------
    record : dict
        Example of a record that is to be inserted into BigQuery
    timestamp_parser : function, optional
        Unary function taking a ``str`` and returning and ``bool`` that is
        True if the string represents a date

    Returns
    -------
    Schema: list
    """
    return [describe_field(k, v, timestamp_parser=timestamp_parser)
            for k, v in list(record.items())]

def describe_field(k, v, timestamp_parser=default_timestamp_parser):
    """Given a key representing a column name and value representing the value
    stored in the column, return a representation of the BigQuery schema
    element describing that field. Raise errors if invalid value types are
    provided.

    Parameters
    ----------
    k : Union[str, unicode]
        Key representing the column
    v : Union[str, unicode, int, float, datetime, object]
        Value mapped to by `k`

    Returns
    -------
    object
        Describing the field

    Raises
    ------
    Exception
        If invalid value types are provided.

    Examples
    --------
    >>> describe_field("username", "Bob")
    {"name": "username", "type": "string", "mode": "nullable"}
    >>> describe_field("users", [{"username": "Bob"}])
    {"name": "users", "type": "record", "mode": "repeated",
     "fields": [{"name":"username","type":"string","mode":"nullable"}]}
    """

    def bq_schema_field(name, bq_type, mode):
        return {"name": name, "type": bq_type, "mode": mode}

    if isinstance(v, list):
        if len(v) == 0:
            raise Exception(
                "Can't describe schema because of empty list {0}:[]".format(k))
        v = v[0]
        mode = "repeated"
    else:
        mode = "nullable"

    bq_type = bigquery_type(v, timestamp_parser=timestamp_parser)
    if not bq_type:
        raise InvalidTypeException(k, v)

    field = bq_schema_field(k, bq_type, mode)
    if bq_type == "record":
        try:
            field['fields'] = schema_from_record(v, timestamp_parser)
        except InvalidTypeException as e:
            # recursively construct the key causing the error
            raise InvalidTypeException("%s.%s" % (k, e.key), e.value)

    return field

def bigquery_type(o, timestamp_parser=default_timestamp_parser):
    """Given a value, return the matching BigQuery type of that value. Must be
    one of str/unicode/int/float/datetime/record, where record is a dict
    containing value which have matching BigQuery types.

    Parameters
    ----------
    o : object
        A Python object
    time_stamp_parser : function, optional
        Unary function taking a ``str`` and returning and ``bool`` that is
        True if the string represents a date

    Returns
    -------
    Union[str, None]
        Name of the corresponding BigQuery type for `o`, or None if no type
        could be found

    Examples
    --------
    >>> bigquery_type("abc")
    "string"
    >>> bigquery_type(123)
    "integer"
    """

    t = type(o)
    if t == int or t == long:
        return "integer"
    elif (t == six.binary_type and six.PY2) or t == six.text_type:
        if timestamp_parser and timestamp_parser(o):
            return "timestamp"
        else:
            return "string"
    elif t == float:
        return "float"
    elif t == bool:
        return "boolean"
    elif t == dict:
        return "record"
    elif t == datetime:
        return "timestamp"
    else:
        return None  # failed to find a type
<|endoftext|>


<commit_before>import argparse

import whack.args

env_default = whack.args.env_default(prefix="WHACK")

def main(argv, create_operations):
    args = parse_args(argv)
    operations = create_operations(
        caching_enabled=args.caching_enabled,
        indices=args.indices,
        enable_build=args.enable_build,
    )
    args.func(operations, args)

def parse_args(argv):
    commands = [
        InstallCommand("install"),
        InstallCommand("get-package"),
        DeployCommand(),
        CreateSourceTarballCommand(),
        BuildPackageTarballCommand(),
    ]

    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()

    for command in commands:
        subparser = subparsers.add_parser(command.name)
        _add_common_args(subparser)
        subparser.set_defaults(func=command.execute)
        command.create_parser(subparser)

    return parser.parse_args(argv[1:])

class KeyValueAction(argparse.Action):
    def __init__(self, default=None, **kwargs):
        if default is None:
            default = {}

        super(type(self), self).__init__(default=default, **kwargs)

    def __call__(self, parser, namespace, values, option_string=None):
        if getattr(namespace, self.dest, None) is None:
            setattr(namespace, self.dest, {})

        pairs = getattr(namespace, self.dest)
        if "=" in values:
            key, value = values.split("=", 1)
            pairs[key] = value
        else:
            pairs[values] = ""

        setattr(namespace, self.dest, pairs)

class InstallCommand(object):
    def __init__(self, name):
        self.name = name

    def create_parser(self, subparser):
        subparser.add_argument('package')
        subparser.add_argument('target_dir', metavar="target-dir")
        _add_build_params_args(subparser)

    def execute(self, operations, args):
        if self.name == "install":
            operation = operations.install
        elif self.name == "get-package":
            operation = operations.get_package
        else:
            raise Exception("Unrecognised operation")

        operation(args.package, args.target_dir, params=args.params)

class DeployCommand(object):
    name = "deploy"

    def create_parser(self, subparser):
        subparser.add_argument('package_dir', metavar="package-dir")

        target_group = subparser.add_mutually_exclusive_group(required=True)
        target_group.add_argument("--in-place", action="store_true")
        target_group.add_argument("target_dir", metavar="target-dir", nargs="?")

    def execute(self, operations, args):
        operations.deploy(args.package_dir, args.target_dir)

class CreateSourceTarballCommand(object):
    name = "create-source-tarball"

    def create_parser(self, subparser):
        subparser.add_argument("package_source_dir", metavar="package-source-dir")
        subparser.add_argument("source_tarball_dir", metavar="source-tarball-dir")

    def execute(self, operations, args):
        source_tarball = operations.create_source_tarball(
            args.package_source_dir,
            args.source_tarball_dir
        )
        print source_tarball.full_name
        print source_tarball.path

class BuildPackageTarballCommand(object):
    name = "get-package-tarball"

    def create_parser(self, subparser):
        subparser.add_argument("package")
        subparser.add_argument("package_tarball_dir", metavar="package-tarball-dir")
        _add_build_params_args(subparser)

    def execute(self, operations, args):
        package_tarball = operations.get_package_tarball(
            args.package,
            args.package_tarball_dir,
            params=args.params,
        )
        print package_tarball.path

def _add_common_args(parser):
    _add_caching_args(parser)
    _add_index_args(parser)
    _add_build_args(parser)

def _add_caching_args(parser):
    parser.add_argument("--disable-cache", action="store_false", dest="caching_enabled")

def _add_index_args(parser):
    parser.add_argument("--add-index", action="append", default=[], dest="indices")

def _add_build_args(parser):
    parser.add_argument("--disable-build", action="store_false", dest="enable_build")

def _add_build_params_args(parser):
    parser.add_argument(
        "--add-parameter", "-p",
        action=KeyValueAction,
        dest="params",
        metavar="KEY=VALUE",
    )
<commit_msg>Change metavar for --add-index to be singular<commit_after>import argparse

import whack.args

env_default = whack.args.env_default(prefix="WHACK")

def main(argv, create_operations):
    args = parse_args(argv)
    operations = create_operations(
        caching_enabled=args.caching_enabled,
        indices=args.indices,
        enable_build=args.enable_build,
    )
    args.func(operations, args)

def parse_args(argv):
    commands = [
        InstallCommand("install"),
        InstallCommand("get-package"),
        DeployCommand(),
        CreateSourceTarballCommand(),
        BuildPackageTarballCommand(),
    ]

    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()

    for command in commands:
        subparser = subparsers.add_parser(command.name)
        _add_common_args(subparser)
        subparser.set_defaults(func=command.execute)
        command.create_parser(subparser)

    return parser.parse_args(argv[1:])

class KeyValueAction(argparse.Action):
    def __init__(self, default=None, **kwargs):
        if default is None:
            default = {}

        super(type(self), self).__init__(default=default, **kwargs)

    def __call__(self, parser, namespace, values, option_string=None):
        if getattr(namespace, self.dest, None) is None:
            setattr(namespace, self.dest, {})

        pairs = getattr(namespace, self.dest)
        if "=" in values:
            key, value = values.split("=", 1)
            pairs[key] = value
        else:
            pairs[values] = ""

        setattr(namespace, self.dest, pairs)

class InstallCommand(object):
    def __init__(self, name):
        self.name = name

    def create_parser(self, subparser):
        subparser.add_argument('package')
        subparser.add_argument('target_dir', metavar="target-dir")
        _add_build_params_args(subparser)

    def execute(self, operations, args):
        if self.name == "install":
            operation = operations.install
        elif self.name == "get-package":
            operation = operations.get_package
        else:
            raise Exception("Unrecognised operation")

        operation(args.package, args.target_dir, params=args.params)

class DeployCommand(object):
    name = "deploy"

    def create_parser(self, subparser):
        subparser.add_argument('package_dir', metavar="package-dir")

        target_group = subparser.add_mutually_exclusive_group(required=True)
        target_group.add_argument("--in-place", action="store_true")
        target_group.add_argument("target_dir", metavar="target-dir", nargs="?")

    def execute(self, operations, args):
        operations.deploy(args.package_dir, args.target_dir)

class CreateSourceTarballCommand(object):
    name = "create-source-tarball"

    def create_parser(self, subparser):
        subparser.add_argument("package_source_dir", metavar="package-source-dir")
        subparser.add_argument("source_tarball_dir", metavar="source-tarball-dir")

    def execute(self, operations, args):
        source_tarball = operations.create_source_tarball(
            args.package_source_dir,
            args.source_tarball_dir
        )
        print source_tarball.full_name
        print source_tarball.path

class BuildPackageTarballCommand(object):
    name = "get-package-tarball"

    def create_parser(self, subparser):
        subparser.add_argument("package")
        subparser.add_argument("package_tarball_dir", metavar="package-tarball-dir")
        _add_build_params_args(subparser)

    def execute(self, operations, args):
        package_tarball = operations.get_package_tarball(
            args.package,
            args.package_tarball_dir,
            params=args.params,
        )
        print package_tarball.path

def _add_common_args(parser):
    _add_caching_args(parser)
    _add_index_args(parser)
    _add_build_args(parser)

def _add_caching_args(parser):
    parser.add_argument("--disable-cache", action="store_false", dest="caching_enabled")

def _add_index_args(parser):
    parser.add_argument(
        "--add-index",
        action="append",
        default=[],
        dest="indices",
        metavar="INDEX",
    )

def _add_build_args(parser):
    parser.add_argument("--disable-build", action="store_false", dest="enable_build")

def _add_build_params_args(parser):
    parser.add_argument(
        "--add-parameter", "-p",
        action=KeyValueAction,
        dest="params",
        metavar="KEY=VALUE",
    )
<|endoftext|>


<commit_before>""" Contains PascalVOC dataset and labels for different tasks """

import os
from os.path import dirname, basename
from io import BytesIO
import tarfile
import tempfile
from collections import defaultdict

import PIL
import tqdm
import numpy as np
import requests

from . import ImagesOpenset
from .. import DatasetIndex

class BasePascal(ImagesOpenset):
    """ The base class for PascalVOC dataset.
    The archive contains 17125 images. Total size 1.9GB.
    You can unpack the archive to the directory its been downloaded by specifing `unpack` flag.

    Tracks of the PascalVOC challenge:
        1. Classification
        2. Detection
        3. Segmentation
        4. Action Classification Task
        5. Boxless Action Classification
        6. Person Layout

    Notes
    -----
    Each track contains only the subset of the total images with labels provided.
    """
    SOURCE_URL = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar'
    SETS_PATH = 'VOCdevkit/VOC2012/ImageSets'
    task = None

    def __init__(self, *args, unpack=False, preloaded=None, train_test=True, **kwargs):
        self.localname = None
        super().__init__(*args, preloaded=preloaded, train_test=train_test, **kwargs)
        if unpack:
            with tarfile.open((self.localname), "r") as archive:
                archive.extractall(dirname(self.localname))

    def download_archive(self, path=None):
        """ Download archive"""
        if path is None:
            path = tempfile.gettempdir()
        filename = os.path.basename(self.SOURCE_URL)
        localname = os.path.join(path, filename)
        self.localname = localname

        if not os.path.isfile(localname):
            r = requests.get(self.SOURCE_URL, stream=True)
            file_size = int(r.headers['Content-Length'])
            chunk = 1
            chunk_size = 1024
            num_bars = int(file_size / chunk_size)
            with open(localname, 'wb') as f:
                for chunk in tqdm.tqdm(r.iter_content(chunk_size=chunk_size), total=num_bars, unit='MB',
                                       desc=filename, leave=True):
                    f.write(chunk)

    def _name(self, path):
        """ Return file name without format """
        return basename(path).split('.')[0]

    def _imagepath(self, name):
        """ Return the path to the .jpg image in the archive by its name """
        return os.path.join(dirname(self.SETS_PATH), 'JPEGImages', name + '.jpg')

    def _extract(self, archive, file):
        data = archive.extractfile(file).read()
        return PIL.Image.open(BytesIO(data))

    def _get_ids(self, archive, part):
        """ Train and test images ids are located in specific for each task folder"""
        part_path = os.path.join(self.SETS_PATH, self.task, part) + '.txt'
        raw_ids = archive.extractfile(part_path)
        list_ids = raw_ids.read().decode().split('\n')
        return list_ids[:-1]

class PascalSegmentation(BasePascal):
    """ Contains 2913 images and masks.

    Notes
    -----
    Index 0 corresponds to background and index 255 corresponds to 'void' or unlabelled.
    """
    task = 'Segmentation'
    classes = ['background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair',
               'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa',
               'train', 'tvmonitor']

    def _maskpath(self, name):
        """ Return the path in the archive to the mask which is .png image by its name"""
        return os.path.join(dirname(self.SETS_PATH), 'SegmentationClass', name + '.png')

    def download(self, path):
        self.download_archive(path)
        with tarfile.open(self.localname, "r") as archive:
            train_ids = self._get_ids(archive, 'train')
            test_ids = self._get_ids(archive, 'val')

            train_images = np.array([self._extract(archive, self._imagepath(name)) \
                                     for name in train_ids], dtype=object)
            test_images = np.array([self._extract(archive, self._imagepath(name)) \
                                    for name in test_ids], dtype=object)

            train_masks = np.array([self._extract(archive, self._maskpath(name)) \
                                    for name in train_ids], dtype=object)
            test_masks = np.array([self._extract(archive, self._maskpath(name)) \
                                   for name in test_ids], dtype=object)

            self._train_index = DatasetIndex(np.arange(len(train_images)))
            self._test_index = DatasetIndex(np.arange(len(test_images)))

            return (train_images, train_masks), (test_images, test_masks)

class PascalClassification(BasePascal):
    """ Contains 11540 images and corresponding classes

    Notes
    -----
    - Labels are represented by the one-hot vector of size 20. '1' stands for the presence of at least one object from
    coresponding class on the image. '-1' stands for the absence. '0' indicates that the object is presented,
    but can hardly be detected.
    """
    task = 'Main'
    classes = ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow',
               'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train',
               'tvmonitor']

    def download(self, path):
        self.download_archive()
        with tarfile.open(self.localname, "r") as archive:
            d = defaultdict(list)
            class_files = [os.path.join(self.SETS_PATH, self.task, name) + '_trainval.txt' for name in self.classes]
            for class_file in class_files:
                data = archive.extractfile(class_file).read()
                for row in data.decode().split('\n')[:-1]:
                    key = row.split()[0]
                    value = int(row.split()[1])
                    d[key].append(value)

            train_ids = self._get_ids(archive, 'train')
            test_ids = self._get_ids(archive, 'val')

            train_images = np.array([self._extract(archive, self._imagepath(name)) for name in train_ids], dtype=object)
            test_images = np.array([self._extract(archive, self._imagepath(name)) for name in test_ids], dtype=object)

            train_labels = np.array([d[self._name(name)] for name in train_ids])
            test_labels = np.array([d[self._name(name)] for name in test_ids])

            self._train_index = DatasetIndex(np.arange(len(train_images)))
            self._test_index = DatasetIndex(np.arange(len(test_images)))

        return (train_images, train_labels), (test_images, test_labels)
<commit_msg>KB to MB<commit_after>""" Contains PascalVOC dataset and labels for different tasks """

import os
from os.path import dirname, basename
from io import BytesIO
import tarfile
import tempfile
from collections import defaultdict

import PIL
import tqdm
import numpy as np
import requests

from . import ImagesOpenset
from .. import DatasetIndex

class BasePascal(ImagesOpenset):
    """ The base class for PascalVOC dataset.
    The archive contains 17125 images. Total size 1.9GB.
    You can unpack the archive to the directory its been downloaded by specifing `unpack` flag.

    Tracks of the PascalVOC challenge:
        1. Classification
        2. Detection
        3. Segmentation
        4. Action Classification Task
        5. Boxless Action Classification
        6. Person Layout

    Notes
    -----
    Each track contains only the subset of the total images with labels provided.
    """
    SOURCE_URL = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar'
    SETS_PATH = 'VOCdevkit/VOC2012/ImageSets'
    task = None

    def __init__(self, *args, unpack=False, preloaded=None, train_test=True, **kwargs):
        self.localname = None
        super().__init__(*args, preloaded=preloaded, train_test=train_test, **kwargs)
        if unpack:
            with tarfile.open((self.localname), "r") as archive:
                archive.extractall(dirname(self.localname))

    def download_archive(self, path=None):
        """ Download archive"""
        if path is None:
            path = tempfile.gettempdir()
        filename = os.path.basename(self.SOURCE_URL)
        localname = os.path.join(path, filename)
        self.localname = localname

        if not os.path.isfile(localname):
            r = requests.get(self.SOURCE_URL, stream=True)
            file_size = int(r.headers['Content-Length'])
            chunk = 1
            chunk_size = 1024
            num_bars = int(file_size / chunk_size)
            with open(localname, 'wb') as f:
                for chunk in tqdm.tqdm(r.iter_content(chunk_size=chunk_size), total=num_bars, unit='KB',
                                       desc=filename, leave=True):
                    f.write(chunk)

    def _name(self, path):
        """ Return file name without format """
        return basename(path).split('.')[0]

    def _imagepath(self, name):
        """ Return the path to the .jpg image in the archive by its name """
        return os.path.join(dirname(self.SETS_PATH), 'JPEGImages', name + '.jpg')

    def _extract(self, archive, file):
        data = archive.extractfile(file).read()
        return PIL.Image.open(BytesIO(data))

    def _get_ids(self, archive, part):
        """ Train and test images ids are located in specific for each task folder"""
        part_path = os.path.join(self.SETS_PATH, self.task, part) + '.txt'
        raw_ids = archive.extractfile(part_path)
        list_ids = raw_ids.read().decode().split('\n')
        return list_ids[:-1]

class PascalSegmentation(BasePascal):
    """ Contains 2913 images and masks.

    Notes
    -----
    Index 0 corresponds to background and index 255 corresponds to 'void' or unlabelled.
    """
    task = 'Segmentation'
    classes = ['background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair',
               'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa',
               'train', 'tvmonitor']

    def _maskpath(self, name):
        """ Return the path in the archive to the mask which is .png image by its name"""
        return os.path.join(dirname(self.SETS_PATH), 'SegmentationClass', name + '.png')

    def download(self, path):
        self.download_archive(path)
        with tarfile.open(self.localname, "r") as archive:
            train_ids = self._get_ids(archive, 'train')
            test_ids = self._get_ids(archive, 'val')

            train_images = np.array([self._extract(archive, self._imagepath(name)) \
                                     for name in train_ids], dtype=object)
            test_images = np.array([self._extract(archive, self._imagepath(name)) \
                                    for name in test_ids], dtype=object)

            train_masks = np.array([self._extract(archive, self._maskpath(name)) \
                                    for name in train_ids], dtype=object)
            test_masks = np.array([self._extract(archive, self._maskpath(name)) \
                                   for name in test_ids], dtype=object)

            self._train_index = DatasetIndex(np.arange(len(train_images)))
            self._test_index = DatasetIndex(np.arange(len(test_images)))

            return (train_images, train_masks), (test_images, test_masks)

class PascalClassification(BasePascal):
    """ Contains 11540 images and corresponding classes

    Notes
    -----
    - Labels are represented by the one-hot vector of size 20. '1' stands for the presence of at least one object from
    coresponding class on the image. '-1' stands for the absence. '0' indicates that the object is presented,
    but can hardly be detected.
    """
    task = 'Main'
    classes = ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow',
               'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train',
               'tvmonitor']

    def download(self, path):
        self.download_archive()
        with tarfile.open(self.localname, "r") as archive:
            d = defaultdict(list)
            class_files = [os.path.join(self.SETS_PATH, self.task, name) + '_trainval.txt' for name in self.classes]
            for class_file in class_files:
                data = archive.extractfile(class_file).read()
                for row in data.decode().split('\n')[:-1]:
                    key = row.split()[0]
                    value = int(row.split()[1])
                    d[key].append(value)

            train_ids = self._get_ids(archive, 'train')
            test_ids = self._get_ids(archive, 'val')

            train_images = np.array([self._extract(archive, self._imagepath(name)) for name in train_ids], dtype=object)
            test_images = np.array([self._extract(archive, self._imagepath(name)) for name in test_ids], dtype=object)

            train_labels = np.array([d[self._name(name)] for name in train_ids])
            test_labels = np.array([d[self._name(name)] for name in test_ids])

            self._train_index = DatasetIndex(np.arange(len(train_images)))
            self._test_index = DatasetIndex(np.arange(len(test_images)))

        return (train_images, train_labels), (test_images, test_labels)
<|endoftext|>