<commit_before>#!/usr/bin/env python

import csv
import json
import sys
import traceback

#: which columns are always lists
list_cols={"collections", "dm", "homophones", "tags", "pronunciations", "small_collection"}
#: which columns might be lists if space is present
list_maybe={"word"}
#: which columns are for the google spreadsheet only ie not to be put into the output.
list_ignore_columns={"Collection changes", "Recording Needed", "Recording notes"}
#: which collections to ignore.
list_ignore_collections={"EXCLUDE", "pickup", "hard_medical"}
def process_token(v):
    v=v.strip()
    return v

def process_dict(v, list=False):
    d={}
    for i in v.split():
        l=i.split(":", 1)
        if list:
            d[l.pop().strip()] = [l.pop().strip()]
        else:
            d[l.pop().strip()] = l.pop().strip()
    return d

def convert(infile, outfile):
    incsv=csv.DictReader(infile)
    res=[]
    lastgoodline=None
    for lineno, line in enumerate(incsv):
        try:
            row=line.copy()
            if row["collections"] in list_ignore_collections:
                continue
            for k,v in row.items():
                if not v or (k in list_ignore_columns):
                    del row[k]
                    continue
                if ":" in v:
                    if "[" in v:
                        #find and process the embedded list separately then reassemble
                        lvwz=v[v.find("[")+1:v.find("]")]
                        lvwz=[process_token(lvwz) for lvwz in lvwz.split()]

                        v=v[:v.find("[")]+"lvwz"+v[v.find("]")+1:]
                        d=process_dict(v, k in list_cols)
                        for m,n in d.items():
                            if n[0] == "lvwz":
                                d[m]=lvwz
                        row[k]=d
                    else:
                        row[k]=process_dict(v, k in list_cols)
                elif k in list_cols or (k in list_maybe and len(v.split())>1):
                    row[k]=[process_token(v) for v in v.split()]
                elif k in list_maybe:
                    row[k]=process_token(v)

            res.append(row)
            lastgoodline=line
        except:
            traceback.print_exc()
            print >> sys.stderr, "While processing row #", lineno+2
            print >> sys.stderr, "Last good line\n"+json.dumps(lastgoodline, sort_keys=True, indent=4)
            sys.exit(1)

    print >> outfile, json.dumps({"words": res}, sort_keys=True, indent=4)

if __name__=='__main__':
    import argparse

    p=argparse.ArgumentParser(description="Converts the word list from CSV into json")
    p.add_argument("--input", default=sys.stdin, type=argparse.FileType('r'), help="Input file [stdin]")
    p.add_argument("--output", default=sys.stdout, type=argparse.FileType('w'), help="Output file [stdout]")

    options=p.parse_args()

    if options.input.isatty():
        p.error("Input is a terminal.  Pipe files or provide filenames")

    convert(options.input, options.output)
<commit_msg>Getting tool working with explanations<commit_after>#!/usr/bin/env python

import csv
import json
import sys
import traceback

#: which columns are always lists
list_cols={"collections", "dm", "homophones", "tags", "pronunciations", "small_collection", "explanations"}
#: which columns might be lists if space is present
list_maybe={"word"}
#: which columns are for the google spreadsheet only ie not to be put into the output.
list_ignore_columns={"Collection changes", "Recording Needed", "Recording notes", "frequent_2000_fifths"}
#: which collections to ignore.
list_ignore_collections={"EXCLUDE", "pickup", "hard_medical"}
def process_token(v):
    v=v.strip()
    return v

def process_dict(v, list=False):
    d={}
    for i in v.split():
        l=i.split(":", 1)
        if list:
            d[l.pop().strip()] = [l.pop().strip()]
        else:
            d[l.pop().strip()] = l.pop().strip()
    return d

def convert(infile, outfile):
    incsv=csv.DictReader(infile)
    res=[]
    lastgoodline=None
    for lineno, line in enumerate(incsv):
        try:
            row=line.copy()
            if row["collections"] in list_ignore_collections or row["collections"].startswith("tbr_"):
                continue
            for k,v in row.items():
                if not v or (k in list_ignore_columns):
                    del row[k]
                    continue
                if ":" in v:
                    if "[" in v:
                        #find and process the embedded list separately then reassemble
                        lvwz=v[v.find("[")+1:v.find("]")]
                        lvwz=[process_token(lvwz) for lvwz in lvwz.split()]

                        v=v[:v.find("[")]+"lvwz"+v[v.find("]")+1:]
                        d=process_dict(v, k in list_cols)
                        for m,n in d.items():
                            if n[0] == "lvwz":
                                d[m]=lvwz
                        row[k]=d
                    else:
                        row[k]=process_dict(v, k in list_cols)
                elif k in list_cols or (k in list_maybe and len(v.split())>1):
                    row[k]=[process_token(v) for v in v.split()]
                elif k in list_maybe:
                    row[k]=process_token(v)

            res.append(row)
            lastgoodline=line
        except:
            traceback.print_exc()
            print >> sys.stderr, "While processing row #", lineno+2
            print >> sys.stderr, "Last good line\n"+json.dumps(lastgoodline, sort_keys=True, indent=4)
            sys.exit(1)

    print >> outfile, json.dumps({"words": res}, sort_keys=True, indent=4)

if __name__=='__main__':
    import argparse

    p=argparse.ArgumentParser(description="Converts the word list from CSV into json")
    p.add_argument("--input", default=sys.stdin, type=argparse.FileType('r'), help="Input file [stdin]")
    p.add_argument("--output", default=sys.stdout, type=argparse.FileType('w'), help="Output file [stdout]")

    options=p.parse_args()

    if options.input.isatty():
        p.error("Input is a terminal.  Pipe files or provide filenames")

    convert(options.input, options.output)
<|endoftext|>


<commit_before>import numpy
import re
import json
import copy

# TODO: rename to QcaDatafile
# TODO: parse dates as datetime
#       e.g. datetime.datetime.strptime(ds, "%a %b %d %H:%M:%S %Y -0600")
# TODO: change date, so it conforms to RFC 2822 (email), e.g. 'Thu, 28 Jun 2001 14:17:15 +0000'

class DatafileError(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

class Datafile:
    EMPTY_DOCUMENT = {
            'info': {
                'program': None,
                'version': None,
                'date': None,
                'dataFormat': None
            },
            'params': None,
            'data': None
        }

    def __init__ (self, filename):
        self._filename = filename
        self._indices = []
        self.parse(filename)

    def parse (self, filename):
        f = open(filename)
        empty = 0
        fpos = 0
        fpos_end = 0
        ln = 1
        ln_end = 0
        ids = self._indices
        # start first index and first block
        ids.append([[(ln, fpos)]])

        for l in f:
            if l.strip(' \t') == '\n':
                if empty == 0:
                    ln_end = ln
                    fpos_end = fpos
                empty += 1

            else:
                cindex = ids[-1]
                cblock = cindex[-1]
                if empty > 0:
                    if (len(cblock) < 2):
                        # no data => begin of data = end of block
                        cblock.append((ln_end, fpos_end))
                    # end of current block
                    cblock.append((ln_end, fpos_end))
                    if empty == 1:
                        # start new block
                        cindex.append([(ln, fpos)])
                    elif empty >= 2:
                        # start new index and new block
                        ids.append([[(ln, fpos)]])
                    if l[0] != '#':
                        # no header for new block
                        ids[-1][-1].append((ln, fpos))
                elif empty == 0 and l[0] != '#' and len(cblock) < 2:
                    # end of header and start of data
                    cblock.append((ln, fpos))
                empty = 0

            ln += 1
            fpos += len(l)

        # close last block
        cindex = ids[-1]
        cblock = cindex[-1]
        if (len(cblock) < 2):
            cblock.append(cblock[0])
        cblock.append((ln, fpos))
        f.close()

    def info (self):
        ids = self._indices
        for i, index in enumerate(ids):
            print "Index", i
            print "   " + str(index[-1][-1][0] - index[0][0][0]) + " lines"
            print
            for j, block in enumerate(index):
                print "   Block", j
                print "      " + str(block[-1][0] - block[0][0]) + " lines"
                print "      Begin Header", block[0]
                print "      Begin Data", block[1]
                print "      End  ", block[2]
            print

    def getAsString (self, index, block=-1):
        start = 0
        end = 0
        if block == -1:
            start = self._indices[index][0][0][1]
            end = self._indices[index][-1][-1][1]
        else:
            start = self._indices[index][block][0][1]
            end = self._indices[index][block][-1][1]
        return self._readfile(start, end)

    def getAsMatrix (self, index, block=-1):
        ls = self.getAsString(index, block)
        return numpy.loadtxt(self._lines(ls))

    def getHeader (self, index, block):
        # read header from file
        start = self._indices[index][block][0][1]
        end = self._indices[index][block][1][1]
        header = self._readfile(start, end)
        # remove leading pound character
        header = re.sub(r'^# ?', '', header, 0, re.MULTILINE)
        # parse header
        version = None
        date = None
        params = None
        tableHeaders = []
        m = re.search(r'^program version: (.+)$', header, re.MULTILINE)
        if m: version = m.group(1)
        m = re.search(r'^date: (.+)$', header, re.MULTILINE)
        if m: date = m.group(1)
        m = re.search(r'^{.*^}', header, re.MULTILINE | re.DOTALL)
        if m:
            paramsJson = m.group(0)
            params = json.loads(paramsJson)
            params = self._updateLeaves(params, self._convert)
        m = re.search(r'\n?.+$', header)
        if m:
            lastLine = m.group(0)
            tableHeaders = lastLine.split()
        return {
                'version': version,
                'date': date,
                'params': params,
                'tableHeaders': tableHeaders
               };

    def getBody (self, index, block):
        # read body from file
        start = self._indices[index][block][1][1]
        end = self._indices[index][block][2][1]
        body = self._readfile(start, end)
        # remove comments and leading white spaces
        body = re.sub(r'^#.+$\n?', '', body, 0, re.MULTILINE)
        body = re.sub(r'^\s*', '', body, 0, re.MULTILINE)
        # parse body
        rows = []
        for l in body.splitlines():
            cols = [float(n) for n in l.split()]
            rows.append(cols)
        return rows

    def get (self, index, block):
        h = self.getHeader(index, block)
        b = self.getBody(index, block)
        if len(b) < 1:
            raise DatafileError('Body is empty')
        if len(h['tableHeaders']) != len(b[0]):
            raise DatafileError('Number of table headers does not match number of table columns')
        o = copy.deepcopy(self.EMPTY_DOCUMENT)
        o['info']['version'] = h['version']
        o['info']['date'] = h['date']
        o['params'] = h['params']
        o['data'] = {'headers': h['tableHeaders'], 'rows': b}
        if not (h['version'] is None and
                h['date'] is None and
                h['params'] is None):
            o['info']['program'] = 'runQca'
            o['info']['dataFormat'] = 'QcaDataFileV1'
        return o

    def getAll (self):
        os = []
        l = copy.deepcopy(self.EMPTY_DOCUMENT)
        for i, index in enumerate(self._indices):
            for j, block in enumerate(index):
                o = self.get(i,j)
                if (o['info']['date'] is None and
                    o['info']['version'] is None and
                    o['params'] is None):
                        o['info'] = copy.deepcopy(l['info'])
                        o['params'] = copy.deepcopy(l['params'])
                else:
                    l['info'] = copy.deepcopy(o['info'])
                    l['params'] = copy.deepcopy(o['params'])
                os.append(o)
        return os

    def toJson (self, prettyPrint=True):
        if prettyPrint:
            return json.dumps(self.getAll(), indent=2, separators=(',', ': '))
        else:
            return json.dumps(self.getAll())

    def _readfile (self, start, end):
        f = open(self._filename)
        f.seek(start)
        ls = ""
        while True:
            l = f.readline()
            if l != "" and f.tell() <= end:
                ls += l
            else:
                break
        f.close()
        return ls

    def _lines (self, lsString):
        ls = lsString.splitlines()
        for l in ls:
            yield l

    def _convert (self, s):
        try:
            return int(s)
        except ValueError:
            try:
                return float(s)
            except ValueError:
                return s

    def _updateLeaves (self, d, f):
        if isinstance(d, dict):
            n = {}
            for k, v in d.iteritems():
                n[k] = self._updateLeaves(v, f)
            return n
        elif isinstance(d, list):
            n = []
            for v in d:
                n.append(self._updateLeaves(v, f))
            return n
        else:
            return f(d)
<commit_msg>Python datafile parser: Rec. convert files to json<commit_after>import numpy
import re
import json
import copy
import datetime
from os import walk as oswalk
import fnmatch
import sys

# TODO: rename to QcaDatafile
# TODO: change date, so it conforms to RFC 2822 (email), e.g. 'Thu, 28 Jun 2001 14:17:15 +0000'

# # we ignore the timezone information, e.g. -0600
# dateString = m.group(1)
# date = datetime.datetime.strptime(dateString[:-6], "%a %b %d %H:%M:%S %Y")

class DatafileError(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

class Datafile:
    EMPTY_DOCUMENT = {
            'info': {
                'program': None,
                'version': None,
                'date': None,
                'dataFormat': None
            },
            'params': None,
            'data': None
        }

    def __init__ (self, filename):
        self._filename = filename
        self._indices = []
        self.parse(filename)

    def parse (self, filename):
        f = open(filename)
        empty = 0
        fpos = 0
        fpos_end = 0
        ln = 1
        ln_end = 0
        ids = self._indices
        # start first index and first block
        ids.append([[(ln, fpos)]])

        for l in f:
            if l.strip(' \t') == '\n':
                if empty == 0:
                    ln_end = ln
                    fpos_end = fpos
                empty += 1

            else:
                cindex = ids[-1]
                cblock = cindex[-1]
                if empty > 0:
                    if (len(cblock) < 2):
                        # no data => begin of data = end of block
                        cblock.append((ln_end, fpos_end))
                    # end of current block
                    cblock.append((ln_end, fpos_end))
                    if empty == 1:
                        # start new block
                        cindex.append([(ln, fpos)])
                    elif empty >= 2:
                        # start new index and new block
                        ids.append([[(ln, fpos)]])
                    if l[0] != '#':
                        # no header for new block
                        ids[-1][-1].append((ln, fpos))
                elif empty == 0 and l[0] != '#' and len(cblock) < 2:
                    # end of header and start of data
                    cblock.append((ln, fpos))
                empty = 0

            ln += 1
            fpos += len(l)

        # close last block
        cindex = ids[-1]
        cblock = cindex[-1]
        if (len(cblock) < 2):
            cblock.append(cblock[0])
        cblock.append((ln, fpos))
        f.close()

    def info (self):
        ids = self._indices
        for i, index in enumerate(ids):
            print "Index", i
            print "   " + str(index[-1][-1][0] - index[0][0][0]) + " lines"
            print
            for j, block in enumerate(index):
                print "   Block", j
                print "      " + str(block[-1][0] - block[0][0]) + " lines"
                print "      Begin Header", block[0]
                print "      Begin Data", block[1]
                print "      End  ", block[2]
            print

    def getAsString (self, index, block=-1):
        start = 0
        end = 0
        if block == -1:
            start = self._indices[index][0][0][1]
            end = self._indices[index][-1][-1][1]
        else:
            start = self._indices[index][block][0][1]
            end = self._indices[index][block][-1][1]
        return self._readfile(start, end)

    def getAsMatrix (self, index, block=-1):
        ls = self.getAsString(index, block)
        return numpy.loadtxt(self._lines(ls))

    def getHeader (self, index):
        # read header from file
        start = self._indices[index][0][0][1]
        end = self._indices[index][0][1][1]
        header = self._readfile(start, end)
        # remove leading pound character
        header = re.sub(r'^# ?', '', header, 0, re.MULTILINE)
        # parse header
        version = None
        date = None
        params = None
        tableHeaders = []
        m = re.search(r'^program version: (.+)$', header, re.MULTILINE)
        if m: version = m.group(1)
        m = re.search(r'^date: (.+)$', header, re.MULTILINE)
        if m: date = m.group(1)
        m = re.search(r'^{.*^}', header, re.MULTILINE | re.DOTALL)
        if m:
            paramsJson = m.group(0)
            params = json.loads(paramsJson)
            params = self._updateLeaves(params, self._convert)
        m = re.search(r'\n?.+$', header)
        if m:
            lastLine = m.group(0)
            tableHeaders = lastLine.split()
        return {
                'version': version,
                'date': date,
                'params': params,
                'tableHeaders': tableHeaders
               };

    def getBody (self, index):
        # read body from file
        # we ignore the blocks and just read the data of all blocks as one body
        start = self._indices[index][0][1][1]
        end = self._indices[index][-1][2][1]
        body = self._readfile(start, end)
        # remove comments, leading white spaces, and empty lines
        body = re.sub(r'^#.+$\n?', '', body, 0, re.MULTILINE)
        body = re.sub(r'^\s*', '', body, 0, re.MULTILINE)
        body = re.sub(r'\n\n', '\n', body, 0, re.MULTILINE)
        # parse body
        rows = []
        for l in body.splitlines():
            cols = [float(n) for n in l.split()]
            rows.append(cols)
        return rows

    def get (self, index):
        h = self.getHeader(index)
        b = self.getBody(index)
        if len(b) < 1:
            raise DatafileError('Body is empty')
        if len(h['tableHeaders']) != len(b[0]):
            raise DatafileError('Number of table headers does not match number of table columns')
        o = copy.deepcopy(self.EMPTY_DOCUMENT)
        o['info']['version'] = h['version']
        o['info']['date'] = h['date']
        o['params'] = h['params']
        o['data'] = {'headers': h['tableHeaders'], 'rows': b}
        if not (h['version'] is None and
                h['date'] is None and
                h['params'] is None):
            o['info']['program'] = 'runQca'
            o['info']['dataFormat'] = 'QcaDataFileV1'
        return o

    def getAll (self):
        os = []
        l = copy.deepcopy(self.EMPTY_DOCUMENT)
        for i, index in enumerate(self._indices):
            o = self.get(i)
            if (o['info']['date'] is None and
                o['info']['version'] is None and
                o['params'] is None):
                    o['info'] = copy.deepcopy(l['info'])
                    o['params'] = copy.deepcopy(l['params'])
            else:
                l['info'] = copy.deepcopy(o['info'])
                l['params'] = copy.deepcopy(o['params'])
            os.append(o)
        return os

    def toJson (self, prettyPrint=True):
        if prettyPrint:
            return json.dumps(self.getAll(), indent=2, separators=(',', ': '))
        else:
            return json.dumps(self.getAll())

    def _readfile (self, start, end):
        f = open(self._filename)
        f.seek(start)
        ls = ""
        while True:
            l = f.readline()
            if l != "" and f.tell() <= end:
                ls += l
            else:
                break
        f.close()
        return ls

    def _lines (self, lsString):
        ls = lsString.splitlines()
        for l in ls:
            yield l

    def _convert (self, s):
        try:
            return int(s)
        except ValueError:
            try:
                return float(s)
            except ValueError:
                return s

    def _updateLeaves (self, d, f):
        if isinstance(d, dict):
            n = {}
            for k, v in d.iteritems():
                n[k] = self._updateLeaves(v, f)
            return n
        elif isinstance(d, list):
            n = []
            for v in d:
                n.append(self._updateLeaves(v, f))
            return n
        else:
            return f(d)

class PrintJson:
    def __init__ (self, prettyPrint=True):
        self._prettyPrint = prettyPrint
        self._count = 0

    def __call__ (self, o):
        if self._count > 0:
            print(',')
        if self._prettyPrint:
            print(json.dumps(o, indent=2, separators=(',', ': ')))
        else:
            print(json.dumps(o))
        self._count += 1

def processFiles (directory, function):
    for d, ds, fs in oswalk(directory):
        for f in fnmatch.filter(fs, '*.dat'):
            sys.stderr.write('Processing ' + f + '\n')
            df = Datafile(f)
            os = df.getAll()
            for o in os:
                function(o)

def filesToJson (directory, prettyPrint=True):
    print('[')
    processFiles(directory, PrintJson(prettyPrint))
    print(']')

if __name__ == '__main__':
    if len(sys.argv) > 1:
        d = sys.argv[1]
        filesToJson(d)
<|endoftext|>


<commit_before>#!/usr/bin/python
import sys, os, urllib

root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(root)
import admin, store, config

cfg = config.Config(root+'/config.ini')
st = store.Store(cfg)

# classifiers
for c in urllib.urlopen("http://pypi.python.org/pypi?%3Aaction=list_classifiers").read().splitlines():
    admin.add_classifier(st, c)

# Demo data starts here

# an admin
otk = st.store_user('fred', 'fredpw', 'fred@python.test')
st.delete_otk(otk)
st.add_role('fred', 'Admin', None)
# an owner
otk = st.store_user('barney', 'barneypw', 'barney@python.test')
st.delete_otk(otk)

# package spam
st.set_user('barney', '127.0.0.1', True)
for version in ('0.8', '0.9', '1.0'):
    st.store_package('spam', version, {
            'author':'Barney Geroellheimer',
            'author_email':'barney@python.test',
            'homepage':'http://spam.python.test/',
            'license':'GPL',
            'summary':'The spam package',
            'description':'Does anybody want to provide real data here?',
            'classifiers':["Development Status :: 6 - Mature",
                           "Programming Language :: Python :: 2"],
            '_pypi_hidden':False
            })

# package eggs
for version in ('0.1', '0.2', '0.3', '0.4'):
    st.store_package('eggs', version, {
            'author':'Barney Geroellheimer',
            'author_email':'barney@python.test',
            'homepage':'http://eggs.python.test/',
            'license':'GPL',
            'summary':'The eggs package',
            'description':'Does anybody want to provide real data here?',
            'classifiers':["Development Status :: 3 - Alpha",
                           "Programming Language :: Python :: 3"],
            '_pypi_hidden':version!='0.4'
            })

st.commit()
<commit_msg>create a file<commit_after>#!/usr/bin/python
import sys, os, urllib

root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(root)
import admin, store, config

cfg = config.Config(root+'/config.ini')
st = store.Store(cfg)

# classifiers
for c in urllib.urlopen("http://pypi.python.org/pypi?%3Aaction=list_classifiers").read().splitlines():
    admin.add_classifier(st, c)

# Demo data starts here

# an admin
otk = st.store_user('fred', 'fredpw', 'fred@python.test')
st.delete_otk(otk)
st.add_role('fred', 'Admin', None)
# an owner
otk = st.store_user('barney', 'barneypw', 'barney@python.test')
st.delete_otk(otk)

# package spam
st.set_user('barney', '127.0.0.1', True)
for version in ('0.8', '0.9', '1.0'):
    st.store_package('spam', version, {
            'author':'Barney Geroellheimer',
            'author_email':'barney@python.test',
            'homepage':'http://spam.python.test/',
            'license':'GPL',
            'summary':'The spam package',
            'description':'Does anybody want to provide real data here?',
            'classifiers':["Development Status :: 6 - Mature",
                           "Programming Language :: Python :: 2"],
            '_pypi_hidden':False
            })

# package eggs
for version in ('0.1', '0.2', '0.3', '0.4'):
    st.store_package('eggs', version, {
            'author':'Barney Geroellheimer',
            'author_email':'barney@python.test',
            'homepage':'http://eggs.python.test/',
            'license':'GPL',
            'summary':'The eggs package',
            'description':'Does anybody want to provide real data here?',
            'classifiers':["Development Status :: 3 - Alpha",
                           "Programming Language :: Python :: 3"],
            '_pypi_hidden':version!='0.4'
            })

st.add_file('spam', '1.0', 'THIS IS SOME CONTENT', '1234', 'text/plain',
            'any', '', 'demo.txt', None)

st.commit()
<|endoftext|>


<commit_before><commit_msg>Adding tests for the serious selection magic<commit_after><|endoftext|>


<commit_before><commit_msg>remove commented out sections<commit_after><|endoftext|>


<commit_before><commit_msg>Add test file<commit_after><|endoftext|>


<commit_before><commit_msg>modified import (cherry picked from commit c2e4878)<commit_after><|endoftext|>


<commit_before><commit_msg>Updated version to 0.1.0<commit_after><|endoftext|>


<commit_before>import sys
import logging
from contextlib import contextmanager
import csv
import sqlite3
import xml.etree.ElementTree as ET
import argparse
from collections import defaultdict

logger = logging.getLogger(__name__)

@contextmanager
def printer(msg):
    msg = msg.strip()
    logging.info("Started "+msg)
    yield
    logging.info("Finished "+msg)

def get_existing_osm_tags(xml_el):
    """Given a XML element for an object, return (as dict) the current OSM tags"""
    return {el.get('k'): el.get('v') for el in xml_el.findall("tag")}

def logainm_tags(xml_el, logainm_data):
    """Given an XML element for an OSM object, return (as dict) the tags to add
    to it, presuming it is to be matched up to this logainm_data"""
    tags = get_existing_osm_tags(xml_el)
    new_tags = {
        'logainm:ref': logainm_data['logainm_id'],
        'logainm:url': 'http://www.logainm.ie/en/{}'.format(logainm_data['logainm_id'])
    }

    # Not adding official_name:XX tags because that is not accurate for NI.

    if logainm_data['name_ga'] not in ["", None]:
        if 'name:ga' not in tags:
            new_tags['name:ga'] = logainm_data['name_ga']
        else:
            pass
            #if tags['name:ga'] != logainm_data['name_ga']:
            #    new_tags['official_name:ga'] = logainm_data['name_ga']

    if logainm_data['name_en'] not in ["", None]:
        if 'name:en' not in tags:
            new_tags['name:en'] = logainm_data['name_en']
        else:
            pass
            #if tags['name:en'] != logainm_data['name_en']:
            #    new_tags['official_name:en'] = logainm_data['name_en']

    #if 'official_name:en' not in tags:
    #    new_tags['official_name:en'] = logainm_data['name_en']

    #if 'official_name:ga' not in tags:
    #    new_tags['official_name:ga'] = logainm_data['name_ga']

    return new_tags

def get_logainm_tags(cursor, logainm_id):
    cursor.execute("select logainm_id, name_en, name_ga from names where logainm_id = ?", [logainm_id])
    data = cursor.fetchone()
    return {'logainm_id': data[0], 'name_en': data[1], 'name_ga': data[2]}

def unicodeify_dict(dct):
    for key, value in dct.items():
        dct[key] = value.decode("utf-8")
    return dct

def name_en(obj):
    if obj['NAME_EN'] != '':
        return obj['NAME_EN']
    else:
        return obj['NAME_TAG']

def read_logainm_data():
    results = {}
    data_to_load = [('townlands-no-geom.csv', 'townlands'), ('civil_parishes-no-geom.csv', 'civil_parishes'), ('counties-no-geom.csv', 'counties'), ('baronies-no-geom.csv', 'baronies')]
    for filename, keyname in data_to_load:
        with open(filename) as fp:
            reader = csv.DictReader(fp)
            results[keyname] = list(unicodeify_dict(x) for x in reader)

    # create a dict
    results['index'] = {}
    for obj_key in  ['OSM_ID', 'BAR_OSM_ID', 'CP_OSM_ID']:
        results['index'][obj_key] = {}
        for parent_key in  ['BAR_OSM_ID', 'CP_OSM_ID', 'CO_OSM_ID']:
            results['index'][obj_key][parent_key] = defaultdict(set)

    for t in results['townlands']:
        for obj_key in  ['OSM_ID', 'BAR_OSM_ID', 'CP_OSM_ID']:
            for parent_key in  ['BAR_OSM_ID', 'CP_OSM_ID', 'CO_OSM_ID']:
                if t[parent_key] != '':
                    results['index'][obj_key][parent_key][t[obj_key]].add(t[parent_key])

    results['index']['osmid_to_logainm_ref'] = {}
    for filename, keyname in data_to_load:
        for x in results[keyname]:
            if x['LOGAINM_RE'].strip() not in (None, ''):
                results['index']['osmid_to_logainm_ref'][x['OSM_ID']] = x['LOGAINM_RE']

    return results

def osmid_to_logainm_ref(logainm_data, osm_id, existing_match_ups):
    try:
        result = logainm_data['index']['osmid_to_logainm_ref'][osm_id]
    except (KeyError, IndexError):
        # is this already seen on a higher level match up?
        # [1:] to remove the -
        result = existing_match_ups[('relation', osm_id[1:])]['logainm_id']
        logger.debug("Found a Logainm id (%s) for %s based on a previous match up", result, osm_id)
    return result

def barony_osmid_for_civil_parish_osmid(logainm_data, civil_parish_id):
    return set(t['BAR_OSM_ID'] for t in logainm_data['townlands'] if t['CP_OSM_ID'] == civil_parish_id and t['BAR_OSM_ID'] != '' )

def parent_osmid_for_obj_osmid(logainm_data, obj_osmid, obj_key, parent_key):
    result = logainm_data['index'][obj_key][parent_key][obj_osmid]
    return result

def baronies_matchup(logainm_data, cursor, existing_match_ups):
    return hierachial_matchup(logainm_data, cursor,
                key='baronies', obj_logainm_code="BAR", parent_logainm_code='CON',
                obj_key="BAR_OSM_ID", parent_key="CO_OSM_ID", parent_name="county",
                existing_match_ups=existing_match_ups,
         )

def civil_parish_matchup(logainm_data, cursor, existing_match_ups):
    return hierachial_matchup(logainm_data, cursor,
                key='civil_parishes', obj_logainm_code="PAR", parent_logainm_code='BAR',
                obj_key="CP_OSM_ID", parent_key="BAR_OSM_ID", parent_name="barony",
                existing_match_ups=existing_match_ups,
         )

def townlands_matchup(logainm_data, cursor, existing_match_ups):
    return hierachial_matchup(logainm_data, cursor,
                key='townlands', obj_logainm_code="BF", parent_logainm_code='PAR',
                obj_key="OSM_ID", parent_key="CP_OSM_ID", parent_name="civil parish",
                existing_match_ups=existing_match_ups,
         )

def hierachial_matchup(logainm_data, cursor, key, obj_logainm_code, parent_logainm_code, obj_key, parent_key, parent_name, existing_match_ups=None):
    existing_match_ups = existing_match_ups or {}
    logger.info("Have %d %s in total", len(logainm_data[key]), key)
    results = {}

    possibles = [b for b in logainm_data[key] if b['LOGAINM_RE'] == '']
    logger.info("Found %d %s without logainm ref", len(possibles), key)

    for obj in possibles:
        logger.debug("Starting to look at %s %s (osm:%s)", key, name_en(obj), obj['OSM_ID'])
        parent_osm_id = parent_osmid_for_obj_osmid(logainm_data, obj['OSM_ID'], obj_key, parent_key)
        if len(parent_osm_id) == 0:
            logger.error("ERROR No %s found for %s %s (%s) in OSM: Tie: http://www.townlands.ie/by/osm_id/%s", parent_name, key, name_en(obj), obj['OSM_ID'], obj['OSM_ID'])
            continue
        elif len(parent_osm_id) > 1:
            logger.error("ERROR Found %s (%s) %s for %s %s (osmid=%s) in OSM", len(parent_osm_id), ",".join(parent_osm_id), parent_name, key, name_en(obj), obj['OSM_ID'])
            continue
        elif len(parent_osm_id) == 1:
            parent_osm_id = parent_osm_id.pop()
            try:
                parent_logainm_id = osmid_to_logainm_ref(logainm_data, parent_osm_id, existing_match_ups)
                if ";" in parent_logainm_id:
                    logger.error("ERROR %s %s (%s) is in %s %s in OSM which is many logainms: %s Tie: http://www.townlands.ie/by/osm_id/%s", key, name_en(obj), obj['OSM_ID'], parent_name, parent_osm_id, parent_logainm_id, obj['OSM_ID'])
                    continue
                else:
                    logger.debug("OK %s %s (%s) is in %s %s in OSM which is logainm id %s", key, name_en(obj), obj['OSM_ID'], parent_name, parent_osm_id, parent_logainm_id)
            except (IndexError, KeyError):
                logger.error("ERROR %s %s (%s) is in %s %s in OSM which has no known logainm", key, name_en(obj), obj['OSM_ID'], parent_name, parent_osm_id)
                continue
        else:
            assert False

        # Now we have the logainm ref of the parent that this obj is in.
        # Look at the logainm data for the objs in that bar
        cursor.execute("select obj.logainm_id from names as parent join geometric_contains as con on (parent.logainm_id = con.outer_obj_id) join names as obj on (obj.logainm_id = con.inner_obj_id) where parent.logainm_category_code = ? and obj.logainm_category_code = ? and parent.logainm_id = ? and obj.name_en = ?;", [parent_logainm_code, obj_logainm_code, parent_logainm_id, name_en(obj)])
        data = cursor.fetchall()
        data_str = ", ".join(x[0] for x in data)
        if len(data) == 0:
            logger.error("ERROR %s %s (%s) is in %s OSM:%s (logainm:%s) http://www.townlands.ie/by/osm_id/%s/ which has no %s in logainm for this name", key, name_en(obj), obj['OSM_ID'], parent_name, parent_osm_id, parent_logainm_id, obj['OSM_ID'], key)
            continue
        elif len(data) > 1:
            logger.error("ERROR %s %s (%s) is in %s OSM:%s (logainm:%s) has >1 %s in logainm for this name: %s", key, name_en(obj), obj['OSM_ID'], parent_name, parent_osm_id, parent_logainm_id, key, data_str)
            continue
        elif len(data) == 1:
            logger.info("OK %s %s (%s) is in %s OSM:%s (logainm:%s) has 1 %s in logainm for this name: %s", key, name_en(obj), obj['OSM_ID'], parent_name, parent_osm_id, parent_logainm_id, data_str, key)
            # remove leading '-' character
            results[('relation', obj['OSM_ID'][1:])] = get_logainm_tags(cursor, data[0][0])
        else:
            assert False

    logger.info("Matched up %d %s of %d without a logainm id (%s%%)", len(results), key, len(possibles), (len(results)*100)/len(possibles))
    return results

def remove_and_warn_dupes(logainm_candidates):
    logainm_ref_to_osm = defaultdict(set)
    for key, tags in logainm_candidates.items():
        logainm_ref = tags['logainm_id']
        logainm_ref_to_osm[logainm_ref].add(key)

    for logainm_ref in logainm_ref_to_osm:
        osms = logainm_ref_to_osm[logainm_ref]
        if len(osms) > 1:
            logger.error("ERROR Duplicates - %s has been matched to the following OSM ids: %s", logainm_ref, ", ".join("{} {}".format(*x) for x in osms))
            for key in osms:
                del logainm_candidates[key]

    return logainm_candidates

def limit(logainm_candidates, limit):
    if limit is None:
        return logainm_candidates

    limit = int(limit)
    logainm_candidates = dict((sorted(logainm_candidates.items()))[:limit])

    return logainm_candidates

def main():

    parser = argparse.ArgumentParser()
    parser.add_argument("-i", "--input")
    parser.add_argument("-o", "--output")
    parser.add_argument("--baronies", action="store_true")
    parser.add_argument("--civil-parishes", action="store_true")
    parser.add_argument("--townlands", action="store_true")
    parser.add_argument("-v", "--verbose", action="store_true")
    parser.add_argument("-n", "--dry-run", action="store_true")

    parser.add_argument("-l", "--limit")

    args = parser.parse_args()

    ch = logging.StreamHandler(sys.stdout)
    if args.verbose:
        ch.setLevel(logging.DEBUG)
    else:
        ch.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s\t%(levelname)s\tL%(lineno)s\t%(message)s')
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    conn = sqlite3.connect("logainm.sqlite")
    cursor = conn.cursor()

    logger.setLevel(logging.DEBUG)

    logainm_data = read_logainm_data()

    logainm_candidates = {}

    if args.baronies:
        logainm_candidates.update(baronies_matchup(logainm_data, cursor, logainm_candidates))

    if args.civil_parishes:
        logainm_candidates.update(civil_parish_matchup(logainm_data, cursor, logainm_candidates))

    if args.townlands:
        logainm_candidates.update(townlands_matchup(logainm_data, cursor, logainm_candidates))

    logainm_candidates = remove_and_warn_dupes(logainm_candidates)
    logainm_candidates = limit(logainm_candidates, args.limit)

    if args.dry_run:
        return

    # read in OSM XML
    with printer("reading in OSM XML"):
        tree = ET.parse(args.input)
        root = tree.getroot()

    # add new tags to OSM XML
    with printer("adding XML tags"):
        for rel in root.findall("relation"):
            osm_id = rel.get("id", None)
            if ('relation', osm_id) in logainm_candidates:

                # This tag is needed so JOSM knows to upload it
                rel.set("action", "modify")

                logging.debug("Adding tags to OSM_ID %d", osm_id)
                logaimn_data = logainm_candidates[('relation', osm_id)]
                for k, v in logainm_tags(rel, logaimn_data).items():
                    ET.SubElement(rel, 'tag', {'k': k, 'v': v})
            else:
                root.remove(rel)

    # write out OSM XML
    tree.write(args.output, encoding='utf-8', xml_declaration=True)

if __name__ == '__main__':
    main()

<commit_msg>show percentage that are missing<commit_after>import sys
import logging
from contextlib import contextmanager
import csv
import sqlite3
import xml.etree.ElementTree as ET
import argparse
from collections import defaultdict

logger = logging.getLogger(__name__)

@contextmanager
def printer(msg):
    msg = msg.strip()
    logging.info("Started "+msg)
    yield
    logging.info("Finished "+msg)

def get_existing_osm_tags(xml_el):
    """Given a XML element for an object, return (as dict) the current OSM tags"""
    return {el.get('k'): el.get('v') for el in xml_el.findall("tag")}

def logainm_tags(xml_el, logainm_data):
    """Given an XML element for an OSM object, return (as dict) the tags to add
    to it, presuming it is to be matched up to this logainm_data"""
    tags = get_existing_osm_tags(xml_el)
    new_tags = {
        'logainm:ref': logainm_data['logainm_id'],
        'logainm:url': 'http://www.logainm.ie/en/{}'.format(logainm_data['logainm_id'])
    }

    # Not adding official_name:XX tags because that is not accurate for NI.

    if logainm_data['name_ga'] not in ["", None]:
        if 'name:ga' not in tags:
            new_tags['name:ga'] = logainm_data['name_ga']
        else:
            pass
            #if tags['name:ga'] != logainm_data['name_ga']:
            #    new_tags['official_name:ga'] = logainm_data['name_ga']

    if logainm_data['name_en'] not in ["", None]:
        if 'name:en' not in tags:
            new_tags['name:en'] = logainm_data['name_en']
        else:
            pass
            #if tags['name:en'] != logainm_data['name_en']:
            #    new_tags['official_name:en'] = logainm_data['name_en']

    #if 'official_name:en' not in tags:
    #    new_tags['official_name:en'] = logainm_data['name_en']

    #if 'official_name:ga' not in tags:
    #    new_tags['official_name:ga'] = logainm_data['name_ga']

    return new_tags

def get_logainm_tags(cursor, logainm_id):
    cursor.execute("select logainm_id, name_en, name_ga from names where logainm_id = ?", [logainm_id])
    data = cursor.fetchone()
    return {'logainm_id': data[0], 'name_en': data[1], 'name_ga': data[2]}

def unicodeify_dict(dct):
    for key, value in dct.items():
        dct[key] = value.decode("utf-8")
    return dct

def name_en(obj):
    if obj['NAME_EN'] != '':
        return obj['NAME_EN']
    else:
        return obj['NAME_TAG']

def read_logainm_data():
    results = {}
    data_to_load = [('townlands-no-geom.csv', 'townlands'), ('civil_parishes-no-geom.csv', 'civil_parishes'), ('counties-no-geom.csv', 'counties'), ('baronies-no-geom.csv', 'baronies')]
    for filename, keyname in data_to_load:
        with open(filename) as fp:
            reader = csv.DictReader(fp)
            results[keyname] = list(unicodeify_dict(x) for x in reader)

    # create a dict
    results['index'] = {}
    for obj_key in  ['OSM_ID', 'BAR_OSM_ID', 'CP_OSM_ID']:
        results['index'][obj_key] = {}
        for parent_key in  ['BAR_OSM_ID', 'CP_OSM_ID', 'CO_OSM_ID']:
            results['index'][obj_key][parent_key] = defaultdict(set)

    for t in results['townlands']:
        for obj_key in  ['OSM_ID', 'BAR_OSM_ID', 'CP_OSM_ID']:
            for parent_key in  ['BAR_OSM_ID', 'CP_OSM_ID', 'CO_OSM_ID']:
                if t[parent_key] != '':
                    results['index'][obj_key][parent_key][t[obj_key]].add(t[parent_key])

    results['index']['osmid_to_logainm_ref'] = {}
    for filename, keyname in data_to_load:
        for x in results[keyname]:
            if x['LOGAINM_RE'].strip() not in (None, ''):
                results['index']['osmid_to_logainm_ref'][x['OSM_ID']] = x['LOGAINM_RE']

    return results

def osmid_to_logainm_ref(logainm_data, osm_id, existing_match_ups):
    try:
        result = logainm_data['index']['osmid_to_logainm_ref'][osm_id]
    except (KeyError, IndexError):
        # is this already seen on a higher level match up?
        # [1:] to remove the -
        result = existing_match_ups[('relation', osm_id[1:])]['logainm_id']
        logger.debug("Found a Logainm id (%s) for %s based on a previous match up", result, osm_id)
    return result

def barony_osmid_for_civil_parish_osmid(logainm_data, civil_parish_id):
    return set(t['BAR_OSM_ID'] for t in logainm_data['townlands'] if t['CP_OSM_ID'] == civil_parish_id and t['BAR_OSM_ID'] != '' )

def parent_osmid_for_obj_osmid(logainm_data, obj_osmid, obj_key, parent_key):
    result = logainm_data['index'][obj_key][parent_key][obj_osmid]
    return result

def baronies_matchup(logainm_data, cursor, existing_match_ups):
    return hierachial_matchup(logainm_data, cursor,
                key='baronies', obj_logainm_code="BAR", parent_logainm_code='CON',
                obj_key="BAR_OSM_ID", parent_key="CO_OSM_ID", parent_name="county",
                existing_match_ups=existing_match_ups,
         )

def civil_parish_matchup(logainm_data, cursor, existing_match_ups):
    return hierachial_matchup(logainm_data, cursor,
                key='civil_parishes', obj_logainm_code="PAR", parent_logainm_code='BAR',
                obj_key="CP_OSM_ID", parent_key="BAR_OSM_ID", parent_name="barony",
                existing_match_ups=existing_match_ups,
         )

def townlands_matchup(logainm_data, cursor, existing_match_ups):
    return hierachial_matchup(logainm_data, cursor,
                key='townlands', obj_logainm_code="BF", parent_logainm_code='PAR',
                obj_key="OSM_ID", parent_key="CP_OSM_ID", parent_name="civil parish",
                existing_match_ups=existing_match_ups,
         )

def hierachial_matchup(logainm_data, cursor, key, obj_logainm_code, parent_logainm_code, obj_key, parent_key, parent_name, existing_match_ups=None):
    existing_match_ups = existing_match_ups or {}
    logger.info("Have %d %s in total", len(logainm_data[key]), key)
    results = {}

    possibles = [b for b in logainm_data[key] if b['LOGAINM_RE'] == '']
    logger.info("Found %(len)d (%(percent)f%%) %(type)s without logainm ref", dict(len=len(possibles), type=key, percent=(len(possibles)*100/len(logainm_data[key]))))

    for obj in possibles:
        logger.debug("Starting to look at %s %s (osm:%s)", key, name_en(obj), obj['OSM_ID'])
        parent_osm_id = parent_osmid_for_obj_osmid(logainm_data, obj['OSM_ID'], obj_key, parent_key)
        if len(parent_osm_id) == 0:
            logger.error("ERROR No %s found for %s %s (%s) in OSM: Tie: http://www.townlands.ie/by/osm_id/%s", parent_name, key, name_en(obj), obj['OSM_ID'], obj['OSM_ID'])
            continue
        elif len(parent_osm_id) > 1:
            logger.error("ERROR Found %s (%s) %s for %s %s (osmid=%s) in OSM", len(parent_osm_id), ",".join(parent_osm_id), parent_name, key, name_en(obj), obj['OSM_ID'])
            continue
        elif len(parent_osm_id) == 1:
            parent_osm_id = parent_osm_id.pop()
            try:
                parent_logainm_id = osmid_to_logainm_ref(logainm_data, parent_osm_id, existing_match_ups)
                if ";" in parent_logainm_id:
                    logger.error("ERROR %s %s (%s) is in %s %s in OSM which is many logainms: %s Tie: http://www.townlands.ie/by/osm_id/%s", key, name_en(obj), obj['OSM_ID'], parent_name, parent_osm_id, parent_logainm_id, obj['OSM_ID'])
                    continue
                else:
                    logger.debug("OK %s %s (%s) is in %s %s in OSM which is logainm id %s", key, name_en(obj), obj['OSM_ID'], parent_name, parent_osm_id, parent_logainm_id)
            except (IndexError, KeyError):
                logger.error("ERROR %s %s (%s) is in %s %s in OSM which has no known logainm", key, name_en(obj), obj['OSM_ID'], parent_name, parent_osm_id)
                continue
        else:
            assert False

        # Now we have the logainm ref of the parent that this obj is in.
        # Look at the logainm data for the objs in that bar
        cursor.execute("select obj.logainm_id from names as parent join geometric_contains as con on (parent.logainm_id = con.outer_obj_id) join names as obj on (obj.logainm_id = con.inner_obj_id) where parent.logainm_category_code = ? and obj.logainm_category_code = ? and parent.logainm_id = ? and obj.name_en = ?;", [parent_logainm_code, obj_logainm_code, parent_logainm_id, name_en(obj)])
        data = cursor.fetchall()
        data_str = ", ".join(x[0] for x in data)
        if len(data) == 0:
            logger.error("ERROR %s %s (%s) is in %s OSM:%s (logainm:%s) http://www.townlands.ie/by/osm_id/%s/ which has no %s in logainm for this name", key, name_en(obj), obj['OSM_ID'], parent_name, parent_osm_id, parent_logainm_id, obj['OSM_ID'], key)
            continue
        elif len(data) > 1:
            logger.error("ERROR %s %s (%s) is in %s OSM:%s (logainm:%s) has >1 %s in logainm for this name: %s", key, name_en(obj), obj['OSM_ID'], parent_name, parent_osm_id, parent_logainm_id, key, data_str)
            continue
        elif len(data) == 1:
            logger.info("OK %s %s (%s) is in %s OSM:%s (logainm:%s) has 1 %s in logainm for this name: %s", key, name_en(obj), obj['OSM_ID'], parent_name, parent_osm_id, parent_logainm_id, data_str, key)
            # remove leading '-' character
            results[('relation', obj['OSM_ID'][1:])] = get_logainm_tags(cursor, data[0][0])
        else:
            assert False

    logger.info("Matched up %d %s of %d without a logainm id (%s%%)", len(results), key, len(possibles), (len(results)*100)/len(possibles))
    return results

def remove_and_warn_dupes(logainm_candidates):
    logainm_ref_to_osm = defaultdict(set)
    for key, tags in logainm_candidates.items():
        logainm_ref = tags['logainm_id']
        logainm_ref_to_osm[logainm_ref].add(key)

    for logainm_ref in logainm_ref_to_osm:
        osms = logainm_ref_to_osm[logainm_ref]
        if len(osms) > 1:
            logger.error("ERROR Duplicates - %s has been matched to the following OSM ids: %s", logainm_ref, ", ".join("{} {}".format(*x) for x in osms))
            for key in osms:
                del logainm_candidates[key]

    return logainm_candidates

def limit(logainm_candidates, limit):
    if limit is None:
        return logainm_candidates

    limit = int(limit)
    logainm_candidates = dict((sorted(logainm_candidates.items()))[:limit])

    return logainm_candidates

def main():

    parser = argparse.ArgumentParser()
    parser.add_argument("-i", "--input")
    parser.add_argument("-o", "--output")
    parser.add_argument("--baronies", action="store_true")
    parser.add_argument("--civil-parishes", action="store_true")
    parser.add_argument("--townlands", action="store_true")
    parser.add_argument("-v", "--verbose", action="store_true")
    parser.add_argument("-n", "--dry-run", action="store_true")

    parser.add_argument("-l", "--limit")

    args = parser.parse_args()

    ch = logging.StreamHandler(sys.stdout)
    if args.verbose:
        ch.setLevel(logging.DEBUG)
    else:
        ch.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s\t%(levelname)s\tL%(lineno)s\t%(message)s')
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    conn = sqlite3.connect("logainm.sqlite")
    cursor = conn.cursor()

    logger.setLevel(logging.DEBUG)

    logainm_data = read_logainm_data()

    logainm_candidates = {}

    if args.baronies:
        logainm_candidates.update(baronies_matchup(logainm_data, cursor, logainm_candidates))

    if args.civil_parishes:
        logainm_candidates.update(civil_parish_matchup(logainm_data, cursor, logainm_candidates))

    if args.townlands:
        logainm_candidates.update(townlands_matchup(logainm_data, cursor, logainm_candidates))

    logainm_candidates = remove_and_warn_dupes(logainm_candidates)
    logainm_candidates = limit(logainm_candidates, args.limit)

    if args.dry_run:
        return

    # read in OSM XML
    with printer("reading in OSM XML"):
        tree = ET.parse(args.input)
        root = tree.getroot()

    # add new tags to OSM XML
    with printer("adding XML tags"):
        for rel in root.findall("relation"):
            osm_id = rel.get("id", None)
            if ('relation', osm_id) in logainm_candidates:

                # This tag is needed so JOSM knows to upload it
                rel.set("action", "modify")

                logging.debug("Adding tags to OSM_ID %d", osm_id)
                logaimn_data = logainm_candidates[('relation', osm_id)]
                for k, v in logainm_tags(rel, logaimn_data).items():
                    ET.SubElement(rel, 'tag', {'k': k, 'v': v})
            else:
                root.remove(rel)

    # write out OSM XML
    tree.write(args.output, encoding='utf-8', xml_declaration=True)

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before><commit_msg>Follow-up to r4655, fix displaying of ticket property changes in the Ticket diff view.<commit_after><|endoftext|>


<commit_before><commit_msg>1.1.4dev: In the `db33` upgrade step, the `create_and_assign` action is only added if the `assigned` state exists in the workflow. Refs #11948.<commit_after><|endoftext|>


<commit_before><commit_msg>minor refactor of json_task.py to address issues 561 and 562 (#568)<commit_after><|endoftext|>


<commit_before><commit_msg>working borders for UR<commit_after><|endoftext|>


<commit_before><commit_msg>Pass format to TypeTree constructor<commit_after><|endoftext|>


<commit_before><commit_msg>Reference for future method<commit_after><|endoftext|>


<commit_before><commit_msg>bump to 0.0.2<commit_after><|endoftext|>


<commit_before><commit_msg>Give default bra for apps that require a second bra to be built (compare)<commit_after><|endoftext|>


<commit_before><commit_msg>do not catch errors in limit() method<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Phil Adams http://philadams.net

hrpg: commandline interface for http://habitrpg.com
http://github.com/philadams/hrpg
"""

def cli():
    pass

if '__main__' == __name__:
    cli()
<commit_msg>spec out interface to API GET endpoints<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Phil Adams http://philadams.net

hrpg: commandline interface for http://habitrpg.com
http://github.com/philadams/hrpg
"""

from docopt import docopt

def cli():
    """HabitRPG command-line interface.

    usage:
      hrpg status
      hrpg user [<uid>]
      hrpg tasks (--habit | --daily | --todo | --reward) [<uid>]
      hrpg task <tid> [<uid>]
      hrpg
      hrpg --version

    options:
      -h --help     Show this screen.
      --version     Show version.

    Subcommands:
      status        Get status of HabitRPG service
      user          Get user <uid> status
      tasks         Get user <uid> tasks
      task          Get task <tid> details
      <none>        Get user status for this user (convenience)
    """
    args = docopt(cli.__doc__, version='hrpg version 0.0.2')

if __name__ == '__main__':
    cli()
<|endoftext|>


<commit_before># coding: utf-8
from bs4 import BeautifulSoup
from datetime import datetime
import os
import urllib.parse
import urllib.request
import yaml

def main():
    dirpath = os.path.abspath(os.path.dirname(__file__))
    conf = yaml.load(open(dirpath + '/conf.yml').read())
    url = conf['weather_url']
    html = urllib.request.urlopen(url).read().decode('utf-8')
    soup = BeautifulSoup(html, 'lxml')
    soup_today = soup.find('section', 'today-weather')

    # テキトー
    icon_d = {
        '01': ':sunny:',
        '10': ':rain_cloud:',
        '15': ':umbrella:',
    }
    #  ':mostly_sunny:',
    #  ':partly_sunny:',
    #  ':barely_sunny:',
    #  ':partly_sunny_lain:',
    #  ':cloud:',
    #  ':rain_cloud:',
    #  ':thunder_cloud_and_rain:',
    #  ':lightning:',
    #  ':umbrella:',

    icon = soup_today.find('div', 'weather-icon').find('img')
    icon_png = icon['src'].split('/')[-1].rstrip('_n.png')
    text = icon['title']

    if icon_png in icon_d:
        emoji = icon_d[icon_png]
    else:
        emoji = ':question:'
        text = icon_png + text

    text = text + ' 最高: ' + soup_today.find('dd', 'high-temp').get_text() + soup_today.find('dd', 'high-temp tempdiff').string
    text = text + ' 最低: ' + soup_today.find('dd', 'low-temp').get_text() + soup_today.find('dd', 'low-temp tempdiff').string
    text = text + ' 取得: ' + datetime.now().strftime("%H:%M")

    params = urllib.parse.urlencode({
        'token': conf['token'],
        'profile': {
            'status_text': text,
            'status_emoji': emoji,
        },
    })
    params = params.encode('ascii')
    slack_url = 'https://slack.com/api/users.profile.set'
    req = urllib.request.Request(slack_url, params, {'Content-type': 'application/x-www-form-urlencoded'})
    urllib.request.urlopen(req)

if __name__ == "__main__":
    main()
<commit_msg>update icon_d<commit_after># coding: utf-8
from bs4 import BeautifulSoup
from datetime import datetime
import os
import urllib.parse
import urllib.request
import yaml

def main():
    dirpath = os.path.abspath(os.path.dirname(__file__))
    conf = yaml.load(open(dirpath + '/conf.yml').read())
    url = conf['weather_url']
    html = urllib.request.urlopen(url).read().decode('utf-8')
    soup = BeautifulSoup(html, 'lxml')
    soup_today = soup.find('section', 'today-weather')

    # テキトー
    icon_d = {
        '01': ':sunny:',
        '10': ':rain_cloud:',
        '15': ':umbrella:',
        '20': ':rain_cloud:',
    }
    #  ':mostly_sunny:',
    #  ':partly_sunny:',
    #  ':barely_sunny:',
    #  ':partly_sunny_lain:',
    #  ':cloud:',
    #  ':rain_cloud:',
    #  ':thunder_cloud_and_rain:',
    #  ':lightning:',
    #  ':umbrella:',

    icon = soup_today.find('div', 'weather-icon').find('img')
    icon_png = icon['src'].split('/')[-1].rstrip('_n.png')
    text = icon['title']

    if icon_png in icon_d:
        emoji = icon_d[icon_png]
    else:
        emoji = ':question:'
        text = icon_png + text

    text = text + ' 最高: ' + soup_today.find('dd', 'high-temp').get_text() + soup_today.find('dd', 'high-temp tempdiff').string
    text = text + ' 最低: ' + soup_today.find('dd', 'low-temp').get_text() + soup_today.find('dd', 'low-temp tempdiff').string
    text = text + ' 取得: ' + datetime.now().strftime("%H:%M")

    params = urllib.parse.urlencode({
        'token': conf['token'],
        'profile': {
            'status_text': text,
            'status_emoji': emoji,
        },
    })
    params = params.encode('ascii')
    slack_url = 'https://slack.com/api/users.profile.set'
    req = urllib.request.Request(slack_url, params, {'Content-type': 'application/x-www-form-urlencoded'})
    urllib.request.urlopen(req)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import logging
import h5py
import numpy as np
from pygwas.core import genotype
import sys
import os
import json
import gzip
from subprocess import Popen, PIPE
import shlex

log = logging.getLogger(__name__)
def die(msg):
    sys.stderr.write('Error: ' + msg + '\n')
    sys.exit(1)

def getSamples(inVCF):
    vname, vext = os.path.splitext(inVCF)
    if vext == '.vcf':
        v = open(inVCF, 'r')
    elif vext == '.gz':
        v = gzip.open(inVCF)
    else:
        die('please provide VCF file either gunzipped or text!')
    eline = v.readline()
    while eline[0] == '#':
        if eline[0:6] == '#CHROM':
            info = eline.rstrip().split('\t')
        eline = v.readline()
    samples = np.array(info[9:])
    return samples

def getCSV(inVCF, outFile, bcfpath = ''):
    outcsv = open(outFile, "w")
    samples = getSamples(inVCF)
    outcsv.write('Chromosome,Position')
    for es in samples:
        outcsv.write(',%s' % es)
    outcsv.write('\n')
    outcsv.truncate()
    log.info('running bcftools!')
    if bcfpath == '':
        bcftool_command = "bcftools query -f \"%CHROM,%POS[,%GT]\n\" " + inVCF
    else:
        bcftool_command = bcfpath + '/' + "bcftools query -f \"%CHROM,%POS[,%GT]\n\" " + inVCF
    sed_command = "| sed 's/,0[\/|]0/,0/g' | sed 's/,1[\/|]1/,1/g' | sed 's/,0[\/|]1/,2/g' | sed 's/,\.[\/|]\./,-1/g'"
    full_command = bcftool_command + sed_command
    convertcsv = Popen(full_command, shell=True, stdout = outcsv)
    convertcsv.wait()
    outcsv.close()
    log.info('done!')

def save_as_hdf5_acc(g, outHDF5):
    NumAcc = len(g.accessions)
    log.info("Writing into HDF5 file acc wise")
    h5file = h5py.File(outHDF5, 'w')
    NumSNPs = len(g.snps)
    h5file.create_dataset('accessions', data=g.accessions, shape=(NumAcc,))
    h5file.create_dataset('positions', data=g.positions, shape=(NumSNPs,),dtype='i4')
    h5file['positions'].attrs['chrs'] = g.chrs
    h5file['positions'].attrs['chr_regions'] = g.chr_regions
    h5file.create_dataset('snps', shape=(NumSNPs, NumAcc), dtype='int8', compression="gzip", chunks=((NumSNPs, 1)))
    for i in range(NumAcc):
        h5file['snps'][:,i] = np.array(g.snps)[:,i]
        if i+1 % 10 == 0:
            log.info("written SNP info for %s accessions", i+1)
    h5file['snps'].attrs['data_format'] = g.data_format
    h5file['snps'].attrs['num_snps'] = NumSNPs
    h5file['snps'].attrs['num_accessions'] = NumAcc
    h5file.close()

def makeHDF5s(csvFile, outFile):
    GenotypeData = genotype.load_csv_genotype_data(csvFile)
    log.info("saving CSV file into HDF5 file chunked rowwise")
    GenotypeData.save_as_hdf5(outFile + '.hdf5')
    log.info("done!")
    log.info("saving CSV file into HDF5 file chunked accession wise")
    save_as_hdf5_acc(GenotypeData, outFile + '.acc.hdf5')
    logging.info("done!")

def makedb_from_vcf(args):
    log.info("converting VCF to CSV")
    getCSV(args['inFile'], args['db_id'] + '.csv', args['bcfpath'])
    log.info("converting CSV to hdf5!")
    makeHDF5s(args['db_id'] + '.csv', args['db_id'])
    log.info('done!')
<commit_msg>checks making hdf5 files<commit_after>import logging
import h5py
import numpy as np
from pygwas.core import genotype
import sys
import os
import os.path
import json
import gzip
from subprocess import Popen, PIPE
import shlex

log = logging.getLogger(__name__)
def die(msg):
    sys.stderr.write('Error: ' + msg + '\n')
    sys.exit(1)

def getSamples(inVCF):
    vname, vext = os.path.splitext(inVCF)
    if vext == '.vcf':
        v = open(inVCF, 'r')
    elif vext == '.gz':
        v = gzip.open(inVCF)
    else:
        die('please provide VCF file either gunzipped or text!')
    eline = v.readline()
    while eline[0] == '#':
        if eline[0:6] == '#CHROM':
            info = eline.rstrip().split('\t')
        eline = v.readline()
    samples = np.array(info[9:])
    return samples

def getCSV(inVCF, outFile, bcfpath = ''):
    outcsv = open(outFile, "w")
    samples = getSamples(inVCF)
    outcsv.write('Chromosome,Position')
    for es in samples:
        outcsv.write(',%s' % es)
    outcsv.write('\n')
    outcsv.truncate()
    log.info('running bcftools!')
    if bcfpath == '':
        bcftool_command = "bcftools query -f \"%CHROM,%POS[,%GT]\n\" " + inVCF
    else:
        bcftool_command = bcfpath + '/' + "bcftools query -f \"%CHROM,%POS[,%GT]\n\" " + inVCF
    sed_command = "| sed 's/,0[\/|]0/,0/g' | sed 's/,1[\/|]1/,1/g' | sed 's/,0[\/|]1/,2/g' | sed 's/,\.[\/|]\./,-1/g'"
    full_command = bcftool_command + sed_command
    convertcsv = Popen(full_command, shell=True, stdout = outcsv)
    convertcsv.wait()
    outcsv.close()
    log.info('done!')

def save_as_hdf5_acc(g, outHDF5):
    NumAcc = len(g.accessions)
    log.info("Writing into HDF5 file acc wise")
    h5file = h5py.File(outHDF5, 'w')
    NumSNPs = len(g.snps)
    h5file.create_dataset('accessions', data=g.accessions, shape=(NumAcc,))
    h5file.create_dataset('positions', data=g.positions, shape=(NumSNPs,),dtype='i4')
    h5file['positions'].attrs['chrs'] = g.chrs
    h5file['positions'].attrs['chr_regions'] = g.chr_regions
    h5file.create_dataset('snps', shape=(NumSNPs, NumAcc), dtype='int8', compression="gzip", chunks=((NumSNPs, 1)))
    for i in range(NumAcc):
        h5file['snps'][:,i] = np.array(g.snps)[:,i]
        if i+1 % 10 == 0:
            log.info("written SNP info for %s accessions", i+1)
    h5file['snps'].attrs['data_format'] = g.data_format
    h5file['snps'].attrs['num_snps'] = NumSNPs
    h5file['snps'].attrs['num_accessions'] = NumAcc
    h5file.close()

def makeHDF5s(csvFile, outFile):
    GenotypeData = genotype.load_csv_genotype_data(csvFile)
    log.info("saving CSV file into HDF5 file chunked rowwise")
    GenotypeData.save_as_hdf5(outFile + '.hdf5')
    log.info("done!")
    log.info("saving CSV file into HDF5 file chunked accession wise")
    save_as_hdf5_acc(GenotypeData, outFile + '.acc.hdf5')
    logging.info("done!")

def makedb_from_vcf(args):
    _,inType = os.path.splitext(args['inFile'])
    if inType == '.vcf':
        log.info("converting VCF to CSV")
        getCSV(args['inFile'], args['db_id'] + '.csv', args['bcfpath'])
        log.info("converting CSV to hdf5!")
        makeHDF5s(args['db_id'] + '.csv', args['db_id'])
        log.info('done!')
    elif inType == '.csv':
        log.info("converting CSV to hdf5!")
        makeHDF5s(args['db_id'] + '.csv', args['db_id'])
        log.info('done!')
    else:
        die("please provide either a VCF file or a CSV!")
<|endoftext|>


<commit_before>__version__ = '3.3.2'
<commit_msg>Version bump<commit_after>__version__ = '3.3.3'
<|endoftext|>


<commit_before>from .dc_water_classifier import wofs_classify
import xarray as xr
import numpy as np
# This import is only for
from .vegetation import EVI, EVI2, NDVI

def EVI(*args, **kwargs):
    """
    Instead of this function, please use the EVI() function in vegetation.py.
    """
    return EVI(*args, **kwargs)

def EVI2(*args, **kwargs):
    """
    Instead of this function, please use the EVI2() function in vegetation.py.
    """
    return EVI2(*args, **kwargs)

def NDVI(*args, **kwargs):
    """
    Instead of this function, please use the NDVI() function in vegetation.py.
    """
    return NDVI(*args, **kwargs)

def compute_ndvi_anomaly(baseline_data,
                         scene_data,
                         baseline_clear_mask=None,
                         selected_scene_clear_mask=None,
                         no_data=-9999):
    """Compute the scene+baseline median ndvi values and the difference

    Args:
        basleine_data: xarray dataset with dims lat, lon, t
        scene_data: xarray dataset with dims lat, lon - should be mosaicked already.
        baseline_clear_mask: boolean mask signifying clear pixels for the baseline data
        selected_scene_clear_mask: boolean mask signifying lcear pixels for the baseline data
        no_data: nodata value for the datasets

    Returns:
        xarray dataset with scene_ndvi, baseline_ndvi(median), ndvi_difference, and ndvi_percentage_change.
    """

    assert selected_scene_clear_mask is not None and baseline_clear_mask is not None, "Both the selected scene and baseline data must have associated clear mask data."

    #cloud filter + nan out all nodata.
    baseline_data = baseline_data.where((baseline_data != no_data) & baseline_clear_mask)

    baseline_ndvi = (baseline_data.nir - baseline_data.red) / (baseline_data.nir + baseline_data.red)
    median_ndvi = baseline_ndvi.median('time')

    #scene should already be mosaicked.
    water_class = wofs_classify(scene_data, clean_mask=selected_scene_clear_mask, mosaic=True).wofs
    scene_cleaned = scene_data.copy(deep=True).where((scene_data != no_data) & (water_class == 0))
    scene_ndvi = (scene_cleaned.nir - scene_cleaned.red) / (scene_cleaned.nir + scene_cleaned.red)

    ndvi_difference = scene_ndvi - median_ndvi
    ndvi_percentage_change = (scene_ndvi - median_ndvi) / median_ndvi

    #convert to conventional nodata vals.
    scene_ndvi.values[~np.isfinite(scene_ndvi.values)] = no_data
    ndvi_difference.values[~np.isfinite(ndvi_difference.values)] = no_data
    ndvi_percentage_change.values[~np.isfinite(ndvi_percentage_change.values)] = no_data

    scene_ndvi_dataset = xr.Dataset(
        {
            'scene_ndvi': scene_ndvi,
            'baseline_ndvi': median_ndvi,
            'ndvi_difference': ndvi_difference,
            'ndvi_percentage_change': ndvi_percentage_change
        },
        coords={'latitude': scene_data.latitude,
                'longitude': scene_data.longitude})

    return scene_ndvi_dataset
<commit_msg>Fixed name clash in dc_ndvi_anomaly.py.<commit_after>from .dc_water_classifier import wofs_classify
import xarray as xr
import numpy as np
# This import is only for
from .vegetation import EVI as _EVI_orig, EVI2 as _EVI2_orig, NDVI as _NDVI_orig

def EVI(*args, **kwargs):
    """
    Instead of this function, please use the EVI() function in vegetation.py.
    """
    return _EVI_orig(*args, **kwargs)

def EVI2(*args, **kwargs):
    """
    Instead of this function, please use the EVI2() function in vegetation.py.
    """
    return _EVI2_orig(*args, **kwargs)

def NDVI(*args, **kwargs):
    """
    Instead of this function, please use the NDVI() function in vegetation.py.
    """
    return _NDVI_orig(*args, **kwargs)

def compute_ndvi_anomaly(baseline_data,
                         scene_data,
                         baseline_clear_mask=None,
                         selected_scene_clear_mask=None,
                         no_data=-9999):
    """Compute the scene+baseline median ndvi values and the difference

    Args:
        basleine_data: xarray dataset with dims lat, lon, t
        scene_data: xarray dataset with dims lat, lon - should be mosaicked already.
        baseline_clear_mask: boolean mask signifying clear pixels for the baseline data
        selected_scene_clear_mask: boolean mask signifying lcear pixels for the baseline data
        no_data: nodata value for the datasets

    Returns:
        xarray dataset with scene_ndvi, baseline_ndvi(median), ndvi_difference, and ndvi_percentage_change.
    """

    assert selected_scene_clear_mask is not None and baseline_clear_mask is not None, "Both the selected scene and baseline data must have associated clear mask data."

    #cloud filter + nan out all nodata.
    baseline_data = baseline_data.where((baseline_data != no_data) & baseline_clear_mask)

    baseline_ndvi = (baseline_data.nir - baseline_data.red) / (baseline_data.nir + baseline_data.red)
    median_ndvi = baseline_ndvi.median('time')

    #scene should already be mosaicked.
    water_class = wofs_classify(scene_data, clean_mask=selected_scene_clear_mask, mosaic=True).wofs
    scene_cleaned = scene_data.copy(deep=True).where((scene_data != no_data) & (water_class == 0))
    scene_ndvi = (scene_cleaned.nir - scene_cleaned.red) / (scene_cleaned.nir + scene_cleaned.red)

    ndvi_difference = scene_ndvi - median_ndvi
    ndvi_percentage_change = (scene_ndvi - median_ndvi) / median_ndvi

    #convert to conventional nodata vals.
    scene_ndvi.values[~np.isfinite(scene_ndvi.values)] = no_data
    ndvi_difference.values[~np.isfinite(ndvi_difference.values)] = no_data
    ndvi_percentage_change.values[~np.isfinite(ndvi_percentage_change.values)] = no_data

    scene_ndvi_dataset = xr.Dataset(
        {
            'scene_ndvi': scene_ndvi,
            'baseline_ndvi': median_ndvi,
            'ndvi_difference': ndvi_difference,
            'ndvi_percentage_change': ndvi_percentage_change
        },
        coords={'latitude': scene_data.latitude,
                'longitude': scene_data.longitude})

    return scene_ndvi_dataset
<|endoftext|>


<commit_before>import boto
import uuid
import urllib
import datetime
from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.orm.exc import MultipleResultsFound
from dataactcore.config import CONFIG_BROKER

class sesEmail(object):

    SIGNING_KEY  ="1234"
    INVALID_LINK  = 1
    LINK_EXPIRED =  2
    LINK_ALREADY_USED = 3
    LINK_VALID  = 0
    isLocal = False
    emailLog = "Email.log"
    def __init__(self,toAddress,fromAddress,content="",subject="",templateType=None,parameters=None, database=None):
        self.toAddress = toAddress
        self.fromAddress = fromAddress
        if(templateType is None):
            self.content = content
            self.subject = subject
        else:
            template = database.getEmailTemplate(templateType)
            self.subject = template.subject
            self.content = template.content

            for key in parameters :
                if(not parameters[key] is None) :
                    self.content = self.content.replace(key,parameters[key])
                else :
                    self.content = self.content.replace(key,"")

    def send(self):
        if(not sesEmail.isLocal):
            raise NotImplementedError("SES through different AWS account not yet implemented")
            connection = boto.ses.connect_to_region(CONFIG_BROKER["aws_region"])
            return connection.send_email(self.fromAddress, self.subject,self.content,self.toAddress,format='html')
        else:
            newEmailText = "\n\n".join(["","Time",str(datetime.datetime.now()),"Subject",self.subject,"From",self.fromAddress,"To",self.toAddress,"Content",self.content])
            open (sesEmail.emailLog,"a").write(newEmailText)

    @staticmethod
    def createToken(emailAddress,database,token_type) :
        """Creates a token to be used and saves it with the salt in the database"""
        salt = uuid.uuid1().int
        ts = URLSafeTimedSerializer(sesEmail.SIGNING_KEY)
        token = ts.dumps(emailAddress, salt=str(salt)+token_type)
        #saves the token and salt pair
        database.saveToken(str(salt),str(token))
        return urllib.quote_plus(str(token))

    @staticmethod
    def checkToken(token,database,token_type):
        """Gets token's salt and decodes it"""
        saltValue = None
        try:
            saltValue = database.getTokenSalt(token)
        except MultipleResultsFound as e:
            #duplicate tokens
            return False ,"Invalid Link", sesEmail.INVALID_LINK
        except NoResultFound as e:
            #Token already used or never existed in the first place
            return False,"Link already used",sesEmail.LINK_ALREADY_USED
        ts = URLSafeTimedSerializer(sesEmail.SIGNING_KEY)
        try:
            emailAddress = ts.loads(token, salt=saltValue[0]+token_type, max_age=86400)
            return True,emailAddress,sesEmail.LINK_VALID
        except BadSignature as e:
            #Token is malformed
            return False,"Invalid Link",sesEmail.INVALID_LINK
        except SignatureExpired:
            #Token is to old
            return False,"Link Expired",sesEmail.LINK_EXPIRED
<commit_msg>Adding keys to see to send emails<commit_after>import boto
import uuid
import urllib
import datetime
from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.orm.exc import MultipleResultsFound
from dataactcore.config import CONFIG_BROKER

class sesEmail(object):

    SIGNING_KEY  ="1234"
    INVALID_LINK  = 1
    LINK_EXPIRED =  2
    LINK_ALREADY_USED = 3
    LINK_VALID  = 0
    isLocal = False
    emailLog = "Email.log"
    def __init__(self,toAddress,fromAddress,content="",subject="",templateType=None,parameters=None, database=None):
        self.toAddress = toAddress
        self.fromAddress = fromAddress
        if(templateType is None):
            self.content = content
            self.subject = subject
        else:
            template = database.getEmailTemplate(templateType)
            self.subject = template.subject
            self.content = template.content

            for key in parameters :
                if(not parameters[key] is None) :
                    self.content = self.content.replace(key,parameters[key])
                else :
                    self.content = self.content.replace(key,"")

    def send(self):
        if(not sesEmail.isLocal):
            connection = boto.connect_ses(aws_access_key_id=CONFIG_BROKER['aws_access_key_id'], aws_secret_access_key=CONFIG_BROKER['aws_secret_access_key'])
            return connection.send_email(self.fromAddress, self.subject,self.content,self.toAddress,format='html')
        else:
            newEmailText = "\n\n".join(["","Time",str(datetime.datetime.now()),"Subject",self.subject,"From",self.fromAddress,"To",self.toAddress,"Content",self.content])
            open (sesEmail.emailLog,"a").write(newEmailText)

    @staticmethod
    def createToken(emailAddress,database,token_type) :
        """Creates a token to be used and saves it with the salt in the database"""
        salt = uuid.uuid1().int
        ts = URLSafeTimedSerializer(sesEmail.SIGNING_KEY)
        token = ts.dumps(emailAddress, salt=str(salt)+token_type)
        #saves the token and salt pair
        database.saveToken(str(salt),str(token))
        return urllib.quote_plus(str(token))

    @staticmethod
    def checkToken(token,database,token_type):
        """Gets token's salt and decodes it"""
        saltValue = None
        try:
            saltValue = database.getTokenSalt(token)
        except MultipleResultsFound as e:
            #duplicate tokens
            return False ,"Invalid Link", sesEmail.INVALID_LINK
        except NoResultFound as e:
            #Token already used or never existed in the first place
            return False,"Link already used",sesEmail.LINK_ALREADY_USED
        ts = URLSafeTimedSerializer(sesEmail.SIGNING_KEY)
        try:
            emailAddress = ts.loads(token, salt=saltValue[0]+token_type, max_age=86400)
            return True,emailAddress,sesEmail.LINK_VALID
        except BadSignature as e:
            #Token is malformed
            return False,"Invalid Link",sesEmail.INVALID_LINK
        except SignatureExpired:
            #Token is to old
            return False,"Link Expired",sesEmail.LINK_EXPIRED
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-

from django.db import models

class ExampleModel(models.Model):
    name = models.CharField(max_length=15)
    date_created = models.DateTimeField(auto_now_add=True)
    related = models.ForeignKey('RelatedModel', blank=True, null=True)
    relateds = models.ManyToManyField('RelatedM2MModel', blank=True)

    def __unicode__(self):
        return "<ExampleModel: %d:'%s'>" % (self.pk, self.name,)

    def __repr__(self):
        return "<ExampleModel: %d:'%s'>" % (self.pk, self.name,)

    def get_absolute_url(self):
        return "#{pk}".format(pk=self.pk)

class RelatedModel(models.Model):
    name = models.CharField(max_length=15)

    def get_absolute_url(self):
        return "#{pk}".format(pk=self.pk)

class RelatedM2MModel(models.Model):
    name = models.CharField(max_length=15)

class ReverseRelatedModel(models.Model):
    name = models.CharField(max_length=15)
    example = models.ForeignKey('ExampleModel')
<commit_msg>Smarter<commit_after># -*- encoding: utf-8 -*-

from django.db import models

class ExampleModel(models.Model):
    name = models.CharField(max_length=15)
    date_created = models.DateTimeField(auto_now_add=True)
    related = models.ForeignKey('RelatedModel', blank=True, null=True)
    relateds = models.ManyToManyField('RelatedM2MModel', blank=True)

    def __unicode__(self):
        return "ExampleModel %d" % (self.pk,)

    def __repr__(self):
        return "<ExampleModel: %d:'%s'>" % (self.pk, self.name,)

    def get_absolute_url(self):
        return "#{pk}".format(pk=self.pk)

class RelatedModel(models.Model):
    name = models.CharField(max_length=15)

    def get_absolute_url(self):
        return "#{pk}".format(pk=self.pk)

class RelatedM2MModel(models.Model):
    name = models.CharField(max_length=15)

class ReverseRelatedModel(models.Model):
    name = models.CharField(max_length=15)
    example = models.ForeignKey('ExampleModel')
<|endoftext|>


<commit_before>VERSION = (0, 1, 'pre')
<commit_msg>Removed version information which was accidentally copied to templatetags\__init__.py<commit_after><|endoftext|>


<commit_before>from django.db import models
from django.contrib.auth.models import User

class Profile(models.Model):

    user = models.ForeignKey(User, unique=True)
    name = models.CharField(max_length=50, null=True, blank=True)
    about = models.TextField(null=True, blank=True)
    location = models.CharField(max_length=40, null=True, blank=True)
    website = models.URLField(null=True, blank=True)

    class Admin:
        pass<commit_msg>added a __unicode__ for Profile<commit_after>from django.db import models
from django.contrib.auth.models import User

class Profile(models.Model):

    user = models.ForeignKey(User, unique=True)
    name = models.CharField(max_length=50, null=True, blank=True)
    about = models.TextField(null=True, blank=True)
    location = models.CharField(max_length=40, null=True, blank=True)
    website = models.URLField(null=True, blank=True)

    def __unicode__(self):
        return self.user.username

    class Admin:
        pass<|endoftext|>


<commit_before>import MySQLdb as mdb

def connect_db():
    connection = mdb.connect('108.167.133.34', 'connie_usr_aed', 'SK5CTTs8zXV9', 'connie_facebook')
    #connection = mdb.connect('localhost', 'root', 'root', 'facebook_data')
    connection.set_character_set('utf8mb4')
    return connection

def get_cursor(connect):
    cursor = connect.cursor()
    cursor.execute('SET NAMES utf8mb4;')
    cursor.execute('SET CHARACTER SET utf8mb4;')
    cursor.execute('SET character_set_connection=utf8mb4;')
    return cursor

def get_comment(id, inicio, fin):
    query_comment = "SELECT message FROM comment WHERE id_post = " + str(id) + " AND STR_TO_DATE(created_time, '%Y-%m-%dT%H:%i:%s+0000') BETWEEN STR_TO_DATE('"+ str(inicio) +"', '%Y-%m-%dT%H:%i:%s+0000') AND STR_TO_DATE('"+ str(fin) +"', '%Y-%m-%dT%H:%i:%s+0000');"
    return query_comment

def get_post(inicio, fin):
    query_post = "SELECT id_post, message AS POSTS FROM post WHERE STR_TO_DATE(created_time, '%Y-%m-%dT%H:%i:%s+0000') BETWEEN STR_TO_DATE('"+ str(inicio) +"', '%Y-%m-%dT%H:%i:%s+0000') AND STR_TO_DATE('"+ str(fin) +"', '%Y-%m-%dT%H:%i:%s+0000');"
    return query_post

if __name__ == '__main__':
    connection = connect_db()
    cursor = get_cursor(connection)

    query_post = "SELECT id_post, message AS POSTS FROM post WHERE STR_TO_DATE(created_time, '%Y-%m-%dT%H:%i:%s+0000') BETWEEN "
    query_post2 = "STR_TO_DATE('2016-06-17T01:22:44+0000', '%Y-%m-%dT%H:%i:%s+0000') AND "
    query_post3 = "STR_TO_DATE('2016-06-17T04:01:30+0000', '%Y-%m-%dT%H:%i:%s+0000');"
    #query_comment = "SELECT message AS COMMENTS FROM comment WHERE id_post = %s AND STR_TO_DATE(created_time, '%Y-%m-%dT%H:%i:%s+0000') BETWEEN STR_TO_DATE('2016-06-17T01:22:44+0000', '%Y-%m-%dT%H:%i:%s+0000') AND STR_TO_DATE('2016-06-17T04:01:30+0000', '%Y-%m-%dT%H:%i:%s+0000');"
    query_comment = "SELECT message FROM comment WHERE id_post = %d AND STR_TO_DATE(created_time, '%Y-%m-%dT%H:%i:%s+0000') BETWEEN STR_TO_DATE('2016-06-17T01:22:44+0000', '%Y-%m-%dT%H:%i:%s+0000') AND STR_TO_DATE('2016-06-17T04:01:30+0000', '%Y-%m-%dT%H:%i:%s+0000');"

    inicio = '2016-06-26T00:24:34+0000'
    fin = '2016-06-26T03:50:36+0000'
    cursor.execute(get_post(inicio, fin))
    posts = cursor.fetchall()
    count = 0
    for post in posts:
        count = count + 1
        name = "post" + str(count) + ".txt"
        f = open(name, 'w')
        id_post = post[0]
        message = post[1]
        f.write(message+'\n')
        relation = (id_post)
        cursor.execute(get_comment(id_post, inicio, fin))
        comments = cursor.fetchall()
        for comment in comments:
            c_message = comment[0]
            f.write(c_message + '\n')
    f.close()

<commit_msg>post de comunidades<commit_after>import MySQLdb as mdb

def connect_db():
    connection = mdb.connect('108.167.133.34', 'connie_usr_aed', 'SK5CTTs8zXV9', 'connie_facebook')
    #connection = mdb.connect('localhost', 'root', 'root', 'facebook_data')
    connection.set_character_set('utf8mb4')
    return connection

def get_cursor(connect):
    cursor = connect.cursor()
    cursor.execute('SET NAMES utf8mb4;')
    cursor.execute('SET CHARACTER SET utf8mb4;')
    cursor.execute('SET character_set_connection=utf8mb4;')
    return cursor

def get_comment(id, inicio, fin):
    query_comment = "SELECT message FROM comment WHERE id_post = " + str(id) + " AND STR_TO_DATE(created_time, '%Y-%m-%dT%H:%i:%s+0000') BETWEEN STR_TO_DATE('"+ str(inicio) +"', '%Y-%m-%dT%H:%i:%s+0000') AND STR_TO_DATE('"+ str(fin) +"', '%Y-%m-%dT%H:%i:%s+0000');"
    return query_comment

def get_post(inicio, fin):
    query_post = "SELECT id_post, message AS POSTS FROM post WHERE STR_TO_DATE(created_time, '%Y-%m-%dT%H:%i:%s+0000') BETWEEN STR_TO_DATE('"+ str(inicio) +"', '%Y-%m-%dT%H:%i:%s+0000') AND STR_TO_DATE('"+ str(fin) +"', '%Y-%m-%dT%H:%i:%s+0000');"
    return query_post

if __name__ == '__main__':
    '''
    inicio = '2016-06-17T01:22:44+0000' fin = '2016-06-17T04:01:30+0000'#EEUU_VS_Ecuador
    inicio = '2016-06-17T23:42:20+0000' fin = '2016-06-18T02:55:17+0000'#Colombia_VS_Peru
    inicio = '2016-06-19T01:26:00+0000' fin = '2016-06-19T04:23:23+0000'#Mexico_VS_Chile
    inicio = '2016-06-22T02:02:20+0000' fin = '2016-06-22T04:20:43+0000'#EEUU_VS_Argentina
    inicio = '2016-06-22T23:40:45+0000' fin = '2016-06-23T05:12:30+0000'#Chile_VS_Colombia
    inicio = '2016-06-26T00:24:34+0000' fin = '2016-06-26T03:50:36+0000'#EEUU_VS_Colombia
    inicio = '2016-06-26T23:24:26+0000' fin = '2016-06-27T05:42:49+0000'#Argentina_VS_Chile
    '''
    inicio = '2016-06-17T01:22:44+0000'
    fin = '2016-06-17T04:01:30+0000'  # EEUU_VS_Ecuador
    connection = connect_db()
    cursor = get_cursor(connection)
    query_post = "SELECT id_post, message AS POSTS FROM post WHERE STR_TO_DATE(created_time, '%Y-%m-%dT%H:%i:%s+0000') BETWEEN "
    query_post2 = "STR_TO_DATE('2016-06-17T01:22:44+0000', '%Y-%m-%dT%H:%i:%s+0000') AND "
    query_post3 = "STR_TO_DATE('2016-06-17T04:01:30+0000', '%Y-%m-%dT%H:%i:%s+0000');"
    #query_comment = "SELECT message AS COMMENTS FROM comment WHERE id_post = %s AND STR_TO_DATE(created_time, '%Y-%m-%dT%H:%i:%s+0000') BETWEEN STR_TO_DATE('2016-06-17T01:22:44+0000', '%Y-%m-%dT%H:%i:%s+0000') AND STR_TO_DATE('2016-06-17T04:01:30+0000', '%Y-%m-%dT%H:%i:%s+0000');"
    query_comment = "SELECT message FROM comment WHERE id_post = %d AND STR_TO_DATE(created_time, '%Y-%m-%dT%H:%i:%s+0000') BETWEEN STR_TO_DATE('2016-06-17T01:22:44+0000', '%Y-%m-%dT%H:%i:%s+0000') AND STR_TO_DATE('2016-06-17T04:01:30+0000', '%Y-%m-%dT%H:%i:%s+0000');"

    inicio = '2016-06-26T00:24:34+0000'
    fin = '2016-06-26T03:50:36+0000'
    cursor.execute(get_post(inicio, fin))
    posts = cursor.fetchall()
    count = 0
    for post in posts:
        count = count + 1
        name = "community" + str(count) + ".txt"
        f = open(name, 'w')
        id_post = post[0]
        message = post[1]
        f.write(message+'\n')
        relation = (id_post)
        #cursor.execute(get_comment(id_post, inicio, fin))
        #comments = cursor.fetchall()
        #for comment in comments:
            #c_message = comment[0]
            #f.write(c_message + '\n')
    f.close()

<|endoftext|>