<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Rico Sennrich

"""Use operations learned with learn_bpe.py to encode a new text.
The text will not be smaller, but use only a fixed vocabulary, with rare words
encoded as variable-length sequences of subword units.

Reference:
Rico Sennrich, Barry Haddow and Alexandra Birch (2015). Neural Machine Translation of Rare Words with Subword Units.
Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (ACL 2016). Berlin, Germany.
"""

from __future__ import unicode_literals, division

import sys
import os
import inspect
import codecs
import io
import argparse
import re
import warnings
import random

# hack for python2/3 compatibility
from io import open
argparse.open = open

class BPE(object):

    def __init__(self, codes, merges=-1, separator='@@', vocab=None, glossaries=None):

        codes.seek(0)
        offset=1

        # check version information
        firstline = codes.readline()
        if firstline.startswith('#version:'):
            self.version = tuple([int(x) for x in re.sub(r'(\.0+)*$','', firstline.split()[-1]).split(".")])
            offset += 1
        else:
            self.version = (0, 1)
            codes.seek(0)

        self.bpe_codes = [tuple(item.strip('\r\n ').split(' ')) for (n, item) in enumerate(codes) if (n < merges or merges == -1)]

        for i, item in enumerate(self.bpe_codes):
            if len(item) != 2:
                sys.stderr.write('Error: invalid line {0} in BPE codes file: {1}\n'.format(i+offset, ' '.join(item)))
                sys.stderr.write('The line should exist of exactly two subword units, separated by whitespace\n')
                sys.exit(1)

        # some hacking to deal with duplicates (only consider first instance)
        self.bpe_codes = dict([(code,i) for (i,code) in reversed(list(enumerate(self.bpe_codes)))])

        self.bpe_codes_reverse = dict([(pair[0] + pair[1], pair) for pair,i in self.bpe_codes.items()])

        self.separator = separator

        self.vocab = vocab

        self.glossaries = glossaries if glossaries else []

        self.glossaries_regex = re.compile('^({})$'.format('|'.join(glossaries))) if glossaries else None

        self.cache = {}

    def process_line(self, line, dropout=0):
        """segment line, dealing with leading and trailing whitespace"""

        out = ""

        leading_whitespace = len(line)-len(line.lstrip('\r\n '))
        if leading_whitespace:
            out += line[:leading_whitespace]

        out += self.segment(line, dropout)

        trailing_whitespace = len(line)-len(line.rstrip('\r\n '))
        if trailing_whitespace and trailing_whitespace != len(line):
            out += line[-trailing_whitespace:]

        return out

    def segment(self, sentence, dropout=0):
        """segment single sentence (whitespace-tokenized string) with BPE encoding"""
        segments = self.segment_tokens(sentence.strip('\r\n ').split(' '), dropout)
        return ' '.join(segments)

    def segment_tokens(self, tokens, dropout=0):
        """segment a sequence of tokens with BPE encoding"""
        output = []
        for word in tokens:
            # eliminate double spaces
            if not word:
                continue
            new_word = [out for segment in self._isolate_glossaries(word)
                        for out in encode(segment,
                                          self.bpe_codes,
                                          self.bpe_codes_reverse,
                                          self.vocab,
                                          self.separator,
                                          self.version,
                                          self.cache,
                                          self.glossaries_regex,
                                          dropout)]

            for item in new_word[:-1]:
                output.append(item + self.separator)
            output.append(new_word[-1])

        return output

    def _isolate_glossaries(self, word):
        word_segments = [word]
        for gloss in self.glossaries:
            word_segments = [out_segments for segment in word_segments
                                 for out_segments in isolate_glossary(segment, gloss)]
        return word_segments

def create_parser(subparsers=None):

    if subparsers:
        parser = subparsers.add_parser('apply-bpe',
            formatter_class=argparse.RawDescriptionHelpFormatter,
            description="learn BPE-based word segmentation")
    else:
        parser = argparse.ArgumentParser(
            formatter_class=argparse.RawDescriptionHelpFormatter,
            description="learn BPE-based word segmentation")

    parser.add_argument(
        '--input', '-i', type=argparse.FileType('r'), default=sys.stdin,
        metavar='PATH',
        help="Input file (default: standard input).")
    parser.add_argument(
        '--codes', '-c', type=argparse.FileType('r'), metavar='PATH',
        required=True,
        help="File with BPE codes (created by learn_bpe.py).")
    parser.add_argument(
        '--merges', '-m', type=int, default=-1,
        metavar='INT',
        help="Use this many BPE operations (<= number of learned symbols)"+
             "default: Apply all the learned merge operations")
    parser.add_argument(
        '--output', '-o', type=argparse.FileType('w'), default=sys.stdout,
        metavar='PATH',
        help="Output file (default: standard output)")
    parser.add_argument(
        '--separator', '-s', type=str, default='@@', metavar='STR',
        help="Separator between non-final subword units (default: '%(default)s'))")
    parser.add_argument(
        '--vocabulary', type=argparse.FileType('r'), default=None,
        metavar="PATH",
        help="Vocabulary file (built with get_vocab.py). If provided, this script reverts any merge operations that produce an OOV.")
    parser.add_argument(
        '--vocabulary-threshold', type=int, default=None,
        metavar="INT",
        help="Vocabulary threshold. If vocabulary is provided, any word with frequency < threshold will be treated as OOV")
    parser.add_argument(
        '--dropout', type=float, default=0,
        metavar="P",
        help="Dropout BPE merge operations with probability P (Provilkov et al., 2019). Use this on training data only.")
    parser.add_argument(
        '--glossaries', type=str, nargs='+', default=None,
        metavar="STR",
        help="Glossaries. Words matching any of the words/regex provided in glossaries will not be affected "+
             "by the BPE (i.e. they will neither be broken into subwords, nor concatenated with other subwords. "+
             "Can be provided as a list of words/regex after the --glossaries argument. Enclose each regex in quotes.")

    return parser

def encode(orig, bpe_codes, bpe_codes_reverse, vocab, separator, version, cache, glossaries_regex=None, dropout=0):
    """Encode word based on list of BPE merge operations, which are applied consecutively
    """

    if not dropout and orig in cache:
        return cache[orig]

    if glossaries_regex and glossaries_regex.match(orig):
        cache[orig] = (orig,)
        return (orig,)

    if len(orig) == 1:
        return orig

    if version == (0, 1):
        word = list(orig) + ['</w>']
    elif version == (0, 2): # more consistent handling of word-final segments
        word = list(orig[:-1]) + [orig[-1] + '</w>']
    else:
        raise NotImplementedError

    while len(word) > 1:

        # get list of symbol pairs; optionally apply dropout
        pairs = [(bpe_codes[pair],i,pair) for (i,pair) in enumerate(zip(word, word[1:])) if (not dropout or random.random() > dropout) and pair in bpe_codes]

        if not pairs:
            break

        #get first merge operation in list of BPE codes
        bigram = min(pairs)[2]

        # find start position of all pairs that we want to merge
        positions = [i for (rank,i,pair) in pairs if pair == bigram]

        i = 0
        new_word = []
        bigram = ''.join(bigram)
        for j in positions:
            # merges are invalid if they start before current position. This can happen if there are overlapping pairs: (x x x -> xx x)
            if j < i:
                continue
            new_word.extend(word[i:j]) # all symbols before merged pair
            new_word.append(bigram) # merged pair
            i = j+2 # continue after merged pair
        new_word.extend(word[i:]) # add all symbols until end of word
        word = new_word

    # don't print end-of-word symbols
    if word[-1] == '</w>':
        word = word[:-1]
    elif word[-1].endswith('</w>'):
        word[-1] = word[-1][:-4]

    word = tuple(word)
    if vocab:
        word = check_vocab_and_split(word, bpe_codes_reverse, vocab, separator)

    cache[orig] = word
    return word

def recursive_split(segment, bpe_codes, vocab, separator, final=False):
    """Recursively split segment into smaller units (by reversing BPE merges)
    until all units are either in-vocabulary, or cannot be split futher."""

    try:
        if final:
            left, right = bpe_codes[segment + '</w>']
            right = right[:-4]
        else:
            left, right = bpe_codes[segment]
    except:
        #sys.stderr.write('cannot split {0} further.\n'.format(segment))
        yield segment
        return

    if left + separator in vocab:
        yield left
    else:
        for item in recursive_split(left, bpe_codes, vocab, separator, False):
            yield item

    if (final and right in vocab) or (not final and right + separator in vocab):
        yield right
    else:
        for item in recursive_split(right, bpe_codes, vocab, separator, final):
            yield item

def check_vocab_and_split(orig, bpe_codes, vocab, separator):
    """Check for each segment in word if it is in-vocabulary,
    and segment OOV segments into smaller units by reversing the BPE merge operations"""

    out = []

    for segment in orig[:-1]:
        if segment + separator in vocab:
            out.append(segment)
        else:
            #sys.stderr.write('OOV: {0}\n'.format(segment))
            for item in recursive_split(segment, bpe_codes, vocab, separator, False):
                out.append(item)

    segment = orig[-1]
    if segment in vocab:
        out.append(segment)
    else:
        #sys.stderr.write('OOV: {0}\n'.format(segment))
        for item in recursive_split(segment, bpe_codes, vocab, separator, True):
            out.append(item)

    return out

def read_vocabulary(vocab_file, threshold):
    """read vocabulary file produced by get_vocab.py, and filter according to frequency threshold.
    """

    vocabulary = set()

    for line in vocab_file:
        word, freq = line.strip('\r\n ').split(' ')
        freq = int(freq)
        if threshold == None or freq >= threshold:
            vocabulary.add(word)

    return vocabulary

def isolate_glossary(word, glossary):
    """
    Isolate a glossary present inside a word.

    Returns a list of subwords. In which all 'glossary' glossaries are isolated

    For example, if 'USA' is the glossary and '1934USABUSA' the word, the return value is:
        ['1934', 'USA', 'B', 'USA']
    """
    # regex equivalent of (if word == glossary or glossary not in word)
    if re.match('^'+glossary+'$', word) or not re.search(glossary, word):
        return [word]
    else:
        segments = re.split(r'({})'.format(glossary), word)
        segments, ending = segments[:-1], segments[-1]
        segments = list(filter(None, segments)) # Remove empty strings in regex group.
        return segments + [ending.strip('\r\n ')] if ending != '' else segments

if __name__ == '__main__':

    currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
    newdir = os.path.join(currentdir, 'subword_nmt')
    if os.path.isdir(newdir):
        warnings.simplefilter('default')
        warnings.warn(
            "this script's location has moved to {0}. This symbolic link will be removed in a future version. Please point to the new location, or install the package and use the command 'subword-nmt'".format(newdir),
            DeprecationWarning
        )

    # python 2/3 compatibility
    if sys.version_info < (3, 0):
        sys.stderr = codecs.getwriter('UTF-8')(sys.stderr)
        sys.stdout = codecs.getwriter('UTF-8')(sys.stdout)
        sys.stdin = codecs.getreader('UTF-8')(sys.stdin)
    else:
        sys.stdin = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8')
        sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
        sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', write_through=True, line_buffering=True)

    parser = create_parser()
    args = parser.parse_args()

    # read/write files as UTF-8
    args.codes = codecs.open(args.codes.name, encoding='utf-8')
    if args.input.name != '<stdin>':
        args.input = codecs.open(args.input.name, encoding='utf-8')
    if args.output.name != '<stdout>':
        args.output = codecs.open(args.output.name, 'w', encoding='utf-8')
    if args.vocabulary:
        args.vocabulary = codecs.open(args.vocabulary.name, encoding='utf-8')

    if args.vocabulary:
        vocabulary = read_vocabulary(args.vocabulary, args.vocabulary_threshold)
    else:
        vocabulary = None

    if sys.version_info < (3, 0):
        args.separator = args.separator.decode('UTF-8')
        if args.glossaries:
            args.glossaries = [g.decode('UTF-8') for g in args.glossaries]

    bpe = BPE(args.codes, args.merges, args.separator, vocabulary, args.glossaries)

    for line in args.input:
        args.output.write(bpe.process_line(line, args.dropout))
<commit_msg>Add --seed command-line argument to apply-bpe to make --dropout reproducible<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Rico Sennrich

"""Use operations learned with learn_bpe.py to encode a new text.
The text will not be smaller, but use only a fixed vocabulary, with rare words
encoded as variable-length sequences of subword units.

Reference:
Rico Sennrich, Barry Haddow and Alexandra Birch (2015). Neural Machine Translation of Rare Words with Subword Units.
Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (ACL 2016). Berlin, Germany.
"""

from __future__ import unicode_literals, division

import sys
import os
import inspect
import codecs
import io
import argparse
import re
import warnings
import random

# hack for python2/3 compatibility
from io import open
argparse.open = open

class BPE(object):

    def __init__(self, codes, merges=-1, separator='@@', vocab=None, glossaries=None):

        codes.seek(0)
        offset=1

        # check version information
        firstline = codes.readline()
        if firstline.startswith('#version:'):
            self.version = tuple([int(x) for x in re.sub(r'(\.0+)*$','', firstline.split()[-1]).split(".")])
            offset += 1
        else:
            self.version = (0, 1)
            codes.seek(0)

        self.bpe_codes = [tuple(item.strip('\r\n ').split(' ')) for (n, item) in enumerate(codes) if (n < merges or merges == -1)]

        for i, item in enumerate(self.bpe_codes):
            if len(item) != 2:
                sys.stderr.write('Error: invalid line {0} in BPE codes file: {1}\n'.format(i+offset, ' '.join(item)))
                sys.stderr.write('The line should exist of exactly two subword units, separated by whitespace\n')
                sys.exit(1)

        # some hacking to deal with duplicates (only consider first instance)
        self.bpe_codes = dict([(code,i) for (i,code) in reversed(list(enumerate(self.bpe_codes)))])

        self.bpe_codes_reverse = dict([(pair[0] + pair[1], pair) for pair,i in self.bpe_codes.items()])

        self.separator = separator

        self.vocab = vocab

        self.glossaries = glossaries if glossaries else []

        self.glossaries_regex = re.compile('^({})$'.format('|'.join(glossaries))) if glossaries else None

        self.cache = {}

    def process_line(self, line, dropout=0):
        """segment line, dealing with leading and trailing whitespace"""

        out = ""

        leading_whitespace = len(line)-len(line.lstrip('\r\n '))
        if leading_whitespace:
            out += line[:leading_whitespace]

        out += self.segment(line, dropout)

        trailing_whitespace = len(line)-len(line.rstrip('\r\n '))
        if trailing_whitespace and trailing_whitespace != len(line):
            out += line[-trailing_whitespace:]

        return out

    def segment(self, sentence, dropout=0):
        """segment single sentence (whitespace-tokenized string) with BPE encoding"""
        segments = self.segment_tokens(sentence.strip('\r\n ').split(' '), dropout)
        return ' '.join(segments)

    def segment_tokens(self, tokens, dropout=0):
        """segment a sequence of tokens with BPE encoding"""
        output = []
        for word in tokens:
            # eliminate double spaces
            if not word:
                continue
            new_word = [out for segment in self._isolate_glossaries(word)
                        for out in encode(segment,
                                          self.bpe_codes,
                                          self.bpe_codes_reverse,
                                          self.vocab,
                                          self.separator,
                                          self.version,
                                          self.cache,
                                          self.glossaries_regex,
                                          dropout)]

            for item in new_word[:-1]:
                output.append(item + self.separator)
            output.append(new_word[-1])

        return output

    def _isolate_glossaries(self, word):
        word_segments = [word]
        for gloss in self.glossaries:
            word_segments = [out_segments for segment in word_segments
                                 for out_segments in isolate_glossary(segment, gloss)]
        return word_segments

def create_parser(subparsers=None):

    if subparsers:
        parser = subparsers.add_parser('apply-bpe',
            formatter_class=argparse.RawDescriptionHelpFormatter,
            description="learn BPE-based word segmentation")
    else:
        parser = argparse.ArgumentParser(
            formatter_class=argparse.RawDescriptionHelpFormatter,
            description="learn BPE-based word segmentation")

    parser.add_argument(
        '--input', '-i', type=argparse.FileType('r'), default=sys.stdin,
        metavar='PATH',
        help="Input file (default: standard input).")
    parser.add_argument(
        '--codes', '-c', type=argparse.FileType('r'), metavar='PATH',
        required=True,
        help="File with BPE codes (created by learn_bpe.py).")
    parser.add_argument(
        '--merges', '-m', type=int, default=-1,
        metavar='INT',
        help="Use this many BPE operations (<= number of learned symbols)"+
             "default: Apply all the learned merge operations")
    parser.add_argument(
        '--output', '-o', type=argparse.FileType('w'), default=sys.stdout,
        metavar='PATH',
        help="Output file (default: standard output)")
    parser.add_argument(
        '--separator', '-s', type=str, default='@@', metavar='STR',
        help="Separator between non-final subword units (default: '%(default)s'))")
    parser.add_argument(
        '--vocabulary', type=argparse.FileType('r'), default=None,
        metavar="PATH",
        help="Vocabulary file (built with get_vocab.py). If provided, this script reverts any merge operations that produce an OOV.")
    parser.add_argument(
        '--vocabulary-threshold', type=int, default=None,
        metavar="INT",
        help="Vocabulary threshold. If vocabulary is provided, any word with frequency < threshold will be treated as OOV")
    parser.add_argument(
        '--dropout', type=float, default=0,
        metavar="P",
        help="Dropout BPE merge operations with probability P (Provilkov et al., 2019). Use this on training data only.")
    parser.add_argument(
        '--glossaries', type=str, nargs='+', default=None,
        metavar="STR",
        help="Glossaries. Words matching any of the words/regex provided in glossaries will not be affected "+
             "by the BPE (i.e. they will neither be broken into subwords, nor concatenated with other subwords. "+
             "Can be provided as a list of words/regex after the --glossaries argument. Enclose each regex in quotes.")
    parser.add_argument(
        '--seed', type=int, default=None,
        metavar="S",
        help="Random seed for the random number generators (e.g. for BPE dropout with --dropout).")

    return parser

def encode(orig, bpe_codes, bpe_codes_reverse, vocab, separator, version, cache, glossaries_regex=None, dropout=0):
    """Encode word based on list of BPE merge operations, which are applied consecutively
    """

    if not dropout and orig in cache:
        return cache[orig]

    if glossaries_regex and glossaries_regex.match(orig):
        cache[orig] = (orig,)
        return (orig,)

    if len(orig) == 1:
        return orig

    if version == (0, 1):
        word = list(orig) + ['</w>']
    elif version == (0, 2): # more consistent handling of word-final segments
        word = list(orig[:-1]) + [orig[-1] + '</w>']
    else:
        raise NotImplementedError

    while len(word) > 1:

        # get list of symbol pairs; optionally apply dropout
        pairs = [(bpe_codes[pair],i,pair) for (i,pair) in enumerate(zip(word, word[1:])) if (not dropout or random.random() > dropout) and pair in bpe_codes]

        if not pairs:
            break

        #get first merge operation in list of BPE codes
        bigram = min(pairs)[2]

        # find start position of all pairs that we want to merge
        positions = [i for (rank,i,pair) in pairs if pair == bigram]

        i = 0
        new_word = []
        bigram = ''.join(bigram)
        for j in positions:
            # merges are invalid if they start before current position. This can happen if there are overlapping pairs: (x x x -> xx x)
            if j < i:
                continue
            new_word.extend(word[i:j]) # all symbols before merged pair
            new_word.append(bigram) # merged pair
            i = j+2 # continue after merged pair
        new_word.extend(word[i:]) # add all symbols until end of word
        word = new_word

    # don't print end-of-word symbols
    if word[-1] == '</w>':
        word = word[:-1]
    elif word[-1].endswith('</w>'):
        word[-1] = word[-1][:-4]

    word = tuple(word)
    if vocab:
        word = check_vocab_and_split(word, bpe_codes_reverse, vocab, separator)

    cache[orig] = word
    return word

def recursive_split(segment, bpe_codes, vocab, separator, final=False):
    """Recursively split segment into smaller units (by reversing BPE merges)
    until all units are either in-vocabulary, or cannot be split futher."""

    try:
        if final:
            left, right = bpe_codes[segment + '</w>']
            right = right[:-4]
        else:
            left, right = bpe_codes[segment]
    except:
        #sys.stderr.write('cannot split {0} further.\n'.format(segment))
        yield segment
        return

    if left + separator in vocab:
        yield left
    else:
        for item in recursive_split(left, bpe_codes, vocab, separator, False):
            yield item

    if (final and right in vocab) or (not final and right + separator in vocab):
        yield right
    else:
        for item in recursive_split(right, bpe_codes, vocab, separator, final):
            yield item

def check_vocab_and_split(orig, bpe_codes, vocab, separator):
    """Check for each segment in word if it is in-vocabulary,
    and segment OOV segments into smaller units by reversing the BPE merge operations"""

    out = []

    for segment in orig[:-1]:
        if segment + separator in vocab:
            out.append(segment)
        else:
            #sys.stderr.write('OOV: {0}\n'.format(segment))
            for item in recursive_split(segment, bpe_codes, vocab, separator, False):
                out.append(item)

    segment = orig[-1]
    if segment in vocab:
        out.append(segment)
    else:
        #sys.stderr.write('OOV: {0}\n'.format(segment))
        for item in recursive_split(segment, bpe_codes, vocab, separator, True):
            out.append(item)

    return out

def read_vocabulary(vocab_file, threshold):
    """read vocabulary file produced by get_vocab.py, and filter according to frequency threshold.
    """

    vocabulary = set()

    for line in vocab_file:
        word, freq = line.strip('\r\n ').split(' ')
        freq = int(freq)
        if threshold == None or freq >= threshold:
            vocabulary.add(word)

    return vocabulary

def isolate_glossary(word, glossary):
    """
    Isolate a glossary present inside a word.

    Returns a list of subwords. In which all 'glossary' glossaries are isolated

    For example, if 'USA' is the glossary and '1934USABUSA' the word, the return value is:
        ['1934', 'USA', 'B', 'USA']
    """
    # regex equivalent of (if word == glossary or glossary not in word)
    if re.match('^'+glossary+'$', word) or not re.search(glossary, word):
        return [word]
    else:
        segments = re.split(r'({})'.format(glossary), word)
        segments, ending = segments[:-1], segments[-1]
        segments = list(filter(None, segments)) # Remove empty strings in regex group.
        return segments + [ending.strip('\r\n ')] if ending != '' else segments

if __name__ == '__main__':

    currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
    newdir = os.path.join(currentdir, 'subword_nmt')
    if os.path.isdir(newdir):
        warnings.simplefilter('default')
        warnings.warn(
            "this script's location has moved to {0}. This symbolic link will be removed in a future version. Please point to the new location, or install the package and use the command 'subword-nmt'".format(newdir),
            DeprecationWarning
        )

    # python 2/3 compatibility
    if sys.version_info < (3, 0):
        sys.stderr = codecs.getwriter('UTF-8')(sys.stderr)
        sys.stdout = codecs.getwriter('UTF-8')(sys.stdout)
        sys.stdin = codecs.getreader('UTF-8')(sys.stdin)
    else:
        sys.stdin = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8')
        sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
        sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', write_through=True, line_buffering=True)

    parser = create_parser()
    args = parser.parse_args()

    # read/write files as UTF-8
    args.codes = codecs.open(args.codes.name, encoding='utf-8')
    if args.input.name != '<stdin>':
        args.input = codecs.open(args.input.name, encoding='utf-8')
    if args.output.name != '<stdout>':
        args.output = codecs.open(args.output.name, 'w', encoding='utf-8')
    if args.vocabulary:
        args.vocabulary = codecs.open(args.vocabulary.name, encoding='utf-8')

    if args.vocabulary:
        vocabulary = read_vocabulary(args.vocabulary, args.vocabulary_threshold)
    else:
        vocabulary = None

    if sys.version_info < (3, 0):
        args.separator = args.separator.decode('UTF-8')
        if args.glossaries:
            args.glossaries = [g.decode('UTF-8') for g in args.glossaries]

    if args.seed is not None:
        random.seed(args.seed)

    bpe = BPE(args.codes, args.merges, args.separator, vocabulary, args.glossaries)

    for line in args.input:
        args.output.write(bpe.process_line(line, args.dropout))
<|endoftext|>


<commit_before>"""
The MIT License (MIT)

Copyright (c) 2015-2016 Olof Kindgren <olof.kindgren@gmail.com>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

import yaml
import xml.etree.ElementTree as ET

from . import ipxact_yaml
import sys
if sys.version_info[0] > 2:
    UNICODE = 'unicode'
else:
    UNICODE = 'utf-8'

class IpxactInt(int):
    def __new__(cls, *args, **kwargs):
        if not args:
            return super(IpxactInt, cls).__new__(cls)

        expr = args[0]
        base = 10

        if len(expr) > 2 and expr[0:2] == '0x':
            # handle non-standard SystemVerilog (but commonly-used) syntax
            expr = expr[2:]
            base = 16
        elif "'" in expr:
            sep = expr.find("'")
            # ignore any bit size specified before ' because size is handled by other IP-XACT properties
            if expr[sep+1] in ["h", "H"]:
                base = 16
            elif expr[sep+1] in ["d", "D"]:
                base = 10
            elif expr[sep+1] in ["o", "O"]:
                base = 8
            elif expr[sep+1] in ["b", "B"]:
                base = 2
            else:
                raise ValueError("Could not convert expression to an integer: {}".format(args[0]))
            expr = expr[sep+2:]

        return super(IpxactInt, cls).__new__(cls, expr, base)

class IpxactBool(str):
    def __new__(cls, *args, **kwargs):
        if not args:
            return None
        elif args[0] in ['true', 'false']:
            return super(IpxactBool, cls).__new__(cls, args[0])
        else:
            raise Exception

class IpxactItem(object):
    nsmap = {'1.4'  : ('spirit' , 'http://www.spiritconsortium.org/XMLSchema/SPIRIT/1.4'),
             '1.5'  : ('spirit' , 'http://www.spiritconsortium.org/XMLSchema/SPIRIT/1.5'),
             '2009' : ('spirit' , 'http://www.spiritconsortium.org/XMLSchema/SPIRIT/1685-2009'),
             '2014' : ('ipxact' , 'http://www.accellera.org/XMLSchema/IPXACT/1685-2014'),
    }
    nsversion = '1.5'

    ATTRIBS = {}
    MEMBERS = {}
    CHILDREN = []
    CHILD = []
    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            if k in self.MEMBERS:
                setattr(self, k, v)
            elif k in self.ATTRIBS:
                setattr(self, k, v)
            else:
                s = "{} has no member or attribute '{}'"
                raise AttributeError(s.format(self.__class__.__name__, k))

    def load(self, f):
        tree = ET.parse(f)
        root = tree.getroot()

        #Warning: Semi-horrible hack to find out which IP-Xact version that is used
        for key, value in self.nsmap.items():
            if root.tag[1:].startswith(value[1]):
                self.nsversion = key

        S = '{%s}' % self.nsmap[self.nsversion][1]
        if not (root.tag == S+self._tag):
            raise Exception

        self.parse_tree(root, self.nsmap[self.nsversion])

    def parse_tree(self, root, ns):
        if self.ATTRIBS:
            for _name, _type in self.ATTRIBS.items():
                _tagname = '{' + ns[1] + '}' + _name
                if _tagname in root.attrib:
                    setattr(self, _name, eval(_type)(root.attrib[_tagname]))
        for _name, _type in self.MEMBERS.items():
            tmp = root.find('./{}:{}'.format(ns[0], _name), {ns[0] : ns[1]})
            if tmp is not None and tmp.text is not None:
                setattr(self, _name, eval(_type)(tmp.text))
            else:
                setattr(self, _name, eval(_type)())

        for c in self.CHILDREN:
            for f in root.findall("./{}:{}".format(ns[0], c), {ns[0] : ns[1]}):
                child = getattr(self, c)[:]
                class_name = c[0].upper() + c[1:]
                t = __import__(self.__module__)
                t = getattr(t, 'ipyxact')
                t = getattr(t, class_name)()
                #t = eval(class_name)()
                t.parse_tree(f, ns)
                child.append(t)
                setattr(self, c, child)
        for c in self.CHILD:
            f = root.find("./{}:{}".format(ns[0], c), {ns[0] : ns[1]})
            if f is not None:
                child = getattr(self, c)
                class_name = c[0].upper() + c[1:]
                t = eval(class_name)()
                t.parse_tree(f, ns)
                setattr(self, c, t)

    def _write(self, root, S):
        for a in self.ATTRIBS:
            root.attrib[S+a] = getattr(self, a)

        for m in self.MEMBERS:
            tmp = getattr(self, m)
            if tmp is not None:
                ET.SubElement(root, S+m).text = str(tmp)

        for c in self.CHILDREN:
            for child_obj in getattr(self, c):
                subel = ET.SubElement(root, S+c)
                child_obj._write(subel, S)
        for c in self.CHILD:
            tmp = getattr(self, c)
            if tmp is not None:
                subel = ET.SubElement(root, S+c)
                tmp._write(subel, S)

    def write(self, f):
        ET.register_namespace(self.nsmap[self.nsversion][0], self.nsmap[self.nsversion][1])
        S = '{%s}' % self.nsmap[self.nsversion][1]
        root = ET.Element(S+self._tag)
        self._write(root, S)

        et = ET.ElementTree(root)
        et.write(f, xml_declaration=True, encoding=UNICODE)

def _generate_classes(j):
    for tag, _items in j.items():
        if 'ATTRIBS' in _items:
            for key, value in _items['ATTRIBS'].items():
                _items.update({key : eval(value)})
        if 'MEMBERS' in _items:
            for key, value in _items['MEMBERS'].items():
                _items.update({key : None})
        if 'CHILDREN' in _items:
            for key in _items['CHILDREN']:
                _items.update({key : []})
        if 'CHILD' in _items:
            for key in _items['CHILD']:
                _items.update({key : None})
        _items.update({'_tag' : tag})

        generatedClass = type(tag[0].upper()+tag[1:], (IpxactItem,), _items)
        globals()[generatedClass.__name__] = generatedClass

_generate_classes(yaml.load(ipxact_yaml.description))
<commit_msg>Support integer expressions containing underscore<commit_after>"""
The MIT License (MIT)

Copyright (c) 2015-2016 Olof Kindgren <olof.kindgren@gmail.com>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

import yaml
import xml.etree.ElementTree as ET

from . import ipxact_yaml
import sys
if sys.version_info[0] > 2:
    UNICODE = 'unicode'
else:
    UNICODE = 'utf-8'

class IpxactInt(int):
    def __new__(cls, *args, **kwargs):
        if not args:
            return super(IpxactInt, cls).__new__(cls)

        expr = args[0]
        base = 10

        if len(expr) > 2 and expr[0:2] == '0x':
            # handle non-standard SystemVerilog (but commonly-used) syntax
            expr = expr[2:]
            base = 16
        elif "'" in expr:
            sep = expr.find("'")
            # ignore any bit size specified before ' because size is handled by other IP-XACT properties
            if expr[sep+1] in ["h", "H"]:
                base = 16
            elif expr[sep+1] in ["d", "D"]:
                base = 10
            elif expr[sep+1] in ["o", "O"]:
                base = 8
            elif expr[sep+1] in ["b", "B"]:
                base = 2
            else:
                raise ValueError("Could not convert expression to an integer: {}".format(args[0]))
            expr = expr[sep+2:]

        return super(IpxactInt, cls).__new__(cls, expr.replace('_', ''), base)

class IpxactBool(str):
    def __new__(cls, *args, **kwargs):
        if not args:
            return None
        elif args[0] in ['true', 'false']:
            return super(IpxactBool, cls).__new__(cls, args[0])
        else:
            raise Exception

class IpxactItem(object):
    nsmap = {'1.4'  : ('spirit' , 'http://www.spiritconsortium.org/XMLSchema/SPIRIT/1.4'),
             '1.5'  : ('spirit' , 'http://www.spiritconsortium.org/XMLSchema/SPIRIT/1.5'),
             '2009' : ('spirit' , 'http://www.spiritconsortium.org/XMLSchema/SPIRIT/1685-2009'),
             '2014' : ('ipxact' , 'http://www.accellera.org/XMLSchema/IPXACT/1685-2014'),
    }
    nsversion = '1.5'

    ATTRIBS = {}
    MEMBERS = {}
    CHILDREN = []
    CHILD = []
    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            if k in self.MEMBERS:
                setattr(self, k, v)
            elif k in self.ATTRIBS:
                setattr(self, k, v)
            else:
                s = "{} has no member or attribute '{}'"
                raise AttributeError(s.format(self.__class__.__name__, k))

    def load(self, f):
        tree = ET.parse(f)
        root = tree.getroot()

        #Warning: Semi-horrible hack to find out which IP-Xact version that is used
        for key, value in self.nsmap.items():
            if root.tag[1:].startswith(value[1]):
                self.nsversion = key

        S = '{%s}' % self.nsmap[self.nsversion][1]
        if not (root.tag == S+self._tag):
            raise Exception

        self.parse_tree(root, self.nsmap[self.nsversion])

    def parse_tree(self, root, ns):
        if self.ATTRIBS:
            for _name, _type in self.ATTRIBS.items():
                _tagname = '{' + ns[1] + '}' + _name
                if _tagname in root.attrib:
                    setattr(self, _name, eval(_type)(root.attrib[_tagname]))
        for _name, _type in self.MEMBERS.items():
            tmp = root.find('./{}:{}'.format(ns[0], _name), {ns[0] : ns[1]})
            if tmp is not None and tmp.text is not None:
                setattr(self, _name, eval(_type)(tmp.text))
            else:
                setattr(self, _name, eval(_type)())

        for c in self.CHILDREN:
            for f in root.findall("./{}:{}".format(ns[0], c), {ns[0] : ns[1]}):
                child = getattr(self, c)[:]
                class_name = c[0].upper() + c[1:]
                t = __import__(self.__module__)
                t = getattr(t, 'ipyxact')
                t = getattr(t, class_name)()
                #t = eval(class_name)()
                t.parse_tree(f, ns)
                child.append(t)
                setattr(self, c, child)
        for c in self.CHILD:
            f = root.find("./{}:{}".format(ns[0], c), {ns[0] : ns[1]})
            if f is not None:
                child = getattr(self, c)
                class_name = c[0].upper() + c[1:]
                t = eval(class_name)()
                t.parse_tree(f, ns)
                setattr(self, c, t)

    def _write(self, root, S):
        for a in self.ATTRIBS:
            root.attrib[S+a] = getattr(self, a)

        for m in self.MEMBERS:
            tmp = getattr(self, m)
            if tmp is not None:
                ET.SubElement(root, S+m).text = str(tmp)

        for c in self.CHILDREN:
            for child_obj in getattr(self, c):
                subel = ET.SubElement(root, S+c)
                child_obj._write(subel, S)
        for c in self.CHILD:
            tmp = getattr(self, c)
            if tmp is not None:
                subel = ET.SubElement(root, S+c)
                tmp._write(subel, S)

    def write(self, f):
        ET.register_namespace(self.nsmap[self.nsversion][0], self.nsmap[self.nsversion][1])
        S = '{%s}' % self.nsmap[self.nsversion][1]
        root = ET.Element(S+self._tag)
        self._write(root, S)

        et = ET.ElementTree(root)
        et.write(f, xml_declaration=True, encoding=UNICODE)

def _generate_classes(j):
    for tag, _items in j.items():
        if 'ATTRIBS' in _items:
            for key, value in _items['ATTRIBS'].items():
                _items.update({key : eval(value)})
        if 'MEMBERS' in _items:
            for key, value in _items['MEMBERS'].items():
                _items.update({key : None})
        if 'CHILDREN' in _items:
            for key in _items['CHILDREN']:
                _items.update({key : []})
        if 'CHILD' in _items:
            for key in _items['CHILD']:
                _items.update({key : None})
        _items.update({'_tag' : tag})

        generatedClass = type(tag[0].upper()+tag[1:], (IpxactItem,), _items)
        globals()[generatedClass.__name__] = generatedClass

_generate_classes(yaml.load(ipxact_yaml.description))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from collections import OrderedDict

raw_information = OrderedDict()
raw_information['Kousaka Honoka'] = {
    'year': 'Second',
    'main_unit': 'μ\'s',
    'sub_unit': 'Printemps',
    'cv_nickname': 'Emitsun',
    'cv_twitter': 'Nitta_Emi',
    'cv_instagram': 'Nitta_Emi',
    'image': 'http://i.schoolido.lu/cards/transparent/28Transparent.png',
    'color': '#f6ab29',
}
raw_information['Nishikino Maki'] = {
    'year': 'First',
    'main_unit': 'μ\'s',
    'sub_unit': 'Bibi',
    'cv_nickname': 'Pile',
    'cv_twitter': 'pile_eric',
    'cv_instagram': 'pile_eric',
    'image': 'http://i.schoolido.lu/cards/transparent/33Transparent.png',
    'color': '#ee7b78',
}
raw_information['Ayase Eli'] = {
    'year': 'Third',
    'main_unit': 'μ\'s',
    'sub_unit': 'Bibi',
    'cv_nickname': 'Nanjolno',
    'cv_twitter': 'nanjolno',
    'cv_instagram': 'nanjolno',
    'image': 'http://i.schoolido.lu/cards/transparent/29Transparent.png',
    'color': '#b1daef',
}
raw_information['Minami Kotori'] = {
    'year': 'Second',
    'main_unit': 'μ\'s',
    'sub_unit': 'Printemps',
    'cv_nickname': 'Ucchi',
    'cv_twitter': 'aya_uchida',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/30Transparent.png',
    'color': '#c2b9bc',
}
raw_information['Hoshizora Rin'] = {
    'year': 'First',
    'main_unit': 'μ\'s',
    'sub_unit': 'Lily White',
    'cv_nickname': 'Rippi',
    'cv_twitter': 'rippialoha',
    'cv_instagram': 'rippialoha',
    'image': 'http://i.schoolido.lu/cards/transparent/32Transparent.png',
    'color': '#98d3cc',
}
raw_information['Koizumi Hanayo'] = {
    'year': 'First',
    'main_unit': 'μ\'s',
    'sub_unit': 'Printemps',
    'cv_nickname': 'Shikaco',
    'cv_twitter': 'shi_ka_co',
    'cv_instagram': 'shi_ka_co',
    'image': 'http://i.schoolido.lu/cards/transparent/35Transparent.png',
    'color': '#89c370',
}
raw_information['Sonoda Umi'] = {
    'year': 'Second',
    'main_unit': 'μ\'s',
    'sub_unit': 'Lily White',
    'cv_nickname': 'Mimorin',
    'cv_twitter': 'mimori_suzuko',
    'cv_instagram': 'mimori_suzuko',
    'image': 'http://i.schoolido.lu/cards/transparent/31Transparent.png',
    'color': '#288bcc',
}
raw_information['Toujou Nozomi'] = {
    'year': 'Third',
    'main_unit': 'μ\'s',
    'sub_unit': 'Lily White',
    'cv_nickname': 'Kussun',
    'cv_twitter': 'kusudaaina',
    'cv_instagram': 'kusudaaina',
    'image': 'http://i.schoolido.lu/cards/transparent/34Transparent.png',
    'color': '#8c70ad',
}
raw_information['Yazawa Nico'] = {
    'year': 'Third',
    'main_unit': 'μ\'s',
    'sub_unit': 'Bibi',
    'cv_nickname': 'Soramaru',
    'cv_twitter': 'tokui_sorangley',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/36Transparent.png',
    'color': '#f1a6c7',
}
raw_information['Takami Chika'] = {
    'year': 'Second',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Inami Anju',
    'cv_twitter': 'anju_inami',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/786Transparent.png',
    'color': '#ff9646',
    'chibi': [
        ('http://i.schoolido.lu/chibi/B2A4v7nh.png', 'umidah'),
    ],
}
raw_information['Sakurauchi Riko'] = {
    'year': 'Second',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Aida Rikako',
    'cv_twitter': 'Rikako_Aida',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/787Transparent.png',
    'color': '#ff7777',
    'chibi': [
        ('http://i.schoolido.lu/chibi/d2XP46rt.png', 'umidah'),
    ],
}
raw_information['Matsuura Kanan'] = {
    'year': 'Third',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Suwa Nanaka',
    'cv_twitter': 'suwananaka',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/788Transparent.png',
    'color': '#16d5a6',
}
raw_information['Kurosawa Dia'] = {
    'year': 'Third',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Komiya Arisa',
    'cv_twitter': 'arisakomiya',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/789Transparent.png',
    'color': '#ff4242',
}
raw_information['Watanabe You'] = {
    'year': 'Second',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Saitou Shuka',
    'cv_twitter': 'Saito_Shuka',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/790Transparent.png',
    'color': '#49bbee',
    'chibi': [
        ('http://i.schoolido.lu/chibi/iriel.png', 'iriel'),
        ('http://i.schoolido.lu/chibi/Mochaqt.png', 'Mochaqt'),
        ('http://i.schoolido.lu/chibi/sanjista.png', 'sanjista'),
    ],
}
raw_information['Tsushima Yoshiko'] = {
    'year': 'First',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Aikakyan',
    'cv_twitter': 'Aikyan_',
    'cv_instagram': 'aikyan_',
    'image': 'http://i.schoolido.lu/cards/transparent/791Transparent.png',
    'color': '#c3c3c3',
    'chibi': [
        ('http://i.schoolido.lu/chibi/p6Bb6Wyb.png', 'Erana'),
        ('http://i.schoolido.lu/chibi/yohanechibi.png', 'ankira'),
    ],
}
raw_information['Kunikida Hanamaru'] = {
    'year': 'First',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Takatsuki Kanako',
    'cv_twitter': 'Kanako_tktk',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/792Transparent.png',
    'color': '#e7cf0b',
    'chibi': [
        ('http://i.schoolido.lu/chibi/dTTgaSi.png', 'Eine'),
    ],
}
raw_information['Ohara Mari'] = {
    'year': 'Third',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Suzuki Aina',
    'cv_twitter': 'aina_suzuki723',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/793Transparent.png',
    'color': '#c450ff',
    'chibi': [
        ('http://i.schoolido.lu/chibi/sorutan.png', 'sorutan'),
    ],
}
raw_information['Kurosawa Ruby'] = {
    'year': 'First',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Furihata Ai',
    'cv_twitter': 'furihata_ai',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/794Transparent.png',
    'color': '#f273c4',
    'chibi': [
        ('http://i.schoolido.lu/chibi/amukagamine.png', 'amukagamine'),
    ],
}

raw_information_n = {
    'Saeki Reine': {
        'chibi': [
            ('http://i.schoolido.lu/chibi/Arne.png', 'Arne'),
        ],
    },
    'Aizawa Yuu': {
        'chibi': [
            ('http://i.schoolido.lu/chibi/blueberrytrain.png', 'blueberrytrain'),
        ],
    },
    'Shirase Koyuki': {
        'chibi': [
            ('http://i.schoolido.lu/chibi/Dokisumi.png', 'Dokisumi'),
        ],
    },
}

app_data = {
    'android': {
        'version': '1.0',
        'download_link': 'http://i.schoolido.lu/android/sukutomo1_0.apk',
        'sources': 'https://github.com/SchoolIdolTomodachi/Sukutomo-Android',
        'website_url': 'http://schoolido.lu/android/',
        'icon': 'http://i.schoolido.lu/android/icon.png',
        'author': {
            'username': 'Arukantara',
            'website_url': 'http://schoolido.lu/user/Arukantara/',
            'contact': 'arukantara@schoolido.lu',
        },
    },
}
<commit_msg>Community cute chibi drawings ✨<commit_after># -*- coding: utf-8 -*-
from collections import OrderedDict

raw_information = OrderedDict()
raw_information['Kousaka Honoka'] = {
    'year': 'Second',
    'main_unit': 'μ\'s',
    'sub_unit': 'Printemps',
    'cv_nickname': 'Emitsun',
    'cv_twitter': 'Nitta_Emi',
    'cv_instagram': 'Nitta_Emi',
    'image': 'http://i.schoolido.lu/cards/transparent/28Transparent.png',
    'color': '#f6ab29',
}
raw_information['Nishikino Maki'] = {
    'year': 'First',
    'main_unit': 'μ\'s',
    'sub_unit': 'Bibi',
    'cv_nickname': 'Pile',
    'cv_twitter': 'pile_eric',
    'cv_instagram': 'pile_eric',
    'image': 'http://i.schoolido.lu/cards/transparent/33Transparent.png',
    'color': '#ee7b78',
}
raw_information['Ayase Eli'] = {
    'year': 'Third',
    'main_unit': 'μ\'s',
    'sub_unit': 'Bibi',
    'cv_nickname': 'Nanjolno',
    'cv_twitter': 'nanjolno',
    'cv_instagram': 'nanjolno',
    'image': 'http://i.schoolido.lu/cards/transparent/29Transparent.png',
    'color': '#b1daef',
}
raw_information['Minami Kotori'] = {
    'year': 'Second',
    'main_unit': 'μ\'s',
    'sub_unit': 'Printemps',
    'cv_nickname': 'Ucchi',
    'cv_twitter': 'aya_uchida',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/30Transparent.png',
    'color': '#c2b9bc',
}
raw_information['Hoshizora Rin'] = {
    'year': 'First',
    'main_unit': 'μ\'s',
    'sub_unit': 'Lily White',
    'cv_nickname': 'Rippi',
    'cv_twitter': 'rippialoha',
    'cv_instagram': 'rippialoha',
    'image': 'http://i.schoolido.lu/cards/transparent/32Transparent.png',
    'color': '#98d3cc',
}
raw_information['Koizumi Hanayo'] = {
    'year': 'First',
    'main_unit': 'μ\'s',
    'sub_unit': 'Printemps',
    'cv_nickname': 'Shikaco',
    'cv_twitter': 'shi_ka_co',
    'cv_instagram': 'shi_ka_co',
    'image': 'http://i.schoolido.lu/cards/transparent/35Transparent.png',
    'color': '#89c370',
}
raw_information['Sonoda Umi'] = {
    'year': 'Second',
    'main_unit': 'μ\'s',
    'sub_unit': 'Lily White',
    'cv_nickname': 'Mimorin',
    'cv_twitter': 'mimori_suzuko',
    'cv_instagram': 'mimori_suzuko',
    'image': 'http://i.schoolido.lu/cards/transparent/31Transparent.png',
    'color': '#288bcc',
}
raw_information['Toujou Nozomi'] = {
    'year': 'Third',
    'main_unit': 'μ\'s',
    'sub_unit': 'Lily White',
    'cv_nickname': 'Kussun',
    'cv_twitter': 'kusudaaina',
    'cv_instagram': 'kusudaaina',
    'image': 'http://i.schoolido.lu/cards/transparent/34Transparent.png',
    'color': '#8c70ad',
}
raw_information['Yazawa Nico'] = {
    'year': 'Third',
    'main_unit': 'μ\'s',
    'sub_unit': 'Bibi',
    'cv_nickname': 'Soramaru',
    'cv_twitter': 'tokui_sorangley',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/36Transparent.png',
    'color': '#f1a6c7',
}
raw_information['Takami Chika'] = {
    'year': 'Second',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Inami Anju',
    'cv_twitter': 'anju_inami',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/786Transparent.png',
    'color': '#ff9646',
    'chibi': [
        ('http://i.schoolido.lu/chibi/B2A4v7nh.png', 'umidah'),
    ],
}
raw_information['Sakurauchi Riko'] = {
    'year': 'Second',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Aida Rikako',
    'cv_twitter': 'Rikako_Aida',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/787Transparent.png',
    'color': '#ff7777',
    'chibi': [
        ('http://i.schoolido.lu/chibi/d2XP46rt.png', 'umidah'),
    ],
}
raw_information['Matsuura Kanan'] = {
    'year': 'Third',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Suwa Nanaka',
    'cv_twitter': 'suwananaka',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/788Transparent.png',
    'color': '#16d5a6',
    'chibi': [
        ('http://i.schoolido.lu/chibi/kirisakisun.png', 'kirisakisun'),
    ],
}
raw_information['Kurosawa Dia'] = {
    'year': 'Third',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Komiya Arisa',
    'cv_twitter': 'arisakomiya',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/789Transparent.png',
    'color': '#ff4242',
}
raw_information['Watanabe You'] = {
    'year': 'Second',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Saitou Shuka',
    'cv_twitter': 'Saito_Shuka',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/790Transparent.png',
    'color': '#49bbee',
    'chibi': [
        ('http://i.schoolido.lu/chibi/iriel.png', 'iriel'),
        ('http://i.schoolido.lu/chibi/Mochaqt.png', 'Mochaqt'),
        ('http://i.schoolido.lu/chibi/sanjista.png', 'sanjista'),
    ],
}
raw_information['Tsushima Yoshiko'] = {
    'year': 'First',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Aikakyan',
    'cv_twitter': 'Aikyan_',
    'cv_instagram': 'aikyan_',
    'image': 'http://i.schoolido.lu/cards/transparent/791Transparent.png',
    'color': '#c3c3c3',
    'chibi': [
        ('http://i.schoolido.lu/chibi/p6Bb6Wyb.png', 'Erana'),
        ('http://i.schoolido.lu/chibi/yohanechibi.png', 'ankira'),
    ],
}
raw_information['Kunikida Hanamaru'] = {
    'year': 'First',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Takatsuki Kanako',
    'cv_twitter': 'Kanako_tktk',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/792Transparent.png',
    'color': '#e7cf0b',
    'chibi': [
        ('http://i.schoolido.lu/chibi/dTTgaSi.png', 'Eine'),
    ],
}
raw_information['Ohara Mari'] = {
    'year': 'Third',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Suzuki Aina',
    'cv_twitter': 'aina_suzuki723',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/793Transparent.png',
    'color': '#c450ff',
    'chibi': [
        ('http://i.schoolido.lu/chibi/sorutan.png', 'sorutan'),
    ],
}
raw_information['Kurosawa Ruby'] = {
    'year': 'First',
    'main_unit': 'Aqours',
    'sub_unit': 'Aqours',
    'cv_nickname': 'Furihata Ai',
    'cv_twitter': 'furihata_ai',
    'cv_instagram': None,
    'image': 'http://i.schoolido.lu/cards/transparent/794Transparent.png',
    'color': '#f273c4',
    'chibi': [
        ('http://i.schoolido.lu/chibi/amukagamine.png', 'amukagamine'),
    ],
}

raw_information_n = {
    'Saeki Reine': {
        'chibi': [
            ('http://i.schoolido.lu/chibi/Arne.png', 'Arne'),
        ],
    },
    'Aizawa Yuu': {
        'chibi': [
            ('http://i.schoolido.lu/chibi/blueberrytrain.png', 'blueberrytrain'),
        ],
    },
    'Shirase Koyuki': {
        'chibi': [
            ('http://i.schoolido.lu/chibi/Dokisumi.png', 'Dokisumi'),
        ],
    },
}

app_data = {
    'android': {
        'version': '1.0',
        'download_link': 'http://i.schoolido.lu/android/sukutomo1_0.apk',
        'sources': 'https://github.com/SchoolIdolTomodachi/Sukutomo-Android',
        'website_url': 'http://schoolido.lu/android/',
        'icon': 'http://i.schoolido.lu/android/icon.png',
        'author': {
            'username': 'Arukantara',
            'website_url': 'http://schoolido.lu/user/Arukantara/',
            'contact': 'arukantara@schoolido.lu',
        },
    },
}
<|endoftext|>


<commit_before>import logging
from typing import Tuple, List, Sequence

import sqlite3
from twisted.internet import defer
from twisted.enterprise import adbapi

from torba.hash import TXRefImmutable
from torba.basetransaction import BaseTransaction
from torba.baseaccount import BaseAccount

log = logging.getLogger(__name__)

def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
    sql, values = [], {}
    for key, constraint in constraints.items():
        col, op, key = key, '=', key.replace('.', '_')
        if key.startswith('$'):
            values[key] = constraint
            continue
        elif key.endswith('__not'):
            col, op = col[:-len('__not')], '!='
        elif key.endswith('__lt'):
            col, op = col[:-len('__lt')], '<'
        elif key.endswith('__lte'):
            col, op = col[:-len('__lte')], '<='
        elif key.endswith('__gt'):
            col, op = col[:-len('__gt')], '>'
        elif key.endswith('__like'):
            col, op = col[:-len('__like')], 'LIKE'
        elif key.endswith('__in') or key.endswith('__not_in'):
            if key.endswith('__in'):
                col, op = col[:-len('__in')], 'IN'
            else:
                col, op = col[:-len('__not_in')], 'NOT IN'
            if isinstance(constraint, (list, set)):
                items = ', '.join(
                    "'{}'".format(item) if isinstance(item, str) else str(item)
                    for item in constraint
                )
            elif isinstance(constraint, str):
                items = constraint
            else:
                raise ValueError("{} requires a list, set or string as constraint value.".format(col))
            sql.append('{} {} ({})'.format(col, op, items))
            continue
        elif key.endswith('__any'):
            where, subvalues = constraints_to_sql(constraint, ' OR ', key+'_')
            sql.append('({})'.format(where))
            values.update(subvalues)
            continue
        sql.append('{} {} :{}'.format(col, op, prepend_key+key))
        values[prepend_key+key] = constraint
    return joiner.join(sql) if sql else '', values

def query(select, **constraints):
    sql = [select]
    limit = constraints.pop('limit', None)
    offset = constraints.pop('offset', None)
    order_by = constraints.pop('order_by', None)

    constraints.pop('my_account', None)
    account = constraints.pop('account', None)
    if account is not None:
        if not isinstance(account, list):
            account = [account]
        constraints['account__in'] = [
            (a.public_key.address if isinstance(a, BaseAccount) else a) for a in account
        ]

    where, values = constraints_to_sql(constraints)
    if where:
        sql.append('WHERE')
        sql.append(where)

    if order_by is not None:
        sql.append('ORDER BY')
        if isinstance(order_by, str):
            sql.append(order_by)
        elif isinstance(order_by, list):
            sql.append(', '.join(order_by))
        else:
            raise ValueError("order_by must be string or list")

    if offset is not None:
        sql.append('OFFSET {}'.format(offset))

    if limit is not None:
        sql.append('LIMIT {}'.format(limit))

    return ' '.join(sql), values

def rows_to_dict(rows, fields):
    if rows:
        return [dict(zip(fields, r)) for r in rows]
    else:
        return []

def row_dict_or_default(rows, fields, default=None):
    dicts = rows_to_dict(rows, fields)
    return dicts[0] if dicts else default

class SQLiteMixin:

    CREATE_TABLES_QUERY: Sequence[str] = ()

    def __init__(self, path):
        self._db_path = path
        self.db: adbapi.ConnectionPool = None
        self.ledger = None

    def open(self):
        log.info("connecting to database: %s", self._db_path)
        self.db = adbapi.ConnectionPool(
            'sqlite3', self._db_path, cp_min=1, cp_max=1, check_same_thread=False
        )
        return self.db.runInteraction(
            lambda t: t.executescript(self.CREATE_TABLES_QUERY)
        )

    def close(self):
        self.db.close()
        return defer.succeed(True)

    @staticmethod
    def _insert_sql(table: str, data: dict) -> Tuple[str, List]:
        columns, values = [], []
        for column, value in data.items():
            columns.append(column)
            values.append(value)
        sql = "INSERT INTO {} ({}) VALUES ({})".format(
            table, ', '.join(columns), ', '.join(['?'] * len(values))
        )
        return sql, values

    @staticmethod
    def _update_sql(table: str, data: dict, where: str, constraints: list) -> Tuple[str, list]:
        columns, values = [], []
        for column, value in data.items():
            columns.append("{} = ?".format(column))
            values.append(value)
        values.extend(constraints)
        sql = "UPDATE {} SET {} WHERE {}".format(
            table, ', '.join(columns), where
        )
        return sql, values

    @staticmethod
    def execute(t, sql, values):
        log.debug(sql)
        log.debug(values)
        return t.execute(sql, values)

    def run_operation(self, sql, values):
        log.debug(sql)
        log.debug(values)
        return self.db.runOperation(sql, values)

    def run_query(self, sql, values):
        log.debug(sql)
        log.debug(values)
        return self.db.runQuery(sql, values)

class BaseDatabase(SQLiteMixin):

    CREATE_PUBKEY_ADDRESS_TABLE = """
        create table if not exists pubkey_address (
            address text primary key,
            account text not null,
            chain integer not null,
            position integer not null,
            pubkey blob not null,
            history text,
            used_times integer not null default 0
        );
    """

    CREATE_TX_TABLE = """
        create table if not exists tx (
            txid text primary key,
            raw blob not null,
            height integer not null,
            position integer not null,
            is_verified boolean not null default 0
        );
    """

    CREATE_TXO_TABLE = """
        create table if not exists txo (
            txid text references tx,
            txoid text primary key,
            address text references pubkey_address,
            position integer not null,
            amount integer not null,
            script blob not null,
            is_reserved boolean not null default 0
        );
    """

    CREATE_TXI_TABLE = """
        create table if not exists txi (
            txid text references tx,
            txoid text references txo,
            address text references pubkey_address
        );
    """

    CREATE_TABLES_QUERY = (
        CREATE_TX_TABLE +
        CREATE_PUBKEY_ADDRESS_TABLE +
        CREATE_TXO_TABLE +
        CREATE_TXI_TABLE
    )

    @staticmethod
    def txo_to_row(tx, address, txo):
        return {
            'txid': tx.id,
            'txoid': txo.id,
            'address': address,
            'position': txo.position,
            'amount': txo.amount,
            'script': sqlite3.Binary(txo.script.source)
        }

    def save_transaction_io(self, save_tx, tx: BaseTransaction, address, txhash, history):

        def _steps(t):
            if save_tx == 'insert':
                self.execute(t, *self._insert_sql('tx', {
                    'txid': tx.id,
                    'raw': sqlite3.Binary(tx.raw),
                    'height': tx.height,
                    'position': tx.position,
                    'is_verified': tx.is_verified
                }))
            elif save_tx == 'update':
                self.execute(t, *self._update_sql("tx", {
                    'height': tx.height, 'position': tx.position, 'is_verified': tx.is_verified
                }, 'txid = ?', (tx.id,)))

            existing_txos = [r[0] for r in self.execute(t, *query(
                "SELECT position FROM txo", txid=tx.id
            )).fetchall()]

            for txo in tx.outputs:
                if txo.position in existing_txos:
                    continue
                if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == txhash:
                    self.execute(t, *self._insert_sql("txo", self.txo_to_row(tx, address, txo)))
                elif txo.script.is_pay_script_hash:
                    # TODO: implement script hash payments
                    print('Database.save_transaction_io: pay script hash is not implemented!')

            # lookup the address associated with each TXI (via its TXO)
            txoid_to_address = {r[0]: r[1] for r in self.execute(t, *query(
                "SELECT txoid, address FROM txo", txoid__in=[txi.txo_ref.id for txi in tx.inputs]
            )).fetchall()}

            # list of TXIs that have already been added
            existing_txis = [r[0] for r in self.execute(t, *query(
                "SELECT txoid FROM txi", txid=tx.id
            )).fetchall()]

            for txi in tx.inputs:
                txoid = txi.txo_ref.id
                new_txi = txoid not in existing_txis
                address_matches = txoid_to_address.get(txoid) == address
                if new_txi and address_matches:
                    self.execute(t, *self._insert_sql("txi", {
                        'txid': tx.id,
                        'txoid': txoid,
                        'address': address,
                    }))

            self._set_address_history(t, address, history)

        return self.db.runInteraction(_steps)

    def reserve_outputs(self, txos, is_reserved=True):
        txoids = [txo.id for txo in txos]
        return self.run_operation(
            "UPDATE txo SET is_reserved = ? WHERE txoid IN ({})".format(
                ', '.join(['?']*len(txoids))
            ), [is_reserved]+txoids
        )

    def release_outputs(self, txos):
        return self.reserve_outputs(txos, is_reserved=False)

    def rewind_blockchain(self, above_height):  # pylint: disable=no-self-use
        # TODO:
        # 1. delete transactions above_height
        # 2. update address histories removing deleted TXs
        return defer.succeed(True)

    def select_transactions(self, cols, account=None, **constraints):
        if 'txid' not in constraints and account is not None:
            constraints['$account'] = account.public_key.address
            constraints['txid__in'] = """
                SELECT txo.txid FROM txo
                JOIN pubkey_address USING (address) WHERE pubkey_address.account = :$account
              UNION
                SELECT txi.txid FROM txi
                JOIN txo USING (txoid)
                JOIN pubkey_address USING (address) WHERE pubkey_address.account = :$account
            """
        return self.run_query(*query("SELECT {} FROM tx".format(cols), **constraints))

    @defer.inlineCallbacks
    def get_transactions(self, my_account=None, **constraints):
        my_account = my_account or constraints.get('account', None)

        tx_rows = yield self.select_transactions(
            'txid, raw, height, position, is_verified',
            order_by=["height DESC", "position DESC"],
            **constraints
        )

        txids, txs = [], []
        for row in tx_rows:
            txids.append(row[0])
            txs.append(self.ledger.transaction_class(
                raw=row[1], height=row[2], position=row[3], is_verified=row[4]
            ))

        annotated_txos = {
            txo.id: txo for txo in
            (yield self.get_txos(
                my_account=my_account,
                txid__in=txids
            ))
        }

        referenced_txos = {
            txo.id: txo for txo in
            (yield self.get_txos(
                my_account=my_account,
                txoid__in=query("SELECT txoid FROM txi", **{'txid__in': txids})[0]
            ))
        }

        for tx in txs:
            for txi in tx.inputs:
                txo = referenced_txos.get(txi.txo_ref.id)
                if txo:
                    txi.txo_ref = txo.ref
            for txo in tx.outputs:
                _txo = annotated_txos.get(txo.id)
                if _txo:
                    txo.update_annotations(_txo)
                else:
                    txo.update_annotations(None)

        return txs

    @defer.inlineCallbacks
    def get_transaction_count(self, **constraints):
        count = yield self.select_transactions('count(*)', **constraints)
        return count[0][0]

    @defer.inlineCallbacks
    def get_transaction(self, **constraints):
        txs = yield self.get_transactions(limit=1, **constraints)
        if txs:
            return txs[0]

    def select_txos(self, cols, **constraints):
        return self.run_query(*query(
            "SELECT {} FROM txo JOIN pubkey_address USING (address)".format(cols), **constraints
        ))

    @defer.inlineCallbacks
    def get_txos(self, my_account=None, **constraints):
        my_account = my_account or constraints.get('account', None)
        if isinstance(my_account, BaseAccount):
            my_account = my_account.public_key.address
        rows = yield self.select_txos(
            "amount, script, txid, txo.position, chain, account", **constraints
        )
        output_class = self.ledger.transaction_class.output_class
        return [
            output_class(
                amount=row[0],
                script=output_class.script_class(row[1]),
                tx_ref=TXRefImmutable.from_id(row[2]),
                position=row[3],
                is_change=row[4] == 1,
                is_my_account=row[5] == my_account
            ) for row in rows
        ]

    @defer.inlineCallbacks
    def get_txo_count(self, **constraints):
        count = yield self.select_txos('count(*)', **constraints)
        return count[0][0]

    @staticmethod
    def constrain_utxo(constraints):
        constraints['is_reserved'] = False
        constraints['txoid__not_in'] = "SELECT txoid FROM txi"

    def get_utxos(self, **constraints):
        self.constrain_utxo(constraints)
        return self.get_txos(**constraints)

    def get_utxos_count(self, **constraints):
        self.constrain_utxo(constraints)
        return self.get_txo_count(**constraints)

    @defer.inlineCallbacks
    def get_balance(self, **constraints):
        self.constrain_utxo(constraints)
        balance = yield self.select_txos('SUM(amount)', **constraints)
        return balance[0][0] or 0

    def select_addresses(self, cols, **constraints):
        return self.run_query(*query(
            "SELECT {} FROM pubkey_address".format(cols), **constraints
        ))

    @defer.inlineCallbacks
    def get_addresses(self, cols=('address', 'account', 'chain', 'position', 'used_times'), **constraints):
        addresses = yield self.select_addresses(', '.join(cols), **constraints)
        return rows_to_dict(addresses, cols)

    @defer.inlineCallbacks
    def get_address_count(self, **constraints):
        count = yield self.select_addresses('count(*)', **constraints)
        return count[0][0]

    @defer.inlineCallbacks
    def get_address(self, address):
        addresses = yield self.get_addresses(
            cols=('address', 'account', 'chain', 'position', 'pubkey', 'history', 'used_times'),
            address=address, limit=1
        )
        if addresses:
            return addresses[0]

    def add_keys(self, account, chain, keys):
        sql = (
            "insert into pubkey_address "
            "(address, account, chain, position, pubkey) "
            "values "
        ) + ', '.join(['(?, ?, ?, ?, ?)'] * len(keys))
        values = []
        for position, pubkey in keys:
            values.append(pubkey.address)
            values.append(account.public_key.address)
            values.append(chain)
            values.append(position)
            values.append(sqlite3.Binary(pubkey.pubkey_bytes))
        return self.run_operation(sql, values)

    @classmethod
    def _set_address_history(cls, t, address, history):
        cls.execute(
            t, "UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?",
            (history, history.count(':')//2, address)
        )

    def set_address_history(self, address, history):
        return self.db.runInteraction(lambda t: self._set_address_history(t, address, history))
<commit_msg>db.get_utxos_count -> db.get_utxo_count<commit_after>import logging
from typing import Tuple, List, Sequence

import sqlite3
from twisted.internet import defer
from twisted.enterprise import adbapi

from torba.hash import TXRefImmutable
from torba.basetransaction import BaseTransaction
from torba.baseaccount import BaseAccount

log = logging.getLogger(__name__)

def constraints_to_sql(constraints, joiner=' AND ', prepend_key=''):
    sql, values = [], {}
    for key, constraint in constraints.items():
        col, op, key = key, '=', key.replace('.', '_')
        if key.startswith('$'):
            values[key] = constraint
            continue
        elif key.endswith('__not'):
            col, op = col[:-len('__not')], '!='
        elif key.endswith('__lt'):
            col, op = col[:-len('__lt')], '<'
        elif key.endswith('__lte'):
            col, op = col[:-len('__lte')], '<='
        elif key.endswith('__gt'):
            col, op = col[:-len('__gt')], '>'
        elif key.endswith('__like'):
            col, op = col[:-len('__like')], 'LIKE'
        elif key.endswith('__in') or key.endswith('__not_in'):
            if key.endswith('__in'):
                col, op = col[:-len('__in')], 'IN'
            else:
                col, op = col[:-len('__not_in')], 'NOT IN'
            if isinstance(constraint, (list, set)):
                items = ', '.join(
                    "'{}'".format(item) if isinstance(item, str) else str(item)
                    for item in constraint
                )
            elif isinstance(constraint, str):
                items = constraint
            else:
                raise ValueError("{} requires a list, set or string as constraint value.".format(col))
            sql.append('{} {} ({})'.format(col, op, items))
            continue
        elif key.endswith('__any'):
            where, subvalues = constraints_to_sql(constraint, ' OR ', key+'_')
            sql.append('({})'.format(where))
            values.update(subvalues)
            continue
        sql.append('{} {} :{}'.format(col, op, prepend_key+key))
        values[prepend_key+key] = constraint
    return joiner.join(sql) if sql else '', values

def query(select, **constraints):
    sql = [select]
    limit = constraints.pop('limit', None)
    offset = constraints.pop('offset', None)
    order_by = constraints.pop('order_by', None)

    constraints.pop('my_account', None)
    account = constraints.pop('account', None)
    if account is not None:
        if not isinstance(account, list):
            account = [account]
        constraints['account__in'] = [
            (a.public_key.address if isinstance(a, BaseAccount) else a) for a in account
        ]

    where, values = constraints_to_sql(constraints)
    if where:
        sql.append('WHERE')
        sql.append(where)

    if order_by is not None:
        sql.append('ORDER BY')
        if isinstance(order_by, str):
            sql.append(order_by)
        elif isinstance(order_by, list):
            sql.append(', '.join(order_by))
        else:
            raise ValueError("order_by must be string or list")

    if offset is not None:
        sql.append('OFFSET {}'.format(offset))

    if limit is not None:
        sql.append('LIMIT {}'.format(limit))

    return ' '.join(sql), values

def rows_to_dict(rows, fields):
    if rows:
        return [dict(zip(fields, r)) for r in rows]
    else:
        return []

def row_dict_or_default(rows, fields, default=None):
    dicts = rows_to_dict(rows, fields)
    return dicts[0] if dicts else default

class SQLiteMixin:

    CREATE_TABLES_QUERY: Sequence[str] = ()

    def __init__(self, path):
        self._db_path = path
        self.db: adbapi.ConnectionPool = None
        self.ledger = None

    def open(self):
        log.info("connecting to database: %s", self._db_path)
        self.db = adbapi.ConnectionPool(
            'sqlite3', self._db_path, cp_min=1, cp_max=1, check_same_thread=False
        )
        return self.db.runInteraction(
            lambda t: t.executescript(self.CREATE_TABLES_QUERY)
        )

    def close(self):
        self.db.close()
        return defer.succeed(True)

    @staticmethod
    def _insert_sql(table: str, data: dict) -> Tuple[str, List]:
        columns, values = [], []
        for column, value in data.items():
            columns.append(column)
            values.append(value)
        sql = "INSERT INTO {} ({}) VALUES ({})".format(
            table, ', '.join(columns), ', '.join(['?'] * len(values))
        )
        return sql, values

    @staticmethod
    def _update_sql(table: str, data: dict, where: str, constraints: list) -> Tuple[str, list]:
        columns, values = [], []
        for column, value in data.items():
            columns.append("{} = ?".format(column))
            values.append(value)
        values.extend(constraints)
        sql = "UPDATE {} SET {} WHERE {}".format(
            table, ', '.join(columns), where
        )
        return sql, values

    @staticmethod
    def execute(t, sql, values):
        log.debug(sql)
        log.debug(values)
        return t.execute(sql, values)

    def run_operation(self, sql, values):
        log.debug(sql)
        log.debug(values)
        return self.db.runOperation(sql, values)

    def run_query(self, sql, values):
        log.debug(sql)
        log.debug(values)
        return self.db.runQuery(sql, values)

class BaseDatabase(SQLiteMixin):

    CREATE_PUBKEY_ADDRESS_TABLE = """
        create table if not exists pubkey_address (
            address text primary key,
            account text not null,
            chain integer not null,
            position integer not null,
            pubkey blob not null,
            history text,
            used_times integer not null default 0
        );
    """

    CREATE_TX_TABLE = """
        create table if not exists tx (
            txid text primary key,
            raw blob not null,
            height integer not null,
            position integer not null,
            is_verified boolean not null default 0
        );
    """

    CREATE_TXO_TABLE = """
        create table if not exists txo (
            txid text references tx,
            txoid text primary key,
            address text references pubkey_address,
            position integer not null,
            amount integer not null,
            script blob not null,
            is_reserved boolean not null default 0
        );
    """

    CREATE_TXI_TABLE = """
        create table if not exists txi (
            txid text references tx,
            txoid text references txo,
            address text references pubkey_address
        );
    """

    CREATE_TABLES_QUERY = (
        CREATE_TX_TABLE +
        CREATE_PUBKEY_ADDRESS_TABLE +
        CREATE_TXO_TABLE +
        CREATE_TXI_TABLE
    )

    @staticmethod
    def txo_to_row(tx, address, txo):
        return {
            'txid': tx.id,
            'txoid': txo.id,
            'address': address,
            'position': txo.position,
            'amount': txo.amount,
            'script': sqlite3.Binary(txo.script.source)
        }

    def save_transaction_io(self, save_tx, tx: BaseTransaction, address, txhash, history):

        def _steps(t):
            if save_tx == 'insert':
                self.execute(t, *self._insert_sql('tx', {
                    'txid': tx.id,
                    'raw': sqlite3.Binary(tx.raw),
                    'height': tx.height,
                    'position': tx.position,
                    'is_verified': tx.is_verified
                }))
            elif save_tx == 'update':
                self.execute(t, *self._update_sql("tx", {
                    'height': tx.height, 'position': tx.position, 'is_verified': tx.is_verified
                }, 'txid = ?', (tx.id,)))

            existing_txos = [r[0] for r in self.execute(t, *query(
                "SELECT position FROM txo", txid=tx.id
            )).fetchall()]

            for txo in tx.outputs:
                if txo.position in existing_txos:
                    continue
                if txo.script.is_pay_pubkey_hash and txo.script.values['pubkey_hash'] == txhash:
                    self.execute(t, *self._insert_sql("txo", self.txo_to_row(tx, address, txo)))
                elif txo.script.is_pay_script_hash:
                    # TODO: implement script hash payments
                    print('Database.save_transaction_io: pay script hash is not implemented!')

            # lookup the address associated with each TXI (via its TXO)
            txoid_to_address = {r[0]: r[1] for r in self.execute(t, *query(
                "SELECT txoid, address FROM txo", txoid__in=[txi.txo_ref.id for txi in tx.inputs]
            )).fetchall()}

            # list of TXIs that have already been added
            existing_txis = [r[0] for r in self.execute(t, *query(
                "SELECT txoid FROM txi", txid=tx.id
            )).fetchall()]

            for txi in tx.inputs:
                txoid = txi.txo_ref.id
                new_txi = txoid not in existing_txis
                address_matches = txoid_to_address.get(txoid) == address
                if new_txi and address_matches:
                    self.execute(t, *self._insert_sql("txi", {
                        'txid': tx.id,
                        'txoid': txoid,
                        'address': address,
                    }))

            self._set_address_history(t, address, history)

        return self.db.runInteraction(_steps)

    def reserve_outputs(self, txos, is_reserved=True):
        txoids = [txo.id for txo in txos]
        return self.run_operation(
            "UPDATE txo SET is_reserved = ? WHERE txoid IN ({})".format(
                ', '.join(['?']*len(txoids))
            ), [is_reserved]+txoids
        )

    def release_outputs(self, txos):
        return self.reserve_outputs(txos, is_reserved=False)

    def rewind_blockchain(self, above_height):  # pylint: disable=no-self-use
        # TODO:
        # 1. delete transactions above_height
        # 2. update address histories removing deleted TXs
        return defer.succeed(True)

    def select_transactions(self, cols, account=None, **constraints):
        if 'txid' not in constraints and account is not None:
            constraints['$account'] = account.public_key.address
            constraints['txid__in'] = """
                SELECT txo.txid FROM txo
                JOIN pubkey_address USING (address) WHERE pubkey_address.account = :$account
              UNION
                SELECT txi.txid FROM txi
                JOIN txo USING (txoid)
                JOIN pubkey_address USING (address) WHERE pubkey_address.account = :$account
            """
        return self.run_query(*query("SELECT {} FROM tx".format(cols), **constraints))

    @defer.inlineCallbacks
    def get_transactions(self, my_account=None, **constraints):
        my_account = my_account or constraints.get('account', None)

        tx_rows = yield self.select_transactions(
            'txid, raw, height, position, is_verified',
            order_by=["height DESC", "position DESC"],
            **constraints
        )

        txids, txs = [], []
        for row in tx_rows:
            txids.append(row[0])
            txs.append(self.ledger.transaction_class(
                raw=row[1], height=row[2], position=row[3], is_verified=row[4]
            ))

        annotated_txos = {
            txo.id: txo for txo in
            (yield self.get_txos(
                my_account=my_account,
                txid__in=txids
            ))
        }

        referenced_txos = {
            txo.id: txo for txo in
            (yield self.get_txos(
                my_account=my_account,
                txoid__in=query("SELECT txoid FROM txi", **{'txid__in': txids})[0]
            ))
        }

        for tx in txs:
            for txi in tx.inputs:
                txo = referenced_txos.get(txi.txo_ref.id)
                if txo:
                    txi.txo_ref = txo.ref
            for txo in tx.outputs:
                _txo = annotated_txos.get(txo.id)
                if _txo:
                    txo.update_annotations(_txo)
                else:
                    txo.update_annotations(None)

        return txs

    @defer.inlineCallbacks
    def get_transaction_count(self, **constraints):
        count = yield self.select_transactions('count(*)', **constraints)
        return count[0][0]

    @defer.inlineCallbacks
    def get_transaction(self, **constraints):
        txs = yield self.get_transactions(limit=1, **constraints)
        if txs:
            return txs[0]

    def select_txos(self, cols, **constraints):
        return self.run_query(*query(
            "SELECT {} FROM txo JOIN pubkey_address USING (address)".format(cols), **constraints
        ))

    @defer.inlineCallbacks
    def get_txos(self, my_account=None, **constraints):
        my_account = my_account or constraints.get('account', None)
        if isinstance(my_account, BaseAccount):
            my_account = my_account.public_key.address
        rows = yield self.select_txos(
            "amount, script, txid, txo.position, chain, account", **constraints
        )
        output_class = self.ledger.transaction_class.output_class
        return [
            output_class(
                amount=row[0],
                script=output_class.script_class(row[1]),
                tx_ref=TXRefImmutable.from_id(row[2]),
                position=row[3],
                is_change=row[4] == 1,
                is_my_account=row[5] == my_account
            ) for row in rows
        ]

    @defer.inlineCallbacks
    def get_txo_count(self, **constraints):
        count = yield self.select_txos('count(*)', **constraints)
        return count[0][0]

    @staticmethod
    def constrain_utxo(constraints):
        constraints['is_reserved'] = False
        constraints['txoid__not_in'] = "SELECT txoid FROM txi"

    def get_utxos(self, **constraints):
        self.constrain_utxo(constraints)
        return self.get_txos(**constraints)

    def get_utxo_count(self, **constraints):
        self.constrain_utxo(constraints)
        return self.get_txo_count(**constraints)

    @defer.inlineCallbacks
    def get_balance(self, **constraints):
        self.constrain_utxo(constraints)
        balance = yield self.select_txos('SUM(amount)', **constraints)
        return balance[0][0] or 0

    def select_addresses(self, cols, **constraints):
        return self.run_query(*query(
            "SELECT {} FROM pubkey_address".format(cols), **constraints
        ))

    @defer.inlineCallbacks
    def get_addresses(self, cols=('address', 'account', 'chain', 'position', 'used_times'), **constraints):
        addresses = yield self.select_addresses(', '.join(cols), **constraints)
        return rows_to_dict(addresses, cols)

    @defer.inlineCallbacks
    def get_address_count(self, **constraints):
        count = yield self.select_addresses('count(*)', **constraints)
        return count[0][0]

    @defer.inlineCallbacks
    def get_address(self, address):
        addresses = yield self.get_addresses(
            cols=('address', 'account', 'chain', 'position', 'pubkey', 'history', 'used_times'),
            address=address, limit=1
        )
        if addresses:
            return addresses[0]

    def add_keys(self, account, chain, keys):
        sql = (
            "insert into pubkey_address "
            "(address, account, chain, position, pubkey) "
            "values "
        ) + ', '.join(['(?, ?, ?, ?, ?)'] * len(keys))
        values = []
        for position, pubkey in keys:
            values.append(pubkey.address)
            values.append(account.public_key.address)
            values.append(chain)
            values.append(position)
            values.append(sqlite3.Binary(pubkey.pubkey_bytes))
        return self.run_operation(sql, values)

    @classmethod
    def _set_address_history(cls, t, address, history):
        cls.execute(
            t, "UPDATE pubkey_address SET history = ?, used_times = ? WHERE address = ?",
            (history, history.count(':')//2, address)
        )

    def set_address_history(self, address, history):
        return self.db.runInteraction(lambda t: self._set_address_history(t, address, history))
<|endoftext|>