<commit_before>#! /usr/bin/env python
# -*- coding: UTF-8 -*- #

from pool import ParquetPool
from pyspark.sql import SQLContext
from query import BaseQuery

__all__ = ['ParquetDAL']

class BaseDAL(object):
    def __init__(self):
        self._connections = dict()
        self._tables = dict()
        self._sql = None
        self._uri = None
        self._query = {}

    @property
    def tables(self):
        return self._tables

    @property
    def sql(self):
        return self._sql

    @property
    def query(self):
        return self.base_query.query

    @property
    def base_query(self):
        return (
            self._query.get('select', BaseQuery()) +
            self._from() +
            self._query.get('where', BaseQuery()) +
            self._query.get('order_by', BaseQuery()) +
            self._query.get('group_by', BaseQuery()) +
            self._query.get('having', BaseQuery()) +
            self._query.get('limit', BaseQuery())
        )

    def connect(self, *args, **kwargs):
        raise NotImplementedError()

    def __call__(self, table_name):
        return self._tables.get(table_name)

    def execute(self):
        raise NotImplementedError()

    def set_table(self, name, table_obj):
        self._tables[name] = table_obj

    def configure(self, name, **params):
        table = self._tables.get(name)
        if table:
            table.config = params

    def select(self, *fields):
        self._query['select'] = self._make_simple_statement(
            BaseQuery("SELECT", fields)
        )

        return self

    def _from(self):
        tables = []
        for query in self._query.values():
            tables += [field.table.name for field in query.fields]

        query = BaseQuery("FROM")
        query += BaseQuery(", ".join(set(tables)))
        return query

    def where(self, *base_queries):
        self._query['where'] = self._make_conditional_statement(
            BaseQuery("WHERE"), base_queries
        )

        return self

    def having(self, *base_queries):
        self._query['having'] = self._make_conditional_statement(
            BaseQuery("HAVING"), base_queries
        )

        return self

    def order_by(self, *fields):
        self._query['order_by'] = self._make_simple_statement(
            BaseQuery("ORDER BY", fields),
            extra_action=self._treat_order_by
        )

        return self

    def group_by(self, *fields):
        self._query['group_by'] = self._make_simple_statement(
            BaseQuery("GROUP BY", fields)
        )

        return self

    def limit(self, limit):
        self._query['limit'] = BaseQuery("LIMIT %s" % limit)

        return self

    def _treat_order_by(self, field):
        sql_name = field.sql('ASC')
        if field.invert:
            sql_name = field.sql('DESC')

        return sql_name

    def _make_conditional_statement(self, query, base_queries):
        for base_query in base_queries:
            query += base_query

        return query

    def _make_simple_statement(self, base_query, extra_action=None):
        query_fields = []
        for field in base_query.fields:

            sql_name = field.sql()
            if extra_action:
                sql_name = extra_action(field)

            query_fields.append(sql_name)

        base_query += BaseQuery(", ".join(query_fields))
        return base_query

class ParquetDAL(BaseDAL):
    def __init__(self, uri, *args, **kwargs):
        super(ParquetDAL, self).__init__()
        self._tables = ParquetPool()
        self._uri = uri
        self._sql = SQLContext(*args, **kwargs)

    def set_table(self, name, table_obj):
        super(ParquetDAL, self).set_table(name, table_obj)
        self._connect_for_schema(name)

    def _connect_for_schema(self, name):
        table = self._tables.get(name)
        if table:
            table.config['files'] = [table.schema_index_file]
            table.connection = self.connect(name)
        else:
            raise ValueError("Table %s not found" % name)

    def execute(self):
        for field in self.base_query.fields:
            self.connect(field.table.name).registerTempTable(field.table.name)
        return self._sql.sql(self.query)

    def connect(self, name):
        table = self._tables.get(name)
        files = table.config.get('files')

        if table and files:
            parquet_list = list()
            for filename in files:
                parquet_list.append("%s/%s/%s" % (self._uri,
                                                  table.name,
                                                  filename))
            return self._sql.read.parquet(*parquet_list)
        raise ValueError("Table (%s) and files needed" % name)
<commit_msg>Changing methods order in BaseDAL.<commit_after>#! /usr/bin/env python
# -*- coding: UTF-8 -*- #

from pool import ParquetPool
from pyspark.sql import SQLContext
from query import BaseQuery

__all__ = ['ParquetDAL']

class BaseDAL(object):
    def __init__(self):
        self._connections = dict()
        self._tables = dict()
        self._sql = None
        self._uri = None
        self._query = {}

    def __call__(self, table_name):
        return self._tables.get(table_name)

    @property
    def base_query(self):
        return (
            self._query.get('select', BaseQuery()) +
            self._from() +
            self._query.get('where', BaseQuery()) +
            self._query.get('order_by', BaseQuery()) +
            self._query.get('group_by', BaseQuery()) +
            self._query.get('having', BaseQuery()) +
            self._query.get('limit', BaseQuery())
        )

    @property
    def query(self):
        return self.base_query.query

    @property
    def sql(self):
        return self._sql

    @property
    def tables(self):
        return self._tables

    def configure(self, name, **params):
        table = self._tables.get(name)
        if table:
            table.config = params

    def connect(self, *args, **kwargs):
        raise NotImplementedError()

    def execute(self):
        raise NotImplementedError()

    def set_table(self, name, table_obj):
        self._tables[name] = table_obj

    def group_by(self, *fields):
        self._query['group_by'] = self._make_simple_statement(
            BaseQuery("GROUP BY", fields)
        )

        return self

    def having(self, *base_queries):
        self._query['having'] = self._make_conditional_statement(
            BaseQuery("HAVING"), base_queries
        )

        return self

    def limit(self, limit):
        self._query['limit'] = BaseQuery("LIMIT %s" % limit)

        return self

    def order_by(self, *fields):
        self._query['order_by'] = self._make_simple_statement(
            BaseQuery("ORDER BY", fields),
            extra_action=self._treat_order_by
        )

        return self

    def select(self, *fields):
        self._query['select'] = self._make_simple_statement(
            BaseQuery("SELECT", fields)
        )

        return self

    def where(self, *base_queries):
        self._query['where'] = self._make_conditional_statement(
            BaseQuery("WHERE"), base_queries
        )

        return self

    def _from(self):
        tables = []
        for query in self._query.values():
            tables += [field.table.name for field in query.fields]

        query = BaseQuery("FROM")
        query += BaseQuery(", ".join(set(tables)))
        return query

    def _make_conditional_statement(self, query, base_queries):
        for base_query in base_queries:
            query += base_query

        return query

    def _make_simple_statement(self, base_query, extra_action=None):
        query_fields = []
        for field in base_query.fields:

            sql_name = field.sql()
            if extra_action:
                sql_name = extra_action(field)

            query_fields.append(sql_name)

        base_query += BaseQuery(", ".join(query_fields))
        return base_query

    def _treat_order_by(self, field):
        sql_name = field.sql('ASC')
        if field.invert:
            sql_name = field.sql('DESC')

        return sql_name

class ParquetDAL(BaseDAL):
    def __init__(self, uri, *args, **kwargs):
        super(ParquetDAL, self).__init__()
        self._tables = ParquetPool()
        self._uri = uri
        self._sql = SQLContext(*args, **kwargs)

    def set_table(self, name, table_obj):
        super(ParquetDAL, self).set_table(name, table_obj)
        self._connect_for_schema(name)

    def _connect_for_schema(self, name):
        table = self._tables.get(name)
        if table:
            table.config['files'] = [table.schema_index_file]
            table.connection = self.connect(name)
        else:
            raise ValueError("Table %s not found" % name)

    def execute(self):
        for field in self.base_query.fields:
            self.connect(field.table.name).registerTempTable(field.table.name)
        return self._sql.sql(self.query)

    def connect(self, name):
        table = self._tables.get(name)
        files = table.config.get('files')

        if table and files:
            parquet_list = list()
            for filename in files:
                parquet_list.append("%s/%s/%s" % (self._uri,
                                                  table.name,
                                                  filename))
            return self._sql.read.parquet(*parquet_list)
        raise ValueError("Table (%s) and files needed" % name)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

""" This file contains lists of tags. First entry is always the tag
representant and the rest are the synonyms. """

tags_001 = [
    # ???
    ['Gelatine', 'Gelatin'],
    ['Hefe', 'Yeast', 'Barm'],
    ['Schokolade', 'Chocolate'],
    ['Semmelbrösel', 'Bread Crumbs', 'Panko Crumbs'],
    ['Speisestärke', 'Starch'],

    # Cheese/Milk/Cream
    ['Brie'],
    ['Butter'],
    ['Buttermilch', 'Buttermilk'],
    ['Cheddar'],
    ['Feta'],
    ['Frischkäse', 'Cream Cheese'],
    ['Gorgonzola'],
    ['Gruyere '],
    ['Joghurt', 'Yogurt'],
    ['Käse', 'Cheese'],
    ['Margarine'],
    ['Mascarpone'],
    ['Milch', 'Milk'],
    ['Mozzarella'],
    ['Nutella', 'Hazelnut Cream'],
    ['Parmesan'],
    ['Quark', 'Curd'],
    ['Ricotta'],
    ['Sahne', 'Rahm', 'Cream', 'Heavy Cream'],
    ['Saure Sahne', 'Sour Cream'],
    ['Sojamilch', 'Soy Milk'],
    ['Ziegenkäse', 'Goat Cheese'],

    # Countries
    ['Amerikanisch', 'American'],
    ['Asiatisch', 'Asian'],
    ['Französisch', 'French'],
    ['Deutsch', 'German'],
    ['Englisch', 'English'],
    ['Italienisch', 'Italian'],
    ['Japanisch', 'Japanese'],
    ['Mexikanisch', 'Mexican'],
    ['Russisch', 'Russian'],
    ['Spanisch', 'Spanish'],

    # Finished
    ['Baguette'],
    ['Blätterteig', 'Puff Pastry'],
    ['Brot', 'Bread'],
    ['Brötchen', 'Bun'],
    ['Eiscreme', 'Speiseeis', 'Ice Cream'],
    ['Fladen', 'Flatbreads', 'Pita'],
    ['Gnocchi'],
    ['Kekse', 'Cracker'],
    ['Löffelbiskuits', 'Sponge Finger'],
    ['Marmelade', 'Konfitüre', 'Jam'],
    ['Miso'],
    ['Nudeln', 'Pasta'],
    ['Pastetenteig', 'Pie Crust'],
    ['Pesto'],
    ['Pizzateig', 'Pizza Dough'],
    ['Toast'],
    ['Tortellini'],
    ['Tortillas'],

    # Fruits
    ['Apfel', 'Äpfel', 'Apple'],
    ['Aprikose', 'Apricot'],
    ['Ananas', 'Pineapple'],
    ['Avocado'],
    ['Banane', 'Banana'],
    ['Berberitzen', 'Barberry'],
    ['Birne', 'Pear'],
    ['Brombeeren', 'Blackberry'],
    ['Cranberry'],
    ['Dattel', 'Date'],
    ['Erdbeeren', 'Strawberry'],
    ['Feige', 'Fig'],
    ['Grapefrucht', 'Grapefruit'],
    ['Heidelbeeren', 'Blaubeeren', 'Blueberry'],
    ['Himbeeren', 'Raspberry'],
    ['Holunderbeeren', 'Elderberry'],
    ['Honigmelone', 'Rock Melon', 'Honeydew Melon'],
    ['Johannisbeeren', 'Currant'],
    ['Kirsche', 'Cherry'],
    ['Kiwi'],
    ['Kumquat'],
    ['Limette', 'Lime'],
    ['Mandarine', 'Mandarin'],
    ['Mango'],
    ['Melone', 'Melon'],
    ['Maracuja', 'Passionsfrucht', 'Passion Fruit'],
    ['Nektarine', 'Nectarine'],
    ['Orange'],
    ['Pfirsich', 'Peach'],
    ['Pflaumen', 'Zwetschge', 'Plum', 'Prune'],
    ['Preiselbeeren', 'Cowberry', 'Lingonberry', 'Lowbush Cranberry', 'Mountain Cranberry'],
    ['Quitte', 'Quince'],
    ['Rosinen', 'Korinthen', 'Raisin', 'Currants'],
    ['Trauben', 'Grapes'],
    ['Wassermelone', 'Watermelon'],
    ['Zitrone', 'Lemon'],

    # Grains/Nuts/Seeds
    ['Cashew'],
    ['Chia'],
    ['Couscous', 'Kuskus'],
    ['Erdnuss', 'Peanut'],
    ['Erdnussbutter', 'Peanut Butter'],
    ['Fenchelsamen', 'Fennel Seeds'],
    ['Grieß', 'Semolina'],
    ['Haferflocken', 'Rolled Oats', 'Oats', 'Oat Flakes'],
    ['Haselnuss', 'Hazelnut'],
    ['Kokosnuss', 'Coconut'],
    ['Kokosnussmilch', 'Kokosmilch', 'Coconut Milk', 'Creme De Coconut', 'Cream Of Coconut'],
    ['Kürbiskerne', 'Pumpkin Seed'],
    ['Leinsamen', 'Flax Seed'],
    ['Maismehl', 'Cornmeal'],
    ['Mandeln', 'Almond'],
    ['Marzipan'],
    ['Mehl', 'Flour'],
    ['Milchreis', 'Rice Pudding'],
    ['Mohn', 'Poppy Seeds'],
    ['Müsli', 'Granola'],
    ['Nüsse', 'Nuss', 'Nuts'],
    ['Pinienkerne', 'Pine Nut', 'Pignolia'],
    ['Pistazien', 'Pistachio'],
    ['Polenta'],
    ['Reis', 'Rice'],
    ['Sesam', 'Sesame'],
    ['Sonnenblumenkerne', 'Sunflower Seed'],
    ['Walnuss', 'Walnut'],

    # Herbs
    ['Bärlauch', 'Bear\'s Garlic', 'Ramsons'],
    ['Basilikum', 'Basil'],
    ['Dill'],
    ['Lavendel', 'Lavender'],
    ['Minze', 'Mint', 'Pfefferminze', 'Peppermint'],
    ['Kresse', 'Cress'],
    ['Petersilie', 'Parsley'],
    ['Rosen', 'Roses'],
    ['Salbei', 'Sage'],
    ['Schnittlauch', 'Chives'],
    ['Thymian', 'Thyme'],
    ['Zitronenmelisse', 'Balm'],

    # Meat/Fish/Egg
    ['Corned Beef'],
    ['Huhn', 'Chicken'],
    ['Eier', 'Eggs'],
    ['Eigelb', 'Eidotter', 'Egg Yolk', 'Yolk'],
    ['Eiweiß', 'Egg White'],
    ['Ente', 'Duck'],
    ['Fisch', 'Fish'],
    ['Garnelen', 'Shrimps', 'Prawn'],
    ['Hackfleisch', 'Hash', 'Minced Meat', 'Ground Meat'],
    ['Krebs', 'Languste', 'Crawfish'],
    ['Lachs', 'Salmon'],
    ['Lamm', 'Lamb'],
    ['Rind', 'Cow', 'Beef'],
    ['Schwein', 'Pork'],
    ['Schinken', 'Ham', 'Prosciutto'],
    ['Speck', 'Bacon', 'Pancetta'],
    ['Thunfisch', 'Tuna'],
    ['Tilapia'],
    ['Truthahn', 'Turkey'],
    ['Wild', 'Venison'],
    ['Wildschwein', 'Boar'],
    ['Würste', 'Wurst', 'Sausage'],

    # Oil
    ['Hanföl', 'Hempseed Oil'],
    ['Olivenöl', 'Olive Oil'],
    ['Rapsöl', 'Canola Oil'],
    ['Sonnenblumenöl', 'Sunflower Oil'],
    ['Walnussöl', 'Walnut Oil'],

    # Spice
    ['Agavensirup', 'Agave Nectar', 'Agave Syrup'],
    ['Ahornsirup', 'Maple Syrup'],
    ['Anis', 'Anise'],
    ['Aubergine', 'Eggplant'],
    ['Balsamico', 'Balsamic Vinegar'],
    ['Bittermandelöl', 'Bitter Almond Oil'],
    ['Bohnenkraut', 'Savory'],
    ['Brauner Zucker', 'Brown Sugar'],
    ['Cayennepfeffer', 'Cayenne Pepper'],
    ['Chilisauce', 'Hot Sauce'],
    ['Curry'],
    ['Essig', 'Vinegar'],
    ['Estragon', 'Tarragon'],
    ['Honig', 'Honey'],
    ['Ingwer', 'Ginger'],
    ['Kakao', 'Cocoa'],
    ['Kardamom', 'Cardamom'],
    ['Ketchup'],
    ['Koriander', 'Coriander', 'Cilantro'],
    ['Kreuzkümmel', 'Kumin', 'Cumin'],
    ['Kurkuma', 'Turmeric'],
    ['Lorbeerblätter', 'Bay Leaves'],
    ['Mayonnaise', 'Mayo'],
    ['Muskat', 'Nutmeg'],
    ['Nelken', 'Clove'],
    ['Orangenschale', 'Orange Zest'],
    ['Oregano', 'Majoran'],
    ['Paprika [Gewürz]', 'Paprika [Spice]'],
    ['Pfeffer', 'Pepper'],
    ['Piment', 'Allspice'],
    ['Puderzucker', 'Powdered Sugar'],
    ['Reisessig', 'Rice Vinegar'],
    ['Rosmarin', 'Rosemary'],
    ['Salz', 'Salt'],
    ['Senf', 'Mustard'],
    ['Sriracha'],
    ['Sojasauce', 'Sojasoße', 'Soy Sauce'],
    ['Vanille', 'Vanilla'],
    ['Wacholderbeeren', 'Juniper Berry'],
    ['Wasabi'],
    ['Worcestershiresauce', 'Worcestersauce ' 'Worcestershire Sauce'],
    ['Zimt', 'Cinnamon'],
    ['Zitronenschale', 'Lemon Zest'],
    ['Zucker', 'Sugar'],

    # Spirits/Juice/Sirups...
    ['Amaretto'],
    ['Ananassaft', 'Pineapple Juice'],
    ['Apfelsaft', 'Apple Juice'],
    ['Bier', 'Beer', 'Ale', 'Stout'],
    ['Bitter'],
    ['Bourbon Whiskey', 'Rye Whiskey'],
    ['Cachaça', 'Cachaca'],
    ['Calvados'],
    ['Chartreuse'],
    ['Cola'],
    ['Cranberry Saft', 'Cranberry Juice'],
    ['Curaçao', 'Curacao'],
    ['Fanta'],
    ['Gin'],
    ['Ginger Ale'],
    ['Ginger Beer'],
    ['Grapefruchtsaft', 'Grapefruit Juice'],
    ['Grenadine'],
    ['Grüner Tee', 'Green Tea'],
    ['Himbeersaft', 'Raspberry Juice'],
    ['Irish Cream', 'Bailey\'s Irish Cream'],
    ['Irish Whiskey'],
    ['Kaffee', 'Coffee', 'Espresso'],
    ['Kaffeelikör', 'Coffee Liqueur', 'Kahlua'],
    ['Kräuterlikör', 'Herbal Liqueur'],
    ['Kornbrand', 'Korn'],
    ['Lavendel Sirup', 'Lavender Syrup'],
    ['Limonade', 'Lemonade'],
    ['Mandelsirup', 'Almond Syrup', 'Orgeat Syrup'],
    ['Mineralwasser', 'Club Soda', 'Soda Water'],
    ['Oolong'],
    ['Orangen Bitter', 'Orange Bitter'],
    ['Orangenlikör', 'Orange Liqueur', 'Grand Marnier', 'Triple Sec', 'Cointreau'],
    ['Orangensaft', 'Orange Juice'],
    ['Pfirsich Likör', 'Peach Liqueur'],
    ['Pfirsichsirup', 'Peach Syrup'],
    ['Rosmarin Sirup', 'Rosemary Syrup'],
    ['Rotwein', 'Red Wine'],
    ['Rum'],
    ['Rum [Dark]'],
    ['Rum [Light]'],
    ['Saffron'],
    ['Schnaps', 'Booze'],
    ['Schwarzer Tee', 'Black Tea'],
    ['Scotch Whisky'],
    ['Sekt', 'Champagne', 'Sparkling Wine'],
    ['Sour Mix'],
    ['Tee', 'Tea'],
    ['Tequila'],
    ['Tomatensaft', 'Tomato Juice'],
    ['Tonic Water'],
    ['Vermouth [Dry'],
    ['Vermouth [Sweet]'],
    ['Vodka'],
    ['Wein', 'Wine'],
    ['Weinbrand',  'Brandy', 'Cognac', 'Armagnac'],
    ['Weißwein', 'White Wine'],
    ['Whisky', 'Whiskey'],
    ['Zimt Sirup', 'Cinnamon Syrup'],
    ['Zuckersirup', 'Simple Syrup', 'Sugar Syrup'],

    # Vegetables
    ['Anschovis', 'Anchovis', 'Anchovy'],
    ['Artischocke', 'Artichoke'],
    ['Blumenkohl', 'Cauliflower'],
    ['Bohnen', 'Beans'],
    ['Brokkoli', 'Broccoli'],
    ['Chicoree', 'Chicory'],
    ['Chili', 'Chilli', 'Peperoni'],
    ['Erbsen', 'Peas'],
    ['Fenchel', 'Fennel'],
    ['Gurke', 'Cucumber'],
    ['Kapern', 'Capers'],
    ['Kartoffeln', 'Potatoes'],
    ['Kichererbsen', 'Chickpea'],
    ['Knoblauch', 'Garlic'],
    ['Kohl', 'Kale', 'Cabbage'],
    ['Kohlrabi', 'Cabbage Turnip'],
    ['Kürbis', 'Pumpkin', 'Squash'],
    ['Lauch', 'Porree', 'Leek', 'Ramp'],
    ['Linsen', 'Lentils'],
    ['Mais', 'Corn'],
    ['Mangold', 'Chard'],
    ['Meerrettich', 'Horseradish'],
    ['Möhren', 'Mohrrüben', 'Karotten', 'Carrot'],
    ['Noriblätter', 'Nori'],
    ['Oliven', 'Olives'],
    ['Paprika', 'Bell Pepper'],
    ['Pilze', 'Mushroom'],
    ['Rettich', 'Radish'],
    ['Rhabarber', 'Rhubarb'],
    ['Rübe', 'Turnip'],
    ['Rucola', 'Arugula'],
    ['Salat', 'Lettuce'],
    ['Schalotten', 'Shallot', 'Scallions'],
    ['Sellerie', 'Celery'],
    ['Spargel', 'Asparagus'],
    ['Spinat', 'Spinach'],
    ['Süßkartoffel', 'Sweet Potato'],
    ['Tofu'],
    ['Tomaten', 'Tomatoes'],
    ['Zucchini', 'Courgette'],
    ['Zwiebeln', 'Onions'],
]<commit_msg>Updated tags.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

""" This file contains lists of tags. First entry is always the tag
representant and the rest are the synonyms. """

tags_001 = [
    # ???
    ['Gelatine', 'Gelatin'],
    ['Hefe', 'Yeast', 'Barm'],
    ['Schokolade', 'Chocolate'],
    ['Semmelbrösel', 'Bread Crumbs', 'Panko Crumbs'],
    ['Speisestärke', 'Starch'],

    # Cheese/Milk/Cream
    ['Brie'],
    ['Butter'],
    ['Buttermilch', 'Buttermilk'],
    ['Cheddar'],
    ['Feta'],
    ['Frischkäse', 'Cream Cheese'],
    ['Gorgonzola'],
    ['Gruyere '],
    ['Joghurt', 'Yogurt'],
    ['Käse', 'Cheese'],
    ['Margarine'],
    ['Mascarpone'],
    ['Milch', 'Milk'],
    ['Mozzarella'],
    ['Nutella', 'Hazelnut Cream'],
    ['Parmesan'],
    ['Quark', 'Curd'],
    ['Ricotta'],
    ['Sahne', 'Rahm', 'Cream', 'Heavy Cream'],
    ['Saure Sahne', 'Sour Cream'],
    ['Sojamilch', 'Soy Milk'],
    ['Ziegenkäse', 'Goat Cheese'],

    # Countries
    ['Amerikanisch', 'American'],
    ['Asiatisch', 'Asian'],
    ['Französisch', 'French'],
    ['Deutsch', 'German'],
    ['Englisch', 'English'],
    ['Italienisch', 'Italian'],
    ['Japanisch', 'Japanese'],
    ['Mexikanisch', 'Mexican'],
    ['Russisch', 'Russian'],
    ['Spanisch', 'Spanish'],

    # Finished
    ['Baguette'],
    ['Blätterteig', 'Puff Pastry'],
    ['Brot', 'Bread'],
    ['Brötchen', 'Bun'],
    ['Eiscreme', 'Speiseeis', 'Ice Cream'],
    ['Fladen', 'Flatbreads', 'Pita'],
    ['Gnocchi'],
    ['Kekse', 'Cracker'],
    ['Löffelbiskuits', 'Sponge Finger'],
    ['Marmelade', 'Konfitüre', 'Jam'],
    ['Miso'],
    ['Nudeln', 'Pasta'],
    ['Pastetenteig', 'Pie Crust'],
    ['Pesto'],
    ['Pizzateig', 'Pizza Dough'],
    ['Toast'],
    ['Tortellini'],
    ['Tortillas'],

    # Fruits
    ['Apfel', 'Äpfel', 'Apple'],
    ['Aprikose', 'Apricot'],
    ['Ananas', 'Pineapple'],
    ['Avocado'],
    ['Banane', 'Banana'],
    ['Berberitzen', 'Barberry'],
    ['Birne', 'Pear'],
    ['Brombeeren', 'Blackberry'],
    ['Cranberry'],
    ['Dattel', 'Date'],
    ['Erdbeeren', 'Strawberry'],
    ['Feige', 'Fig'],
    ['Grapefrucht', 'Grapefruit'],
    ['Heidelbeeren', 'Blaubeeren', 'Blueberry'],
    ['Himbeeren', 'Raspberry'],
    ['Holunderbeeren', 'Elderberry'],
    ['Honigmelone', 'Rock Melon', 'Honeydew Melon'],
    ['Johannisbeeren', 'Currant'],
    ['Kirsche', 'Cherry'],
    ['Kiwi'],
    ['Kumquat'],
    ['Limette', 'Lime'],
    ['Mandarine', 'Mandarin'],
    ['Mango'],
    ['Melone', 'Melon'],
    ['Maracuja', 'Passionsfrucht', 'Passion Fruit'],
    ['Nektarine', 'Nectarine'],
    ['Orange'],
    ['Pfirsich', 'Peach'],
    ['Pflaumen', 'Zwetschge', 'Plum', 'Prune'],
    ['Preiselbeeren', 'Cowberry', 'Lingonberry', 'Lowbush Cranberry', 'Mountain Cranberry'],
    ['Quitte', 'Quince'],
    ['Rosinen', 'Korinthen', 'Raisin', 'Currants'],
    ['Trauben', 'Grapes'],
    ['Wassermelone', 'Watermelon'],
    ['Zitrone', 'Lemon'],

    # Grains/Nuts/Seeds
    ['Cashew'],
    ['Chia'],
    ['Couscous', 'Kuskus'],
    ['Erdnuss', 'Peanut'],
    ['Erdnussbutter', 'Peanut Butter'],
    ['Fenchelsamen', 'Fennel Seeds'],
    ['Grieß', 'Semolina'],
    ['Haferflocken', 'Rolled Oats', 'Oats', 'Oat Flakes'],
    ['Haselnuss', 'Hazelnut'],
    ['Kokosnuss', 'Coconut'],
    ['Kokosnussmilch', 'Kokosmilch', 'Coconut Milk', 'Creme De Coconut', 'Cream Of Coconut'],
    ['Kürbiskerne', 'Pumpkin Seed', 'Pepita'],
    ['Leinsamen', 'Flax Seed'],
    ['Maismehl', 'Cornmeal'],
    ['Mandeln', 'Almond'],
    ['Marzipan'],
    ['Mehl', 'Flour'],
    ['Milchreis', 'Rice Pudding'],
    ['Mohn', 'Poppy Seeds'],
    ['Müsli', 'Granola'],
    ['Nüsse', 'Nuss', 'Nuts'],
    ['Pinienkerne', 'Pine Nut', 'Pignolia'],
    ['Pistazien', 'Pistachio'],
    ['Polenta'],
    ['Reis', 'Rice'],
    ['Sesam', 'Sesame'],
    ['Sonnenblumenkerne', 'Sunflower Seed'],
    ['Walnuss', 'Walnut'],

    # Herbs
    ['Bärlauch', 'Bear\'s Garlic', 'Ramsons'],
    ['Basilikum', 'Basil'],
    ['Dill'],
    ['Lavendel', 'Lavender'],
    ['Minze', 'Mint', 'Pfefferminze', 'Peppermint'],
    ['Kresse', 'Cress'],
    ['Petersilie', 'Parsley'],
    ['Rosen', 'Roses'],
    ['Salbei', 'Sage'],
    ['Schnittlauch', 'Chives'],
    ['Thymian', 'Thyme'],
    ['Zitronenmelisse', 'Balm'],

    # Meat/Fish/Egg
    ['Corned Beef'],
    ['Huhn', 'Chicken'],
    ['Eier', 'Eggs'],
    ['Eigelb', 'Eidotter', 'Egg Yolk', 'Yolk'],
    ['Eiweiß', 'Egg White'],
    ['Ente', 'Duck'],
    ['Fisch', 'Fish'],
    ['Garnelen', 'Shrimps', 'Prawn'],
    ['Hackfleisch', 'Hash', 'Minced Meat', 'Ground Meat'],
    ['Krebs', 'Languste', 'Crawfish'],
    ['Lachs', 'Salmon'],
    ['Lamm', 'Lamb'],
    ['Rind', 'Cow', 'Beef'],
    ['Schwein', 'Pork'],
    ['Schinken', 'Ham', 'Prosciutto'],
    ['Speck', 'Bacon', 'Pancetta'],
    ['Thunfisch', 'Tuna'],
    ['Tilapia'],
    ['Truthahn', 'Turkey'],
    ['Wild', 'Venison'],
    ['Wildschwein', 'Boar'],
    ['Würste', 'Wurst', 'Sausage'],

    # Oil
    ['Hanföl', 'Hempseed Oil'],
    ['Olivenöl', 'Olive Oil'],
    ['Rapsöl', 'Canola Oil'],
    ['Sonnenblumenöl', 'Sunflower Oil'],
    ['Walnussöl', 'Walnut Oil'],

    # Spice
    ['Agavensirup', 'Agave Nectar', 'Agave Syrup'],
    ['Ahornsirup', 'Maple Syrup'],
    ['Anis', 'Anise'],
    ['Aubergine', 'Eggplant'],
    ['Balsamico', 'Balsamic Vinegar'],
    ['Bittermandelöl', 'Bitter Almond Oil'],
    ['Bohnenkraut', 'Savory'],
    ['Brauner Zucker', 'Brown Sugar'],
    ['Cayennepfeffer', 'Cayenne Pepper'],
    ['Chilisauce', 'Hot Sauce'],
    ['Curry'],
    ['Essig', 'Vinegar'],
    ['Estragon', 'Tarragon'],
    ['Honig', 'Honey'],
    ['Ingwer', 'Ginger'],
    ['Kakao', 'Cocoa'],
    ['Kardamom', 'Cardamom'],
    ['Ketchup'],
    ['Koriander', 'Coriander', 'Cilantro'],
    ['Kreuzkümmel', 'Kumin', 'Cumin'],
    ['Kurkuma', 'Turmeric'],
    ['Lorbeerblätter', 'Bay Leaves'],
    ['Mayonnaise', 'Mayo'],
    ['Muskat', 'Nutmeg'],
    ['Nelken', 'Clove'],
    ['Orangenschale', 'Orange Zest'],
    ['Oregano', 'Majoran'],
    ['Paprika [Gewürz]', 'Paprika [Spice]'],
    ['Pfeffer', 'Pepper'],
    ['Piment', 'Allspice'],
    ['Puderzucker', 'Powdered Sugar'],
    ['Reisessig', 'Rice Vinegar'],
    ['Rosmarin', 'Rosemary'],
    ['Salz', 'Salt'],
    ['Senf', 'Mustard'],
    ['Sriracha'],
    ['Sojasauce', 'Sojasoße', 'Soy Sauce'],
    ['Vanille', 'Vanilla'],
    ['Wacholderbeeren', 'Juniper Berry'],
    ['Wasabi'],
    ['Worcestershiresauce', 'Worcestersauce ' 'Worcestershire Sauce'],
    ['Zimt', 'Cinnamon'],
    ['Zitronenschale', 'Lemon Zest'],
    ['Zucker', 'Sugar'],

    # Spirits/Juice/Sirups...
    ['Amaretto'],
    ['Ananassaft', 'Pineapple Juice'],
    ['Apfelsaft', 'Apple Juice'],
    ['Bier', 'Beer', 'Ale', 'Stout'],
    ['Bitter'],
    ['Bourbon Whiskey', 'Rye Whiskey'],
    ['Cachaça', 'Cachaca'],
    ['Calvados'],
    ['Chartreuse'],
    ['Cola'],
    ['Cranberry Saft', 'Cranberry Juice'],
    ['Curaçao', 'Curacao'],
    ['Fanta'],
    ['Gin'],
    ['Ginger Ale'],
    ['Ginger Beer'],
    ['Grapefruchtsaft', 'Grapefruit Juice'],
    ['Grenadine'],
    ['Grüner Tee', 'Green Tea'],
    ['Himbeersaft', 'Raspberry Juice'],
    ['Irish Cream', 'Bailey\'s Irish Cream'],
    ['Irish Whiskey'],
    ['Kaffee', 'Coffee', 'Espresso'],
    ['Kaffeelikör', 'Coffee Liqueur', 'Kahlua'],
    ['Kräuterlikör', 'Herbal Liqueur'],
    ['Kornbrand', 'Korn'],
    ['Lavendel Sirup', 'Lavender Syrup'],
    ['Limonade', 'Lemonade'],
    ['Mandelsirup', 'Almond Syrup', 'Orgeat Syrup'],
    ['Mineralwasser', 'Club Soda', 'Soda Water'],
    ['Oolong'],
    ['Orangen Bitter', 'Orange Bitter'],
    ['Orangenlikör', 'Orange Liqueur', 'Grand Marnier', 'Triple Sec', 'Cointreau'],
    ['Orangensaft', 'Orange Juice'],
    ['Pfirsich Likör', 'Peach Liqueur'],
    ['Pfirsichsirup', 'Peach Syrup'],
    ['Rosmarin Sirup', 'Rosemary Syrup'],
    ['Rotwein', 'Red Wine'],
    ['Rum'],
    ['Rum [Dark]'],
    ['Rum [Light]'],
    ['Saffron'],
    ['Schnaps', 'Booze'],
    ['Schwarzer Tee', 'Black Tea'],
    ['Scotch Whisky'],
    ['Sekt', 'Champagne', 'Sparkling Wine'],
    ['Sour Mix'],
    ['Tee', 'Tea'],
    ['Tequila'],
    ['Tomatensaft', 'Tomato Juice'],
    ['Tonic Water'],
    ['Vermouth [Dry'],
    ['Vermouth [Sweet]'],
    ['Vodka'],
    ['Wein', 'Wine'],
    ['Weinbrand',  'Brandy', 'Cognac', 'Armagnac'],
    ['Weißwein', 'White Wine'],
    ['Whisky', 'Whiskey'],
    ['Zimt Sirup', 'Cinnamon Syrup'],
    ['Zuckersirup', 'Simple Syrup', 'Sugar Syrup'],

    # Vegetables
    ['Anschovis', 'Anchovis', 'Anchovy'],
    ['Artischocke', 'Artichoke'],
    ['Blumenkohl', 'Cauliflower'],
    ['Bohnen', 'Beans'],
    ['Brokkoli', 'Broccoli'],
    ['Chicoree', 'Chicory'],
    ['Chili', 'Chilli', 'Peperoni'],
    ['Erbsen', 'Peas'],
    ['Fenchel', 'Fennel'],
    ['Gurke', 'Cucumber'],
    ['Kapern', 'Capers'],
    ['Kartoffeln', 'Potatoes'],
    ['Kichererbsen', 'Chickpea'],
    ['Knoblauch', 'Garlic'],
    ['Kohl', 'Kale', 'Cabbage'],
    ['Kohlrabi', 'Cabbage Turnip'],
    ['Kürbis', 'Pumpkin', 'Squash'],
    ['Lauch', 'Porree', 'Leek', 'Ramp'],
    ['Linsen', 'Lentils'],
    ['Mais', 'Corn'],
    ['Mangold', 'Chard'],
    ['Meerrettich', 'Horseradish'],
    ['Möhren', 'Mohrrüben', 'Karotten', 'Carrot'],
    ['Noriblätter', 'Nori'],
    ['Oliven', 'Olives'],
    ['Paprika', 'Bell Pepper'],
    ['Pilze', 'Mushroom'],
    ['Rettich', 'Radish'],
    ['Rhabarber', 'Rhubarb'],
    ['Rübe', 'Turnip'],
    ['Rucola', 'Arugula'],
    ['Salat', 'Lettuce'],
    ['Schalotten', 'Shallot', 'Scallions'],
    ['Sellerie', 'Celery'],
    ['Spargel', 'Asparagus'],
    ['Spinat', 'Spinach'],
    ['Süßkartoffel', 'Sweet Potato'],
    ['Tofu'],
    ['Tomaten', 'Tomatoes'],
    ['Zucchini', 'Courgette'],
    ['Zwiebeln', 'Onions'],
]<|endoftext|>


<commit_before># coding=utf-8

from sys import version_info

if version_info >= (3,):
    unichr = chr
    unicode = lambda s, e: s
    xrange = range

# Surrounding field for printing clarity
field = u'\u2591'
# List of all homoglyph strings
all_hgs = []
# Index dict of all homoglyphs
hg_index = {}

def fill_homoglyphs():
    """
    Use http://dev.networkerror.org/utf8/?start=0&end=255&cols=10&show_uni_hex=on
    with the stupid table width forced to auto.
    This dataset is for ASCII characters mapped to UTF-8 homoglyphs (some approximate).
    Some of the entries are also selected from the results of search(), below.
    """
    all_hgs.extend((
        u' \u00A0\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u202F\u205F',
        u'!\uFF01\u01C3\u2D51\uFE15\uFE57',
        u'"\uFF02',
        u'#\uFF03\uFE5F',
        u'$\uFF04\uFE69',
        u'%\uFF05\u066A\u2052\uFE6A',
        u'&\uFF06\uFE60',
        u"'\uFF07\u02B9\u0374",
        u'(\uFF08\uFE59',
        u')\uFF09\uFE5A',
        u'*\uFF0A\u22C6\uFE61',
        u'+\uFF0B\u16ED\uFE62',
        u',\uFF0C\u02CF\u16E7\u201A',
        u'-\uFF0D\u02D7\u2212\u23BC\u2574\uFE63',
        u'.\uFF0E\u2024',
        u'/\uFF0F\u1735\u2044\u2215\u29F8',
        # u'0',
        # u'1',
        u'2\u14BF',
        u'3\u01B7\u2128',
        u'4\u13CE',
        # u'5',
        u'6\u13EE',
        # u'7',
        # u'8',
        u'9\u13ED',
        u':\uFF1A\u02D0\u02F8\u0589\u1361\u16EC\u205A\u2236\u2806\uFE13\uFE55',
        u';\uFF1B\u037E\uFE14\uFE54',
        u'<\uFF1C\u02C2\u2039\u227A\u276E\u2D66\uFE64',
        u'=\uFF1D\u2550\u268C\uFE66',
        u'>\uFF1E\u02C3\u203A\u227B\u276F\uFE65',
        u'?\uFF1F\uFE16\uFE56',
        u'@\uFF20\uFE6B',
        u'A\u0391\u0410\u13AA',
        u'B\u0392\u0412\u13F4\u15F7\u2C82',
        u'C\u03F9\u0421\u13DF\u216D\u2CA4',
        u'D\u13A0\u15EA\u216E',
        u'E\u0395\u0415\u13AC',
        u'F\u15B4',
        u'G\u050C\u13C0',
        u'H\u0397\u041D\u12D8\u13BB\u157C\u2C8E',
        u'I\u0399\u0406\u2160',
        u'J\u0408\u13AB\u148D',
        u'K\u039A\u13E6\u16D5\u212A\u2C94',
        u'L\u13DE\u14AA\u216C',
        u'M\u039C\u03FA\u041C\u13B7\u216F',
        u'N\u039D\u2C9A',
        u'O\u039F\u041E\u2C9E',
        u'P\u03A1\u0420\u13E2\u2CA2',
        u'Q\u051A\u2D55',
        u'R\u13A1\u13D2\u1587',
        u'S\u0405\u13DA',
        u'T\u03A4\u0422\u13A2',
        # u'U',
        u'V\u13D9\u2164',
        u'W\u13B3\u13D4',
        u'X\u03A7\u0425\u2169\u2CAC',
        u'Y\u03A5\u2CA8',
        u'Z\u0396\u13C3',
        u'[\uFF3B',
        u'\\\uFF3C\u2216\u29F5\u29F9\uFE68',
        u']\uFF3D',
        u'^\uFF3E\u02C4\u02C6\u1DBA\u2303',
        u'_\uFF3F\u02CD\u268A',
        u'`\uFF40\u02CB\u1FEF\u2035',
        u'a\u0251\u0430',
        # u'b',
        u'c\u03F2\u0441\u217D',
        u'd\u0501\u217E',
        u'e\u0435\u1971',
        # u'f',
        u'g\u0261',
        u'h\u04BB',
        u'i\u0456\u2170',
        u'j\u03F3\u0458',
        # u'k',
        u'l\u217C',
        u'm\u217F',
        u'n\u1952',
        u'o\u03BF\u043E\u0D20\u2C9F',
        u'p\u0440\u2CA3',
        # u'q',
        # u'r',
        u's\u0455',
        # u't',
        u'u\u1959\u222A',
        u'v\u1D20\u2174\u2228\u22C1',
        u'w\u1D21',
        u'x\u0445\u2179\u2CAD',
        u'y\u0443\u1EFF',
        u'z\u1D22',
        u'{\uFF5B\uFE5B',
        u'|\uFF5C\u01C0\u16C1\u239C\u239F\u23A2\u23A5\u23AA\u23AE\uFFE8',
        u'}\uFF5D\uFE5C',
        u'~\uFF5E\u02DC\u2053\u223C'
    ))
    hg_index.update({c: hgs for hgs in all_hgs for c in hgs})

def homoglyphs_for_char(char):
    """
    :param char: The char to look up
    :return: A string of homoglyphs for that char
    """
    return hg_index[char]

def has_homoglyphs(char):
    """
    :param char: A character to check
    :return: Whether we have homoglyphs for it
    """
    return char in hg_index

def is_ascii(char):
    return ord(' ') <= ord(char) <= ord('~')

def get_writer():
    """
    :return: A codec writer for stdout. Necessary for output piping to work.
    """
    from codecs import getwriter
    from sys import stdout

    if version_info >= (3,):
        return stdout
    return getwriter(stdout.encoding or 'utf-8')(stdout)

def read_line():
    from sys import stdin

    if version_info >= (3,):
        return input()
    return raw_input().decode(stdin.encoding or 'utf-8')

def listing():
    """
    Show a list of all known homoglyphs
    """
    out = get_writer()
    for hgs in all_hgs:
        for c in hgs:
            out.write(field + c)
        out.write(field + '\n')

def explain(char):
    """
    Show an explanation of all known homoglyphs for the given ASCII char
    :param char: An ASCII char to explain
    """
    if not has_homoglyphs(char):
        print('No homoglyphs.')
        return

    try:
        import unicodedata
    except ImportError:
        print('Install docutils.')
        return

    out = get_writer()

    out.write('Char\tPoint %20s Cat Name\n' % 'Normal')

    for hg in homoglyphs_for_char(char):
        norms = ''
        for form in ('NFC', 'NFKC', 'NFD', 'NFKD'):
            h = unicodedata.normalize(form, hg)
            if h == hg[0]:
                if norms:
                    norms += ' '
                norms += form
        out.write(' %(field)c%(hg)c%(field)c\t'
                  'u%(point)04X %(norms)20s %(cat)3s %(name)s\n' % {
                      'field': field,
                      'hg': hg,
                      'point': ord(hg),
                      'norms': norms,
                      'cat': unicodedata.category(hg),
                      'name': unicodedata.name(hg, '<unnamed>')
                  })

def search():
    """
    (Not useful to the user) Troll the unicode DB for normalization matches, which are potentially homoglyphs.
    """
    try:
        import unicodedata
    except ImportError:
        print('Install docutils.')
        return

    out = get_writer()

    for point in xrange(ord('~') + 1, 0x10000):
        u = unichr(point)
        for form in ('NFC', 'NFKC', 'NFD', 'NFKD'):
            if u in hg_index:
                continue
            h = unicodedata.normalize(form, u)
            if len(h) == 1 and ord(h) != ord(u) and (
                        has_homoglyphs(h) or is_ascii(h)):
                out.write('%(ascii)c %(form)s->  %(hg)c\tu%(point)04X %(cat)s/%(name)s\n' % {
                    'ascii': h,
                    'form': form,
                    'hg': u,
                    'point': ord(u),
                    'cat': unicodedata.category(u),
                    'name': unicodedata.name(u, '<unnamed>')
                })
                break

def pipe(hardness, reverse=False):
    """
    Pipe from input to output, replacing chars with homoglyphs
    End with ctrl+C or EOF
    :param hardness: Percent probability to replace a char
    """
    from random import random, randrange

    out = get_writer()

    # "for line in stdin" works for piped input but not keyboard input

    while True:
        try:
            line = read_line()
        except EOFError:
            return
        for c in line:
            if reverse:
                c = hg_index.get(c, c)[0]
            elif random() < hardness / 100 and has_homoglyphs(c):
                hms = homoglyphs_for_char(c)
                index = randrange(len(hms))
                c = hms[index]
            out.write(c)
        out.write('\n')

def parse():
    from optparse import OptionParser

    parser = OptionParser()
    parser.add_option('-m', '--me-harder', dest='chance', type='float', default=1,
                      help='replacement percent')
    parser.add_option('-e', '--explain', dest='char',
                      help="show a char's homoglyphs")
    parser.add_option('-l', '--list', action='store_true',
                      help='show all homoglyphs')
    parser.add_option('-r', '--reverse', action='store_true',
                      help='reverse operation, clean a mimicked file')
    return parser.parse_args()

def main():
    try:
        (options, args) = parse()
        if options.list:
            listing()
        elif options.char:
            explain(unicode(options.char, 'utf-8'))
        else:
            pipe(options.chance, options.reverse)
    except KeyboardInterrupt:
        pass

fill_homoglyphs()

<commit_msg>Add check feature. Fix integer div bug.<commit_after># coding=utf-8

from sys import version_info

if version_info >= (3,):
    unichr = chr
    unicode = lambda s, e: s
    xrange = range

# Surrounding field for printing clarity
field = u'\u2591'
# List of all homoglyph strings
all_hgs = []
# Index dict of all homoglyphs
hg_index = {}

def fill_homoglyphs():
    """
    Use http://dev.networkerror.org/utf8/?start=0&end=255&cols=10&show_uni_hex=on
    with the stupid table width forced to auto.
    This dataset is for ASCII characters mapped to UTF-8 homoglyphs (some approximate).
    Some of the entries are also selected from the results of search(), below.
    """
    all_hgs.extend((
        u' \u00A0\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u202F\u205F',
        u'!\uFF01\u01C3\u2D51\uFE15\uFE57',
        u'"\uFF02',
        u'#\uFF03\uFE5F',
        u'$\uFF04\uFE69',
        u'%\uFF05\u066A\u2052\uFE6A',
        u'&\uFF06\uFE60',
        u"'\uFF07\u02B9\u0374",
        u'(\uFF08\uFE59',
        u')\uFF09\uFE5A',
        u'*\uFF0A\u22C6\uFE61',
        u'+\uFF0B\u16ED\uFE62',
        u',\uFF0C\u02CF\u16E7\u201A',
        u'-\uFF0D\u02D7\u2212\u23BC\u2574\uFE63',
        u'.\uFF0E\u2024',
        u'/\uFF0F\u1735\u2044\u2215\u29F8',
        # u'0',
        # u'1',
        u'2\u14BF',
        u'3\u01B7\u2128',
        u'4\u13CE',
        # u'5',
        u'6\u13EE',
        # u'7',
        # u'8',
        u'9\u13ED',
        u':\uFF1A\u02D0\u02F8\u0589\u1361\u16EC\u205A\u2236\u2806\uFE13\uFE55',
        u';\uFF1B\u037E\uFE14\uFE54',
        u'<\uFF1C\u02C2\u2039\u227A\u276E\u2D66\uFE64',
        u'=\uFF1D\u2550\u268C\uFE66',
        u'>\uFF1E\u02C3\u203A\u227B\u276F\uFE65',
        u'?\uFF1F\uFE16\uFE56',
        u'@\uFF20\uFE6B',
        u'A\u0391\u0410\u13AA',
        u'B\u0392\u0412\u13F4\u15F7\u2C82',
        u'C\u03F9\u0421\u13DF\u216D\u2CA4',
        u'D\u13A0\u15EA\u216E',
        u'E\u0395\u0415\u13AC',
        u'F\u15B4',
        u'G\u050C\u13C0',
        u'H\u0397\u041D\u12D8\u13BB\u157C\u2C8E',
        u'I\u0399\u0406\u2160',
        u'J\u0408\u13AB\u148D',
        u'K\u039A\u13E6\u16D5\u212A\u2C94',
        u'L\u13DE\u14AA\u216C',
        u'M\u039C\u03FA\u041C\u13B7\u216F',
        u'N\u039D\u2C9A',
        u'O\u039F\u041E\u2C9E',
        u'P\u03A1\u0420\u13E2\u2CA2',
        u'Q\u051A\u2D55',
        u'R\u13A1\u13D2\u1587',
        u'S\u0405\u13DA',
        u'T\u03A4\u0422\u13A2',
        # u'U',
        u'V\u13D9\u2164',
        u'W\u13B3\u13D4',
        u'X\u03A7\u0425\u2169\u2CAC',
        u'Y\u03A5\u2CA8',
        u'Z\u0396\u13C3',
        u'[\uFF3B',
        u'\\\uFF3C\u2216\u29F5\u29F9\uFE68',
        u']\uFF3D',
        u'^\uFF3E\u02C4\u02C6\u1DBA\u2303',
        u'_\uFF3F\u02CD\u268A',
        u'`\uFF40\u02CB\u1FEF\u2035',
        u'a\u0251\u0430',
        # u'b',
        u'c\u03F2\u0441\u217D',
        u'd\u0501\u217E',
        u'e\u0435\u1971',
        # u'f',
        u'g\u0261',
        u'h\u04BB',
        u'i\u0456\u2170',
        u'j\u03F3\u0458',
        # u'k',
        u'l\u217C',
        u'm\u217F',
        u'n\u1952',
        u'o\u03BF\u043E\u0D20\u2C9F',
        u'p\u0440\u2CA3',
        # u'q',
        # u'r',
        u's\u0455',
        # u't',
        u'u\u1959\u222A',
        u'v\u1D20\u2174\u2228\u22C1',
        u'w\u1D21',
        u'x\u0445\u2179\u2CAD',
        u'y\u0443\u1EFF',
        u'z\u1D22',
        u'{\uFF5B\uFE5B',
        u'|\uFF5C\u01C0\u16C1\u239C\u239F\u23A2\u23A5\u23AA\u23AE\uFFE8',
        u'}\uFF5D\uFE5C',
        u'~\uFF5E\u02DC\u2053\u223C'
    ))
    hg_index.update({c: hgs for hgs in all_hgs for c in hgs})

def homoglyphs_for_char(char):
    """
    :param char: The char to look up
    :return: A string of homoglyphs for that char
    """
    return hg_index[char]

def has_homoglyphs(char):
    """
    :param char: A character to check
    :return: Whether we have homoglyphs for it
    """
    return char in hg_index

def is_ascii(char):
    return ord(' ') <= ord(char) <= ord('~')

def get_writer():
    """
    :return: A codec writer for stdout. Necessary for output piping to work.
    """
    from codecs import getwriter
    from sys import stdout

    if version_info >= (3,):
        return stdout
    return getwriter(stdout.encoding or 'utf-8')(stdout)

def read_line():
    from sys import stdin

    if version_info >= (3,):
        return input()
    return raw_input().decode(stdin.encoding or 'utf-8')

def listing():
    """
    Show a list of all known homoglyphs
    """
    out = get_writer()
    for hgs in all_hgs:
        for c in hgs:
            out.write(field + c)
        out.write(field + '\n')

def explain(char):
    """
    Show an explanation of all known homoglyphs for the given ASCII char
    :param char: An ASCII char to explain
    """
    if not has_homoglyphs(char):
        print('No homoglyphs.')
        return

    try:
        import unicodedata
    except ImportError:
        print('Install docutils.')
        return

    out = get_writer()

    out.write('Char\t%6s %20s Cat Name\n' % ('Point', 'Normal'))

    for hg in homoglyphs_for_char(char):
        norms = ''
        for form in ('NFC', 'NFKC', 'NFD', 'NFKD'):
            h = unicodedata.normalize(form, hg)
            if h == hg[0]:
                if norms:
                    norms += ' '
                norms += form
        out.write(' %(field)c%(hg)c%(field)c\t'
                  'U+%(point)04X %(norms)20s %(cat)3s %(name)s\n' % {
                      'field': field,
                      'hg': hg,
                      'point': ord(hg),
                      'norms': norms,
                      'cat': unicodedata.category(hg),
                      'name': unicodedata.name(hg, '<unnamed>')
                  })

def search():
    """
    (Not useful to the user) Troll the unicode DB for normalization matches, which are potentially homoglyphs.
    """
    try:
        import unicodedata
    except ImportError:
        print('Install docutils.')
        return

    out = get_writer()

    for point in xrange(ord('~') + 1, 0x10000):
        u = unichr(point)
        for form in ('NFC', 'NFKC', 'NFD', 'NFKD'):
            if u in hg_index:
                continue
            h = unicodedata.normalize(form, u)
            if len(h) == 1 and ord(h) != ord(u) and (
                        has_homoglyphs(h) or is_ascii(h)):
                out.write('%(ascii)c %(form)s->  %(hg)c\tU+%(point)04X %(cat)s/%(name)s\n' % {
                    'ascii': h,
                    'form': form,
                    'hg': u,
                    'point': ord(u),
                    'cat': unicodedata.category(u),
                    'name': unicodedata.name(u, '<unnamed>')
                })
                break

def pipe(replace):
    """
    Pipe from input to output
    End with ctrl+C or EOF
    :param replace: A function to replace each char
    """

    out = get_writer()

    # "for line in stdin" works for piped input but not keyboard input

    while True:
        try:
            line = read_line()
        except EOFError:
            return
        for c in line:
            out.write(replace(c))
        out.write('\n')

def pipe_mimic(hardness):
    """
    Pipe from input to output, replacing chars with homoglyphs
    :param hardness: Percent probability to replace a char
    """
    from random import random, randrange

    def replace(c):
        if random() > hardness / 100. or not has_homoglyphs(c):
            return c
        hms = homoglyphs_for_char(c)
        index = randrange(len(hms))
        return hms[index]

    pipe(replace)

def replace_reverse(c):
    """
    Undo the damage to c
    """
    return hg_index.get(c, c)[0]

def replace_check(c):
    """
    Replace non-ASCII chars with their code point
    """
    if ord(c) <= ord('~'):
        return c
    return '<%(orig)c:U+%(point)04X>' % {
        'orig': c,
        'point': ord(c)
    }

def parse():
    from optparse import OptionParser

    parser = OptionParser()
    parser.add_option('-m', '--me-harder', dest='chance', type='float', default=1,
                      help='replacement percent')
    parser.add_option('-e', '--explain', dest='char',
                      help="show a char's homoglyphs")
    parser.add_option('-l', '--list', action='store_true',
                      help='show all homoglyphs')
    parser.add_option('-c', '--check', action='store_true',
                      help='check input for suspicious chars')
    parser.add_option('-r', '--reverse', action='store_true',
                      help='reverse operation, clean a mimicked file')
    return parser.parse_args()

def main():
    try:
        (options, args) = parse()
        if options.list:
            listing()
        elif options.char:
            explain(unicode(options.char, 'utf-8'))
        elif options.check:
            pipe(replace_check)
        elif options.reverse:
            pipe(replace_reverse)
        else:
            pipe_mimic(options.chance)
    except KeyboardInterrupt:
        pass

fill_homoglyphs()

<|endoftext|>


<commit_before>'''
The goal of a tempo estimation algorithm is to automatically detect the tempo
of a piece of music, measured in beats per minute (BPM).

See http://www.music-ir.org/mirex/wiki/2014:Audio_Tempo_Estimation for a
description of the task and evaluation criteria.

Conventions
-----------

Reference tempi should be strictly positive, and provided in ascending order
as a numpy array of length 2.  Estimated tempi are allowed to be 0, but
otherwise are subject to the same constraints as reference.

The weighting value from the reference must be a float in the range [0, 1].

Metrics
-------
:func:`mir_eval.tempo.detection`
    Relative error, hits, and weighted precision of tempo estimation.

'''

import numpy as np
import collections
from . import util
import warnings

def validate(reference_tempi, reference_weight, estimated_tempi):
    """Checks that the input annotations to a metric look like valid tempo
    annotations.

    Parameters
    ----------
    reference_onsets : np.ndarray
        reference tempo values, in bpm

    reference_weight : float
        perceptual weight of slow vs fast in reference

    estimated_onsets : np.ndarray
        estimated tempo values, in bpm

    """
    # If reference or estimated onsets are empty, warn because metric will be 0
    if reference_tempi.size != 2:
        raise ValueError("Reference tempi must have two values.")

    if estimated_tempi.size != 2:
        raise ValueError("Estimated tempi must have two values.")

    if np.any(reference_tempi <= 0):
        warnings.warn('Detected non-positive reference tempo')

    if reference_weight < 0 or reference_weight > 1:
        raise ValueError('Reference weight must lie in range [0, 1]')

def detection(reference_tempi, reference_weight, estimated_tempi, tol=0.08):
    """Compute the tempo detection accuracy metric.

    Parameters
    ----------
    reference_tempi : np.ndarray, shape=(2,)
        Two non-negative reference tempi, t_slow and t_fast,
        such that t_slow < t_fast.
    reference_weight : float > 0
        The relative strength of t_slow vs t_fast in the reference.
    estimated_tempi : np.ndarray, shape=(2,)
        Two non-negative estimated tempi, r_slow and r_fast.
    tol : float in [0, 1]:
        The maximum allowable deviation from a reference tempo to
        count as a hit.
        ``|est_t - ref_t| <= tol * ref_t``
        (Default value = 0.08)

    Returns
    -------
    p_score : float in [0, 1]
        Weighted average of recalls:
        ``reference_weight * hits[0] + (1 - reference_weight) * hits[1]``
    one_correct : bool
        True if at least one reference tempo was correctly estimated
    both_correct : bool
        True if both reference tempi were correctly estimated

    """

    validate(reference_tempi, reference_weight, estimated_tempi)

    relative_errors = []
    hits = []

    for ref_t in reference_tempi:
        # Compute the relative error for tihs reference tempo
        relative_errors.append(np.min(np.abs(ref_t - estimated_tempi) / float(ref_t)))

        # Count the hits
        hits.append(bool(relative_errors[-1] <= tol))

    p_score = reference_weight * hits[0] + (1.0-reference_weight) * hits[1]

    one_correct = bool(np.max(hits))
    both_correct = bool(np.min(hits))

    return p_score, one_correct, both_correct

def evaluate(reference_tempi, reference_weight, estimated_tempi, **kwargs):
    """Compute all metrics for the given reference and estimated annotations.

    Parameters
    ----------
    kwargs
        Additional keyword arguments which will be passed to the
        appropriate metric or preprocessing functions.

    Returns
    -------
    scores : dict
        Dictionary of scores, where the key is the metric name (str) and
        the value is the (float) score achieved.

    """
    # Compute all metrics
    scores = collections.OrderedDict()

    (scores['P-score'],
     scores['One-correct'],
     scores['Both-correct']) = util.filter_kwargs(detection, reference_tempi,
                                                  reference_weight,
                                                  estimated_tempi,
                                                  **kwargs)

    return scores
<commit_msg>improved tempo.evaluate docstring #123<commit_after>'''
The goal of a tempo estimation algorithm is to automatically detect the tempo
of a piece of music, measured in beats per minute (BPM).

See http://www.music-ir.org/mirex/wiki/2014:Audio_Tempo_Estimation for a
description of the task and evaluation criteria.

Conventions
-----------

Reference tempi should be strictly positive, and provided in ascending order
as a numpy array of length 2.  Estimated tempi are allowed to be 0, but
otherwise are subject to the same constraints as reference.

The weighting value from the reference must be a float in the range [0, 1].

Metrics
-------
:func:`mir_eval.tempo.detection`
    Relative error, hits, and weighted precision of tempo estimation.

'''

import numpy as np
import collections
from . import util
import warnings

def validate(reference_tempi, reference_weight, estimated_tempi):
    """Checks that the input annotations to a metric look like valid tempo
    annotations.

    Parameters
    ----------
    reference_onsets : np.ndarray
        reference tempo values, in bpm

    reference_weight : float
        perceptual weight of slow vs fast in reference

    estimated_onsets : np.ndarray
        estimated tempo values, in bpm

    """
    # If reference or estimated onsets are empty, warn because metric will be 0
    if reference_tempi.size != 2:
        raise ValueError("Reference tempi must have two values.")

    if estimated_tempi.size != 2:
        raise ValueError("Estimated tempi must have two values.")

    if np.any(reference_tempi <= 0):
        warnings.warn('Detected non-positive reference tempo')

    if reference_weight < 0 or reference_weight > 1:
        raise ValueError('Reference weight must lie in range [0, 1]')

def detection(reference_tempi, reference_weight, estimated_tempi, tol=0.08):
    """Compute the tempo detection accuracy metric.

    Parameters
    ----------
    reference_tempi : np.ndarray, shape=(2,)
        Two non-negative reference tempi, t_slow and t_fast,
        such that t_slow < t_fast.
    reference_weight : float > 0
        The relative strength of t_slow vs t_fast in the reference.
    estimated_tempi : np.ndarray, shape=(2,)
        Two non-negative estimated tempi, r_slow and r_fast.
    tol : float in [0, 1]:
        The maximum allowable deviation from a reference tempo to
        count as a hit.
        ``|est_t - ref_t| <= tol * ref_t``
        (Default value = 0.08)

    Returns
    -------
    p_score : float in [0, 1]
        Weighted average of recalls:
        ``reference_weight * hits[0] + (1 - reference_weight) * hits[1]``
    one_correct : bool
        True if at least one reference tempo was correctly estimated
    both_correct : bool
        True if both reference tempi were correctly estimated

    """

    validate(reference_tempi, reference_weight, estimated_tempi)

    relative_errors = []
    hits = []

    for ref_t in reference_tempi:
        # Compute the relative error for tihs reference tempo
        relative_errors.append(np.min(np.abs(ref_t - estimated_tempi) / float(ref_t)))

        # Count the hits
        hits.append(bool(relative_errors[-1] <= tol))

    p_score = reference_weight * hits[0] + (1.0-reference_weight) * hits[1]

    one_correct = bool(np.max(hits))
    both_correct = bool(np.min(hits))

    return p_score, one_correct, both_correct

def evaluate(reference_tempi, reference_weight, estimated_tempi, **kwargs):
    """Compute all metrics for the given reference and estimated annotations.

    Parameters
    ----------
    reference_tempi : np.ndarray, shape=(2,)
        Two non-negative reference tempi, t_slow and t_fast,
        such that t_slow < t_fast.

    reference_weight : float > 0
        The relative strength of t_slow vs t_fast in the reference.

    estimated_tempi : np.ndarray, shape=(2,)
        Two non-negative estimated tempi, r_slow and r_fast.

    kwargs
        Additional keyword arguments which will be passed to the
        appropriate metric or preprocessing functions.

    Returns
    -------
    scores : dict
        Dictionary of scores, where the key is the metric name (str) and
        the value is the (float) score achieved.
    """
    # Compute all metrics
    scores = collections.OrderedDict()

    (scores['P-score'],
     scores['One-correct'],
     scores['Both-correct']) = util.filter_kwargs(detection, reference_tempi,
                                                  reference_weight,
                                                  estimated_tempi,
                                                  **kwargs)

    return scores
<|endoftext|>


<commit_before>"""
Pure Python implementation of some builtins.
This code is not going to be executed anywhere.
These implementations are not always correct, but should work as good as
possible for the auto completion.
"""
def next(iterator, default=None):
    if hasattr("next"):
        return iterator.next()
    else:
        return iterator.__next__()
    return default

class property():
    def __init__(self, fget, fset=None, fdel=None, doc=None):
        self.fget = fget
        self.fset = fset
        self.fdel = fdel
        self.__doc__ = doc

    def __get__(self, obj, cls):
        return self.fget(obj)

    def __set__(self, obj, value):
        self.fset(obj, value)

    def __delete__(self, obj):
        self.fdel(obj)

    def setter(self, func):
        self.fset = func
        return self

    def getter(self, func):
        self.fget = func
        return self

    def deleter(self, func):
        self.fdel = func
        return self

class staticmethod():
    def __init__(self, func):
        self.func = func

    def __get__(self, obj, cls):
        return self.func

class classmethod():
    def __init__(self, func):
        self._func = func

    def __get__(self, obj, cls):
        def _method(*args, **kwargs):
            return self._func(cls, *args, **kwargs)
        return _method

class list():
    def __init__(self, iterable=[]):
        self.iterable = []
        for i in iterable:
            self.iterable += [i]

    def __iter__(self):
        for i in self.iterable:
            yield i

    def __getitem__(self, y):
        return self.iterable[y]

    def pop(self):
        return self.iterable[-1]

class set():
    def __init__(self, iterable=[]):
        self.iterable = iterable

    def __iter__(self):
        for i in self.iterable:
            yield i

    def add(self, elem):
        self.iterable += [elem]

    def pop(self):
        return self.iterable.pop()

    def copy(self):
        return self

def iter(collection, sentinel=None):
    if sentinel:
        yield collection()
    else:
        yield next(collection)

# basic types
class int():
    def __init__(self, x, base=None):
        self.x = x

class str():
    def __init__(self, obj):
        self.obj = obj
<commit_msg>rearranged builtin mixins<commit_after>"""
Pure Python implementation of some builtins.
This code is not going to be executed anywhere.
These implementations are not always correct, but should work as good as
possible for the auto completion.
"""

def next(iterator, default=None):
    if hasattr("next"):
        return iterator.next()
    else:
        return iterator.__next__()
    return default

def iter(collection, sentinel=None):
    if sentinel:
        yield collection()
    else:
        yield next(collection)

#--------------------------------------------------------
# descriptors
#--------------------------------------------------------
class property():
    def __init__(self, fget, fset=None, fdel=None, doc=None):
        self.fget = fget
        self.fset = fset
        self.fdel = fdel
        self.__doc__ = doc

    def __get__(self, obj, cls):
        return self.fget(obj)

    def __set__(self, obj, value):
        self.fset(obj, value)

    def __delete__(self, obj):
        self.fdel(obj)

    def setter(self, func):
        self.fset = func
        return self

    def getter(self, func):
        self.fget = func
        return self

    def deleter(self, func):
        self.fdel = func
        return self

class staticmethod():
    def __init__(self, func):
        self.func = func

    def __get__(self, obj, cls):
        return self.func

class classmethod():
    def __init__(self, func):
        self._func = func

    def __get__(self, obj, cls):
        def _method(*args, **kwargs):
            return self._func(cls, *args, **kwargs)
        return _method

#--------------------------------------------------------
# array stuff
#--------------------------------------------------------
class list():
    def __init__(self, iterable=[]):
        self.iterable = []
        for i in iterable:
            self.iterable += [i]

    def __iter__(self):
        for i in self.iterable:
            yield i

    def __getitem__(self, y):
        return self.iterable[y]

    def pop(self):
        return self.iterable[-1]

class set():
    def __init__(self, iterable=[]):
        self.iterable = iterable

    def __iter__(self):
        for i in self.iterable:
            yield i

    def add(self, elem):
        self.iterable += [elem]

    def pop(self):
        return self.iterable.pop()

    def copy(self):
        return self

#--------------------------------------------------------
# basic types
#--------------------------------------------------------
class int():
    def __init__(self, x, base=None):
        self.x = x

class str():
    def __init__(self, obj):
        self.obj = obj
<|endoftext|>


<commit_before>"""Conversion tool from Neuroscan CNT to FIF
"""

# Author: Jaakko Leppakangas <jaeilepp@student.jyu.fi>
#
# License: BSD (3-clause)
from os import path
import datetime
import calendar

import numpy as np

from ...utils import warn, verbose
from ..constants import FIFF
from ..utils import _mult_cal_one
from ..meas_info import _empty_info
from ..base import _BaseRaw

def read_raw_cnt(input_fname, read_blocks=True, preload=False, verbose=None):
    """

    Parameters
    ----------
    input_fname : str
        Path to the data file.
    read_blocks : bool
        Whether to read data in blocks. This is for dealing with different
        kinds of CNT data formats.
    preload : bool or str (default False)
        Preload data into memory for data manipulation and faster indexing.
        If True, the data will be preloaded into memory (fast, requires
        large amount of memory). If preload is a string, preload is the
        file name of a memory-mapped file which is used to store the data
        on the hard drive (slower, requires less memory).
    verbose : bool, str, int, or None
        If not None, override default verbose level (see mne.verbose).

    Returns
    -------
    Instance of RawCNT.
    """
    return RawCNT(input_fname, read_blocks=read_blocks, preload=preload,
                  verbose=verbose)

def _get_cnt_info(input_fname, read_blocks):
    """Helper for reading the cnt header."""
    dtype = '<i4'
    n_bytes = np.dtype(dtype).itemsize
    offset = 900  # Size of the 'SETUP' header.
    start = 0
    cnt_info = dict()
    # Reading only the fields of interest. Structure of the whole header at
    # http://paulbourke.net/dataformats/eeg/
    with open(input_fname, 'rb', buffering=0) as fid:
        fid.seek(225)
        session_date = ''.join(np.fromfile(fid, dtype='S1', count=10))
        time = ''.join(np.fromfile(fid, dtype='S1', count=12))
        date = session_date.split('/')
        if date[2].startswith('9'):
            date[2] = '19' + date[2]
        elif len(date[2]) == 2:
            date[2] = '20' + date[2]
        time = time.split(':')
        # Assuming mm/dd/yy
        date = datetime.datetime(int(date[2]), int(date[0]), int(date[1]),
                                 int(time[0]), int(time[1]), int(time[2]))
        fid.seek(370)
        n_channels = np.fromfile(fid, dtype='<u2', count=1)[0]

        data_offset = n_channels * start * n_bytes + offset
        fid.seek(376)
        cnt_info['sfreq'] = np.fromfile(fid, dtype='<u2', count=1)[0]
        fid.seek(438)
        lowpass_toggle = np.fromfile(fid, 'i1', count=1)[0]
        highpass_toggle = np.fromfile(fid, 'i1', count=1)[0]
        fid.seek(869)
        lowcutoff = np.fromfile(fid, dtype='f4', count=1)[0]
        fid.seek(2, 1)
        highcutoff = np.fromfile(fid, dtype='f4', count=1)[0]

        fid.seek(886)
        cnt_info['event_offset'] = np.fromfile(fid, dtype='<i4', count=1)[0]
        cnt_info['continuous_seconds'] = np.fromfile(fid, dtype='<f4',
                                                     count=1)[0]
        cnt_info['channel_offset'] = np.fromfile(fid, dtype='<i4', count=1)[0]
        if cnt_info['channel_offset'] > 1 and read_blocks:
            cnt_info['channel_offset'] /= 2
            warn('Reading in data in blocks of %d. If this fails, try using '
                 'read_blocks=False.')
        cnt_info['n_samples'] = (cnt_info['event_offset'] -
                                 (900 + 75 * n_channels)) / (2 * n_channels)
        ch_names = list()
        cals = list()
        baselines = list()  # Baselines are subtracted before scaling the data.
        chs = list()
        for ch_idx in range(n_channels):
            fid.seek(data_offset + 75 * ch_idx)
            ch_names.append(''.join(np.fromfile(fid, dtype='S1', count=10)))
            fid.seek(data_offset + 75 * ch_idx + 47)
            baselines.append(np.fromfile(fid, dtype='i2', count=1)[0])
            fid.seek(data_offset + 75 * ch_idx + 59)
            sensitivity = np.fromfile(fid, dtype='f4', count=1)[0]
            fid.seek(data_offset + 75 * ch_idx + 71)
            cal = np.fromfile(fid, dtype='f4', count=1)
            cals.append(cal * sensitivity * 1e-6 / 204.8)
    cnt_info['n_channels'] = n_channels

    info = _empty_info(cnt_info['sfreq'])
    if lowpass_toggle == 1:
        info['lowpass'] = highcutoff
    if highpass_toggle == 1:
        info['highpass'] = lowcutoff
    info.update(filename=input_fname,
                meas_date=calendar.timegm(date.utctimetuple()),
                description=None, buffer_size_sec=10.)
    for idx, ch_name in enumerate(ch_names):
        ch_coil = FIFF.FIFFV_COIL_EEG
        ch_kind = FIFF.FIFFV_EEG_CH
        chan_info = {'cal': cals[idx], 'logno': idx + 1, 'scanno': idx + 1,
                     'range': 1.0, 'unit_mul': 0., 'ch_name': ch_name,
                     'unit': FIFF.FIFF_UNIT_V,
                     'coord_frame': FIFF.FIFFV_COORD_HEAD, 'loc': np.zeros(12),
                     'coil_type': ch_coil, 'kind': ch_kind}
        chs.append(chan_info)

    # Add the stim channel.
    chan_info = {'cal': 1.0, 'logno': len(chs) + 1, 'scanno': len(chs) + 1,
                 'range': 1.0, 'unit_mul': 0., 'ch_name': 'STI 014',
                 'unit': FIFF.FIFF_UNIT_NONE,
                 'coord_frame': FIFF.FIFFV_COORD_UNKNOWN, 'loc': np.zeros(12),
                 'coil_type': FIFF.FIFFV_COIL_NONE, 'kind': FIFF.FIFFV_STIM_CH}
    chs.append(chan_info)
    baselines.append(0)  # For stim channel
    cnt_info['baselines'] = np.array(baselines)
    info['chs'] = chs
    info._check_consistency()
    return info, cnt_info

class RawCNT(_BaseRaw):
    """Raw object from Neuroscan CNT file.

    Parameters
    ----------
    input_fname : str
        Path to the CNT file.
    read_blocks : bool
        Whether to read data in blocks. This is for dealing with different
        kinds of CNT data formats.
    preload : bool or str (default False)
        Preload data into memory for data manipulation and faster indexing.
        If True, the data will be preloaded into memory (fast, requires
        large amount of memory). If preload is a string, preload is the
        file name of a memory-mapped file which is used to store the data
        on the hard drive (slower, requires less memory).
    verbose : bool, str, int, or None
        If not None, override default verbose level (see mne.verbose).

    See Also
    --------
    mne.io.Raw : Documentation of attribute and methods.
    """
    def __init__(self, input_fname, read_blocks=True, preload=False,
                 verbose=None):
        input_fname = path.abspath(input_fname)
        info, cnt_info = _get_cnt_info(input_fname, read_blocks)
        last_samps = [cnt_info['n_samples'] - 1]
        super(RawCNT, self).__init__(
            info, preload, filenames=[input_fname], raw_extras=[cnt_info],
            last_samps=last_samps, orig_format='int',
            verbose=verbose)

    @verbose
    def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
        """Take a chunk of raw data, multiply by mult or cals, and store"""
        n_channels = self.info['nchan'] - 1
        n_samples = self._raw_extras[0]['n_samples']
        channel_offset = self._raw_extras[0]['channel_offset']
        baselines = self._raw_extras[0]['baselines']
        event_offset = self._raw_extras[0]['event_offset']
        n_bytes = 2
        # The data is divided into blocks of samples / channel.
        # channel_offset determines the amount of successive samples.
        # Here we use sample offset to align the data because start can be in
        # the middle of these blocks.
        s_offset = start % channel_offset
        sel = np.arange(n_channels + 1)[idx]
        with open(self._filenames[fi], 'rb', buffering=0) as fid:
            fid.seek(event_offset)
            event_type = np.fromfile(fid, dtype='<i1', count=1)[0]
            event_size = np.fromfile(fid, dtype='<i4', count=1)
            if event_type == 1:
                event_bytes = 8
            elif event_type == 2:
                event_bytes = 19
            else:
                raise IOError('Unexpected event size.')
            n_events = event_size // event_bytes
            events = list()
            for i in range(n_events):
                fid.seek(event_offset + 9 + i * event_bytes + 4)
                offset = np.fromfile(fid, dtype='<i4', count=1)
                events.append((offset - 900 - 75 * n_channels) //
                              (n_channels * 2))

            event_ch = np.zeros(n_samples)

            for event in events:
                event_ch[event - 1] = 1

            fid.seek(900 + n_channels * (75 + (start - s_offset) * n_bytes))
            data_ = np.empty((n_channels + 1, data.shape[1]))
            n_samps = stop - start

            # In case channel offset and start time do not align perfectly, one
            # extra sample set is read here to cover the desired time window.
            # The whole block is read at once and then reshaped to
            # (n_channels, n_samples).
            extra_samps = channel_offset * n_channels if s_offset != 0 else 0
            count = (n_samps * n_channels + extra_samps)
            n_samps = count // n_channels // channel_offset
            samps = np.fromfile(fid, dtype='<i2', count=count)
            samps = samps.reshape((n_samps, n_channels, channel_offset),
                                  order='C')

        block = np.zeros((n_channels + 1, channel_offset * n_samps))
        for set_idx, row in enumerate(samps):
            block[:-1, set_idx * channel_offset:(set_idx +
                                                 1) * channel_offset] = row
        data_[sel] = block[sel, s_offset:stop - start + s_offset]
        data_[-1] = event_ch[start:stop]
        data -= baselines[sel, None]
        _mult_cal_one(data, data_, idx, cals, mult=None)
<commit_msg>Stim_channel constructed once. Extra block read if needed.<commit_after>"""Conversion tool from Neuroscan CNT to FIF
"""

# Author: Jaakko Leppakangas <jaeilepp@student.jyu.fi>
#
# License: BSD (3-clause)
from os import path
import datetime
import calendar

import numpy as np

from ...utils import warn, verbose
from ..constants import FIFF
from ..utils import _mult_cal_one
from ..meas_info import _empty_info
from ..base import _BaseRaw

def read_raw_cnt(input_fname, read_blocks=True, preload=False, verbose=None):
    """

    Parameters
    ----------
    input_fname : str
        Path to the data file.
    read_blocks : bool
        Whether to read data in blocks. This is for dealing with different
        kinds of CNT data formats.
    preload : bool or str (default False)
        Preload data into memory for data manipulation and faster indexing.
        If True, the data will be preloaded into memory (fast, requires
        large amount of memory). If preload is a string, preload is the
        file name of a memory-mapped file which is used to store the data
        on the hard drive (slower, requires less memory).
    verbose : bool, str, int, or None
        If not None, override default verbose level (see mne.verbose).

    Returns
    -------
    Instance of RawCNT.
    """
    return RawCNT(input_fname, read_blocks=read_blocks, preload=preload,
                  verbose=verbose)

def _get_cnt_info(input_fname, read_blocks):
    """Helper for reading the cnt header."""
    dtype = '<i4'
    n_bytes = np.dtype(dtype).itemsize
    offset = 900  # Size of the 'SETUP' header.
    start = 0
    cnt_info = dict()
    # Reading only the fields of interest. Structure of the whole header at
    # http://paulbourke.net/dataformats/eeg/
    with open(input_fname, 'rb', buffering=0) as fid:
        fid.seek(225)
        session_date = ''.join(np.fromfile(fid, dtype='S1', count=10))
        time = ''.join(np.fromfile(fid, dtype='S1', count=12))
        date = session_date.split('/')
        if date[2].startswith('9'):
            date[2] = '19' + date[2]
        elif len(date[2]) == 2:
            date[2] = '20' + date[2]
        time = time.split(':')
        # Assuming mm/dd/yy
        date = datetime.datetime(int(date[2]), int(date[0]), int(date[1]),
                                 int(time[0]), int(time[1]), int(time[2]))
        fid.seek(370)
        n_channels = np.fromfile(fid, dtype='<u2', count=1)[0]

        data_offset = n_channels * start * n_bytes + offset
        fid.seek(376)
        cnt_info['sfreq'] = np.fromfile(fid, dtype='<u2', count=1)[0]
        fid.seek(438)
        lowpass_toggle = np.fromfile(fid, 'i1', count=1)[0]
        highpass_toggle = np.fromfile(fid, 'i1', count=1)[0]
        fid.seek(869)
        lowcutoff = np.fromfile(fid, dtype='f4', count=1)[0]
        fid.seek(2, 1)
        highcutoff = np.fromfile(fid, dtype='f4', count=1)[0]

        fid.seek(886)
        event_offset = np.fromfile(fid, dtype='<i4', count=1)[0]
        cnt_info['continuous_seconds'] = np.fromfile(fid, dtype='<f4',
                                                     count=1)[0]
        # Channel offset refers to the size of blocks per channel in the file.
        cnt_info['channel_offset'] = np.fromfile(fid, dtype='<i4', count=1)[0]
        if cnt_info['channel_offset'] > 1 and read_blocks:
            cnt_info['channel_offset'] /= 2  # Data read as 2 byte ints.
            warn('Reading in data in blocks of %d. If this fails, try using '
                 'read_blocks=False.')
        n_samples = (event_offset - (900 + 75 * n_channels)) / (2 * n_channels)
        ch_names, cals, baselines, chs, pos = (list(), list(), list(), list(),
                                               list())
        size = list()
        for ch_idx in range(n_channels):  # ELECTLOC fields
            fid.seek(data_offset + 75 * ch_idx)
            ch_names.append(''.join(np.fromfile(fid, dtype='S1', count=10)))
            fid.seek(data_offset + 75 * ch_idx + 19)
            pos.append(np.fromfile(fid, dtype='f4', count=2))  # x and y pos
            fid.seek(data_offset + 75 * ch_idx + 47)
            # Baselines are subtracted before scaling the data.
            baselines.append(np.fromfile(fid, dtype='i2', count=1)[0])
            fid.seek(data_offset + 75 * ch_idx + 59)
            sensitivity = np.fromfile(fid, dtype='f4', count=1)[0]
            fid.seek(data_offset + 75 * ch_idx + 66)
            size.append(np.fromfile(fid, dtype='u1', count=5))
            fid.seek(data_offset + 75 * ch_idx + 71)
            cal = np.fromfile(fid, dtype='f4', count=1)
            cals.append(cal * sensitivity * 1e-6 / 204.8)

        fid.seek(event_offset)
        event_type = np.fromfile(fid, dtype='<i1', count=1)[0]
        event_size = np.fromfile(fid, dtype='<i4', count=1)
        if event_type == 1:
            event_bytes = 8
        elif event_type == 2:
            event_bytes = 19
        else:
            raise IOError('Unexpected event size.')
        n_events = event_size // event_bytes
        events = list()
        for i in range(n_events):
            fid.seek(event_offset + 9 + i * event_bytes + 4)
            offset = np.fromfile(fid, dtype='<i4', count=1)
            events.append((offset - 900 - 75 * n_channels) //
                          (n_channels * 2))
        stim_channel = np.zeros(n_samples)

        for event in events:
            stim_channel[event - 1] = 1

    info = _empty_info(cnt_info['sfreq'])
    if lowpass_toggle == 1:
        info['lowpass'] = highcutoff
    if highpass_toggle == 1:
        info['highpass'] = lowcutoff
    info.update(filename=input_fname,
                meas_date=calendar.timegm(date.utctimetuple()),
                description=None, buffer_size_sec=10.)
    for idx, ch_name in enumerate(ch_names):
        ch_coil = FIFF.FIFFV_COIL_EEG
        ch_kind = FIFF.FIFFV_EEG_CH
        loc = [pos[idx][0], pos[idx][1], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
        chan_info = {'cal': cals[idx], 'logno': idx + 1, 'scanno': idx + 1,
                     'range': 1.0, 'unit_mul': 0., 'ch_name': ch_name,
                     'unit': FIFF.FIFF_UNIT_V,
                     'coord_frame': FIFF.FIFFV_COORD_HEAD, 'loc': loc,
                     'coil_type': ch_coil, 'kind': ch_kind}
        chs.append(chan_info)

    # Add the stim channel.
    chan_info = {'cal': 1.0, 'logno': len(chs) + 1, 'scanno': len(chs) + 1,
                 'range': 1.0, 'unit_mul': 0., 'ch_name': 'STI 014',
                 'unit': FIFF.FIFF_UNIT_NONE,
                 'coord_frame': FIFF.FIFFV_COORD_UNKNOWN, 'loc': np.zeros(12),
                 'coil_type': FIFF.FIFFV_COIL_NONE, 'kind': FIFF.FIFFV_STIM_CH}
    chs.append(chan_info)
    baselines.append(0)  # For stim channel
    cnt_info.update(baselines=np.array(baselines), n_samples=n_samples,
                    n_channels=n_channels, stim_channel=stim_channel)
    info['chs'] = chs
    info._check_consistency()
    return info, cnt_info

class RawCNT(_BaseRaw):
    """Raw object from Neuroscan CNT file.

    Parameters
    ----------
    input_fname : str
        Path to the CNT file.
    read_blocks : bool
        Whether to read data in blocks. This is for dealing with different
        kinds of CNT data formats.
    preload : bool or str (default False)
        Preload data into memory for data manipulation and faster indexing.
        If True, the data will be preloaded into memory (fast, requires
        large amount of memory). If preload is a string, preload is the
        file name of a memory-mapped file which is used to store the data
        on the hard drive (slower, requires less memory).
    verbose : bool, str, int, or None
        If not None, override default verbose level (see mne.verbose).

    See Also
    --------
    mne.io.Raw : Documentation of attribute and methods.
    """
    def __init__(self, input_fname, read_blocks=True, preload=False,
                 verbose=None):
        input_fname = path.abspath(input_fname)
        info, cnt_info = _get_cnt_info(input_fname, read_blocks)
        last_samps = [cnt_info['n_samples'] - 1]
        super(RawCNT, self).__init__(
            info, preload, filenames=[input_fname], raw_extras=[cnt_info],
            last_samps=last_samps, orig_format='int',
            verbose=verbose)

    @verbose
    def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
        """Take a chunk of raw data, multiply by mult or cals, and store"""
        n_channels = self.info['nchan'] - 1
        channel_offset = self._raw_extras[0]['channel_offset']
        baselines = self._raw_extras[0]['baselines']
        n_bytes = 2
        # The data is divided into blocks of samples / channel.
        # channel_offset determines the amount of successive samples.
        # Here we use sample offset to align the data because start can be in
        # the middle of these blocks.
        s_offset = start % channel_offset
        sel = np.arange(n_channels + 1)[idx]
        block_size = channel_offset * n_channels  # Size of blocks in file.
        with open(self._filenames[fi], 'rb', buffering=0) as fid:
            fid.seek(900 + n_channels * (75 + (start - s_offset) * n_bytes))
            data_ = np.empty((n_channels + 1, data.shape[1]))
            n_samps = stop - start

            # In case channel offset and start time do not align perfectly,
            # extra sample sets are read here to cover the desired time window.
            # The whole block is read at once and then reshaped to
            # (n_channels, n_samples).
            extra_samps = block_size if (s_offset != 0 or
                                         n_samps % channel_offset != 0) else 0
            if s_offset >= (channel_offset // 2):  # Extend at the end.
                extra_samps += block_size
            count = n_samps // channel_offset * block_size + extra_samps
            n_samps = count // block_size
            samps = np.fromfile(fid, dtype='<i2', count=count)
            samps = samps.reshape((n_samps, n_channels, channel_offset),
                                  order='C')
        # Intermediate shaping to block sizes.
        block = np.zeros((n_channels + 1, channel_offset * n_samps))
        for set_idx, row in enumerate(samps):  # Final shape.
            block[:-1, set_idx * channel_offset:(set_idx +
                                                 1) * channel_offset] = row
        block = block[sel, s_offset:stop - start + s_offset]
        data_[sel] = block
        data_[-1] = self._raw_extras[0]['stim_channel'][start:stop]
        data -= baselines[sel, None]
        _mult_cal_one(data, data_, idx, cals, mult=None)
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

from xia2.XIA2Version import Version

# the import implicitly updates the .gitversion file
print(Version)
<commit_msg>Enable precommitbx nagging<commit_after>from __future__ import absolute_import, division, print_function

import dials.precommitbx.nagger
from xia2.XIA2Version import Version

dials.precommitbx.nagger.nag()

# the import implicitly updates the .gitversion file
print(Version)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
An example of implementing multinomial logistic (softmax) regression with a single layer of
perceptrons using Tensorflow

Ouput: Confidence prediction (as an array) of which class an observation in the class belongs to
"""

import time

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

with tf.Graph().as_default():
    # Inputs
    x = tf.placeholder(tf.float32, shape=[None, 784], name="image_inputs")
    y_ = tf.placeholder(tf.float32, shape=[None, 10], name="actual_class")
    """
    x : 28px by 28px images converted into a [(Batch Size * 28^2) x 1] column vector
    y : [Batch Size * 10] matrix of one-hot encodings representing the actual class of the image
       (ie. [ 0, 0, 0, 0, 0, 1, 0, 0, 0, 0 ] where the index of 1 is the class)
    """
    with tf.name_scope("hidden1"):
        #
        W = tf.Variable(tf.zeros([784,10]), name="weights")
        b = tf.Variable(tf.zeros([10]), name="biases")

    # Sigmoid unit
    y = tf.nn.softmax(tf.matmul(x,W) + b)
    """
    function in the form of:
        f(x_i, W, b) = Wx_i + b
        which is a linear mapping of image pixels to class scores.
        W and b are the parameters of this function which change after each iteration

        1) W * x_i => [ 0.2, 0.5, 0.6, 0.3, 1.2, .5, .2, .9, .2, .6]  # does not sum to 1
           return a K element array representing the probabilities that an image belongs to each class K

        2) + b => Adds biases to each of the classes

        3) softmax() => [ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1] # sums to 1
           returns a K element array w/ normalized probabilities that an image belongs to each class K

    Variables (Learning Parameters)
    x_i : an image with all its pixels flattened out into a [D x 1] vector
    b : "bias" vector of size [K x 1]
    W : "weight" matrix of size [D * K] (transpose of x)
    """
    cross_entropy = -tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1],
                                   name="xentropy")
    """
    Represents the cross-entropy between the p distribution and the estimated distribution q

    Defined as:
        H(p,q) = - summation{p(x)*log(q(x))}

    This represents a a second order equation with a defined minima so gradient descent
    converges to only 1 minima.
    """

    # Loss function
    loss = tf.reduce_mean(cross_entropy, name="xentropy_mean")

    global_step = tf.Variable(0, name="global_step", trainable=False)
    train_op = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
    """
    cross_entropy example

    assume inaccurate output:
    output = [0.3, 0.2, 0.5]
    expected = [0, 1, 0]
    cross entropy would be -0.2

    assume accurate output:
    output = [0.3, 0.5, 0.2]
    expected = [0, 1, 0]
    cross entropy would be -0.5

    Notice that the accurate output has a more negative value and therefore favored since
    the loss function aims to minimize the cross entropy
    """

    # SUMMARIES
    tf.scalar_summary(loss.op.name, loss)
    summary_op = tf.merge_all_summaries()

    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())

        summary_writer = tf.train.SummaryWriter("/tmp/tf-summaries/", sess.graph)

        start = time.time()
        for step in range(1000):
            image_inputs, actual_classes = mnist.train.next_batch(50)
            _, loss_value = sess.run([train_op, loss], feed_dict={x: image_inputs, y_: actual_classes})

            summary_str = sess.run(summary_op, feed_dict={x: image_inputs, y_: actual_classes})
            summary_writer.add_summary(summary_str, step)

            if step % 100 == 0:
                duration = time.time() - start
                print "Step {}: loss = {:.2f} ({:.3f} sec)".format(step, loss_value,
                                                                 duration)

        correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
        """
        Arguements of the maxima (argmax) refers to the point/s of the
        domain of a function where the function is maximized

        In this context, argmax returns the index of the greatest value in the array
        """

        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

        # Calling `Tensor.eval()` == `tf.get_default_session().run(Tensor)`
        print(accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels}))
<commit_msg>Update mnist_basic<commit_after>#!/usr/bin/env python
"""
An example of implementing multinomial logistic (softmax) regression with a single layer of
perceptrons using Tensorflow

Ouput: Confidence prediction (as an array) of which class an observation in the class belongs to
"""

import time

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

with tf.Graph().as_default():
    # Inputs
    x = tf.placeholder(tf.float32, shape=[None, 784], name="image_inputs")
    y_ = tf.placeholder(tf.float32, shape=[None, 10], name="actual_class")
    """
    x : 28px by 28px images converted into a [(Batch Size * 28^2) x 1] column vector
    y : [Batch Size * 10] matrix of one-hot encodings representing the actual class of the image
       (ie. [ 0, 0, 0, 0, 0, 1, 0, 0, 0, 0 ] where the index of 1 is the class)
    """
    with tf.name_scope("hidden1"):
        W = tf.Variable(tf.zeros([784,10]), name="weights")
        b = tf.Variable(tf.zeros([10]), name="biases")

    # Sigmoid unit
    y = tf.nn.softmax(tf.matmul(x,W) + b)
    """
    function in the form of:
        f(x_i, W, b) = Wx_i + b
        which is a linear mapping of image pixels to class scores.
        W and b are the parameters of this function which change after each iteration

        1) W * x_i => [ 0.2, 0.5, 0.6, 0.3, 1.2, .5, .2, .9, .2, .6]  # does not sum to 1
           return a K element array representing the probabilities that an image belongs to each class K

        2) + b => Adds biases to each of the classes

        3) softmax() => [ 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1] # sums to 1
           returns a K element array w/ normalized probabilities that an image belongs to each class K

    Variables (Learning Parameters)
    x_i : an image with all its pixels flattened out into a [D x 1] vector
    b : "bias" vector of size [K x 1]
    W : "weight" matrix of size [D * K] (transpose of x)
    """
    cross_entropy = -tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1],
                                   name="xentropy")
    """
    Represents the cross-entropy between the true (p) distribution and the estimated (q) distribution

    Defined as:
        H(p,q) = - summation{p(x)*log(q(x))}

    As q converges to p, the product of p*log(q) will increase and therefore, H(p,q) will become
    more negative

    The cross-entropy function p*log(q) represents a a second order equation with a defined minima so gradient descent
    converges to only 1 minima.
    """

    # Loss function
    loss = tf.reduce_mean(cross_entropy, name="xentropy_mean")

    global_step = tf.Variable(0, name="global_step", trainable=False)
    train_op = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
    """
    cross_entropy example

    assume inaccurate output:
    output = [0.3, 0.2, 0.5]
    expected = [0, 1, 0]
    cross entropy would be -0.2

    assume accurate output:
    output = [0.3, 0.5, 0.2]
    expected = [0, 1, 0]
    cross entropy would be -0.5

    Notice that the accurate output has a more negative value and therefore favored since
    the loss function aims to minimize the cross entropy
    """

    # SUMMARIES
    tf.scalar_summary(loss.op.name, loss)
    summary_op = tf.merge_all_summaries()

    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())

        summary_writer = tf.train.SummaryWriter("/tmp/tf-summaries/", sess.graph)

        start = time.time()
        for step in range(1000):
            image_inputs, actual_classes = mnist.train.next_batch(50)
            _, loss_value = sess.run([train_op, loss], feed_dict={x: image_inputs, y_: actual_classes})

            summary_str = sess.run(summary_op, feed_dict={x: image_inputs, y_: actual_classes})
            summary_writer.add_summary(summary_str, step)

            if step % 100 == 0:
                duration = time.time() - start
                print "Step {}: loss = {:.2f} ({:.3f} sec)".format(step, loss_value,
                                                                 duration)

        correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
        """
        Arguements of the maxima (argmax) refers to the point/s of the
        domain of a function where the function is maximized

        In this context, argmax returns the index of the greatest value in the array
        """

        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

        # Calling `Tensor.eval()` == `tf.get_default_session().run(Tensor)`
        print(accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels}))
<|endoftext|>


<commit_before>

__all__ = ('GateStage', 'DigitalGateStage', 'AnalogGateStage')

from kivy.properties import (BooleanProperty, StringProperty,
    BoundedNumericProperty, ObjectProperty, NumericProperty,
    ReferenceListProperty)
from kivy.clock import Clock
from time import clock
from moa.stage import MoaStage

class GateStage(MoaStage):

    last_state = None
    ''' Gets reset to None before starting. Gets assigned the value after a
    check.
    '''
    _start_hold_time = None

    def _hold_timeout(self, *largs):
        t = self.hold_time
        elapsed = clock() - self._start_hold_time

        if t <= elapsed:
            device = self.device
            device.deactivate(self)
            device.unbind(**{self.state_attr: self._port_callback})
            self.step_stage()
        else:
            Clock.schedule_once(self._hold_timeout, t - elapsed)

    def _port_callback(self, instance, value):
        last_val = self.last_state
        self.last_state = value
        if last_val == value:
            return

        if self.check_exit(value, last_val):
            t = self.hold_time
            if t:
                self._start_hold_time = clock()
                Clock.schedule_once(self._hold_timeout, t)
            else:
                device = self.device
                device.deactivate(self)
                device.unbind(**{self.state_attr: self._port_callback})
                self.step_stage()
                return False
        else:
            Clock.unschedule(self._hold_timeout)

    def pause(self, *largs, **kwargs):
        if super(GateStage, self).pause(*largs, **kwargs):
            device = self.device
            if device is not None:
                device.deactivate(self)
                device.unbind(**{self.state_attr: self._port_callback})
            Clock.unschedule(self._hold_timeout)
            return True
        return False

    def unpause(self, *largs, **kwargs):
        if super(GateStage, self).unpause(*largs, **kwargs):
            device = self.device
            if device is None:
                raise AttributeError('A device has not been assigned to this '
                                     'stage, {}'.format(self))
            self.last_state = None
            device.activate(self)
            device.bind(**{self.state_attr: self._port_callback})
            self._port_callback(device, getattr(device, self.state_attr))
            return True
        return False

    def stop(self, *largs, **kwargs):
        if super(GateStage, self).stop(*largs, **kwargs):
            device = self.device
            if device is not None:
                device.deactivate(self)
                device.unbind(**{self.state_attr: self._port_callback})
            Clock.unschedule(self._hold_timeout)
            return True
        return False

    def step_stage(self, *largs, **kwargs):
        device = self.device
        if device is None:
            raise AttributeError('A device has not been assigned to this '
                                 'stage, {}'.format(self))

        if not super(GateStage, self).step_stage(*largs, **kwargs):
            return False

        self.last_state = None
        device.activate(self)
        device.bind(**{self.state_attr: self._port_callback})
        return (self._port_callback(device, getattr(device, self.state_attr))
                is not False)

    def check_exit(self, value, last_val):
        pass

    device = ObjectProperty(None, allownone=True)
    '''The input device.
    '''

    state_attr = StringProperty('state')
    ''' The name of the attr in device to bind.
    '''

    hold_time = BoundedNumericProperty(0, min=0)
    ''' How long the state must be held to finish.
    '''

class DigitalGateStage(MoaStage):

    def check_exit(self, value, last_val):
        return value == self.state and (self.use_initial or
                                    last_val is not None)

    state = BooleanProperty(False)
    '''The state the device has to be on in order to exit from this stage.
    '''

    use_initial = BooleanProperty(True)
    ''' Whether we can complete the stage if when entering, the channel is
    already at this exit_state.
    '''

class AnalogGateStage(MoaStage):

    def check_exit(self, value, last_val):
        return self.min <= value <= self.max

    max = NumericProperty(0)

    min = NumericProperty(0)

    range = ReferenceListProperty(min, max)
<commit_msg>Add priority to clock.<commit_after>

__all__ = ('GateStage', 'DigitalGateStage', 'AnalogGateStage')

from kivy.properties import (BooleanProperty, StringProperty,
    BoundedNumericProperty, ObjectProperty, NumericProperty,
    ReferenceListProperty)
from kivy.clock import Clock
from time import clock
from moa.stage import MoaStage

class GateStage(MoaStage):

    last_state = None
    ''' Gets reset to None before starting. Gets assigned the value after a
    check.
    '''
    _start_hold_time = None

    def _hold_timeout(self, *largs):
        t = self.hold_time
        elapsed = clock() - self._start_hold_time

        if t <= elapsed:
            device = self.device
            device.deactivate(self)
            device.unbind(**{self.state_attr: self._port_callback})
            self.step_stage()
        else:
            Clock.schedule_once(self._hold_timeout, t - elapsed, priority=True)

    def _port_callback(self, instance, value):
        last_val = self.last_state
        self.last_state = value
        if last_val == value:
            return

        if self.check_exit(value, last_val):
            t = self.hold_time
            if t:
                self._start_hold_time = clock()
                Clock.schedule_once(self._hold_timeout, t, priority=True)
            else:
                device = self.device
                device.deactivate(self)
                device.unbind(**{self.state_attr: self._port_callback})
                self.step_stage()
                return False
        else:
            Clock.unschedule(self._hold_timeout)

    def pause(self, *largs, **kwargs):
        if super(GateStage, self).pause(*largs, **kwargs):
            device = self.device
            if device is not None:
                device.deactivate(self)
                device.unbind(**{self.state_attr: self._port_callback})
            Clock.unschedule(self._hold_timeout)
            return True
        return False

    def unpause(self, *largs, **kwargs):
        if super(GateStage, self).unpause(*largs, **kwargs):
            device = self.device
            if device is None:
                raise AttributeError('A device has not been assigned to this '
                                     'stage, {}'.format(self))
            self.last_state = None
            device.activate(self)
            device.bind(**{self.state_attr: self._port_callback})
            self._port_callback(device, getattr(device, self.state_attr))
            return True
        return False

    def stop(self, *largs, **kwargs):
        if super(GateStage, self).stop(*largs, **kwargs):
            device = self.device
            if device is not None:
                device.deactivate(self)
                device.unbind(**{self.state_attr: self._port_callback})
            Clock.unschedule(self._hold_timeout)
            return True
        return False

    def step_stage(self, *largs, **kwargs):
        device = self.device
        if device is None:
            raise AttributeError('A device has not been assigned to this '
                                 'stage, {}'.format(self))

        if not super(GateStage, self).step_stage(*largs, **kwargs):
            return False

        self.last_state = None
        device.activate(self)
        device.bind(**{self.state_attr: self._port_callback})
        return (self._port_callback(device, getattr(device, self.state_attr))
                is not False)

    def check_exit(self, value, last_val):
        pass

    device = ObjectProperty(None, allownone=True)
    '''The input device.
    '''

    state_attr = StringProperty('state')
    ''' The name of the attr in device to bind.
    '''

    hold_time = BoundedNumericProperty(0, min=0)
    ''' How long the state must be held to finish.
    '''

class DigitalGateStage(MoaStage):

    def check_exit(self, value, last_val):
        return value == self.state and (self.use_initial or
                                    last_val is not None)

    state = BooleanProperty(False)
    '''The state the device has to be on in order to exit from this stage.
    '''

    use_initial = BooleanProperty(True)
    ''' Whether we can complete the stage if when entering, the channel is
    already at this exit_state.
    '''

class AnalogGateStage(MoaStage):

    def check_exit(self, value, last_val):
        return self.min <= value <= self.max

    max = NumericProperty(0)

    min = NumericProperty(0)

    range = ReferenceListProperty(min, max)
<|endoftext|>