<commit_before>import logging
import sys
import inspect
import itertools
import typing
from collections import OrderedDict

from katagawa.exc import NoSuchColumnError, SchemaError
from katagawa import kg as md_kg
from katagawa.orm.schema import column as md_column
from katagawa.orm.schema import row as md_row
from katagawa.orm.schema import relationship as md_relationship

PY36 = sys.version_info[0:2] >= (3, 6)
logger = logging.getLogger(__name__)

class TableMetadata(object):
    """
    The root class for table metadata.
    This stores a registry of tables, and is responsible for calculating relationships etc.

    .. code-block:: python

        meta = TableMetadata()
        Table = table_base(metadata=meta)
    """

    def __init__(self):
        #: A registry of table name -> table object for this metadata.
        self.tables = {}

        #: The DB object bound to this metadata.
        self.bind = None  # type: md_kg.Katagawa

    def register_table(self, tbl: 'TableMeta') -> 'TableMeta':
        """
        Registers a new table object.

        :param tbl: The table to register.
        """
        tbl._metadata = self
        self.tables[tbl.__tablename__] = tbl

        return tbl

    def resolve_floating_relationships(self):
        """
        Resolves any "floating" relationships - i.e any relationship/foreign keys that don't
        directly reference a column object.
        """
        for tbl in self.tables.values():
            for column in tbl._columns.values():
                if column.foreign_key is None:
                    continue

                foreignkey = column.foreign_key
                if foreignkey.foreign_column is None:
                    table, column = foreignkey._f_name.split(".")
                    table = self.tables[table]
                    if table is None:
                        raise SchemaError("No such table '{}' exists".format(table))

                    try:
                        col = next(filter(lambda col: col.name == column, table.iter_columns()))
                    except StopIteration:
                        raise SchemaError("No such column '{}' exists on table '{}'"
                                          .format(table, column))

                    foreignkey.foreign_column = col

            for relation in tbl._relationships.values():
                assert isinstance(relation, md_relationship.Relationship)

                if relation.via_column is None:
                    table, column = relation._via_name.split(".")
                    # TODO: Add get_table method
                    table = self.tables[table]

                    if table is None:
                        raise SchemaError("No such table '{}' exists".format(table))

                    try:
                        col = next(filter(lambda col: col.name == column, table.iter_columns()))
                    except StopIteration:
                        raise SchemaError("No such column '{}' exists on table '{}'"
                                          .format(table, column))

                    relation.via_column = col

    def get_referring_relationships(self, table: 'TableMeta'):
        """
        Gets a list of relationships that "refer" to this table.
        """
        rels = []
        for tbl in self.tables.values():
            for relationship in tbl.iter_relationships():
                if relationship.table == table:
                    rels.append(relationship)

        return rels

class TableMeta(type):
    def __prepare__(*args, **kwargs):
        return OrderedDict()

    def __new__(mcs, n, b, c, register: bool = True):
        # hijack columns
        columns = OrderedDict()
        relationships = OrderedDict()
        for col_name, value in c.copy().items():
            if isinstance(value, md_column.Column):
                columns[col_name] = value
                # nuke the column
                c.pop(col_name)
            elif isinstance(value, md_relationship.Relationship):
                relationships[col_name] = value
                c.pop(col_name)

        c["_columns"] = columns
        c["_relationships"] = relationships
        return type.__new__(mcs, n, b, c)

    def __init__(self, tblname: str, tblbases: tuple, class_body: dict, register: bool = True):
        """
        Creates a new Table instance.
        """
        # create the new type object
        super().__init__(tblname, tblbases, class_body)

        # emulate `__set_name__` on Python 3.5
        # also, set names on columns unconditionally
        it = itertools.chain(self._columns.items(), self._relationships.items())
        if not PY36:
            it = itertools.chain(class_body.items(), it)

        for name, value in it:
            if hasattr(value, "__set_name__"):
                value.__set_name__(self, name)

        if register is False:
            return

        # ================ #
        # TABLE ATTRIBUTES #
        # ================ #

        try:
            self.__tablename__
        except AttributeError:
            #: The name of this table.
            self.__tablename__ = tblname.lower()

        #: The :class:`.Katagawa` this table is bound to.
        self.__bind = None

        #: A dict of columns for this table.
        self._columns = self._columns  # type: typing.Dict[str, md_column.Column]

        #: A dict of relationships for this table.
        self._relationships = self._relationships  # type: typing.Dict[str, md_relationship.Relationship]

        #: The primary key for this table.
        #: This should be a :class:`.PrimaryKey`.
        self._primary_key = self._calculate_primary_key()

        logger.debug("Registered new table {}".format(tblname))
        self._metadata.register_table(self)

    def __call__(self, *args, **kwargs):
        return self._get_table_row(**kwargs)

    def __getattr__(self, item):
        try:
            return next(filter(lambda col: col.name == item, self.columns))
        except StopIteration:
            try:
                return next(filter(lambda tup: tup[0] == item, self._relationships.items()))[1]
            except StopIteration:
                raise AttributeError(item) from None

    @property
    def __quoted_name__(self):
        return '"{}"'.format(self.__tablename__)

    @property
    def columns(self) -> 'typing.List[md_column.Column]':
        """
        :return: A list of :class:`.Column` this Table has.
        """
        return list(self.iter_columns())

    def __repr__(self):
        try:
            return "
".format(self.__name__, self.__tablename__)
        except AttributeError:
            return super().__repr__()

    def iter_relationships(self) -> 'typing.Generator[md_relationship.Relationship, None, None]':
        """
        :return: A generator that yields :class:`.Relationship` objects for this table.
        """
        for rel in self._relationships.values():
            yield rel

    def iter_columns(self) -> 'typing.Generator[md_column.Column, None, None]':
        """
        :return: A generator that yields :class:`.Column` objects for this table.
        """
        for col in self._columns.values():
            yield col

    def _calculate_primary_key(self) -> typing.Union['PrimaryKey', None]:
        """
        Calculates the current primary key for a table, given all the columns.

        If no columns are marked as a primary key, the key will not be generated.
        """
        pk_cols = []
        for col in self.iter_columns():
            if col.primary_key is True:
                pk_cols.append(col)

        if pk_cols:
            pk = PrimaryKey(*pk_cols)
            pk.table = self
            logger.debug("Calculated new primary key {}".format(pk))
            return pk

        return None

    @property
    def primary_key(self) -> 'PrimaryKey':
        """
        :getter: The :class:`.PrimaryKey` for this table.
        :setter: A new :class:`.PrimaryKey` for this table.

        .. note::
            A primary key will automatically be calculated from columns at define time, if any
            columns have ``primary_key`` set to True.
        """
        return self._primary_key

    @primary_key.setter
    def primary_key(self, key: 'PrimaryKey'):
        key.table = self
        self._primary_key = key

    def _get_table_row(self, **kwargs) -> 'md_row.TableRow':
        """
        Gets a :class:`.TableRow` that represents this table.
        """
        col_map = {col.name: col for col in self.columns}
        row = md_row.TableRow(tbl=self)

        # lol
        if self.__init__ != TableMeta.__init__ and self.__init__ != object.__init__:
            self.__init__(row, **kwargs)
        else:
            for name, val in kwargs.items():
                if name not in col_map:
                    raise NoSuchColumnError(name)

                col_map[name].type.on_set(row, val)

        return row

def table_base(name: str = "Table", meta: 'TableMetadata' = None):
    """
    Gets a new base object to use for OO-style tables.
    This object is the parent of all tables created in the object-oriented style; it provides some
    key configuration to the relationship calculator and the Katagawa object itself.

    To use this object, you call this function to create the new object, and subclass it in your
    table classes:

    .. code-block:: python
        Table = table_base()

        class User(Table):
            ...

    Binding the base object to the database object is essential for querying:

    .. code-block:: python
        # ensure the table is bound to that database
        db.bind_tables(Table)

        # now we can do queries
        sess = db.get_session()
        user = await sess.select(User).where(User.id == 2).first()

    Each Table object is associated with a database interface, which it uses for special querying
    inside the object, such as :meth:`.Table.get`.

    .. code-block:: python
        class User(Table):
            id = Column(Integer, primary_key=True)
            ...

        db.bind_tables(Table)
        # later on, in some worker code
        user = await User.get(1)

    :param name: The name of the new class to produce. By default, it is ``Table``.
    :param meta: The :class:`.TableMetadata` to use as metadata.
    :return: A new Table class that can be used for OO tables.
    """
    if meta is None:
        meta = TableMetadata()

    class Table(metaclass=TableMeta, register=False):
        _metadata = meta

    Table.__name__ = name
    return Table

class PrimaryKey(object):
    """
    Represents the primary key of a table.

    A primary key can be on any 1 to N columns in a table.

    .. code-block:: python
        class Something(Table):
            first_id = Column(Integer)
            second_id = Column(Integer)

        pkey = PrimaryKey(Something.first_id, Something.second_id)
        Something.primary_key = pkey

    Alternatively, the primary key can be automatically calculated by passing ``primary_key=True``
    to columns in their constructor:

    .. code-block:: python
        class Something(Table):
            id = Column(Integer, primary_key=True)

        print(Something.primary_key)
    """

    def __init__(self, *cols: 'md_column.Column'):
        #: A list of :class:`.Column` that this primary key encompasses.
        self.columns = list(cols)  # type: typing.List[md_column.Column]

        #: The table this primary key is bound to.
        self.table = None

    def __repr__(self):
        return "<PrimaryKey table='{}' columns='{}'>".format(self.table, self.columns)
<commit_msg>`table_name` argument is used over `__tablename__` for table names.<commit_after>import logging
import sys
import inspect
import itertools
import typing
from collections import OrderedDict

from katagawa.exc import NoSuchColumnError, SchemaError
from katagawa import kg as md_kg
from katagawa.orm.schema import column as md_column
from katagawa.orm.schema import row as md_row
from katagawa.orm.schema import relationship as md_relationship

PY36 = sys.version_info[0:2] >= (3, 6)
logger = logging.getLogger(__name__)

class TableMetadata(object):
    """
    The root class for table metadata.
    This stores a registry of tables, and is responsible for calculating relationships etc.

    .. code-block:: python

        meta = TableMetadata()
        Table = table_base(metadata=meta)
    """

    def __init__(self):
        #: A registry of table name -> table object for this metadata.
        self.tables = {}

        #: The DB object bound to this metadata.
        self.bind = None  # type: md_kg.Katagawa

    def register_table(self, tbl: 'TableMeta') -> 'TableMeta':
        """
        Registers a new table object.

        :param tbl: The table to register.
        """
        tbl._metadata = self
        self.tables[tbl.__tablename__] = tbl

        return tbl

    def resolve_floating_relationships(self):
        """
        Resolves any "floating" relationships - i.e any relationship/foreign keys that don't
        directly reference a column object.
        """
        for tbl in self.tables.values():
            for column in tbl._columns.values():
                if column.foreign_key is None:
                    continue

                foreignkey = column.foreign_key
                if foreignkey.foreign_column is None:
                    table, column = foreignkey._f_name.split(".")
                    table = self.tables[table]
                    if table is None:
                        raise SchemaError("No such table '{}' exists".format(table))

                    try:
                        col = next(filter(lambda col: col.name == column, table.iter_columns()))
                    except StopIteration:
                        raise SchemaError("No such column '{}' exists on table '{}'"
                                          .format(table, column))

                    foreignkey.foreign_column = col

            for relation in tbl._relationships.values():
                assert isinstance(relation, md_relationship.Relationship)

                if relation.via_column is None:
                    table, column = relation._via_name.split(".")
                    # TODO: Add get_table method
                    table = self.tables[table]

                    if table is None:
                        raise SchemaError("No such table '{}' exists".format(table))

                    try:
                        col = next(filter(lambda col: col.name == column, table.iter_columns()))
                    except StopIteration:
                        raise SchemaError("No such column '{}' exists on table '{}'"
                                          .format(table, column))

                    relation.via_column = col

    def get_referring_relationships(self, table: 'TableMeta'):
        """
        Gets a list of relationships that "refer" to this table.
        """
        rels = []
        for tbl in self.tables.values():
            for relationship in tbl.iter_relationships():
                if relationship.table == table:
                    rels.append(relationship)

        return rels

class TableMeta(type):
    def __prepare__(*args, **kwargs):
        return OrderedDict()

    def __new__(mcs, n, b, c, register: bool = True, *args, **kwargs):
        # hijack columns
        columns = OrderedDict()
        relationships = OrderedDict()
        for col_name, value in c.copy().items():
            if isinstance(value, md_column.Column):
                columns[col_name] = value
                # nuke the column
                c.pop(col_name)
            elif isinstance(value, md_relationship.Relationship):
                relationships[col_name] = value
                c.pop(col_name)

        c["_columns"] = columns
        c["_relationships"] = relationships
        return type.__new__(mcs, n, b, c)

    def __init__(self, tblname: str, tblbases: tuple, class_body: dict, register: bool = True,
                 *, table_name: str = None):
        """
        Creates a new Table instance.

        :param register: Should this table be registered in the TableMetadata?
        :param table_name: The name for this table.
        """
        # create the new type object
        super().__init__(tblname, tblbases, class_body)

        # emulate `__set_name__` on Python 3.5
        # also, set names on columns unconditionally
        it = itertools.chain(self._columns.items(), self._relationships.items())
        if not PY36:
            it = itertools.chain(class_body.items(), it)

        for name, value in it:
            if hasattr(value, "__set_name__"):
                value.__set_name__(self, name)

        if register is False:
            return

        # ================ #
        # TABLE ATTRIBUTES #
        # ================ #

        try:
            self.__tablename__
        except AttributeError:
            #: The name of this table.
            self.__tablename__ = table_name or tblname.lower()

        #: The :class:`.Katagawa` this table is bound to.
        self.__bind = None

        #: A dict of columns for this table.
        self._columns = self._columns  # type: typing.Dict[str, md_column.Column]

        #: A dict of relationships for this table.
        self._relationships = self._relationships  # type: typing.Dict[str, md_relationship.Relationship]

        #: The primary key for this table.
        #: This should be a :class:`.PrimaryKey`.
        self._primary_key = self._calculate_primary_key()

        logger.debug("Registered new table {}".format(tblname))
        self._metadata.register_table(self)

    def __call__(self, *args, **kwargs):
        return self._get_table_row(**kwargs)

    def __getattr__(self, item):
        try:
            return next(filter(lambda col: col.name == item, self.columns))
        except StopIteration:
            try:
                return next(filter(lambda tup: tup[0] == item, self._relationships.items()))[1]
            except StopIteration:
                raise AttributeError(item) from None

    @property
    def __quoted_name__(self):
        return '"{}"'.format(self.__tablename__)

    @property
    def columns(self) -> 'typing.List[md_column.Column]':
        """
        :return: A list of :class:`.Column` this Table has.
        """
        return list(self.iter_columns())

    def __repr__(self):
        try:
            return "
".format(self.__name__, self.__tablename__)
        except AttributeError:
            return super().__repr__()

    def iter_relationships(self) -> 'typing.Generator[md_relationship.Relationship, None, None]':
        """
        :return: A generator that yields :class:`.Relationship` objects for this table.
        """
        for rel in self._relationships.values():
            yield rel

    def iter_columns(self) -> 'typing.Generator[md_column.Column, None, None]':
        """
        :return: A generator that yields :class:`.Column` objects for this table.
        """
        for col in self._columns.values():
            yield col

    def _calculate_primary_key(self) -> typing.Union['PrimaryKey', None]:
        """
        Calculates the current primary key for a table, given all the columns.

        If no columns are marked as a primary key, the key will not be generated.
        """
        pk_cols = []
        for col in self.iter_columns():
            if col.primary_key is True:
                pk_cols.append(col)

        if pk_cols:
            pk = PrimaryKey(*pk_cols)
            pk.table = self
            logger.debug("Calculated new primary key {}".format(pk))
            return pk

        return None

    @property
    def primary_key(self) -> 'PrimaryKey':
        """
        :getter: The :class:`.PrimaryKey` for this table.
        :setter: A new :class:`.PrimaryKey` for this table.

        .. note::
            A primary key will automatically be calculated from columns at define time, if any
            columns have ``primary_key`` set to True.
        """
        return self._primary_key

    @primary_key.setter
    def primary_key(self, key: 'PrimaryKey'):
        key.table = self
        self._primary_key = key

    def _get_table_row(self, **kwargs) -> 'md_row.TableRow':
        """
        Gets a :class:`.TableRow` that represents this table.
        """
        col_map = {col.name: col for col in self.columns}
        row = md_row.TableRow(tbl=self)

        # lol
        if self.__init__ != TableMeta.__init__ and self.__init__ != object.__init__:
            self.__init__(row, **kwargs)
        else:
            for name, val in kwargs.items():
                if name not in col_map:
                    raise NoSuchColumnError(name)

                col_map[name].type.on_set(row, val)

        return row

def table_base(name: str = "Table", meta: 'TableMetadata' = None):
    """
    Gets a new base object to use for OO-style tables.
    This object is the parent of all tables created in the object-oriented style; it provides some
    key configuration to the relationship calculator and the Katagawa object itself.

    To use this object, you call this function to create the new object, and subclass it in your
    table classes:

    .. code-block:: python
        Table = table_base()

        class User(Table):
            ...

    Binding the base object to the database object is essential for querying:

    .. code-block:: python
        # ensure the table is bound to that database
        db.bind_tables(Table)

        # now we can do queries
        sess = db.get_session()
        user = await sess.select(User).where(User.id == 2).first()

    Each Table object is associated with a database interface, which it uses for special querying
    inside the object, such as :meth:`.Table.get`.

    .. code-block:: python
        class User(Table):
            id = Column(Integer, primary_key=True)
            ...

        db.bind_tables(Table)
        # later on, in some worker code
        user = await User.get(1)

    :param name: The name of the new class to produce. By default, it is ``Table``.
    :param meta: The :class:`.TableMetadata` to use as metadata.
    :return: A new Table class that can be used for OO tables.
    """
    if meta is None:
        meta = TableMetadata()

    class Table(metaclass=TableMeta, register=False):
        _metadata = meta

    Table.__name__ = name
    return Table

class PrimaryKey(object):
    """
    Represents the primary key of a table.

    A primary key can be on any 1 to N columns in a table.

    .. code-block:: python
        class Something(Table):
            first_id = Column(Integer)
            second_id = Column(Integer)

        pkey = PrimaryKey(Something.first_id, Something.second_id)
        Something.primary_key = pkey

    Alternatively, the primary key can be automatically calculated by passing ``primary_key=True``
    to columns in their constructor:

    .. code-block:: python
        class Something(Table):
            id = Column(Integer, primary_key=True)

        print(Something.primary_key)
    """

    def __init__(self, *cols: 'md_column.Column'):
        #: A list of :class:`.Column` that this primary key encompasses.
        self.columns = list(cols)  # type: typing.List[md_column.Column]

        #: The table this primary key is bound to.
        self.table = None

    def __repr__(self):
        return "<PrimaryKey table='{}' columns='{}'>".format(self.table, self.columns)
<|endoftext|>


<commit_before>from MelodyGenerate import GetData
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout
from keras.layers import GRU
from keras.optimizers import SGD, Adam
import numpy as np
import os
import copy
from Preprocess import GlobalConstant as G

class MelodyGenerate:
    """
    Generate melody using GRU-RNN(GNN).
    """

    def _init_(self):

        self.rhythm = Sequential()
        self.melody = Sequential()

#         These variables cost too much memory.
#         self.pitch_train = []
#         self.duration_train = []
#         self.pitch_test = []
#         self.duration_test = []
#
        self.melody_train = []
        self.rhythm_train = []
        self.melody_train_labels = []
        self.rhythm_train_labels = []

        self.melody_test = []
        self.rhythm_test = []
        self.melody_test_labels = []
        self.rhythm_test_labels = []

        self.pitch_generate = []
        self.duration_generate = []

        self.batch_size = 128  #to be changed
        self.nb_epoch = 1 #to be changed
        self.data_dim = 56  #23-dimension plus 33-dimension
        self.train_timesteps = 1
        self.test_timesteps = 1

        self.timestep = 1

    def getData(self, timestep):
        """
        Get the rhythm batch size.
        Prepare the melody and rhythm arrays to be the input of the model.
        """

        self._init_()
        self.timestep = timestep

        getDataor = GetData.getData()

        #padding the senquence
        temp = 0
        G.pitch_train, self.train_timesteps = getDataor.padSequences(G.pitch_train, 33)
        G.pitch_test, self.test_timesteps = getDataor.padSequences(G.pitch_test, 33)
        G.duration_train, temp = getDataor.padSequences(G.duration_train, 23)
        G.duration_test, temp= getDataor.padSequences(G.duration_test, 23)

        difference = self.train_timesteps - self.test_timesteps
        if difference < 0:
            G.pitch_train, G.duration_train = getDataor.trainTimestepsNormalization(difference, G.pitch_train, G.duration_train)
        elif difference > 0:
            G.pitch_test, G.duration_test = getDataor.testTimestepsNormalization(difference, G.pitch_test, G.duration_test)
        else:
            pass

#         pitch_train = getDataor.getPitch(pitch_train_file_name)
#         duration_train = getDataor.getDuration(duration_train_file_name)
#
#         pitch_test = getDataor.getPitch(pitch_test_file_name)
#         duration_test = getDataor.getDuration(duration_test_file_name)

        melody_train, rhythm_train, melody_train_labels, rhythm_train_labels = getDataor.getMelodyRhythm(G.pitch_train, G.duration_train,
                                                                                timestep, int(max(self.train_timesteps, self.test_timesteps) / timestep))

        melody_test, rhythm_test, melody_test_labels, rhythm_test_labels = getDataor.getMelodyRhythm(G.pitch_test, G.duration_test,
                                                                                timestep, int(max(self.train_timesteps, self.test_timesteps) / timestep))

#         print(melody_train, melody_train[10])

#         melody_train_labels, rhythm_train_labels = getDataor.getLabels(melody_train,
#                                                                         rhythm_train)
#         melody_test_labels, rhythm_test_labels = getDataor.getLabels(melody_test,
#                                                                         rhythm_test)

        self.melody_train = np.array(melody_train)
        self.rhythm_train = np.array(rhythm_train)
        self.melody_train_labels = np.array(melody_train_labels)
        self.rhythm_train_labels = np.array(rhythm_train_labels)
        self.melody_test = np.array(melody_test)
        self.rhythm_test = np.array(rhythm_test)
        self.melody_test_labels = np.array(melody_test_labels)
        self.rhythm_test_labels = np.array(rhythm_test_labels)

    def modelConstruction(self):
        """
        Construct the melody and rhythm model.
        """
        self.melody.add(GRU(128, consume_less = 'mem', return_sequences = True,
                           input_shape = (self.timestep, 56)))
        self.melody.add(Dropout(0.5))
        self.rhythm.add(GRU(128, consume_less = 'mem', return_sequences = True,
                           input_shape = (self.timestep, 56)))
        self.rhythm.add(Dropout(0.5))

        for i in range(2):
            self.melody.add(GRU(128, return_sequences = True))
            self.melody.add(Dropout(0.5))
            self.rhythm.add(GRU(128, return_sequences = True))
            self.rhythm.add(Dropout(0.5))

#         self.melody.add(GRU(128, consume_less = 'mem', return_sequences=True,
#                            input_shape = (None, self.data_dim)))
#         self.melody.add(Dropout(0.5))
#         self.rhythm.add(GRU(128, consume_less = 'mem', return_sequences=True,
#                            input_shape = (None, self.data_dim)))
#         self.rhythm.add(Dropout(0.5))
#
#         for i in range(2):
#             self.melody.add(GRU(128, consume_less = 'mem', return_sequences=False,
#                            input_shape = (None, self.data_dim)))
#             self.melody.add(Dropout(0.5))
#             self.rhythm.add(GRU(128, consume_less = 'mem', return_sequences=False,
#                            input_shape = (None, self.data_dim)))
#             self.rhythm.add(Dropout(0.5))

        self.melody.add(Dense(33, activation = 'softmax'))
        self.rhythm.add(Dense(23, activation = 'softmax'))

        #compile part
        self.melody.compile(optimizer = 'adam',
                            #loss = self.my_loss_function()
                            loss = 'categorical_crossentropy')
        self.rhythm.compile(optimizer = 'adam',
                            #loss = self.my_loss_function()
                            loss = 'categorical_crossentropy')

    def trainProcess(self):
        """
        Start training the model.
        Remember the difference between the input of the melody and rhythm model.
        """

        self.melody.fit(self.melody_train, self.melody_train_labels, batch_size = self.batch_size, nb_epoch = self.nb_epoch)
        self.rhythm.fit(self.rhythm_train, self.rhythm_train_labels, batch_size = self.batch_size, nb_epoch = self.nb_epoch)

    def evaluateProcess(self):
        score_melody = self.melody.evaluate(self.melody_test, self.melody_test_labels, batch_size = self.batch_size)
        score_rhythm = self.rhythm.evaluate(self.rhythm_test, self.rhythm_test_labels, batch_size = self.batch_size)

        print('The test error of melody is', score_melody)
        print('The test error of rhythm is', score_rhythm)

    #need be fixed
    def generater(self, length, pitch_index, duration_index):
        """
        The generater to generate new melody and rhythm.
        Rhythm first and Melody next.
        """

        melody_x = np.zeros((1, self.timestep, 56))
        rhythm_x = np.zeros((1, self.timestep, 56))

        next_duration_index = [[0] * self.timestep]
        next_pitch_index = [[0] * self.timestep]

        for i in range(self.timestep):
            melody_x[0][i][pitch_index[i]] = 1

            rhythm_x[0][i][pitch_index[i]] = 1
            rhythm_x[0][i][duration_index[i] + 33] = 1

        for i in range(self.timestep):
            self.pitch_generate.append(list(rhythm_x[0][i][0 : 33]))
            self.duration_generate.append(list(rhythm_x[0][i][33 : 56]))

        for i in range(length):

            preds_rhythm = self.rhythm.predict(rhythm_x, verbose=0)
            print(preds_rhythm)

            for j in range(self.timestep):
                next_duration_index[0][j] = self.getIndex(preds_rhythm[0, j], 23)
                next_duration_index[0][j] = next_duration_index[0][j] + 33

            for j in range(self.timestep):
                melody_x[0][j][next_duration_index[0][j]] = 1
            preds_melody = self.melody.predict(melody_x, verbose=0)

            for j in range(self.timestep):
                next_pitch_index[0][j] = self.getIndex(preds_melody[0, j], 33)

            print(preds_melody.shape)
            print(preds_rhythm.shape)

            next_melody_x = np.zeros((1, self.timestep, 56))
            next_rhythm_x = np.zeros((1, self.timestep, 56))

            for j in range(self.timestep):
                next_melody_x[0][j][next_pitch_index[0][j]] = 1

                next_rhythm_x[0][j][next_pitch_index[0][j]] = 1
                next_rhythm_x[0][j][next_duration_index[0][j]] = 1

            for j in range(self.timestep):
                self.pitch_generate.append(list(next_rhythm_x[0][j][0 : 33]))
                self.duration_generate.append(list(next_rhythm_x[0][j][33 : 56]))

            melody_x = copy.deepcopy(next_melody_x)
            rhythm_x = copy.deepcopy(next_rhythm_x)

        return self.pitch_generate, self.duration_generate

    def getIndex(self, list, length):
        """
        Get the index of the largest probability of the current pitch and duration.
        """
        maxIndex = 0
        maxValue = list[0]

        for i in range(length - 1):
            if(list[i + 1] > maxValue):
                maxIndex = i + 1
                maxValue = list[i + 1]

        return maxIndex

    def abcFileWriter(self, pitch, duration, file_name):
        """
        Write pitch and duration in files.
        """
        pitch_index = 0
        duration_index = 0

        file = open(file_name, 'w')
        file.write('X:1\nT:Melody Generated\nM:6/8\nL:1/8\nK:C\n')
        duration_index_list = ['1/4', '1/3', '1/2', '3/4', '3/8',
                               '5/8', '2/3', '2/5', '4/9', '4/5',
                               '1',  '3/2', '4/3', '9/8', '2',
                               '8/3', '9/4', '7/2', '3', '4',
                               '9/2', '6', '8']

        pitch_index_list=['G,', '^G,', 'A,', '^A,', 'B,', 'C', '^C', 'D',
                          '^D', 'E', 'F', '^F', 'G', '^G', 'A', '^A',
                          'B', 'c', '^c', 'd', '^d', 'e', 'f', '^f',
                          'g', '^g', 'a', '^a', 'b', 'c\'', '^c\'', 'd\'', '^d\'']
        for i in range(int(len(pitch) / self.timestep)):
            for j in range(self.timestep):
                pitch_index = pitch[i * self.timestep + j].index(1)
                file.write(pitch_index_list[pitch_index].__str__())

                duration_index = duration[i * self.timestep + j].index(1)
                file.write(duration_index_list[duration_index].__str__())

                file.write(' ')
            file.write('|')

        file.write('|')
        file.close()

    def saveModels(self):
        self.melody.save('melody.h5')
        self.rhythm.save('rhythm.h5')

           <commit_msg>Update the MelodyGenerating part<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Kay sessionstore.

:Copyright: (c) 2009 Takashi Matsuo <tmatsuo@candit.jp> All rights reserved.
:license: BSD, see LICENSE for more details.
"""

import base64
import datetime
import cPickle as pickle

from werkzeug.contrib import (
  sessions, securecookie
)
from google.appengine.ext import db
from google.appengine.api import memcache

from kay.conf import settings
from kay.utils.decorators import retry_on_timeout
from models import GAESession

class GAESessionStore(sessions.SessionStore):
  """
  A session store class with GAE Datastore backend.
  """
  def __init__(self, session_class=None, session_prefix=None):
    self.session_prefix = session_prefix or settings.SESSION_PREFIX
    super(GAESessionStore, self).__init__(session_class)

  def get_key_name(self, sid):
    return '%s:%s' %(self.session_prefix ,sid)

  @retry_on_timeout(retries=5, secs=0.2)
  def save_to_db(self, key_name, session):
    gae_session = GAESession(
      key_name = key_name,
      data = self.encode(dict(session)),
      expire_date = (datetime.datetime.now() +
                     datetime.timedelta(seconds=settings.COOKIE_AGE))
    )
    gae_session.put()
    return gae_session

  def save(self, session):
    key_name = self.get_key_name(session.sid)
    gae_session = self.save_to_db(key_name, session)
    memcache.set(key_name, gae_session, settings.SESSION_MEMCACHE_AGE)

  @retry_on_timeout(retries=5, secs=0.2)
  def delete(self, session):
    s = GAESession.get_by_key_name(self.get_key_name(session.sid))
    if s:
      s.delete()

  @retry_on_timeout(retries=5, secs=0.2)
  def get(self, sid):
    key_name = self.get_key_name(sid)
    s = memcache.get(key_name)
    if s is None:
      s = GAESession.get_by_key_name(key_name)
    if not self.is_valid_key(sid) or s is None:
      return self.new()
    else:
      data = self.decode(s.data)
    return self.session_class(data, sid, False)

  def encode(self, session):
    "Returns the given session instance pickled and encoded as a string."
    pickled = pickle.dumps(session, pickle.HIGHEST_PROTOCOL)
    return base64.encodestring(pickled)

  def decode(self, session_data):
    encoded_data = base64.decodestring(session_data)
    try:
      return pickle.loads(encoded_data)
    # Unpickling can cause a variety of exceptions. If something happens,
    # just return an empty dictionary (an empty session).
    except Exception:
      return {}

  def get_data(self, session):
    return session.sid

class SecureCookieSessionStore(object):
  """
  A session store class with GAE Datastore backend.
  """
  def __init__(self, session_prefix=None, secret_key=None):
    self.session_prefix = session_prefix or settings.SESSION_PREFIX
    self.secret_key = secret_key or settings.SECRET_KEY

  def new(self):
    return securecookie.SecureCookie(secret_key=self.secret_key)

  def get(self, data):
    return securecookie.SecureCookie.unserialize(data, self.secret_key)

  def save(self, session):
    pass

  def get_data(self, session):
    return session.serialize()

  def delete(self, session):
    return True
<commit_msg>Catch db.BadValueError.<commit_after># -*- coding: utf-8 -*-

"""
Kay sessionstore.

:Copyright: (c) 2009 Takashi Matsuo <tmatsuo@candit.jp> All rights reserved.
:license: BSD, see LICENSE for more details.
"""

import logging
import base64
import datetime
import cPickle as pickle

from werkzeug.contrib import (
  sessions, securecookie
)
from google.appengine.ext import db
from google.appengine.api import memcache

from kay.conf import settings
from kay.utils.decorators import retry_on_timeout
from models import GAESession

class GAESessionStore(sessions.SessionStore):
  """
  A session store class with GAE Datastore backend.
  """
  def __init__(self, session_class=None, session_prefix=None):
    self.session_prefix = session_prefix or settings.SESSION_PREFIX
    super(GAESessionStore, self).__init__(session_class)

  def get_key_name(self, sid):
    return '%s:%s' %(self.session_prefix ,sid)

  @retry_on_timeout(retries=5, secs=0.2)
  def save_to_db(self, key_name, session):
    gae_session = GAESession(
      key_name = key_name,
      data = self.encode(dict(session)),
      expire_date = (datetime.datetime.now() +
                     datetime.timedelta(seconds=settings.COOKIE_AGE))
    )
    gae_session.put()
    return gae_session

  def save(self, session):
    key_name = self.get_key_name(session.sid)
    gae_session = self.save_to_db(key_name, session)
    memcache.set(key_name, gae_session, settings.SESSION_MEMCACHE_AGE)

  @retry_on_timeout(retries=5, secs=0.2)
  def delete(self, session):
    s = GAESession.get_by_key_name(self.get_key_name(session.sid))
    if s:
      s.delete()

  @retry_on_timeout(retries=5, secs=0.2)
  def get(self, sid):
    key_name = self.get_key_name(sid)
    s = memcache.get(key_name)
    if s is None:
      try:
        s = GAESession.get_by_key_name(key_name)
      except db.BadValueError, e:
        logging.warn("get_by_key_name failed with db.BadValueError: %s" % e)
    if not self.is_valid_key(sid) or s is None:
      return self.new()
    else:
      data = self.decode(s.data)
    return self.session_class(data, sid, False)

  def encode(self, session):
    "Returns the given session instance pickled and encoded as a string."
    pickled = pickle.dumps(session, pickle.HIGHEST_PROTOCOL)
    return base64.encodestring(pickled)

  def decode(self, session_data):
    encoded_data = base64.decodestring(session_data)
    try:
      return pickle.loads(encoded_data)
    # Unpickling can cause a variety of exceptions. If something happens,
    # just return an empty dictionary (an empty session).
    except Exception:
      return {}

  def get_data(self, session):
    return session.sid

class SecureCookieSessionStore(object):
  """
  A session store class with GAE Datastore backend.
  """
  def __init__(self, session_prefix=None, secret_key=None):
    self.session_prefix = session_prefix or settings.SESSION_PREFIX
    self.secret_key = secret_key or settings.SECRET_KEY

  def new(self):
    return securecookie.SecureCookie(secret_key=self.secret_key)

  def get(self, data):
    return securecookie.SecureCookie.unserialize(data, self.secret_key)

  def save(self, session):
    pass

  def get_data(self, session):
    return session.serialize()

  def delete(self, session):
    return True
<|endoftext|>


<commit_before>from __future__ import division
import numpy as np
import netCDF4
from collections import OrderedDict
from kid_readout.analysis.resonator import Resonator

class Dataset(object):

    def __init__(self, filename):
        self.ds = netCDF4.Dataset(filename)

    def sweep(self, sweep_name):
        """
        Return the netCDF4.Group object corresponding to sweep_name.
        """
        match = [group for name, group in self.ds.groups['sweeps'].groups.items()
                 if sweep_name in name]
        if not match:
            raise ValueError("No sweep names contain {0}".format(sweep_name))
        elif len(match) > 1:
            raise ValueError("Multiple sweep names contain {0}".format(sweep_name))
        else:
            return match.pop()

    def sweep_names(self):
        return self.ds.groups['sweeps'].groups.keys()

class CoarseSweep(object):
    """
    This interface is going to change soon -- it will figure out
    everything from the netCDF group, like the FineSweep.
    """
    def __init__(self, f, s21, coarse_resolution, n_subsweeps):
        self.f = f
        self.s21 = s21
        self.coarse_resolution = coarse_resolution
        self.n_subsweeps = n_subsweeps
        self.resolution = self.coarse_resolution / self.n_subsweeps

    def nth_subsweep(self, n):
        f_int = np.round((self.f-self.f[0])/self.resolution)
        mask = (f_int % self.n_subsweeps == n)
        return mask

    def subsweep_derivative_rms(self):
        return np.array([np.std(np.diff(np.abs(self.s21[self.nth_subsweep(n)])) /
                                np.diff(self.f[self.nth_subsweep(n)]))
                         for n in range(self.n_subsweeps)])

class FineSweep(OrderedDict):
    """
    A container class for a fine sweep.

    This class assumes that the frequency data is in MHz and converts
    it to Hz.

    Parameters:
    group -- a netCDF Group that contains fine sweep data.

    Keywords:
    fit_subsweeps -- if True, the class will attempt to fit the data
    correponding to each index using the current resonator defaults.
    """

    def __init__(self, group, fit_subsweeps=False):
        super(FineSweep, self).__init__()
        self.group = group
        self.f = 1e6 * group.variables['frequency'][:]
        self.s21 = group.variables['s21'][:].view('complex128')
        # This allows the class to load old data without the subsweep
        # index. Use build_index() instead.
        try:
            self.index = group.variables['index'][:]
        except KeyError:
            pass
        if fit_subsweeps:
            self.fit_subsweeps()

    def fit_subsweeps(self, deglitch=True, threshold=2):
        """
        Attempt to fit the data corresponding to each value of the
        index by instantiating a Resonator object with default
        settings. If *deglitch* is true then points identified as
        glitches with be masked.
        """
        for index in np.unique(self.index):
            mask = self.index == index
            f = self.f[mask]
            s21 = self.s21[mask]
            if deglitch:
                good = ~self.glitch(s21, threshold)
            else:
                good = np.ones_like(s21).astype('bool')
            self[index] = Resonator(f, s21, mask=good)

    # Replace this with a method that uses gaps in the frequencies.
    def build_index(self, f_0, span):
        """
        Build the index for old data that doesn't have an index. This
        works well if the subsweeps don't overlap, but fails if they
        do. Data outside all ranges f_0 +/- span will fall into index
        -1, which is a sign that the frequencies or span should be
        adjusted.
        """
        self.index = -np.ones_like(self.f, dtype=np.int)
        for n, f in enumerate(f_0):
            self.index[np.abs(f - self.f) <= span] = n

    def glitch(self, data, threshold):
        """
        Return a boolean array with the same size as the data that is
        True where the absolute value of the second derivative is at
        least *threshold* standard deviations above the mean.
        """
        # This is a poor man's second derivative.
        k = np.array([1, -2, 1])
        x = np.convolve(k, abs(data), mode='same')
        # This supuriously flags the (k.size - )/2 points at each end
        # due to zero padding; it also flags one point on either side
        # of a single-point glitch.
        glitch = abs(x) > threshold * np.std(x)
        glitch[:(k.size - 1) / 2] = glitch[-(k.size - 1) / 2:] = False
        return glitch
<commit_msg>Frequency scaling now an option in dataset.py<commit_after>from __future__ import division
import numpy as np
import netCDF4
from collections import OrderedDict
from kid_readout.analysis.resonator import Resonator

class Dataset(object):

    def __init__(self, filename):
        self.ds = netCDF4.Dataset(filename)

    def sweep(self, sweep_name):
        """
        Return the netCDF4.Group object corresponding to sweep_name.
        """
        match = [group for name, group in self.ds.groups['sweeps'].groups.items()
                 if sweep_name in name]
        if not match:
            raise ValueError("No sweep names contain {0}".format(sweep_name))
        elif len(match) > 1:
            raise ValueError("Multiple sweep names contain {0}".format(sweep_name))
        else:
            return match.pop()

    def sweep_names(self):
        return self.ds.groups['sweeps'].groups.keys()

class CoarseSweep(object):
    """
    This interface is going to change soon -- it will figure out
    everything from the netCDF group, like the FineSweep.
    """
    def __init__(self, f, s21, coarse_resolution, n_subsweeps):
        self.f = f
        self.s21 = s21
        self.coarse_resolution = coarse_resolution
        self.n_subsweeps = n_subsweeps
        self.resolution = self.coarse_resolution / self.n_subsweeps

    def nth_subsweep(self, n):
        f_int = np.round((self.f-self.f[0])/self.resolution)
        mask = (f_int % self.n_subsweeps == n)
        return mask

    def subsweep_derivative_rms(self):
        return np.array([np.std(np.diff(np.abs(self.s21[self.nth_subsweep(n)])) /
                                np.diff(self.f[self.nth_subsweep(n)]))
                         for n in range(self.n_subsweeps)])

class FineSweep(OrderedDict):
    """
    A container class for a fine sweep.

    Parameters:
    group -- a netCDF Group that contains fine sweep data.

    Keywords:
    fit_subsweeps -- if True, the class will attempt to fit the data
    correponding to each index using the current resonator defaults.
    scale_frequency -- frequency values from the data are multiplied
    by this number; the default of 1e6 assumes that the frequency is
    stored in MHz and converts it to Hz.
    """

    def __init__(self, group, fit_subsweeps=False, scale_frequency=1e6):
        super(FineSweep, self).__init__()
        self.group = group
        self.f = scale_frequency * group.variables['frequency'][:]
        self.s21 = group.variables['s21'][:].view('complex128')
        # This allows the class to load old data without the subsweep
        # index. Use build_index() instead.
        try:
            self.index = group.variables['index'][:]
        except KeyError:
            pass
        if fit_subsweeps:
            self.fit_subsweeps()

    def fit_subsweeps(self, deglitch=True, threshold=2):
        """
        Attempt to fit the data corresponding to each value of the
        index by instantiating a Resonator object with default
        settings. If *deglitch* is true then points identified as
        glitches with be masked.
        """
        for index in np.unique(self.index):
            mask = self.index == index
            f = self.f[mask]
            s21 = self.s21[mask]
            if deglitch:
                good = ~self.glitch(s21, threshold)
            else:
                good = np.ones_like(s21).astype('bool')
            self[index] = Resonator(f, s21, mask=good)

    # Replace this with a method that uses gaps in the frequencies.
    def build_index(self, f_0, span):
        """
        Build the index for old data that doesn't have an index. This
        works well if the subsweeps don't overlap, but fails if they
        do. Data outside all ranges f_0 +/- span will fall into index
        -1, which is a sign that the frequencies or span should be
        adjusted.
        """
        self.index = -np.ones_like(self.f, dtype=np.int)
        for n, f in enumerate(f_0):
            self.index[np.abs(f - self.f) <= span] = n

    def glitch(self, data, threshold):
        """
        Return a boolean array with the same size as the data that is
        True where the absolute value of the second derivative is at
        least *threshold* standard deviations above the mean.
        """
        # This is a poor man's second derivative.
        k = np.array([1, -2, 1])
        x = np.convolve(k, abs(data), mode='same')
        # This supuriously flags the (k.size - )/2 points at each end
        # due to zero padding; it also flags one point on either side
        # of a single-point glitch.
        glitch = abs(x) > threshold * np.std(x)
        glitch[:(k.size - 1) / 2] = glitch[-(k.size - 1) / 2:] = False
        return glitch
<|endoftext|>


<commit_before>from auth_helper import *
from output_helper import *

# You must provide credentials in auth_helper.py.

# Summary
# This example demonstrates how to get keyword ideas and traffic estimates for search advertising campaigns.

def output_status_message(message):
    print(message)

def output_array_of_long(items):
    if items is None or len(items) == 0:
        return
    output_status_message("Array Of long:")
    for item in items['long']:
        output_status_message("Value of the long: {0}".format(item))

def output_keywordidea(data_object):
    if data_object is None:
        return
    output_status_message("KeywordIdea (Data Object):")
    output_status_message("AdGroupId: {0}".format(data_object.AdGroupId))
    output_status_message("AdGroupName: {0}".format(data_object.AdGroupName))
    output_status_message("AdImpressionShare: {0}".format(data_object.AdImpressionShare))
    output_status_message("Competition: {0}".format(data_object.Competition))
    output_status_message("Keyword: {0}".format(data_object.Keyword))
    output_status_message("MonthlySearchCounts (Element Name):")
    output_array_of_long(data_object.MonthlySearchCounts)
    output_status_message("Relevance: {0}".format(data_object.Relevance))
    output_status_message("Source: {0}".format(data_object.Source))
    output_status_message("SuggestedBid: {0}".format(data_object.SuggestedBid))

def output_array_of_keywordidea(data_objects):
    if data_objects is None or len(data_objects) == 0:
        return
    output_status_message("Array Of KeywordIdea:\n")
    for data_object in data_objects['KeywordIdea']:
        output_keywordidea(data_object)
        output_status_message("\n")

def main(authorization_data):

    try:

        getkeywordideacategories_response = adinsight_service.GetKeywordIdeaCategories()

        # You must specify the attributes that you want in each returned KeywordIdea.

        ideas_attributes = adinsight_service.factory.create('ns2:ArrayOfKeywordIdeaAttribute')
        ideas_attributes.KeywordIdeaAttribute.append([
            'AdGroupId',
            'AdGroupName',
            'AdImpressionShare',
            'Competition',
            'Keyword',
            'MonthlySearchCounts',
            'Relevance',
            'Source',
            'SuggestedBid'
        ])

        # Only one of each SearchParameter type can be specified per call.

        search_parameters = adinsight_service.factory.create('ns4:ArrayOfSearchParameter')
        '''
        DateRangeSearchParameter
        Determines the start and end month for MonthlySearchCounts data returned with each KeywordIdea.
        The date range search parameter is optional. If you do not include the DateRangeSearchParameter
        in the GetKeywordIdeas request, then you will not be able to confirm whether the first list item
        within MonthlySearchCounts is data for the previous month, or the month prior. If the date range is
        specified and the most recent month's data is not yet available, then GetKeywordIdeas will return an error.
        '''

        date_range_search_parameter = adinsight_service.factory.create('ns4:DateRangeSearchParameter')
        end_date = adinsight_service.factory.create('ns1:DayMonthAndYear')
        end_date.Day = 31
        end_date.Month = 12
        end_date.Year = int(strftime("%Y", gmtime()))-1

        start_date = adinsight_service.factory.create('ns1:DayMonthAndYear')
        start_date.Day = 1
        start_date.Month = 1
        start_date.Year = int(strftime("%Y", gmtime()))-1

        date_range_search_parameter.EndDate = end_date
        date_range_search_parameter.StartDate = start_date

        '''
        The CategorySearchParameter corresponds to filling in 'Your product category' under
        'Search for new keywords using a phrase, website, or category' in the
        Bing Ads web application's Keyword Planner tool.
        One or more CategorySearchParameter, QuerySearchParameter, or UrlSearchParameter is required.
        '''

        category_search_parameter = adinsight_service.factory.create('ns4:CategorySearchParameter')
        # Use the GetKeywordIdeaCategories operation to get a list of valid category identifiers.
        category_search_parameter.CategoryId = getkeywordideacategories_response['KeywordIdeaCategory'][0].CategoryId

        '''
        The QuerySearchParameter corresponds to filling in 'Product or service' under
        'Search for new keywords using a phrase, website, or category' in the
        Bing Ads web application's Keyword Planner tool.
        One or more CategorySearchParameter, QuerySearchParameter, or UrlSearchParameter is required.
        When calling GetKeywordIdeas, if ExpandIdeas = false the QuerySearchParameter is required.
        '''

        query_search_parameter = adinsight_service.factory.create('ns4:QuerySearchParameter')
        queries = adinsight_service.factory.create('ns7:ArrayOfstring')
        queries.string.append([
            'tennis',
            'tennis shoes',
            'running',
            'running shoes',
            'cross training',
            'running'])
        query_search_parameter.Queries = queries

        '''
        UrlSearchParameter
        The UrlSearchParameter corresponds to filling in 'Your landing page' under
        'Search for new keywords using a phrase, website, or category' in the
        Bing Ads web application's Keyword Planner tool.
        One or more CategorySearchParameter, QuerySearchParameter, or UrlSearchParameter is required.
        '''

        url_search_parameter = adinsight_service.factory.create('ns4:UrlSearchParameter')
        url_search_parameter.Url = 'contoso.com'

        '''
        The LanguageSearchParameter, LocationSearchParameter, and NetworkSearchParameter
        correspond to the 'Keyword Planner' -> 'Search for new keywords using a phrase, website, or category' ->
        'Targeting' workflow in the Bing Ads web application.
        Each of these search parameters are required.
        '''

        language_search_parameter = adinsight_service.factory.create('ns4:LanguageSearchParameter')
        languages = adinsight_service.factory.create('ns3:ArrayOfLanguageCriterion')
        language = adinsight_service.factory.create('ns3:LanguageCriterion')
        # You must specify exactly one language
        language.Language = 'English'
        languages.LanguageCriterion.append([language])
        language_search_parameter.Languages = languages

        location_search_parameter = adinsight_service.factory.create('ns4:LocationSearchParameter')
        locations = adinsight_service.factory.create('ns3:ArrayOfLocationCriterion')
        # You must specify between 1 and 100 locations
        location = adinsight_service.factory.create('ns3:LocationCriterion')
        # United States
        location.LocationId = '190'
        locations.LocationCriterion.append([location])
        location_search_parameter.Locations = locations

        network_search_parameter = adinsight_service.factory.create('ns4:NetworkSearchParameter')
        network = adinsight_service.factory.create('ns3:NetworkCriterion')
        network.Network = 'OwnedAndOperatedAndSyndicatedSearch'
        network_search_parameter.Network = network

        '''
        The CompetitionSearchParameter, ExcludeAccountKeywordsSearchParameter, IdeaTextSearchParameter,
        ImpressionShareSearchParameter, SearchVolumeSearchParameter, and SuggestedBidSearchParameter
        correspond to the 'Keyword Planner' -> 'Search for new keywords using a phrase, website, or category' ->
        'Search options' workflow in the Bing Ads web application.
        Use these options to refine what keywords we suggest. You can limit the keywords by historical data,
        hide keywords already in your account, and include or exclude specific keywords.
        Each of these search parameters are optional.
        '''

        competition_search_parameter = adinsight_service.factory.create('ns4:CompetitionSearchParameter')
        competition_levels = adinsight_service.factory.create('ns2:ArrayOfCompetitionLevel')
        competition_levels.CompetitionLevel.append([
            'High',
            'Medium',
            'Low'])
        competition_search_parameter.CompetitionLevels = competition_levels

        # ExcludeAccountKeywordsSearchParameter
        exclude_account_keyword_search_parameter = adinsight_service.factory.create('ns4:ExcludeAccountKeywordsSearchParameter')
        exclude_account_keyword_search_parameter.ExcludeAccountKeywords = False

        # IdeaTextSearchParameter
        idea_text_search_parameter = adinsight_service.factory.create('ns4:IdeaTextSearchParameter')
        # The match type is required.Only Broad is supported.
        excluded_list = adinsight_service.factory.create('ns2:ArrayOfKeyword')
        excluded_keyword = adinsight_service.factory.create('ns2:Keyword')
        excluded_keyword.MatchType = 'Broad'
        excluded_keyword.Text = 'tennis court'
        excluded_list.Keyword.append([excluded_keyword])

        included_list = adinsight_service.factory.create('ns2:ArrayOfKeyword')
        included_keyword = adinsight_service.factory.create('ns2:Keyword')
        included_keyword.MatchType = 'Broad'
        included_keyword.Text = 'athletic clothing'
        included_list.Keyword.append([included_keyword])

        idea_text_search_parameter.Excluded = excluded_list
        idea_text_search_parameter.Included = included_list

        # ImpressionShareSearchParameter
        impression_share_search_parameter = adinsight_service.factory.create('ns4:ImpressionShareSearchParameter')
        impression_share_search_parameter.Maximum = '50'
        impression_share_search_parameter.Minimum = '0'

        # SearchVolumeSearchParameter
        search_volume_search_parameter = adinsight_service.factory.create('ns4:SearchVolumeSearchParameter')
        search_volume_search_parameter.Maximum = None
        search_volume_search_parameter.Minimum = '50'

        # SuggestedBidSearchParameter
        suggested_bid_search_parameter = adinsight_service.factory.create('ns4:SuggestedBidSearchParameter')
        suggested_bid_search_parameter.Maximum = '1000'
        suggested_bid_search_parameter.Minimum = '4'

        '''
        Setting the device criterion is not available in the
        'Keyword Planner' -> 'Search for new keywords using a phrase, website, or category'
        workflow in the Bing Ads web application.
        The DeviceSearchParameter is optional and by default the keyword ideas data
        are aggregated for all devices.
        '''

        device_search_parameter = adinsight_service.factory.create('ns4:DeviceSearchParameter')
        device = adinsight_service.factory.create('ns3:DeviceCriterion')
        # Possible values are All, Computers, Tablets, Smartphones
        device.DeviceName = 'All'
        device_search_parameter.Device = device

        # Populate ArrayOfSearchParameter

        search_parameters.SearchParameter.append([
            network_search_parameter,
            query_search_parameter,
            language_search_parameter,
            location_search_parameter,
            url_search_parameter,
            category_search_parameter,
            search_volume_search_parameter,
            suggested_bid_search_parameter,
            exclude_account_keyword_search_parameter,
            impression_share_search_parameter,
            device_search_parameter,
            idea_text_search_parameter,
            date_range_search_parameter,
            competition_search_parameter])
        # Call the GetKeywordIdeas Function
        # If ExpandIdeas is false, the QuerySearchParameter is required.

        get_keyword_ideas_response=adinsight_service.GetKeywordIdeas(
            IdeaAttributes=ideas_attributes,
            SearchParameters=search_parameters,
            ExpandIdeas= True
#/ false
        )

        output_array_of_keywordidea(get_keyword_ideas_response)
        output_status_message("Program execution completed")

    except WebFault as ex:

        output_webfault_errors(ex)

    except Exception as ex:

        output_status_message(ex)

# Main execution

if __name__ == '__main__':

    authorization_data=AuthorizationData(
        account_id=None,
        customer_id=None,
        developer_token=DEVELOPER_TOKEN,
        authentication=None,
    )

    adinsight_service=ServiceClient(
        service='AdInsightService',
        authorization_data=authorization_data,
        environment=ENVIRONMENT,
        version=11
    )

    # You should authenticate for Bing Ads production services with a Microsoft Account,
    # instead of providing the Bing Ads username and password set.
    # Authentication with a Microsoft Account is currently not supported in Sandbox.

    authenticate(authorization_data)

    main(authorization_data)
<commit_msg>Keyword Planner - Update comments<commit_after>from auth_helper import *
from output_helper import *

# You must provide credentials in auth_helper.py.

# Summary
# This example demonstrates how to get keyword ideas and traffic estimates for search advertising campaigns.

def output_status_message(message):
    print(message)

def output_array_of_long(items):
    if items is None or len(items) == 0:
        return
    output_status_message("Array Of long:")
    for item in items['long']:
        output_status_message("Value of the long: {0}".format(item))

def output_keywordidea(data_object):
    if data_object is None:
        return
    output_status_message("KeywordIdea (Data Object):")
    output_status_message("AdGroupId: {0}".format(data_object.AdGroupId))
    output_status_message("AdGroupName: {0}".format(data_object.AdGroupName))
    output_status_message("AdImpressionShare: {0}".format(data_object.AdImpressionShare))
    output_status_message("Competition: {0}".format(data_object.Competition))
    output_status_message("Keyword: {0}".format(data_object.Keyword))
    output_status_message("MonthlySearchCounts (Element Name):")
    output_array_of_long(data_object.MonthlySearchCounts)
    output_status_message("Relevance: {0}".format(data_object.Relevance))
    output_status_message("Source: {0}".format(data_object.Source))
    output_status_message("SuggestedBid: {0}".format(data_object.SuggestedBid))

def output_array_of_keywordidea(data_objects):
    if data_objects is None or len(data_objects) == 0:
        return
    output_status_message("Array Of KeywordIdea:\n")
    for data_object in data_objects['KeywordIdea']:
        output_keywordidea(data_object)
        output_status_message("\n")

def main(authorization_data):

    try:

        getkeywordideacategories_response = adinsight_service.GetKeywordIdeaCategories()

        # You must specify the attributes that you want in each returned KeywordIdea.

        ideas_attributes = adinsight_service.factory.create('ns2:ArrayOfKeywordIdeaAttribute')
        ideas_attributes.KeywordIdeaAttribute.append([
            'AdGroupId',
            'AdGroupName',
            'AdImpressionShare',
            'Competition',
            'Keyword',
            'MonthlySearchCounts',
            'Relevance',
            'Source',
            'SuggestedBid'
        ])

        # Only one of each SearchParameter type can be specified per call.

        search_parameters = adinsight_service.factory.create('ns4:ArrayOfSearchParameter')
        '''
        DateRangeSearchParameter
        Determines the start and end month for MonthlySearchCounts data returned with each KeywordIdea.
        The date range search parameter is optional. If you do not include the DateRangeSearchParameter
        in the GetKeywordIdeas request, then you will not be able to confirm whether the first list item
        within MonthlySearchCounts is data for the previous month, or the month prior. If the date range is
        specified and the most recent month's data is not yet available, then GetKeywordIdeas will return an error.
        '''

        date_range_search_parameter = adinsight_service.factory.create('ns4:DateRangeSearchParameter')
        end_date = adinsight_service.factory.create('ns1:DayMonthAndYear')
        end_date.Day = 31
        end_date.Month = 12
        end_date.Year = int(strftime("%Y", gmtime()))-1

        start_date = adinsight_service.factory.create('ns1:DayMonthAndYear')
        start_date.Day = 1
        start_date.Month = 1
        start_date.Year = int(strftime("%Y", gmtime()))-1

        date_range_search_parameter.EndDate = end_date
        date_range_search_parameter.StartDate = start_date

        '''
        The CategorySearchParameter corresponds to filling in 'Your product category' under
        'Search for new keywords using a phrase, website, or category' in the
        Bing Ads web application's Keyword Planner tool.
        One or more CategorySearchParameter, QuerySearchParameter, or UrlSearchParameter is required.
        '''

        category_search_parameter = adinsight_service.factory.create('ns4:CategorySearchParameter')
        # Use the GetKeywordIdeaCategories operation to get a list of valid category identifiers.
        category_search_parameter.CategoryId = getkeywordideacategories_response['KeywordIdeaCategory'][0].CategoryId

        '''
        The QuerySearchParameter corresponds to filling in 'Product or service' under
        'Search for new keywords using a phrase, website, or category' in the
        Bing Ads web application's Keyword Planner tool.
        One or more CategorySearchParameter, QuerySearchParameter, or UrlSearchParameter is required.
        When calling GetKeywordIdeas, if ExpandIdeas = false the QuerySearchParameter is required.
        '''

        query_search_parameter = adinsight_service.factory.create('ns4:QuerySearchParameter')
        queries = adinsight_service.factory.create('ns7:ArrayOfstring')
        queries.string.append([
            'tennis',
            'tennis shoes',
            'running',
            'running shoes',
            'cross training',
            'running'])
        query_search_parameter.Queries = queries

        '''
        UrlSearchParameter
        The UrlSearchParameter corresponds to filling in 'Your landing page' under
        'Search for new keywords using a phrase, website, or category' in the
        Bing Ads web application's Keyword Planner tool.
        One or more CategorySearchParameter, QuerySearchParameter, or UrlSearchParameter is required.
        '''

        url_search_parameter = adinsight_service.factory.create('ns4:UrlSearchParameter')
        url_search_parameter.Url = 'contoso.com'

        '''
        The LanguageSearchParameter, LocationSearchParameter, and NetworkSearchParameter
        correspond to the 'Keyword Planner' -> 'Search for new keywords using a phrase, website, or category' ->
        'Targeting' workflow in the Bing Ads web application.
        Each of these search parameters are required.
        '''

        language_search_parameter = adinsight_service.factory.create('ns4:LanguageSearchParameter')
        languages = adinsight_service.factory.create('ns3:ArrayOfLanguageCriterion')
        language = adinsight_service.factory.create('ns3:LanguageCriterion')
        # You must specify exactly one language
        language.Language = 'English'
        languages.LanguageCriterion.append([language])
        language_search_parameter.Languages = languages

        location_search_parameter = adinsight_service.factory.create('ns4:LocationSearchParameter')
        locations = adinsight_service.factory.create('ns3:ArrayOfLocationCriterion')
        # You must specify between 1 and 100 locations
        location = adinsight_service.factory.create('ns3:LocationCriterion')
        # United States
        location.LocationId = '190'
        locations.LocationCriterion.append([location])
        location_search_parameter.Locations = locations

        network_search_parameter = adinsight_service.factory.create('ns4:NetworkSearchParameter')
        network = adinsight_service.factory.create('ns3:NetworkCriterion')
        network.Network = 'OwnedAndOperatedAndSyndicatedSearch'
        network_search_parameter.Network = network

        '''
        The CompetitionSearchParameter, ExcludeAccountKeywordsSearchParameter, IdeaTextSearchParameter,
        ImpressionShareSearchParameter, SearchVolumeSearchParameter, and SuggestedBidSearchParameter
        correspond to the 'Keyword Planner' -> 'Search for new keywords using a phrase, website, or category' ->
        'Search options' workflow in the Bing Ads web application.
        Use these options to refine what keywords we suggest. You can limit the keywords by historical data,
        hide keywords already in your account, and include or exclude specific keywords.
        Each of these search parameters are optional.
        '''

        competition_search_parameter = adinsight_service.factory.create('ns4:CompetitionSearchParameter')
        competition_levels = adinsight_service.factory.create('ns2:ArrayOfCompetitionLevel')
        competition_levels.CompetitionLevel.append([
            'High',
            'Medium',
            'Low'])
        competition_search_parameter.CompetitionLevels = competition_levels

        # ExcludeAccountKeywordsSearchParameter
        exclude_account_keyword_search_parameter = adinsight_service.factory.create('ns4:ExcludeAccountKeywordsSearchParameter')
        exclude_account_keyword_search_parameter.ExcludeAccountKeywords = False

        # IdeaTextSearchParameter
        idea_text_search_parameter = adinsight_service.factory.create('ns4:IdeaTextSearchParameter')
        # The match type is required.Only Broad is supported.
        excluded_list = adinsight_service.factory.create('ns2:ArrayOfKeyword')
        excluded_keyword = adinsight_service.factory.create('ns2:Keyword')
        excluded_keyword.MatchType = 'Broad'
        excluded_keyword.Text = 'tennis court'
        excluded_list.Keyword.append([excluded_keyword])

        included_list = adinsight_service.factory.create('ns2:ArrayOfKeyword')
        included_keyword = adinsight_service.factory.create('ns2:Keyword')
        included_keyword.MatchType = 'Broad'
        included_keyword.Text = 'athletic clothing'
        included_list.Keyword.append([included_keyword])

        idea_text_search_parameter.Excluded = excluded_list
        idea_text_search_parameter.Included = included_list

        # ImpressionShareSearchParameter
        impression_share_search_parameter = adinsight_service.factory.create('ns4:ImpressionShareSearchParameter')
        impression_share_search_parameter.Maximum = '50'
        impression_share_search_parameter.Minimum = '0'

        # SearchVolumeSearchParameter
        search_volume_search_parameter = adinsight_service.factory.create('ns4:SearchVolumeSearchParameter')
        search_volume_search_parameter.Maximum = None
        search_volume_search_parameter.Minimum = '50'

        # SuggestedBidSearchParameter
        suggested_bid_search_parameter = adinsight_service.factory.create('ns4:SuggestedBidSearchParameter')
        suggested_bid_search_parameter.Maximum = '1000'
        suggested_bid_search_parameter.Minimum = '4'

        '''
        Setting the device criterion is not available in the
        'Keyword Planner' -> 'Search for new keywords using a phrase, website, or category'
        workflow in the Bing Ads web application.
        The DeviceSearchParameter is optional and by default the keyword ideas data
        are aggregated for all devices.
        '''

        device_search_parameter = adinsight_service.factory.create('ns4:DeviceSearchParameter')
        device = adinsight_service.factory.create('ns3:DeviceCriterion')
        # Possible values are All, Computers, Tablets, Smartphones
        device.DeviceName = 'All'
        device_search_parameter.Device = device

        # Populate ArrayOfSearchParameter

        search_parameters.SearchParameter.append([
            network_search_parameter,
            query_search_parameter,
            language_search_parameter,
            location_search_parameter,
            url_search_parameter,
            category_search_parameter,
            search_volume_search_parameter,
            suggested_bid_search_parameter,
            exclude_account_keyword_search_parameter,
            impression_share_search_parameter,
            device_search_parameter,
            idea_text_search_parameter,
            date_range_search_parameter,
            competition_search_parameter])

        # Call the GetKeywordIdeas Function
        # If ExpandIdeas is false, the QuerySearchParameter is required.

        get_keyword_ideas_response=adinsight_service.GetKeywordIdeas(
            IdeaAttributes=ideas_attributes,
            SearchParameters=search_parameters,
            ExpandIdeas=True
        )

        output_array_of_keywordidea(get_keyword_ideas_response)
        output_status_message("Program execution completed")

    except WebFault as ex:

        output_webfault_errors(ex)

    except Exception as ex:

        output_status_message(ex)

# Main execution

if __name__ == '__main__':

    authorization_data=AuthorizationData(
        account_id=None,
        customer_id=None,
        developer_token=DEVELOPER_TOKEN,
        authentication=None,
    )

    adinsight_service=ServiceClient(
        service='AdInsightService',
        authorization_data=authorization_data,
        environment=ENVIRONMENT,
        version=11
    )

    # You should authenticate for Bing Ads production services with a Microsoft Account,
    # instead of providing the Bing Ads username and password set.
    # Authentication with a Microsoft Account is currently not supported in Sandbox.

    authenticate(authorization_data)

    main(authorization_data)
<|endoftext|>


<commit_before>"""KLEE test cases"""
# vim: set sw=4 ts=4 softtabstop=4 expandtab:

import os
import re
import glob
import logging
from collections import namedtuple
from ..exceptions import InputError

_logger = logging.getLogger(__name__)

def _force_match(regex, line, message, path):
    match = regex.fullmatch(line)
    if not match:
        raise InputError(message.format(path))
    return match

Early = namedtuple("Early", ["message"])
def _parse_early(path):
    """Load a .early file"""
    assert os.path.exists(os.path.dirname(path))
    try:
        with open(path) as file:
            return Early(file.readlines())
    except FileNotFoundError:
        return None

ErrorFile = namedtuple("ErrorFile", ["message", "file", "line", "assembly_line", "stack"])

_RE_ERROR = re.compile(r"Error: (.*)\r?\n")
_RE_FILE = re.compile(r"File: (.*)\r?\n")
_RE_LINE = re.compile(r"Line: (\d+)\r?\n")
_RE_ASSEMBLY_LINE = re.compile(r"assembly.ll line: (\d+)\r?\n")
_RE_ERROR_FILE = re.compile(r"^test(\d+)\.")
_RE_KTEST_FILE = re.compile(r"^(test(\d+))\.ktest$")

def _parse_error(path):
    assert os.path.exists(os.path.dirname(path))
    try:
        with open(path) as file:
            match = _force_match(_RE_ERROR, file.readline(), "{}: Invalid error message in line 1", path)
            message = match.group(1)
            match = _force_match(_RE_FILE, file.readline(), "{}: Invalid file in line 2", path)
            filename = match.group(1)
            match = _force_match(_RE_LINE, file.readline(), "{}: Invalid line number in line 3", path)
            line = int(match.group(1))
            match = _force_match(_RE_ASSEMBLY_LINE, file.readline(), "{}: Invalid assembly.ll line number in line 4", path)
            assline = int(match.group(1))
            if file.readline().rstrip() != "Stack:":
                raise InputError("{}: Invalid begin stacktrace stack in line 5".format(path))
            stack = file.readlines()
            return ErrorFile(message, filename, line, assline, stack)
    except FileNotFoundError:
        return None

class Test:
    """
    A KLEE test case

    Attributes:
        early -- early termination info (None if it did not happen)
        error -- execution error info (None if it did not happen)
        abort -- abortion error info (None if it did not happen)
        assertion -- assertion error info (None if it did not happen)
        division -- division error info (None if it did not happen)
    """
    def __str__(self):
        msg = "Test {\n"
        msg += "identifier: {},\n".format(self.identifier)
        msg += "type: \"{}\",\n".format(self.type_string)
        if self.early:
            msg += "Return early reason: \"{}\"\n".format(self.early.message)
        elif self.error:
            msg += "Error: {},\n".format(self.error)
        msg += "}\n"
        return msg

    @property
    def type_string(self):
        msg = "type: "
        if self.is_successful_termination:
            return "successful termination"
        if self.early:
            return "early termination"
        if self.execution_error:
            return "execution error"
        if self.abort:
            return "abort"
        if self.division:
            return "division by zero"
        if self.assertion:
            return "assertion failure"
        if self.free:
            return "use after free"
        if self.ptr:
            return "invalid pointer dereference"
        if self.overshift:
            return "overshift"
        if self.readonly_error:
            return "read only error"
        if self.user_error:
            return "user error"
        if self.overflow:
            return "integer overflow"
        if self.misc_error:
            return "misc error"
        raise Exception('Unhandled test type')

    def __init__(self, path: "path to ktest file"):
      # pylint: disable=too-many-branches
        """Load a KLEE test case"""
        if not path.endswith('.ktest'):
            raise Exception('path is not a ktest file')
        if not os.path.exists(path):
            raise Exception('{} does not exist'.format(path))

        # Get identifier and path stub
        self.ktest_file = os.path.abspath(path)
        _logger.debug('Creating test with path "{}"'.format(self.ktest_file))
        basename = os.path.basename(path)
        m = _RE_KTEST_FILE.match(basename)
        if m is None:
            raise Exception('Failed to match KTest file')
        self.__pathstub = m.group(1)
        assert self.__pathstub.startswith('test')
        self.identifier = int(m.group(2))
        assert self.identifier >= 0

        self.error = None
        self.execution_error = None
        self.abort = None
        self.division = None
        self.assertion = None
        self.free = None
        self.ptr = None
        self.overshift = None
        self.readonly_error = None
        self.user_error = None
        self.overflow = None
        self.misc_error = None

        klee_dir_path = os.path.dirname(path)
        assert os.path.exists(klee_dir_path)

        early_path = os.path.join(klee_dir_path, self.__pathstub) + ".early"
        self.early = _parse_early(early_path) # FIXME: Mutually exclusive?

        _logger.debug('klee_dir_path: "{}"'.format(klee_dir_path))
        error_file_map = Test._get_error_file_map_for(klee_dir_path)
        error_file_path = None
        try:
          error_file_path = error_file_map[self.identifier]
        except KeyError:
          # No error file
          pass

        if error_file_path is not None:
            error = os.path.join(klee_dir_path, error_file_path)
            if not os.path.exists(error):
              raise Exception('Error file "{}" does not exist'.format(error))
            self.error = _parse_error(error)
            error = error[:-4]
            error = error[error.rfind(".")+1:]
            if error == "abort":
                self.abort = self.error
            elif error == "assert":
                self.assertion = self.error
            elif error == "div":
                self.division = self.error
            elif error == "exec":
                self.execution_error = self.error
            elif error == "free":
                self.free = self.error
            elif error == "overflow":
                self.overflow = self.error
            elif error == "overshift":
                self.overshift = self.error
            elif error == "ptr":
                self.ptr = self.error
            elif error == "readonly":
                self.readonly_error = self.error
            elif error == "user":
                self.user_error = self.error
            else:
                self.misc_error = self.error

            # Sanity check
            if self.error:
                assert self.early is None

    @property
    def ktest_path(self):
        """Path to the matching .ktest file"""
        return self.__pathstub + ".ktest"

    @property
    def pc_path(self):
        """Path to the matching .pc file"""
        return self.__pathstub + ".pc"

    _error_file_map_cache = dict()
    @classmethod
    def _get_error_file_map_for(cls, path):
      """
        This returns a map from identifiers
        to error files for the particular
        `path` (a KLEE directory).

        This is essentially a cache which
        avoids traversing a KLEE directory
        multiple times.
      """
      # FIXME: There should be a lock on this!
      error_file_map = None
      try:
        return cls._error_file_map_cache[path]
      except KeyError:
        # This KLEE directory has not been visited before
        error_file_map = dict()
        cls._error_file_map_cache[path] = error_file_map

        # Initialise the map
        errorFiles = glob.glob(os.path.join(glob.escape(path),'test*.*.err'))
        for errorFileFullPath in errorFiles:
          # Get identifier from the file name
          basename = os.path.basename(errorFileFullPath)
          m = _RE_ERROR_FILE.match(basename)
          if m is None:
            raise Exception('Could not get identifier from test file name')
          identifier = int(m.group(1))
          _logger.debug("Adding mapping [{}] => \"{}\"".format(
            identifier,
            basename))
          if identifier in error_file_map:
            raise Exception("Identifier should not already be in the map")
          error_file_map[identifier] = basename

      return error_file_map

    @property
    def is_error(self):
      return self.error is not None

    @property
    def is_successful_termination(self):
      return (not self.is_error) and (self.early is None)
<commit_msg>When dumping string representation of a test show the absolute path to the ktest file.<commit_after>"""KLEE test cases"""
# vim: set sw=4 ts=4 softtabstop=4 expandtab:

import os
import re
import glob
import logging
from collections import namedtuple
from ..exceptions import InputError

_logger = logging.getLogger(__name__)

def _force_match(regex, line, message, path):
    match = regex.fullmatch(line)
    if not match:
        raise InputError(message.format(path))
    return match

Early = namedtuple("Early", ["message"])
def _parse_early(path):
    """Load a .early file"""
    assert os.path.exists(os.path.dirname(path))
    try:
        with open(path) as file:
            return Early(file.readlines())
    except FileNotFoundError:
        return None

ErrorFile = namedtuple("ErrorFile", ["message", "file", "line", "assembly_line", "stack"])

_RE_ERROR = re.compile(r"Error: (.*)\r?\n")
_RE_FILE = re.compile(r"File: (.*)\r?\n")
_RE_LINE = re.compile(r"Line: (\d+)\r?\n")
_RE_ASSEMBLY_LINE = re.compile(r"assembly.ll line: (\d+)\r?\n")
_RE_ERROR_FILE = re.compile(r"^test(\d+)\.")
_RE_KTEST_FILE = re.compile(r"^(test(\d+))\.ktest$")

def _parse_error(path):
    assert os.path.exists(os.path.dirname(path))
    try:
        with open(path) as file:
            match = _force_match(_RE_ERROR, file.readline(), "{}: Invalid error message in line 1", path)
            message = match.group(1)
            match = _force_match(_RE_FILE, file.readline(), "{}: Invalid file in line 2", path)
            filename = match.group(1)
            match = _force_match(_RE_LINE, file.readline(), "{}: Invalid line number in line 3", path)
            line = int(match.group(1))
            match = _force_match(_RE_ASSEMBLY_LINE, file.readline(), "{}: Invalid assembly.ll line number in line 4", path)
            assline = int(match.group(1))
            if file.readline().rstrip() != "Stack:":
                raise InputError("{}: Invalid begin stacktrace stack in line 5".format(path))
            stack = file.readlines()
            return ErrorFile(message, filename, line, assline, stack)
    except FileNotFoundError:
        return None

class Test:
    """
    A KLEE test case

    Attributes:
        early -- early termination info (None if it did not happen)
        error -- execution error info (None if it did not happen)
        abort -- abortion error info (None if it did not happen)
        assertion -- assertion error info (None if it did not happen)
        division -- division error info (None if it did not happen)
    """
    def __str__(self):
        msg = "Test {\n"
        msg += "ktest_file: {}".format(self.ktest_file)
        msg += "identifier: {},\n".format(self.identifier)
        msg += "type: \"{}\",\n".format(self.type_string)
        if self.early:
            msg += "Return early reason: \"{}\"\n".format(self.early.message)
        elif self.error:
            msg += "Error: {},\n".format(self.error)
        msg += "}\n"
        return msg

    @property
    def type_string(self):
        msg = "type: "
        if self.is_successful_termination:
            return "successful termination"
        if self.early:
            return "early termination"
        if self.execution_error:
            return "execution error"
        if self.abort:
            return "abort"
        if self.division:
            return "division by zero"
        if self.assertion:
            return "assertion failure"
        if self.free:
            return "use after free"
        if self.ptr:
            return "invalid pointer dereference"
        if self.overshift:
            return "overshift"
        if self.readonly_error:
            return "read only error"
        if self.user_error:
            return "user error"
        if self.overflow:
            return "integer overflow"
        if self.misc_error:
            return "misc error"
        raise Exception('Unhandled test type')

    def __init__(self, path: "path to ktest file"):
      # pylint: disable=too-many-branches
        """Load a KLEE test case"""
        if not path.endswith('.ktest'):
            raise Exception('path is not a ktest file')
        if not os.path.exists(path):
            raise Exception('{} does not exist'.format(path))

        # Get identifier and path stub
        self.ktest_file = os.path.abspath(path)
        _logger.debug('Creating test with path "{}"'.format(self.ktest_file))
        basename = os.path.basename(path)
        m = _RE_KTEST_FILE.match(basename)
        if m is None:
            raise Exception('Failed to match KTest file')
        self.__pathstub = m.group(1)
        assert self.__pathstub.startswith('test')
        self.identifier = int(m.group(2))
        assert self.identifier >= 0

        self.error = None
        self.execution_error = None
        self.abort = None
        self.division = None
        self.assertion = None
        self.free = None
        self.ptr = None
        self.overshift = None
        self.readonly_error = None
        self.user_error = None
        self.overflow = None
        self.misc_error = None

        klee_dir_path = os.path.dirname(path)
        assert os.path.exists(klee_dir_path)

        early_path = os.path.join(klee_dir_path, self.__pathstub) + ".early"
        self.early = _parse_early(early_path) # FIXME: Mutually exclusive?

        _logger.debug('klee_dir_path: "{}"'.format(klee_dir_path))
        error_file_map = Test._get_error_file_map_for(klee_dir_path)
        error_file_path = None
        try:
          error_file_path = error_file_map[self.identifier]
        except KeyError:
          # No error file
          pass

        if error_file_path is not None:
            error = os.path.join(klee_dir_path, error_file_path)
            if not os.path.exists(error):
              raise Exception('Error file "{}" does not exist'.format(error))
            self.error = _parse_error(error)
            error = error[:-4]
            error = error[error.rfind(".")+1:]
            if error == "abort":
                self.abort = self.error
            elif error == "assert":
                self.assertion = self.error
            elif error == "div":
                self.division = self.error
            elif error == "exec":
                self.execution_error = self.error
            elif error == "free":
                self.free = self.error
            elif error == "overflow":
                self.overflow = self.error
            elif error == "overshift":
                self.overshift = self.error
            elif error == "ptr":
                self.ptr = self.error
            elif error == "readonly":
                self.readonly_error = self.error
            elif error == "user":
                self.user_error = self.error
            else:
                self.misc_error = self.error

            # Sanity check
            if self.error:
                assert self.early is None

    @property
    def ktest_path(self):
        """Path to the matching .ktest file"""
        return self.__pathstub + ".ktest"

    @property
    def pc_path(self):
        """Path to the matching .pc file"""
        return self.__pathstub + ".pc"

    _error_file_map_cache = dict()
    @classmethod
    def _get_error_file_map_for(cls, path):
      """
        This returns a map from identifiers
        to error files for the particular
        `path` (a KLEE directory).

        This is essentially a cache which
        avoids traversing a KLEE directory
        multiple times.
      """
      # FIXME: There should be a lock on this!
      error_file_map = None
      try:
        return cls._error_file_map_cache[path]
      except KeyError:
        # This KLEE directory has not been visited before
        error_file_map = dict()
        cls._error_file_map_cache[path] = error_file_map

        # Initialise the map
        errorFiles = glob.glob(os.path.join(glob.escape(path),'test*.*.err'))
        for errorFileFullPath in errorFiles:
          # Get identifier from the file name
          basename = os.path.basename(errorFileFullPath)
          m = _RE_ERROR_FILE.match(basename)
          if m is None:
            raise Exception('Could not get identifier from test file name')
          identifier = int(m.group(1))
          _logger.debug("Adding mapping [{}] => \"{}\"".format(
            identifier,
            basename))
          if identifier in error_file_map:
            raise Exception("Identifier should not already be in the map")
          error_file_map[identifier] = basename

      return error_file_map

    @property
    def is_error(self):
      return self.error is not None

    @property
    def is_successful_termination(self):
      return (not self.is_error) and (self.early is None)
<|endoftext|>


<commit_before># Copyright (c) 2014, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

import stix
import stix.utils
import stix.indicator.test_mechanism
from stix.indicator.test_mechanism import _BaseTestMechanism
import stix.bindings.extensions.test_mechanism.open_ioc_2010 as open_ioc_tm_binding
from lxml import etree

class OpenIOCTestMechanism(_BaseTestMechanism):
    _namespace = "http://stix.mitre.org/extensions/TestMechanism#OpenIOC2010-1"
    _binding = open_ioc_tm_binding
    _binding_class = _binding.OpenIOC2010TestMechanismType
    _XSI_TYPE = "stix-openioc:OpenIOC2010TestMechanismType"

    def __init__(self, id_=None, idref=None):
        super(OpenIOCTestMechanism, self).__init__(id_=id_, idref=idref)
        self.ioc = None

    @property
    def ioc(self):
        return self._ioc

    @ioc.setter
    def ioc(self, value):
        if not value:
            self._ioc = None
        elif isinstance(value, etree._ElementTree):
            self._ioc = value
        elif isinstance(value, etree._Element):
            self._ioc = etree.ElementTree(value)
        else:
            raise ValueError('ioc must be instance of lxml.etree._Element or lxml.etree._ElementTree')

    @classmethod
    def from_obj(cls, obj, return_obj=None):
        if not obj:
            return None
        if not return_obj:
            return_obj = cls()

        super(OpenIOCTestMechanism, cls).from_obj(obj, return_obj)
        return_obj.ioc = obj.get_ioc()
        return return_obj

    def to_obj(self, return_obj=None):
        if not return_obj:
            return_obj = self._binding_class()

        super(OpenIOCTestMechanism, self).to_obj(return_obj)
        return_obj.set_ioc(self.ioc)
        return return_obj

    @classmethod
    def from_dict(cls, d, return_obj=None):
        if not d:
            return None
        if not return_obj:
            return_obj = cls()

        super(OpenIOCTestMechanism, cls).from_dict(d, return_obj)
        if 'ioc' in d:
            parser = etree.ETCompatXMLParser(huge_tree=True)
            return_obj.ioc = etree.parse(d['ioc'], parser=parser)

        return return_obj

stix.indicator.test_mechanism.add_extension(OpenIOCTestMechanism)
<commit_msg>Updated OpenIOC test mechanism extension, specifically the behavior of the ioc property setter<commit_after># Copyright (c) 2014, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

import stix
import stix.utils
import stix.indicator.test_mechanism
from stix.indicator.test_mechanism import _BaseTestMechanism
import stix.bindings.extensions.test_mechanism.open_ioc_2010 as open_ioc_tm_binding
from lxml import etree

class OpenIOCTestMechanism(_BaseTestMechanism):
    _namespace = "http://stix.mitre.org/extensions/TestMechanism#OpenIOC2010-1"
    _binding = open_ioc_tm_binding
    _binding_class = _binding.OpenIOC2010TestMechanismType
    _XSI_TYPE = "stix-openioc:OpenIOC2010TestMechanismType"

    def __init__(self, id_=None, idref=None):
        super(OpenIOCTestMechanism, self).__init__(id_=id_, idref=idref)
        self.ioc = None

    @property
    def ioc(self):
        return self._ioc

    @ioc.setter
    def ioc(self, value):
        if not value:
            self._ioc = None
            return
        elif isinstance(value, etree._ElementTree):
            tree = value
        elif isinstance(value, etree._Element):
            tree = etree.ElementTree(value)
        else:
            raise ValueError('ioc must be instance of lxml.etree._Element '
                             'or lxml.etree._ElementTree')

        root = tree.getroot()
        expected_node_tag = "{%s}ioc" % (self._namespace)
        if root.tag != expected_node_tag:
            ns_ioc = "http://schemas.mandiant.com/2010/ioc"
            node_ns = root.tag[1:].split("}")[0] if root.tag.startswith("{") else None
            if node_ns == ns_ioc:
                # attempt to cast
                root.tag = expected_node_tag
            else:
                raise ValueError("Cannot set ioc property. Expected tag %s found %s"
                                 % (expected_node_tag, root.tag))

        self._ioc = tree

    @classmethod
    def from_obj(cls, obj, return_obj=None):
        if not obj:
            return None
        if not return_obj:
            return_obj = cls()

        super(OpenIOCTestMechanism, cls).from_obj(obj, return_obj)
        return_obj.ioc = obj.get_ioc()
        return return_obj

    def to_obj(self, return_obj=None):
        if not return_obj:
            return_obj = self._binding_class()

        super(OpenIOCTestMechanism, self).to_obj(return_obj)
        return_obj.set_ioc(self.ioc)
        return return_obj

    @classmethod
    def from_dict(cls, d, return_obj=None):
        if not d:
            return None
        if not return_obj:
            return_obj = cls()

        super(OpenIOCTestMechanism, cls).from_dict(d, return_obj)
        if 'ioc' in d:
            parser = etree.ETCompatXMLParser(huge_tree=True)
            return_obj.ioc = etree.parse(d['ioc'], parser=parser)

        return return_obj

stix.indicator.test_mechanism.add_extension(OpenIOCTestMechanism)
<|endoftext|>