<commit_before># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import absolute_import, unicode_literals

import os.path
import re
import sqlite3

import simplesqlite as sqlite
import typepy

from ._const import MAX_VERBOSITY_LEVEL, SQLITE_SYSTEM_TABLE_LIST, Header
from ._error import DataNotFoundError, OperationalError
from ._logger import logger
from ._schema import SQLiteTableSchema

class SQLiteSchemaExtractor(object):
    """
    A SQLite database file schema extractor class.

    :param str database_path: Path to the SQLite database file.
    """

    _SQLITE_MASTER_TABLE_NAME = "master_"

    _RE_ATTR_DESCRIPTION = re.compile("[(].*[)]")
    _RE_FOREIGN_KEY = re.compile("FOREIGN KEY")
    _RE_ATTR_NAME = re.compile("[\'].*?[\']")

    def __init__(self, database_source):
        is_connection_required = True

        try:
            if database_source.is_connected():
                # datasource is a SimpleSQLite instance
                self.__con = database_source.connection
                is_connection_required = False
        except AttributeError:
            pass

        if isinstance(database_source, sqlite3.Connection):
            self.__con = database_source
            is_connection_required = False

        if is_connection_required:
            if not os.path.isfile(database_source):
                raise IOError("file not found: {}".format(database_source))

            try:
                self.__con = sqlite3.connect(database_source)
            except sqlite3.OperationalError as e:
                raise OperationalError(e)

        self.__cur = self.__con.cursor()
        self.__con_sqlite_master = None
        self._total_changes = None
        self._stream = None

    def fetch_table_name_list(self, include_system_table=False):
        """
        :return: List of table names in the database.
        :rtype: list
        """

        result = self.__cur.execute("SELECT name FROM sqlite_master WHERE TYPE='table'")
        if result is None:
            return []

        table_name_list = [record[0] for record in result.fetchall()]

        if include_system_table:
            return table_name_list

        return [table for table in table_name_list if table not in SQLITE_SYSTEM_TABLE_LIST]

    def fetch_table_schema(self, table_name):
        return SQLiteTableSchema(table_name, schema_data=self.__fetch_table_metadata(table_name))

    def fetch_database_schema(self):
        for table_name in self.fetch_table_name_list():
            yield self.fetch_table_schema(table_name)

    def fetch_database_schema_as_dict(self):
        database_schema = {}
        for table_schema in self.fetch_database_schema():
            database_schema.update(table_schema.as_dict())

        return database_schema

    def fetch_sqlite_master(self):
        """
        Get sqlite_master table information as a list of dictionaries.

        :return: sqlite_master table information.
        :rtype: list

        :Sample Code:
            .. code:: python

                from sqliteschema import SQLiteSchemaExtractor

                print(json.dumps(SQLiteSchemaExtractor("sample.sqlite").fetch_sqlite_master(), indent=4))

        :Output:
            .. code-block:: json

                [
                    {
                        "tbl_name": "sample_table",
                        "sql": "CREATE TABLE 'sample_table' ('a' INTEGER, 'b' REAL, 'c' TEXT, 'd' REAL, 'e' TEXT)",
                        "type": "table",
                        "name": "sample_table",
                        "rootpage": 2
                    },
                    {
                        "tbl_name": "sample_table",
                        "sql": "CREATE INDEX sample_table_a_index ON sample_table('a')",
                        "type": "index",
                        "name": "sample_table_a_index",
                        "rootpage": 3
                    }
                ]
        """

        sqlite_master_list = []
        result = self.__cur.execute("SELECT {:s} FROM sqlite_master".format(
            ", ".join(["tbl_name", "sql", "type", "name", "rootpage"])))
        return result.fetchall()

        for item in result.fetchall():
            sqlite_master_list.append(dict([[key, item[key]] for key in item.keys()]))

        return sqlite_master_list

    def dumps(self, output_format=None, verbosity_level=MAX_VERBOSITY_LEVEL):
        dump_list = []

        for table_schema in self.fetch_database_schema():
            dump_list.append(table_schema.dumps(
                output_format=output_format, verbosity_level=verbosity_level))

        return "\n".join(dump_list)

    def _extract_attr_name(self, schema):
        re_quote = re.compile("[\"\[\]\']")

        match_attr_name = self._RE_ATTR_NAME.search(schema)
        if not match_attr_name:
            return re_quote.sub("", schema.split()[0])

        return re_quote.sub("", match_attr_name.group())

    def _extract_attr_type(self, schema):
        match_attr_name = self._RE_ATTR_NAME.search(schema)
        if not match_attr_name:
            return schema.split()[1]

        schema_wo_name = self._RE_ATTR_NAME.sub("", schema).strip()

        return schema_wo_name.split()[0]

    def _extract_attr_constraints(self, schema):
        attr_name_match = self._RE_ATTR_NAME.search(schema)
        if not attr_name_match:
            return " ".join(schema.split()[2:])

        schema_wo_name = self._RE_ATTR_NAME.sub("", schema).strip()

        return " ".join(schema_wo_name.split()[1:])

    def _fetch_attr_schema(self, table_name, schema_type):
        if table_name in SQLITE_SYSTEM_TABLE_LIST:
            logger.debug("skip fetching sqlite system table: {:s}".format(table_name))
            return []

        self.__update_sqlite_master_db()

        try:
            result = self.__con_sqlite_master.execute(
                "SELECT {:s} FROM {:s} WHERE {:s} AND {:s}".format(
                    "sql",
                    self._SQLITE_MASTER_TABLE_NAME,
                    "{:s} = '{:s}'".format("tbl_name", table_name),
                    "{:s} = '{:s}'".format("type", schema_type)))
        except sqlite.TableNotFoundError:
            raise DataNotFoundError("table not found: '{}'".format(self._SQLITE_MASTER_TABLE_NAME))

        error_message_format = "data not found in '{}' table"

        try:
            table_schema = result.fetchone()[0]
        except TypeError:
            raise DataNotFoundError(
                error_message_format.format(self._SQLITE_MASTER_TABLE_NAME))

        match = self._RE_ATTR_DESCRIPTION.search(table_schema)
        if match is None:
            raise DataNotFoundError(error_message_format.format(table_schema))

        return [
            attr.strip() for attr in match.group().strip("()").split(",")
            if self._RE_FOREIGN_KEY.search(attr) is None
        ]

    def _fetch_index_schema(self, table_name):
        self.__update_sqlite_master_db()

        try:
            result = self.__con_sqlite_master.execute(
                "SELECT {:s} FROM {:s} WHERE {:s} AND {:s}".format(
                    "sql",
                    self._SQLITE_MASTER_TABLE_NAME,
                    "{:s} = '{:s}'".format("tbl_name", table_name),
                    "{:s} = '{:s}'".format("type", "index")))
        except sqlite.TableNotFoundError as e:
            raise DataNotFoundError(e)

        try:
            return [
                record[0] for record in result.fetchall()
                if typepy.is_not_empty_sequence(record[0])
            ]
        except TypeError:
            raise DataNotFoundError("index not found in '{}'".format(table_name))

    def __fetch_table_metadata(self, table_name):
        regexp_primary_key = re.compile("PRIMARY KEY", re.IGNORECASE)
        regexp_not_null = re.compile("NOT NULL", re.IGNORECASE)
        regexp_unique = re.compile("UNIQUE", re.IGNORECASE)
        index_query_list = self._fetch_index_schema(table_name)

        metadata = {}
        for attr_schema in self._fetch_attr_schema(table_name, "table"):
            values = {}
            attr_name = self._extract_attr_name(attr_schema)
            re_index = re.compile(re.escape(attr_name))

            values[Header.ATTR_NAME] = attr_name
            values[Header.INDEX] = False

            for index_query in index_query_list:
                if re_index.search(index_query) is not None:
                    values[Header.INDEX] = True
                    break

            try:
                values[Header.DATA_TYPE] = self._extract_attr_type(attr_schema)
            except IndexError:
                continue

            try:
                constraint = self._extract_attr_constraints(attr_schema)
            except IndexError:
                continue

            values[Header.PRIMARY_KEY] = regexp_primary_key.search(constraint) is not None
            values[Header.NOT_NULL] = regexp_not_null.search(constraint) is not None
            values[Header.UNIQUE] = regexp_unique.search(constraint) is not None

            metadata.setdefault(table_name, []).append(values)

        if not metadata:
            pass

        return metadata

    def __update_sqlite_master_db(self):
        try:
            total_changes = self.__con.total_changes
            if self._total_changes == total_changes:
                return
        except AttributeError:
            pass

        if self.__con_sqlite_master:
            self.__con_sqlite_master.close()

        self.__con_sqlite_master = sqlite3.connect(":memory:")

        sqlite_master = self.fetch_sqlite_master()
        if typepy.is_empty_sequence(sqlite_master):
            return

        self.__con_sqlite_master.execute("""CREATE TABLE {:s} (
            tbl_name TEXT NOT NULL,
            sql TEXT,
            type TEXT NOT NULL,
            name TEXT NOT NULL,
            rootpage INTEGER NOT NULL
            )""".format(self._SQLITE_MASTER_TABLE_NAME))
        self.__con_sqlite_master.executemany(
            "INSERT INTO {:s} VALUES (?,?,?,?,?)".format(self._SQLITE_MASTER_TABLE_NAME),
            sqlite_master)
        self.__con_sqlite_master.commit()

        self._total_changes = self.__con.total_changes
<commit_msg>Rename a member variable<commit_after># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import absolute_import, unicode_literals

import os.path
import re
import sqlite3

import simplesqlite as sqlite
import typepy

from ._const import MAX_VERBOSITY_LEVEL, SQLITE_SYSTEM_TABLE_LIST, Header
from ._error import DataNotFoundError, OperationalError
from ._logger import logger
from ._schema import SQLiteTableSchema

class SQLiteSchemaExtractor(object):
    """
    A SQLite database file schema extractor class.

    :param str database_path: Path to the SQLite database file.
    """

    _SQLITE_MASTER_TABLE_NAME = "master_"

    _RE_ATTR_DESCRIPTION = re.compile("[(].*[)]")
    _RE_FOREIGN_KEY = re.compile("FOREIGN KEY")
    _RE_ATTR_NAME = re.compile("[\'].*?[\']")

    def __init__(self, database_source):
        is_connection_required = True

        try:
            if database_source.is_connected():
                # datasource is a SimpleSQLite instance
                self.__con = database_source.connection
                is_connection_required = False
        except AttributeError:
            pass

        if isinstance(database_source, sqlite3.Connection):
            self.__con = database_source
            is_connection_required = False

        if is_connection_required:
            if not os.path.isfile(database_source):
                raise IOError("file not found: {}".format(database_source))

            try:
                self.__con = sqlite3.connect(database_source)
            except sqlite3.OperationalError as e:
                raise OperationalError(e)

        self.__cur = self.__con.cursor()
        self.__con_sqlite_master = None
        self.__total_changes = None
        self._stream = None

    def fetch_table_name_list(self, include_system_table=False):
        """
        :return: List of table names in the database.
        :rtype: list
        """

        result = self.__cur.execute("SELECT name FROM sqlite_master WHERE TYPE='table'")
        if result is None:
            return []

        table_name_list = [record[0] for record in result.fetchall()]

        if include_system_table:
            return table_name_list

        return [table for table in table_name_list if table not in SQLITE_SYSTEM_TABLE_LIST]

    def fetch_table_schema(self, table_name):
        return SQLiteTableSchema(table_name, schema_data=self.__fetch_table_metadata(table_name))

    def fetch_database_schema(self):
        for table_name in self.fetch_table_name_list():
            yield self.fetch_table_schema(table_name)

    def fetch_database_schema_as_dict(self):
        database_schema = {}
        for table_schema in self.fetch_database_schema():
            database_schema.update(table_schema.as_dict())

        return database_schema

    def fetch_sqlite_master(self):
        """
        Get sqlite_master table information as a list of dictionaries.

        :return: sqlite_master table information.
        :rtype: list

        :Sample Code:
            .. code:: python

                from sqliteschema import SQLiteSchemaExtractor

                print(json.dumps(SQLiteSchemaExtractor("sample.sqlite").fetch_sqlite_master(), indent=4))

        :Output:
            .. code-block:: json

                [
                    {
                        "tbl_name": "sample_table",
                        "sql": "CREATE TABLE 'sample_table' ('a' INTEGER, 'b' REAL, 'c' TEXT, 'd' REAL, 'e' TEXT)",
                        "type": "table",
                        "name": "sample_table",
                        "rootpage": 2
                    },
                    {
                        "tbl_name": "sample_table",
                        "sql": "CREATE INDEX sample_table_a_index ON sample_table('a')",
                        "type": "index",
                        "name": "sample_table_a_index",
                        "rootpage": 3
                    }
                ]
        """

        sqlite_master_list = []
        result = self.__cur.execute("SELECT {:s} FROM sqlite_master".format(
            ", ".join(["tbl_name", "sql", "type", "name", "rootpage"])))
        return result.fetchall()

        for item in result.fetchall():
            sqlite_master_list.append(dict([[key, item[key]] for key in item.keys()]))

        return sqlite_master_list

    def dumps(self, output_format=None, verbosity_level=MAX_VERBOSITY_LEVEL):
        dump_list = []

        for table_schema in self.fetch_database_schema():
            dump_list.append(table_schema.dumps(
                output_format=output_format, verbosity_level=verbosity_level))

        return "\n".join(dump_list)

    def _extract_attr_name(self, schema):
        re_quote = re.compile("[\"\[\]\']")

        match_attr_name = self._RE_ATTR_NAME.search(schema)
        if not match_attr_name:
            return re_quote.sub("", schema.split()[0])

        return re_quote.sub("", match_attr_name.group())

    def _extract_attr_type(self, schema):
        match_attr_name = self._RE_ATTR_NAME.search(schema)
        if not match_attr_name:
            return schema.split()[1]

        schema_wo_name = self._RE_ATTR_NAME.sub("", schema).strip()

        return schema_wo_name.split()[0]

    def _extract_attr_constraints(self, schema):
        attr_name_match = self._RE_ATTR_NAME.search(schema)
        if not attr_name_match:
            return " ".join(schema.split()[2:])

        schema_wo_name = self._RE_ATTR_NAME.sub("", schema).strip()

        return " ".join(schema_wo_name.split()[1:])

    def _fetch_attr_schema(self, table_name, schema_type):
        if table_name in SQLITE_SYSTEM_TABLE_LIST:
            logger.debug("skip fetching sqlite system table: {:s}".format(table_name))
            return []

        self.__update_sqlite_master_db()

        try:
            result = self.__con_sqlite_master.execute(
                "SELECT {:s} FROM {:s} WHERE {:s} AND {:s}".format(
                    "sql",
                    self._SQLITE_MASTER_TABLE_NAME,
                    "{:s} = '{:s}'".format("tbl_name", table_name),
                    "{:s} = '{:s}'".format("type", schema_type)))
        except sqlite.TableNotFoundError:
            raise DataNotFoundError("table not found: '{}'".format(self._SQLITE_MASTER_TABLE_NAME))

        error_message_format = "data not found in '{}' table"

        try:
            table_schema = result.fetchone()[0]
        except TypeError:
            raise DataNotFoundError(
                error_message_format.format(self._SQLITE_MASTER_TABLE_NAME))

        match = self._RE_ATTR_DESCRIPTION.search(table_schema)
        if match is None:
            raise DataNotFoundError(error_message_format.format(table_schema))

        return [
            attr.strip() for attr in match.group().strip("()").split(",")
            if self._RE_FOREIGN_KEY.search(attr) is None
        ]

    def _fetch_index_schema(self, table_name):
        self.__update_sqlite_master_db()

        try:
            result = self.__con_sqlite_master.execute(
                "SELECT {:s} FROM {:s} WHERE {:s} AND {:s}".format(
                    "sql",
                    self._SQLITE_MASTER_TABLE_NAME,
                    "{:s} = '{:s}'".format("tbl_name", table_name),
                    "{:s} = '{:s}'".format("type", "index")))
        except sqlite.TableNotFoundError as e:
            raise DataNotFoundError(e)

        try:
            return [
                record[0] for record in result.fetchall()
                if typepy.is_not_empty_sequence(record[0])
            ]
        except TypeError:
            raise DataNotFoundError("index not found in '{}'".format(table_name))

    def __fetch_table_metadata(self, table_name):
        regexp_primary_key = re.compile("PRIMARY KEY", re.IGNORECASE)
        regexp_not_null = re.compile("NOT NULL", re.IGNORECASE)
        regexp_unique = re.compile("UNIQUE", re.IGNORECASE)
        index_query_list = self._fetch_index_schema(table_name)

        metadata = {}
        for attr_schema in self._fetch_attr_schema(table_name, "table"):
            values = {}
            attr_name = self._extract_attr_name(attr_schema)
            re_index = re.compile(re.escape(attr_name))

            values[Header.ATTR_NAME] = attr_name
            values[Header.INDEX] = False

            for index_query in index_query_list:
                if re_index.search(index_query) is not None:
                    values[Header.INDEX] = True
                    break

            try:
                values[Header.DATA_TYPE] = self._extract_attr_type(attr_schema)
            except IndexError:
                continue

            try:
                constraint = self._extract_attr_constraints(attr_schema)
            except IndexError:
                continue

            values[Header.PRIMARY_KEY] = regexp_primary_key.search(constraint) is not None
            values[Header.NOT_NULL] = regexp_not_null.search(constraint) is not None
            values[Header.UNIQUE] = regexp_unique.search(constraint) is not None

            metadata.setdefault(table_name, []).append(values)

        if not metadata:
            pass

        return metadata

    def __update_sqlite_master_db(self):
        try:
            total_changes = self.__con.total_changes
            if self.__total_changes == total_changes:
                return
        except AttributeError:
            pass

        if self.__con_sqlite_master:
            self.__con_sqlite_master.close()

        self.__con_sqlite_master = sqlite3.connect(":memory:")

        sqlite_master = self.fetch_sqlite_master()
        if typepy.is_empty_sequence(sqlite_master):
            return

        self.__con_sqlite_master.execute("""CREATE TABLE {:s} (
            tbl_name TEXT NOT NULL,
            sql TEXT,
            type TEXT NOT NULL,
            name TEXT NOT NULL,
            rootpage INTEGER NOT NULL
            )""".format(self._SQLITE_MASTER_TABLE_NAME))
        self.__con_sqlite_master.executemany(
            "INSERT INTO {:s} VALUES (?,?,?,?,?)".format(self._SQLITE_MASTER_TABLE_NAME),
            sqlite_master)
        self.__con_sqlite_master.commit()

        self.__total_changes = self.__con.total_changes
<|endoftext|>


<commit_before>import numpy as np
import pandas as pd
import sys
sys.path.append("..")
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.linear_model import LogisticRegression
import src.data.extractor.preprocessing as preprocessing
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier

df = pd.read_csv("victim_info_2012_2017.csv")

def predict_logistic_regression(df):
    df = preprocessing.clean_data(df)
    df = preprocessing.get_dummies(df)
    #get X variables
    X = df[['age','time','Cause__Stabbing','Cause__Assault','Cause__Auto Crash','Cause__Other','Cause__Shooting','Cause__Strangulation']]
    y = df['Race__White']
    print ("splitting test and training data")
    X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=101,test_size=0.3)
    lr = LogisticRegression()
    print ("Fitting the training data into the linear regression model")
    lr.fit(X_train,y_train)
    print ("Predicting values of test data")
    predictions = lr.predict(X_test)
    print ("Printing classification report")
    print(classification_report(y_test, predictions))
    print(confusion_matrix(y_test, predictions))

def predict_random_forest(df):
    df = preprocessing.clean_data(df)
    df = preprocessing.get_dummies(df)
    # get X variables
    X = df[['age', 'time', 'Cause__Stabbing', 'Cause__Assault', 'Cause__Auto Crash', 'Cause__Other', 'Cause__Shooting',
            'Cause__Strangulation']]
    y = df['Race__White']
    print ("splitting test and training data")
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=101, test_size=0.3)
    rf = RandomForestClassifier(n_estimators=800)
    print ("Fitting the training data into the random forest")
    rf.fit(X_train, y_train)
    print ("Predicting values of test data")
    predictions = rf.predict(X_test)
    print ("Printing classification report")
    print(classification_report(y_test, predictions))
    print(confusion_matrix(y_test, predictions))

def predict_k_nearest_neighbours(df):
    df = preprocessing.clean_data(df)
    df = preprocessing.get_dummies(df)
    # get X variables
    X = df[['age', 'time', 'Cause__Stabbing', 'Cause__Assault', 'Cause__Auto Crash', 'Cause__Other', 'Cause__Shooting',
            'Cause__Strangulation']]
    y = df['Race__White']
    print ("splitting test and training data")
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=101, test_size=0.3)
    knn = KNeighborsClassifier(n_neighbors=5)
    knn.fit(X_train, y_train)
    print ("Fitting the training data into k-nearest neighbour")
    print ("Predicting values of test data")
    predictions = knn.predict(X_test)
    print ("Printing classification report")
    print(classification_report(y_test, predictions))
    print(confusion_matrix(y_test, predictions))

def sigmoid(x):
    return 1/(1+ np.exp(-x))

def forward(X, W1, b1, W2, b2):
    Z = sigmoid(np.dot(X,W1) + b1)
    A = np.dot(Z,W2) + b2
    expA = np.exp(A)
    Y = expA / expA.sum(axis=1, keepdims=True)
    return Y

def classification_rate(Y, P):
    n_correct = 0
    n_total = 0
    for i in range(len(Y)):
        n_total += 1
        if Y[i] == P[i]:
            n_correct += 1
    return float(n_correct) / n_total

def predict_using_neural_network(df):
    df = preprocessing.clean_data(df)
    df = preprocessing.get_dummies(df)
    X = df[['age', 'time', 'Cause__Stabbing', 'Cause__Assault', 'Cause__Auto Crash', 'Cause__Other', 'Cause__Shooting',
            'Cause__Strangulation']]
    y = df['Race__White']
    print ("converting to matrices")
    X = X.as_matrix
    y = y.as_matrix
    D = 8
    M = 8
    K = 2
    W1 = np.random.randn(D, M)
    b1 = np.random.randn(M)
    W2 = np.random.randn(M, K)
    b2 = np.random.randn(K)
    P = forward(X,W1,b1,W2,b2)
    P = np.argmax(P, axis=1)

    print (classification_rate(y,P))

predict_using_neural_network(df)<commit_msg>debugged numpy array conversion<commit_after>import numpy as np
import pandas as pd
import sys
sys.path.append("..")
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.linear_model import LogisticRegression
import src.data.extractor.preprocessing as preprocessing
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier

df = pd.read_csv("victim_info_2012_2017.csv")

def predict_logistic_regression(df):
    df = preprocessing.clean_data(df)
    df = preprocessing.get_dummies(df)
    #get X variables
    X = df[['age','time','Cause__Stabbing','Cause__Assault','Cause__Auto Crash','Cause__Other','Cause__Shooting','Cause__Strangulation']]
    y = df['Race__White']
    print ("splitting test and training data")
    X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=101,test_size=0.3)
    lr = LogisticRegression()
    print ("Fitting the training data into the linear regression model")
    lr.fit(X_train,y_train)
    print ("Predicting values of test data")
    predictions = lr.predict(X_test)
    print ("Printing classification report")
    print(classification_report(y_test, predictions))
    print(confusion_matrix(y_test, predictions))

def predict_random_forest(df):
    df = preprocessing.clean_data(df)
    df = preprocessing.get_dummies(df)
    # get X variables
    X = df[['age', 'time', 'Cause__Stabbing', 'Cause__Assault', 'Cause__Auto Crash', 'Cause__Other', 'Cause__Shooting',
            'Cause__Strangulation']]
    y = df['Race__White']
    print ("splitting test and training data")
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=101, test_size=0.3)
    rf = RandomForestClassifier(n_estimators=800)
    print ("Fitting the training data into the random forest")
    rf.fit(X_train, y_train)
    print ("Predicting values of test data")
    predictions = rf.predict(X_test)
    print ("Printing classification report")
    print(classification_report(y_test, predictions))
    print(confusion_matrix(y_test, predictions))

def predict_k_nearest_neighbours(df):
    df = preprocessing.clean_data(df)
    df = preprocessing.get_dummies(df)
    # get X variables
    X = df[['age', 'time', 'Cause__Stabbing', 'Cause__Assault', 'Cause__Auto Crash', 'Cause__Other', 'Cause__Shooting',
            'Cause__Strangulation']]
    y = df['Race__White']
    print ("splitting test and training data")
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=101, test_size=0.3)
    knn = KNeighborsClassifier(n_neighbors=5)
    knn.fit(X_train, y_train)
    print ("Fitting the training data into k-nearest neighbour")
    print ("Predicting values of test data")
    predictions = knn.predict(X_test)
    print ("Printing classification report")
    print(classification_report(y_test, predictions))
    print(confusion_matrix(y_test, predictions))

def sigmoid(x):
    return 1/(1+ np.exp(-x))

def forward(X, W1, b1, W2, b2):
    Z = sigmoid(np.dot(X,W1) + b1)
    A = np.dot(Z,W2) + b2
    expA = np.exp(A)
    Y = expA / expA.sum(axis=1, keepdims=True)
    return Y

def classification_rate(Y, P):
    n_correct = 0
    n_total = 0
    for i in range(len(Y)):
        n_total += 1
        if Y[i] == P[i]:
            n_correct += 1
    return float(n_correct) / n_total

def predict_using_neural_network(df):
    df = preprocessing.clean_data(df)
    df = preprocessing.get_dummies(df)
    X = df[['age', 'time', 'Cause__Stabbing', 'Cause__Assault', 'Cause__Auto Crash', 'Cause__Other', 'Cause__Shooting',
            'Cause__Strangulation']]
    y = df['Race__White']
    print ("converting to matrices")
    X = X.values
    y = y.values
    print (type(X))
    D = 8
    M = 8
    K = 2
    W1 = np.random.randn(D, M)
    b1 = np.random.randn(M)
    W2 = np.random.randn(M, K)
    b2 = np.random.randn(K)
    P = forward(X,W1,b1,W2,b2)
    P = np.argmax(P, axis=1)

    print (classification_rate(y,P))

predict_using_neural_network(df)<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import absolute_import, unicode_literals

import abc
import re
from collections import OrderedDict

import simplesqlite as sqlite
import six
import typepy
from simplesqlite.sqlquery import SqlQuery

from ._error import DataNotFoundError
from ._logger import logger

@six.add_metaclass(abc.ABCMeta)
class SqliteSchemaExtractorInterface(object):

    @abc.abstractproperty
    def con(self):  # pragma: no cover
        pass

    @abc.abstractproperty
    def verbosity_level(self):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_table_name_list(self):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_table_schema(self, table_name):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_table_schema_text(self, table_name):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_database_schema(self):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_num_records(self, table_name):  # pragma: no cover
        pass

    @abc.abstractmethod
    def dumps(self):  # pragma: no cover
        pass

class AbstractSqliteSchemaExtractor(SqliteSchemaExtractorInterface):
    """
    Abstract class of a SQLite database file schema extractor.

    :param str database_path: Path to the SQLite database file.
    """

    _SQLITE_MASTER_TABLE_NAME = "master"

    _RE_ATTR_DESCRIPTION = re.compile("[(].*[)]")
    _RE_FOREIGN_KEY = re.compile("FOREIGN KEY")
    _RE_ATTR_NAME = re.compile("[\'].*?[\']")

    @property
    def con(self):
        return self._con

    def __init__(self, database_source):
        is_connection_required = True

        try:
            if database_source.is_connected():
                self._con = database_source
                is_connection_required = False
        except AttributeError:
            pass

        if is_connection_required:
            self._con = sqlite.SimpleSQLite(database_source, "r")

        self.__con_sqlite_master = None
        self._total_changes = None
        self._stream = None

    def get_table_name_list(self):
        return self._con.get_table_name_list()

    def get_database_schema(self):
        database_schema = OrderedDict()
        for table_name in self.get_table_name_list():
            table_schema = self.get_table_schema(table_name)
            if table_schema is None:
                continue

            database_schema[table_name] = table_schema

        return database_schema

    def get_table_schema(self, table_name):
        return self._get_table_schema_v0(table_name)

    def get_num_records(self, table_name):
        return self._con.get_num_records(table_name)

    def _get_table_schema_v0(self, table_name):
        return [
            self._get_attr_name(attr)
            for attr in self._get_attr_schema(table_name, "table")
        ]

    def _get_table_schema_v1(self, table_name):
        return OrderedDict([
            [self._get_attr_name(attr), self._get_attr_type(attr)]
            for attr in self._get_attr_schema(table_name, "table")
        ])

    def _get_table_schema_v2(self, table_name):
        def get_schema_item(attr):
            element_list = [self._get_attr_type(attr)]

            constraints = self._get_attr_constraints(attr)
            if constraints:
                element_list.append(constraints)

            return " ".join(element_list)

        return OrderedDict([
            [self._get_attr_name(attr), get_schema_item(attr)]
            for attr in self._get_attr_schema(table_name, "table")
        ])

    def dumps(self):
        self._stream = six.StringIO()
        self._write_database_schema()

        text = self._stream.getvalue()

        self._stream.close()
        self._stream = None

        return text

    @abc.abstractmethod
    def _write_database_schema(self):  # pragma: no cover
        pass

    def _validate_table_existence(self, table_name):
        try:
            self._con.verify_table_existence(table_name)
        except sqlite.TableNotFoundError as e:
            raise DataNotFoundError(e)

    def _get_attr_name(self, schema):
        re_quote = re.compile("[\"\[\]\']")

        match_attr_name = self._RE_ATTR_NAME.search(schema)
        if not match_attr_name:
            return re_quote.sub("", schema.split()[0])

        return re_quote.sub("", match_attr_name.group())

    def _get_attr_type(self, schema):
        match_attr_name = self._RE_ATTR_NAME.search(schema)
        if not match_attr_name:
            return schema.split()[1]

        schema_wo_name = self._RE_ATTR_NAME.sub("", schema).strip()

        return schema_wo_name.split()[0]

    def _get_attr_constraints(self, schema):
        attr_name_match = self._RE_ATTR_NAME.search(schema)
        if not attr_name_match:
            return " ".join(schema.split()[2:])

        schema_wo_name = self._RE_ATTR_NAME.sub("", schema).strip()

        return " ".join(schema_wo_name.split()[1:])

    def _get_attr_schema(self, table_name, schema_type):
        self._validate_table_existence(table_name)

        if table_name in sqlite.SQLITE_SYSTEM_TABLE_LIST:
            logger.debug("skip sqlite system table: {:s}".format(table_name))
            return []

        self.__update_sqlite_master_db()

        try:
            result = self.__con_sqlite_master.select(
                "sql", table_name=self._SQLITE_MASTER_TABLE_NAME,
                where=" AND ".join([
                    SqlQuery.make_where("tbl_name", table_name),
                    SqlQuery.make_where("type", schema_type),
                ]))
        except sqlite.TableNotFoundError:
            raise DataNotFoundError("table not found: '{}'".format(self._SQLITE_MASTER_TABLE_NAME))

        error_message_format = "data not found in '{}' table"

        try:
            table_schema = result.fetchone()[0]
        except TypeError:
            raise DataNotFoundError(
                error_message_format.format(self._SQLITE_MASTER_TABLE_NAME))

        match = self._RE_ATTR_DESCRIPTION.search(table_schema)
        if match is None:
            raise DataNotFoundError(error_message_format.format(table_schema))

        return [
            attr.strip() for attr in match.group().strip("()").split(",")
            if self._RE_FOREIGN_KEY.search(attr) is None
        ]

    def _get_index_schema(self, table_name):
        self.__update_sqlite_master_db()

        try:
            result = self.__con_sqlite_master.select(
                "sql", table_name=self._SQLITE_MASTER_TABLE_NAME,
                where=" AND ".join([
                    SqlQuery.make_where("tbl_name", table_name),
                    SqlQuery.make_where("type", "index"),
                ]))
        except sqlite.TableNotFoundError as e:
            raise DataNotFoundError(e)

        try:
            return [
                record[0] for record in result.fetchall()
                if typepy.is_not_empty_sequence(record[0])
            ]
        except TypeError:
            raise DataNotFoundError("index not found in '{}'".format(table_name))

    def __update_sqlite_master_db(self):
        try:
            total_changes = self._con.get_total_changes()
            if self._total_changes == total_changes:
                return
        except AttributeError:
            pass

        self.__con_sqlite_master = sqlite.connect_sqlite_memdb()

        sqlite_master = self._con.get_sqlite_master()
        if typepy.is_empty_sequence(sqlite_master):
            return

        self.__con_sqlite_master.create_table_from_data_matrix(
            table_name=self._SQLITE_MASTER_TABLE_NAME,
            attr_name_list=["tbl_name", "sql", "type", "name", "rootpage"],
            data_matrix=sqlite_master)

        self._total_changes = self._con.get_total_changes()
<commit_msg>Refactor SQLite queries<commit_after>#!/usr/bin/env python
# encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import absolute_import, unicode_literals

import abc
import re
from collections import OrderedDict

import simplesqlite as sqlite
import six
import typepy
from simplesqlite.query import And, Where

from ._error import DataNotFoundError
from ._logger import logger

@six.add_metaclass(abc.ABCMeta)
class SqliteSchemaExtractorInterface(object):

    @abc.abstractproperty
    def con(self):  # pragma: no cover
        pass

    @abc.abstractproperty
    def verbosity_level(self):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_table_name_list(self):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_table_schema(self, table_name):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_table_schema_text(self, table_name):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_database_schema(self):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_num_records(self, table_name):  # pragma: no cover
        pass

    @abc.abstractmethod
    def dumps(self):  # pragma: no cover
        pass

class AbstractSqliteSchemaExtractor(SqliteSchemaExtractorInterface):
    """
    Abstract class of a SQLite database file schema extractor.

    :param str database_path: Path to the SQLite database file.
    """

    _SQLITE_MASTER_TABLE_NAME = "master"

    _RE_ATTR_DESCRIPTION = re.compile("[(].*[)]")
    _RE_FOREIGN_KEY = re.compile("FOREIGN KEY")
    _RE_ATTR_NAME = re.compile("[\'].*?[\']")

    @property
    def con(self):
        return self._con

    def __init__(self, database_source):
        is_connection_required = True

        try:
            if database_source.is_connected():
                self._con = database_source
                is_connection_required = False
        except AttributeError:
            pass

        if is_connection_required:
            self._con = sqlite.SimpleSQLite(database_source, "r")

        self.__con_sqlite_master = None
        self._total_changes = None
        self._stream = None

    def get_table_name_list(self):
        return self._con.get_table_name_list()

    def get_database_schema(self):
        database_schema = OrderedDict()
        for table_name in self.get_table_name_list():
            table_schema = self.get_table_schema(table_name)
            if table_schema is None:
                continue

            database_schema[table_name] = table_schema

        return database_schema

    def get_table_schema(self, table_name):
        return self._get_table_schema_v0(table_name)

    def get_num_records(self, table_name):
        return self._con.get_num_records(table_name)

    def _get_table_schema_v0(self, table_name):
        return [
            self._get_attr_name(attr)
            for attr in self._get_attr_schema(table_name, "table")
        ]

    def _get_table_schema_v1(self, table_name):
        return OrderedDict([
            [self._get_attr_name(attr), self._get_attr_type(attr)]
            for attr in self._get_attr_schema(table_name, "table")
        ])

    def _get_table_schema_v2(self, table_name):
        def get_schema_item(attr):
            element_list = [self._get_attr_type(attr)]

            constraints = self._get_attr_constraints(attr)
            if constraints:
                element_list.append(constraints)

            return " ".join(element_list)

        return OrderedDict([
            [self._get_attr_name(attr), get_schema_item(attr)]
            for attr in self._get_attr_schema(table_name, "table")
        ])

    def dumps(self):
        self._stream = six.StringIO()
        self._write_database_schema()

        text = self._stream.getvalue()

        self._stream.close()
        self._stream = None

        return text

    @abc.abstractmethod
    def _write_database_schema(self):  # pragma: no cover
        pass

    def _validate_table_existence(self, table_name):
        try:
            self._con.verify_table_existence(table_name)
        except sqlite.TableNotFoundError as e:
            raise DataNotFoundError(e)

    def _get_attr_name(self, schema):
        re_quote = re.compile("[\"\[\]\']")

        match_attr_name = self._RE_ATTR_NAME.search(schema)
        if not match_attr_name:
            return re_quote.sub("", schema.split()[0])

        return re_quote.sub("", match_attr_name.group())

    def _get_attr_type(self, schema):
        match_attr_name = self._RE_ATTR_NAME.search(schema)
        if not match_attr_name:
            return schema.split()[1]

        schema_wo_name = self._RE_ATTR_NAME.sub("", schema).strip()

        return schema_wo_name.split()[0]

    def _get_attr_constraints(self, schema):
        attr_name_match = self._RE_ATTR_NAME.search(schema)
        if not attr_name_match:
            return " ".join(schema.split()[2:])

        schema_wo_name = self._RE_ATTR_NAME.sub("", schema).strip()

        return " ".join(schema_wo_name.split()[1:])

    def _get_attr_schema(self, table_name, schema_type):
        self._validate_table_existence(table_name)

        if table_name in sqlite.SQLITE_SYSTEM_TABLE_LIST:
            logger.debug("skip sqlite system table: {:s}".format(table_name))
            return []

        self.__update_sqlite_master_db()

        try:
            result = self.__con_sqlite_master.select(
                "sql",
                table_name=self._SQLITE_MASTER_TABLE_NAME,
                where=And([Where("tbl_name", table_name), Where("type", schema_type)]).to_query())
        except sqlite.TableNotFoundError:
            raise DataNotFoundError("table not found: '{}'".format(self._SQLITE_MASTER_TABLE_NAME))

        error_message_format = "data not found in '{}' table"

        try:
            table_schema = result.fetchone()[0]
        except TypeError:
            raise DataNotFoundError(
                error_message_format.format(self._SQLITE_MASTER_TABLE_NAME))

        match = self._RE_ATTR_DESCRIPTION.search(table_schema)
        if match is None:
            raise DataNotFoundError(error_message_format.format(table_schema))

        return [
            attr.strip() for attr in match.group().strip("()").split(",")
            if self._RE_FOREIGN_KEY.search(attr) is None
        ]

    def _get_index_schema(self, table_name):
        self.__update_sqlite_master_db()

        try:
            result = self.__con_sqlite_master.select(
                "sql",
                table_name=self._SQLITE_MASTER_TABLE_NAME,
                where=And([Where("tbl_name", table_name), Where("type", "index")]).to_query())
        except sqlite.TableNotFoundError as e:
            raise DataNotFoundError(e)

        try:
            return [
                record[0] for record in result.fetchall()
                if typepy.is_not_empty_sequence(record[0])
            ]
        except TypeError:
            raise DataNotFoundError("index not found in '{}'".format(table_name))

    def __update_sqlite_master_db(self):
        try:
            total_changes = self._con.get_total_changes()
            if self._total_changes == total_changes:
                return
        except AttributeError:
            pass

        self.__con_sqlite_master = sqlite.connect_sqlite_memdb()

        sqlite_master = self._con.get_sqlite_master()
        if typepy.is_empty_sequence(sqlite_master):
            return

        self.__con_sqlite_master.create_table_from_data_matrix(
            table_name=self._SQLITE_MASTER_TABLE_NAME,
            attr_name_list=["tbl_name", "sql", "type", "name", "rootpage"],
            data_matrix=sqlite_master)

        self._total_changes = self._con.get_total_changes()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import absolute_import, unicode_literals

import abc
import re
from collections import OrderedDict

import simplesqlite as sqlite
import six
import typepy
from simplesqlite.sqlquery import SqlQuery

from ._error import DataNotFoundError

@six.add_metaclass(abc.ABCMeta)
class SqliteSchemaExtractorInterface(object):

    @abc.abstractproperty
    def con(self):  # pragma: no cover
        pass

    @abc.abstractproperty
    def verbosity_level(self):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_table_name_list(self):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_table_schema(self, table_name):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_table_schema_text(self, table_name):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_database_schema(self):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_num_records(self, table_name):  # pragma: no cover
        pass

    @abc.abstractmethod
    def dumps(self):  # pragma: no cover
        pass

class AbstractSqliteSchemaExtractor(SqliteSchemaExtractorInterface):
    """
    Abstract class of a SQLite database file schema extractor.

    :param str database_path: Path to the SQLite database file.
    """

    _SQLITE_MASTER_TABLE_NAME = "master"

    _RE_ATTR_DESCRIPTION = re.compile("[(].*[)]")
    _RE_FOREIGN_KEY = re.compile("FOREIGN KEY")
    _RE_ATTR_NAME = re.compile("[\'].*?[\']")

    @property
    def con(self):
        return self._con

    def __init__(self, database_source):
        is_connection_required = True

        try:
            if database_source.is_connected():
                self._con = database_source
                is_connection_required = False
        except AttributeError:
            pass

        if is_connection_required:
            self._con = sqlite.SimpleSQLite(database_source, "r")

        self._con_sql_master = None
        self._total_changes = None
        self._stream = None

    def get_table_name_list(self):
        return self._con.get_table_name_list()

    def get_database_schema(self):
        database_schema = OrderedDict()
        for table_name in self.get_table_name_list():
            table_schema = self.get_table_schema(table_name)
            if table_schema is None:
                continue

            database_schema[table_name] = table_schema

        return database_schema

    def get_table_schema(self, table_name):
        return self._get_table_schema_v0(table_name)

    def get_num_records(self, table_name):
        return self._con.get_num_records(table_name)

    def _get_table_schema_v0(self, table_name):
        return [
            self._get_attr_name(attr)
            for attr in self._get_attr_schema(table_name, "table")
        ]

    def _get_table_schema_v1(self, table_name):
        return OrderedDict([
            [self._get_attr_name(attr), self._get_attr_type(attr)]
            for attr in self._get_attr_schema(table_name, "table")
        ])

    def _get_table_schema_v2(self, table_name):
        def get_schema_item(attr):
            element_list = [self._get_attr_type(attr)]

            constraints = self._get_attr_constraints(attr)
            if constraints:
                element_list.append(constraints)

            return " ".join(element_list)

        return OrderedDict([
            [self._get_attr_name(attr), get_schema_item(attr)]
            for attr in self._get_attr_schema(table_name, "table")
        ])

    def dumps(self):
        self._stream = six.StringIO()
        self._write_database_schema()

        text = self._stream.getvalue()

        self._stream.close()
        self._stream = None

        return text

    @abc.abstractmethod
    def _write_database_schema(self):  # pragma: no cover
        pass

    def _validate_table_existence(self, table_name):
        try:
            self._con.verify_table_existence(table_name)
        except sqlite.TableNotFoundError as e:
            raise DataNotFoundError(e)

    def _get_attr_name(self, schema):
        re_quote = re.compile("[\"\[\]\']")

        match_attr_name = self._RE_ATTR_NAME.search(schema)
        if not match_attr_name:
            return re_quote.sub("", schema.split()[0])

        return re_quote.sub("", match_attr_name.group())

    def _get_attr_type(self, schema):
        match_attr_name = self._RE_ATTR_NAME.search(schema)
        if not match_attr_name:
            return schema.split()[1]

        schema_wo_name = self._RE_ATTR_NAME.sub("", schema).strip()

        return schema_wo_name.split()[0]

    def _get_attr_constraints(self, schema):
        attr_name_match = self._RE_ATTR_NAME.search(schema)
        if not attr_name_match:
            return " ".join(schema.split()[2:])

        schema_wo_name = self._RE_ATTR_NAME.sub("", schema).strip()

        return " ".join(schema_wo_name.split()[1:])

    def _get_attr_schema(self, table_name, schema_type):
        self._validate_table_existence(table_name)

        if table_name in sqlite.SQLITE_SYSTEM_TABLE_LIST:
            return []

        self.__update_sqlite_master_db()

        try:
            result = self._con_sql_master.select(
                "sql", table_name=self._SQLITE_MASTER_TABLE_NAME,
                where=" AND ".join([
                    SqlQuery.make_where("tbl_name", table_name),
                    SqlQuery.make_where("type", schema_type),
                ]))
        except sqlite.TableNotFoundError:
            raise DataNotFoundError("table not found: '{}'".format(self._SQLITE_MASTER_TABLE_NAME))

        error_message_format = "data not found in '{}' table"

        try:
            table_schema = result.fetchone()[0]
        except TypeError:
            raise DataNotFoundError(
                error_message_format.format(self._SQLITE_MASTER_TABLE_NAME))

        match = self._RE_ATTR_DESCRIPTION.search(table_schema)
        if match is None:
            raise DataNotFoundError(error_message_format.format(table_schema))

        return [
            attr.strip() for attr in match.group().strip("()").split(",")
            if self._RE_FOREIGN_KEY.search(attr) is None
        ]

    def _get_index_schema(self, table_name):
        self.__update_sqlite_master_db()

        try:
            result = self._con_sql_master.select(
                "sql", table_name=self._SQLITE_MASTER_TABLE_NAME,
                where=" AND ".join([
                    SqlQuery.make_where("tbl_name", table_name),
                    SqlQuery.make_where("type", "index"),
                ]))
        except sqlite.TableNotFoundError as e:
            raise DataNotFoundError(e)

        try:
            return [
                record[0] for record in result.fetchall()
                if typepy.is_not_empty_sequence(record[0])
            ]
        except TypeError:
            raise DataNotFoundError("index not found in '{}'".format(table_name))

    def __update_sqlite_master_db(self):
        try:
            total_changes = self._con.get_total_changes()
            if self._total_changes == total_changes:
                return
        except AttributeError:
            pass

        self._con_sql_master = sqlite.connect_sqlite_memdb()

        sqlite_master = self._con.get_sqlite_master()
        if typepy.is_empty_sequence(sqlite_master):
            return

        self._con_sql_master.create_table_from_data_matrix(
            table_name=self._SQLITE_MASTER_TABLE_NAME,
            attr_name_list=["tbl_name", "sql", "type", "name", "rootpage"],
            data_matrix=sqlite_master)

        self._total_changes = self._con.get_total_changes()
<commit_msg>Add a debug message<commit_after>#!/usr/bin/env python
# encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import absolute_import, unicode_literals

import abc
import re
from collections import OrderedDict

import simplesqlite as sqlite
import six
import typepy
from simplesqlite.sqlquery import SqlQuery

from ._error import DataNotFoundError
from ._logger import logger

@six.add_metaclass(abc.ABCMeta)
class SqliteSchemaExtractorInterface(object):

    @abc.abstractproperty
    def con(self):  # pragma: no cover
        pass

    @abc.abstractproperty
    def verbosity_level(self):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_table_name_list(self):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_table_schema(self, table_name):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_table_schema_text(self, table_name):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_database_schema(self):  # pragma: no cover
        pass

    @abc.abstractmethod
    def get_num_records(self, table_name):  # pragma: no cover
        pass

    @abc.abstractmethod
    def dumps(self):  # pragma: no cover
        pass

class AbstractSqliteSchemaExtractor(SqliteSchemaExtractorInterface):
    """
    Abstract class of a SQLite database file schema extractor.

    :param str database_path: Path to the SQLite database file.
    """

    _SQLITE_MASTER_TABLE_NAME = "master"

    _RE_ATTR_DESCRIPTION = re.compile("[(].*[)]")
    _RE_FOREIGN_KEY = re.compile("FOREIGN KEY")
    _RE_ATTR_NAME = re.compile("[\'].*?[\']")

    @property
    def con(self):
        return self._con

    def __init__(self, database_source):
        is_connection_required = True

        try:
            if database_source.is_connected():
                self._con = database_source
                is_connection_required = False
        except AttributeError:
            pass

        if is_connection_required:
            self._con = sqlite.SimpleSQLite(database_source, "r")

        self._con_sql_master = None
        self._total_changes = None
        self._stream = None

    def get_table_name_list(self):
        return self._con.get_table_name_list()

    def get_database_schema(self):
        database_schema = OrderedDict()
        for table_name in self.get_table_name_list():
            table_schema = self.get_table_schema(table_name)
            if table_schema is None:
                continue

            database_schema[table_name] = table_schema

        return database_schema

    def get_table_schema(self, table_name):
        return self._get_table_schema_v0(table_name)

    def get_num_records(self, table_name):
        return self._con.get_num_records(table_name)

    def _get_table_schema_v0(self, table_name):
        return [
            self._get_attr_name(attr)
            for attr in self._get_attr_schema(table_name, "table")
        ]

    def _get_table_schema_v1(self, table_name):
        return OrderedDict([
            [self._get_attr_name(attr), self._get_attr_type(attr)]
            for attr in self._get_attr_schema(table_name, "table")
        ])

    def _get_table_schema_v2(self, table_name):
        def get_schema_item(attr):
            element_list = [self._get_attr_type(attr)]

            constraints = self._get_attr_constraints(attr)
            if constraints:
                element_list.append(constraints)

            return " ".join(element_list)

        return OrderedDict([
            [self._get_attr_name(attr), get_schema_item(attr)]
            for attr in self._get_attr_schema(table_name, "table")
        ])

    def dumps(self):
        self._stream = six.StringIO()
        self._write_database_schema()

        text = self._stream.getvalue()

        self._stream.close()
        self._stream = None

        return text

    @abc.abstractmethod
    def _write_database_schema(self):  # pragma: no cover
        pass

    def _validate_table_existence(self, table_name):
        try:
            self._con.verify_table_existence(table_name)
        except sqlite.TableNotFoundError as e:
            raise DataNotFoundError(e)

    def _get_attr_name(self, schema):
        re_quote = re.compile("[\"\[\]\']")

        match_attr_name = self._RE_ATTR_NAME.search(schema)
        if not match_attr_name:
            return re_quote.sub("", schema.split()[0])

        return re_quote.sub("", match_attr_name.group())

    def _get_attr_type(self, schema):
        match_attr_name = self._RE_ATTR_NAME.search(schema)
        if not match_attr_name:
            return schema.split()[1]

        schema_wo_name = self._RE_ATTR_NAME.sub("", schema).strip()

        return schema_wo_name.split()[0]

    def _get_attr_constraints(self, schema):
        attr_name_match = self._RE_ATTR_NAME.search(schema)
        if not attr_name_match:
            return " ".join(schema.split()[2:])

        schema_wo_name = self._RE_ATTR_NAME.sub("", schema).strip()

        return " ".join(schema_wo_name.split()[1:])

    def _get_attr_schema(self, table_name, schema_type):
        self._validate_table_existence(table_name)

        if table_name in sqlite.SQLITE_SYSTEM_TABLE_LIST:
            logger.debug("skip sqlite system table: {:s}".format(table_name))
            return []

        self.__update_sqlite_master_db()

        try:
            result = self._con_sql_master.select(
                "sql", table_name=self._SQLITE_MASTER_TABLE_NAME,
                where=" AND ".join([
                    SqlQuery.make_where("tbl_name", table_name),
                    SqlQuery.make_where("type", schema_type),
                ]))
        except sqlite.TableNotFoundError:
            raise DataNotFoundError("table not found: '{}'".format(self._SQLITE_MASTER_TABLE_NAME))

        error_message_format = "data not found in '{}' table"

        try:
            table_schema = result.fetchone()[0]
        except TypeError:
            raise DataNotFoundError(
                error_message_format.format(self._SQLITE_MASTER_TABLE_NAME))

        match = self._RE_ATTR_DESCRIPTION.search(table_schema)
        if match is None:
            raise DataNotFoundError(error_message_format.format(table_schema))

        return [
            attr.strip() for attr in match.group().strip("()").split(",")
            if self._RE_FOREIGN_KEY.search(attr) is None
        ]

    def _get_index_schema(self, table_name):
        self.__update_sqlite_master_db()

        try:
            result = self._con_sql_master.select(
                "sql", table_name=self._SQLITE_MASTER_TABLE_NAME,
                where=" AND ".join([
                    SqlQuery.make_where("tbl_name", table_name),
                    SqlQuery.make_where("type", "index"),
                ]))
        except sqlite.TableNotFoundError as e:
            raise DataNotFoundError(e)

        try:
            return [
                record[0] for record in result.fetchall()
                if typepy.is_not_empty_sequence(record[0])
            ]
        except TypeError:
            raise DataNotFoundError("index not found in '{}'".format(table_name))

    def __update_sqlite_master_db(self):
        try:
            total_changes = self._con.get_total_changes()
            if self._total_changes == total_changes:
                return
        except AttributeError:
            pass

        self._con_sql_master = sqlite.connect_sqlite_memdb()

        sqlite_master = self._con.get_sqlite_master()
        if typepy.is_empty_sequence(sqlite_master):
            return

        self._con_sql_master.create_table_from_data_matrix(
            table_name=self._SQLITE_MASTER_TABLE_NAME,
            attr_name_list=["tbl_name", "sql", "type", "name", "rootpage"],
            data_matrix=sqlite_master)

        self._total_changes = self._con.get_total_changes()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import json
import datetime

def get_time():
    """
    Generate a formatted timestamp for currenttime
    """
    return str(datetime.datetime.now())

def get_autogen_header(path=None):
    file_path = os.path.join(path, "list_raw.txt")

    with open(file_path, "r") as f:
        data_version = f.readlines()[2]
        data_version = data_version[2:]
        data_version = data_version.rstrip()

        f.close()

    # Autogenerated header
    autogen_header = "This file was generated by The-Big-Username-Blacklist %s (at %s)" % (  # NOQA
        data_version, get_time()
    )

    return autogen_header

def convert(path=None):
    """
    Loads data from list_raw and generates files in the following formats:
        - newline
        - json
        - python
        - js (ES5)
        - js (ES6)
        - php

    """
    # Load raw
    file_path = os.path.join(path, "list_raw.txt")

    with open(file_path, "r") as f:
        raw_usernames = f.readlines()

        # Remote newline
        usernames = [x.strip("\n") for x in raw_usernames]

        # Remove trailing #
        usernames = [x for x in usernames if x and not x.startswith("#")]

        # Sort list
        usernames.sort()

        f.close()

    # Write optimized newline file
    file_path = os.path.join(path, "list.txt")

    optimized_data = "".join("%s\n" % e for e in usernames)
    optimized_data = optimized_data.strip()

    autogen_header = get_autogen_header(path=path)

    with open(file_path, "w") as f:
        f.write(optimized_data)
        f.close()

    # Write optimized json file
    file_path = os.path.join(path, "list.json")

    with open(file_path, "w") as f:
        json.dump(usernames, f, indent=4, ensure_ascii=False)
        f.close()

    # Write python file
    file_path = os.path.join(path, "list.py")

    with open(file_path, "w") as f:
        f.write("# %s\n" % autogen_header)
        f.write("data = %s" % str(usernames))
        f.close()

    # Write es6 module
    file_path = os.path.join(path, "list.js")

    with open(file_path, "w") as f:
        f.write("// %s\n" % autogen_header)
        f.write("export default %s;" % str(usernames))
        f.close()

    # Write commonjs module
    file_path = os.path.join(path, "list-commonjs.js")

    with open(file_path, "w") as f:
        f.write("// %s\n" % autogen_header)
        f.write("module.exports = %s;" % str(usernames))
        f.close()

    # write PHP file
    file_path = os.path.join(path, "list.php")

    with open(file_path, "w") as f:
        f.write("<?php \n")
        f.write("// %s\n" % autogen_header)
        f.write("return %s;" % str(usernames))
        f.close()

if __name__ == "__main__":
    current_dir = os.path.dirname(os.path.realpath(__file__))
    list_dir = os.path.join(current_dir, os.pardir)

    convert(path=list_dir)
<commit_msg>Updated generator to use python3 string template<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import json
import datetime

def get_time():
    """
    Generate a formatted timestamp for currenttime
    """
    return str(datetime.datetime.now())

def get_autogen_header(path=None):
    file_path = os.path.join(path, "list_raw.txt")

    with open(file_path, "r") as f:
        data_version = f.readlines()[2]
        data_version = data_version[2:]
        data_version = data_version.rstrip()

        f.close()

    # Autogenerated header
    autogen_header = "This file was generated by The-Big-Username-Blacklist {} (at {})".format(  # NOQA
        data_version, get_time()
    )

    return autogen_header

def convert(path=None):
    """
    Loads data from list_raw and generates files in the following formats:
        - newline
        - json
        - python
        - js (ES5)
        - js (ES6)
        - php
    """
    # Load raw
    file_path = os.path.join(path, "list_raw.txt")

    with open(file_path, "r") as f:
        raw_usernames = f.readlines()

        # Remote newline
        usernames = [x.strip("\n") for x in raw_usernames]

        # Remove trailing #
        usernames = [x for x in usernames if x and not x.startswith("#")]

        # Sort list
        usernames.sort()

        f.close()

    # Write optimized newline file
    file_path = os.path.join(path, "list.txt")

    optimized_data = "".join("{}\n".format(e) for e in usernames)
    optimized_data = optimized_data.strip()

    autogen_header = get_autogen_header(path=path)

    with open(file_path, "w") as f:
        f.write(optimized_data)
        f.close()

    # Write optimized json file
    file_path = os.path.join(path, "list.json")

    with open(file_path, "w") as f:
        json.dump(usernames, f, indent=4, ensure_ascii=False)
        f.close()

    # Write python file
    file_path = os.path.join(path, "list.py")

    with open(file_path, "w") as f:
        f.write(f"# {autogen_header}\n")
        f.write(f"data = {usernames}")
        f.close()

    # Write es6 module
    file_path = os.path.join(path, "list.js")

    with open(file_path, "w") as f:
        f.write(f"// {autogen_header}\n")
        f.write(f"export default {usernames};")
        f.close()

    # Write commonjs module
    file_path = os.path.join(path, "list-commonjs.js")

    with open(file_path, "w") as f:
        f.write(f"// {autogen_header}\n")
        f.write(f"module.exports = {usernames};")
        f.close()

    # write PHP file
    file_path = os.path.join(path, "list.php")

    with open(file_path, "w") as f:
        f.write("<?php \n")
        f.write(f"// {autogen_header}\n")
        f.write(f"return {usernames};")
        f.close()

if __name__ == "__main__":
    current_dir = os.path.dirname(os.path.realpath(__file__))
    list_dir = os.path.join(current_dir, os.pardir)

    convert(path=list_dir)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Arne Neumann <discoursegraphs.programming@arne.cl>

"""
The ``merging`` module combines several document graphs into one.
So far, it is able to merge rhetorical structure theory (RS3), syntax
(TigerXML) and anaphora (ad-hoc format) annotations of the same document.
"""

import os
import sys
import argparse
from networkx import write_dot

from discoursegraphs.relabel import relabel_nodes
from discoursegraphs.readwrite.anaphoricity import AnaphoraDocumentGraph
from discoursegraphs.readwrite.rst import RSTGraph
from discoursegraphs.readwrite.tiger import TigerDocumentGraph
from discoursegraphs.readwrite.conano import ConanoDocumentGraph
from discoursegraphs.readwrite.neo4j import convert_to_geoff, upload_to_neo4j

def merging_cli():
    """
    simple commandline interface of the merging module.

    This function is called when you use the ``discoursegraphs`` application
    directly on the command line.
    """
    parser = argparse.ArgumentParser()

    parser.add_argument('-t', '--tiger-file',
                        help='TigerXML (syntax) file to be merged')
    parser.add_argument('-r', '--rst-file',
                        help='RS3 (rhetorical structure) file to be merged')
    parser.add_argument('-a', '--anaphoricity-file',
                        help='anaphoricity file to be merged')
    parser.add_argument('-c', '--conano-file',
                        help='conano file to be merged')
    parser.add_argument('-o', '--output-format',
                        default='dot',
                        help='output format: dot, geoff, neo4j, no-output')
    parser.add_argument('output_file', nargs='?', default=sys.stdout)

    args = parser.parse_args(sys.argv[1:])

    assert args.tiger_file, \
        "You'll need to provide at least a TigerXML file."

    for filepath in (args.tiger_file, args.rst_file, args.anaphoricity_file,
                     args.conano_file):
        if filepath:  # if it was specified on the command line
            assert os.path.isfile(filepath), \
                "File '{}' doesn't exist".format(filepath)

    tiger_docgraph = TigerDocumentGraph(args.tiger_file)

    if args.rst_file:
        rst_graph = RSTGraph(args.rst_file)
        tiger_docgraph.merge_graphs(rst_graph)

    if args.anaphoricity_file:
        anaphora_graph = AnaphoraDocumentGraph(args.anaphoricity_file)
        tiger_docgraph.merge_graphs(anaphora_graph)
        # the anaphora doc graph only contains trivial edges from its root
        # node.
        try:
            tiger_docgraph.remove_node('anaphoricity:root_node')
        except:
            pass

    if args.conano_file:
        conano_graph = ConanoDocumentGraph(args.conano_file)
        tiger_docgraph.merge_graphs(conano_graph)

    if args.output_format == 'dot':
        write_dot(tiger_docgraph, args.output_file)
    elif args.output_format == 'geoff':
        args.output_file.write(convert_to_geoff(tiger_docgraph))
        print ''
    elif args.output_format == 'neo4j':
        import requests
        try:
            upload_to_neo4j(tiger_docgraph)
        except requests.exceptions.ConnectionError as e:
            sys.stderr.write(
                ("Can't upload graph to Neo4j server. "
                 "Is it running?\n{}\n".format(e)))
    elif args.output_format == 'no-output':
        pass  # just testing if the merging works
    else:
        raise ValueError(
            "Unsupported output format: {}".format(args.output_format))

if __name__ == '__main__':
    merging_cli()
<commit_msg>flake8: merging.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Arne Neumann <discoursegraphs.programming@arne.cl>

"""
The ``merging`` module combines several document graphs into one.
So far, it is able to merge rhetorical structure theory (RS3), syntax
(TigerXML) and anaphora (ad-hoc format) annotations of the same document.
"""

import os
import sys
import argparse
from networkx import write_dot

from discoursegraphs.readwrite.anaphoricity import AnaphoraDocumentGraph
from discoursegraphs.readwrite.rst import RSTGraph
from discoursegraphs.readwrite.tiger import TigerDocumentGraph
from discoursegraphs.readwrite.conano import ConanoDocumentGraph
from discoursegraphs.readwrite.neo4j import convert_to_geoff, upload_to_neo4j

def merging_cli():
    """
    simple commandline interface of the merging module.

    This function is called when you use the ``discoursegraphs`` application
    directly on the command line.
    """
    parser = argparse.ArgumentParser()

    parser.add_argument('-t', '--tiger-file',
                        help='TigerXML (syntax) file to be merged')
    parser.add_argument('-r', '--rst-file',
                        help='RS3 (rhetorical structure) file to be merged')
    parser.add_argument('-a', '--anaphoricity-file',
                        help='anaphoricity file to be merged')
    parser.add_argument('-c', '--conano-file',
                        help='conano file to be merged')
    parser.add_argument('-o', '--output-format',
                        default='dot',
                        help='output format: dot, geoff, neo4j, no-output')
    parser.add_argument('output_file', nargs='?', default=sys.stdout)

    args = parser.parse_args(sys.argv[1:])

    assert args.tiger_file, \
        "You'll need to provide at least a TigerXML file."

    for filepath in (args.tiger_file, args.rst_file, args.anaphoricity_file,
                     args.conano_file):
        if filepath:  # if it was specified on the command line
            assert os.path.isfile(filepath), \
                "File '{}' doesn't exist".format(filepath)

    tiger_docgraph = TigerDocumentGraph(args.tiger_file)

    if args.rst_file:
        rst_graph = RSTGraph(args.rst_file)
        tiger_docgraph.merge_graphs(rst_graph)

    if args.anaphoricity_file:
        anaphora_graph = AnaphoraDocumentGraph(args.anaphoricity_file)
        tiger_docgraph.merge_graphs(anaphora_graph)
        # the anaphora doc graph only contains trivial edges from its root
        # node.
        try:
            tiger_docgraph.remove_node('anaphoricity:root_node')
        except:
            pass

    if args.conano_file:
        conano_graph = ConanoDocumentGraph(args.conano_file)
        tiger_docgraph.merge_graphs(conano_graph)

    if args.output_format == 'dot':
        write_dot(tiger_docgraph, args.output_file)
    elif args.output_format == 'geoff':
        args.output_file.write(convert_to_geoff(tiger_docgraph))
        print ''
    elif args.output_format == 'neo4j':
        import requests
        try:
            upload_to_neo4j(tiger_docgraph)
        except requests.exceptions.ConnectionError as e:
            sys.stderr.write(
                ("Can't upload graph to Neo4j server. "
                 "Is it running?\n{}\n".format(e)))
    elif args.output_format == 'no-output':
        pass  # just testing if the merging works
    else:
        raise ValueError(
            "Unsupported output format: {}".format(args.output_format))

if __name__ == '__main__':
    merging_cli()
<|endoftext|>


<commit_before>__author__ = 'katharine'

import csv
import struct
import time

def generate_files(source_dir, target_dir):
    stops_txt = [x for x in csv.DictReader(open("%s/stops.txt" % source_dir, 'rb')) if x['location_type'] == '0']
    print "%d stops" % len(stops_txt)

    stop_parent_map = {}
    stop_map = {}
    stops = []
    for s in stops_txt:
        if s['parent_station'] in stop_parent_map:
            stop_map[int(s['stop_code'])] = stop_parent_map[s['parent_station']]
            continue
        stop_map[int(s['stop_code'])] = len(stops)
        stop_parent_map[s['parent_station']] = len(stops)
        stops.append({
            'name': s['stop_name'].replace('Caltrain', '').replace('Station', '').replace("Mt View", "Mountain View").rstrip(),
            'zone': int(s['zone_id']) if s['zone_id'] != '' else 0,
            'lat': float(s['stop_lat']),
            'lon': float(s['stop_lon'])
        })

    with open('%s/stops.dat' % target_dir, 'wb') as f:
        f.write(struct.pack('<B', len(stops)))
        for stop in stops:
            f.write(struct.pack('<B18sii', stop['zone'], stop['name'], int(stop['lat'] * 1000000), int(stop['lon'] * 1000000)))

    calendar_txt = list(csv.DictReader(open("%s/calendar.txt" % source_dir, 'rb')))

    cal = []
    cal_map = {}
    for i, x in enumerate(calendar_txt):
        cal_map[x['service_id']] = i
        cal.append({
            'id': i,
            'start': time.mktime(time.strptime(x['start_date'], '%Y%m%d')),
            'end': time.mktime(time.strptime(x['end_date'], '%Y%m%d')),
            'days': (
                (int(x['monday'])    << 0) |
                (int(x['tuesday'])   << 1) |
                (int(x['wednesday']) << 2) |
                (int(x['thursday'])  << 3) |
                (int(x['friday'])    << 4) |
                (int(x['saturday'])  << 5) |
                (int(x['sunday'])    << 6)
            )
        })

    with open('%s/calendar.dat' % target_dir, 'wb') as f:
        f.write(struct.pack('<B', len(cal)))
        for c in cal:
            f.write(struct.pack('<IIB', int(c['start']), int(c['end']), c['days']))

    trips_txt = list(csv.DictReader(open("%s/trips.txt" % source_dir, "rb")))

    tr = []
    tr_map = {}

    # These shouldn't be hardcoded, and should instead be inferred from routes.txt.
    route_map = {
        "Bu-120": 0,
        "Li-120": 1,
        "Lo-120": 2,
        "TaSj-120": 3,
        "Bu-121": 0,
        "Li-121": 1,
        "Lo-121": 2,
        "TaSj-121": 3,
    }

    for i, trip in enumerate(trips_txt):
        tr.append({
            'direction': int(not int(trip['direction_id'])),  # We picked opposing values for north/south.
            'route': route_map[trip['route_id']],
            'service': cal_map[trip['service_id']],
            'trip_name': int(trip['trip_short_name'])}),
        tr_map[trip['trip_id']] = i

    with open('%s/trips.dat' % target_dir, 'wb') as f:
        f.write(struct.pack('<H', len(tr)))
        for t in tr:
            f.write(struct.pack('<HBBB', t['trip_name'], t['direction'], t['route'], t['service']))

    times_txt = list(csv.DictReader(open("%s/stop_times.txt" % source_dir)))

    tm = sorted([{
        'time': (int(x['arrival_time'].split(':')[0])*60 + int(x['arrival_time'].split(':')[1])),
        'stop': stop_map[int(x['stop_id'])],
        'sequence': int(x['stop_sequence']),
        'trip': tr_map[x['trip_id']]
    } for x in times_txt], key=lambda y: y['time'])

    with open('%s/times.dat' % target_dir, 'wb') as f:
        f.write(struct.pack('<H', len(tm)))
        for t in tm:
            f.write(struct.pack('<HHBB', t['trip'], t['time'], t['stop'], t['sequence']))

    stop_times = {stop: sorted([i for i, x in enumerate(tm) if x['stop'] == stop], key=lambda t: tm[t]['time']) for stop, s in enumerate(stops)}
    lengths = [len(x) for x in stop_times.values()]

    with open('%s/stop_index.dat' % target_dir, 'wb') as f:
        f.write(struct.pack('<B', len(lengths)))
        counter = len(lengths)*4 + 1
        for l in lengths:
            f.write(struct.pack('<HH', counter, l))
            counter += l*2

        for s in stop_times.values():
            for x in s:
                f.write(struct.pack('<H', x))

    trip_stops = [sorted([i for i, x in enumerate(tm) if x['trip'] == trip], key=lambda k: tm[k]['stop']) for trip, s in enumerate(tr)]
    lengths = map(len, trip_stops)
    with open('%s/trip_index.dat' % target_dir, 'wb') as f:
        f.write(struct.pack('<H', len(lengths)))
        counter = len(lengths) * 3 + 2
        data_start = counter
        for l in lengths:
            f.write(struct.pack('<HB', counter, l))
            counter += l*2

        if data_start != f.tell():
            raise Exception("%d != %d" % (counter, f.tell()))

        for s in trip_stops:
            for x in s:
                f.write(struct.pack('<H', x))

        if f.tell() != counter:
            raise Exception("Not the expected length!")

if __name__ == "__main__":
    generate_files("/Users/katharine/Downloads/GTFS Caltrain Devs/", "/Users/katharine/projects/pebble-caltrain/resources/data/")<commit_msg>scripts/generate.py: Add the South SF replacement<commit_after>__author__ = 'katharine'

import csv
import struct
import time

def generate_files(source_dir, target_dir):
    stops_txt = [x for x in csv.DictReader(open("%s/stops.txt" % source_dir, 'rb')) if x['location_type'] == '0']
    print "%d stops" % len(stops_txt)

    name_replacements = (
        ('Caltrain', ''),
        ('Station', ''),
        ('Mt View', 'Mountain View'),
        ('So. San Francisco', 'South SF'),
    )

    stop_parent_map = {}
    stop_map = {}
    stops = []
    for s in stops_txt:
        if s['parent_station'] in stop_parent_map:
            stop_map[int(s['stop_code'])] = stop_parent_map[s['parent_station']]
            continue
        stop_map[int(s['stop_code'])] = len(stops)
        stop_parent_map[s['parent_station']] = len(stops)
        for replacement in name_replacements:
            s['stop_name'] = s['stop_name'].replace(*replacement)
        s['stop_name'] = s['stop_name'].rstrip()
        stops.append({
            'name': s['stop_name'],
            'zone': int(s['zone_id']) if s['zone_id'] != '' else 0,
            'lat': float(s['stop_lat']),
            'lon': float(s['stop_lon'])
        })

    with open('%s/stops.dat' % target_dir, 'wb') as f:
        f.write(struct.pack('<B', len(stops)))
        for stop in stops:
            f.write(struct.pack('<B18sii', stop['zone'], stop['name'], int(stop['lat'] * 1000000), int(stop['lon'] * 1000000)))

    calendar_txt = list(csv.DictReader(open("%s/calendar.txt" % source_dir, 'rb')))

    cal = []
    cal_map = {}
    for i, x in enumerate(calendar_txt):
        cal_map[x['service_id']] = i
        cal.append({
            'id': i,
            'start': time.mktime(time.strptime(x['start_date'], '%Y%m%d')),
            'end': time.mktime(time.strptime(x['end_date'], '%Y%m%d')),
            'days': (
                (int(x['monday'])    << 0) |
                (int(x['tuesday'])   << 1) |
                (int(x['wednesday']) << 2) |
                (int(x['thursday'])  << 3) |
                (int(x['friday'])    << 4) |
                (int(x['saturday'])  << 5) |
                (int(x['sunday'])    << 6)
            )
        })

    with open('%s/calendar.dat' % target_dir, 'wb') as f:
        f.write(struct.pack('<B', len(cal)))
        for c in cal:
            f.write(struct.pack('<IIB', int(c['start']), int(c['end']), c['days']))

    trips_txt = list(csv.DictReader(open("%s/trips.txt" % source_dir, "rb")))

    tr = []
    tr_map = {}

    # These shouldn't be hardcoded, and should instead be inferred from routes.txt.
    route_map = {
        "Bu-120": 0,
        "Li-120": 1,
        "Lo-120": 2,
        "TaSj-120": 3,
        "Bu-121": 0,
        "Li-121": 1,
        "Lo-121": 2,
        "TaSj-121": 3,
    }

    for i, trip in enumerate(trips_txt):
        tr.append({
            'direction': int(not int(trip['direction_id'])),  # We picked opposing values for north/south.
            'route': route_map[trip['route_id']],
            'service': cal_map[trip['service_id']],
            'trip_name': int(trip['trip_short_name'])}),
        tr_map[trip['trip_id']] = i

    with open('%s/trips.dat' % target_dir, 'wb') as f:
        f.write(struct.pack('<H', len(tr)))
        for t in tr:
            f.write(struct.pack('<HBBB', t['trip_name'], t['direction'], t['route'], t['service']))

    times_txt = list(csv.DictReader(open("%s/stop_times.txt" % source_dir)))

    tm = sorted([{
        'time': (int(x['arrival_time'].split(':')[0])*60 + int(x['arrival_time'].split(':')[1])),
        'stop': stop_map[int(x['stop_id'])],
        'sequence': int(x['stop_sequence']),
        'trip': tr_map[x['trip_id']]
    } for x in times_txt], key=lambda y: y['time'])

    with open('%s/times.dat' % target_dir, 'wb') as f:
        f.write(struct.pack('<H', len(tm)))
        for t in tm:
            f.write(struct.pack('<HHBB', t['trip'], t['time'], t['stop'], t['sequence']))

    stop_times = {stop: sorted([i for i, x in enumerate(tm) if x['stop'] == stop], key=lambda t: tm[t]['time']) for stop, s in enumerate(stops)}
    lengths = [len(x) for x in stop_times.values()]

    with open('%s/stop_index.dat' % target_dir, 'wb') as f:
        f.write(struct.pack('<B', len(lengths)))
        counter = len(lengths)*4 + 1
        for l in lengths:
            f.write(struct.pack('<HH', counter, l))
            counter += l*2

        for s in stop_times.values():
            for x in s:
                f.write(struct.pack('<H', x))

    trip_stops = [sorted([i for i, x in enumerate(tm) if x['trip'] == trip], key=lambda k: tm[k]['stop']) for trip, s in enumerate(tr)]
    lengths = map(len, trip_stops)
    with open('%s/trip_index.dat' % target_dir, 'wb') as f:
        f.write(struct.pack('<H', len(lengths)))
        counter = len(lengths) * 3 + 2
        data_start = counter
        for l in lengths:
            f.write(struct.pack('<HB', counter, l))
            counter += l*2

        if data_start != f.tell():
            raise Exception("%d != %d" % (counter, f.tell()))

        for s in trip_stops:
            for x in s:
                f.write(struct.pack('<H', x))

        if f.tell() != counter:
            raise Exception("Not the expected length!")

if __name__ == "__main__":
    generate_files("/Users/katharine/Downloads/GTFS Caltrain Devs/", "/Users/katharine/projects/pebble-caltrain/resources/data/")<|endoftext|>


<commit_before>
from into.backends.hdfstore import discover
from contextlib import contextmanager
from into.utils import tmpfile
from into.chunks import chunks
from into import into, append, convert, resource, discover
import datashape
import pandas as pd
from datetime import datetime
import numpy as np
import os

df = pd.DataFrame([['a', 1, 10., datetime(2000, 1, 1)],
                   ['ab', 2, 20., datetime(2000, 2, 2)],
                   ['abc', 3, 30., datetime(2000, 3, 3)],
                   ['abcd', 4, 40., datetime(2000, 4, 4)]],
                   columns=['name', 'a', 'b', 'time'])

@contextmanager
def file(df):
    with tmpfile('.hdf5') as fn:
        f = pd.HDFStore(fn)
        f.put('/data', df, format='table', append=True)

        try:
            yield fn, f, f.get_storer('/data')
        finally:
            f.close()

def test_discover():
    with file(df) as (fn, f, dset):
        assert str(discover(dset)) == str(discover(df))
        assert str(discover(f)) == str(discover({'data': df}))

def test_discover():
    with tmpfile('hdf5') as fn:
        df.to_hdf(fn, '/a/b/data')
        df.to_hdf(fn, '/a/b/data2')
        df.to_hdf(fn, '/a/data')

        hdf = pd.HDFStore(fn)

        assert discover(hdf) == discover({'a': {'b': {'data': df, 'data2': df},
                                                'data': df}})

def eq(a, b):
    if isinstance(a, pd.DataFrame):
        a = into(np.ndarray, a)
    if isinstance(b, pd.DataFrame):
        b = into(np.ndarray, b)
    c = a == b
    if isinstance(c, np.ndarray):
        c = c.all()
    return c

def test_chunks():
    with file(df) as (fn, f, dset):
        c = convert(chunks(pd.DataFrame), dset)
        assert eq(convert(np.ndarray, c), df)

def test_resource_no_info():
    with tmpfile('.hdf5') as fn:
        assert isinstance(resource('hdfstore://' + fn), pd.HDFStore)

def test_resource_of_dataset():
    with tmpfile('.hdf5') as fn:
        ds = datashape.dshape('{x: int32, y: 3 * int32}')
        r = resource('hdfstore://'+fn+'::/x', dshape=ds)
        assert r

def test_append():
    with file(df) as (fn, f, dset):
        append(dset, df)
        append(dset, df)
        assert discover(dset).shape == (len(df) * 3,)

def test_into_resource():
    with tmpfile('.hdf5') as fn:
        d = into('hdfstore://' + fn + '::/x', df)
        assert discover(d) == discover(df)
        assert eq(into(pd.DataFrame, d), df)

def test_convert_pandas():
    with file(df) as (fn, f, dset):
        assert eq(convert(pd.DataFrame, dset), df)

def test_convert_chunks():
    with file(df) as (fn, f, dset):
        c = convert(chunks(pd.DataFrame), dset, chunksize=len(df) / 2)
        assert len(list(c)) == 2
        assert eq(convert(pd.DataFrame, c), df)

def test_append_chunks():
    with file(df) as (fn, f, dset):
        append(dset, chunks(pd.DataFrame)([df, df]))

        assert discover(dset).shape[0] == len(df) * 3

def test_append_other():
    with tmpfile('.hdf5') as fn:
        x = into(np.ndarray, df)
        dset = into('hdfstore://'+fn+'::/data', x)
        assert discover(dset) == discover(df)

def test_fixed_shape():
    with tmpfile('.hdf5') as fn:
        df.to_hdf(fn, 'foo')
        r = resource('hdfstore://'+fn+'::/foo')
        assert isinstance(r.shape, list)
        assert discover(r).shape == (len(df),)

def test_fixed_convert():
    with tmpfile('.hdf5') as fn:
        df.to_hdf(fn, 'foo')
        r = resource('hdfstore://'+fn+'::/foo')
        assert eq(convert(pd.DataFrame, r), df)
<commit_msg>hdfstore tests close resources<commit_after>
from into.backends.hdfstore import discover
from contextlib import contextmanager
from into.utils import tmpfile
from into.chunks import chunks
from into import into, append, convert, resource, discover
import datashape
import pandas as pd
from datetime import datetime
import numpy as np
import os

df = pd.DataFrame([['a', 1, 10., datetime(2000, 1, 1)],
                   ['ab', 2, 20., datetime(2000, 2, 2)],
                   ['abc', 3, 30., datetime(2000, 3, 3)],
                   ['abcd', 4, 40., datetime(2000, 4, 4)]],
                   columns=['name', 'a', 'b', 'time'])

@contextmanager
def file(df):
    with tmpfile('.hdf5') as fn:
        f = pd.HDFStore(fn)
        f.put('/data', df, format='table', append=True)

        try:
            yield fn, f, f.get_storer('/data')
        finally:
            f.close()

def test_discover():
    with file(df) as (fn, f, dset):
        assert str(discover(dset)) == str(discover(df))
        assert str(discover(f)) == str(discover({'data': df}))

def test_discover():
    with tmpfile('hdf5') as fn:
        df.to_hdf(fn, '/a/b/data')
        df.to_hdf(fn, '/a/b/data2')
        df.to_hdf(fn, '/a/data')

        hdf = pd.HDFStore(fn)

        try:
            assert discover(hdf) == discover({'a': {'b': {'data': df, 'data2': df},
                                                    'data': df}})
        finally:
            hdf.close()

def eq(a, b):
    if isinstance(a, pd.DataFrame):
        a = into(np.ndarray, a)
    if isinstance(b, pd.DataFrame):
        b = into(np.ndarray, b)
    c = a == b
    if isinstance(c, np.ndarray):
        c = c.all()
    return c

def test_chunks():
    with file(df) as (fn, f, dset):
        c = convert(chunks(pd.DataFrame), dset)
        assert eq(convert(np.ndarray, c), df)

def test_resource_no_info():
    with tmpfile('.hdf5') as fn:
        r = resource('hdfstore://' + fn)
        assert isinstance(r, pd.HDFStore)
        r.close()

def test_resource_of_dataset():
    with tmpfile('.hdf5') as fn:
        ds = datashape.dshape('{x: int32, y: 3 * int32}')
        r = resource('hdfstore://'+fn+'::/x', dshape=ds)
        assert r
        r.parent.close()

def test_append():
    with file(df) as (fn, f, dset):
        append(dset, df)
        append(dset, df)
        assert discover(dset).shape == (len(df) * 3,)

def test_into_resource():
    with tmpfile('.hdf5') as fn:
        d = into('hdfstore://' + fn + '::/x', df)
        assert discover(d) == discover(df)
        assert eq(into(pd.DataFrame, d), df)
        d.parent.close()

def test_convert_pandas():
    with file(df) as (fn, f, dset):
        assert eq(convert(pd.DataFrame, dset), df)

def test_convert_chunks():
    with file(df) as (fn, f, dset):
        c = convert(chunks(pd.DataFrame), dset, chunksize=len(df) / 2)
        assert len(list(c)) == 2
        assert eq(convert(pd.DataFrame, c), df)

def test_append_chunks():
    with file(df) as (fn, f, dset):
        append(dset, chunks(pd.DataFrame)([df, df]))

        assert discover(dset).shape[0] == len(df) * 3

def test_append_other():
    with tmpfile('.hdf5') as fn:
        x = into(np.ndarray, df)
        dset = into('hdfstore://'+fn+'::/data', x)
        assert discover(dset) == discover(df)
        dset.parent.close()

def test_fixed_shape():
    with tmpfile('.hdf5') as fn:
        df.to_hdf(fn, 'foo')
        r = resource('hdfstore://'+fn+'::/foo')
        assert isinstance(r.shape, list)
        assert discover(r).shape == (len(df),)
        r.parent.close()

def test_fixed_convert():
    with tmpfile('.hdf5') as fn:
        df.to_hdf(fn, 'foo')
        r = resource('hdfstore://'+fn+'::/foo')
        assert eq(convert(pd.DataFrame, r), df)
        r.parent.close()
<|endoftext|>