<commit_before>"""Provide an easy interface for loading data into L{Dataframe}s for Spark.
"""
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from sparklingpandas.utils import add_pyspark_path

add_pyspark_path()
import pandas
from StringIO import StringIO as sio
from pyspark.context import SparkContext
from sparklingpandas.dataframe import Dataframe, _normalize_index_names

class PSparkContext():
    """This is a thin wrapper around SparkContext from PySpark which makes it
    easy to load data into L{PRDD}s."""

    def __init__(self, spark_context, sql_ctx=None):
        """Initialize a PSparkContext with the associacted spark context,
        and spark sql context if provided.
        :param spark_context: Initialized and configured spark context.
        :param sql_ctx: Initialized and configured SQL context, if relevant.
        :return: Correctly initialized SparklingPandasContext.
        """
        self.spark_ctx = spark_context
        if sql_ctx:
            self.sql_ctx = sql_ctx
        else:
            from pyspark.sql import SQLContext
            self.sql_ctx = SQLContext(self.spark_ctx)

    @classmethod
    def simple(cls, *args, **kwargs):
        """Takes the same arguments as SparkContext and constructs a
        PSparkContext"""
        return PSparkContext(SparkContext(*args, **kwargs))

    def read_csv(self, name, use_whole_file=False, names=None, skiprows=0,
                 *args, **kwargs):
        """Read a CSV file in and parse it into Pandas DataFrames.
        If no names is provided we use the first row for the names.
        header=0 is the default unless names is provided in which case
        header=None is the default.
        skiprows indicates how many rows of input to skip. This will
        only be applied to the first partition of the data (so if
        #skiprows > #row in first partition this will not work). Generally
        this shouldn't be an issue for small values of skiprows.
        No other values of header is supported.
        All additional parameters are passed to the read_csv function.
        TODO: Use spark-csv package if the request could be fulfilled by it.
        """
        def csv_file(partition_number, files):
            # pylint: disable=unexpected-keyword-arg
            file_count = 0
            for _, contents in files:
                # Only skip lines on the first file
                if partition_number == 0 and file_count == 0 and _skiprows > 0:
                    yield pandas.read_csv(
                        sio(contents), *args,
                        header=None,
                        names=mynames,
                        skiprows=_skiprows,
                        **kwargs)
                else:
                    file_count += 1
                    yield pandas.read_csv(
                        sio(contents), *args,
                        header=None,
                        names=mynames,
                        **kwargs)

        def csv_rows(partition_number, rows):
            # pylint: disable=unexpected-keyword-arg
            in_str = "\n".join(rows)
            if partition_number == 0:
                return iter([
                    pandas.read_csv(
                        sio(in_str), *args, header=None,
                        names=mynames,
                        skiprows=_skiprows,
                        **kwargs)])
            else:
                # could use .iterows instead?
                return iter([pandas.read_csv(sio(in_str), *args, header=None,
                                             names=mynames, **kwargs)])

        # If we need to peak at the first partition and determine the column
        # names
        mynames = None
        _skiprows = skiprows
        if names:
            mynames = names
        else:
            # In the future we could avoid this expensive call.
            first_line = self.spark_ctx.textFile(name).first()
            frame = pandas.read_csv(sio(first_line), **kwargs)
            # pylint sees frame as a tuple despite it being a Dataframe
            mynames = list(frame.columns)
            _skiprows += 1

        # Do the actual load
        if use_whole_file:
            return self.from_pandas_rdd(
                self.spark_ctx.wholeTextFiles(name)
                .mapPartitionsWithIndex(csv_file))
        else:
            return self.from_pandas_rdd(
                self.spark_ctx.textFile(name).mapPartitionsWithIndex(csv_rows))

    def parquetFile(self, *paths):
        """Loads a Parquet file, returning the result as a L{Dataframe}.
        """
        return self.from_spark_rdd(self.sql_ctx.parquetFile(paths),
                                   self.sql_ctx)

    def jsonFile(self, path, schema=None, sampling_ratio=1.0):
        """Loads a text file storing one JSON object per line as a
        L{Dataframe}.
        """
        schema_rdd = self.sql_ctx.jsonFile(path, schema, sampling_ratio)
        return self.from_spark_rdd(schema_rdd, self.sql_ctx)

    def from_pd_data_frame(self, local_df):
        """Make a distributed dataframe from a local dataframe. The intend use
        is for testing. Note: dtypes are re-infered, so they may not match."""
        def frame_to_rows(frame):
            """Convert a Panda's DataFrame into Spark SQL Rows"""
            # TODO: Convert to row objects directly?
            return [r.tolist() for r in frame.to_records()]
        schema = list(local_df.columns)
        index_names = list(local_df.index.names)
        index_names = _normalize_index_names(index_names)
        schema = index_names + schema
        rows = self.spark_ctx.parallelize(frame_to_rows(local_df))
        sp_df = Dataframe.from_schema_rdd(
            self.sql_ctx.createDataFrame(
                rows,
                schema=schema,
                # Look at all the rows, should be ok since coming from
                # a local dataset
                samplingRatio=1))
        sp_df._index_names = index_names
        return sp_df

    def sql(self, query):
        """Perform a SQL query and create a L{Dataframe} of the result."""
        return Dataframe.from_spark_rdd(self.sql_ctx.sql(query), self.sql_ctx)

    def table(self, table):
        """Returns the provided table as a L{Dataframe}"""
        return Dataframe.from_spark_rdd(self.sql_ctx.table(table),
                                        self.sql_ctx)

    def from_spark_rdd(self, spark_rdd, sql_ctx):
        """
        Translates a Spark DataFrame Rdd into a SparklingPandas dataframe.
        :param dataframe_rdd: Input dataframe RDD to convert
        :return: Matchign SparklingPandas dataframe
        """
        return Dataframe.from_spark_rdd(spark_rdd, sql_ctx)

    def DataFrame(self, elements, *args, **kwargs):
        """Wraps the pandas.DataFrame operation."""
        return self.from_pd_data_frame(pandas.DataFrame(
            elements,
            *args,
            **kwargs))

    def from_pandas_rdd(self, pandas_rdd):
        def _extract_records(data):
            return [r for r in data.to_records(index=False).tolist()]

        def _from_pandas_rdd_records(pandas_rdd_records, schema):
            """Create a L{Dataframe} from an RDD of records with schema"""
            return Dataframe.from_spark_rdd(
                self.sql_ctx.createDataFrame(pandas_rdd_records,
                                             schema.values.tolist()),
                self.sql_ctx)

        schema = pandas_rdd.map(lambda x: x.columns).first()
        rdd_records = pandas_rdd.flatMap(_extract_records)
        return _from_pandas_rdd_records(rdd_records, schema)

    def read_json(self, name,
                  *args, **kwargs):
        """Read a json file in and parse it into Pandas DataFrames.
        If no names is provided we use the first row for the names.
        Currently, it is not possible to skip the first n rows of a file.
        Headers are provided in the json file and not specified separately.
        """
        def json_file_to_df(files):
            """ Transforms a JSON file into a list of data"""
            for _, contents in files:
                yield pandas.read_json(sio(contents), *args, **kwargs)

        return self.from_pandas_rdd(self.spark_ctx.wholeTextFiles(name)
                                    .mapPartitions(json_file_to_df))

    def stop(self):
        """Stop the underlying SparkContext
        """
        self.spark_ctx.stop()
<commit_msg>Add numpy style docs to read_* methods.<commit_after>"""Provide an easy interface for loading data into L{Dataframe}s for Spark.
"""
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from sparklingpandas.utils import add_pyspark_path

add_pyspark_path()
import pandas
from StringIO import StringIO as sio
from pyspark.context import SparkContext
from sparklingpandas.dataframe import Dataframe, _normalize_index_names

class PSparkContext():
    """This is a thin wrapper around SparkContext from PySpark which makes it
    easy to load data into L{PRDD}s."""

    def __init__(self, spark_context, sql_ctx=None):
        """Initialize a PSparkContext with the associacted spark context,
        and spark sql context if provided.
        :param spark_context: Initialized and configured spark context.
        :param sql_ctx: Initialized and configured SQL context, if relevant.
        :return: Correctly initialized SparklingPandasContext.
        """
        self.spark_ctx = spark_context
        if sql_ctx:
            self.sql_ctx = sql_ctx
        else:
            from pyspark.sql import SQLContext
            self.sql_ctx = SQLContext(self.spark_ctx)

    @classmethod
    def simple(cls, *args, **kwargs):
        """Takes the same arguments as SparkContext and constructs a
        PSparkContext"""
        return PSparkContext(SparkContext(*args, **kwargs))

    def read_csv(self, file_path, use_whole_file=False, names=None, skiprows=0,
                 *args, **kwargs):
        """Read a CSV file in and parse it into Pandas DataFrames. By default,
        the first row from the first partition of that data is parsed and used
        as the column names for the data from. If no 'names' param is
        provided we parse the first row of the first partition of data and
        use it for column names.

        Parameters
        ----------
        file_path: string
            Path to input. Any valid file path in Spark works here, eg:
            'my/path/in/local/file/system' or 'hdfs:/user/juliet/'
        use_whole_file: boolean
            Whether of not to use the whole file.
        names: list of strings, optional
        skiprows: integer, optional
            indicates how many rows of input to skip. This will
            only be applied to the first partition of the data (so if
            #skiprows > #row in first partition this will not work). Generally
            this shouldn't be an issue for small values of skiprows.
        No other value of header is supported.
        All additional parameters available in pandas.read_csv() are usable
        here.

        Returns
        -------
        A SparklingPandas DataFrame that contains the data from the
        specified file.
        """
        def csv_file(partition_number, files):
            # pylint: disable=unexpected-keyword-arg
            file_count = 0
            for _, contents in files:
                # Only skip lines on the first file
                if partition_number == 0 and file_count == 0 and _skiprows > 0:
                    yield pandas.read_csv(
                        sio(contents), *args,
                        header=None,
                        names=mynames,
                        skiprows=_skiprows,
                        **kwargs)
                else:
                    file_count += 1
                    yield pandas.read_csv(
                        sio(contents), *args,
                        header=None,
                        names=mynames,
                        **kwargs)

        def csv_rows(partition_number, rows):
            # pylint: disable=unexpected-keyword-arg
            in_str = "\n".join(rows)
            if partition_number == 0:
                return iter([
                    pandas.read_csv(
                        sio(in_str), *args, header=None,
                        names=mynames,
                        skiprows=_skiprows,
                        **kwargs)])
            else:
                # could use .iterows instead?
                return iter([pandas.read_csv(sio(in_str), *args, header=None,
                                             names=mynames, **kwargs)])

        # If we need to peak at the first partition and determine the column
        # names
        mynames = None
        _skiprows = skiprows
        if names:
            mynames = names
        else:
            # In the future we could avoid this expensive call.
            first_line = self.spark_ctx.textFile(file_path).first()
            frame = pandas.read_csv(sio(first_line), **kwargs)
            # pylint sees frame as a tuple despite it being a Dataframe
            mynames = list(frame.columns)
            _skiprows += 1

        # Do the actual load
        if use_whole_file:
            return self.from_pandas_rdd(
                self.spark_ctx.wholeTextFiles(name)
                .mapPartitionsWithIndex(csv_file))
        else:
            return self.from_pandas_rdd(
                self.spark_ctx.textFile(name).mapPartitionsWithIndex(csv_rows))

    def parquetFile(self, *paths):
        """Loads a Parquet file, returning the result as a L{Dataframe}.
        """
        return self.from_spark_rdd(self.sql_ctx.parquetFile(paths),
                                   self.sql_ctx)

    def jsonFile(self, path, schema=None, sampling_ratio=1.0):
        """Loads a text file storing one JSON object per line as a
        L{Dataframe}.
        """
        schema_rdd = self.sql_ctx.jsonFile(path, schema, sampling_ratio)
        return self.from_spark_rdd(schema_rdd, self.sql_ctx)

    def from_pd_data_frame(self, local_df):
        """Make a distributed dataframe from a local dataframe. The intend use
        is for testing. Note: dtypes are re-infered, so they may not match."""
        def frame_to_rows(frame):
            """Convert a Panda's DataFrame into Spark SQL Rows"""
            # TODO: Convert to row objects directly?
            return [r.tolist() for r in frame.to_records()]
        schema = list(local_df.columns)
        index_names = list(local_df.index.names)
        index_names = _normalize_index_names(index_names)
        schema = index_names + schema
        rows = self.spark_ctx.parallelize(frame_to_rows(local_df))
        sp_df = Dataframe.from_schema_rdd(
            self.sql_ctx.createDataFrame(
                rows,
                schema=schema,
                # Look at all the rows, should be ok since coming from
                # a local dataset
                samplingRatio=1))
        sp_df._index_names = index_names
        return sp_df

    def sql(self, query):
        """Perform a SQL query and create a L{Dataframe} of the result."""
        return Dataframe.from_spark_rdd(self.sql_ctx.sql(query), self.sql_ctx)

    def table(self, table):
        """Returns the provided table as a L{Dataframe}"""
        return Dataframe.from_spark_rdd(self.sql_ctx.table(table),
                                        self.sql_ctx)

    def from_spark_rdd(self, spark_rdd, sql_ctx):
        """
        Translates a Spark DataFrame Rdd into a SparklingPandas dataframe.
        :param dataframe_rdd: Input dataframe RDD to convert
        :return: Matchign SparklingPandas dataframe
        """
        return Dataframe.from_spark_rdd(spark_rdd, sql_ctx)

    def DataFrame(self, elements, *args, **kwargs):
        """Wraps the pandas.DataFrame operation."""
        return self.from_pd_data_frame(pandas.DataFrame(
            elements,
            *args,
            **kwargs))

    def from_pandas_rdd(self, pandas_rdd):
        def _extract_records(data):
            return [r for r in data.to_records(index=False).tolist()]

        def _from_pandas_rdd_records(pandas_rdd_records, schema):
            """Create a L{Dataframe} from an RDD of records with schema"""
            return Dataframe.from_spark_rdd(
                self.sql_ctx.createDataFrame(pandas_rdd_records,
                                             schema.values.tolist()),
                self.sql_ctx)

        schema = pandas_rdd.map(lambda x: x.columns).first()
        rdd_records = pandas_rdd.flatMap(_extract_records)
        return _from_pandas_rdd_records(rdd_records, schema)

    def read_json(self, file_path,
                  *args, **kwargs):
        """Read a json file in and parse it into Pandas DataFrames.
        If no names is provided we use the first row for the names.
        Currently, it is not possible to skip the first n rows of a file.
        Headers are provided in the json file and not specified separately.

        Parameters
        ----------
        file_path: string
            Path to input. Any valid file path in Spark works here, eg:
            'my/path/in/local/file/system' or 'hdfs:/user/juliet/'
        Other than skipRows, all additional parameters available in
        pandas.read_csv() are usable here.

        Returns
        -------
            A SparklingPandas DataFrame that contains the data from the
        specified file.
        """
        def json_file_to_df(files):
            """ Transforms a JSON file into a list of data"""
            for _, contents in files:
                yield pandas.read_json(sio(contents), *args, **kwargs)

        return self.from_pandas_rdd(self.spark_ctx.wholeTextFiles(file_path)
                                    .mapPartitions(json_file_to_df))

    def stop(self):
        """Stop the underlying SparkContext
        """
        self.spark_ctx.stop()
<|endoftext|>


<commit_before>import decimal
import enum
import jqsh.filter
import string
import unicodedata

class Incomplete(Exception):
    pass

TokenType = enum.Enum('TokenType', [
    'close_array',
    'close_object',
    'close_paren',
    'colon',
    'comma',
    'comment',
    'illegal',
    'name',
    'number',
    'open_array',
    'open_object',
    'open_paren',
    'string',
    'trailing_whitespace'
], module=__name__)

class Token:
    def __eq__(self, other):
        return self.type is other.type and self.text == other.text

    def __init__(self, token_type, token_string=None, text=None):
        self.type = token_type
        self.string = token_string # ''.join(token.string for token in tokenize(jqsh_string)) == jqsh_string
        self.text = text # metadata like the name of a name token or the digits of a number literal. None for simple tokens

    def __repr__(self):
        return 'jqsh.parser.Token(' + repr(self.type) + ('' if self.string is None else ', token_string=' + repr(self.string)) + ('' if self.text is None else ', text=' + repr(self.text)) + ')'

    def __str__(self):
        if self.string is None:
            return "'" + repr(self) + "'"
        else:
            return self.string

json_tokens = [ # token types that are allowed in pure JSON
    TokenType.close_array,
    TokenType.close_object,
    TokenType.colon,
    TokenType.comma,
    TokenType.name,
    TokenType.number,
    TokenType.open_array,
    TokenType.open_object,
    TokenType.string,
    TokenType.trailing_whitespace
]

matching_parens = { # a dictionary that maps opening parenthesis-like tokens (parens) to the associated closing parens
    TokenType.open_array: TokenType.close_array,
    TokenType.open_object: TokenType.close_object,
    TokenType.open_paren: TokenType.close_paren
}

paren_filters = {
    TokenType.open_array: jqsh.filter.Array,
    TokenType.open_paren: jqsh.filter.Parens
}

symbols = {
    '(': TokenType.open_paren,
    ')': TokenType.close_paren,
    ',': TokenType.comma,
    ':': TokenType.colon,
    '[': TokenType.open_array,
    ']': TokenType.close_array,
    '{': TokenType.open_object,
    '}': TokenType.close_object
}

def illegal_token_exception(token, position=None, expected=None):
    if token.type is TokenType.illegal:
        return SyntaxError('illegal character' + ('' if position is None else ' at position ' + repr(position)) + ': ' + repr(token.text[0]) + ' (U+' + format(ord(token.text[0]), 'x').upper() + ' ' + unicodedata.name(token.text[0], 'unknown character') + ')')
    else:
        return SyntaxError('illegal ' + token.type.name + ' token' + ('' if position is None else ' at position ' + repr(position)) + ('' if expected is None else ' (expected ' + ' or '.join(sorted(expected_token_type.name for expected_token_type in expected)) + ')'))

def parse(tokens):
    if isinstance(tokens, str):
        tokens = list(tokenize(tokens))
    tokens = [token for token in tokens if isinstance(token, jqsh.filter.Filter) or token.type is not TokenType.comment]
    if not len(tokens):
        return jqsh.filter.Filter() # token list is empty, return an empty filter
    for token in tokens:
        if token.type is TokenType.illegal:
            raise illegal_token_exception(token)
    if isinstance(tokens[-1], Token) and tokens[-1].type is TokenType.trailing_whitespace:
        if len(tokens) == 1:
            return jqsh.filter.Filter() # token list consists entirely of whitespace, return an empty filter
        else:
            tokens[-2].string += tokens[-1].string # merge the trailing whitespace into the second-to-last token
            tokens.pop() # remove the trailing_whitespace token
    paren_balance = 0
    paren_start = None
    for i, token in reversed(list(enumerate(tokens))): # iterating over the token list in reverse because we modify it in the process
        if not isinstance(token, Token):
            continue
        elif token.type in matching_parens.values():
            if paren_balance == 0:
                paren_start = i
            paren_balance += 1
        elif token.type in matching_parens.keys():
            paren_balance -= 1
            if paren_balance < 0:
                raise Incomplete('too many opening parens of type ' + repr(token.type))
            elif paren_balance == 0:
                if matching_parens[token.type] is tokens[paren_start].type:
                    tokens[i:paren_start + 1] = [paren_filters[token.type](attribute=parse(tokens[i + 1:paren_start]))] # parse the inside of the parens
                else:
                    raise SyntaxError('opening paren of type ' + repr(token.type) + ' does not match closing paren of type ' + repr(tokens[paren_start].type))
                paren_start = None
    if paren_balance != 0:
        raise SyntaxError('mismatched parens')
    if len(tokens) == 1 and isinstance(tokens[0], jqsh.filter.Filter):
        return tokens[0] # finished parsing
    else:
        raise SyntaxError('could not parse token list')

def parse_json(tokens):
    if isinstance(tokens, str):
        tokens = list(tokenize(tokens))
    if not len(tokens):
        raise Incomplete('JSON is empty')
    for token in tokens:
        if token.type not in json_tokens:
            raise illegal_token_exception(token)
    ret = None
    key_path = []
    token_index = 0
    while token_index < len(tokens):
        token = tokens[token_index]
        if token.type is TokenType.name:
            if token.text == 'false':
                ret = set_value_at_key_path(ret, key_path, False)
                token_index += 1
            elif token.text == 'null':
                ret = set_value_at_key_path(ret, key_path, None)
                token_index += 1
            elif token.text == 'true':
                ret = set_value_at_key_path(ret, key_path, True)
                token_index += 1
            else:
                raise SyntaxError('illegal name token ' + repr(token.text) + ' at position ' + repr(token_index) + ' (expected false, null, or true)')
        elif token.type is TokenType.number:
            ret = set_value_at_key_path(ret, key_path, decimal.Decimal(token.text))
            token_index += 1
        elif token.type is TokenType.open_array:
            ret = set_value_at_key_path(ret, key_path, [])
            token_index += 1
            if token_index >= len(tokens):
                raise Incomplete('unclosed JSON array at key_path ' + repr(key_path))
            if tokens[token_index].type is TokenType.close_array:
                token_index += 1 # empty array parsed
            else:
                key_path.append(0)
                continue
        elif token.type is TokenType.open_object:
            ret = set_value_at_key_path(ret, key_path, {})
            token_index += 1
            if token_index >= len(tokens):
                raise Incomplete('unclosed JSON object at key path ' + repr(key_path))
            token = tokens[token_index]
            if token.type is TokenType.close_object:
                token_index += 1 # empty object parsed
            elif token.type is TokenType.string:
                key_path.append(token.text)
                token_index += 1
                if token_index >= len(tokens):
                    raise Incomplete('unclosed JSON object at key path ' + repr(key_path[:-1]))
                elif tokens[token_index].type is not TokenType.colon:
                    raise illegal_token_exception(token, position=token_index, expected={TokenType.colon})
                else:
                    token_index += 1
                    continue
            else:
                raise illegal_token_exception(token, position=token_index, expected={TokenType.close_object, TokenType.string})
        elif token.type is TokenType.string:
            ret = set_value_at_key_path(ret, key_path, token.text)
            token_index += 1
        elif token.type is TokenType.trailing_whitespace:
            token_index += 1
            if token_index < len(tokens):
                raise SyntaxError('found a trailing_whitespace token in a non-trailing position')
        else:
            raise illegal_token_exception(token, position=token_index, expected={TokenType.name, TokenType.number, TokenType.open_array, TokenType.open_object, TokenType.string, TokenType.trailing_whitespace})
        keep_closing = True
        while keep_closing and len(key_path):
            if isinstance(key_path[-1], str): # we are in an object, get the next key or close it
                if token_index >= len(tokens):
                    raise Incomplete('unclosed JSON object at key path ' + repr(key_path[:-1]))
                token = tokens[token_index]
                if token.type is TokenType.close_object:
                    key_path.pop()
                    token_index += 1
                elif token.type is TokenType.comma:
                    token_index += 1
                    if token_index >= len(tokens):
                        raise Incomplete('unclosed JSON object at key path ' + repr(key_path[:-1]))
                    token = tokens[token_index]
                    if token.type is TokenType.string:
                        key_path[-1] = token.text
                        token_index += 1
                        if token_index >= len(tokens):
                            raise Incomplete('unclosed JSON object at key path ' + repr(key_path[:-1]))
                        elif tokens[token_index].type is not TokenType.colon:
                            raise illegal_token_exception(token, position=token_index, expected={TokenType.colon})
                        else:
                            token_index += 1
                            keep_closing = False
                    else:
                        raise illegal_token_exception(token, position=token_index, expected={TokenType.string})
                else:
                    raise illegal_token_exception(token, position=token_index, expected={TokenType.close_object, TokenType.comma})
            else: # we are in an array, check if it continues
                if token_index >= len(tokens):
                    raise Incomplete('unclosed JSON array at key path ' + repr(key_path[:-1]))
                token = tokens[token_index]
                if token.type is TokenType.close_array:
                    key_path.pop()
                    token_index += 1
                elif token.type is TokenType.comma:
                    key_path[-1] += 1
                    token_index += 1
                    keep_closing = False
                else:
                    raise illegal_token_exception(token, position=token_index, expected={TokenType.close_array, TokenType.comma})
    if token_index < len(tokens):
        raise SyntaxError('multiple top-level JSON values found')
    return ret

def set_value_at_key_path(structure, key_path, value):
    if len(key_path):
        substructure = structure
        for key in key_path[:-1]:
            substructure = substructure[key]
        if isinstance(key_path[-1], int) and len(substructure) == key_path[-1]:
            substructure.append(value)
        else:
            substructure[key_path[-1]] = value
        return structure
    else:
        return value

def tokenize(jqsh_string):
    rest_string = jqsh_string
    if not isinstance(rest_string, str):
        rest_string = rest_string.decode('utf-8')
    whitespace_prefix = ''
    if rest_string.startswith('\ufeff'):
        whitespace_prefix += rest_string[0]
        rest_string = rest_string[1:]
    while len(rest_string):
        if rest_string[0] in string.whitespace:
            whitespace_prefix += rest_string[0]
            rest_string = rest_string[1:]
        elif rest_string[0] == '#':
            rest_string = rest_string[1:]
            comment = ''
            while len(rest_string):
                if rest_string[0] == '\n':
                    break
                comment += rest_string[0]
                rest_string = rest_string[1:]
            yield Token(TokenType.comment, token_string=whitespace_prefix + '#' + comment, text=comment)
            whitespace_prefix = ''
        elif any(rest_string.startswith(symbol) for symbol in symbols):
            for symbol, token_type in sorted(symbols.items(), key=lambda pair: -len(pair[0])): # look at longer symbols first, so that a += is not mistakenly tokenized as a +
                if rest_string.startswith(symbol):
                    yield Token(token_type, token_string=whitespace_prefix + rest_string[:len(symbol)])
                    whitespace_prefix = ''
                    rest_string = rest_string[len(symbol):]
                    break
        else:
            yield Token(TokenType.illegal, token_string=whitespace_prefix + rest_string, text=rest_string)
            whitespace_prefix = ''
            rest_string = ''
    if len(whitespace_prefix):
        yield Token(TokenType.trailing_whitespace, token_string=whitespace_prefix)
<commit_msg>Fix JSON parser, add json_to_tokens, parse names and basic number literals<commit_after>import decimal
import enum
import jqsh.filter
import string
import unicodedata

class Incomplete(Exception):
    pass

TokenType = enum.Enum('TokenType', [
    'close_array',
    'close_object',
    'close_paren',
    'colon',
    'comma',
    'comment',
    'dot',
    'illegal',
    'minus',
    'name',
    'number',
    'open_array',
    'open_object',
    'open_paren',
    'string',
    'string_end',
    'string_end_incomplete',
    'string_incomplete',
    'string_middle',
    'string_start',
    'trailing_whitespace'
], module=__name__)

class Token:
    def __eq__(self, other):
        return self.type is other.type and self.text == other.text

    def __init__(self, token_type, token_string=None, text=None):
        self.type = token_type
        self.string = token_string # ''.join(token.string for token in tokenize(jqsh_string)) == jqsh_string
        self.text = text # metadata like the name of a name token or the digits of a number literal. None for simple tokens

    def __repr__(self):
        return 'jqsh.parser.Token(' + repr(self.type) + ('' if self.string is None else ', token_string=' + repr(self.string)) + ('' if self.text is None else ', text=' + repr(self.text)) + ')'

    def __str__(self):
        if self.string is None:
            return "'" + repr(self) + "'"
        else:
            return self.string

json_tokens = [ # token types that are allowed in pure JSON
    TokenType.close_array,
    TokenType.close_object,
    TokenType.colon,
    TokenType.comma,
    TokenType.name,
    TokenType.number,
    TokenType.open_array,
    TokenType.open_object,
    TokenType.string,
    TokenType.trailing_whitespace
]

matching_parens = { # a dictionary that maps opening parenthesis-like tokens (parens) to the associated closing parens
    TokenType.open_array: TokenType.close_array,
    TokenType.open_object: TokenType.close_object,
    TokenType.open_paren: TokenType.close_paren
}

paren_filters = {
    TokenType.open_array: jqsh.filter.Array,
    TokenType.open_paren: jqsh.filter.Parens
}

symbols = {
    '(': TokenType.open_paren,
    ')': TokenType.close_paren,
    ',': TokenType.comma,
    '-': TokenType.minus,
    '.': TokenType.dot,
    ':': TokenType.colon,
    '[': TokenType.open_array,
    ']': TokenType.close_array,
    '{': TokenType.open_object,
    '}': TokenType.close_object
}

def illegal_token_exception(token, position=None, expected=None):
    if token.type is TokenType.illegal:
        return SyntaxError('illegal character' + ('' if position is None else ' at position ' + repr(position)) + ': ' + repr(token.text[0]) + ' (U+' + format(ord(token.text[0]), 'x').upper() + ' ' + unicodedata.name(token.text[0], 'unknown character') + ')')
    else:
        return SyntaxError('illegal ' + token.type.name + ' token' + ('' if position is None else ' at position ' + repr(position)) + ('' if expected is None else ' (expected ' + ' or '.join(sorted(expected_token_type.name for expected_token_type in expected)) + ')'))

def json_to_tokens(json_value):
    if json_value is False:
        yield Token(TokenType.name, text='false')
    elif json_value is None:
        yield Token(TokenType.name, text='null')
    elif json_value is True:
        yield Token(TokenType.name, text='true')
    elif isinstance(json_value, decimal.Decimal) or isinstance(json_value, int) or isinstance(json_value, float):
        yield Token(TokenType.number, text=str(json_value))
    elif isinstance(json_value, list):
        yield Token(TokenType.open_array)
        for i, item in enumerate(json_value):
            if i > 0:
                yield Token(TokenType.comma)
            yield from json_to_tokens(item)
        yield Token(TokenType.close_array)
    elif isinstance(json_value, dict):
        yield Token(TokenType.open_object)
        for i, (key, value) in enumerate(sorted(json_value.items(), key=lambda pair: pair[0])): # sort the pairs of an object by their names
            if i > 0:
                yield Token(TokenType.comma)
            yield Token(TokenType.string, text=key)
            yield Token(TokenType.colon)
            yield from json_to_tokens(value)
        yield Token(TokenType.close_object)
    else:
        yield Token(TokenType.illegal)

def parse(tokens):
    if isinstance(tokens, str):
        tokens = list(tokenize(tokens))
    tokens = [token for token in tokens if isinstance(token, jqsh.filter.Filter) or token.type is not TokenType.comment]
    if not len(tokens):
        return jqsh.filter.Filter() # token list is empty, return an empty filter
    for token in tokens:
        if token.type is TokenType.illegal:
            raise illegal_token_exception(token)
    if isinstance(tokens[-1], Token) and tokens[-1].type is TokenType.trailing_whitespace:
        if len(tokens) == 1:
            return jqsh.filter.Filter() # token list consists entirely of whitespace, return an empty filter
        else:
            tokens[-2].string += tokens[-1].string # merge the trailing whitespace into the second-to-last token
            tokens.pop() # remove the trailing_whitespace token
    paren_balance = 0
    paren_start = None
    for i, token in reversed(list(enumerate(tokens))): # iterating over the token list in reverse because we modify it in the process
        if not isinstance(token, Token):
            continue
        elif token.type in matching_parens.values():
            if paren_balance == 0:
                paren_start = i
            paren_balance += 1
        elif token.type in matching_parens.keys():
            paren_balance -= 1
            if paren_balance < 0:
                raise Incomplete('too many opening parens of type ' + repr(token.type))
            elif paren_balance == 0:
                if matching_parens[token.type] is tokens[paren_start].type:
                    tokens[i:paren_start + 1] = [paren_filters[token.type](attribute=parse(tokens[i + 1:paren_start]))] # parse the inside of the parens
                else:
                    raise SyntaxError('opening paren of type ' + repr(token.type) + ' does not match closing paren of type ' + repr(tokens[paren_start].type))
                paren_start = None
    if paren_balance != 0:
        raise SyntaxError('mismatched parens')
    if len(tokens) == 1 and isinstance(tokens[0], jqsh.filter.Filter):
        return tokens[0] # finished parsing
    else:
        raise SyntaxError('could not parse token list')

def parse_json(tokens):
    if isinstance(tokens, str):
        tokens = list(tokenize(tokens))
    if not len(tokens):
        raise Incomplete('JSON is empty')
    for token in tokens:
        if token.type not in json_tokens:
            raise illegal_token_exception(token)
    ret = None
    key_path = []
    token_index = 0
    while token_index < len(tokens):
        token = tokens[token_index]
        if token.type is TokenType.name:
            if token.text == 'false':
                ret = set_value_at_key_path(ret, key_path, False)
                token_index += 1
            elif token.text == 'null':
                ret = set_value_at_key_path(ret, key_path, None)
                token_index += 1
            elif token.text == 'true':
                ret = set_value_at_key_path(ret, key_path, True)
                token_index += 1
            else:
                raise SyntaxError('illegal name token ' + repr(token.text) + ' at position ' + repr(token_index) + ' (expected false, null, or true)')
        elif token.type is TokenType.number:
            ret = set_value_at_key_path(ret, key_path, decimal.Decimal(token.text))
            token_index += 1
        elif token.type is TokenType.open_array:
            ret = set_value_at_key_path(ret, key_path, [])
            token_index += 1
            if token_index >= len(tokens):
                raise Incomplete('unclosed JSON array at key_path ' + repr(key_path))
            if tokens[token_index].type is TokenType.close_array:
                token_index += 1 # empty array parsed
            else:
                key_path.append(0)
                continue
        elif token.type is TokenType.open_object:
            ret = set_value_at_key_path(ret, key_path, {})
            token_index += 1
            if token_index >= len(tokens):
                raise Incomplete('unclosed JSON object at key path ' + repr(key_path))
            token = tokens[token_index]
            if token.type is TokenType.close_object:
                token_index += 1 # empty object parsed
            elif token.type is TokenType.string:
                key_path.append(token.text)
                token_index += 1
                if token_index >= len(tokens):
                    raise Incomplete('unclosed JSON object at key path ' + repr(key_path[:-1]))
                elif tokens[token_index].type is not TokenType.colon:
                    raise illegal_token_exception(token, position=token_index, expected={TokenType.colon})
                else:
                    token_index += 1
                    continue
            else:
                raise illegal_token_exception(token, position=token_index, expected={TokenType.close_object, TokenType.string})
        elif token.type is TokenType.string:
            ret = set_value_at_key_path(ret, key_path, token.text)
            token_index += 1
        elif token.type is TokenType.trailing_whitespace:
            token_index += 1
            if token_index < len(tokens):
                raise SyntaxError('found a trailing_whitespace token in a non-trailing position')
        else:
            raise illegal_token_exception(token, position=token_index, expected={TokenType.name, TokenType.number, TokenType.open_array, TokenType.open_object, TokenType.string, TokenType.trailing_whitespace})
        keep_closing = True
        while keep_closing and len(key_path):
            if isinstance(key_path[-1], str): # we are in an object, get the next key or close it
                if token_index >= len(tokens):
                    raise Incomplete('unclosed JSON object at key path ' + repr(key_path[:-1]))
                token = tokens[token_index]
                if token.type is TokenType.close_object:
                    key_path.pop()
                    token_index += 1
                elif token.type is TokenType.comma:
                    token_index += 1
                    if token_index >= len(tokens):
                        raise Incomplete('unclosed JSON object at key path ' + repr(key_path[:-1]))
                    token = tokens[token_index]
                    if token.type is TokenType.string:
                        key_path[-1] = token.text
                        token_index += 1
                        if token_index >= len(tokens):
                            raise Incomplete('unclosed JSON object at key path ' + repr(key_path[:-1]))
                        elif tokens[token_index].type is not TokenType.colon:
                            raise illegal_token_exception(token, position=token_index, expected={TokenType.colon})
                        else:
                            token_index += 1
                            if token_index >= len(tokens):
                                raise Incomplete('unclosed JSON object at key path ' + repr(key_path[:-1]))
                            keep_closing = False
                    else:
                        raise illegal_token_exception(token, position=token_index, expected={TokenType.string})
                else:
                    raise illegal_token_exception(token, position=token_index, expected={TokenType.close_object, TokenType.comma})
            else: # we are in an array, check if it continues
                if token_index >= len(tokens):
                    raise Incomplete('unclosed JSON array at key path ' + repr(key_path[:-1]))
                token = tokens[token_index]
                if token.type is TokenType.close_array:
                    key_path.pop()
                    token_index += 1
                elif token.type is TokenType.comma:
                    key_path[-1] += 1
                    token_index += 1
                    if token_index >= len(tokens):
                        raise Incomplete('unclosed JSON array at key path ' + repr(key_path[:-1]))
                    keep_closing = False
                else:
                    raise illegal_token_exception(token, position=token_index, expected={TokenType.close_array, TokenType.comma})
    if token_index < len(tokens):
        raise SyntaxError('multiple top-level JSON values found')
    return ret

def set_value_at_key_path(structure, key_path, value):
    if len(key_path):
        substructure = structure
        for key in key_path[:-1]:
            substructure = substructure[key]
        if isinstance(key_path[-1], int) and len(substructure) == key_path[-1]:
            substructure.append(value)
        else:
            substructure[key_path[-1]] = value
        return structure
    else:
        return value

def tokenize(jqsh_string):
    rest_string = jqsh_string
    if not isinstance(rest_string, str):
        rest_string = rest_string.decode('utf-8')
    whitespace_prefix = ''
    if rest_string.startswith('\ufeff'):
        whitespace_prefix += rest_string[0]
        rest_string = rest_string[1:]
    while len(rest_string):
        if rest_string[0] in string.whitespace:
            whitespace_prefix += rest_string[0]
            rest_string = rest_string[1:]
        elif rest_string[0] == '#':
            rest_string = rest_string[1:]
            comment = ''
            while len(rest_string):
                if rest_string[0] == '\n':
                    break
                comment += rest_string[0]
                rest_string = rest_string[1:]
            yield Token(TokenType.comment, token_string=whitespace_prefix + '#' + comment, text=comment)
            whitespace_prefix = ''
        elif rest_string[0] in string.ascii_letters:
            name = ''
            while len(rest_string) and rest_string[0] in string.ascii_letters:
                name += rest_string[0]
                rest_string = rest_string[1:]
            yield Token(TokenType.name, token_string=whitespace_prefix + name, text=name)
            whitespace_prefix = ''
        elif rest_string[0] in string.digits:
            number = ''
            while len(rest_string) and rest_string[0] in string.digits:
                number += rest_string[0]
                rest_string = rest_string[1:]
            yield Token(TokenType.number, token_string=whitespace_prefix + number, text=number)
            whitespace_prefix = ''
        elif any(rest_string.startswith(symbol) for symbol in symbols):
            for symbol, token_type in sorted(symbols.items(), key=lambda pair: -len(pair[0])): # look at longer symbols first, so that a += is not mistakenly tokenized as a +
                if rest_string.startswith(symbol):
                    yield Token(token_type, token_string=whitespace_prefix + rest_string[:len(symbol)])
                    whitespace_prefix = ''
                    rest_string = rest_string[len(symbol):]
                    break
        else:
            yield Token(TokenType.illegal, token_string=whitespace_prefix + rest_string, text=rest_string)
            whitespace_prefix = ''
            rest_string = ''
    if len(whitespace_prefix):
        yield Token(TokenType.trailing_whitespace, token_string=whitespace_prefix)
<|endoftext|>


<commit_before>from importlib import reload

import pandas as pd
import numpy as np
from textblob import TextBlob
from sklearn.decomposition import LatentDirichletAllocation, NMF
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from scipy.spatial.distance import pdist
from sklearn.preprocessing import MultiLabelBinarizer, LabelBinarizer
from nltk.corpus import reuters, brown
from sklearn.datasets import fetch_20newsgroups
from sklearn.model_selection import GridSearchCV
from sklearn.svm import LinearSVC, SVC
from sklearn import metrics
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.model_selection import cross_val_score
from sklearn.metrics.pairwise import cosine_similarity

# Add general functions to the project
from os import path
import sys
sys.path.append(path.abspath('../methods'))

def print_top_words(model, feature_names, n_top_words):
    for topic_idx, topic in enumerate(model.components_):
        message = "Topic #%d: " % topic_idx
        message += " ".join([feature_names[i]
                             for i in topic.argsort()[:-n_top_words - 1:-1]])
        print(message)
    print()

def get_top_words(model, feature_names, n_top_words):
    out = []
    for topic_idx, topic in enumerate(model.components_):
        topics = [feature_names[i] for i in topic.argsort()[:-n_top_words - 1:-1]]
        out.extend(topics)

    return set(out)

# DATA Loading
raw = np.load("../datasets/full.pkl")

# Remove Unclear from groups/clusters
# raw["Clusters"] = data.apply(
#     lambda row: row["Clusters"]
#     .replace(";Unclear", "")
#     .replace(";;", ";"), axis=1
#     )

# train/test split
# test = raw[raw["DOI"].str.contains("2013|2012")]  # len = 197
# train = raw.drop(test.index)  # len = 1280
label_column = ["Clusters"]

# data
# y
y = np.array(raw["Title"])

# x
abstract_tfidf = TfidfVectorizer(stop_words="english").fit(raw["Abstract"])
abstract_train = abstract_tfidf.transform(raw["Abstract"])

full_text_tfidf = TfidfVectorizer(stop_words="english").fit(raw["Fulltext"])
fulltext_train = full_text_tfidf.transform(raw["Fulltext"])

# params
n_dim = [4,6,8,10,12,15,20,25,30]
algorithms = ["lda", "nmf"]
datasets = ["abstract", "fulltext"]

n_runs_per_setting = 10
topic_consistency_thresholds = [5, 10, 20]

# Create all parameter permutations
protocol = pd.DataFrame(columns=["Algorithm", "Dataset", "Dimensions", "NumberRuns", "Doc", "JaccTop5Median", "JaccTop10Median", "JaccTop20Median"])
runs = np.stack(np.meshgrid(algorithms, datasets, n_dim), -1).reshape(-1, 3)

# Compute
for run in runs:
    ldas = []
    vecs = []
    similarities = []
    row = []

    # Set variables
    if run[1] == "abstract":
        used = abstract_train
        tfidf = abstract_tfidf
    if run[1] == "fulltext":
        used = fulltext_train
        tfidf = full_text_tfidf

    # Fill protocol
    row.extend(run)
    row.append(n_runs_per_setting)
    # Compute models
    for iteration in range(0, n_runs_per_setting):
        if run[0] == "lda":
            lda = LatentDirichletAllocation(int(run[2]), learning_method="batch")
            vec = lda.fit_transform(used)
        if run[0] == "nmf":
            lda = NMF(int(run[2]))
            vec = lda.fit_transform(used)

        similarity = cosine_similarity(vec)

        ldas.append(lda)
        vecs.append(vec)
        similarities.append(similarity)

    # Document neighborhood consistency
    doc_sim = []
    for i in range(0, len(similarities)):
        for j in range(i+1, len(similarities)):
            doc_sim.append(abs(similarities[i] - similarities[j]).sum())

    row.append(np.median(doc_sim))
    # print(y[similarity1[156].argsort()[-10:][::-1]])
    # print(y[similarity2[156].argsort()[-10:][::-1]])

    # Topic consistency
    for thres in topic_consistency_thresholds:
        top_sim = []

        for i in range(0, len(similarities)):
            topics1 = get_top_words(ldas[i], tfidf.get_feature_names(), thres)
            for j in range(i+1, len(similarities)):
                topics2 = get_top_words(ldas[j], tfidf.get_feature_names(), thres)

                top_sim.append(len(topics1.intersection(topics2))/len(topics1.union(topics2)))

        row.append(np.median(top_sim))
        # print_top_words(lda1, tfidf.get_feature_names(), 10)
        # print_top_words(lda2, tfidf.get_feature_names(), 10)

    protocol = protocol.append(pd.DataFrame([row], columns=protocol.columns))

# LDA Fulltext dim 6 is best (drop on 8)
# LDA Abstract dim 4 is best (far worse then the NMF 4 or 6)
# NMF Fulltext dim 10 is best (drop on 12)
# NMF Abstract dim 6 is best (drop on 8)
protocol.to_csv("runs.csv")

# Filter dimensions
# np.median(vecs1, axis=0)
# np.std(vecs1, axis=0)

print_top_words(ldas[1], tfidf.get_feature_names(), 10)
print_top_words(ldas[0], tfidf.get_feature_names(), 10)

# Create embeddings
dois = raw["DOI"]

# LDA_NMF data
# lda = LatentDirichletAllocation(6, learning_method="batch")
# vecs_lda = lda.fit_transform(fulltext_train)
#
# vecs_lda = np.asarray(vecs_lda, dtype=np.object)
# vecs_lda = np.insert(vecs_lda, 0, dois, axis=1)
#
# emb1 = pd.DataFrame(vecs_lda)
# emb1.to_csv("lda_nmf_1.csv", header=False)
#
# nmf = NMF(10)
# vecs_nmf = nmf.fit_transform(fulltext_train)
#
# vecs_nmf = np.asarray(vecs_nmf, dtype=np.object)
# vecs_nmf = np.insert(vecs_nmf, 0, dois, axis=1)
#
# emb2 = pd.DataFrame(vecs_nmf)
# emb2.to_csv("lda_nmf_2.csv", header=False)

# abstract_fulltext data
lda = NMF(6)
vecs_lda = lda.fit_transform(abstract_train)

vecs_lda = np.asarray(vecs_lda, dtype=np.object)
vecs_lda = np.insert(vecs_lda, 0, dois, axis=1)

emb1 = pd.DataFrame(vecs_lda)
emb1.to_csv("full_abstract_1.csv", header=False)

nmf = NMF(10)
vecs_nmf = nmf.fit_transform(fulltext_train)

vecs_nmf = np.asarray(vecs_nmf, dtype=np.object)
vecs_nmf = np.insert(vecs_nmf, 0, dois, axis=1)

emb2 = pd.DataFrame(vecs_nmf)
emb2.to_csv("full_abstract_2.csv", header=False)
<commit_msg>Preprocessing implemented and gensim lda work in prgress<commit_after>from importlib import reload

import pandas as pd
import numpy as np
import spacy
from textblob import TextBlob
from sklearn.decomposition import LatentDirichletAllocation, NMF
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from scipy.spatial.distance import pdist
from sklearn.preprocessing import MultiLabelBinarizer, LabelBinarizer
from nltk.corpus import reuters, brown
from sklearn.datasets import fetch_20newsgroups
from sklearn.model_selection import GridSearchCV
from sklearn.svm import LinearSVC, SVC
from sklearn import metrics
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.model_selection import cross_val_score
from sklearn.metrics.pairwise import cosine_similarity
import gensim.corpora as corpora
from nltk.corpus import stopwords

# Add general functions to the project
from os import path
import sys
sys.path.append(path.abspath('../methods'))

nlp = spacy.load('en_core_web_md', disable=['ner'])

def print_top_words(model, feature_names, n_top_words):
    for topic_idx, topic in enumerate(model.components_):
        message = "Topic #%d: " % topic_idx
        message += " ".join([feature_names[i]
                             for i in topic.argsort()[:-n_top_words - 1:-1]])
        print(message)
    print()

def get_top_words(model, feature_names, n_top_words):
    out = []
    for topic_idx, topic in enumerate(model.components_):
        topics = [feature_names[i] for i in topic.argsort()[:-n_top_words - 1:-1]]
        out.extend(topics)

    return set(out)

def preprocessing(text, stopwords, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
    """https://spacy.io/api/annotation"""
    texts_out = []
    for sent in text.sents:
        texts_out.extend([token.lemma_ for token in sent if
                          token.pos_ in allowed_postags and
                          token.lemma_ not in stop_words and
                          not token.like_num and
                          not token.like_url and
                          not token.like_email and
                          not token.lemma_ == "-PRON-" and
                          token.is_alpha and
                          len(token.lemma_) > 1])
    return texts_out

# DATA Loading
raw = np.load("../datasets/full.pkl")

# Remove Unclear from groups/clusters
# raw["Clusters"] = data.apply(
#     lambda row: row["Clusters"]
#     .replace(";Unclear", "")
#     .replace(";;", ";"), axis=1
#     )

# train/test split
# test = raw[raw["DOI"].str.contains("2013|2012")]  # len = 197
# train = raw.drop(test.index)  # len = 1280
label_column = ["Clusters"]

# data
# y
y = np.array(raw["Title"])

# preprocessing
stop_words = stopwords.words('english')

# test = raw.iloc[:100]

docs = [nlp(text) for text in raw["Fulltext"].tolist()]
clean = [preprocessing(doc, stop_words) for doc in docs]

# x
abstract_tfidf = TfidfVectorizer(stop_words=stop_words).fit(raw["Abstract"])
abstract_train = abstract_tfidf.transform(raw["Abstract"])

full_text_tfidf = TfidfVectorizer(stop_words=stop_words).fit(raw["Fulltext"])
fulltext_train = full_text_tfidf.transform(raw["Fulltext"])

id2word = corpora.Dictionary(clean)
corpus = [id2word.doc2bow(text) for text in clean]

# params
n_dim = [4,6,8,10,12,15,20,25,30]
algorithms = ["lda", "nmf"]
datasets = ["abstract", "fulltext"]

n_runs_per_setting = 10
topic_consistency_thresholds = [5, 10, 20]

# Create all parameter permutations
protocol = pd.DataFrame(columns=["Algorithm", "Dataset", "Dimensions", "NumberRuns", "Doc", "JaccTop5Median", "JaccTop10Median", "JaccTop20Median"])
runs = np.stack(np.meshgrid(algorithms, datasets, n_dim), -1).reshape(-1, 3)

# Compute
for run in runs:
    ldas = []
    vecs = []
    similarities = []
    row = []

    # Set variables
    if run[1] == "abstract":
        used = abstract_train
        tfidf = abstract_tfidf
    if run[1] == "fulltext":
        used = fulltext_train
        tfidf = full_text_tfidf

    # Fill protocol
    row.extend(run)
    row.append(n_runs_per_setting)
    # Compute models
    for iteration in range(0, n_runs_per_setting):
        if run[0] == "lda":
            lda = LatentDirichletAllocation(int(run[2]), learning_method="batch")
            vec = lda.fit_transform(used)
        if run[0] == "nmf":
            lda = NMF(int(run[2]))
            vec = lda.fit_transform(used)

        similarity = cosine_similarity(vec)

        ldas.append(lda)
        vecs.append(vec)
        similarities.append(similarity)

    # Document neighborhood consistency
    doc_sim = []
    for i in range(0, len(similarities)):
        for j in range(i+1, len(similarities)):
            doc_sim.append(abs(similarities[i] - similarities[j]).sum())

    row.append(np.median(doc_sim))
    # print(y[similarity1[156].argsort()[-10:][::-1]])
    # print(y[similarity2[156].argsort()[-10:][::-1]])

    # Topic consistency
    for thres in topic_consistency_thresholds:
        top_sim = []

        for i in range(0, len(similarities)):
            topics1 = get_top_words(ldas[i], tfidf.get_feature_names(), thres)
            for j in range(i+1, len(similarities)):
                topics2 = get_top_words(ldas[j], tfidf.get_feature_names(), thres)

                top_sim.append(len(topics1.intersection(topics2))/len(topics1.union(topics2)))

        row.append(np.median(top_sim))
        # print_top_words(lda1, tfidf.get_feature_names(), 10)
        # print_top_words(lda2, tfidf.get_feature_names(), 10)

    protocol = protocol.append(pd.DataFrame([row], columns=protocol.columns))

# LDA Fulltext dim 6 is best (drop on 8)
# LDA Abstract dim 4 is best (far worse then the NMF 4 or 6)
# NMF Fulltext dim 10 is best (drop on 12)
# NMF Abstract dim 6 is best (drop on 8)
protocol.to_csv("runs.csv")

# Filter dimensions
# np.median(vecs1, axis=0)
# np.std(vecs1, axis=0)

print_top_words(ldas[1], tfidf.get_feature_names(), 10)
print_top_words(ldas[0], tfidf.get_feature_names(), 10)

# Create embeddings
dois = raw["DOI"]

# LDA_NMF data
# lda = LatentDirichletAllocation(6, learning_method="batch")
# vecs_lda = lda.fit_transform(fulltext_train)
#
# vecs_lda = np.asarray(vecs_lda, dtype=np.object)
# vecs_lda = np.insert(vecs_lda, 0, dois, axis=1)
#
# emb1 = pd.DataFrame(vecs_lda)
# emb1.to_csv("lda_nmf_1.csv", header=False, index=False)
#
# nmf = NMF(10)
# vecs_nmf = nmf.fit_transform(fulltext_train)
#
# vecs_nmf = np.asarray(vecs_nmf, dtype=np.object)
# vecs_nmf = np.insert(vecs_nmf, 0, dois, axis=1)
#
# emb2 = pd.DataFrame(vecs_nmf)
# emb2.to_csv("lda_nmf_2.csv", header=False, index=False)

# abstract_fulltext data
lda = NMF(6)
vecs_lda = lda.fit_transform(abstract_train)

vecs_lda = np.asarray(vecs_lda, dtype=np.object)
vecs_lda = np.insert(vecs_lda, 0, dois, axis=1)

emb1 = pd.DataFrame(vecs_lda)
emb1.to_csv("full_abstract_1.csv", header=False, index=False)

nmf = NMF(10)
vecs_nmf = nmf.fit_transform(fulltext_train)

vecs_nmf = np.asarray(vecs_nmf, dtype=np.object)
vecs_nmf = np.insert(vecs_nmf, 0, dois, axis=1)

emb2 = pd.DataFrame(vecs_nmf)
emb2.to_csv("full_abstract_2.csv", header=False, index=False)
<|endoftext|>


<commit_before>import json
from copy import copy

from django.conf import settings
from kafka import KafkaConsumer
from kafka.common import ConsumerTimeout

from corehq.apps.change_feed.data_sources import get_document_store
from corehq.apps.change_feed.exceptions import UnknownDocumentStore
from corehq.apps.change_feed.topics import get_multi_topic_offset, validate_offsets
from dimagi.utils.logging import notify_error
from pillowtop.checkpoints.manager import PillowCheckpointEventHandler
from pillowtop.models import kafka_seq_to_str
from pillowtop.feed.interface import ChangeFeed, Change, ChangeMeta

MIN_TIMEOUT = 100

class KafkaChangeFeed(ChangeFeed):
    """
    Kafka-based implementation of a ChangeFeed
    """
    sequence_format = 'json'

    def __init__(self, topics, group_id, strict=False):
        """
        Create a change feed listener for a list of kafka topics, a group ID, and partition.

        See http://kafka.apache.org/documentation.html#introduction for a description of what these are.
        """
        self._topics = topics
        self._group_id = group_id
        self._processed_topic_offsets = {}
        self.strict = strict

    def __unicode__(self):
        return u'KafkaChangeFeed: topics: {}, group: {}'.format(self._topics, self._group_id)

    @property
    def topics(self):
        return self._topics

    def _get_single_topic_or_fail(self):
        if len(self._topics) != 1:
            raise ValueError("This function requires a single topic but found {}!".format(self._topics))
        return self._topics[0]

    def iter_changes(self, since, forever):
        """
        Since must be a dictionary of topic partition offsets.
        """
        # a special value of since=None will start from the end of the change stream
        if since is not None and (not isinstance(since, dict) or not since):
            raise ValueError("'since' must be None or a topic offset dictionary")

        # in milliseconds, -1 means wait forever for changes
        timeout = -1 if forever else MIN_TIMEOUT

        start_from_latest = since is None

        reset = 'largest' if start_from_latest else 'smallest'
        consumer = self._get_consumer(timeout, auto_offset_reset=reset)
        if not start_from_latest:
            if self.strict:
                validate_offsets(since)

            checkpoint_topics = {tp[0] for tp in since}
            extra_topics = checkpoint_topics - set(self._topics)
            if extra_topics:
                raise ValueError("'since' contains extra topics: {}".format(list(extra_topics)))

            self._processed_topic_offsets = copy(since)

            offsets = [
                copy(self._processed_topic_offsets)
            ]
            topics_missing = set(self._topics) - checkpoint_topics
            for topic in topics_missing:
                offsets.append(topic)  # consume all available partitions

            # this is how you tell the consumer to start from a certain point in the sequence
            consumer.set_topic_partitions(*offsets)

        try:
            for message in consumer:
                self._processed_topic_offsets[(message.topic, message.partition)] = message.offset
                yield change_from_kafka_message(message)
        except ConsumerTimeout:
            assert not forever, 'Kafka pillow should not timeout when waiting forever!'
            # no need to do anything since this is just telling us we've reached the end of the feed

    def get_current_checkpoint_offsets(self):
        # the way kafka works, the checkpoint should increment by 1 because
        # querying the feed is inclusive of the value passed in.
        return {
            topic_partition: sequence + 1 for topic_partition, sequence in self._processed_topic_offsets.items()
        }

    def get_processed_offsets(self):
        return copy(self._processed_topic_offsets)

    def get_latest_offsets(self):
        return get_multi_topic_offset(self.topics)

    def get_latest_offsets_json(self):
        return json.loads(kafka_seq_to_str(self.get_latest_offsets()))

    def get_latest_offsets_as_checkpoint_value(self):
        return self.get_latest_offsets()

    def _get_consumer(self, timeout, auto_offset_reset='smallest'):
        config = {
            'group_id': self._group_id,
            'bootstrap_servers': [settings.KAFKA_URL],
            'consumer_timeout_ms': timeout,
            'auto_offset_reset': auto_offset_reset,
        }
        return KafkaConsumer(
            *self._topics,
            **config
        )

class PartitionedKafkaChangeFeed(KafkaChangeFeed):
    def __init__(self, topics, group_id, strict=False, num_processes=1, process_num=0):
        self.num_processes = num_processes
        self.process_num = process_num
        super(PartitionedKafkaChangeFeed, self).__init__(topics, group_id, strict)

    def iter_changes(self, since, forever):
        """
        Since must be a dictionary of topic partition offsets.
        """
        since = self._get_partitioned_offsets(since)
        return super(PartitionedKafkaChangeFeed, self).iter_changes(since, forever)

    def get_current_checkpoint_offsets(self):
        offsets = super(PartitionedKafkaChangeFeed, self).get_current_checkpoint_offsets()
        return self._get_partitioned_offsets(offsets)

    def get_latest_offsets(self):
        offsets = super(PartitionedKafkaChangeFeed, self).get_latest_offsets()
        return self._get_partitioned_offsets(offsets)

    def _get_partitioned_offsets(self, offsets):
        topic_partitions = sorted(list(offsets))

        partitioned_topic_partitions = [
            topic_partitions[num::self.num_processes]
            for num in range(self.num_processes)
        ][self.process_num]

        return {
            tp: offset
            for tp, offset in offsets.items()
            if tp in partitioned_topic_partitions
        }

class KafkaCheckpointEventHandler(PillowCheckpointEventHandler):
    """
    Event handler that supports checkpoints when subscribing to multiple topics.
    """

    def __init__(self, checkpoint, checkpoint_frequency, change_feed):
        super(KafkaCheckpointEventHandler, self).__init__(checkpoint, checkpoint_frequency)
        assert isinstance(change_feed, KafkaChangeFeed)
        self.change_feed = change_feed

    def fire_change_processed(self, change, context):
        if self.should_update_checkpoint(context):
            updated_to = self.change_feed.get_current_checkpoint_offsets()
            self.update_checkpoint(updated_to)
            return True

        return False

def change_from_kafka_message(message):
    change_meta = change_meta_from_kafka_message(message.value)
    try:
        document_store = get_document_store(
            data_source_type=change_meta.data_source_type,
            data_source_name=change_meta.data_source_name,
            domain=change_meta.domain
        )
    except UnknownDocumentStore:
        document_store = None
        notify_error("Unknown document store: {}".format(change_meta.data_source_type))
    return Change(
        id=change_meta.document_id,
        sequence_id=message.offset,
        document=None,
        deleted=change_meta.is_deletion,
        metadata=change_meta,
        document_store=document_store,
        topic=message.topic,
    )

def change_meta_from_kafka_message(message):
    return ChangeMeta.wrap(json.loads(message))
<commit_msg>memoized filtering<commit_after>import json
from copy import copy

from django.conf import settings
from kafka import KafkaConsumer
from kafka.common import ConsumerTimeout

from corehq.apps.change_feed.data_sources import get_document_store
from corehq.apps.change_feed.exceptions import UnknownDocumentStore
from corehq.apps.change_feed.topics import get_multi_topic_offset, validate_offsets
from dimagi.utils.decorators.memoized import memoized
from dimagi.utils.logging import notify_error
from pillowtop.checkpoints.manager import PillowCheckpointEventHandler
from pillowtop.models import kafka_seq_to_str
from pillowtop.feed.interface import ChangeFeed, Change, ChangeMeta

MIN_TIMEOUT = 100

class KafkaChangeFeed(ChangeFeed):
    """
    Kafka-based implementation of a ChangeFeed
    """
    sequence_format = 'json'

    def __init__(self, topics, group_id, strict=False):
        """
        Create a change feed listener for a list of kafka topics, a group ID, and partition.

        See http://kafka.apache.org/documentation.html#introduction for a description of what these are.
        """
        self._topics = topics
        self._group_id = group_id
        self._processed_topic_offsets = {}
        self.strict = strict

    def __unicode__(self):
        return u'KafkaChangeFeed: topics: {}, group: {}'.format(self._topics, self._group_id)

    @property
    def topics(self):
        return self._topics

    @property
    @memoized
    def topic_and_partitions(self):
        return list(self._processed_topic_offsets)

    def _get_single_topic_or_fail(self):
        if len(self._topics) != 1:
            raise ValueError("This function requires a single topic but found {}!".format(self._topics))
        return self._topics[0]

    def iter_changes(self, since, forever):
        """
        Since must be a dictionary of topic partition offsets.
        """
        # a special value of since=None will start from the end of the change stream
        if since is not None and (not isinstance(since, dict) or not since):
            raise ValueError("'since' must be None or a topic offset dictionary")

        # in milliseconds, -1 means wait forever for changes
        timeout = -1 if forever else MIN_TIMEOUT

        start_from_latest = since is None

        reset = 'largest' if start_from_latest else 'smallest'
        consumer = self._get_consumer(timeout, auto_offset_reset=reset)
        if not start_from_latest:
            if self.strict:
                validate_offsets(since)

            checkpoint_topics = {tp[0] for tp in since}
            extra_topics = checkpoint_topics - set(self._topics)
            if extra_topics:
                raise ValueError("'since' contains extra topics: {}".format(list(extra_topics)))

            self._processed_topic_offsets = copy(since)

            offsets = [
                copy(self._processed_topic_offsets)
            ]
            topics_missing = set(self._topics) - checkpoint_topics
            for topic in topics_missing:
                offsets.append(topic)  # consume all available partitions

            # this is how you tell the consumer to start from a certain point in the sequence
            consumer.set_topic_partitions(*offsets)

        try:
            for message in consumer:
                self._processed_topic_offsets[(message.topic, message.partition)] = message.offset
                yield change_from_kafka_message(message)
        except ConsumerTimeout:
            assert not forever, 'Kafka pillow should not timeout when waiting forever!'
            # no need to do anything since this is just telling us we've reached the end of the feed

    def get_current_checkpoint_offsets(self):
        # the way kafka works, the checkpoint should increment by 1 because
        # querying the feed is inclusive of the value passed in.
        return {
            topic_partition: sequence + 1 for topic_partition, sequence in self._processed_topic_offsets.items()
        }

    def get_processed_offsets(self):
        return copy(self._processed_topic_offsets)

    def get_latest_offsets(self):
        return get_multi_topic_offset(self.topics)

    def get_latest_offsets_json(self):
        return json.loads(kafka_seq_to_str(self.get_latest_offsets()))

    def get_latest_offsets_as_checkpoint_value(self):
        return self.get_latest_offsets()

    def _get_consumer(self, timeout, auto_offset_reset='smallest'):
        config = {
            'group_id': self._group_id,
            'bootstrap_servers': [settings.KAFKA_URL],
            'consumer_timeout_ms': timeout,
            'auto_offset_reset': auto_offset_reset,
        }
        return KafkaConsumer(
            *self._topics,
            **config
        )

class PartitionedKafkaChangeFeed(KafkaChangeFeed):
    def __init__(self, topics, group_id, strict=False, num_processes=1, process_num=0):
        self.num_processes = num_processes
        self.process_num = process_num
        super(PartitionedKafkaChangeFeed, self).__init__(topics, group_id, strict)

    @property
    @memoized
    def topic_and_partitions(self):
        return list(self._get_partitioned_offsets(get_multi_topic_offset(self.topics)))

    def iter_changes(self, since, forever):
        """
        Since must be a dictionary of topic partition offsets.
        """
        since = self._filter_offsets(since)
        return super(PartitionedKafkaChangeFeed, self).iter_changes(since, forever)

    def get_current_checkpoint_offsets(self):
        offsets = super(PartitionedKafkaChangeFeed, self).get_current_checkpoint_offsets()
        return self._filter_offsets(offsets)

    def get_latest_offsets(self):
        offsets = super(PartitionedKafkaChangeFeed, self).get_latest_offsets()
        return self._filter_offsets(offsets)

    def _filter_offsets(self, offsets):
        return {
            tp: offsets[tp] for tp in self.topic_and_partitions
        }

    def _get_partitioned_offsets(self, offsets):
        topic_partitions = sorted(list(offsets))

        partitioned_topic_partitions = [
            topic_partitions[num::self.num_processes]
            for num in range(self.num_processes)
        ][self.process_num]

        return {
            tp: offset
            for tp, offset in offsets.items()
            if tp in partitioned_topic_partitions
        }

class KafkaCheckpointEventHandler(PillowCheckpointEventHandler):
    """
    Event handler that supports checkpoints when subscribing to multiple topics.
    """

    def __init__(self, checkpoint, checkpoint_frequency, change_feed):
        super(KafkaCheckpointEventHandler, self).__init__(checkpoint, checkpoint_frequency)
        assert isinstance(change_feed, KafkaChangeFeed)
        self.change_feed = change_feed

    def fire_change_processed(self, change, context):
        if self.should_update_checkpoint(context):
            updated_to = self.change_feed.get_current_checkpoint_offsets()
            self.update_checkpoint(updated_to)
            return True

        return False

def change_from_kafka_message(message):
    change_meta = change_meta_from_kafka_message(message.value)
    try:
        document_store = get_document_store(
            data_source_type=change_meta.data_source_type,
            data_source_name=change_meta.data_source_name,
            domain=change_meta.domain
        )
    except UnknownDocumentStore:
        document_store = None
        notify_error("Unknown document store: {}".format(change_meta.data_source_type))
    return Change(
        id=change_meta.document_id,
        sequence_id=message.offset,
        document=None,
        deleted=change_meta.is_deletion,
        metadata=change_meta,
        document_store=document_store,
        topic=message.topic,
    )

def change_meta_from_kafka_message(message):
    return ChangeMeta.wrap(json.loads(message))
<|endoftext|>


<commit_before># CloudShell L1 resource autoload XML helper
#
# It should not be necessary to edit this file.
#
# - Generates the autoload XML resource format to return to CloudShell
# - Subresources are also represented with nested instances of this class
# - See example usage in <project>_l1_handler.py

class L1DriverResourceInfo:
    def __init__(self, name, full_address, family, model, map_path=None, serial='-1'):
        """
        :param name: str
        :param full_address: str
        :param family: str
        :param model: str
        :param map_path: str
        :param serial: str
        """
        self.name = name
        self.address = full_address
        self.family = family
        self.model = model
        self.serial = serial
        self.map_path = map_path
        self.subresources = []
        self.attrname2typevaluetuple = {}
        if False:
            self.subresources.append(L1DriverResourceInfo(None, None, None, None))

    def add_subresource(self, subresource):
        """
        :param subresource: L1DriverResourceInfo
        :return: None
        """
        self.subresources.append(subresource)

    def set_attribute(self, name, value, typename='String'):
        """
        :param name: str
        :param value: str
        :param typename: str
        :return: None
        """
        self.attrname2typevaluetuple[name] = (typename, value)

    def get_attribute(self, name):
        """
        :param name: str
        :return: str
        """
        return self.attrname2typevaluetuple[name][1]

    def to_string(self, tabs=''):
        """
        :param tabs: str
        :return: str
        """
        def indent(t, s):
            return t + (('\n' + t).join(s.split('\n'))).strip()

        return indent(tabs,
                      '''<ResourceInfo
        Name="''' + self.name + '''"
        Address="''' + self.address + '''"
        ResourceFamilyName="''' + self.family + '''"
        ResourceModelName="''' + self.model + '''"
        SerialNumber="''' + self.serial + '''">
    <ChildResources>
''' + ('\n'.join(
                          [
                              x.to_string(tabs=tabs + '    ')
                              for x in self.subresources
                              ]
                      )) + '''    </ChildResources>
    <ResourceAttributes>
''' + (''.join(
                          [
'''        <Attribute
                Name="''' + attrname + '''"
                Type="''' + self.attrname2typevaluetuple[attrname][0] + '''"
                Value="''' + self.attrname2typevaluetuple[attrname][1] + '''" />\n'''
                              for attrname in self.attrname2typevaluetuple.keys()
                              ]
                      )) + '''    </ResourceAttributes>
''' +
                      ('''    <ResourceMapping><IncomingMapping>''' + self.map_path + '''</IncomingMapping></ResourceMapping>''' if self.map_path else '') +
                      '''
</ResourceInfo>
                      ''')
<commit_msg>Update l1_driver_resource_info.py<commit_after># CloudShell L1 resource autoload XML helper
#
# It should not be necessary to edit this file.
#
# - Generates the autoload XML resource format to return to CloudShell
# - Subresources are also represented with nested instances of this class
# - See example usage in <project>_l1_handler.py

class L1DriverResourceInfo:
    def __init__(self, name, full_address, family, model, map_path=None, serial='-1'):
        """
        :param name: str
        :param full_address: str
        :param family: str
        :param model: str
        :param map_path: str
        :param serial: str
        """
        self.name = name
        self.address = full_address
        self.family = family
        self.model = model
        self.serial = serial
        self.map_path = map_path
        self.subresources = []
        self.attrname2typevaluetuple = {}
        if False:
            self.subresources.append(L1DriverResourceInfo(None, None, None, None))

    def add_subresource(self, subresource):
        """
        :param subresource: L1DriverResourceInfo
        :return: None
        """
        self.subresources.append(subresource)

    def set_attribute(self, name, value, typename='String'):
        """
        :param name: str
        :param value: str
        :param typename: str
        :return: None
        """
        self.attrname2typevaluetuple[name] = (typename, value)

    def get_attribute(self, name):
        """
        :param name: str
        :return: str
        """
        return self.attrname2typevaluetuple[name][1]

    def to_string(self, tabs=''):
        """
        :param tabs: str
        :return: str
        """
        def indent(t, s):
            return t + (('\n' + t).join(s.split('\n'))).strip()

        return indent(tabs,
'''<ResourceInfo
        Name="''' + self.name + '''"
        Address="''' + self.address + '''"
        ResourceFamilyName="''' + self.family + '''"
        ResourceModelName="''' + self.model + '''"
        SerialNumber="''' + self.serial + '''">
    <ChildResources>
''' + ('\n'.join(
                          [
                              x.to_string(tabs=tabs + '    ')
                              for x in self.subresources
                              ]
                      )) + '''    </ChildResources>
    <ResourceAttributes>
''' + (''.join(
                          [
'''        <Attribute
                Name="''' + attrname + '''"
                Type="''' + self.attrname2typevaluetuple[attrname][0] + '''"
                Value="''' + self.attrname2typevaluetuple[attrname][1] + '''" />\n'''
                              for attrname in self.attrname2typevaluetuple.keys()
                              ]
                      )) + '''    </ResourceAttributes>
''' +
                      ('''    <ResourceMapping><IncomingMapping>''' + self.map_path + '''</IncomingMapping></ResourceMapping>\n''' if self.map_path else '\n') +
'''</ResourceInfo>
''')
<|endoftext|>


<commit_before>
from pykml import parser
from pykml.parser import Schema
from pykml.factory import KML_ElementMaker as KML
from os import path
import json
import geojson
from lxml import etree
import sys, os
from random import randrange

sampleTrip = '{"date": "20140413", "lastUpdate": "20140414T064442Z", "segments": [], "summary": null}'
sampleSegment = '{"place": {"type": "unknown", "id": 54095848, "location": {"lat": 37.3910149202, "lon": -122.0865010796}}, "endTime": "20140413T234434-0700", "type": "place", "startTime": "20140412T190446-0700", "lastUpdate": "20140414T064442Z"}'

def kml_to_geojson(infile_path, outfile_path, geojson_properties={}):
	"""
	Converts a KML file to geojson to store in an output file.
	"""
	coordinatePoints = geojson.MultiPoint(__get_coords(infile_path))
	dump = geojson.dumps(geojson.Feature(geometry=coordinatePoints, properties=geojson_properties)) # can add properties and id to feature (perhaps trip/section id?)
	outfile = open(outfile_path, 'w')
	outfile.write(dump)
	outfile.close()

def kml_to_json(infile_path, outfile_path):
	"""
	Converts a KML file to geojson to store in an output file.
	"""
	coordinatePoints = __get_coords(infile_path)
	segmentsList = []
	for coord in coordinatePoints:
		j = json.loads(sampleSegment)
		j["place"]["location"]["lat"] = coord[0] # 0th index might be longitude
		j["place"]["location"]["lon"] = coord[1]
		segmentsList.append(j)
	j = json.loads(sampleTrip)
	j["segments"] = segmentsList
	with open(outfile_path, 'w') as outfile:
		json.dump(j, outfile)

def __get_coords(infile_path):
	data = ""
	with open(infile_path, "r") as temp:
		data = temp.read()
	assert(data != "")
	doc = parser.fromstring(data)
	dataPoints = doc.Document.Placemark.LineString.coordinates.text.split(" ")
	return [[float(i) for i in x.split(",")[:2]] for x in dataPoints]

def validate_kml(filepath, schema = "https://developers.google.com/kml/schema/kml21.xsd"):
        """
        Validates kml located at filepath.
        TODO: Even imported kml files do not validate against default xml
        """
        schema_gomaps = Schema(schema)
	data = ""
	with open(filepath, "r") as temp:
		data = temp.read()
	assert(data != "")
	doc = parser.fromstring(data)
        return schema_ogc.assertValid(doc)

def section_to_kml(section, outfile_path="", write=True):
        """
        Converts a section(s) into a kml file
        """
        color = "".join([hex(randrange(255))[-2:] for i in range(3)])
        make_coord = lambda p: (",".join(map(lambda x: str(x),
                         p["track_location"]["coordinates"]) + ["0.0"]))
        make_coord_point = lambda p: (",".join(map(lambda x: str(x),
                         p["coordinates"]) + ["0.0"]))
        pm = KML.Placemark(
                KML.styleUrl("#line-%s-5" % section["section_start_time"]),
                KML.name(section['_id']),
                KML.LineString(
                        KML.tessellate(1),
                        KML.coordinates(" ".join(
                                map(lambda track_point: make_coord(track_point)
                                    ,section['track_points'][1:-1])))
                )
        )
        start_point = KML.Placemark(
		KML.styleUrl("#icon-%s" % section["section_start_time"]),
                KML.name("Start"),
                KML.description(section['section_start_time']),
                KML.Point(KML.coordinates(make_coord_point(section['section_start_point'])))
        )
        end_point = KML.Placemark(
		KML.styleUrl("#icon-%s" % section["section_start_time"]),
                KML.name("End"),
                KML.description(section['section_end_time']),
                KML.Point(KML.coordinates(make_coord_point(section['section_end_point'])))
        )
        line_style = KML.Style(
                KML.LineStyle(
                        KML.color(color),
                        KML.width("5")
                )
        )
        line_style.set("id","line-%s-5" % section["section_start_time"])
        icon_style = KML.Style(
                KML.LineStyle(
                        KML.color(color)
                )
        )
        icon_style.set("id","icon-%s" % section["section_start_time"])
        fld = KML.Folder(
                KML.name(section['_id']),
                pm,
                start_point,
                end_point
        )

        if write:
                kml = KML.kml(KML.Document(fld))
                path = os.path.join(outfile_path, str(section['user_id']) +'.kml')
                outfile = file(path,'w')
                outfile.write(etree.tostring(kml, pretty_print=True))
        else:
                return (fld, icon_style, line_style)

def sections_to_kml(filename, sections, outfile_path=""):
        kml = KML.kml(KML.Document(*map(lambda section: section_to_kml(section, write=False) if len(section['track_points']) > 1 else None ,sections)))
        path = os.path.join(outfile_path, filename +'.kml')
        outfile = file(path,'w')
        outfile.write(etree.tostring(kml, pretty_print=True))

def chunks(l,n):
        """
        Generates evenly sized chunks of a list
        """
        for i in xrange(0, len(l), n):
                yield l[i:i+n]

if __name__ == "__main__":
        sys.path.append("%s/../" % os.getcwd())
        from get_database import get_section_db
        from uuid import UUID
        Sections = get_section_db()
        user_id = "__ID_GOES_HERE__"
        sections = Sections.find({'$and': [{'user_id':user_id}]})
        sections_to_kml(user_id,sections)
<commit_msg>Fix issues with generating trips with different colors<commit_after>
from pykml import parser
from pykml.parser import Schema
from pykml.factory import KML_ElementMaker as KML
from os import path
import json
import geojson
from lxml import etree
import sys, os
from random import randrange

sampleTrip = '{"date": "20140413", "lastUpdate": "20140414T064442Z", "segments": [], "summary": null}'
sampleSegment = '{"place": {"type": "unknown", "id": 54095848, "location": {"lat": 37.3910149202, "lon": -122.0865010796}}, "endTime": "20140413T234434-0700", "type": "place", "startTime": "20140412T190446-0700", "lastUpdate": "20140414T064442Z"}'

def kml_to_geojson(infile_path, outfile_path, geojson_properties={}):
	"""
	Converts a KML file to geojson to store in an output file.
	"""
	coordinatePoints = geojson.MultiPoint(__get_coords(infile_path))
	dump = geojson.dumps(geojson.Feature(geometry=coordinatePoints, properties=geojson_properties)) # can add properties and id to feature (perhaps trip/section id?)
	outfile = open(outfile_path, 'w')
	outfile.write(dump)
	outfile.close()

def kml_to_json(infile_path, outfile_path):
	"""
	Converts a KML file to geojson to store in an output file.
	"""
	coordinatePoints = __get_coords(infile_path)
	segmentsList = []
	for coord in coordinatePoints:
		j = json.loads(sampleSegment)
		j["place"]["location"]["lat"] = coord[0] # 0th index might be longitude
		j["place"]["location"]["lon"] = coord[1]
		segmentsList.append(j)
	j = json.loads(sampleTrip)
	j["segments"] = segmentsList
	with open(outfile_path, 'w') as outfile:
		json.dump(j, outfile)

def __get_coords(infile_path):
	data = ""
	with open(infile_path, "r") as temp:
		data = temp.read()
	assert(data != "")
	doc = parser.fromstring(data)
	dataPoints = doc.Document.Placemark.LineString.coordinates.text.split(" ")
	return [[float(i) for i in x.split(",")[:2]] for x in dataPoints]

def validate_kml(filepath, schema = "https://developers.google.com/kml/schema/kml21.xsd"):
        """
        Validates kml located at filepath.
        TODO: Even imported kml files do not validate against default xml
        """
        schema_gomaps = Schema(schema)
	data = ""
	with open(filepath, "r") as temp:
		data = temp.read()
	assert(data != "")
	doc = parser.fromstring(data)
        return schema_ogc.assertValid(doc)

def section_to_kml(section, outfile_path="", write=True):
        """
        Converts a section(s) into a kml file
        """
        color = "".join(['ff'] + [hex(randrange(255))[-2:] for i in range(3)])
        make_coord = lambda p: (",".join(map(lambda x: str(x),
                         p["track_location"]["coordinates"]) + ["0.0"]))
        make_coord_point = lambda p: (",".join(map(lambda x: str(x),
                         p["coordinates"]) + ["0.0"]))
	style_id = "style-%s" % section['section_start_time']
        pm = KML.Placemark(
                KML.styleUrl("#%s" % style_id),
                KML.name(section['_id']),
                KML.LineString(
                        KML.tessellate(1),
                        KML.coordinates(" ".join(
                                map(lambda track_point: make_coord(track_point)
                                    ,section['track_points'][1:-1])))
                )
        )
        start_point = KML.Placemark(
		KML.styleUrl("#%s" % style_id),
                KML.name("Start"),
                KML.description(section['section_start_time']),
                KML.Point(KML.coordinates(make_coord_point(section['section_start_point'])))
        )
        end_point = KML.Placemark(
		KML.styleUrl("#%s" % style_id),
                KML.name("End"),
                KML.description(section['section_end_time']),
                KML.Point(KML.coordinates(make_coord_point(section['section_end_point'])))
        )
        style = KML.Style(
                KML.LineStyle(
                        KML.color(color),
                        KML.width(3)
                ),
                KML.IconStyle(
                        KML.color(color),
			KML.scale(1.1),
			KML.Icon(
			   KML.href("http://www.gstatic.com/mapspro/images/stock/503-wht-blank_maps.png")
			)
                )
        )
        style.set("id",style_id)
        fld = KML.Folder(
                KML.name(section['_id']),
                pm,
                start_point,
                end_point
        )

        if write:
                kml = KML.kml(KML.Document(fld))
                path = os.path.join(outfile_path, str(section['user_id']) +'.kml')
                outfile = file(path,'w')
                outfile.write(etree.tostring(kml, pretty_print=True))
        else:
                return (fld, style)

def sections_to_kml(filename, sections, outfile_path=""):
        # kml = KML.kml(KML.Document(*map(lambda section: section_to_kml(section, write=False) if len(section['track_points']) > 1 else None ,sections)))
	foldersAndStyles = map(lambda section: section_to_kml(section, write=False) if len(section['track_points']) > 1 else None ,sections)
	folders = [fs[0] for fs in foldersAndStyles]
	styles = [fs[1] for fs in foldersAndStyles]

        kml = KML.kml(KML.Document(*map(lambda fs: fs, folders + styles)))
        path = os.path.join(outfile_path, filename +'.kml')
        outfile = file(path,'w')
        outfile.write(etree.tostring(kml, pretty_print=True))

def chunks(l,n):
        """
        Generates evenly sized chunks of a list
        """
        for i in xrange(0, len(l), n):
                yield l[i:i+n]

if __name__ == "__main__":
        sys.path.append("%s/../" % os.getcwd())
        from get_database import get_section_db
        from uuid import UUID
        Sections = get_section_db()
        user_id = "__ID_GOES_HERE__"
        sections = Sections.find({'$and': [{'user_id':user_id}]})
        sections_to_kml(user_id,sections)
<|endoftext|>


<commit_before>import pkg_resources

loaded = set()

def load_entry_points(name=None):
    """Load extension packages into the segpy.ext namespace.

    Any packages registered against the 'segpy.ext' entry-point group will be
    installed dynamically into the segpy.ext namespace.
    """
    for entry_point in pkg_resources.iter_entry_points(group='segpy.ext', name=name):
        package = entry_point.load()
        if package not in loaded:
            loaded.add(package)
            __path__.extend(package.__path__)
            package.load()

load_entry_points()
<commit_msg>Remove the ext subpackage.<commit_after><|endoftext|>


<commit_before>from django.utils.translation import ugettext_lazy as _
from django.contrib import admin
from mezzanine.core.admin import TabularDynamicInlineAdmin, DisplayableAdmin

from fullcalendar.models import *

class EventCategoryAdmin(admin.ModelAdmin):
    list_display = ('name',)

class OccurrenceInline(TabularDynamicInlineAdmin):
    model = Occurrence
    extra = 1

class EventAdmin(DisplayableAdmin):
    list_display = ('title', 'event_category')
    list_filter = ('event_category',)
    search_fields = ('title', 'description', 'content', 'keywords')

    fieldsets = (
        (None, {
            "fields": [
                "title", "status", ("publish_date", "expiry_date"),
                "event_category", "content"
            ]
        }),
        (_("Meta data"), {
            "fields": [
                "_meta_title", "slug",
                ("description", "gen_description"),
                "keywords", "in_sitemap"
            ],
            "classes": ("collapse-closed",)
        }),
    )

    inlines = [OccurrenceInline]

admin.site.register(Event, EventAdmin)
admin.site.register(EventCategory, EventCategoryAdmin)

<commit_msg>Reorder fields for occurence inline<commit_after>from django.utils.translation import ugettext_lazy as _
from django.contrib import admin
from mezzanine.core.admin import TabularDynamicInlineAdmin, DisplayableAdmin

from fullcalendar.models import *

class EventCategoryAdmin(admin.ModelAdmin):
    list_display = ('name',)

class OccurrenceInline(TabularDynamicInlineAdmin):
    model = Occurrence
    extra = 1

    fields = ('start_time', 'end_time', 'description')

class EventAdmin(DisplayableAdmin):
    list_display = ('title', 'event_category')
    list_filter = ('event_category',)
    search_fields = ('title', 'description', 'content', 'keywords')

    fieldsets = (
        (None, {
            "fields": [
                "title", "status", ("publish_date", "expiry_date"),
                "event_category", "content"
            ]
        }),
        (_("Meta data"), {
            "fields": [
                "_meta_title", "slug",
                ("description", "gen_description"),
                "keywords", "in_sitemap"
            ],
            "classes": ("collapse-closed",)
        }),
    )

    inlines = [OccurrenceInline]

admin.site.register(Event, EventAdmin)
admin.site.register(EventCategory, EventCategoryAdmin)

<|endoftext|>