<commit_before>import abc
import inspect
from pathlib import Path
from typing import Any, Callable, Mapping, Optional

import numpy as np
import pandas as pd
import pandas.testing as tm
import pytest

import ibis
import ibis.backends.base_sqlalchemy.compiler as comp
import ibis.expr.types as ir

# TODO: Merge into BackendTest, #2564
class RoundingConvention:
    @staticmethod
    @abc.abstractmethod
    def round(series: pd.Series, decimals: int = 0) -> pd.Series:
        """Round a series to `decimals` number of decimal values."""

# TODO: Merge into BackendTest, #2564
class RoundAwayFromZero(RoundingConvention):
    @staticmethod
    def round(series: pd.Series, decimals: int = 0) -> pd.Series:
        if not decimals:
            return (
                -(np.sign(series)) * np.ceil(-(series.abs()) - 0.5)
            ).astype(np.int64)
        return series.round(decimals=decimals)

# TODO: Merge into BackendTest, #2564
class RoundHalfToEven(RoundingConvention):
    @staticmethod
    def round(series: pd.Series, decimals: int = 0) -> pd.Series:
        result = series.round(decimals=decimals)
        return result if decimals else result.astype(np.int64)

# TODO: Merge into BackendTest, #2564
class UnorderedComparator:
    @classmethod
    def assert_series_equal(
        cls, left: pd.Series, right: pd.Series, *args: Any, **kwargs: Any
    ) -> None:
        left = left.sort_values().reset_index(drop=True)
        right = right.sort_values().reset_index(drop=True)
        return super().assert_series_equal(left, right, *args, **kwargs)

    @classmethod
    def assert_frame_equal(
        cls, left: pd.DataFrame, right: pd.DataFrame, *args: Any, **kwargs: Any
    ) -> None:
        columns = list(set(left.columns) & set(right.columns))
        left = left.sort_values(by=columns)
        right = right.sort_values(by=columns)
        return super().assert_frame_equal(left, right, *args, **kwargs)

class BackendTest(abc.ABC):
    check_dtype = True
    check_names = True
    supports_arrays = True
    supports_arrays_outside_of_select = supports_arrays
    supports_window_operations = True
    additional_skipped_operations = frozenset()
    supports_divide_by_zero = False
    returned_timestamp_unit = 'us'
    supported_to_timestamp_units = {'s', 'ms', 'us'}
    supports_floating_modulus = True

    def __init__(self, data_directory: Path) -> None:
        self.api  # skips if we can't access the backend
        self.connection = self.connect(data_directory)

    def __str__(self):
        return f'<BackendTest {self.name()}>'

    @classmethod
    def name(cls) -> str:
        backend_tests_path = inspect.getmodule(cls).__file__
        return Path(backend_tests_path).resolve().parent.parent.name

    @staticmethod
    @abc.abstractmethod
    def connect(data_directory: Path) -> ibis.client.Client:
        """Return a connection with data loaded from `data_directory`."""

    @classmethod
    def assert_series_equal(
        cls, left: pd.Series, right: pd.Series, *args: Any, **kwargs: Any
    ) -> None:
        kwargs.setdefault('check_dtype', cls.check_dtype)
        kwargs.setdefault('check_names', cls.check_names)
        tm.assert_series_equal(left, right, *args, **kwargs)

    @classmethod
    def assert_frame_equal(
        cls, left: pd.DataFrame, right: pd.DataFrame, *args: Any, **kwargs: Any
    ) -> None:
        left = left.reset_index(drop=True)
        right = right.reset_index(drop=True)
        tm.assert_frame_equal(left, right, *args, **kwargs)

    @staticmethod
    def default_series_rename(
        series: pd.Series, name: str = 'tmp'
    ) -> pd.Series:
        return series.rename(name)

    @staticmethod
    def greatest(
        f: Callable[..., ir.ValueExpr], *args: ir.ValueExpr
    ) -> ir.ValueExpr:
        return f(*args)

    @staticmethod
    def least(
        f: Callable[..., ir.ValueExpr], *args: ir.ValueExpr
    ) -> ir.ValueExpr:
        return f(*args)

    @property
    def db(self) -> ibis.client.Database:
        return self.connection.database()

    @property
    def functional_alltypes(self) -> ir.TableExpr:
        return self.db.functional_alltypes

    @property
    def batting(self) -> ir.TableExpr:
        return self.db.batting

    @property
    def awards_players(self) -> ir.TableExpr:
        return self.db.awards_players

    @property
    def geo(self) -> Optional[ir.TableExpr]:
        return None

    @property
    def api(self):
        return getattr(ibis.backends, self.name()).Backend()

    def make_context(
        self, params: Optional[Mapping[ir.ValueExpr, Any]] = None
    ) -> comp.QueryContext:
        return self.api.dialect.make_context(params=params)

# TODO move to the spark/pyspark backends, #2565
_spark_testing_client = None
_pyspark_testing_client = None

# TODO move to the sparn/pyspark backends, #2565
def get_spark_testing_client(data_directory):
    global _spark_testing_client
    if _spark_testing_client is None:
        _spark_testing_client = get_common_spark_testing_client(
            data_directory,
            lambda session: ibis.backends.spark.Backend().connect(session),
        )
    return _spark_testing_client

# TODO move to the spark/pyspark backends, #2565
def get_pyspark_testing_client(data_directory):
    global _pyspark_testing_client
    if _pyspark_testing_client is None:
        _pyspark_testing_client = get_common_spark_testing_client(
            data_directory,
            lambda session: ibis.backends.pyspark.Backend().connect(session),
        )
    return _pyspark_testing_client

# TODO move to the spark/pyspark backends, #2565
def get_common_spark_testing_client(data_directory, connect):
    pytest.importorskip('pyspark')
    import pyspark.sql.types as pt
    from pyspark.sql import SparkSession

    spark = SparkSession.builder.config(
        'spark.default.parallelism', 4
    ).getOrCreate()
    _spark_testing_client = connect(spark)
    s = _spark_testing_client._session
    num_partitions = 4

    df_functional_alltypes = (
        s.read.csv(
            path=str(data_directory / 'functional_alltypes.csv'),
            schema=pt.StructType(
                [
                    pt.StructField('index', pt.IntegerType(), True),
                    pt.StructField('Unnamed: 0', pt.IntegerType(), True),
                    pt.StructField('id', pt.IntegerType(), True),
                    # cast below, Spark can't read 0/1 as bool
                    pt.StructField('bool_col', pt.ByteType(), True),
                    pt.StructField('tinyint_col', pt.ByteType(), True),
                    pt.StructField('smallint_col', pt.ShortType(), True),
                    pt.StructField('int_col', pt.IntegerType(), True),
                    pt.StructField('bigint_col', pt.LongType(), True),
                    pt.StructField('float_col', pt.FloatType(), True),
                    pt.StructField('double_col', pt.DoubleType(), True),
                    pt.StructField('date_string_col', pt.StringType(), True),
                    pt.StructField('string_col', pt.StringType(), True),
                    pt.StructField('timestamp_col', pt.TimestampType(), True),
                    pt.StructField('year', pt.IntegerType(), True),
                    pt.StructField('month', pt.IntegerType(), True),
                ]
            ),
            mode='FAILFAST',
            header=True,
        )
        .repartition(num_partitions)
        .sort('index')
    )

    df_functional_alltypes = df_functional_alltypes.withColumn(
        "bool_col", df_functional_alltypes["bool_col"].cast("boolean")
    )
    df_functional_alltypes.createOrReplaceTempView('functional_alltypes')

    df_batting = (
        s.read.csv(
            path=str(data_directory / 'batting.csv'),
            schema=pt.StructType(
                [
                    pt.StructField('playerID', pt.StringType(), True),
                    pt.StructField('yearID', pt.IntegerType(), True),
                    pt.StructField('stint', pt.IntegerType(), True),
                    pt.StructField('teamID', pt.StringType(), True),
                    pt.StructField('lgID', pt.StringType(), True),
                    pt.StructField('G', pt.IntegerType(), True),
                    pt.StructField('AB', pt.DoubleType(), True),
                    pt.StructField('R', pt.DoubleType(), True),
                    pt.StructField('H', pt.DoubleType(), True),
                    pt.StructField('X2B', pt.DoubleType(), True),
                    pt.StructField('X3B', pt.DoubleType(), True),
                    pt.StructField('HR', pt.DoubleType(), True),
                    pt.StructField('RBI', pt.DoubleType(), True),
                    pt.StructField('SB', pt.DoubleType(), True),
                    pt.StructField('CS', pt.DoubleType(), True),
                    pt.StructField('BB', pt.DoubleType(), True),
                    pt.StructField('SO', pt.DoubleType(), True),
                    pt.StructField('IBB', pt.DoubleType(), True),
                    pt.StructField('HBP', pt.DoubleType(), True),
                    pt.StructField('SH', pt.DoubleType(), True),
                    pt.StructField('SF', pt.DoubleType(), True),
                    pt.StructField('GIDP', pt.DoubleType(), True),
                ]
            ),
            header=True,
        )
        .repartition(num_partitions)
        .sort('playerID')
    )
    df_batting.createOrReplaceTempView('batting')

    df_awards_players = (
        s.read.csv(
            path=str(data_directory / 'awards_players.csv'),
            schema=pt.StructType(
                [
                    pt.StructField('playerID', pt.StringType(), True),
                    pt.StructField('awardID', pt.StringType(), True),
                    pt.StructField('yearID', pt.IntegerType(), True),
                    pt.StructField('lgID', pt.StringType(), True),
                    pt.StructField('tie', pt.StringType(), True),
                    pt.StructField('notes', pt.StringType(), True),
                ]
            ),
            header=True,
        )
        .repartition(num_partitions)
        .sort('playerID')
    )
    df_awards_players.createOrReplaceTempView('awards_players')

    df_simple = s.createDataFrame([(1, 'a')], ['foo', 'bar'])
    df_simple.createOrReplaceTempView('simple')

    df_struct = s.createDataFrame([((1, 2, 'a'),)], ['struct_col'])
    df_struct.createOrReplaceTempView('struct')

    df_nested_types = s.createDataFrame(
        [([1, 2], [[3, 4], [5, 6]], {'a': [[2, 4], [3, 5]]})],
        [
            'list_of_ints',
            'list_of_list_of_ints',
            'map_string_list_of_list_of_ints',
        ],
    )
    df_nested_types.createOrReplaceTempView('nested_types')

    df_complicated = s.createDataFrame(
        [({(1, 3): [[2, 4], [3, 5]]},)], ['map_tuple_list_of_list_of_ints']
    )
    df_complicated.createOrReplaceTempView('complicated')

    df_udf = s.createDataFrame(
        [('a', 1, 4.0, 'a'), ('b', 2, 5.0, 'a'), ('c', 3, 6.0, 'b')],
        ['a', 'b', 'c', 'key'],
    )
    df_udf.createOrReplaceTempView('udf')

    df_udf_nan = s.createDataFrame(
        pd.DataFrame(
            {
                'a': np.arange(10, dtype=float),
                'b': [3.0, np.NaN] * 5,
                'key': list('ddeefffggh'),
            }
        )
    )
    df_udf_nan.createOrReplaceTempView('udf_nan')

    df_udf_null = s.createDataFrame(
        [
            (float(i), None if i % 2 else 3.0, 'ddeefffggh'[i])
            for i in range(10)
        ],
        ['a', 'b', 'key'],
    )
    df_udf_null.createOrReplaceTempView('udf_null')

    df_udf_random = s.createDataFrame(
        pd.DataFrame(
            {
                'a': np.arange(4, dtype=float).tolist()
                + np.random.rand(3).tolist(),
                'b': np.arange(4, dtype=float).tolist()
                + np.random.rand(3).tolist(),
                'key': list('ddeefff'),
            }
        )
    )
    df_udf_random.createOrReplaceTempView('udf_random')

    return _spark_testing_client
<commit_msg>CI: Fixing Spark tests by settings the ip address to bind (#2732)<commit_after>import abc
import inspect
from pathlib import Path
from typing import Any, Callable, Mapping, Optional

import numpy as np
import pandas as pd
import pandas.testing as tm
import pytest

import ibis
import ibis.backends.base_sqlalchemy.compiler as comp
import ibis.expr.types as ir

# TODO: Merge into BackendTest, #2564
class RoundingConvention:
    @staticmethod
    @abc.abstractmethod
    def round(series: pd.Series, decimals: int = 0) -> pd.Series:
        """Round a series to `decimals` number of decimal values."""

# TODO: Merge into BackendTest, #2564
class RoundAwayFromZero(RoundingConvention):
    @staticmethod
    def round(series: pd.Series, decimals: int = 0) -> pd.Series:
        if not decimals:
            return (
                -(np.sign(series)) * np.ceil(-(series.abs()) - 0.5)
            ).astype(np.int64)
        return series.round(decimals=decimals)

# TODO: Merge into BackendTest, #2564
class RoundHalfToEven(RoundingConvention):
    @staticmethod
    def round(series: pd.Series, decimals: int = 0) -> pd.Series:
        result = series.round(decimals=decimals)
        return result if decimals else result.astype(np.int64)

# TODO: Merge into BackendTest, #2564
class UnorderedComparator:
    @classmethod
    def assert_series_equal(
        cls, left: pd.Series, right: pd.Series, *args: Any, **kwargs: Any
    ) -> None:
        left = left.sort_values().reset_index(drop=True)
        right = right.sort_values().reset_index(drop=True)
        return super().assert_series_equal(left, right, *args, **kwargs)

    @classmethod
    def assert_frame_equal(
        cls, left: pd.DataFrame, right: pd.DataFrame, *args: Any, **kwargs: Any
    ) -> None:
        columns = list(set(left.columns) & set(right.columns))
        left = left.sort_values(by=columns)
        right = right.sort_values(by=columns)
        return super().assert_frame_equal(left, right, *args, **kwargs)

class BackendTest(abc.ABC):
    check_dtype = True
    check_names = True
    supports_arrays = True
    supports_arrays_outside_of_select = supports_arrays
    supports_window_operations = True
    additional_skipped_operations = frozenset()
    supports_divide_by_zero = False
    returned_timestamp_unit = 'us'
    supported_to_timestamp_units = {'s', 'ms', 'us'}
    supports_floating_modulus = True

    def __init__(self, data_directory: Path) -> None:
        self.api  # skips if we can't access the backend
        self.connection = self.connect(data_directory)

    def __str__(self):
        return f'<BackendTest {self.name()}>'

    @classmethod
    def name(cls) -> str:
        backend_tests_path = inspect.getmodule(cls).__file__
        return Path(backend_tests_path).resolve().parent.parent.name

    @staticmethod
    @abc.abstractmethod
    def connect(data_directory: Path) -> ibis.client.Client:
        """Return a connection with data loaded from `data_directory`."""

    @classmethod
    def assert_series_equal(
        cls, left: pd.Series, right: pd.Series, *args: Any, **kwargs: Any
    ) -> None:
        kwargs.setdefault('check_dtype', cls.check_dtype)
        kwargs.setdefault('check_names', cls.check_names)
        tm.assert_series_equal(left, right, *args, **kwargs)

    @classmethod
    def assert_frame_equal(
        cls, left: pd.DataFrame, right: pd.DataFrame, *args: Any, **kwargs: Any
    ) -> None:
        left = left.reset_index(drop=True)
        right = right.reset_index(drop=True)
        tm.assert_frame_equal(left, right, *args, **kwargs)

    @staticmethod
    def default_series_rename(
        series: pd.Series, name: str = 'tmp'
    ) -> pd.Series:
        return series.rename(name)

    @staticmethod
    def greatest(
        f: Callable[..., ir.ValueExpr], *args: ir.ValueExpr
    ) -> ir.ValueExpr:
        return f(*args)

    @staticmethod
    def least(
        f: Callable[..., ir.ValueExpr], *args: ir.ValueExpr
    ) -> ir.ValueExpr:
        return f(*args)

    @property
    def db(self) -> ibis.client.Database:
        return self.connection.database()

    @property
    def functional_alltypes(self) -> ir.TableExpr:
        return self.db.functional_alltypes

    @property
    def batting(self) -> ir.TableExpr:
        return self.db.batting

    @property
    def awards_players(self) -> ir.TableExpr:
        return self.db.awards_players

    @property
    def geo(self) -> Optional[ir.TableExpr]:
        return None

    @property
    def api(self):
        return getattr(ibis.backends, self.name()).Backend()

    def make_context(
        self, params: Optional[Mapping[ir.ValueExpr, Any]] = None
    ) -> comp.QueryContext:
        return self.api.dialect.make_context(params=params)

# TODO move to the spark/pyspark backends, #2565
_spark_testing_client = None
_pyspark_testing_client = None

# TODO move to the sparn/pyspark backends, #2565
def get_spark_testing_client(data_directory):
    global _spark_testing_client
    if _spark_testing_client is None:
        _spark_testing_client = get_common_spark_testing_client(
            data_directory,
            lambda session: ibis.backends.spark.Backend().connect(session),
        )
    return _spark_testing_client

# TODO move to the spark/pyspark backends, #2565
def get_pyspark_testing_client(data_directory):
    global _pyspark_testing_client
    if _pyspark_testing_client is None:
        _pyspark_testing_client = get_common_spark_testing_client(
            data_directory,
            lambda session: ibis.backends.pyspark.Backend().connect(session),
        )
    return _pyspark_testing_client

# TODO move to the spark/pyspark backends, #2565
def get_common_spark_testing_client(data_directory, connect):
    pytest.importorskip('pyspark')
    import pyspark.sql.types as pt
    from pyspark.sql import SparkSession

    spark = (
        SparkSession.builder.config('spark.default.parallelism', 4)
        .config('spark.driver.bindAddress', '127.0.0.1')
        .getOrCreate()
    )
    _spark_testing_client = connect(spark)
    s = _spark_testing_client._session
    num_partitions = 4

    df_functional_alltypes = (
        s.read.csv(
            path=str(data_directory / 'functional_alltypes.csv'),
            schema=pt.StructType(
                [
                    pt.StructField('index', pt.IntegerType(), True),
                    pt.StructField('Unnamed: 0', pt.IntegerType(), True),
                    pt.StructField('id', pt.IntegerType(), True),
                    # cast below, Spark can't read 0/1 as bool
                    pt.StructField('bool_col', pt.ByteType(), True),
                    pt.StructField('tinyint_col', pt.ByteType(), True),
                    pt.StructField('smallint_col', pt.ShortType(), True),
                    pt.StructField('int_col', pt.IntegerType(), True),
                    pt.StructField('bigint_col', pt.LongType(), True),
                    pt.StructField('float_col', pt.FloatType(), True),
                    pt.StructField('double_col', pt.DoubleType(), True),
                    pt.StructField('date_string_col', pt.StringType(), True),
                    pt.StructField('string_col', pt.StringType(), True),
                    pt.StructField('timestamp_col', pt.TimestampType(), True),
                    pt.StructField('year', pt.IntegerType(), True),
                    pt.StructField('month', pt.IntegerType(), True),
                ]
            ),
            mode='FAILFAST',
            header=True,
        )
        .repartition(num_partitions)
        .sort('index')
    )

    df_functional_alltypes = df_functional_alltypes.withColumn(
        "bool_col", df_functional_alltypes["bool_col"].cast("boolean")
    )
    df_functional_alltypes.createOrReplaceTempView('functional_alltypes')

    df_batting = (
        s.read.csv(
            path=str(data_directory / 'batting.csv'),
            schema=pt.StructType(
                [
                    pt.StructField('playerID', pt.StringType(), True),
                    pt.StructField('yearID', pt.IntegerType(), True),
                    pt.StructField('stint', pt.IntegerType(), True),
                    pt.StructField('teamID', pt.StringType(), True),
                    pt.StructField('lgID', pt.StringType(), True),
                    pt.StructField('G', pt.IntegerType(), True),
                    pt.StructField('AB', pt.DoubleType(), True),
                    pt.StructField('R', pt.DoubleType(), True),
                    pt.StructField('H', pt.DoubleType(), True),
                    pt.StructField('X2B', pt.DoubleType(), True),
                    pt.StructField('X3B', pt.DoubleType(), True),
                    pt.StructField('HR', pt.DoubleType(), True),
                    pt.StructField('RBI', pt.DoubleType(), True),
                    pt.StructField('SB', pt.DoubleType(), True),
                    pt.StructField('CS', pt.DoubleType(), True),
                    pt.StructField('BB', pt.DoubleType(), True),
                    pt.StructField('SO', pt.DoubleType(), True),
                    pt.StructField('IBB', pt.DoubleType(), True),
                    pt.StructField('HBP', pt.DoubleType(), True),
                    pt.StructField('SH', pt.DoubleType(), True),
                    pt.StructField('SF', pt.DoubleType(), True),
                    pt.StructField('GIDP', pt.DoubleType(), True),
                ]
            ),
            header=True,
        )
        .repartition(num_partitions)
        .sort('playerID')
    )
    df_batting.createOrReplaceTempView('batting')

    df_awards_players = (
        s.read.csv(
            path=str(data_directory / 'awards_players.csv'),
            schema=pt.StructType(
                [
                    pt.StructField('playerID', pt.StringType(), True),
                    pt.StructField('awardID', pt.StringType(), True),
                    pt.StructField('yearID', pt.IntegerType(), True),
                    pt.StructField('lgID', pt.StringType(), True),
                    pt.StructField('tie', pt.StringType(), True),
                    pt.StructField('notes', pt.StringType(), True),
                ]
            ),
            header=True,
        )
        .repartition(num_partitions)
        .sort('playerID')
    )
    df_awards_players.createOrReplaceTempView('awards_players')

    df_simple = s.createDataFrame([(1, 'a')], ['foo', 'bar'])
    df_simple.createOrReplaceTempView('simple')

    df_struct = s.createDataFrame([((1, 2, 'a'),)], ['struct_col'])
    df_struct.createOrReplaceTempView('struct')

    df_nested_types = s.createDataFrame(
        [([1, 2], [[3, 4], [5, 6]], {'a': [[2, 4], [3, 5]]})],
        [
            'list_of_ints',
            'list_of_list_of_ints',
            'map_string_list_of_list_of_ints',
        ],
    )
    df_nested_types.createOrReplaceTempView('nested_types')

    df_complicated = s.createDataFrame(
        [({(1, 3): [[2, 4], [3, 5]]},)], ['map_tuple_list_of_list_of_ints']
    )
    df_complicated.createOrReplaceTempView('complicated')

    df_udf = s.createDataFrame(
        [('a', 1, 4.0, 'a'), ('b', 2, 5.0, 'a'), ('c', 3, 6.0, 'b')],
        ['a', 'b', 'c', 'key'],
    )
    df_udf.createOrReplaceTempView('udf')

    df_udf_nan = s.createDataFrame(
        pd.DataFrame(
            {
                'a': np.arange(10, dtype=float),
                'b': [3.0, np.NaN] * 5,
                'key': list('ddeefffggh'),
            }
        )
    )
    df_udf_nan.createOrReplaceTempView('udf_nan')

    df_udf_null = s.createDataFrame(
        [
            (float(i), None if i % 2 else 3.0, 'ddeefffggh'[i])
            for i in range(10)
        ],
        ['a', 'b', 'key'],
    )
    df_udf_null.createOrReplaceTempView('udf_null')

    df_udf_random = s.createDataFrame(
        pd.DataFrame(
            {
                'a': np.arange(4, dtype=float).tolist()
                + np.random.rand(3).tolist(),
                'b': np.arange(4, dtype=float).tolist()
                + np.random.rand(3).tolist(),
                'key': list('ddeefff'),
            }
        )
    )
    df_udf_random.createOrReplaceTempView('udf_random')

    return _spark_testing_client
<|endoftext|>


<commit_before>import re, argparse, sys, scholarly, urllib, feedparser, nltk, bs4, string
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('averaged_perceptron_tagger')
nltk.download('maxent_ne_chunker')
nltk.download('words')
nltk.download('stopwords')

from nltk import word_tokenize, sent_tokenize, corpus
from nltk.text import Text
from nltk import chunk
from nltk.chunk import *
from nltk.chunk.util import *
from nltk.chunk.regexp import *
from nltk.tree import Tree
from nltk.collocations import *
from nltk.corpus import wordnet, stopwords
from Bio import Entrez
from bs4 import BeautifulSoup

email = 'your.email.here@you.com'

wnl = nltk.WordNetLemmatizer()

translator = str.maketrans('', '', string.punctuation)

refseq_pattern = re.compile(r'\b([A-Z]{2}_[\d]+)')
insdc_pattern = re.compile(r'\b([SED]R[APRSXZ]\d{7})')
genbank_pattern = re.compile(r'\b([A-Z]{1}[0-9]{5}|[A-Z]{2}[0-9]{6}|[A-Z]{4}[0-9]{8,9}|[A-Z]{5}[0-9]{7})(\.[0-9]{1,3})*')
fallback_json = re.compile(r'\"text\":\"([A-Z0-9]+)\"')

def grab_pmid(pmid=4304705):
    print("Grabbing {0}".format(pmid))
#    url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id={0}'.format(pmid)
    url = 'https://www.ncbi.nlm.nih.gov/pmc/articles/pmid/{0}/'.format(pmid)
    headers = { 'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0' }
    req = urllib.request.Request(url=url, data=b'None', headers=headers)
    with urllib.request.urlopen(req) as response:
        html = response.read()
        soup = BeautifulSoup(html, "lxml")
        for script in soup(["script", "style"]):
            script.decompose()
        text = soup.get_text()

        # break into lines and remove leading and trailing space on each
        lines = (line.strip() for line in text.splitlines())
        # break multi-headlines into a line each
        chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
        # drop blank lines
        return '\n'.join(chunk for chunk in chunks if chunk)

def search(query):
    Entrez.email = email
    handle = Entrez.esearch(db='pubmed',
                            sort='relevance',
                            retmax='20',
                            retmode='xml',
                            term=query)
    results = Entrez.read(handle)
    return results

def fetch_details(id_list):
    ids = ','.join(id_list)
    Entrez.email = email
    handle = Entrez.efetch(db='pubmed',
                           retmode='xml',
                           id=ids)
    results = Entrez.read(handle)
    return results

## create a filter that consults a scored dictionary and a list of query words, from http://stackoverflow.com/questions/23479179/combining-filters-in-nltk-collocations
def create_filter_minfreq_inwords(scored, words, minfreq):
    def bigram_filter(w1, w2):
        return (w1 not in words and w2 not in words) and (
                (w1, w2) in scored and scored[w1, w2] <= minfreq)
    return bigram_filter

## collect word phases, from https://simply-python.com/2014/03/14/saving-output-of-nltk-text-concordance/
def word_phases(target_word, query_text, left_margin = 10, right_margin = 10):
    """
        Function to get all the phases that contain the target word in a text/passage tar_passage.

        str target_word, str tar_passage int left_margin int right_margin --> list of str
        left_margin and right_margin allocate the number of words/pununciation before and after target word
        Left margin will take note of the beginning of the text
    """
    ## Create list of tokens using nltk function
    #tokens = nltk.word_tokenize(abstract)

    ## Create the text of tokens
    #text = nltk.Text(tokens)

    ## Collect all the index or offset position of the target word
    c = nltk.ConcordanceIndex(query_text.tokens, key = lambda s: s.lower())

    ## Collect the range of the words that is within the target word by using text.tokens[start;end].
    ## The map function is use so that when the offset position - the target range < 0, it will be default to zero
    concordance_txt = ([query_text.tokens[list(map(lambda x: x-5 if (x-left_margin)>0 else 0,[offset]))[0]:offset+right_margin] for offset in c.offsets(target_word)])

    ## join the sentences for each of the target phrase and return it
    return [''.join([x+' ' for x in con_sub]) for con_sub in concordance_txt]

def extract_phases(tokens, wordlist):
    all_phases = []
    text = nltk.Text(tokens)
    for word in wordlist:
        phases = word_phases(word, text)
        if phases:
            all_phases.append(phases)
    return all_phases

def chunk_entities(tokens):
    ## produce tags
    tagged = nltk.pos_tag(tokens)

    ## identify named entities, and don't expand NE types
    return nltk.chunk.ne_chunk(tagged, binary=True)

def find_and_filter_bigrams(tokens):
    # initialize finder object with the tokens
    finder = nltk.collocations.BigramCollocationFinder.from_words(tokens)

    # build a dictionary with bigrams and their frequencies
    bigram_measures = nltk.collocations.BigramAssocMeasures()
    scored = dict(finder.score_ngrams(bigram_measures.raw_freq))

    # create the filter...
    myfilter = create_filter_minfreq_inwords(scored, wordlist, 0.1)
#    before_filter = list(finder.score_ngrams(bigram_measures.raw_freq))
#    if args.verbose:
#        print('Before filter:\n', before_filter)

    # apply filter
    finder.apply_ngram_filter(myfilter)
    after_filter = list(finder.score_ngrams(bigram_measures.raw_freq))
    if args.verbose:
        print('\nAfter filter:\n', after_filter)
    return after_filter

def score_chunks(chunkparser, tree, trace=0):
    chunkscore = chunk.ChunkScore()

    tokens = tree.leaves()
    gold = chunkparser.parse(Tree('S', tokens), trace)
    chunkscore.score(gold, tree)

    a = chunkscore.accuracy()*100
    p = chunkscore.precision()*100
    r = chunkscore.recall()*100
    f = chunkscore.f_measure()*100
    i = chunkscore.incorrect()
    m = chunkscore.missed()

    if args.verbose:
        print('Testing: {0}'.format(tree))
        print('Guessed: {0}'.format(chunkscore.guessed()))
        print('\n/'+('='*75)+'\\')
        print('Scoring', chunkparser)
        print(('-'*77))
        print('Accuracy: %5.1f%%' % (a), ' '*4, end=' ')
        print('Precision: %5.1f%%' % (p), ' '*4, end=' ')
        print('Recall: %5.1f%%' % (r), ' '*6, end=' ')
        print('F-Measure: %5.1f%%' % (f))
        print('Missing: {0} -> {1}'.format(len(m), m))
        print('Incorrect: {0} -> {1}'.format(len(i), i))

    return (a, p, r, f, i, m)

def detectohumans(bigrams, phases, title):
    if not bigrams:
       print('No context detected to ascertain humanness')
    else:
       for bigram in bigrams:
           if str.lower(bigram[0][0]) == 'human' or str.lower(bigram[0][1]) == 'human':
               print(title.rstrip('.') + ', in humans.')
               return True
           else:
               print("No humans detected. This paper might be relevant to stuff other than humans.")
    return False

def detectogenome(bigrams, phases, title):
    orig_title_tokens = nltk.word_tokenize(title)
    lc_title_tokens = nltk.word_tokenize(str.lower(title))

    #grammar = "NE: {<JJ>*<NN|JJ|NNP><NN|NNP><IN|TO>*}"
    #chunk_rule = ChunkRule("<.*>+", "Chunk all")
    another_chunk = ChunkRule("<JJ>*<NN.*>+<NN.*>+<IN|OF>*", "[Complete] genome sequence ish")
    chink_rule = ChinkRule("<CC|VB|IN|\.>", "Chink some")
    another_chink = ChinkRule("<NNP><NN>", "Chink some more")
    #split_rule = SplitRule("<JJ>*<NN|NNP><NN|NNP>", "Split some")

#    cp = nltk.RegexpParser(grammar)
    cp = nltk.RegexpChunkParser([another_chunk, chink_rule, another_chink], chunk_label='NP')

    o_chunks = chunk_entities(orig_title_tokens)
    o_tree = cp.parse(o_chunks)
    stats = score_chunks(cp, o_tree)

    maybe = False

    ## check abstract text
    for phase in phases:
        for sent in phase:
            if args.verbose:
                print('\nPhase: {0}'.format(sent))

            phase_tokens = nltk.word_tokenize(sent)
            phase_chunks = chunk_entities(phase_tokens)
            phase_tree = cp.parse(phase_chunks)
            stats = score_chunks(cp, phase_tree)

            if stats[0] > 50 and stats[1] > 50 and stats[2] > 25 and stats[3] > 0:
                print('Looks like we have ourselves a genome paper')
                maybe = True

    for bigram in bigrams:
        if str.lower(bigram[0][0]) == 'complete' and str.lower(bigram[0][1]).endswith('genome'):
            print('We DEFINITELY have a genome paper')
            return True
    return maybe

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-q', '--query')
    parser.add_argument('-w', '--wordfile')
    parser.add_argument('-s', dest='human', action='store_true')
    parser.add_argument('-g', dest='genome', action='store_true')
    parser.add_argument('-v', dest='verbose', action='store_true')
    args = parser.parse_args()

    if args.wordfile is None:
        if args.verbose:
            print('No wordfile. Defaulting to human stuff')
        wordlist = ['GCRh37', 'GCRh38', 'human', 'genome', 'variant']
    else:
        wordlist = []
        if args.verbose:
            print('Wordfile specified. Loading...')
        with open(args.wordfile) as f:
            for t in set(f.read().splitlines()):
#                wordlist.append(wordnet.morphy(str.lower(t)))
                wordlist.append(wnl.lemmatize(t, 'n'))

    results = search(args.query)
    id_list = results['IdList']
    abstract = None
    meshterms = []
    papers = []

    if not id_list:
        print('Nothing in pubmed, looking in arXiv and Google Scholar...')
        # try arxiv
        url = 'http://export.arxiv.org/api/query'
        q = 'ti:"{0}"'.format(args.query)
        params = {'search_query':q, 'start':0, 'max_results':1}
        data = urllib.parse.urlencode(params).encode('ascii')
        req = urllib.request.Request(url, data)
        with urllib.request.urlopen(req) as response:
            res = response.read()
            feed = feedparser.parse(res)
            if not feed.entries:
                print('Not found in arXiv\n')
            else:
                abstract = feed.entries[0].summary
                if args.verbose:
                    print('Found in arXiv:\n' + abstract + '\n')

        # try scholar
        search_query = scholarly.search_pubs_query('%22'+args.query+'%22')
        for scholar_result in search_query:
            sr = scholar_result.fill()
            if not sr.bib:
                print('Not found in Google Scholar\n')
            else:
                if not sr.bib["abstract"]:
                    # ignore
                    continue
                abstract = scholar_result.bib["abstract"]
                if args.verbose:
                    print('Found Google Scholar records:\n' + abstract + '\n')
                break
    else:
        papers = fetch_details(id_list)
        for i, paper in enumerate(papers["PubmedArticle"]):
            if 'MeshHeadingList' in paper["MedlineCitation"]:
                for mesh in paper["MedlineCitation"]['MeshHeadingList']:
                    for term in nltk.word_tokenize(mesh["DescriptorName"]):
                        meshterms.append(wnl.lemmatize(str.lower(term), 'n'))
                if args.verbose:
                    print(meshterms)
            abstract = paper["MedlineCitation"]["Article"]["Abstract"]["AbstractText"][0]

    if not abstract:
        print('No abstracts available')
    else:
        ## tokenise into word context
        tokens = nltk.word_tokenize(abstract)

        ## identify named entities
        entities = chunk_entities(tokens)
        if args.verbose:
            print('Chunks:\n{0}'.format(entities))

        # extract base phases
        base_phases = extract_phases(tokens, wordlist)
        filtered_bigrams = find_and_filter_bigrams(tokens)
        if args.verbose:
            print('Located potential base phases:\n{0}'.format(base_phases))

        if args.human:
            if detectohumans(filtered_bigrams, base_phases, args.query) == False:
                ## inject mesh terms and extract phases
                tokens.extend(meshterms)
                all_phases = extract_phases(tokens, wordlist)
                mesh_bigrams = find_and_filter_bigrams(tokens)
                if args.verbose:
                    print('Located potential MeSH phases:\n{0}'.format(all_phases))
                detectohumans(mesh_bigrams, all_phases, args.query)

        if args.genome:
            detected = detectogenome(filtered_bigrams, base_phases, args.query)
            if detected:
                content = grab_pmid(id_list[0])
                if args.verbose:
                    print(content)
                seq_list = ['GenBank', 'ID', 'No.', 'no.', 'ENA', 'SRA', 'NCBI', 'genome', 'accession', 'annotation', 'DDBJ', 'EMBL', 'deposited']
                full_tokens = nltk.word_tokenize(content)
                no_stop_tokens = [w.translate(translator) for w in full_tokens if not w in stopwords.words('english')]
                entities = chunk_entities(full_tokens)
                full_phases = extract_phases(full_tokens, seq_list)
                full_filtered_bigrams = find_and_filter_bigrams(no_stop_tokens)
                if args.verbose:
                    print('Located full-text phases:\n{0}'.format(full_phases))
                    print(full_filtered_bigrams)

                refseq_matches = set(refseq_pattern.findall(content))
                insdc_matches = set(insdc_pattern.findall(content))
                genbank_matches = set(genbank_pattern.findall(content))

                fallback_matches = fallback_json.findall(content)
                if not refseq_matches and not insdc_matches and not genbank_matches:
                    for fbm in fallback_matches:
                        refseq_matches = set(refseq_pattern.findall(fbm))
                        insdc_matches = set(insdc_pattern.findall(fbm))
                        genbank_matches = set(genbank_pattern.findall(fbm))

                print("Refseq: {0}".format(refseq_matches))
                print("INSDC: {0}".format(insdc_matches))
                print("GenBank: {0}".format(genbank_matches))

<commit_msg>Fix INSDC pattern numeral length<commit_after>import re, argparse, sys, scholarly, urllib, feedparser, nltk, bs4, string
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('averaged_perceptron_tagger')
nltk.download('maxent_ne_chunker')
nltk.download('words')
nltk.download('stopwords')

from nltk import word_tokenize, sent_tokenize, corpus
from nltk.text import Text
from nltk import chunk
from nltk.chunk import *
from nltk.chunk.util import *
from nltk.chunk.regexp import *
from nltk.tree import Tree
from nltk.collocations import *
from nltk.corpus import wordnet, stopwords
from Bio import Entrez
from bs4 import BeautifulSoup

email = 'your.email.here@you.com'

wnl = nltk.WordNetLemmatizer()

translator = str.maketrans('', '', string.punctuation)

refseq_pattern = re.compile(r'\b([A-Z]{2}_[\d]+)')
insdc_pattern = re.compile(r'\b([SED]R[APRSXZ]\d{6,7})')
genbank_pattern = re.compile(r'\b([A-Z]{1}[0-9]{5}|[A-Z]{2}[0-9]{6}|[A-Z]{4}[0-9]{8,9}|[A-Z]{5}[0-9]{7})(\.[0-9]{1,3})*')
fallback_json = re.compile(r'\"text\":\"([A-Z0-9]+)\"')

def grab_pmid(pmid=4304705):
    print("Grabbing {0}".format(pmid))
#    url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id={0}'.format(pmid)
    url = 'https://www.ncbi.nlm.nih.gov/pmc/articles/pmid/{0}/'.format(pmid)
    headers = { 'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0' }
    req = urllib.request.Request(url=url, data=b'None', headers=headers)
    with urllib.request.urlopen(req) as response:
        html = response.read()
        soup = BeautifulSoup(html, "lxml")
        for script in soup(["script", "style"]):
            script.decompose()
        text = soup.get_text()

        # break into lines and remove leading and trailing space on each
        lines = (line.strip() for line in text.splitlines())
        # break multi-headlines into a line each
        chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
        # drop blank lines
        return '\n'.join(chunk for chunk in chunks if chunk)

def search(query):
    Entrez.email = email
    handle = Entrez.esearch(db='pubmed',
                            sort='relevance',
                            retmax='20',
                            retmode='xml',
                            term=query)
    results = Entrez.read(handle)
    return results

def fetch_details(id_list):
    ids = ','.join(id_list)
    Entrez.email = email
    handle = Entrez.efetch(db='pubmed',
                           retmode='xml',
                           id=ids)
    results = Entrez.read(handle)
    return results

## create a filter that consults a scored dictionary and a list of query words, from http://stackoverflow.com/questions/23479179/combining-filters-in-nltk-collocations
def create_filter_minfreq_inwords(scored, words, minfreq):
    def bigram_filter(w1, w2):
        return (w1 not in words and w2 not in words) and (
                (w1, w2) in scored and scored[w1, w2] <= minfreq)
    return bigram_filter

## collect word phases, from https://simply-python.com/2014/03/14/saving-output-of-nltk-text-concordance/
def word_phases(target_word, query_text, left_margin = 10, right_margin = 10):
    """
        Function to get all the phases that contain the target word in a text/passage tar_passage.

        str target_word, str tar_passage int left_margin int right_margin --> list of str
        left_margin and right_margin allocate the number of words/pununciation before and after target word
        Left margin will take note of the beginning of the text
    """
    ## Create list of tokens using nltk function
    #tokens = nltk.word_tokenize(abstract)

    ## Create the text of tokens
    #text = nltk.Text(tokens)

    ## Collect all the index or offset position of the target word
    c = nltk.ConcordanceIndex(query_text.tokens, key = lambda s: s.lower())

    ## Collect the range of the words that is within the target word by using text.tokens[start;end].
    ## The map function is use so that when the offset position - the target range < 0, it will be default to zero
    concordance_txt = ([query_text.tokens[list(map(lambda x: x-5 if (x-left_margin)>0 else 0,[offset]))[0]:offset+right_margin] for offset in c.offsets(target_word)])

    ## join the sentences for each of the target phrase and return it
    return [''.join([x+' ' for x in con_sub]) for con_sub in concordance_txt]

def extract_phases(tokens, wordlist):
    all_phases = []
    text = nltk.Text(tokens)
    for word in wordlist:
        phases = word_phases(word, text)
        if phases:
            all_phases.append(phases)
    return all_phases

def chunk_entities(tokens):
    ## produce tags
    tagged = nltk.pos_tag(tokens)

    ## identify named entities, and don't expand NE types
    return nltk.chunk.ne_chunk(tagged, binary=True)

def find_and_filter_bigrams(tokens):
    # initialize finder object with the tokens
    finder = nltk.collocations.BigramCollocationFinder.from_words(tokens)

    # build a dictionary with bigrams and their frequencies
    bigram_measures = nltk.collocations.BigramAssocMeasures()
    scored = dict(finder.score_ngrams(bigram_measures.raw_freq))

    # create the filter...
    myfilter = create_filter_minfreq_inwords(scored, wordlist, 0.1)
#    before_filter = list(finder.score_ngrams(bigram_measures.raw_freq))
#    if args.verbose:
#        print('Before filter:\n', before_filter)

    # apply filter
    finder.apply_ngram_filter(myfilter)
    after_filter = list(finder.score_ngrams(bigram_measures.raw_freq))
    if args.verbose:
        print('\nAfter filter:\n', after_filter)
    return after_filter

def score_chunks(chunkparser, tree, trace=0):
    chunkscore = chunk.ChunkScore()

    tokens = tree.leaves()
    gold = chunkparser.parse(Tree('S', tokens), trace)
    chunkscore.score(gold, tree)

    a = chunkscore.accuracy()*100
    p = chunkscore.precision()*100
    r = chunkscore.recall()*100
    f = chunkscore.f_measure()*100
    i = chunkscore.incorrect()
    m = chunkscore.missed()

    if args.verbose:
        print('Testing: {0}'.format(tree))
        print('Guessed: {0}'.format(chunkscore.guessed()))
        print('\n/'+('='*75)+'\\')
        print('Scoring', chunkparser)
        print(('-'*77))
        print('Accuracy: %5.1f%%' % (a), ' '*4, end=' ')
        print('Precision: %5.1f%%' % (p), ' '*4, end=' ')
        print('Recall: %5.1f%%' % (r), ' '*6, end=' ')
        print('F-Measure: %5.1f%%' % (f))
        print('Missing: {0} -> {1}'.format(len(m), m))
        print('Incorrect: {0} -> {1}'.format(len(i), i))

    return (a, p, r, f, i, m)

def detectohumans(bigrams, phases, title):
    if not bigrams:
       print('No context detected to ascertain humanness')
    else:
       for bigram in bigrams:
           if str.lower(bigram[0][0]) == 'human' or str.lower(bigram[0][1]) == 'human':
               print(title.rstrip('.') + ', in humans.')
               return True
           else:
               print("No humans detected. This paper might be relevant to stuff other than humans.")
    return False

def detectogenome(bigrams, phases, title):
    orig_title_tokens = nltk.word_tokenize(title)
    lc_title_tokens = nltk.word_tokenize(str.lower(title))

    #grammar = "NE: {<JJ>*<NN|JJ|NNP><NN|NNP><IN|TO>*}"
    #chunk_rule = ChunkRule("<.*>+", "Chunk all")
    another_chunk = ChunkRule("<JJ>*<NN.*>+<NN.*>+<IN|OF>*", "[Complete] genome sequence ish")
    chink_rule = ChinkRule("<CC|VB|IN|\.>", "Chink some")
    another_chink = ChinkRule("<NNP><NN>", "Chink some more")
    #split_rule = SplitRule("<JJ>*<NN|NNP><NN|NNP>", "Split some")

#    cp = nltk.RegexpParser(grammar)
    cp = nltk.RegexpChunkParser([another_chunk, chink_rule, another_chink], chunk_label='NP')

    o_chunks = chunk_entities(orig_title_tokens)
    o_tree = cp.parse(o_chunks)
    stats = score_chunks(cp, o_tree)

    maybe = False

    ## check abstract text
    for phase in phases:
        for sent in phase:
            if args.verbose:
                print('\nPhase: {0}'.format(sent))

            phase_tokens = nltk.word_tokenize(sent)
            phase_chunks = chunk_entities(phase_tokens)
            phase_tree = cp.parse(phase_chunks)
            stats = score_chunks(cp, phase_tree)

            if stats[0] > 50 and stats[1] > 50 and stats[2] > 25 and stats[3] > 0:
                print('Looks like we have ourselves a genome paper')
                maybe = True

    for bigram in bigrams:
        if str.lower(bigram[0][0]) == 'complete' and str.lower(bigram[0][1]).endswith('genome'):
            print('We DEFINITELY have a genome paper')
            return True
    return maybe

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-q', '--query')
    parser.add_argument('-w', '--wordfile')
    parser.add_argument('-s', dest='human', action='store_true')
    parser.add_argument('-g', dest='genome', action='store_true')
    parser.add_argument('-v', dest='verbose', action='store_true')
    args = parser.parse_args()

    if args.wordfile is None:
        if args.verbose:
            print('No wordfile. Defaulting to human stuff')
        wordlist = ['GCRh37', 'GCRh38', 'human', 'genome', 'variant']
    else:
        wordlist = []
        if args.verbose:
            print('Wordfile specified. Loading...')
        with open(args.wordfile) as f:
            for t in set(f.read().splitlines()):
#                wordlist.append(wordnet.morphy(str.lower(t)))
                wordlist.append(wnl.lemmatize(t, 'n'))

    results = search(args.query)
    id_list = results['IdList']
    abstract = None
    meshterms = []
    papers = []

    if not id_list:
        print('Nothing in pubmed, looking in arXiv and Google Scholar...')
        # try arxiv
        url = 'http://export.arxiv.org/api/query'
        q = 'ti:"{0}"'.format(args.query)
        params = {'search_query':q, 'start':0, 'max_results':1}
        data = urllib.parse.urlencode(params).encode('ascii')
        req = urllib.request.Request(url, data)
        with urllib.request.urlopen(req) as response:
            res = response.read()
            feed = feedparser.parse(res)
            if not feed.entries:
                print('Not found in arXiv\n')
            else:
                abstract = feed.entries[0].summary
                if args.verbose:
                    print('Found in arXiv:\n' + abstract + '\n')

        # try scholar
        search_query = scholarly.search_pubs_query('%22'+args.query+'%22')
        for scholar_result in search_query:
            sr = scholar_result.fill()
            if not sr.bib:
                print('Not found in Google Scholar\n')
            else:
                if not sr.bib["abstract"]:
                    # ignore
                    continue
                abstract = scholar_result.bib["abstract"]
                if args.verbose:
                    print('Found Google Scholar records:\n' + abstract + '\n')
                break
    else:
        papers = fetch_details(id_list)
        for i, paper in enumerate(papers["PubmedArticle"]):
            if 'MeshHeadingList' in paper["MedlineCitation"]:
                for mesh in paper["MedlineCitation"]['MeshHeadingList']:
                    for term in nltk.word_tokenize(mesh["DescriptorName"]):
                        meshterms.append(wnl.lemmatize(str.lower(term), 'n'))
                if args.verbose:
                    print(meshterms)
            abstract = paper["MedlineCitation"]["Article"]["Abstract"]["AbstractText"][0]

    if not abstract:
        print('No abstracts available')
    else:
        ## tokenise into word context
        tokens = nltk.word_tokenize(abstract)

        ## identify named entities
        entities = chunk_entities(tokens)
        if args.verbose:
            print('Chunks:\n{0}'.format(entities))

        # extract base phases
        base_phases = extract_phases(tokens, wordlist)
        filtered_bigrams = find_and_filter_bigrams(tokens)
        if args.verbose:
            print('Located potential base phases:\n{0}'.format(base_phases))

        if args.human:
            if detectohumans(filtered_bigrams, base_phases, args.query) == False:
                ## inject mesh terms and extract phases
                tokens.extend(meshterms)
                all_phases = extract_phases(tokens, wordlist)
                mesh_bigrams = find_and_filter_bigrams(tokens)
                if args.verbose:
                    print('Located potential MeSH phases:\n{0}'.format(all_phases))
                detectohumans(mesh_bigrams, all_phases, args.query)

        if args.genome:
            detected = detectogenome(filtered_bigrams, base_phases, args.query)
            if detected:
                content = grab_pmid(id_list[0])
                if args.verbose:
                    print(content)
                seq_list = ['GenBank', 'ID', 'No.', 'no.', 'ENA', 'SRA', 'NCBI', 'genome', 'accession', 'annotation', 'DDBJ', 'EMBL', 'deposited']
                full_tokens = nltk.word_tokenize(content)
                no_stop_tokens = [w.translate(translator) for w in full_tokens if not w in stopwords.words('english')]
                entities = chunk_entities(full_tokens)
                full_phases = extract_phases(full_tokens, seq_list)
                full_filtered_bigrams = find_and_filter_bigrams(no_stop_tokens)
                if args.verbose:
                    print('Located full-text phases:\n{0}'.format(full_phases))
                    print(full_filtered_bigrams)

                refseq_matches = set(refseq_pattern.findall(content))
                insdc_matches = set(insdc_pattern.findall(content))
                genbank_matches = set(genbank_pattern.findall(content))

                fallback_matches = fallback_json.findall(content)
                if not refseq_matches and not insdc_matches and not genbank_matches:
                    for fbm in fallback_matches:
                        refseq_matches = set(refseq_pattern.findall(fbm))
                        insdc_matches = set(insdc_pattern.findall(fbm))
                        genbank_matches = set(genbank_pattern.findall(fbm))

                print("Refseq: {0}".format(refseq_matches))
                print("INSDC: {0}".format(insdc_matches))
                print("GenBank: {0}".format(genbank_matches))

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    flaskbb.management.models
    ~~~~~~~~~~~~~~~~~~~~~~~~~

    This module contains all management related models.

    :copyright: (c) 2014 by the FlaskBB Team.
    :license: BSD, see LICENSE for more details.
"""
import sys
from wtforms import (TextField, IntegerField, FloatField, BooleanField,
                     SelectField, SelectMultipleField, validators)
from flask.ext.wtf import Form
from flaskbb.extensions import db, cache

class SettingsGroup(db.Model):
    __tablename__ = "settingsgroup"

    key = db.Column(db.String(255), primary_key=True)
    name = db.Column(db.String(255), nullable=False)
    description = db.Column(db.Text, nullable=False)
    settings = db.relationship("Setting", lazy="dynamic", backref="group",
                               cascade="all, delete-orphan")

    def save(self):
        """Saves a settingsgroup."""
        db.session.add(self)
        db.session.commit()

    def delete(self):
        """Deletes a settingsgroup."""
        db.session.delete(self)
        db.session.commit()

class Setting(db.Model):
    __tablename__ = "settings"

    key = db.Column(db.String(255), primary_key=True)
    value = db.Column(db.PickleType, nullable=False)
    settingsgroup = db.Column(db.String,
                              db.ForeignKey('settingsgroup.key',
                                            use_alter=True,
                                            name="fk_settingsgroup"),
                              nullable=False)

    # The name (displayed in the form)
    name = db.Column(db.String(200), nullable=False)

    # The description (displayed in the form)
    description = db.Column(db.Text, nullable=False)

    # Available types: string, integer, float, boolean, select, selectmultiple
    value_type = db.Column(db.String(20), nullable=False)

    # Extra attributes like, validation things (min, max length...)
    # For Select*Fields required: choices
    extra = db.Column(db.PickleType)

    @classmethod
    def get_form(cls, group):
        """Returns a Form for all settings found in :class:`SettingsGroup`.

        :param group: The settingsgroup name. It is used to get the settings
                      which are in the specified group.
        """

        class SettingsForm(Form):
            pass

        # now parse the settings in this group
        for setting in group.settings:
            field_validators = []

            # generate the validators
            if "min" in setting.extra:
                # Min number validator
                if setting.value_type in ("integer", "float"):
                    field_validators.append(
                        validators.NumberRange(min=setting.extra["min"])
                    )

                # Min text length validator
                elif setting.value_type in ("string"):
                    field_validators.append(
                        validators.Length(min=setting.extra["min"])
                    )

            if "max" in setting.extra:
                # Max number validator
                if setting.value_type in ("integer", "float"):
                    field_validators.append(
                        validators.NumberRange(max=setting.extra["max"])
                    )

                # Max text length validator
                elif setting.value_type in ("string"):
                    field_validators.append(
                        validators.Length(max=setting.extra["max"])
                    )

            # Generate the fields based on value_type
            # IntegerField
            if setting.value_type == "integer":
                setattr(
                    SettingsForm, setting.key,
                    IntegerField(setting.name, validators=field_validators,
                                 description=setting.description)
                )
            # FloatField
            elif setting.value_type == "float":
                setattr(
                    SettingsForm, setting.key,
                    FloatField(setting.name, validators=field_validators,
                               description=setting.description)
                )

            # TextField
            if setting.value_type == "string":
                setattr(
                    SettingsForm, setting.key,
                    TextField(setting.name, validators=field_validators,
                              description=setting.description)
                )

            # SelectMultipleField
            if setting.value_type == "selectmultiple":
                # if no coerce is found, it will fallback to unicode
                if "coerce" in setting.extra:
                    coerce_to = setting.extra['coerce']
                else:
                    coerce_to = unicode

                setattr(
                    SettingsForm, setting.key,
                    SelectMultipleField(
                        setting.name,
                        choices=setting.extra['choices'](),
                        coerce=coerce_to,
                        description=setting.description
                    )
                )

            # SelectField
            if setting.value_type == "select":
                # if no coerce is found, it will fallback to unicode
                if "coerce" in setting.extra:
                    coerce_to = setting.extra['coerce']
                else:
                    coerce_to = unicode

                setattr(
                    SettingsForm, setting.key,
                    SelectField(
                        setting.name,
                        coerce=coerce_to,
                        choices=setting.extra['choices'](),
                        description=setting.description)
                )

            # BooleanField
            if setting.value_type == "boolean":
                setattr(
                    SettingsForm, setting.key,
                    BooleanField(setting.name, description=setting.description)
                )

        return SettingsForm

    @classmethod
    def get_all(cls):
        return cls.query.all()

    @classmethod
    def update(cls, settings, app=None):
        """Updates the cache and stores the changes in the
        database.

        :param settings: A dictionary with setting items.
        """
        # update the database
        for key, value in settings.iteritems():
            setting = cls.query.filter(Setting.key == key.lower()).first()

            setting.value = value

            db.session.add(setting)
            db.session.commit()

        cls.invalidate_cache()

    @classmethod
    def get_settings(cls, from_group=None):
        """This will return all settings with the key as the key for the dict
        and the values are packed again in a dict which contains
        the remaining attributes.

        :param from_group: Optionally - Returns only the settings from a group.
        """
        result = None
        if from_group is not None:
            result = from_group.settings
        else:
            result = cls.query.all()

        settings = {}
        for setting in result:
            settings[setting.key] = {
                'name': setting.name,
                'description': setting.description,
                'value': setting.value,
                'value_type': setting.value_type,
                'extra': setting.extra
            }

        return settings

    @classmethod
    @cache.memoize(timeout=sys.maxint)
    def as_dict(cls, from_group=None, upper=True):
        """Returns all settings as a dict. This method is cached. If you want
        to invalidate the cache, simply execute ``self.invalidate_cache()``.

        :param from_group: Returns only the settings from the group as a dict.
        :param upper: If upper is ``True``, the key will use upper-case
                      letters. Defaults to ``False``.
        """

        settings = {}
        result = None
        if from_group is not None:
            result = SettingsGroup.query.filter_by(key=from_group).\
                first_or_404()
            result = result.settings
        else:
            result = cls.query.all()

        for setting in result:
            if upper:
                setting_key = setting.key.upper()
            else:
                setting_key = setting.key

            settings[setting_key] = setting.value

        return settings

    @classmethod
    def invalidate_cache(cls):
        """Invalidates this objects cached metadata."""
        cache.delete_memoized(cls.as_dict, cls)

    def save(self):
        """Saves a setting"""
        db.session.add(self)
        db.session.commit()

    def delete(self):
        """Deletes a setting"""
        db.session.delete(self)
        db.session.commit()
<commit_msg>Management/Models: Reduced Complexity<commit_after># -*- coding: utf-8 -*-
"""
    flaskbb.management.models
    ~~~~~~~~~~~~~~~~~~~~~~~~~

    This module contains all management related models.

    :copyright: (c) 2014 by the FlaskBB Team.
    :license: BSD, see LICENSE for more details.
"""
import sys
from wtforms import (TextField, IntegerField, FloatField, BooleanField,
                     SelectField, SelectMultipleField, validators)
from flask.ext.wtf import Form
from flaskbb.extensions import db, cache

class SettingsGroup(db.Model):
    __tablename__ = "settingsgroup"

    key = db.Column(db.String(255), primary_key=True)
    name = db.Column(db.String(255), nullable=False)
    description = db.Column(db.Text, nullable=False)
    settings = db.relationship("Setting", lazy="dynamic", backref="group",
                               cascade="all, delete-orphan")

    def save(self):
        """Saves a settingsgroup."""
        db.session.add(self)
        db.session.commit()

    def delete(self):
        """Deletes a settingsgroup."""
        db.session.delete(self)
        db.session.commit()

class Setting(db.Model):
    __tablename__ = "settings"

    key = db.Column(db.String(255), primary_key=True)
    value = db.Column(db.PickleType, nullable=False)
    settingsgroup = db.Column(db.String,
                              db.ForeignKey('settingsgroup.key',
                                            use_alter=True,
                                            name="fk_settingsgroup"),
                              nullable=False)

    # The name (displayed in the form)
    name = db.Column(db.String(200), nullable=False)

    # The description (displayed in the form)
    description = db.Column(db.Text, nullable=False)

    # Available types: string, integer, float, boolean, select, selectmultiple
    value_type = db.Column(db.String(20), nullable=False)

    # Extra attributes like, validation things (min, max length...)
    # For Select*Fields required: choices
    extra = db.Column(db.PickleType)

    @classmethod
    def get_form(cls, group):
        """Returns a Form for all settings found in :class:`SettingsGroup`.

        :param group: The settingsgroup name. It is used to get the settings
                      which are in the specified group.
        """

        class SettingsForm(Form):
            pass

        # now parse the settings in this group
        for setting in group.settings:
            field_validators = []

            if setting.value_type in ("integer", "float"):
                validator_class = validators.NumberRange
            elif setting.value_type == "string":
                validator_class = validators.Length

            # generate the validators
            if "min" in setting.extra:
                # Min number validator
                field_validators.append(
                    validator_class(min=setting.extra["min"])
                )

            if "max" in setting.extra:
                # Max number validator
                field_validators.append(
                    validator_class(max=setting.extra["max"])
                )

            # Generate the fields based on value_type
            # IntegerField
            if setting.value_type == "integer":
                setattr(
                    SettingsForm, setting.key,
                    IntegerField(setting.name, validators=field_validators,
                                 description=setting.description)
                )
            # FloatField
            elif setting.value_type == "float":
                setattr(
                    SettingsForm, setting.key,
                    FloatField(setting.name, validators=field_validators,
                               description=setting.description)
                )

            # TextField
            elif setting.value_type == "string":
                setattr(
                    SettingsForm, setting.key,
                    TextField(setting.name, validators=field_validators,
                              description=setting.description)
                )

            # SelectMultipleField
            elif setting.value_type == "selectmultiple":
                # if no coerce is found, it will fallback to unicode
                if "coerce" in setting.extra:
                    coerce_to = setting.extra['coerce']
                else:
                    coerce_to = unicode

                setattr(
                    SettingsForm, setting.key,
                    SelectMultipleField(
                        setting.name,
                        choices=setting.extra['choices'](),
                        coerce=coerce_to,
                        description=setting.description
                    )
                )

            # SelectField
            elif setting.value_type == "select":
                # if no coerce is found, it will fallback to unicode
                if "coerce" in setting.extra:
                    coerce_to = setting.extra['coerce']
                else:
                    coerce_to = unicode

                setattr(
                    SettingsForm, setting.key,
                    SelectField(
                        setting.name,
                        coerce=coerce_to,
                        choices=setting.extra['choices'](),
                        description=setting.description)
                )

            # BooleanField
            elif setting.value_type == "boolean":
                setattr(
                    SettingsForm, setting.key,
                    BooleanField(setting.name, description=setting.description)
                )

        return SettingsForm

    @classmethod
    def get_all(cls):
        return cls.query.all()

    @classmethod
    def update(cls, settings, app=None):
        """Updates the cache and stores the changes in the
        database.

        :param settings: A dictionary with setting items.
        """
        # update the database
        for key, value in settings.iteritems():
            setting = cls.query.filter(Setting.key == key.lower()).first()

            setting.value = value

            db.session.add(setting)
            db.session.commit()

        cls.invalidate_cache()

    @classmethod
    def get_settings(cls, from_group=None):
        """This will return all settings with the key as the key for the dict
        and the values are packed again in a dict which contains
        the remaining attributes.

        :param from_group: Optionally - Returns only the settings from a group.
        """
        result = None
        if from_group is not None:
            result = from_group.settings
        else:
            result = cls.query.all()

        settings = {}
        for setting in result:
            settings[setting.key] = {
                'name': setting.name,
                'description': setting.description,
                'value': setting.value,
                'value_type': setting.value_type,
                'extra': setting.extra
            }

        return settings

    @classmethod
    @cache.memoize(timeout=sys.maxint)
    def as_dict(cls, from_group=None, upper=True):
        """Returns all settings as a dict. This method is cached. If you want
        to invalidate the cache, simply execute ``self.invalidate_cache()``.

        :param from_group: Returns only the settings from the group as a dict.
        :param upper: If upper is ``True``, the key will use upper-case
                      letters. Defaults to ``False``.
        """

        settings = {}
        result = None
        if from_group is not None:
            result = SettingsGroup.query.filter_by(key=from_group).\
                first_or_404()
            result = result.settings
        else:
            result = cls.query.all()

        for setting in result:
            if upper:
                setting_key = setting.key.upper()
            else:
                setting_key = setting.key

            settings[setting_key] = setting.value

        return settings

    @classmethod
    def invalidate_cache(cls):
        """Invalidates this objects cached metadata."""
        cache.delete_memoized(cls.as_dict, cls)

    def save(self):
        """Saves a setting"""
        db.session.add(self)
        db.session.commit()

    def delete(self):
        """Deletes a setting"""
        db.session.delete(self)
        db.session.commit()
<|endoftext|>


<commit_before>"""RCS interface module for CVSGit."""

import os.path

from cvsgit.changeset import Change, FILE_ADDED, FILE_MODIFIED, \
    FILE_DELETED

# Our 'rcsparse' module is actually a simple copy of the RCS parser
# code from cvs2svn.
from cvsgit import rcsparse

REV_TIMESTAMP = 0
REV_AUTHOR = 1
REV_STATE = 2
REV_BRANCHES = 3
REV_NEXT = 4
REV_MODE = 5

class RCSFile(rcsparse.Sink):
    "Represents a single RCS file."

    def __init__(self, filename, encoding='iso8859-1'):
        """'encoding' sets the encoding of log messages and delta text
        in RCS files."""
        self.filename = filename
        self.encoding = encoding
        self.parse()

    def parse(self):
        """Parse the RCS file to set the 'head', 'branch', 'revs' and
        'log' attributes."""

        f = file(os.path.join(self.filename), 'r')
        try:
            self.head = None
            self.branch = None
            self.revs = {}
            self.log = {}
            self.mode = ''              # XXX

            rcsparse.parse(f, self)
        finally:
            f.close()

    def changes(self):
        """Yield Change objects corresponding to all trunk revisions,
        generated by following the current head revision of the
        default branch back to its origin.  The order of changes is
        thus from most recent to oldest.
        """

        #print self.filename
        for revision in self.revision_trail(self.default_branch()):
            rev = self.revs[revision]
            #self.print_revision(revision)

            if revision == '1.1.1.1' or revision == '1.1':
                filestatus = FILE_ADDED
            elif rev[REV_STATE] == 'dead':
                filestatus = FILE_DELETED
            else:
                # XXX: Resurrections of dead revisions aren't flagged
                # as FILE_ADDED.
                filestatus = FILE_MODIFIED
            #print filestatus

            yield(Change(timestamp=rev[REV_TIMESTAMP],
                         author=rev[REV_AUTHOR],
                         log=self.log[revision],
                         filestatus=filestatus,
                         filename=self.filename,
                         revision=revision,
                         state=rev[REV_STATE],
                         mode=rev[REV_MODE]))

    def default_branch(self):
        """Return the default branch, normally either '1' or the
        vendor branch '1.1.1'."""

        if self.branch:
            # Usually the branch will be '1.1.1' in this case.
            assert(self.branch == '1.1.1')
            return self.branch
        else:
            # Under normal circumstances the branch will always be '1'.
            branch = self.head.rsplit('.', 1)[0]
            assert(branch == '1')
            return branch

    def revision_trail(self, branch):
        """Yield the complete history of a branch, beginning at the
        head revision of 'branch' and going backwards, tracing the
        previous revisions on that branch and the revision of the
        branch ancestors up to the origin."""

        while branch:
            for revision in self.branch_revisions(branch):
                if revision == '1.1' and '1.1.1.1' in \
                        self.revs[revision][REV_BRANCHES]:
                    # This revision seems to have been created by an
                    # import into the vendor branch.  The actual log
                    # message is that of the initial revision on the
                    # vendor branch.
                    yield('1.1.1.1')
                    return
                elif revision == '1.1.1.1':
                    # Since the initial vendor branch revision
                    # contains the real log message for the initial
                    # import, there is no need to trace the vendor
                    # branch back to the 1.1 revision.  (The initial
                    # import on the vendor branch in a way preceeds
                    # the creation of the initial revision on trunk,
                    # even though that isn't reflected by the revision
                    # numbering scheme.)
                    yield(revision)
                    return
                else:
                    yield(revision)

            # Continue tracing the history of the branch point.
            branch = self.branch_point(branch)

    def branch_revisions(self, branch):
        """Yield all revisions of the branch, from the highest
        revision number to the lowest."""

        count = self.branch_revision_count(branch)
        while count > 0:
            yield(branch + '.' + str(count))
            count -= 1

    def branch_revision_count(self, branch):
        """Return the number of revisions on the given branch."""

        count = 0
        for revision in self.revs.keys():
            rbranch, rnumber = revision.rsplit('.', 1)
            if rbranch == branch:
                count += 1
        return count

    def branch_point(self, branch):
        """Return the ancestor revision number of the given branch or
        None if the branch has no ancestor."""

        initial = branch + '.1'
        for revision in self.revs.keys():
            rev = self.revs[revision]
            for rbranch in rev[REV_BRANCHES]:
                if rbranch == initial:
                    return revision
        return None

    # XXX only for debugging; remove later
    def print_revision(self, revision):
        import time
        rev = self.revs[revision]
        print 'revision:', revision
        print '  timestamp:', time.strftime("%Y-%m-%d %H:%M", time.gmtime(rev[REV_TIMESTAMP]))
        print '  branches:', rev[REV_BRANCHES]
        print '  next:', rev[REV_NEXT]
        print '  log:', self.log[revision].splitlines()[0]

    # rcsparse.Sink methods:

    def set_head_revision(self, revision):
        self.head = revision

    def set_principal_branch(self, branch_name):
        self.branch = branch_name

    def set_expansion(self, mode):
        self.mode = mode

    def define_revision(self, revision, timestamp, author, state,
                        branches, next):
        self.revs[revision] = (timestamp, author, state, branches, next,
                               self.mode) # XXX

    def set_revision_info(self, revision, log, text):
        self.log[revision] = unicode(log, self.encoding)
<commit_msg>Fix an edge-case where 1.1 is created by an add and 1.1.1.1 is created by an import, later<commit_after>"""RCS interface module for CVSGit."""

import os.path

from cvsgit.changeset import Change, FILE_ADDED, FILE_MODIFIED, \
    FILE_DELETED

# Our 'rcsparse' module is actually a simple copy of the RCS parser
# code from cvs2svn.
from cvsgit import rcsparse

REV_TIMESTAMP = 0
REV_AUTHOR = 1
REV_STATE = 2
REV_BRANCHES = 3
REV_NEXT = 4
REV_MODE = 5

class RCSFile(rcsparse.Sink):
    "Represents a single RCS file."

    def __init__(self, filename, encoding='iso8859-1'):
        """'encoding' sets the encoding of log messages and delta text
        in RCS files."""
        self.filename = filename
        self.encoding = encoding
        self.parse()

    def parse(self):
        """Parse the RCS file to set the 'head', 'branch', 'revs' and
        'log' attributes."""

        f = file(os.path.join(self.filename), 'r')
        try:
            self.head = None
            self.branch = None
            self.revs = {}
            self.log = {}
            self.mode = ''              # XXX

            rcsparse.parse(f, self)
        finally:
            f.close()

    def changes(self):
        """Yield Change objects corresponding to all trunk revisions,
        generated by following the current head revision of the
        default branch back to its origin.  The order of changes is
        thus from most recent to oldest.
        """

        #print self.filename
        for revision in self.revision_trail(self.default_branch()):
            rev = self.revs[revision]
            #self.print_revision(revision)

            if revision == '1.1.1.1' or revision == '1.1':
                filestatus = FILE_ADDED
            elif rev[REV_STATE] == 'dead':
                filestatus = FILE_DELETED
            else:
                # XXX: Resurrections of dead revisions aren't flagged
                # as FILE_ADDED.
                filestatus = FILE_MODIFIED
            #print filestatus

            yield(Change(timestamp=rev[REV_TIMESTAMP],
                         author=rev[REV_AUTHOR],
                         log=self.log[revision],
                         filestatus=filestatus,
                         filename=self.filename,
                         revision=revision,
                         state=rev[REV_STATE],
                         mode=rev[REV_MODE]))

    def default_branch(self):
        """Return the default branch, normally either '1' or the
        vendor branch '1.1.1'."""

        if self.branch:
            # Usually the branch will be '1.1.1' in this case.
            assert(self.branch == '1.1.1')
            return self.branch
        else:
            # Under normal circumstances the branch will always be '1'.
            branch = self.head.rsplit('.', 1)[0]
            assert(branch == '1')
            return branch

    def revision_trail(self, branch):
        """Yield the complete history of a branch, beginning at the
        head revision of 'branch' and going backwards, tracing the
        previous revisions on that branch and the revision of the
        branch ancestors up to the origin."""

        while branch:
            for revision in self.branch_revisions(branch):
                if revision == '1.1' and '1.1.1.1' in \
                        self.revs[revision][REV_BRANCHES] and \
                        self.log[revision] == 'Initial revision':
                    # TODO: Time stamps of revisions 1.1 and 1.1.1.1
                    # revision should also match, so verify?
                    # This revision seems to have been created by an
                    # import into the vendor branch.  The actual log
                    # message is that of the initial revision on the
                    # vendor branch.
                    yield('1.1.1.1')
                    return
                elif revision == '1.1.1.1':
                    # Since the initial vendor branch revision
                    # contains the real log message for the initial
                    # import, there is no need to trace the vendor
                    # branch back to the 1.1 revision.
                    yield(revision)
                    return
                else:
                    yield(revision)

            # Continue tracing the history of the branch point.
            branch = self.branch_point(branch)

    def branch_revisions(self, branch):
        """Yield all revisions of the branch, from the highest
        revision number to the lowest."""

        count = self.branch_revision_count(branch)
        while count > 0:
            yield(branch + '.' + str(count))
            count -= 1

    def branch_revision_count(self, branch):
        """Return the number of revisions on the given branch."""

        count = 0
        for revision in self.revs.keys():
            rbranch, rnumber = revision.rsplit('.', 1)
            if rbranch == branch:
                count += 1
        return count

    def branch_point(self, branch):
        """Return the ancestor revision number of the given branch or
        None if the branch has no ancestor."""

        initial = branch + '.1'
        for revision in self.revs.keys():
            rev = self.revs[revision]
            for rbranch in rev[REV_BRANCHES]:
                if rbranch == initial:
                    return revision
        return None

    # XXX only for debugging; remove later
    def print_revision(self, revision):
        import time
        rev = self.revs[revision]
        print 'revision:', revision
        print '  timestamp:', time.strftime("%Y-%m-%d %H:%M", time.gmtime(rev[REV_TIMESTAMP]))
        print '  branches:', rev[REV_BRANCHES]
        print '  next:', rev[REV_NEXT]
        print '  log:', self.log[revision].splitlines()[0]

    # rcsparse.Sink methods:

    def set_head_revision(self, revision):
        self.head = revision

    def set_principal_branch(self, branch_name):
        self.branch = branch_name

    def set_expansion(self, mode):
        self.mode = mode

    def define_revision(self, revision, timestamp, author, state,
                        branches, next):
        self.revs[revision] = (timestamp, author, state, branches, next,
                               self.mode) # XXX

    def set_revision_info(self, revision, log, text):
        self.log[revision] = unicode(log, self.encoding)
<|endoftext|>