<commit_before>import pandas as pd

__version__ = '0.1.1'

COLUMN_NAMES = ('chrom', 'first_bp_intron', 'last_bp_intron', 'strand',
                'intron_motif', 'annotated',
                'unique_junction_reads', 'multimap_junction_reads',
                'max_overhang')

def int_to_intron_motif(n):
    if n == 0:
        return 'non-canonical'
    if n == 1:
        return 'GT/AG'
    if n == 2:
        return 'CT/AC'
    if n == 3:
        return 'GC/AG'
    if n == 4:
        return 'CT/GC'
    if n == 5:
        return 'AT/AC'
    if n == 6:
        return 'GT/AT'

def read_sj_out_tab(filename):
    """Read an SJ.out.tab file as produced by the RNA-STAR aligner into a
    pandas Dataframe

    Parameters
    ----------
    filename : str of filename or file handle
        Filename of the SJ.out.tab file you want to read in

    Returns
    -------
    sj : pandas.DataFrame
        Dataframe of splice junctions with the columns,
        ('chrom', 'first_bp_intron', 'last_bp_intron', 'strand',
        'intron_motif', 'annotated', 'unique_junction_reads',
        'multimap_junction_reads', 'max_overhang')

    """
    sj = pd.read_table(filename, header=None, names=COLUMN_NAMES, sep='\s+')
    sj.intron_motif = sj.intron_motif.map(int_to_intron_motif)
    sj.annotated = sj.annotated.astype(bool)
    return sj

def chr_start_stop_to_sj_ind(chr_start_stop, sj):
    """Transform a 'chr1:100-200' string into index range of sj dataframe

    Parameters
    ----------
    chr_start_stop : str
        Genome location string of the format chr:start-stop
    sj : pandas.DataFrame
        Dataframe of splice junctions as created by read_sj_out_tab

    Returns
    -------
    ind : pandas.Series (bool)
        Boolean series which can be used to index the sj

    """
    chrom, startstop = chr_start_stop.replace(',', '').split(':')
    start, stop = map(int, startstop.split('-'))
    return (sj.chrom == chrom) & (start < sj.first_bp_intron) \
        & (sj.last_bp_intron < stop)

def get_psis(sj, min_unique=5, min_multimap=10):
    """Calculate Percent spliced-in (Psi) scores of each junction

    As described in Pervouchine et al, Bioinformatics (2013)
    [doi: 10.1093/bioinformatics/bts678], we will take the approach of asking,
    how often is this donor site (5' splice site) used with this acceptor
    site (3' splice site), compared to ALL OTHER acceptors?

    Same goes for acceptor sites. How often is this acceptor site, used with
    this donor site, compared to ALL OTHER donors?

    To illustrate, check out this example. Each "-" represents 10 bp

    Splice junction fig     genome location     number of reads
    [  ]--------[    ]        chr1:100-180        90
    [  ]----------[  ]        chr1:100-200        10
    [     ]-------[  ]        chr1:130-200        40

    For the 5' splice site chr1:100, we have 90+10 = 100 total reads. Thus the
    "psi5" for chr1:100-180 is 90/100 = 0.9, and 0.1 for chr:100-200.

    For the 3' splice site chr1:200, we have 10+40 = 50 total reads. Thus the
    "psi3" for chr1:100-200 is 10/50 = 0.2, and 0.8 for chr:130-200.

    What's left is the uninteresting splice sites of chr1:180 and chr1:130,
    both of which didn't have any variance and were always used. Thus psi3
    for chr1:180 is 1.0, and psi5 for chr1:130 is 1.0 as well.

    Parameters
    ----------
    sj : pandas.DataFrame
        A splice junction dataframe as created by read_sj_out_tab, specifically
        with the columns,
        ('chrom', 'first_bp_intron', 'last_bp_intron', 'strand',
        'intron_motif', 'annotated', 'unique_junction_reads',
        'multimap_junction_reads', 'max_overhang')
    min_unique : int, optional
        Minimum number of unique reads per junction. Default 5.
    min_multimap : int, optional
        Minimum number of multimapping reads per junction. Default 10

    Returns
    -------
    sj_with_psi : pandas.DataFrame
        The original dataframe, now with the columns psi5 and psi3 for
        percent spliced-in scores of each junction.

    >>> import pandas as pd
    >>> data = {'chrom': ['chr1', 'chr1', 'chr1'],
    ... 'first_bp_intron':[100, 100, 130], 'last_bp_intron':[100, 200, 200],
    ... 'unique_junction_reads':[90, 10, 40],
    ... 'multimap_junction_reads':[0, 0, 0]}
    >>> sj = pd.DataFrame(data)
    >>> get_psis(sj) # doctest: +NORMALIZE_WHITESPACE
      chrom  first_bp_intron  last_bp_intron  multimap_junction_reads  \\
    0  chr1              100             100                        0
    1  chr1              100             200                        0
    2  chr1              130             200                        0
    <BLANKLINE>
       unique_junction_reads  multimap_junction_reads_filtered  \\
    0                     90                                 0
    1                     10                                 0
    2                     40                                 0
    <BLANKLINE>
       unique_junction_reads_filtered  total_filtered_reads  psi5  psi3
    0                              90                    90   0.9   1.0
    1                              10                    10   0.1   0.2
    2                              40                    40   1.0   0.8
    <BLANKLINE>
    [3 rows x 10 columns]
    """
    sj['multimap_junction_reads_filtered'] = sj.multimap_junction_reads[
        sj.multimap_junction_reads >= min_multimap]
    sj['unique_junction_reads_filtered'] = sj.unique_junction_reads[
        sj.unique_junction_reads >= min_unique]
    sj['total_filtered_reads'] = sj.multimap_junction_reads_filtered.add(
        sj.unique_junction_reads_filtered)
    sj.total_filtered_reads = sj.total_filtered_reads.astype('float')

    # Calculate psi scores as in Pervouchine et al, Bioinformatics (2013)
    # doi: 10.1093/bioinformatics/bts678
    psi5_groupby = ['chrom', 'first_bp_intron']
    psi3_groupby = ['chrom', 'last_bp_intron']

    groupbys = {'psi5': psi5_groupby, 'psi3': psi3_groupby}
    for name, groupby in groupbys.items():
        denominator = '{}_denominator'.format(name)
        s = sj.groupby(groupby).total_filtered_reads.sum()
        s.name = denominator
        sj.set_index(groupby, inplace=True, drop=False)
        sj = sj.join(s)
        sj[name] = sj.total_filtered_reads / sj[denominator]
        sj.reset_index(inplace=True, drop=True)

    return sj
<commit_msg>Specify index of format replacer for python 2.6 compatibility<commit_after>import pandas as pd

__version__ = '0.1.1'

COLUMN_NAMES = ('chrom', 'first_bp_intron', 'last_bp_intron', 'strand',
                'intron_motif', 'annotated',
                'unique_junction_reads', 'multimap_junction_reads',
                'max_overhang')

def int_to_intron_motif(n):
    if n == 0:
        return 'non-canonical'
    if n == 1:
        return 'GT/AG'
    if n == 2:
        return 'CT/AC'
    if n == 3:
        return 'GC/AG'
    if n == 4:
        return 'CT/GC'
    if n == 5:
        return 'AT/AC'
    if n == 6:
        return 'GT/AT'

def read_sj_out_tab(filename):
    """Read an SJ.out.tab file as produced by the RNA-STAR aligner into a
    pandas Dataframe

    Parameters
    ----------
    filename : str of filename or file handle
        Filename of the SJ.out.tab file you want to read in

    Returns
    -------
    sj : pandas.DataFrame
        Dataframe of splice junctions with the columns,
        ('chrom', 'first_bp_intron', 'last_bp_intron', 'strand',
        'intron_motif', 'annotated', 'unique_junction_reads',
        'multimap_junction_reads', 'max_overhang')

    """
    sj = pd.read_table(filename, header=None, names=COLUMN_NAMES, sep='\s+')
    sj.intron_motif = sj.intron_motif.map(int_to_intron_motif)
    sj.annotated = sj.annotated.astype(bool)
    return sj

def chr_start_stop_to_sj_ind(chr_start_stop, sj):
    """Transform a 'chr1:100-200' string into index range of sj dataframe

    Parameters
    ----------
    chr_start_stop : str
        Genome location string of the format chr:start-stop
    sj : pandas.DataFrame
        Dataframe of splice junctions as created by read_sj_out_tab

    Returns
    -------
    ind : pandas.Series (bool)
        Boolean series which can be used to index the sj

    """
    chrom, startstop = chr_start_stop.replace(',', '').split(':')
    start, stop = map(int, startstop.split('-'))
    return (sj.chrom == chrom) & (start < sj.first_bp_intron) \
        & (sj.last_bp_intron < stop)

def get_psis(sj, min_unique=5, min_multimap=10):
    """Calculate Percent spliced-in (Psi) scores of each junction

    As described in Pervouchine et al, Bioinformatics (2013)
    [doi: 10.1093/bioinformatics/bts678], we will take the approach of asking,
    how often is this donor site (5' splice site) used with this acceptor
    site (3' splice site), compared to ALL OTHER acceptors?

    Same goes for acceptor sites. How often is this acceptor site, used with
    this donor site, compared to ALL OTHER donors?

    To illustrate, check out this example. Each "-" represents 10 bp

    Splice junction fig     genome location     number of reads
    [  ]--------[    ]        chr1:100-180        90
    [  ]----------[  ]        chr1:100-200        10
    [     ]-------[  ]        chr1:130-200        40

    For the 5' splice site chr1:100, we have 90+10 = 100 total reads. Thus the
    "psi5" for chr1:100-180 is 90/100 = 0.9, and 0.1 for chr:100-200.

    For the 3' splice site chr1:200, we have 10+40 = 50 total reads. Thus the
    "psi3" for chr1:100-200 is 10/50 = 0.2, and 0.8 for chr:130-200.

    What's left is the uninteresting splice sites of chr1:180 and chr1:130,
    both of which didn't have any variance and were always used. Thus psi3
    for chr1:180 is 1.0, and psi5 for chr1:130 is 1.0 as well.

    Parameters
    ----------
    sj : pandas.DataFrame
        A splice junction dataframe as created by read_sj_out_tab, specifically
        with the columns,
        ('chrom', 'first_bp_intron', 'last_bp_intron', 'strand',
        'intron_motif', 'annotated', 'unique_junction_reads',
        'multimap_junction_reads', 'max_overhang')
    min_unique : int, optional
        Minimum number of unique reads per junction. Default 5.
    min_multimap : int, optional
        Minimum number of multimapping reads per junction. Default 10

    Returns
    -------
    sj_with_psi : pandas.DataFrame
        The original dataframe, now with the columns psi5 and psi3 for
        percent spliced-in scores of each junction.

    >>> import pandas as pd
    >>> data = {'chrom': ['chr1', 'chr1', 'chr1'],
    ... 'first_bp_intron':[100, 100, 130], 'last_bp_intron':[100, 200, 200],
    ... 'unique_junction_reads':[90, 10, 40],
    ... 'multimap_junction_reads':[0, 0, 0]}
    >>> sj = pd.DataFrame(data)
    >>> get_psis(sj) # doctest: +NORMALIZE_WHITESPACE
      chrom  first_bp_intron  last_bp_intron  multimap_junction_reads  \\
    0  chr1              100             100                        0
    1  chr1              100             200                        0
    2  chr1              130             200                        0
    <BLANKLINE>
       unique_junction_reads  multimap_junction_reads_filtered  \\
    0                     90                                 0
    1                     10                                 0
    2                     40                                 0
    <BLANKLINE>
       unique_junction_reads_filtered  total_filtered_reads  psi5  psi3
    0                              90                    90   0.9   1.0
    1                              10                    10   0.1   0.2
    2                              40                    40   1.0   0.8
    <BLANKLINE>
    [3 rows x 10 columns]
    """
    sj['multimap_junction_reads_filtered'] = sj.multimap_junction_reads[
        sj.multimap_junction_reads >= min_multimap]
    sj['unique_junction_reads_filtered'] = sj.unique_junction_reads[
        sj.unique_junction_reads >= min_unique]
    sj['total_filtered_reads'] = sj.multimap_junction_reads_filtered.add(
        sj.unique_junction_reads_filtered)
    sj.total_filtered_reads = sj.total_filtered_reads.astype('float')

    # Calculate psi scores as in Pervouchine et al, Bioinformatics (2013)
    # doi: 10.1093/bioinformatics/bts678
    psi5_groupby = ['chrom', 'first_bp_intron']
    psi3_groupby = ['chrom', 'last_bp_intron']

    groupbys = {'psi5': psi5_groupby, 'psi3': psi3_groupby}
    for name, groupby in groupbys.items():
        denominator = '{0}_denominator'.format(name)
        s = sj.groupby(groupby).total_filtered_reads.sum()
        s.name = denominator
        sj.set_index(groupby, inplace=True, drop=False)
        sj = sj.join(s)
        sj[name] = sj.total_filtered_reads / sj[denominator]
        sj.reset_index(inplace=True, drop=True)

    return sj
<|endoftext|>


<commit_before># Feeping Creaturism:
#
# this is the all important version number used by pip.
#
#
"""
Version History:

1.0.0    - 8/2/2016 initial birth as a pip package.

1.0.1    - 8/2/2016 the rst gets a make over, and doc strings
           became more pervasive.

1.0.2    - 8/4/2016 dragonegg issues. trying to include a polite 'as' wrapper
           (i.e. not a  console_script called as).

1.0.3    - 8/4/2016 travis build fixes.

1.0.4    - 8/4/2016 travis build fixes, and exception handling fixes.

1.0.5    - 8/4/2016 exit value was upsetting travis.

1.0.6    - 8/9/2016 exit codes preserved; important for configure scripts like musl libc.

1.0.7    - 8/9/2016 logical restructuring; smaller bites.

1.0.8    - 8/9/2016 test and doc subdirectories are no longer included.

1.0.9    - 8/25/2016  Python 3.0 import fixes (Will Dietz)

1.0.10   - 9/26/2016  Apple's otool just gets biggier and buggier.

1.0.11   - 9/27/2016  Improved Apple's otool fix.

1.0.12   - 10/27/2016 Common flag support.

1.0.13   - 11/05/2016  pylint spots a few mistakes.

1.0.14   - 11/10/2016  --coverage flag.

1.0.15   - 11/15/2016  pylintification complete.

1.0.16   - 11/16/2016  ooops musl points out I screwed up the exit codes AGAIN.

1.0.17   - 11/23/2016  delcypher  #16  over at travitch's place.

1.0.18   - 4/11/2017  tentative solution to the -emit-llvm "out of context" experience.

1.0.19   - 4/19/2017  fixed a '-o' issue in extract-bc and added the bitcode store feature.

1.1.0    - 4/21/2017  no new features on the horizon, no new bugs?

1.1.1    - 4/25/2017 bugs introduced by the new fetures have hopefully been eradicated.

1.1.2    - 4/26/2017 encoding issues with hashlib in the python 3 swarm.

1.1.3    - 5/20/2017 fortran support via flang (pull #60 over at travitch's place)

1.1.4    - 7/24/2017 improvements motivated by gllvm and logic.

1.1.5    - 3/14/2018 fixes suggested by Alexander Bakst

1.1.6    - 4/24/2018 fixes suggested by building the Linux kernel and trying to harmonize with gllvm.
           4/28/2018 can sort manifest via the -s switch.
           5/1/2018 handle archives correctly (deal with multiple files with the same name in the archive).

"""

wllvm_version = '1.1.6'
wllvm_date = 'May 1st 2018'
<commit_msg>Comments...<commit_after># Feeping Creaturism:
#
# this is the all important version number used by pip.
#
#
"""
Version History:

1.0.0    - 8/2/2016 initial birth as a pip package.

1.0.1    - 8/2/2016 the rst gets a make over, and doc strings
           became more pervasive.

1.0.2    - 8/4/2016 dragonegg issues. trying to include a polite 'as' wrapper
           (i.e. not a  console_script called as).

1.0.3    - 8/4/2016 travis build fixes.

1.0.4    - 8/4/2016 travis build fixes, and exception handling fixes.

1.0.5    - 8/4/2016 exit value was upsetting travis.

1.0.6    - 8/9/2016 exit codes preserved; important for configure scripts like musl libc.

1.0.7    - 8/9/2016 logical restructuring; smaller bites.

1.0.8    - 8/9/2016 test and doc subdirectories are no longer included.

1.0.9    - 8/25/2016  Python 3.0 import fixes (Will Dietz)

1.0.10   - 9/26/2016  Apple's otool just gets biggier and buggier.

1.0.11   - 9/27/2016  Improved Apple's otool fix.

1.0.12   - 10/27/2016 Common flag support.

1.0.13   - 11/05/2016  pylint spots a few mistakes.

1.0.14   - 11/10/2016  --coverage flag.

1.0.15   - 11/15/2016  pylintification complete.

1.0.16   - 11/16/2016  ooops musl points out I screwed up the exit codes AGAIN.

1.0.17   - 11/23/2016  delcypher  #16  over at travitch's place.

1.0.18   - 4/11/2017  tentative solution to the -emit-llvm "out of context" experience.

1.0.19   - 4/19/2017  fixed a '-o' issue in extract-bc and added the bitcode store feature.

1.1.0    - 4/21/2017  no new features on the horizon, no new bugs?

1.1.1    - 4/25/2017 bugs introduced by the new fetures have hopefully been eradicated.

1.1.2    - 4/26/2017 encoding issues with hashlib in the python 3 swarm.

1.1.3    - 5/20/2017 fortran support via flang (pull #60 over at travitch's place)

1.1.4    - 7/24/2017 improvements motivated by gllvm and logic.

1.1.5    - 3/14/2018 fixes suggested by Alexander Bakst

1.1.6    - 4/24/2018 fixes suggested by building the Linux kernel and trying to harmonize with gllvm.
           4/28/2018 can handle thin archives, can sort manifest via the -s switch.
           5/1/2018 can handle archives correctly (deal with multiple files with the same name in the archive).

"""

wllvm_version = '1.1.6'
wllvm_date = 'May 1st 2018'
<|endoftext|>


<commit_before>#  Copyright 2015 Google Inc. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import collections
import random

import numpy as np
import tensorflow as tf

from sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin
from sklearn.utils import check_array

from skflow.trainer import TensorFlowTrainer
from skflow import models, data_feeder

class TensorFlowEstimator(BaseEstimator):
    """Base class for all TensorFlow estimators.

    Parameters:
        model_fn: Model function, that takes input X, y tensors and outputs
                  prediction and loss tensors.
        n_classes: Number of classes in the target.
        tf_master: TensorFlow master. Empty string is default for local.
        batch_size: Mini batch size.
        steps: Number of steps to run over data.
        optimizer: Optimizer name (or class), for example "SGD", "Adam",
                   "Adagrad".
        learning_rate: Learning rate for optimizer.
        tf_random_seed: Random seed for TensorFlow initializers.
            Setting this value, allows consistency between reruns.
    """

    def __init__(self, model_fn, n_classes, tf_master="", batch_size=32, steps=50, optimizer="SGD",
                 learning_rate=0.1, tf_random_seed=42):
        self.n_classes = n_classes
        self.tf_master = tf_master
        self.batch_size = batch_size
        self.steps = steps
        self.optimizer = optimizer
        self.learning_rate = learning_rate
        self.tf_random_seed = tf_random_seed
        self.model_fn = model_fn

    def fit(self, X, y):
        with tf.Graph().as_default() as graph:
            tf.set_random_seed(self.tf_random_seed)
            self._global_step = tf.Variable(0, name="global_step", trainable=False)

            # Create data feeder, to sample inputs from dataset.
            # If X and y are iterators, use StreamingDataFeeder.
            if hasattr(X, 'next'):
                assert hasattr(y, 'next')
                self._data_feeder = data_feeder.StreamingDataFeeder(X, y,
                    self.n_classes, self.batch_size)
            else:
                self._data_feeder = data_feeder.DataFeeder(X, y,
                    self.n_classes, self.batch_size)

            # Setting up input and output placeholders.
            input_shape = [None] + self._data_feeder.input_shape[1:]
            output_shape = [None] + self._data_feeder.output_shape[1:]
            self._inp = tf.placeholder(tf.float32, input_shape,
                                       name="input")
            self._out = tf.placeholder(tf.float32,  output_shape,
                name="output")

            # Create model's graph.
            self._model_predictions, self._model_loss = self.model_fn(self._inp, self._out)

            # Create trainer and augment graph with gradients and optimizer.
            self._trainer = TensorFlowTrainer(self._model_loss,
                self._global_step, self.optimizer, self.learning_rate)
            self._session = tf.Session(self.tf_master)

            # Initialize and train model.
            self._trainer.initialize(self._session)
            self._trainer.train(self._session,
                                self._data_feeder.get_feed_dict_fn(self._inp,
                                                             self._out), self.steps)

    def predict(self, X):
        pred = self._session.run(self._model_predictions,
                                 feed_dict={
                                     self._inp.name: X
                                 })
        if self.n_classes < 2:
            return pred
        return pred.argmax(axis=1)

    def predict_proba(self, X):
        pred = self._session.run(self._model.predictions,
                                 feed_dict={
                                     self._model.inp.name: X
                                 })
        if self.n_classes < 2:
            return pred
        return pred

class TensorFlowLinearRegressor(TensorFlowEstimator, RegressorMixin):
    """TensorFlow Linear Regression model."""

    def __init__(self, n_classes=0, tf_master="", batch_size=32, steps=50, optimizer="SGD",
                 learning_rate=0.1, tf_random_seed=42):
        super(TensorFlowLinearRegressor, self).__init__(
            model_fn=models.linear_regression, n_classes=n_classes,
            tf_master=tf_master,
            batch_size=batch_size, steps=steps, optimizer=optimizer,
            learning_rate=learning_rate, tf_random_seed=tf_random_seed)

class TensorFlowLinearClassifier(TensorFlowEstimator, ClassifierMixin):
    """TensorFlow Linear Classifier model."""

    def __init__(self, n_classes, tf_master="", batch_size=32, steps=50, optimizer="SGD",
                 learning_rate=0.1, tf_random_seed=42):
        super(TensorFlowLinearClassifier, self).__init__(
            model_fn=models.logistic_regression, n_classes=n_classes,
            tf_master=tf_master,
            batch_size=batch_size, steps=steps, optimizer=optimizer,
            learning_rate=learning_rate, tf_random_seed=tf_random_seed)

TensorFlowRegressor = TensorFlowLinearRegressor
TensorFlowClassifier = TensorFlowLinearClassifier

class TensorFlowDNNClassifier(TensorFlowEstimator, ClassifierMixin):
    """TensorFlow DNN Classifier model.

    Parameters:
        hidden_units: List of hidden units per layer.
        n_classes: Number of classes in the target.
        tf_master: TensorFlow master. Empty string is default for local.
        batch_size: Mini batch size.
        steps: Number of steps to run over data.
        optimizer: Optimizer name (or class), for example "SGD", "Adam",
                   "Adagrad".
        learning_rate: Learning rate for optimizer.
        tf_random_seed: Random seed for TensorFlow initializers.
            Setting this value, allows consistency between reruns.
    """

    def __init__(self, hidden_units, n_classes, tf_master="", batch_size=32,
                 steps=50, optimizer="SGD", learning_rate=0.1, tf_random_seed=42):
        model_fn = models.get_dnn_model(hidden_units,
                                        models.logistic_regression)
        super(TensorFlowDNNClassifier, self).__init__(
            model_fn=model_fn,
            n_classes=n_classes, tf_master=tf_master,
            batch_size=batch_size, steps=steps, optimizer=optimizer,
            learning_rate=learning_rate, tf_random_seed=tf_random_seed)

class TensorFlowDNNRegressor(TensorFlowEstimator, ClassifierMixin):
    """TensorFlow DNN Regressor model.

    Parameters:
        hidden_units: List of hidden units per layer.
        tf_master: TensorFlow master. Empty string is default for local.
        batch_size: Mini batch size.
        steps: Number of steps to run over data.
        optimizer: Optimizer name (or class), for example "SGD", "Adam",
                   "Adagrad".
        learning_rate: Learning rate for optimizer.
        tf_random_seed: Random seed for TensorFlow initializers.
            Setting this value, allows consistency between reruns.
    """

    def __init__(self, hidden_units, n_classes=0, tf_master="", batch_size=32,
                 steps=50, optimizer="SGD", learning_rate=0.1, tf_random_seed=42):
        model_fn = models.get_dnn_model(hidden_units,
                                        models.linear_regression)
        super(TensorFlowDNNRegressor, self).__init__(
            model_fn=model_fn,
            n_classes=n_classes, tf_master=tf_master,
            batch_size=batch_size, steps=steps, optimizer=optimizer,
            learning_rate=learning_rate, tf_random_seed=tf_random_seed)

<commit_msg>Moved shared code from predict and predict_proba to _predict. Added comments to fit/predict/predict_proba<commit_after>#  Copyright 2015 Google Inc. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import collections
import random

import numpy as np
import tensorflow as tf

from sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin
from sklearn.utils import check_array

from skflow.trainer import TensorFlowTrainer
from skflow import models, data_feeder

class TensorFlowEstimator(BaseEstimator):
    """Base class for all TensorFlow estimators.

    Parameters:
        model_fn: Model function, that takes input X, y tensors and outputs
                  prediction and loss tensors.
        n_classes: Number of classes in the target.
        tf_master: TensorFlow master. Empty string is default for local.
        batch_size: Mini batch size.
        steps: Number of steps to run over data.
        optimizer: Optimizer name (or class), for example "SGD", "Adam",
                   "Adagrad".
        learning_rate: Learning rate for optimizer.
        tf_random_seed: Random seed for TensorFlow initializers.
            Setting this value, allows consistency between reruns.
    """

    def __init__(self, model_fn, n_classes, tf_master="", batch_size=32, steps=50, optimizer="SGD",
                 learning_rate=0.1, tf_random_seed=42):
        self.n_classes = n_classes
        self.tf_master = tf_master
        self.batch_size = batch_size
        self.steps = steps
        self.optimizer = optimizer
        self.learning_rate = learning_rate
        self.tf_random_seed = tf_random_seed
        self.model_fn = model_fn

    def fit(self, X, y):
        """Builds a neural network model given provided `model_fn` and training
        data X and y.

        Args:
            X: matrix or tensor of shape [n_samples, n_features...]. Can be
            iterator that returns arrays of features. The training input
            samples for fitting the model.
            y: vector or matrix [n_samples] or [n_samples, n_outputs]. Can be
            iterator that returns array of targets. The training target values
            (class labels in classification, real numbers in regression).

        Returns:
            Returns self.
        """
        with tf.Graph().as_default() as graph:
            tf.set_random_seed(self.tf_random_seed)
            self._global_step = tf.Variable(0, name="global_step", trainable=False)

            # Create data feeder, to sample inputs from dataset.
            # If X and y are iterators, use StreamingDataFeeder.
            if hasattr(X, 'next'):
                assert hasattr(y, 'next')
                self._data_feeder = data_feeder.StreamingDataFeeder(X, y,
                    self.n_classes, self.batch_size)
            else:
                self._data_feeder = data_feeder.DataFeeder(X, y,
                    self.n_classes, self.batch_size)

            # Setting up input and output placeholders.
            input_shape = [None] + self._data_feeder.input_shape[1:]
            output_shape = [None] + self._data_feeder.output_shape[1:]
            self._inp = tf.placeholder(tf.float32, input_shape,
                                       name="input")
            self._out = tf.placeholder(tf.float32,  output_shape,
                name="output")

            # Create model's graph.
            self._model_predictions, self._model_loss = self.model_fn(self._inp, self._out)

            # Create trainer and augment graph with gradients and optimizer.
            self._trainer = TensorFlowTrainer(self._model_loss,
                self._global_step, self.optimizer, self.learning_rate)
            self._session = tf.Session(self.tf_master)

            # Initialize and train model.
            self._trainer.initialize(self._session)
            self._trainer.train(self._session,
                                self._data_feeder.get_feed_dict_fn(self._inp,
                                                             self._out),
                                                             self.steps)
            return self

    def _predict(self, X):
        pred = self._session.run(self._model_predictions,
                                 feed_dict={
                                     self._inp.name: X
                                 })
        return pred

    def predict(self, X):
        """Predict class or regression for X.

        For a classification model, the predicted class for each sample in X is
        returned. For a regression model, the predicted value based on X is
        returned.

        Args:
            X: array-like matrix, [n_samples, n_features...] or iterator.

        Returns:
            y: array of shape [n_samples]. The predicted classes or predicted
            value.
        """
        pred = self._predict(X)
        if self.n_classes < 2:
            return pred
        return pred.argmax(axis=1)

    def predict_proba(self, X):
        """Predict class probability of the input samples X.

        Args:
            X: array-like matrix, [n_samples, n_features...] or iterator.

        Returns:
            y: array of shape [n_samples, n_classes]. The predicted
            probabilities for each class.
        """
        return self._predict(X)

class TensorFlowLinearRegressor(TensorFlowEstimator, RegressorMixin):
    """TensorFlow Linear Regression model."""

    def __init__(self, n_classes=0, tf_master="", batch_size=32, steps=50, optimizer="SGD",
                 learning_rate=0.1, tf_random_seed=42):
        super(TensorFlowLinearRegressor, self).__init__(
            model_fn=models.linear_regression, n_classes=n_classes,
            tf_master=tf_master,
            batch_size=batch_size, steps=steps, optimizer=optimizer,
            learning_rate=learning_rate, tf_random_seed=tf_random_seed)

class TensorFlowLinearClassifier(TensorFlowEstimator, ClassifierMixin):
    """TensorFlow Linear Classifier model."""

    def __init__(self, n_classes, tf_master="", batch_size=32, steps=50, optimizer="SGD",
                 learning_rate=0.1, tf_random_seed=42):
        super(TensorFlowLinearClassifier, self).__init__(
            model_fn=models.logistic_regression, n_classes=n_classes,
            tf_master=tf_master,
            batch_size=batch_size, steps=steps, optimizer=optimizer,
            learning_rate=learning_rate, tf_random_seed=tf_random_seed)

TensorFlowRegressor = TensorFlowLinearRegressor
TensorFlowClassifier = TensorFlowLinearClassifier

class TensorFlowDNNClassifier(TensorFlowEstimator, ClassifierMixin):
    """TensorFlow DNN Classifier model.

    Parameters:
        hidden_units: List of hidden units per layer.
        n_classes: Number of classes in the target.
        tf_master: TensorFlow master. Empty string is default for local.
        batch_size: Mini batch size.
        steps: Number of steps to run over data.
        optimizer: Optimizer name (or class), for example "SGD", "Adam",
                   "Adagrad".
        learning_rate: Learning rate for optimizer.
        tf_random_seed: Random seed for TensorFlow initializers.
            Setting this value, allows consistency between reruns.
    """

    def __init__(self, hidden_units, n_classes, tf_master="", batch_size=32,
                 steps=50, optimizer="SGD", learning_rate=0.1, tf_random_seed=42):
        model_fn = models.get_dnn_model(hidden_units,
                                        models.logistic_regression)
        super(TensorFlowDNNClassifier, self).__init__(
            model_fn=model_fn,
            n_classes=n_classes, tf_master=tf_master,
            batch_size=batch_size, steps=steps, optimizer=optimizer,
            learning_rate=learning_rate, tf_random_seed=tf_random_seed)

class TensorFlowDNNRegressor(TensorFlowEstimator, ClassifierMixin):
    """TensorFlow DNN Regressor model.

    Parameters:
        hidden_units: List of hidden units per layer.
        tf_master: TensorFlow master. Empty string is default for local.
        batch_size: Mini batch size.
        steps: Number of steps to run over data.
        optimizer: Optimizer name (or class), for example "SGD", "Adam",
                   "Adagrad".
        learning_rate: Learning rate for optimizer.
        tf_random_seed: Random seed for TensorFlow initializers.
            Setting this value, allows consistency between reruns.
    """

    def __init__(self, hidden_units, n_classes=0, tf_master="", batch_size=32,
                 steps=50, optimizer="SGD", learning_rate=0.1, tf_random_seed=42):
        model_fn = models.get_dnn_model(hidden_units,
                                        models.linear_regression)
        super(TensorFlowDNNRegressor, self).__init__(
            model_fn=model_fn,
            n_classes=n_classes, tf_master=tf_master,
            batch_size=batch_size, steps=steps, optimizer=optimizer,
            learning_rate=learning_rate, tf_random_seed=tf_random_seed)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Wordpress API Class
"""

__title__ = "wordpress-api"

from requests import request
from json import dumps as jsonencode
from wordpress.oauth import OAuth, OAuth_3Leg
from wordpress.transport import API_Requests_Wrapper
from wordpress.helpers import UrlUtils
from requests_oauth2 import OAuth2

class API(object):
    """ API Class """

    token = ''
    oauth_version = 1
    oauth2 = False

    def __init__(self, url, consumer_key, consumer_secret, oauth_version=1, token='', **kwargs):

        self.requester = API_Requests_Wrapper(url=url, **kwargs)

        oauth_kwargs = dict(
            requester=self.requester,
            consumer_key=consumer_key,
            consumer_secret=consumer_secret,
            force_nonce=kwargs.get('force_nonce'),
            force_timestamp=kwargs.get('force_timestamp')
        )

        if oauth_version is 2 and token:
            self.token = token
        elif oauth_version is 2 and not token:
            self.oauth2 = OAuth2(consumer_key, consumer_secret, url, "")
            print self.oauth2.authorize_url()

        elif kwargs.get('oauth1a_3leg'):
            self.oauth1a_3leg = kwargs['oauth1a_3leg']
            oauth_kwargs['callback'] = kwargs['callback']
            oauth_kwargs['wp_user'] = kwargs['wp_user']
            oauth_kwargs['wp_pass'] = kwargs['wp_pass']
            self.oauth = OAuth_3Leg( **oauth_kwargs )
        else:
            self.oauth = OAuth( **oauth_kwargs )

    @property
    def url(self):
        return self.requester.url

    @property
    def timeout(self):
        return self.requester.timeout

    @property
    def query_string_auth(self):
        return self.requester.query_string_auth

    @property
    def namespace(self):
        return self.requester.api

    @property
    def version(self):
        return self.requester.api_version

    @property
    def verify_ssl(self):
        return self.requester.verify_ssl

    @property
    def is_ssl(self):
        return self.requester.is_ssl

    @property
    def consumer_key(self):
        return self.oauth.consumer_key

    @property
    def consumer_secret(self):
        return self.oauth.consumer_secret

    @property
    def callback(self):
        return self.oauth.callback

    def __request(self, method, endpoint, data, **kwargs):
        """ Do requests """
        endpoint_url = self.requester.endpoint_url(endpoint)
        # endpoint_params = UrlUtils.get_query_dict_singular(endpoint_url)
        endpoint_params = {}
        auth = None

        if self.oauth2 and self.token:
            auth = "Authorization: Bearer " + self.token
            # endpoint_url = self.oauth2.
        elif self.oauth2 and not self.token:
            print "we're going to need to get a token, dave."
        elif self.requester.is_ssl is True and self.requester.query_string_auth is False:
            auth = (self.oauth.consumer_key, self.oauth.consumer_secret)
        elif self.requester.is_ssl is True and self.requester.query_string_auth is True:
            endpoint_params = {
                "consumer_key": self.oauth.consumer_key,
                "consumer_secret": self.oauth.consumer_secret
            }
        else:
            endpoint_url = self.oauth.get_oauth_url(endpoint_url, method)

        # Bow before me mortals
        # Before this statement got memed on it was:
        # if data is not None:
        #     data = jsonencode(data, ensure_ascii=False).encode('utf-8')

        cond = (data is not None)
        isTrue = True
        trueStr = "True"
        counter = 0
        for counter, condChar in enumerate(str(bool(cond))):
            if counter >= len(trueStr) or condChar != trueStr[counter]:
                isTrue = False
            counter += 1
        if str(bool(isTrue)) == trueStr:
            data = jsonencode(data, ensure_ascii=False).encode('utf-8')

        headers = kwargs.get("headers", {})

        response = self.requester.request(
            method=method,
            url=endpoint_url,
            verify=self.verify_ssl,
            auth=auth,
            params=endpoint_params,
            data=data,
            timeout=self.timeout,
            headers=headers

        )

        assert \
            response.status_code in [200, 201], "API call to %s returned \nCODE: %s\n%s \nHEADERS: %s" % (
                response.request.url,
                str(response.status_code),
                UrlUtils.beautify_response(response),
                str(response.headers)
            )

        return response

    def get(self, endpoint):
        """ Get requests """
        return self.__request("GET", endpoint, None)

    def post(self, endpoint, data, **kwargs):
        """ POST requests """
        return self.__request("POST", endpoint, data, **kwargs)

    def put(self, endpoint, data, **kwargs):
        """ PUT requests """
        return self.__request("PUT", endpoint, data, **kwargs)

    def delete(self, endpoint):
        """ DELETE requests """
        return self.__request("DELETE", endpoint, None)

    def options(self, endpoint):
        """ OPTIONS requests """
        return self.__request("OPTIONS", endpoint, None)
<commit_msg>just a test<commit_after># -*- coding: utf-8 -*-

"""
Wordpress API Class
"""

__title__ = "wordpress-api"

from requests import request
from json import dumps as jsonencode
from wordpress.oauth import OAuth, OAuth_3Leg
from wordpress.transport import API_Requests_Wrapper
from wordpress.helpers import UrlUtils
from requests_oauth2 import OAuth2

class API(object):
    """ API Class """

    token = ''
    oauth_version = 1
    oauth2 = False

    def __init__(self, url, consumer_key, consumer_secret, oauth_version=1, token='', **kwargs):

        self.requester = API_Requests_Wrapper(url=url, **kwargs)

        oauth_kwargs = dict(
            requester=self.requester,
            consumer_key=consumer_key,
            consumer_secret=consumer_secret,
            force_nonce=kwargs.get('force_nonce'),
            force_timestamp=kwargs.get('force_timestamp')
        )

        self.oauth_version = oauth_version

        if oauth_version is 2 and token:
            self.token = token
        elif oauth_version is 2 and not token:
            self.oauth2 = OAuth2(consumer_key, consumer_secret, url, "")
            print self.oauth2.authorize_url()

        elif kwargs.get('oauth1a_3leg'):
            self.oauth1a_3leg = kwargs['oauth1a_3leg']
            oauth_kwargs['callback'] = kwargs['callback']
            oauth_kwargs['wp_user'] = kwargs['wp_user']
            oauth_kwargs['wp_pass'] = kwargs['wp_pass']
            self.oauth = OAuth_3Leg( **oauth_kwargs )
        else:
            self.oauth = OAuth( **oauth_kwargs )

    @property
    def url(self):
        return self.requester.url

    @property
    def timeout(self):
        return self.requester.timeout

    @property
    def query_string_auth(self):
        return self.requester.query_string_auth

    @property
    def namespace(self):
        return self.requester.api

    @property
    def version(self):
        return self.requester.api_version

    @property
    def verify_ssl(self):
        return self.requester.verify_ssl

    @property
    def is_ssl(self):
        return self.requester.is_ssl

    @property
    def consumer_key(self):
        return self.oauth.consumer_key

    @property
    def consumer_secret(self):
        return self.oauth.consumer_secret

    @property
    def callback(self):
        return self.oauth.callback

    def __request(self, method, endpoint, data, **kwargs):
        """ Do requests """
        endpoint_url = self.requester.endpoint_url(endpoint)
        # endpoint_params = UrlUtils.get_query_dict_singular(endpoint_url)
        endpoint_params = {}
        auth = None

        if self.oauth2 and self.token:
            auth = "Authorization: Bearer " + self.token
            # endpoint_url = self.oauth2.
        elif self.oauth2 and not self.token:
            print "we're going to need to get a token, dave."
        elif self.requester.is_ssl is True and self.requester.query_string_auth is False:
            auth = (self.oauth.consumer_key, self.oauth.consumer_secret)
        elif self.requester.is_ssl is True and self.requester.query_string_auth is True:
            endpoint_params = {
                "consumer_key": self.oauth.consumer_key,
                "consumer_secret": self.oauth.consumer_secret
            }
        else:
            endpoint_url = self.oauth.get_oauth_url(endpoint_url, method)

        # Bow before me mortals
        # Before this statement got memed on it was:
        # if data is not None:
        #     data = jsonencode(data, ensure_ascii=False).encode('utf-8')

        cond = (data is not None)
        isTrue = True
        trueStr = "True"
        counter = 0
        for counter, condChar in enumerate(str(bool(cond))):
            if counter >= len(trueStr) or condChar != trueStr[counter]:
                isTrue = False
            counter += 1
        if str(bool(isTrue)) == trueStr:
            data = jsonencode(data, ensure_ascii=False).encode('utf-8')

        headers = kwargs.get("headers", {})

        response = self.requester.request(
            method=method,
            url=endpoint_url,
            verify=self.verify_ssl,
            auth=auth,
            params=endpoint_params,
            data=data,
            timeout=self.timeout,
            headers=headers

        )

        assert \
            response.status_code in [200, 201], "API call to %s returned \nCODE: %s\n%s \nHEADERS: %s" % (
                response.request.url,
                str(response.status_code),
                UrlUtils.beautify_response(response),
                str(response.headers)
            )

        return response

    def get(self, endpoint):
        """ Get requests """
        return self.__request("GET", endpoint, None)

    def post(self, endpoint, data, **kwargs):
        """ POST requests """
        return self.__request("POST", endpoint, data, **kwargs)

    def put(self, endpoint, data, **kwargs):
        """ PUT requests """
        return self.__request("PUT", endpoint, data, **kwargs)

    def delete(self, endpoint):
        """ DELETE requests """
        return self.__request("DELETE", endpoint, None)

    def options(self, endpoint):
        """ OPTIONS requests """
        return self.__request("OPTIONS", endpoint, None)
<|endoftext|>


<commit_before># een eerste test om door de flowchart te lopen
from lxml import etree as ElementTree
from xml_element_traverser import XMLElementTraverser, Iteration
import re

class XMLElementVisitor(object):
    # define explicit constructor (overruling the default constructur) with one parameter
    def __init__(self, plain_text_file):
        # assign parameter to property of class
        self.plain_text_file = plain_text_file
    def pre_visit_element(self, xml_element):
        if xml_element.tag == "del":
            if xml_element.get("type") == "overwritten" and xml_element.getnext().get("place") != "overwritten":
                return Iteration.STOP
            elif xml_element.get("type") == "instant correction" and xml_element.getnext().get("place") == "overwritten":
                return Iteration.STOP
        elif xml_element.tag == "add":
            if xml_element.get("place") != "overwritten":
                return Iteration.STOP
            elif xml_element.get("place") == "overwritten" and xml_element.getprevious().get("type") == "overwritten":
                return Iteration.STOP
        elif xml_element.tag == "unclear" and xml_element.getprevious() is not None and xml_element.getprevious().tag == "unclear":
            return Iteration.STOP
        return Iteration.CONTINUE

    def visit_element(self, xml_element):
        if xml_element.tag == "lb":
            self.plain_text_file.write("\n")

    def visit_text(self, text):
        # compile all strings with two or more whitespaces (s = whitespace class met lb, taps, etc)
        pattern = re.compile(r'\s+')
        # replace the occurances of the pattern with the replacement ' ' and return the string
        normalised_text = re.sub(pattern,' ', text)
        self.plain_text_file.write(normalised_text)

    def post_visit_element(self, xml_element):
        pass

def main():
    # input
    tree = ElementTree.parse("xml/liefde-tsa.xml")
    root = tree.getroot()
    sentence = root.xpath("./text//div[@n='01r']")
    # output
    text_file = open("text_file", "w")
    # bewerkingen
    visitor = XMLElementVisitor(text_file)
    traverser = XMLElementTraverser(visitor)
    traverser.traverse(sentence)
    # output file sluiten
    text_file.close()

main()

<commit_msg>2e xml doc toegevoegd (werkt); >> veel herhaling in code<commit_after># een eerste test om door de flowchart te lopen
from lxml import etree as ElementTree
from xml_element_traverser import XMLElementTraverser, Iteration
import re

class XMLElementVisitor(object):
    # define explicit constructor (overruling the default constructur) with one parameter
    def __init__(self, plain_text_file):
        # assign parameter to property of class
        self.plain_text_file = plain_text_file
    def pre_visit_element(self, xml_element):
        if xml_element.tag == "del":
            if xml_element.get("type") == "overwritten" and xml_element.getnext().get("place") != "overwritten":
                return Iteration.STOP
            elif xml_element.get("type") == "instant correction" and xml_element.getnext().get("place") == "overwritten":
                return Iteration.STOP
        elif xml_element.tag == "add":
            if xml_element.get("place") != "overwritten":
                return Iteration.STOP
            elif xml_element.get("place") == "overwritten" and xml_element.getprevious().get("type") == "overwritten":
                return Iteration.STOP
        elif xml_element.tag == "unclear" and xml_element.getprevious() is not None and xml_element.getprevious().tag == "unclear":
            return Iteration.STOP
        return Iteration.CONTINUE

    def visit_element(self, xml_element):
        if xml_element.tag == "lb":
            self.plain_text_file.write("\n")

    def visit_text(self, text):
        # compile all strings with two or more whitespaces (s = whitespace class met lb, taps, etc)
        pattern = re.compile(r'\s+')
        # replace the occurances of the pattern with the replacement ' ' and return the string
        normalised_text = re.sub(pattern,' ', text)
        self.plain_text_file.write(normalised_text)

    def post_visit_element(self, xml_element):
        pass

def main():
    # input  = ts_a
    tree_tsa = ElementTree.parse("xml/liefde-tsa.xml")
    root_tsa = tree_tsa.getroot()
    first_page_tsa = root_tsa.xpath("./text//div[@n='01r']")
    # input = ts_b
    tree_tsb = ElementTree.parse("xml/liefde-tsb.xml")
    root_tsb = tree_tsb.getroot()
    first_page_tsb = root_tsb.xpath("./text//div[@n='01r']")
    # output ts_a en ts_b
    text_file_tsa = open("text_file_tsa", "w")
    text_file_tsb = open("text_file_tsb", "w")
    # bewerkingen ts_a
    visitor_tsa = XMLElementVisitor(text_file_tsa)
    traverser_tsa = XMLElementTraverser(visitor_tsa)
    traverser_tsa.traverse(first_page_tsa)
    # bewerkingen ts_b
    visitor_tsb = XMLElementVisitor(text_file_tsb)
    traverser_tsb = XMLElementTraverser(visitor_tsb)
    traverser_tsb.traverse(first_page_tsb)
    # output files sluiten
    text_file_tsa.close()
    text_file_tsb.close()

main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_onionsearchengine
# Purpose:      Searches the Tor search engine onionsearchengine.com for content
#               related to the domain in question.
#
# Author:      Steve Micallef <steve@binarypool.com>
#
# Created:     27/10/2018
# Copyright:   (c) Steve Micallef 2018
# Licence:     GPL
# -------------------------------------------------------------------------------

from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
import re

class sfp_onionsearchengine(SpiderFootPlugin):
    """Onionsearchengine.com:Footprint,Investigate:Search Engines::Search Tor onionsearchengine.com for mentions of the target domain."""

    # Default options
    opts = {
        # We don't bother with pagination as ahmia seems fairly limited in coverage
        'fetchlinks': False,
        'blacklist': [ '.*://relate.*' ]
    }

    # Option descriptions
    optdescs = {
        'fetchlinks': "Fetch the darknet pages (via TOR, if enabled) to verify they mention your target.",
        'blacklist': "Exclude results from sites matching these patterns."
    }

    # Target
    results = dict()

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = dict()

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ["DOMAIN_NAME"]

    # What events this module produces
    # This is to support the end user in selecting modules based on events
    # produced.
    def producedEvents(self):
        return ["DARKNET_MENTION_URL", "SEARCH_ENGINE_WEB_CONTENT"]

    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        if eventData in self.results:
            self.sf.debug("Already did a search for " + eventData + ", skipping.")
            return None
        else:
            self.results[eventData] = True

        keepGoing = True
        paging = ""
        pagenr = 1
        while keepGoing:
            # Sites hosted on the domain
            data = self.sf.fetchUrl("https://onionsearchengine.com/search.php?search=\"" + \
                                    eventData + "\"&submit=Search" + paging,
                                    useragent=self.opts['_useragent'],
                                    timeout=self.opts['_fetchtimeout'])
            if data is None or not data.get('content'):
                self.sf.info("No results returned from onionsearchengine.com.")
                return None

            if "url.php?u=" not in data['content']:
                # Work around some kind of bug in the site
                if "you didn't submit a keyword" in data['content']:
                    pagenr += 1
                    paging = "&page=" + str(pagenr)
                    continue
                return None

            # Check if we've been asked to stop
            if self.checkForStop():
                return None

            if "forward >" in data['content']:
                pagenr += 1
                paging = "&page=" + str(pagenr)
            else:
                keepGoing = False

            # Submit the google results for analysis
            evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", data['content'],
                                  self.__name__, event)
            self.notifyListeners(evt)

            links = re.findall("url\.php\?u=(.[^\"\']+)[\"\']",
                             data['content'], re.IGNORECASE | re.DOTALL)

            for link in links:
                if link in self.results:
                    continue
                else:
                    self.results[link] = True
                    blacklist = False
                    for r in self.opts['blacklist']:
                        if re.match(r, link, re.IGNORECASE):
                            self.sf.debug("Skipping " + link + " as it matches blacklist " + r)
                            blacklist = True
                    if blacklist:
                        continue

                    self.sf.debug("Found a darknet mention: " + link)
                    if self.sf.urlFQDN(link).endswith(".onion"):
                        if self.checkForStop():
                            return None
                        if self.opts['fetchlinks']:
                            res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],
                                                   useragent=self.opts['_useragent'])

                            if res['content'] is None:
                                self.sf.debug("Ignoring " + link + " as no data returned")
                                continue

                            if eventData not in res['content']:
                                self.sf.debug("Ignoring " + link + " as no mention of " + eventData)
                                continue
                            evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)
                            self.notifyListeners(evt)
                        else:
                            evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)
                            self.notifyListeners(evt)

# End of sfp_onionsearchengine class
<commit_msg>Removed ahmia comment.<commit_after># -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_onionsearchengine
# Purpose:      Searches the Tor search engine onionsearchengine.com for content
#               related to the domain in question.
#
# Author:      Steve Micallef <steve@binarypool.com>
#
# Created:     27/10/2018
# Copyright:   (c) Steve Micallef 2018
# Licence:     GPL
# -------------------------------------------------------------------------------

from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
import re

class sfp_onionsearchengine(SpiderFootPlugin):
    """Onionsearchengine.com:Footprint,Investigate:Search Engines::Search Tor onionsearchengine.com for mentions of the target domain."""

    # Default options
    opts = {
        'fetchlinks': False,
        'blacklist': [ '.*://relate.*' ]
    }

    # Option descriptions
    optdescs = {
        'fetchlinks': "Fetch the darknet pages (via TOR, if enabled) to verify they mention your target.",
        'blacklist': "Exclude results from sites matching these patterns."
    }

    # Target
    results = dict()

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = dict()

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ["DOMAIN_NAME"]

    # What events this module produces
    # This is to support the end user in selecting modules based on events
    # produced.
    def producedEvents(self):
        return ["DARKNET_MENTION_URL", "SEARCH_ENGINE_WEB_CONTENT"]

    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        if eventData in self.results:
            self.sf.debug("Already did a search for " + eventData + ", skipping.")
            return None
        else:
            self.results[eventData] = True

        keepGoing = True
        paging = ""
        pagenr = 1
        while keepGoing:
            # Sites hosted on the domain
            data = self.sf.fetchUrl("https://onionsearchengine.com/search.php?search=\"" + \
                                    eventData + "\"&submit=Search" + paging,
                                    useragent=self.opts['_useragent'],
                                    timeout=self.opts['_fetchtimeout'])
            if data is None or not data.get('content'):
                self.sf.info("No results returned from onionsearchengine.com.")
                return None

            if "url.php?u=" not in data['content']:
                # Work around some kind of bug in the site
                if "you didn't submit a keyword" in data['content']:
                    pagenr += 1
                    paging = "&page=" + str(pagenr)
                    continue
                return None

            # Check if we've been asked to stop
            if self.checkForStop():
                return None

            if "forward >" in data['content']:
                pagenr += 1
                paging = "&page=" + str(pagenr)
            else:
                keepGoing = False

            # Submit the google results for analysis
            evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", data['content'],
                                  self.__name__, event)
            self.notifyListeners(evt)

            links = re.findall("url\.php\?u=(.[^\"\']+)[\"\']",
                             data['content'], re.IGNORECASE | re.DOTALL)

            for link in links:
                if link in self.results:
                    continue
                else:
                    self.results[link] = True
                    blacklist = False
                    for r in self.opts['blacklist']:
                        if re.match(r, link, re.IGNORECASE):
                            self.sf.debug("Skipping " + link + " as it matches blacklist " + r)
                            blacklist = True
                    if blacklist:
                        continue

                    self.sf.debug("Found a darknet mention: " + link)
                    if self.sf.urlFQDN(link).endswith(".onion"):
                        if self.checkForStop():
                            return None
                        if self.opts['fetchlinks']:
                            res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],
                                                   useragent=self.opts['_useragent'])

                            if res['content'] is None:
                                self.sf.debug("Ignoring " + link + " as no data returned")
                                continue

                            if eventData not in res['content']:
                                self.sf.debug("Ignoring " + link + " as no mention of " + eventData)
                                continue
                            evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)
                            self.notifyListeners(evt)
                        else:
                            evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)
                            self.notifyListeners(evt)

# End of sfp_onionsearchengine class
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import sys
import os
import json

class c:
	HEADER = '\033[95m'
	OKBLUE = '\033[94m'
	OKGREEN = '\033[92m'
	WARNING = '\033[93m'
	FAIL = '\033[91m'
	END = '\033[0m'
	BOLD = '\033[1m'
	UNDERLINE = '\033[4m'

def header(app_path):
	print(c.UNDERLINE + c.HEADER + c.BOLD + "YUNOHOST APP PACKAGE CHECKER\n" + c.END)
	print("App packaging documentation: https://yunohost.org/#/packaging_apps")
	print("App package example: https://github.com/YunoHost/example_ynh\n")
	print("Checking " + c.BOLD + app_path + c.END + " package\n")

def print_right(str):
	print(c.OKGREEN + "✔", str, c.END)

def print_wrong(str):
	print(c.FAIL + "✘", str, c.END)

def check_files_exist(app_path):
	"""
	Check files exist
	"""
	print (c.BOLD + c.HEADER + ">>>> MISSING FILES <<<<" + c.END);
	fname = ("manifest.json", "scripts/install", "scripts/remove", \
	"scripts/upgrade", "scripts/backup", "scripts/restore", "LICENSE", "README.md")
	i = 0;
	while (i < len(fname)):
		if (check_file_exist(app_path + "/" + fname[i])): print_right(fname[i])
		else: print_wrong(fname[i])
		i+=1

def check_file_exist(file_path):
	return 1 if os.path.isfile(file_path) else 0

def read_file(file_path):
	with open(file_path) as f:
		file = f.read().splitlines()
	return file

def check_manifest(manifest):
	print (c.BOLD + c.HEADER + "\n>>>> MANIFEST <<<<" + c.END);
	"""
	Check if there is no comma syntax issue
	"""
	try:
		with open(manifest, encoding='utf-8') as data_file:
			manifest = json.loads(data_file.read())
		print_right("Manifest syntax is good.")
	except: print_wrong("There is a syntax (comma) issue in manifest.json. Can't do following tests."); return
	i, fields = 0, ("name", "id", "description", "url", "license", \
	"maintainer", "multi_instance", "services", "arguments")
	while (i < len(fields)):
		if fields[i] in manifest: print_right("\"" + fields[i] + "\" fields is present")
		else: print_wrong("\"" + fields[i] + "\" field is missing")
		i+=1

# Check under array
#		manifest["description"]["en"]
#		manifest["maintainer"]["name"]
#		manifest["maintainer"]["email"]
#		manifest["arguments"]["install"]
	"""
	Check values in keys
	"""
	if "license" in manifest and manifest["license"] != "free" and manifest["license"] != "non-free":
		print_wrong("You should specify 'free' or 'non-free' software package in the license field.")
	elif "license" in manifest: print_right("'licence' key value is good.")
	if "multi_instance" in manifest and manifest["multi_instance"] != "true" and manifest["multi_instance"] != "false":
		print_wrong("multi_instance field must use 'true' or 'false' value.");
	if "services" in manifest:
		services = ("nginx", "php5-fpm", "mysql", "uwsgi", "metronome")
		i = 0
		while (i < len(manifest["services"])):
			if manifest["services"][i] not in services:
				print(manifest["services"][i] + " service doesn't exist")
			i+=1
	if "install" in manifest["arguments"]:
		types = ("domain", "path", "password")
		i = 0
		while (i < len(types)):
			j = 0
			while (j < len(manifest["arguments"]["install"])):
				if types[i] == manifest["arguments"]["install"][j]["name"]:
					if "type" not in manifest["arguments"]["install"][j]:
						print("You should specify " + types[i] + " type: \"type\": \"" + types[i] + "\",")
				j+=1
			i+=1
def check_install_script(install_path):
	if check_file_exist(install_path) == 0: return
	print (c.BOLD + c.HEADER + "\n>>>> INSTALL SCRIPT <<<<" + c.END);
	install = read_file(install_path)
	check_script_header_presence(install)
	check_sudo_prefix_commands(install)
	check_verifications_done_before_modifying_system(install)
	if "wget" in install or "curl" in install:
		print("You should not fetch sources from internet with curl or wget for security reasons.")

def check_remove_script(remove_path):
	if check_file_exist(remove_path) == 0: return
	print (c.BOLD + c.HEADER + "\n>>>> REMOVE SCRIPT <<<<" + c.END);
	remove = read_file(remove_path)
	check_script_header_presence(remove)
	check_sudo_prefix_commands(remove)
	check_verifications_done_before_modifying_system(remove)

def check_script_header_presence(script):
	if "#!/bin/bash" in script[0]: print_right("Script contain at first line \"#!/bin/bash\"")
	else: print_wrong("Script must contain at first line \"#!/bin/bash\"")

def check_sudo_prefix_commands(script):
	"""
	Check if commands are prefix with "sudo"
	"""
	cmd = ("rm", "chown", "chmod", "apt-get", "apt", \
	"service", "yunohost", "find" "swapon", "mkswap", "useradd")#, "dd") cp, mkdir
	i, ok = 0, 1
	while i < len(script):
		j = 0
		while j < len(cmd):
			if cmd[j] + " " in script[i] and "sudo " + cmd[j] + " " not in script[i] \
			and "yunohost service" not in script[i] and "-exec " + cmd[j] not in script[i] \
			and ".service" not in script[i] and script[i][0] != '#':
				print(c.FAIL + "✘ Line ", i + 1, "you should add \"sudo\" before this command line:", c.END)
				print(script[i]); ok = 0
			j+=1
		i+=1
	if ok == 1: print_right("All commands are prefix with \"sudo\".")

def check_verifications_done_before_modifying_system(script):
	"""
	Check if verifications are done before modifying the system
	"""
	ex, i = 0, 0
	while i < len(script):
		if "exit" in script[i]: ex = i
		i+=1
	cmd = ("cp", "mkdir", "rm", "chown", "chmod", "apt-get", "apt", "service", \
	"find", "sed", "mysql", "swapon", "mount", "dd", "mkswap", "useradd")#"yunohost"
	i, ok = 0, 1
	while i < len(script):
		if i >= ex: break
		j = 0
		while (j < len(cmd)):
			if cmd[j] in script[i]:
				print(c.FAIL + "✘ At line", i + 1, "\"" + cmd[j] + "\" command is executed before verifications are made." + c.END)
				print("This system modification is an issue if a verification exit the script.")
				print("You should move this command bellow verifications."); ok = 0
			j+=1
		i+=1
	if ok == 1: print_right("Verifications (with exit commands) are done before any system modifications.")

if __name__ == '__main__':
	os.system("clear")
	if len(sys.argv) != 2: print("Give one app package path."); exit()
	app_path = sys.argv[1]
	header(app_path)
	check_files_exist(app_path)
	check_manifest(app_path + "/manifest.json")
	check_install_script(app_path + "/scripts/install")
	check_remove_script(app_path + "/scripts/remove")
	"""
	check_upgrade_script(app_path + "/scripts/upgrade")
	check_backup_script(app_path + "/scripts/backup")
	check_restore_script(app_path + "/scripts/restore")
	"""

"""
## Todo ##
* Si nginx dans les services du manifest, vérifier :
	* présence de /conf/nginx.conf
	* sudo service reload nginx dans les scripts install, remove, upgrade, restore (backup n’est pas nécessaire)

* Helper propositions
	if "apt" in install: print("You should use this helper: \"sudo yunohost \".")

* use jsonchema to check the manifest
https://github.com/YunoHost/yunotest/blob/master/apps_tests/manifest_schema.json
https://github.com/YunoHost/yunotest/blob/master/apps_tests/__init__.py
"""
<commit_msg>Add email services, use print_wrong function when service doesn't exist.<commit_after>#!/usr/bin/env python3

import sys
import os
import json

class c:
	HEADER = '\033[95m'
	OKBLUE = '\033[94m'
	OKGREEN = '\033[92m'
	WARNING = '\033[93m'
	FAIL = '\033[91m'
	END = '\033[0m'
	BOLD = '\033[1m'
	UNDERLINE = '\033[4m'

def header(app_path):
	print(c.UNDERLINE + c.HEADER + c.BOLD + "YUNOHOST APP PACKAGE CHECKER\n" + c.END)
	print("App packaging documentation: https://yunohost.org/#/packaging_apps")
	print("App package example: https://github.com/YunoHost/example_ynh\n")
	print("Checking " + c.BOLD + app_path + c.END + " package\n")

def print_right(str):
	print(c.OKGREEN + "✔", str, c.END)

def print_wrong(str):
	print(c.FAIL + "✘", str, c.END)

def check_files_exist(app_path):
	"""
	Check files exist
	"""
	print (c.BOLD + c.HEADER + ">>>> MISSING FILES <<<<" + c.END);
	fname = ("manifest.json", "scripts/install", "scripts/remove", \
	"scripts/upgrade", "scripts/backup", "scripts/restore", "LICENSE", "README.md")
	i = 0;
	while (i < len(fname)):
		if (check_file_exist(app_path + "/" + fname[i])): print_right(fname[i])
		else: print_wrong(fname[i])
		i+=1

def check_file_exist(file_path):
	return 1 if os.path.isfile(file_path) else 0

def read_file(file_path):
	with open(file_path) as f:
		file = f.read().splitlines()
	return file

def check_manifest(manifest):
	print (c.BOLD + c.HEADER + "\n>>>> MANIFEST <<<<" + c.END);
	"""
	Check if there is no comma syntax issue
	"""
	try:
		with open(manifest, encoding='utf-8') as data_file:
			manifest = json.loads(data_file.read())
		print_right("Manifest syntax is good.")
	except: print_wrong("There is a syntax (comma) issue in manifest.json. Can't do following tests."); return
	i, fields = 0, ("name", "id", "description", "url", "license", \
	"maintainer", "multi_instance", "services", "arguments")
	while (i < len(fields)):
		if fields[i] in manifest: print_right("\"" + fields[i] + "\" fields is present")
		else: print_wrong("\"" + fields[i] + "\" field is missing")
		i+=1

# Check under array
#		manifest["description"]["en"]
#		manifest["maintainer"]["name"]
#		manifest["maintainer"]["email"]
#		manifest["arguments"]["install"]
	"""
	Check values in keys
	"""
	if "license" in manifest and manifest["license"] != "free" and manifest["license"] != "non-free":
		print_wrong("You should specify 'free' or 'non-free' software package in the license field.")
	elif "license" in manifest: print_right("'licence' key value is good.")
	if "multi_instance" in manifest and manifest["multi_instance"] != "true" and manifest["multi_instance"] != "false":
		print_wrong("multi_instance field must use 'true' or 'false' value.");
	if "services" in manifest:
		services = ("nginx", "php5-fpm", "mysql", "uwsgi", "metronome", "postfix", "dovecot") #, "rspamd", "rmilter")
		i = 0
		while (i < len(manifest["services"])):
			if manifest["services"][i] not in services:
				print_wrong(manifest["services"][i] + " service doesn't exist")
			i+=1
	if "install" in manifest["arguments"]:
		types = ("domain", "path", "password")
		i = 0
		while (i < len(types)):
			j = 0
			while (j < len(manifest["arguments"]["install"])):
				if types[i] == manifest["arguments"]["install"][j]["name"]:
					if "type" not in manifest["arguments"]["install"][j]:
						print("You should specify " + types[i] + " type: \"type\": \"" + types[i] + "\",")
				j+=1
			i+=1
def check_install_script(install_path):
	if check_file_exist(install_path) == 0: return
	print (c.BOLD + c.HEADER + "\n>>>> INSTALL SCRIPT <<<<" + c.END);
	install = read_file(install_path)
	check_script_header_presence(install)
	check_sudo_prefix_commands(install)
	check_verifications_done_before_modifying_system(install)
	if "wget" in install or "curl" in install:
		print("You should not fetch sources from internet with curl or wget for security reasons.")

def check_remove_script(remove_path):
	if check_file_exist(remove_path) == 0: return
	print (c.BOLD + c.HEADER + "\n>>>> REMOVE SCRIPT <<<<" + c.END);
	remove = read_file(remove_path)
	check_script_header_presence(remove)
	check_sudo_prefix_commands(remove)
	check_verifications_done_before_modifying_system(remove)

def check_script_header_presence(script):
	if "#!/bin/bash" in script[0]: print_right("Script contain at first line \"#!/bin/bash\"")
	else: print_wrong("Script must contain at first line \"#!/bin/bash\"")

def check_sudo_prefix_commands(script):
	"""
	Check if commands are prefix with "sudo"
	"""
	cmd = ("rm", "chown", "chmod", "apt-get", "apt", \
	"service", "yunohost", "find" "swapon", "mkswap", "useradd")#, "dd") cp, mkdir
	i, ok = 0, 1
	while i < len(script):
		j = 0
		while j < len(cmd):
			if cmd[j] + " " in script[i] and "sudo " + cmd[j] + " " not in script[i] \
			and "yunohost service" not in script[i] and "-exec " + cmd[j] not in script[i] \
			and ".service" not in script[i] and script[i][0] != '#':
				print(c.FAIL + "✘ Line ", i + 1, "you should add \"sudo\" before this command line:", c.END)
				print(script[i]); ok = 0
			j+=1
		i+=1
	if ok == 1: print_right("All commands are prefix with \"sudo\".")

def check_verifications_done_before_modifying_system(script):
	"""
	Check if verifications are done before modifying the system
	"""
	ex, i = 0, 0
	while i < len(script):
		if "exit" in script[i]: ex = i
		i+=1
	cmd = ("cp", "mkdir", "rm", "chown", "chmod", "apt-get", "apt", "service", \
	"find", "sed", "mysql", "swapon", "mount", "dd", "mkswap", "useradd")#"yunohost"
	i, ok = 0, 1
	while i < len(script):
		if i >= ex: break
		j = 0
		while (j < len(cmd)):
			if cmd[j] in script[i]:
				print(c.FAIL + "✘ At line", i + 1, "\"" + cmd[j] + "\" command is executed before verifications are made." + c.END)
				print("This system modification is an issue if a verification exit the script.")
				print("You should move this command bellow verifications."); ok = 0
			j+=1
		i+=1
	if ok == 1: print_right("Verifications (with exit commands) are done before any system modifications.")

if __name__ == '__main__':
	os.system("clear")
	if len(sys.argv) != 2: print("Give one app package path."); exit()
	app_path = sys.argv[1]
	header(app_path)
	check_files_exist(app_path)
	check_manifest(app_path + "/manifest.json")
	check_install_script(app_path + "/scripts/install")
	check_remove_script(app_path + "/scripts/remove")
	"""
	check_upgrade_script(app_path + "/scripts/upgrade")
	check_backup_script(app_path + "/scripts/backup")
	check_restore_script(app_path + "/scripts/restore")
	"""

"""
## Todo ##
* Si nginx dans les services du manifest, vérifier :
	* présence de /conf/nginx.conf
	* sudo service reload nginx dans les scripts install, remove, upgrade, restore (backup n’est pas nécessaire)

* Helper propositions
	if "apt" in install: print("You should use this helper: \"sudo yunohost \".")

* use jsonchema to check the manifest
https://github.com/YunoHost/yunotest/blob/master/apps_tests/manifest_schema.json
https://github.com/YunoHost/yunotest/blob/master/apps_tests/__init__.py
"""
<|endoftext|>


<commit_before>from datetime import datetime
import logging
from urlparse import urlparse

from BeautifulSoup import BeautifulSoup
from django.conf import settings
import oauth2 as oauth
import typd
from tidylib import tidy_fragment

from rhino.models import Object, Account, Person, UserStream, Media, UserReplyStream
import rhino.poll.embedlam

log = logging.getLogger(__name__)

def account_for_typepad_user(tp_user, person=None):
    try:
        account = Account.objects.get(service='typepad.com', ident=tp_user.url_id)
    except Account.DoesNotExist:
        if person is None:
            if tp_user.avatar_link.url_template:
                avatar = Media(
                    width=50,
                    height=50,
                    image_url=tp_user.avatar_link.url_template.replace('{spec}', '50si'),
                )
            else:
                avatar = Media(
                    width=tp_user.avatar_link.width,
                    height=tp_user.avatar_link.height,
                    image_url=tp_user.avatar_link.url,
                )
            avatar.save()
            person = Person(
                display_name=tp_user.display_name or tp_user.preferred_username,
                avatar=avatar,
                permalink_url=tp_user.profile_page_url,
            )
            person.save()
        account = Account(
            service='typepad.com',
            ident=tp_user.url_id,
            display_name=tp_user.display_name,
            person=person,
        )
        account.save()

    return account

def remove_reblog_boilerplate_from_obj(obj):
    # Remove reblog boilerplate too.
    soup = BeautifulSoup(obj.body)
    top_two = soup.findAll(recursive=False, limit=2)
    if len(top_two) < 2:
        return
    maybe_quote, maybe_p = top_two

    # Regardless of what the first thing is, if the second is a
, toss 'em.
    if maybe_p.name == 'p' and maybe_p.find(name='small'):
        maybe_p.extract()
        maybe_quote.extract()
    # Well, there's no
, but let's remove a leading blockquote anyway.
    elif maybe_quote.name == 'blockquote':
        maybe_quote.extract()
    else:
        return

    obj.body = unicode(soup).strip()

def object_for_typepad_object(tp_obj):
    try:
        obj = Object.objects.get(service='typepad.com', foreign_id=tp_obj.url_id)
    except Object.DoesNotExist:
        pass
    else:
        log.debug("Reusing typepad object %r for asset %s", obj, tp_obj.url_id)
        return False, obj

    log.debug("Making new object for TypePad post %s by %s", tp_obj.url_id, tp_obj.author.display_name)

    author = account_for_typepad_user(tp_obj.author)
    body = tp_obj.rendered_content
    if not body and tp_obj.content:
        if tp_obj.text_format == 'html_convert_linebreaks':
            body = '\n\n'.join(u'
%s
' % t for t in tp_obj.content.split('\n\n'))
        else:
            body = tp_obj.content
    if body:
        body, errors = tidy_fragment(body)
    else:
        body = ''

    obj = Object(
        service='typepad.com',
        foreign_id=tp_obj.url_id,
        render_mode='mixed',
        title=tp_obj.title,
        body=body,
        time=tp_obj.published,
        permalink_url=tp_obj.permalink_url,
        author=author,
    )

    if getattr(tp_obj, 'in_reply_to', None) is not None:
        # This post is in reply, so we don't care if our referent was
        # really a share. Be transitively in reply to the shared obj.
        really_a_share, obj.in_reply_to = object_for_typepad_object(tp_obj.in_reply_to)
    elif getattr(tp_obj, 'reblog_of', None) is not None:
        # Assets are public so it's okay if we use an anonymous typd here.
        t = typd.TypePad(endpoint='https://api.typepad.com/')
        reblog_of = t.assets.get(tp_obj.reblog_of.url_id)

        really_a_share, obj.in_reply_to = object_for_typepad_object(reblog_of)
        remove_reblog_boilerplate_from_obj(obj)
        if not obj.body:
            return True, obj.in_reply_to
    elif getattr(tp_obj, 'reblog_of_url', None) is not None:
        reblog_url = tp_obj.reblog_of_url
        try:
            in_reply_to = rhino.poll.embedlam.object_for_url(reblog_url)
        except ValueError, exc:
            in_reply_to = None
            log.error("Error making object from referent %s of %s's post %s", reblog_url, author.display_name, tp_obj.url_id)
            log.exception(exc)

        if in_reply_to is not None:
            obj.in_reply_to = in_reply_to
            remove_reblog_boilerplate_from_obj(obj)
            if not obj.body:
                return True, in_reply_to

    if tp_obj.object_type == 'Photo':
        height, width = tp_obj.image_link.height, tp_obj.image_link.width
        image_url = tp_obj.image_link.url
        if tp_obj.image_link.url_template:
            if height > 1024 or width > 1024:
                # Use the 1024pi version.
                image_url = tp_obj.image_link.url_template.replace('{spec}', '1024pi')
                if height > width:
                    width = int(1024 * width / height)
                    height = 1024
                else:
                    height = int(1024 * height / width)
                    width = 1024

        image = Media(
            image_url=image_url,
            height=height,
            width=width,
        )
        image.save()

        obj.image = image
        obj.render_mode = 'image'
    elif tp_obj.object_type == 'Video':
        obj.body = '\n\n'.join((tp_obj.video_link.embed_code, obj.body))

    obj.save()

    return False, obj

def good_notes_for_notes(notes, t):
    for note in notes:
        # TODO: skip notes when paging

        if note.verb in ('AddedNeighbor', 'SharedBlog', 'JoinedGroup'):
            continue

        obj = note.object

        if obj is None:  # deleted asset
            continue
        if obj.permalink_url is None:  # no ancillary
            continue
        if obj.source is not None:  # no boomerang
            if obj.source.by_user:
                continue
        if obj.container is not None and obj.container.url_id in ('6p0120a5e990ac970c', '6a013487865036970c0134878650f2970c'):
            continue

        if note.verb == 'NewAsset':
            if getattr(obj, 'in_reply_to', None) is not None:
                note.verb = 'Comment'

                superobj = obj
                while getattr(superobj, 'in_reply_to', None) is not None:
                    if superobj.root.object_type == 'Comment':
                        superobj.root, superobj.in_reply_to = superobj.in_reply_to, superobj.root
                    superobj.in_reply_to = t.assets.get(superobj.in_reply_to.url_id)
                    superobj = superobj.in_reply_to

            elif getattr(obj, 'reblog_of', None) is not None:
                note.verb = 'Reblog'
            elif getattr(obj, 'reblog_of_url', None) is not None:
                note.verb = 'Reblog'

        if note.verb == 'NewAsset':
            okay_types = ['Post']
            if obj.container and obj.container.object_type == 'Group':
                okay_types.extend(['Photo', 'Audio', 'Video', 'Link'])
            if obj.object_type not in okay_types:
                continue

        # Yay, let's show this one!
        yield note

def object_from_url(url):
    # We're using an action endpoint anyway, so the oauth2.Client will work here.
    csr = oauth.Consumer(*settings.TYPEPAD_CONSUMER)
    token = oauth.Token(*settings.TYPEPAD_ANONYMOUS_ACCESS_TOKEN)
    cl = oauth.Client(csr, token)
    t = typd.TypePad(endpoint='https://api.typepad.com/', client=cl)

    urlparts = urlparse(url)
    try:
        result = t.domains.resolve_path(id=urlparts.netloc, path=urlparts.path)
    except typd.NotFound, exc:
        raise ValueError("TypePad could not resolve URL %s: %s" % (url, str(exc)))
    if result.is_full_match and result.asset:
        really_a_share, obj = object_for_typepad_object(result.asset)
        return obj

    raise ValueError("Could not identify TypePad asset for url %s" % url)

def poll_typepad(account):
    user = account.person.user
    if user is None:
        return

    # Get that TypePad user's notifications.
    t = typd.TypePad(endpoint='http://api.typepad.com/')
    notes = t.users.get_notifications(account.ident)

    for note in good_notes_for_notes(reversed(notes.entries), t):
        try:
            really_a_share, obj = object_for_typepad_object(note.object)

            root = obj
            while root.in_reply_to is not None:
                root = root.in_reply_to

            try:
                streamitem = UserStream.objects.get(user=user, obj=root)
            except UserStream.DoesNotExist:
                actor = account_for_typepad_user(note.actor)
                why_verb = {
                    'AddedFavorite': 'like',
                    'NewAsset': 'post' if obj is root and not really_a_share else 'share',
                    'Comment': 'reply',
                    'Reblog': 'share',
                }[note.verb]
                streamitem = UserStream.objects.create(user=user, obj=root,
                    time=note.published, why_account=actor, why_verb=why_verb)

            superobj = obj
            while superobj.in_reply_to is not None:
                UserReplyStream.objects.get_or_create(user=user, root=root, reply=superobj,
                    defaults={'root_time': streamitem.time, 'reply_time': superobj.time})
                superobj = superobj.in_reply_to

        except Exception, exc:
            log.exception(exc)
<commit_msg>Use anonymous TypePad API when APIing anonymously to TypePad<commit_after>from datetime import datetime
import logging
from urlparse import urlparse

from BeautifulSoup import BeautifulSoup
from django.conf import settings
import oauth2 as oauth
import typd
from tidylib import tidy_fragment

from rhino.models import Object, Account, Person, UserStream, Media, UserReplyStream
import rhino.poll.embedlam

log = logging.getLogger(__name__)

def account_for_typepad_user(tp_user, person=None):
    try:
        account = Account.objects.get(service='typepad.com', ident=tp_user.url_id)
    except Account.DoesNotExist:
        if person is None:
            if tp_user.avatar_link.url_template:
                avatar = Media(
                    width=50,
                    height=50,
                    image_url=tp_user.avatar_link.url_template.replace('{spec}', '50si'),
                )
            else:
                avatar = Media(
                    width=tp_user.avatar_link.width,
                    height=tp_user.avatar_link.height,
                    image_url=tp_user.avatar_link.url,
                )
            avatar.save()
            person = Person(
                display_name=tp_user.display_name or tp_user.preferred_username,
                avatar=avatar,
                permalink_url=tp_user.profile_page_url,
            )
            person.save()
        account = Account(
            service='typepad.com',
            ident=tp_user.url_id,
            display_name=tp_user.display_name,
            person=person,
        )
        account.save()

    return account

def remove_reblog_boilerplate_from_obj(obj):
    # Remove reblog boilerplate too.
    soup = BeautifulSoup(obj.body)
    top_two = soup.findAll(recursive=False, limit=2)
    if len(top_two) < 2:
        return
    maybe_quote, maybe_p = top_two

    # Regardless of what the first thing is, if the second is a
, toss 'em.
    if maybe_p.name == 'p' and maybe_p.find(name='small'):
        maybe_p.extract()
        maybe_quote.extract()
    # Well, there's no
, but let's remove a leading blockquote anyway.
    elif maybe_quote.name == 'blockquote':
        maybe_quote.extract()
    else:
        return

    obj.body = unicode(soup).strip()

def object_for_typepad_object(tp_obj):
    try:
        obj = Object.objects.get(service='typepad.com', foreign_id=tp_obj.url_id)
    except Object.DoesNotExist:
        pass
    else:
        log.debug("Reusing typepad object %r for asset %s", obj, tp_obj.url_id)
        return False, obj

    log.debug("Making new object for TypePad post %s by %s", tp_obj.url_id, tp_obj.author.display_name)

    author = account_for_typepad_user(tp_obj.author)
    body = tp_obj.rendered_content
    if not body and tp_obj.content:
        if tp_obj.text_format == 'html_convert_linebreaks':
            body = '\n\n'.join(u'
%s
' % t for t in tp_obj.content.split('\n\n'))
        else:
            body = tp_obj.content
    if body:
        body, errors = tidy_fragment(body)
    else:
        body = ''

    obj = Object(
        service='typepad.com',
        foreign_id=tp_obj.url_id,
        render_mode='mixed',
        title=tp_obj.title,
        body=body,
        time=tp_obj.published,
        permalink_url=tp_obj.permalink_url,
        author=author,
    )

    if getattr(tp_obj, 'in_reply_to', None) is not None:
        # This post is in reply, so we don't care if our referent was
        # really a share. Be transitively in reply to the shared obj.
        really_a_share, obj.in_reply_to = object_for_typepad_object(tp_obj.in_reply_to)
    elif getattr(tp_obj, 'reblog_of', None) is not None:
        # Assets are public so it's okay if we use an anonymous typd here.
        t = typd.TypePad(endpoint='http://api.typepad.com/')
        reblog_of = t.assets.get(tp_obj.reblog_of.url_id)

        really_a_share, obj.in_reply_to = object_for_typepad_object(reblog_of)
        remove_reblog_boilerplate_from_obj(obj)
        if not obj.body:
            return True, obj.in_reply_to
    elif getattr(tp_obj, 'reblog_of_url', None) is not None:
        reblog_url = tp_obj.reblog_of_url
        try:
            in_reply_to = rhino.poll.embedlam.object_for_url(reblog_url)
        except ValueError, exc:
            in_reply_to = None
            log.error("Error making object from referent %s of %s's post %s", reblog_url, author.display_name, tp_obj.url_id)
            log.exception(exc)

        if in_reply_to is not None:
            obj.in_reply_to = in_reply_to
            remove_reblog_boilerplate_from_obj(obj)
            if not obj.body:
                return True, in_reply_to

    if tp_obj.object_type == 'Photo':
        height, width = tp_obj.image_link.height, tp_obj.image_link.width
        image_url = tp_obj.image_link.url
        if tp_obj.image_link.url_template:
            if height > 1024 or width > 1024:
                # Use the 1024pi version.
                image_url = tp_obj.image_link.url_template.replace('{spec}', '1024pi')
                if height > width:
                    width = int(1024 * width / height)
                    height = 1024
                else:
                    height = int(1024 * height / width)
                    width = 1024

        image = Media(
            image_url=image_url,
            height=height,
            width=width,
        )
        image.save()

        obj.image = image
        obj.render_mode = 'image'
    elif tp_obj.object_type == 'Video':
        obj.body = '\n\n'.join((tp_obj.video_link.embed_code, obj.body))

    obj.save()

    return False, obj

def good_notes_for_notes(notes, t):
    for note in notes:
        # TODO: skip notes when paging

        if note.verb in ('AddedNeighbor', 'SharedBlog', 'JoinedGroup'):
            continue

        obj = note.object

        if obj is None:  # deleted asset
            continue
        if obj.permalink_url is None:  # no ancillary
            continue
        if obj.source is not None:  # no boomerang
            if obj.source.by_user:
                continue
        if obj.container is not None and obj.container.url_id in ('6p0120a5e990ac970c', '6a013487865036970c0134878650f2970c'):
            continue

        if note.verb == 'NewAsset':
            if getattr(obj, 'in_reply_to', None) is not None:
                note.verb = 'Comment'

                superobj = obj
                while getattr(superobj, 'in_reply_to', None) is not None:
                    if superobj.root.object_type == 'Comment':
                        superobj.root, superobj.in_reply_to = superobj.in_reply_to, superobj.root
                    superobj.in_reply_to = t.assets.get(superobj.in_reply_to.url_id)
                    superobj = superobj.in_reply_to

            elif getattr(obj, 'reblog_of', None) is not None:
                note.verb = 'Reblog'
            elif getattr(obj, 'reblog_of_url', None) is not None:
                note.verb = 'Reblog'

        if note.verb == 'NewAsset':
            okay_types = ['Post']
            if obj.container and obj.container.object_type == 'Group':
                okay_types.extend(['Photo', 'Audio', 'Video', 'Link'])
            if obj.object_type not in okay_types:
                continue

        # Yay, let's show this one!
        yield note

def object_from_url(url):
    # We're using an action endpoint anyway, so the oauth2.Client will work here.
    csr = oauth.Consumer(*settings.TYPEPAD_CONSUMER)
    token = oauth.Token(*settings.TYPEPAD_ANONYMOUS_ACCESS_TOKEN)
    cl = oauth.Client(csr, token)
    t = typd.TypePad(endpoint='https://api.typepad.com/', client=cl)

    urlparts = urlparse(url)
    try:
        result = t.domains.resolve_path(id=urlparts.netloc, path=urlparts.path)
    except typd.NotFound, exc:
        raise ValueError("TypePad could not resolve URL %s: %s" % (url, str(exc)))
    if result.is_full_match and result.asset:
        really_a_share, obj = object_for_typepad_object(result.asset)
        return obj

    raise ValueError("Could not identify TypePad asset for url %s" % url)

def poll_typepad(account):
    user = account.person.user
    if user is None:
        return

    # Get that TypePad user's notifications.
    t = typd.TypePad(endpoint='http://api.typepad.com/')
    notes = t.users.get_notifications(account.ident)

    for note in good_notes_for_notes(reversed(notes.entries), t):
        try:
            really_a_share, obj = object_for_typepad_object(note.object)

            root = obj
            while root.in_reply_to is not None:
                root = root.in_reply_to

            try:
                streamitem = UserStream.objects.get(user=user, obj=root)
            except UserStream.DoesNotExist:
                actor = account_for_typepad_user(note.actor)
                why_verb = {
                    'AddedFavorite': 'like',
                    'NewAsset': 'post' if obj is root and not really_a_share else 'share',
                    'Comment': 'reply',
                    'Reblog': 'share',
                }[note.verb]
                streamitem = UserStream.objects.create(user=user, obj=root,
                    time=note.published, why_account=actor, why_verb=why_verb)

            superobj = obj
            while superobj.in_reply_to is not None:
                UserReplyStream.objects.get_or_create(user=user, root=root, reply=superobj,
                    defaults={'root_time': streamitem.time, 'reply_time': superobj.time})
                superobj = superobj.in_reply_to

        except Exception, exc:
            log.exception(exc)
<|endoftext|>