<commit_before>"""
This Blueprint implements Blog related views.
"""
from datetime import datetime

from flask import Blueprint, render_template, abort, request, redirect, flash
from flask_login import login_required, current_user
from flask_wtf import FlaskForm
import wtforms
from wtforms.ext.sqlalchemy.fields import QuerySelectMultipleField, QuerySelectField

from models import Post, Tag, PostState
from extensions import db

blog = Blueprint('blog', __name__)

@blog.route('/')
def blog_index():
    """Render list of all posts. If user is logged in also display links for creating new post and editing posts."""
    all_posts = Post.query.all()
    all_posts = sorted(all_posts, key=lambda p: p.created_at, reverse=True)
    all_tags = Tag.query.all()
    all_tags = sorted(all_tags, key=lambda t: t.name)
    return render_template('blog_list.html', posts=all_posts, tags=all_tags)

@blog.route('/post/<post_url>')
def blog_post(post_url):
    """Render post of given url."""
    post = Post.query.filter_by(url=post_url).first()
    if post is None:
        abort(404)
    return render_template('blog_post.html', post=post)

@blog.route('/new')
@login_required
def new_post():
    """Create a new post in database and redirect to editing that post."""
    post = Post(title='Untitled', content='', author_id=current_user.id)
    db.session.add(post)
    db.session.commit()
    return redirect('/blog/edit/{}'.format(post.id))

class BlogEditForm(FlaskForm):
    """WTForm to edit a blog post."""
    title = wtforms.StringField('Title', validators=[wtforms.validators.DataRequired()])
    url = wtforms.StringField('Url', validators=[wtforms.validators.DataRequired()])
    content = wtforms.TextAreaField('Content', validators=[wtforms.validators.DataRequired()])
    state = QuerySelectField('State', query_factory=lambda: db.session.query(PostState))
    tags = QuerySelectMultipleField('Tags', query_factory=lambda: db.session.query(Tag), get_label=lambda tag: tag.name)
    submit = wtforms.SubmitField('Save')

@blog.route('/edit/<int:post_id>', methods=['GET', 'POST'])
@login_required
def edit_post(post_id):
    """GET shows post's content in a BlogEditForm. Post gets entered information via the Form into post and save."""
    post = Post.query.filter_by(id=post_id).first()
    form = BlogEditForm(request.form, post)
    if request.method == 'POST' and form.validate_on_submit():
        form_state = PostState.query.filter_by(id=int(request.form.get('state'))).first()
        is_published = post.state.name == 'draft' and form_state.name == 'published'
        is_edited = post.state.name == 'published' and form_state.name == 'published'
        is_drafted = post.state.name == 'published' and form_state.name == 'draft'
        if is_published:
            post.published_at = datetime.utcnow()
        elif is_edited:
            post.edited_at = datetime.utcnow()
        elif is_drafted:
            post.published_at = None
            post.edited_at = None
        form.populate_obj(post)
        db.session.commit()
        flash('(Post saved)')
        return redirect('/blog/edit/{}'.format(post_id))
    return render_template('blog_edit.html', form=form, post=post)

@blog.route('/delete/<int:post_id>', methods=['POST'])
@login_required
def delete_post(post_id):
    """Delete post with given ID from DB."""
    post = Post.query.filter_by(id=post_id).first()
    if post:
        db.session.delete(post)
        db.session.commit()
    return redirect('/blog')

@blog.route('/preview', methods=['POST'])
def preview_post():
    """View to return rendered marked into html to be used via an async request."""
    markdown_text = request.form.get('markdown', '')
    html = Post.render_markdown(markdown_text)
    return html

@blog.route('/tag/<tag_name>')
def blog_tag(tag_name):
    """Render the list of all posts with given tag."""
    tag = Tag.query.filter_by(name=tag_name).first()
    return render_template('blog_tag.html', tag=tag, posts=tag.posts)

@blog.route('/edit_tags')
@login_required
def edit_tags():
    all_tags = Tag.query.order_by(Tag.name).all()
    return render_template('blog_edit_tags.html', tags=all_tags)
<commit_msg>a timedelta due to time zone different is sent to template as tz_diff variable<commit_after>"""
This Blueprint implements Blog related views.
"""
from datetime import datetime, timedelta

from flask import Blueprint, render_template, abort, request, redirect, flash
from flask_login import login_required, current_user
from flask_wtf import FlaskForm
import wtforms
from wtforms.ext.sqlalchemy.fields import QuerySelectMultipleField, QuerySelectField

from models import Post, Tag, PostState
from extensions import db

blog = Blueprint('blog', __name__)

@blog.route('/')
def blog_index():
    """Render list of all posts. If user is logged in also display links for creating new post and editing posts."""
    all_posts = Post.query.all()
    all_posts = sorted(all_posts, key=lambda p: p.created_at, reverse=True)
    all_tags = Tag.query.all()
    all_tags = sorted(all_tags, key=lambda t: t.name)
    return render_template('blog_list.html', posts=all_posts, tags=all_tags)

@blog.route('/post/<post_url>')
def blog_post(post_url):
    """Render post of given url."""
    post = Post.query.filter_by(url=post_url).first()
    timezone_diff = timedelta(hours=post.timezone)
    if post is None:
        abort(404)
    return render_template('blog_post.html', post=post, tz_diff=timezone_diff)

@blog.route('/new')
@login_required
def new_post():
    """Create a new post in database and redirect to editing that post."""
    post = Post(title='Untitled', content='', author_id=current_user.id)
    db.session.add(post)
    db.session.commit()
    return redirect('/blog/edit/{}'.format(post.id))

class BlogEditForm(FlaskForm):
    """WTForm to edit a blog post."""
    title = wtforms.StringField('Title', validators=[wtforms.validators.DataRequired()])
    url = wtforms.StringField('Url', validators=[wtforms.validators.DataRequired()])
    content = wtforms.TextAreaField('Content', validators=[wtforms.validators.DataRequired()])
    state = QuerySelectField('State', query_factory=lambda: db.session.query(PostState))
    tags = QuerySelectMultipleField('Tags', query_factory=lambda: db.session.query(Tag), get_label=lambda tag: tag.name)
    submit = wtforms.SubmitField('Save')

@blog.route('/edit/<int:post_id>', methods=['GET', 'POST'])
@login_required
def edit_post(post_id):
    """GET shows post's content in a BlogEditForm. Post gets entered information via the Form into post and save."""
    post = Post.query.filter_by(id=post_id).first()
    form = BlogEditForm(request.form, post)
    if request.method == 'POST' and form.validate_on_submit():
        form_state = PostState.query.filter_by(id=int(request.form.get('state'))).first()
        is_published = post.state.name == 'draft' and form_state.name == 'published'
        is_edited = post.state.name == 'published' and form_state.name == 'published'
        is_drafted = post.state.name == 'published' and form_state.name == 'draft'
        if is_published:
            post.published_at = datetime.utcnow()
        elif is_edited:
            post.edited_at = datetime.utcnow()
        elif is_drafted:
            post.published_at = None
            post.edited_at = None
        form.populate_obj(post)
        db.session.commit()
        flash('(Post saved)')
        return redirect('/blog/edit/{}'.format(post_id))
    return render_template('blog_edit.html', form=form, post=post)

@blog.route('/delete/<int:post_id>', methods=['POST'])
@login_required
def delete_post(post_id):
    """Delete post with given ID from DB."""
    post = Post.query.filter_by(id=post_id).first()
    if post:
        db.session.delete(post)
        db.session.commit()
    return redirect('/blog')

@blog.route('/preview', methods=['POST'])
def preview_post():
    """View to return rendered marked into html to be used via an async request."""
    markdown_text = request.form.get('markdown', '')
    html = Post.render_markdown(markdown_text)
    return html

@blog.route('/tag/<tag_name>')
def blog_tag(tag_name):
    """Render the list of all posts with given tag."""
    tag = Tag.query.filter_by(name=tag_name).first()
    return render_template('blog_tag.html', tag=tag, posts=tag.posts)

@blog.route('/edit_tags')
@login_required
def edit_tags():
    all_tags = Tag.query.order_by(Tag.name).all()
    return render_template('blog_edit_tags.html', tags=all_tags)
<|endoftext|>


<commit_before># Copyright (C) 2015 UCSC Computational Genomics Lab
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""A demonstration of toil. Sorts the lines of a file into ascending order by doing a parallel merge sort.
"""
from __future__ import absolute_import
from argparse import ArgumentParser
import os
import random
import logging
import shutil
from bd2k.util.humanize import human2bytes

from toil.job import Job
from toil.test.sort.lib import merge, sort, copySubRangeOfFile, getMidPoint

sortMemory = human2bytes('1000M')

def setup(job, inputFile, N, downCheckpoints):
    """Sets up the sort.
    """
    #Write the input file to the file store
    inputFileStoreID = job.fileStore.writeGlobalFile(inputFile, True)
    job.fileStore.logToMaster(" Starting the merge sort ")
    job.addFollowOnJobFn(cleanup, job.addChildJobFn(down,
                        inputFileStoreID, N, downCheckpoints,
                        checkpoint=downCheckpoints).rv(), inputFile)

def down(job, inputFileStoreID, N, downCheckpoints, memory=sortMemory):
    """Input is a file and a range into that file to sort and an output location in which
    to write the sorted file.
    If the range is larger than a threshold N the range is divided recursively and
    a follow on job is then created which merges back the results else
    the file is sorted and placed in the output.
    """
    #Read the file
    inputFile = job.fileStore.readGlobalFile(inputFileStoreID, cache=False)
    length = os.path.getsize(inputFile)
    if length > N:
        #We will subdivide the file
        job.fileStore.logToMaster( "Splitting file: %s of size: %s"
                                      % (inputFileStoreID, length), level=logging.CRITICAL )
        #Split the file into two copies
        midPoint = getMidPoint(inputFile, 0, length)
        t1 = job.fileStore.getLocalTempFile()
        with open(t1, 'w') as fH:
            copySubRangeOfFile(inputFile, 0, midPoint+1, fH)
        t2 = job.fileStore.getLocalTempFile()
        with open(t2, 'w') as fH:
            copySubRangeOfFile(inputFile, midPoint+1, length, fH)
        #Call down recursively
        return job.addFollowOnJobFn(up,
            job.addChildJobFn(down, job.fileStore.writeGlobalFile(t1), N,
                              downCheckpoints, checkpoint=downCheckpoints, memory=sortMemory).rv(),
            job.addChildJobFn(down, job.fileStore.writeGlobalFile(t2), N,
                              downCheckpoints, checkpoint=downCheckpoints, memory=sortMemory).rv()).rv()
    else:
        #We can sort this bit of the file
        job.fileStore.logToMaster( "Sorting file: %s of size: %s"
                                      % (inputFileStoreID, length), level=logging.CRITICAL )
        #Sort the copy and write back to the fileStore
        shutil.copyfile(inputFile, inputFile + '.sort')
        sort(inputFile + '.sort')
        return job.fileStore.writeGlobalFile(inputFile + '.sort')

def up(job, inputFileID1, inputFileID2, memory=sortMemory):
    """Merges the two files and places them in the output.
    """
    with job.fileStore.writeGlobalFileStream() as (fileHandle, outputFileStoreID):
        with job.fileStore.readGlobalFileStream( inputFileID1 ) as inputFileHandle1:
            with job.fileStore.readGlobalFileStream( inputFileID2 ) as inputFileHandle2:
                merge(inputFileHandle1, inputFileHandle2, fileHandle)
                job.fileStore.logToMaster( "Merging %s and %s to %s"
                                       % (inputFileID1, inputFileID2, outputFileStoreID) )
        #Cleanup up the input files - these deletes will occur after the completion is successful.
        job.fileStore.deleteGlobalFile(inputFileID1)
        job.fileStore.deleteGlobalFile(inputFileID2)
        return outputFileStoreID

def cleanup(job, tempOutputFileStoreID, outputFile, cores=1, memory=sortMemory, disk="3G"):
    """Copies back the temporary file to input once we've successfully sorted the temporary file.
    """
    fileName = job.fileStore.readGlobalFile(tempOutputFileStoreID)
    shutil.copyfile(fileName, outputFile)
    job.fileStore.logToMaster("Finished copying sorted file to output: %s" % outputFile)

def main():
    parser = ArgumentParser()
    Job.Runner.addToilOptions(parser)

    parser.add_argument("--fileToSort", dest="fileToSort",
                      help="The file you wish to sort")

    parser.add_argument("--N", dest="N",
                      help="The threshold below which a serial sort function is"
                      "used to sort file. All lines must of length less than or equal to N or program will fail",
                      default=10000)

    options = parser.parse_args()

    if options.fileToSort is None:
        raise RuntimeError("No file to sort given")

    if not os.path.exists(options.fileToSort):
        raise RuntimeError("File to sort does not exist: %s" % options.fileToSort)

    if int(options.N) <= 0:
        raise RuntimeError("Invalid value of N: %s" % options.N)

    #Now we are ready to run
    Job.Runner.startToil(Job.wrapJobFn(setup, options.fileToSort, int(options.N), False,
                                       memory=sortMemory), options)

if __name__ == '__main__':
    main()
<commit_msg>Revert insignificant whitespace changes<commit_after># Copyright (C) 2015 UCSC Computational Genomics Lab
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""A demonstration of toil. Sorts the lines of a file into ascending order by doing a parallel merge sort.
"""
from __future__ import absolute_import
from argparse import ArgumentParser
import os
import random
import logging
import shutil
from bd2k.util.humanize import human2bytes

from toil.job import Job
from toil.test.sort.lib import merge, sort, copySubRangeOfFile, getMidPoint

sortMemory = human2bytes('1000M')

def setup(job, inputFile, N, downCheckpoints):
    """Sets up the sort.
    """
    #Write the input file to the file store
    inputFileStoreID = job.fileStore.writeGlobalFile(inputFile, True)
    job.fileStore.logToMaster(" Starting the merge sort ")
    job.addFollowOnJobFn(cleanup, job.addChildJobFn(down,
                        inputFileStoreID, N, downCheckpoints,
                        checkpoint=downCheckpoints).rv(), inputFile)

def down(job, inputFileStoreID, N, downCheckpoints, memory=sortMemory):
    """Input is a file and a range into that file to sort and an output location in which
    to write the sorted file.
    If the range is larger than a threshold N the range is divided recursively and
    a follow on job is then created which merges back the results else
    the file is sorted and placed in the output.
    """
    #Read the file
    inputFile = job.fileStore.readGlobalFile(inputFileStoreID, cache=False)
    length = os.path.getsize(inputFile)
    if length > N:
        #We will subdivide the file
        job.fileStore.logToMaster( "Splitting file: %s of size: %s"
                                      % (inputFileStoreID, length), level=logging.CRITICAL )
        #Split the file into two copies
        midPoint = getMidPoint(inputFile, 0, length)
        t1 = job.fileStore.getLocalTempFile()
        with open(t1, 'w') as fH:
            copySubRangeOfFile(inputFile, 0, midPoint+1, fH)
        t2 = job.fileStore.getLocalTempFile()
        with open(t2, 'w') as fH:
            copySubRangeOfFile(inputFile, midPoint+1, length, fH)
        #Call down recursively
        return job.addFollowOnJobFn(up,
            job.addChildJobFn(down, job.fileStore.writeGlobalFile(t1), N,
                              downCheckpoints, checkpoint=downCheckpoints, memory=sortMemory).rv(),
            job.addChildJobFn(down, job.fileStore.writeGlobalFile(t2), N,
                              downCheckpoints, checkpoint=downCheckpoints, memory=sortMemory).rv()).rv()
    else:
        #We can sort this bit of the file
        job.fileStore.logToMaster( "Sorting file: %s of size: %s"
                                      % (inputFileStoreID, length), level=logging.CRITICAL )
        #Sort the copy and write back to the fileStore
        shutil.copyfile(inputFile, inputFile + '.sort')
        sort(inputFile + '.sort')
        return job.fileStore.writeGlobalFile(inputFile + '.sort')

def up(job, inputFileID1, inputFileID2, memory=sortMemory):
    """Merges the two files and places them in the output.
    """
    with job.fileStore.writeGlobalFileStream() as (fileHandle, outputFileStoreID):
        with job.fileStore.readGlobalFileStream( inputFileID1 ) as inputFileHandle1:
            with job.fileStore.readGlobalFileStream( inputFileID2 ) as inputFileHandle2:
                merge(inputFileHandle1, inputFileHandle2, fileHandle)
                job.fileStore.logToMaster( "Merging %s and %s to %s"
                                       % (inputFileID1, inputFileID2, outputFileStoreID) )
        #Cleanup up the input files - these deletes will occur after the completion is successful.
        job.fileStore.deleteGlobalFile(inputFileID1)
        job.fileStore.deleteGlobalFile(inputFileID2)
        return outputFileStoreID

def cleanup(job, tempOutputFileStoreID, outputFile, cores=1, memory=sortMemory, disk="3G"):
    """Copies back the temporary file to input once we've successfully sorted the temporary file.
    """
    fileName = job.fileStore.readGlobalFile(tempOutputFileStoreID)
    shutil.copyfile(fileName, outputFile)
    job.fileStore.logToMaster("Finished copying sorted file to output: %s" % outputFile)

def main():
    parser = ArgumentParser()
    Job.Runner.addToilOptions(parser)

    parser.add_argument("--fileToSort", dest="fileToSort",
                      help="The file you wish to sort")

    parser.add_argument("--N", dest="N",
                      help="The threshold below which a serial sort function is"
                      "used to sort file. All lines must of length less than or equal to N or program will fail",
                      default=10000)

    options = parser.parse_args()

    if options.fileToSort is None:
        raise RuntimeError("No file to sort given")

    if not os.path.exists(options.fileToSort):
        raise RuntimeError("File to sort does not exist: %s" % options.fileToSort)

    if int(options.N) <= 0:
        raise RuntimeError("Invalid value of N: %s" % options.N)

    #Now we are ready to run
    Job.Runner.startToil(Job.wrapJobFn(setup, options.fileToSort, int(options.N), False,
                                       memory=sortMemory), options)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from default.queue import tasks
from pyramid.response import Response
from pyramid.view import view_config

from sqlalchemy.exc import DBAPIError

@view_config(route_name='home', renderer='default:templates/mytemplate.mako')
def my_view(request):
    tasks.add.delay(5,5)
    return {'project': 'default'}

conn_err_msg = """\
Pyramid is having a problem using your SQL database.  The problem
might be caused by one of the following things:

1.  You may need to run the "initialize_default_db" script
    to initialize your database tables.  Check your virtual
    environment's "bin" directory for this script and try to run it.

2.  Your database server may not be running.  Check that the
    database server referred to by the "sqlalchemy.url" setting in
    your "development.ini" file is running.

After you fix the problem, please restart the Pyramid application to
try it again.
"""

<commit_msg>insert placeholder for home.py<commit_after>from ~~~PROJNAME~~~.queue import tasks
from pyramid.response import Response
from pyramid.view import view_config

from sqlalchemy.exc import DBAPIError

@view_config(route_name='home', renderer='~~~PROJNAME~~~:templates/mytemplate.mako')
def my_view(request):
    tasks.add.delay(5,5)
    return {'project': '~~~PROJNAME~~~'}
<|endoftext|>


<commit_before>import numpy as np, sys
from rpy2 import robjects
import rpy2.robjects.numpy2ri
rpy2.robjects.numpy2ri.activate()

import selection.randomized.lasso as L; reload(L)
from selection.randomized.lasso import lasso, highdim

def glmnet_lasso(X, y, lambda_val):
    robjects.r('''
                glmnet_LASSO = function(X,y,lambda){
                y = as.matrix(y)
                X = as.matrix(X)
                lam = as.matrix(lambda)[1,1]
                n = nrow(X)
                fit = glmnet(X, y, standardize=TRUE, intercept=FALSE)
                estimate = coef(fit, s=lam)[-1]
                return(list(estimate = estimate))
                }''')

    lambda_R = robjects.globalenv['glmnet_LASSO']
    n, p = X.shape
    r_X = robjects.r.matrix(X, nrow=n, ncol=p)
    r_y = robjects.r.matrix(y, nrow=n, ncol=1)
    r_lam = robjects.r.matrix(lambda_val, nrow=1, ncol=1)
    estimate = np.array(lambda_R(r_X, r_y, r_lam).rx2('estimate'))
    return estimate

def sim_xy(n, p, nval, rho=0, s=5, beta_type=2, snr=1):
    robjects.r('''
    library(bestsubset)
    sim_xy = bestsubset::sim.xy
    ''')

    r_simulate = robjects.globalenv['sim_xy']
    sim = r_simulate(n, p, nval, rho, s, beta_type, snr)
    X = np.array(sim.rx2('x'))
    y = np.array(sim.rx2('y'))
    X_val = np.array(sim.rx2('xval'))
    y_val = np.array(sim.rx2('yval'))
    Sigma = np.array(sim.rx2('Sigma'))
    beta = np.array(sim.rx2('beta'))
    sigma = np.array(sim.rx2('sigma'))

    return X, y, X_val, y_val, Sigma, beta, sigma

def tuned_lasso(X, y, X_val,y_val):
    robjects.r('''
        tuned_lasso_estimator = function(X,Y,X.val,Y.val){
        Y = as.matrix(Y)
        X = as.matrix(X)
        Y.val = as.vector(Y.val)
        X.val = as.matrix(X.val)
        rel.LASSO = lasso(X,Y,intercept=TRUE, nrelax=10, nlam=50, standardize=TRUE)
        LASSO = lasso(X,Y,intercept=TRUE,nlam=50, standardize=TRUE)
        beta.hat.rellasso = as.matrix(coef(rel.LASSO))
        beta.hat.lasso = as.matrix(coef(LASSO))
        min.lam = min(rel.LASSO$lambda)
        max.lam = max(rel.LASSO$lambda)
        #print(paste("max and min values of lambda", max.lam, min.lam))

        lam.seq = exp(seq(log(max.lam),log(min.lam),length=rel.LASSO$nlambda))
        muhat.val.rellasso = as.matrix(predict(rel.LASSO, X.val))
        muhat.val.lasso = as.matrix(predict(LASSO, X.val))
        err.val.rellasso = colMeans((muhat.val.rellasso - Y.val)^2)
        err.val.lasso = colMeans((muhat.val.lasso - Y.val)^2)

        opt_lam = ceiling(which.min(err.val.rellasso)/10)
        lambda.tuned.rellasso = lam.seq[opt_lam]
        lambda.tuned.lasso = lam.seq[which.min(err.val.lasso)]

        fit = glmnet(X, Y, standardize=TRUE, intercept=TRUE)
        estimate.tuned = coef(fit, s=lambda.tuned.lasso)[-1]

        #print(paste("compare estimates", max(abs(estimate.tuned-(beta.hat.lasso[,which.min(err.val.lasso)])[-1])),
        length(which(estimate.tuned!=0)), length(which((beta.hat.lasso[,which.min(err.val.lasso)])[-1]!=0))))

        return(list(beta.hat.rellasso = (beta.hat.rellasso[,which.min(err.val.rellasso)])[-1],
        beta.hat.lasso = (beta.hat.lasso[,which.min(err.val.lasso)])[-1],
        lambda.tuned.rellasso = lambda.tuned.rellasso, lambda.tuned.lasso= lambda.tuned.lasso,
        lambda.seq = lam.seq))
        }''')

    r_lasso = robjects.globalenv['tuned_lasso_estimator']

    n, p = X.shape
    nval, _ = X_val.shape
    r_X = robjects.r.matrix(X, nrow=n, ncol=p)
    r_y = robjects.r.matrix(y, nrow=n, ncol=1)
    r_X_val = robjects.r.matrix(X_val, nrow=nval, ncol=p)
    r_y_val = robjects.r.matrix(y_val, nrow=nval, ncol=1)

    tuned_est = r_lasso(r_X, r_y, r_X_val, r_y_val)
    estimator_rellasso = np.array(tuned_est.rx2('beta.hat.rellasso'))
    estimator_lasso = np.array(tuned_est.rx2('beta.hat.lasso'))
    lam_tuned_rellasso = np.array(tuned_est.rx2('lambda.tuned.rellasso'))
    lam_tuned_lasso = np.array(tuned_est.rx2('lambda.tuned.lasso'))
    lam_seq = np.array(tuned_est.rx2('lambda.seq'))
    return estimator_rellasso, estimator_lasso, lam_tuned_rellasso, lam_tuned_lasso, lam_seq

def relative_risk(est, truth, Sigma):

    return (est-truth).T.dot(Sigma).dot(est-truth)/truth.T.dot(Sigma).dot(truth)

def comparison_risk_inference(n=500, p=100, nval=500, rho=0.35, s=5, beta_type=2, snr=0.2,
                              randomizer_scale=np.sqrt(0.25), target = "selected",
                              full_dispersion = True):

    X, y, X_val, y_val, Sigma, beta, sigma = sim_xy(n=n, p=p, nval=nval, rho=rho,
                                                    s=s, beta_type=beta_type, snr=snr)
    rel_LASSO, est_LASSO, lam_tuned_rellasso, lam_tuned_lasso, lam_seq = tuned_lasso(X, y, X_val, y_val)
    active_nonrand = (est_LASSO != 0)
    nactive_nonrand = active_nonrand.sum()
    true_mean = X.dot(beta)

    _X = X
    X -= X.mean(0)[None, :]
    X /= (X.std(0)[None, :] * np.sqrt(n))
    X_val -= X_val.mean(0)[None, :]
    X_val /= (X_val.std(0)[None, :] * np.sqrt(nval))

    _y = y
    y = y - y.mean()
    y_val = y_val - y_val.mean()

    dispersion = None
    if full_dispersion:
        dispersion = np.linalg.norm(y - X.dot(np.linalg.pinv(X).dot(y))) ** 2 / (n - p)

    sigma_ = np.std(y)
    LASSO_py = lasso.gaussian(X, y, np.asscalar((sigma**2.)*lam_tuned_lasso))
    soln = LASSO_py.fit()
    #print("compare solns", soln, est_LASSO)
    active_LASSO = (soln != 0)
    nactive_LASSO = active_LASSO.sum()

    # LASSO_rand0 = highdim.gaussian(X,
    #                                y,
    #                                np.asscalar((sigma_**2)*lam_tuned_lasso),
    #                                randomizer_scale=0.00000001)
    # signs_rand0 = LASSO_rand0.fit()

    #glm_LASSO = glmnet_lasso(X, y, np.asscalar(lam_tuned_lasso))

    const = highdim.gaussian
    lam_seq = sigma_* np.linspace(0.25, 2.75, num=100) * \
              np.mean(np.fabs(np.dot(X.T, np.random.standard_normal((n, 2000)))).max(0))
    err = np.zeros(100)
    for k in range(100):
        W = lam_seq[k]
        conv = const(X,
                     y,
                     W,
                     randomizer_scale=randomizer_scale * sigma_)
        signs = conv.fit()
        nonzero = signs != 0
        estimate, _, _, _, _, _ = conv.selective_MLE(target=target, dispersion=dispersion)

        full_estimate = np.zeros(p)
        full_estimate[nonzero] = estimate
        err[k] = np.mean((y_val - X_val.dot(full_estimate)) ** 2.)

    lam = lam_seq[np.argmin(err)]
    sys.stderr.write("lambda from tuned relaxed LASSO " + str((sigma_**2)*lam_tuned_lasso) + "\n")
    sys.stderr.write("lambda from randomized LASSO " + str(lam) + "\n")

    randomized_lasso = const(X,
                             y,
                             lam,
                             randomizer_scale=randomizer_scale * sigma_)

    signs = randomized_lasso.fit()
    nonzero = signs != 0
    sys.stderr.write("active variables selected by tuned LASSO " + str(nactive_nonrand) + "\n")
    sys.stderr.write("active variables selected by LASSO in python " + str(nactive_LASSO)+ "\n")
    #sys.stderr.write("recall glmnet at tuned lambda " + str((glm_LASSO!=0).sum()) + "\n")
    sys.stderr.write("active variables selected by randomized LASSO " + str(nonzero.sum()) + "\n")

    sel_MLE = np.zeros(p)
    estimate, _, _, pval, intervals, ind_unbiased_estimator = randomized_lasso.selective_MLE(target=target, dispersion=dispersion)
    sel_MLE[nonzero] = estimate / np.sqrt(n)
    ind_estimator = np.zeros(p)
    ind_estimator[nonzero] = ind_unbiased_estimator / np.sqrt(n)

    return relative_risk(sel_MLE, beta, Sigma),\
           relative_risk(ind_estimator, beta, Sigma),\
           relative_risk(randomized_lasso.initial_soln / np.sqrt(n), beta, Sigma),\
           relative_risk(randomized_lasso._beta_full / np.sqrt(n), beta, Sigma), \
           relative_risk(rel_LASSO, beta, Sigma),\
           relative_risk(est_LASSO, beta, Sigma)

if __name__ == "__main__":

    ndraw = 50
    bias = 0.
    risk_selMLE = 0.
    risk_indest = 0.
    risk_LASSO_rand = 0.
    risk_relLASSO_rand = 0.

    risk_relLASSO_nonrand = 0.
    risk_LASSO_nonrand = 0.

    for i in range(ndraw):
        output = comparison_risk_inference(n=500, p=100, nval=500, rho=0.35, s=5, beta_type=2, snr=0.2,
                                           randomizer_scale=np.sqrt(0.25), target="selected", full_dispersion=True)

        risk_selMLE += output[0]
        risk_indest += output[1]
        risk_LASSO_rand += output[2]
        risk_relLASSO_rand += output[3]
        risk_relLASSO_nonrand += output[4]
        risk_LASSO_nonrand += output[5]

        sys.stderr.write("overall selMLE risk " + str(risk_selMLE / float(i + 1)) + "\n")
        sys.stderr.write("overall indep est risk " + str(risk_indest / float(i + 1)) + "\n")
        sys.stderr.write("overall randomized LASSO est risk " + str(risk_LASSO_rand / float(i + 1)) + "\n")
        sys.stderr.write("overall relaxed rand LASSO est risk " + str(risk_relLASSO_rand / float(i + 1)) + "\n"+ "\n")

        sys.stderr.write("overall relLASSO risk " + str(risk_relLASSO_nonrand / float(i + 1)) + "\n")
        sys.stderr.write("overall LASSO risk " + str(risk_LASSO_nonrand / float(i + 1)) + "\n" + "\n")

        sys.stderr.write("iteration completed" + str(i+1) + "\n")

<commit_msg>removed redundant print<commit_after>import numpy as np, sys
from rpy2 import robjects
import rpy2.robjects.numpy2ri
rpy2.robjects.numpy2ri.activate()

import selection.randomized.lasso as L; reload(L)
from selection.randomized.lasso import lasso, highdim

def glmnet_lasso(X, y, lambda_val):
    robjects.r('''
                glmnet_LASSO = function(X,y,lambda){
                y = as.matrix(y)
                X = as.matrix(X)
                lam = as.matrix(lambda)[1,1]
                n = nrow(X)
                fit = glmnet(X, y, standardize=TRUE, intercept=FALSE)
                estimate = coef(fit, s=lam)[-1]
                return(list(estimate = estimate))
                }''')

    lambda_R = robjects.globalenv['glmnet_LASSO']
    n, p = X.shape
    r_X = robjects.r.matrix(X, nrow=n, ncol=p)
    r_y = robjects.r.matrix(y, nrow=n, ncol=1)
    r_lam = robjects.r.matrix(lambda_val, nrow=1, ncol=1)
    estimate = np.array(lambda_R(r_X, r_y, r_lam).rx2('estimate'))
    return estimate

def sim_xy(n, p, nval, rho=0, s=5, beta_type=2, snr=1):
    robjects.r('''
    library(bestsubset)
    sim_xy = bestsubset::sim.xy
    ''')

    r_simulate = robjects.globalenv['sim_xy']
    sim = r_simulate(n, p, nval, rho, s, beta_type, snr)
    X = np.array(sim.rx2('x'))
    y = np.array(sim.rx2('y'))
    X_val = np.array(sim.rx2('xval'))
    y_val = np.array(sim.rx2('yval'))
    Sigma = np.array(sim.rx2('Sigma'))
    beta = np.array(sim.rx2('beta'))
    sigma = np.array(sim.rx2('sigma'))

    return X, y, X_val, y_val, Sigma, beta, sigma

def tuned_lasso(X, y, X_val,y_val):
    robjects.r('''
        tuned_lasso_estimator = function(X,Y,X.val,Y.val){
        Y = as.matrix(Y)
        X = as.matrix(X)
        Y.val = as.vector(Y.val)
        X.val = as.matrix(X.val)
        rel.LASSO = lasso(X,Y,intercept=TRUE, nrelax=10, nlam=50, standardize=TRUE)
        LASSO = lasso(X,Y,intercept=TRUE,nlam=50, standardize=TRUE)
        beta.hat.rellasso = as.matrix(coef(rel.LASSO))
        beta.hat.lasso = as.matrix(coef(LASSO))
        min.lam = min(rel.LASSO$lambda)
        max.lam = max(rel.LASSO$lambda)
        #print(paste("max and min values of lambda", max.lam, min.lam))

        lam.seq = exp(seq(log(max.lam),log(min.lam),length=rel.LASSO$nlambda))
        muhat.val.rellasso = as.matrix(predict(rel.LASSO, X.val))
        muhat.val.lasso = as.matrix(predict(LASSO, X.val))
        err.val.rellasso = colMeans((muhat.val.rellasso - Y.val)^2)
        err.val.lasso = colMeans((muhat.val.lasso - Y.val)^2)

        opt_lam = ceiling(which.min(err.val.rellasso)/10)
        lambda.tuned.rellasso = lam.seq[opt_lam]
        lambda.tuned.lasso = lam.seq[which.min(err.val.lasso)]

        fit = glmnet(X, Y, standardize=TRUE, intercept=TRUE)
        estimate.tuned = coef(fit, s=lambda.tuned.lasso)[-1]

        #print(paste("compare estimates", max(abs(estimate.tuned-(beta.hat.lasso[,which.min(err.val.lasso)])[-1])),
        #length(which(estimate.tuned!=0)), length(which((beta.hat.lasso[,which.min(err.val.lasso)])[-1]!=0))))

        return(list(beta.hat.rellasso = (beta.hat.rellasso[,which.min(err.val.rellasso)])[-1],
        beta.hat.lasso = (beta.hat.lasso[,which.min(err.val.lasso)])[-1],
        lambda.tuned.rellasso = lambda.tuned.rellasso, lambda.tuned.lasso= lambda.tuned.lasso,
        lambda.seq = lam.seq))
        }''')

    r_lasso = robjects.globalenv['tuned_lasso_estimator']

    n, p = X.shape
    nval, _ = X_val.shape
    r_X = robjects.r.matrix(X, nrow=n, ncol=p)
    r_y = robjects.r.matrix(y, nrow=n, ncol=1)
    r_X_val = robjects.r.matrix(X_val, nrow=nval, ncol=p)
    r_y_val = robjects.r.matrix(y_val, nrow=nval, ncol=1)

    tuned_est = r_lasso(r_X, r_y, r_X_val, r_y_val)
    estimator_rellasso = np.array(tuned_est.rx2('beta.hat.rellasso'))
    estimator_lasso = np.array(tuned_est.rx2('beta.hat.lasso'))
    lam_tuned_rellasso = np.array(tuned_est.rx2('lambda.tuned.rellasso'))
    lam_tuned_lasso = np.array(tuned_est.rx2('lambda.tuned.lasso'))
    lam_seq = np.array(tuned_est.rx2('lambda.seq'))
    return estimator_rellasso, estimator_lasso, lam_tuned_rellasso, lam_tuned_lasso, lam_seq

def relative_risk(est, truth, Sigma):

    return (est-truth).T.dot(Sigma).dot(est-truth)/truth.T.dot(Sigma).dot(truth)

def comparison_risk_inference(n=500, p=100, nval=500, rho=0.35, s=5, beta_type=2, snr=0.2,
                              randomizer_scale=np.sqrt(0.25), target = "selected",
                              full_dispersion = True):

    X, y, X_val, y_val, Sigma, beta, sigma = sim_xy(n=n, p=p, nval=nval, rho=rho,
                                                    s=s, beta_type=beta_type, snr=snr)
    rel_LASSO, est_LASSO, lam_tuned_rellasso, lam_tuned_lasso, lam_seq = tuned_lasso(X, y, X_val, y_val)
    active_nonrand = (est_LASSO != 0)
    nactive_nonrand = active_nonrand.sum()
    true_mean = X.dot(beta)

    _X = X
    X -= X.mean(0)[None, :]
    X /= (X.std(0)[None, :] * np.sqrt(n))
    X_val -= X_val.mean(0)[None, :]
    X_val /= (X_val.std(0)[None, :] * np.sqrt(nval))

    _y = y
    y = y - y.mean()
    y_val = y_val - y_val.mean()

    dispersion = None
    if full_dispersion:
        dispersion = np.linalg.norm(y - X.dot(np.linalg.pinv(X).dot(y))) ** 2 / (n - p)

    sigma_ = np.std(y)
    LASSO_py = lasso.gaussian(X, y, np.asscalar((sigma**2.)*lam_tuned_lasso))
    soln = LASSO_py.fit()
    #print("compare solns", soln, est_LASSO)
    active_LASSO = (soln != 0)
    nactive_LASSO = active_LASSO.sum()

    # LASSO_rand0 = highdim.gaussian(X,
    #                                y,
    #                                np.asscalar((sigma_**2)*lam_tuned_lasso),
    #                                randomizer_scale=0.00000001)
    # signs_rand0 = LASSO_rand0.fit()

    #glm_LASSO = glmnet_lasso(X, y, np.asscalar(lam_tuned_lasso))

    const = highdim.gaussian
    lam_seq = sigma_* np.linspace(0.25, 2.75, num=100) * \
              np.mean(np.fabs(np.dot(X.T, np.random.standard_normal((n, 2000)))).max(0))
    err = np.zeros(100)
    for k in range(100):
        W = lam_seq[k]
        conv = const(X,
                     y,
                     W,
                     randomizer_scale=randomizer_scale * sigma_)
        signs = conv.fit()
        nonzero = signs != 0
        estimate, _, _, _, _, _ = conv.selective_MLE(target=target, dispersion=dispersion)

        full_estimate = np.zeros(p)
        full_estimate[nonzero] = estimate
        err[k] = np.mean((y_val - X_val.dot(full_estimate)) ** 2.)

    lam = lam_seq[np.argmin(err)]
    sys.stderr.write("lambda from tuned relaxed LASSO " + str((sigma_**2)*lam_tuned_lasso) + "\n")
    sys.stderr.write("lambda from randomized LASSO " + str(lam) + "\n")

    randomized_lasso = const(X,
                             y,
                             lam,
                             randomizer_scale=randomizer_scale * sigma_)

    signs = randomized_lasso.fit()
    nonzero = signs != 0
    sys.stderr.write("active variables selected by tuned LASSO " + str(nactive_nonrand) + "\n")
    sys.stderr.write("active variables selected by LASSO in python " + str(nactive_LASSO)+ "\n")
    #sys.stderr.write("recall glmnet at tuned lambda " + str((glm_LASSO!=0).sum()) + "\n")
    sys.stderr.write("active variables selected by randomized LASSO " + str(nonzero.sum()) + "\n")

    sel_MLE = np.zeros(p)
    estimate, _, _, pval, intervals, ind_unbiased_estimator = randomized_lasso.selective_MLE(target=target, dispersion=dispersion)
    sel_MLE[nonzero] = estimate / np.sqrt(n)
    ind_estimator = np.zeros(p)
    ind_estimator[nonzero] = ind_unbiased_estimator / np.sqrt(n)

    return relative_risk(sel_MLE, beta, Sigma),\
           relative_risk(ind_estimator, beta, Sigma),\
           relative_risk(randomized_lasso.initial_soln / np.sqrt(n), beta, Sigma),\
           relative_risk(randomized_lasso._beta_full / np.sqrt(n), beta, Sigma), \
           relative_risk(rel_LASSO, beta, Sigma),\
           relative_risk(est_LASSO, beta, Sigma)

if __name__ == "__main__":

    ndraw = 50
    bias = 0.
    risk_selMLE = 0.
    risk_indest = 0.
    risk_LASSO_rand = 0.
    risk_relLASSO_rand = 0.

    risk_relLASSO_nonrand = 0.
    risk_LASSO_nonrand = 0.

    for i in range(ndraw):
        output = comparison_risk_inference(n=500, p=100, nval=500, rho=0.35, s=5, beta_type=3, snr=0.2,
                                           randomizer_scale=np.sqrt(0.25), target="selected", full_dispersion=True)

        risk_selMLE += output[0]
        risk_indest += output[1]
        risk_LASSO_rand += output[2]
        risk_relLASSO_rand += output[3]
        risk_relLASSO_nonrand += output[4]
        risk_LASSO_nonrand += output[5]

        sys.stderr.write("overall selMLE risk " + str(risk_selMLE / float(i + 1)) + "\n")
        sys.stderr.write("overall indep est risk " + str(risk_indest / float(i + 1)) + "\n")
        sys.stderr.write("overall randomized LASSO est risk " + str(risk_LASSO_rand / float(i + 1)) + "\n")
        sys.stderr.write("overall relaxed rand LASSO est risk " + str(risk_relLASSO_rand / float(i + 1)) + "\n"+ "\n")

        sys.stderr.write("overall relLASSO risk " + str(risk_relLASSO_nonrand / float(i + 1)) + "\n")
        sys.stderr.write("overall LASSO risk " + str(risk_LASSO_nonrand / float(i + 1)) + "\n" + "\n")

        sys.stderr.write("iteration completed" + str(i+1) + "\n")

<|endoftext|>


<commit_before>
import logging
from datetime import datetime, timedelta

from dateutil import parser

from core.errors import ObservableValidationError
from core.feed import Feed
from core.observables import Hash

TYPE_DICT = {
    "MITM": ['mitm'],
    "C&C": ["c2"],
    "distribution": ["payload_delivery"],
    "sinkhole": ["sinkhole"],
}

class SSLBlackListCerts(Feed):

    default_values = {
        "frequency": timedelta(hours=24),
        "name": "SSLBlackListCerts",
        "source": "https://sslbl.abuse.ch/blacklist/sslblacklist.csv",
        "description":
            "SSLBL SSL Certificate Blacklist (SHA1 Fingerprints)",
    }

    def update(self):

        since_last_run = datetime.now() - self.frequency

        for line in self.update_csv(delimiter=',', quotechar='"'):
            if not line or line[0].startswith("#"):
                continue

            first_seen = parser.parse(line[0])

            if self.last_run is not None:
                if since_last_run > first_seen:
                    continue

            self.analyze(line, first_seen)

    def analyze(self, line, first_seen):

        _, sha1, reason = line

        tags = []
        tag = reason.split(" ")
        if len(tag) >= 2:
            family = tag[0]
            tags.append(family.lower())
        _type = tag[-1]

        if TYPE_DICT.get(_type):
            tags += TYPE_DICT[_type]

        tags.append("ssl_fingerprint")

        context_hash = {
            'source': self.name,
            'first_seen': first_seen
        }

        try:
            sha1 = Hash.get_or_create(value=sha1)
            sha1.tag(tags)
            sha1.add_context(context_hash)
        except ObservableValidationError as e:
            logging.error("Invalid line: {}\nLine: {}".format(e, line))
<commit_msg>adapt sslblacklist_fingerprints.py<commit_after>
import logging
from datetime import datetime, timedelta

from core.errors import ObservableValidationError
from core.feed import Feed
from core.observables import Hash

TYPE_DICT = {
    "MITM": ['mitm'],
    "C&C": ["c2"],
    "distribution": ["payload_delivery"],
    "sinkhole": ["sinkhole"],
}

class SSLBlackListCerts(Feed):

    default_values = {
        "frequency": timedelta(hours=24),
        "name": "SSLBlackListCerts",
        "source": "https://sslbl.abuse.ch/blacklist/sslblacklist.csv",
        "description":
            "SSLBL SSL Certificate Blacklist (SHA1 Fingerprints)",
    }

    def update(self):

        since_last_run = datetime.now() - self.frequency

        for index,line in self.update_csv(delimiter=',',
                                          names=['Listingdate','SHA1','Listingreason'],
                                          filter_row='Listingdate'):

            self.analyze(line)

    def analyze(self, line):

        first_seen = line['Listingdate']
        _sha1 = line['SHA1']
        reason = line['Listingreason']

        tags = []
        tag = reason.split(' ')
        if len(tag) >= 2:
            family = tag[0]
            tags.append(family.lower())
        _type = tag[-1]

        if TYPE_DICT.get(_type):
            tags += TYPE_DICT[_type]

        tags.append("ssl_fingerprint")

        context_hash = {
            'source': self.name,
            'first_seen': first_seen
        }

        try:
            sha1 = Hash.get_or_create(value=_sha1)
            sha1.tag(tags)
            sha1.add_context(context_hash)
        except ObservableValidationError as e:
            logging.error("Invalid line: {}\nLine: {}".format(e, line))
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Floyd Hightower <https://github.com/fhightower>
# September 2016
# ----------------------------------------------------------------------------
# Copyright (c) 2016 Floyd Hightower
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ----------------------------------------------------------------------------
"""Find internationalized domain-squats of a given domain."""

import argparse
import itertools
import json
import os
import urllib.parse

import dns.resolver

def init_parser():
    """Initialize the argument parser."""
    parser = argparse.ArgumentParser(description='Find internationalized ' +
                                                 'domain-squats.')

    # get domain from which internationalized domains squats will be derived
    parser.add_argument('domain', help='Domain name from which to search')

    # dataset arguments to create possible internationalized domain squats
    parser.add_argument('-cca', '--cyrillic_complete_advanced',
                        action='store_true',
                        help='Use complete, advanced Cyrillic character set')
    parser.add_argument('-ccb', '--cyrillic_complete_basic',
                        action='store_true',
                        help='Use complete, basic Cyrillic character set')
    parser.add_argument('-cs', '--cyrillic_simplified',
                        action='store_true',
                        help='Use simplified Cyrillic character set')
    parser.add_argument('-lc', '--latin_complete', action='store_true',
                        help='Use complete Latin character set')
    parser.add_argument('-ls', '--latin_simplified', action='store_true',
                        help='Use simplified Latin character set')
    parser.add_argument('-gc', '--greek_complete', action='store_true',
                        help='Use complete Greek character set')
    parser.add_argument('-gs', '--greek_simplified', action='store_true',
                        help='Use simplified Greek character set')
    parser.add_argument('-d', '--dns', action='store_true',
                        help='Query DNS for each domain')

    return parser.parse_args()

def get_spoofable_charset(arguments):
    """Retrieve and return the data from the desired character sets."""
    spoofable_charset = dict()

    base_dataset_path = os.path.join(os.getcwd(), "../data/")
    dataset_paths = {
        'cyrillic_complete_advanced': "complete/cyrillic_advanced.json",
        'cyrillic_complete_basic': "complete/cyrillic_basic.json",
        'cyrillic_simplified': "simplified/cyrillic_simplified.json",
        'latin_complete': "complete/latin_basic.json",
        'latin_simplified': "simplified/latin_simplified.json",
        'greek_complete': "complete/greek_basic.json",
        'greek_simplified': "simplified/greek_simplified.json"
    }

    for arg_name, arg_value in arguments._get_kwargs():
        # if the argument is one that may be specifying a character set...
        if arg_name in dataset_paths:
            # if we want to read this character set...
            if arg_value:
                with open(os.path.join(base_dataset_path,
                                       dataset_paths[arg_name]),
                          'r') as data_file:
                    character_set = json.load(data_file)
                    # iterate through each char and its spoofs from char set
                    for character, spoofs in character_set.items():
                        # if character is already in the spoofable charset...
                        if spoofable_charset.get(character):
                            # append the spoofable chars to the end of the list
                            spoofable_charset[character].extend(spoofs)
                        # if the character is not in the spoofable charset...
                        else:
                            # add character and spoofs to the spoofable charset
                            spoofable_charset[character] = spoofs

    return spoofable_charset

def get_domain_details(domain):
    """Return the domain name and the top level domain (tld)."""
    parsed_domain = urllib.parse.urlparse(domain)

    domain_name = parsed_domain.path.split(".")[0]
    tld = parsed_domain.path.split(".")[1].split("/")[0]

    return domain_name, tld

def get_combinations(index_list):
    """Get all combinations of the domain name's 'spoofable' indices."""
    return (combination
            for combination in itertools.chain.from_iterable(
                itertools.combinations(index_list, i)
                for i in range(1, len(index_list) + 1))
            )

def get_possible_domain_squats(domain_name, combinations, SPOOFABLE_CHARS):
    """Construct all of the possible internationalized domain squats."""
    domains = list()

    for combo in combinations:
        # collect all possible domain squats for this combo.
        temp_combo_domains = list()
        temp_combo_domains.append(domain_name)

        for index in combo:
            # collect domain squats created by changing char. at this index
            temp_index_domains = list()

            # iterate through all domain squats collected so far for this combo
            for domain in temp_combo_domains:
                for spoofed_char in SPOOFABLE_CHARS[domain[index]]:
                    # replace the appropriate, spoofed character
                    new_domain = domain[:index] + spoofed_char

                    # if current index is not the last char. in the domain name
                    if len(domain) > (index + 1):
                        # add the ending of the domain
                        new_domain = new_domain + domain[(index + 1):]

                    # add to list of domains created by changing char at index
                    temp_index_domains.append(new_domain)
            # add to domains created by changing chars. at indices in combo
            temp_combo_domains = temp_index_domains

        # add to list of domain squats
        domains.extend(temp_combo_domains)

    return domains

def get_domain_dns(domain):
    """Get the DNS record, if any, for the given domain."""
    dns_records = None

    try:
        # get the dns resolutions for this domain
        dns_results = dns.resolver.query(domain)
        dns_records = [ip.address for ip in dns_results]
    except dns.resolver.NXDOMAIN as e:
        # the domain does not exist so dns resolutions will remain None
        pass

    return dns_records

def display_possible_domain_squats(possible_domain_squats, tld, dns_query):
    """Display each of the possible, internationalized domain squats."""
    print("{} results found.\n".format(len(possible_domain_squats)))

    for squat in possible_domain_squats:
        domain_name = "xn--{}.{}".format(
                      str(squat.encode("punycode").decode("utf-8")), tld)
        if dns_query:
            domain_dns = set(get_domain_dns(domain_name))

            print("{}.{}".format(squat, tld), domain_name, domain_dns,
                  sep='\t')
        else:
            print("{}.{}".format(squat, tld), domain_name, sep='\t')

def main():
    """Find the internationalized domain squats for a given domain."""
    # parse the arguments
    args = init_parser()

    SPOOFABLE_CHARS = get_spoofable_charset(args)

    # get the domain from which we should look for domain squats
    domain_name, tld = get_domain_details(args.domain)

    count = 0
    spoofable_indices = list()

    # find the index of each 'spoofable' character in the domain
    for char in domain_name:
        if char in SPOOFABLE_CHARS:
            spoofable_indices.append(count)

        count += 1

    # find all possible combinations of the 'spoofable' indices
    combinations = list(get_combinations(spoofable_indices))

    # create each domain squat
    possible_domain_squats = get_possible_domain_squats(domain_name,
                                                        combinations,
                                                        SPOOFABLE_CHARS)

    # display each possible domain squat
    display_possible_domain_squats(possible_domain_squats, tld, args.dns)

if __name__ == '__main__':
    main()
<commit_msg>Updated location of dataset in script<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Floyd Hightower <https://github.com/fhightower>
# September 2016
# ----------------------------------------------------------------------------
# Copyright (c) 2016 Floyd Hightower
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ----------------------------------------------------------------------------
"""Find internationalized domain-squats of a given domain."""

import argparse
import itertools
import json
import os
import urllib.parse

import dns.resolver

def init_parser():
    """Initialize the argument parser."""
    parser = argparse.ArgumentParser(description='Find internationalized ' +
                                                 'domain-squats.')

    # get domain from which internationalized domains squats will be derived
    parser.add_argument('domain', help='Domain name from which to search')

    # dataset arguments to create possible internationalized domain squats
    parser.add_argument('-cca', '--cyrillic_complete_advanced',
                        action='store_true',
                        help='Use complete, advanced Cyrillic character set')
    parser.add_argument('-ccb', '--cyrillic_complete_basic',
                        action='store_true',
                        help='Use complete, basic Cyrillic character set')
    parser.add_argument('-cs', '--cyrillic_simplified',
                        action='store_true',
                        help='Use simplified Cyrillic character set')
    parser.add_argument('-lc', '--latin_complete', action='store_true',
                        help='Use complete Latin character set')
    parser.add_argument('-ls', '--latin_simplified', action='store_true',
                        help='Use simplified Latin character set')
    parser.add_argument('-gc', '--greek_complete', action='store_true',
                        help='Use complete Greek character set')
    parser.add_argument('-gs', '--greek_simplified', action='store_true',
                        help='Use simplified Greek character set')
    parser.add_argument('-d', '--dns', action='store_true',
                        help='Query DNS for each domain')

    return parser.parse_args()

def get_spoofable_charset(arguments):
    """Retrieve and return the data from the desired character sets."""
    spoofable_charset = dict()

    base_dataset_path = os.path.join(os.getcwd(), "./data/")
    dataset_paths = {
        'cyrillic_complete_advanced': "complete/cyrillic_advanced.json",
        'cyrillic_complete_basic': "complete/cyrillic_basic.json",
        'cyrillic_simplified': "simplified/cyrillic_simplified.json",
        'latin_complete': "complete/latin_basic.json",
        'latin_simplified': "simplified/latin_simplified.json",
        'greek_complete': "complete/greek_basic.json",
        'greek_simplified': "simplified/greek_simplified.json"
    }

    for arg_name, arg_value in arguments._get_kwargs():
        # if the argument is one that may be specifying a character set...
        if arg_name in dataset_paths:
            # if we want to read this character set...
            if arg_value:
                with open(os.path.join(base_dataset_path,
                                       dataset_paths[arg_name]),
                          'r') as data_file:
                    character_set = json.load(data_file)
                    # iterate through each char and its spoofs from char set
                    for character, spoofs in character_set.items():
                        # if character is already in the spoofable charset...
                        if spoofable_charset.get(character):
                            # append the spoofable chars to the end of the list
                            spoofable_charset[character].extend(spoofs)
                        # if the character is not in the spoofable charset...
                        else:
                            # add character and spoofs to the spoofable charset
                            spoofable_charset[character] = spoofs

    return spoofable_charset

def get_domain_details(domain):
    """Return the domain name and the top level domain (tld)."""
    parsed_domain = urllib.parse.urlparse(domain)

    domain_name = parsed_domain.path.split(".")[0]
    tld = parsed_domain.path.split(".")[1].split("/")[0]

    return domain_name, tld

def get_combinations(index_list):
    """Get all combinations of the domain name's 'spoofable' indices."""
    return (combination
            for combination in itertools.chain.from_iterable(
                itertools.combinations(index_list, i)
                for i in range(1, len(index_list) + 1))
            )

def get_possible_domain_squats(domain_name, combinations, SPOOFABLE_CHARS):
    """Construct all of the possible internationalized domain squats."""
    domains = list()

    for combo in combinations:
        # collect all possible domain squats for this combo.
        temp_combo_domains = list()
        temp_combo_domains.append(domain_name)

        for index in combo:
            # collect domain squats created by changing char. at this index
            temp_index_domains = list()

            # iterate through all domain squats collected so far for this combo
            for domain in temp_combo_domains:
                for spoofed_char in SPOOFABLE_CHARS[domain[index]]:
                    # replace the appropriate, spoofed character
                    new_domain = domain[:index] + spoofed_char

                    # if current index is not the last char. in the domain name
                    if len(domain) > (index + 1):
                        # add the ending of the domain
                        new_domain = new_domain + domain[(index + 1):]

                    # add to list of domains created by changing char at index
                    temp_index_domains.append(new_domain)
            # add to domains created by changing chars. at indices in combo
            temp_combo_domains = temp_index_domains

        # add to list of domain squats
        domains.extend(temp_combo_domains)

    return domains

def get_domain_dns(domain):
    """Get the DNS record, if any, for the given domain."""
    dns_records = None

    try:
        # get the dns resolutions for this domain
        dns_results = dns.resolver.query(domain)
        dns_records = [ip.address for ip in dns_results]
    except dns.resolver.NXDOMAIN as e:
        # the domain does not exist so dns resolutions will remain None
        pass

    return dns_records

def display_possible_domain_squats(possible_domain_squats, tld, dns_query):
    """Display each of the possible, internationalized domain squats."""
    print("{} results found.\n".format(len(possible_domain_squats)))

    for squat in possible_domain_squats:
        domain_name = "xn--{}.{}".format(
                      str(squat.encode("punycode").decode("utf-8")), tld)
        if dns_query:
            domain_dns = set(get_domain_dns(domain_name))

            print("{}.{}".format(squat, tld), domain_name, domain_dns,
                  sep='\t')
        else:
            print("{}.{}".format(squat, tld), domain_name, sep='\t')

def main():
    """Find the internationalized domain squats for a given domain."""
    # parse the arguments
    args = init_parser()

    SPOOFABLE_CHARS = get_spoofable_charset(args)

    # get the domain from which we should look for domain squats
    domain_name, tld = get_domain_details(args.domain)

    count = 0
    spoofable_indices = list()

    # find the index of each 'spoofable' character in the domain
    for char in domain_name:
        if char in SPOOFABLE_CHARS:
            spoofable_indices.append(count)

        count += 1

    # find all possible combinations of the 'spoofable' indices
    combinations = list(get_combinations(spoofable_indices))

    # create each domain squat
    possible_domain_squats = get_possible_domain_squats(domain_name,
                                                        combinations,
                                                        SPOOFABLE_CHARS)

    # display each possible domain squat
    display_possible_domain_squats(possible_domain_squats, tld, args.dns)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""NBConvert is a utility for conversion of .ipynb files.

Command-line interface for the NbConvert conversion utility.
"""
#-----------------------------------------------------------------------------
#Copyright (c) 2013, the IPython Development Team.
#
#Distributed under the terms of the Modified BSD License.
#
#The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
#Imports
#-----------------------------------------------------------------------------

# Stdlib imports
from __future__ import print_function

import logging
import sys
import os
import glob

# From IPython
from IPython.core.application import BaseIPythonApplication, base_aliases, base_flags
from IPython.config import catch_config_error, Configurable
from IPython.utils.traitlets import (
    Unicode, List, Instance, DottedObjectName, Type, CaselessStrEnum,
)
from IPython.utils.importstring import import_item
from IPython.utils.text import dedent

from .exporters.export import get_export_names, exporter_map
from IPython.nbconvert import exporters, preprocessors, writers, postprocessors
from .utils.base import NbConvertBase
from .utils.exceptions import ConversionException

#-----------------------------------------------------------------------------
#Classes and functions
#-----------------------------------------------------------------------------

class DottedOrNone(DottedObjectName):
    """
    A string holding a valid dotted object name in Python, such as A.b3._c
    Also allows for None type."""

    default_value = u''

    def validate(self, obj, value):
        if value is not None and len(value) > 0:
            return super(DottedOrNone, self).validate(obj, value)
        else:
            return value

nbconvert_aliases = {}
nbconvert_aliases.update(base_aliases)
nbconvert_aliases.update({
    'to' : 'NbConvertApp.export_format',
    'template' : 'Exporter.template_file',
    'writer' : 'NbConvertApp.writer_class',
    'post': 'NbConvertApp.post_processor_class',
    'output': 'NbConvertApp.output_base',
    'offline-slides': 'RevealHelpTransformer.url_prefix',
    'slide-notes': 'RevealHelpTransformer.speaker_notes'
})

nbconvert_flags = {}
nbconvert_flags.update(base_flags)
nbconvert_flags.update({
    'stdout' : (
        {'NbConvertApp' : {'writer_class' : "StdoutWriter"}},
        "Write notebook output to stdout instead of files."
        )
})

class NbConvertApp(BaseIPythonApplication):
    """Application used to convert to and from notebook file type (*.ipynb)"""

    name = 'ipython-nbconvert'
    aliases = nbconvert_aliases
    flags = nbconvert_flags

    def _log_level_default(self):
        return logging.INFO

    def _classes_default(self):
        classes = [NbConvertBase]
        for pkg in (exporters, preprocessors, writers):
            for name in dir(pkg):
                cls = getattr(pkg, name)
                if isinstance(cls, type) and issubclass(cls, Configurable):
                    classes.append(cls)
        return classes

    description = Unicode(
        u"""This application is used to convert notebook files (*.ipynb)
        to various other formats.

        WARNING: THE COMMANDLINE INTERFACE MAY CHANGE IN FUTURE RELEASES.""")

    output_base = Unicode('', config=True, help='''overwrite base name use for output files.
            can only  be use when converting one notebook at a time.
            ''')

    examples = Unicode(u"""
        The simplest way to use nbconvert is

        > ipython nbconvert mynotebook.ipynb

        which will convert mynotebook.ipynb to the default format (probably HTML).

        You can specify the export format with `--to`.
        Options include {0}

        > ipython nbconvert --to latex mynotebook.ipnynb

        Both HTML and LaTeX support multiple output templates. LaTeX includes
        'basic', 'book', and 'article'.  HTML includes 'basic' and 'full'.  You
        can specify the flavor of the format used.

        > ipython nbconvert --to html --template basic mynotebook.ipynb

        You can also pipe the output to stdout, rather than a file

        > ipython nbconvert mynotebook.ipynb --stdout

        A post-processor can be used to compile a PDF

        > ipython nbconvert mynotebook.ipynb --to latex --post PDF

        You can get (and serve) a Reveal.js-powered slideshow

        > ipython nbconvert myslides.ipynb --to slides --post serve

        Multiple notebooks can be given at the command line in a couple of
        different ways:

        > ipython nbconvert notebook*.ipynb
        > ipython nbconvert notebook1.ipynb notebook2.ipynb

        or you can specify the notebooks list in a config file, containing::

            c.NbConvertApp.notebooks = ["my_notebook.ipynb"]

        > ipython nbconvert --config mycfg.py
        """.format(get_export_names()))

    # Writer specific variables
    writer = Instance('IPython.nbconvert.writers.base.WriterBase',
                      help="""Instance of the writer class used to write the
                      results of the conversion.""")
    writer_class = DottedObjectName('FilesWriter', config=True,
                                    help="""Writer class used to write the
                                    results of the conversion""")
    writer_aliases = {'fileswriter': 'IPython.nbconvert.writers.files.FilesWriter',
                      'debugwriter': 'IPython.nbconvert.writers.debug.DebugWriter',
                      'stdoutwriter': 'IPython.nbconvert.writers.stdout.StdoutWriter'}
    writer_factory = Type()

    def _writer_class_changed(self, name, old, new):
        if new.lower() in self.writer_aliases:
            new = self.writer_aliases[new.lower()]
        self.writer_factory = import_item(new)

    # Post-processor specific variables
    post_processor = Instance('IPython.nbconvert.postprocessors.base.PostProcessorBase',
                      help="""Instance of the PostProcessor class used to write the
                      results of the conversion.""")

    post_processor_class = DottedOrNone(config=True,
                                    help="""PostProcessor class used to write the
                                    results of the conversion""")
    post_processor_aliases = {'pdf': 'IPython.nbconvert.postprocessors.pdf.PDFPostProcessor',
                              'serve': 'IPython.nbconvert.postprocessors.serve.ServePostProcessor'}
    post_processor_factory = Type()

    def _post_processor_class_changed(self, name, old, new):
        if new.lower() in self.post_processor_aliases:
            new = self.post_processor_aliases[new.lower()]
        if new:
            self.post_processor_factory = import_item(new)

    # Other configurable variables
    export_format = CaselessStrEnum(get_export_names(),
        default_value="html",
        config=True,
        help="""The export format to be used."""
    )

    notebooks = List([], config=True, help="""List of notebooks to convert.
                     Wildcards are supported.
                     Filenames passed positionally will be added to the list.
                     """)

    @catch_config_error
    def initialize(self, argv=None):
        super(NbConvertApp, self).initialize(argv)
        self.init_syspath()
        self.init_notebooks()
        self.init_writer()
        self.init_post_processor()

    def init_syspath(self):
        """
        Add the cwd to the sys.path ($PYTHONPATH)
        """
        sys.path.insert(0, os.getcwd())

    def init_notebooks(self):
        """Construct the list of notebooks.
        If notebooks are passed on the command-line,
        they override notebooks specified in config files.
        Glob each notebook to replace notebook patterns with filenames.
        """

        # Specifying notebooks on the command-line overrides (rather than adds)
        # the notebook list
        if self.extra_args:
            patterns = self.extra_args
        else:
            patterns = self.notebooks

        # Use glob to replace all the notebook patterns with filenames.
        filenames = []
        for pattern in patterns:

            # Use glob to find matching filenames.  Allow the user to convert
            # notebooks without having to type the extension.
            globbed_files = glob.glob(pattern)
            globbed_files.extend(glob.glob(pattern + '.ipynb'))
            if not globbed_files:
                self.log.warn("pattern %r matched no files", pattern)

            for filename in globbed_files:
                if not filename in filenames:
                    filenames.append(filename)
        self.notebooks = filenames

    def init_writer(self):
        """
        Initialize the writer (which is stateless)
        """
        self._writer_class_changed(None, self.writer_class, self.writer_class)
        self.writer = self.writer_factory(parent=self)

    def init_post_processor(self):
        """
        Initialize the post_processor (which is stateless)
        """
        self._post_processor_class_changed(None, self.post_processor_class,
            self.post_processor_class)
        if self.post_processor_factory:
            self.post_processor = self.post_processor_factory(parent=self)

    def start(self):
        """
        Ran after initialization completed
        """
        super(NbConvertApp, self).start()
        self.convert_notebooks()

    def convert_notebooks(self):
        """
        Convert the notebooks in the self.notebook traitlet
        """
        # Export each notebook
        conversion_success = 0

        if self.output_base != '' and len(self.notebooks) > 1:
            self.log.error(
            """UsageError: --output flag or `NbConvertApp.output_base` config option
            cannot be used when converting multiple notebooks.
            """)
            self.exit(1)

        exporter = exporter_map[self.export_format](config=self.config)

        for notebook_filename in self.notebooks:
            self.log.info("Converting notebook %s to %s", notebook_filename, self.export_format)

            # Get a unique key for the notebook and set it in the resources object.
            basename = os.path.basename(notebook_filename)
            notebook_name = basename[:basename.rfind('.')]
            if self.output_base:
                notebook_name = self.output_base
            resources = {}
            resources['unique_key'] = notebook_name
            resources['output_files_dir'] = '%s_files' % notebook_name
            self.log.info("Support files will be in %s", os.path.join(resources['output_files_dir'], ''))

            # Try to export
            try:
                output, resources = exporter.from_filename(notebook_filename, resources=resources)
            except ConversionException as e:
                self.log.error("Error while converting '%s'", notebook_filename,
                      exc_info=True)
                self.exit(1)
            else:
                write_resultes = self.writer.write(output, resources, notebook_name=notebook_name)

                #Post-process if post processor has been defined.
                if hasattr(self, 'post_processor') and self.post_processor:
                    self.post_processor(write_resultes)
                conversion_success += 1

        # If nothing was converted successfully, help the user.
        if conversion_success == 0:
            self.print_help()
            sys.exit(-1)

#-----------------------------------------------------------------------------
# Main entry point
#-----------------------------------------------------------------------------

launch_new_instance = NbConvertApp.launch_instance
<commit_msg>Also renamed local variables in nbconvertapp<commit_after>#!/usr/bin/env python
"""NBConvert is a utility for conversion of .ipynb files.

Command-line interface for the NbConvert conversion utility.
"""
#-----------------------------------------------------------------------------
#Copyright (c) 2013, the IPython Development Team.
#
#Distributed under the terms of the Modified BSD License.
#
#The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
#Imports
#-----------------------------------------------------------------------------

# Stdlib imports
from __future__ import print_function

import logging
import sys
import os
import glob

# From IPython
from IPython.core.application import BaseIPythonApplication, base_aliases, base_flags
from IPython.config import catch_config_error, Configurable
from IPython.utils.traitlets import (
    Unicode, List, Instance, DottedObjectName, Type, CaselessStrEnum,
)
from IPython.utils.importstring import import_item
from IPython.utils.text import dedent

from .exporters.export import get_export_names, exporter_map
from IPython.nbconvert import exporters, preprocessors, writers, postprocessors
from .utils.base import NbConvertBase
from .utils.exceptions import ConversionException

#-----------------------------------------------------------------------------
#Classes and functions
#-----------------------------------------------------------------------------

class DottedOrNone(DottedObjectName):
    """
    A string holding a valid dotted object name in Python, such as A.b3._c
    Also allows for None type."""

    default_value = u''

    def validate(self, obj, value):
        if value is not None and len(value) > 0:
            return super(DottedOrNone, self).validate(obj, value)
        else:
            return value

nbconvert_aliases = {}
nbconvert_aliases.update(base_aliases)
nbconvert_aliases.update({
    'to' : 'NbConvertApp.export_format',
    'template' : 'Exporter.template_file',
    'writer' : 'NbConvertApp.writer_class',
    'post': 'NbConvertApp.postprocessor_class',
    'output': 'NbConvertApp.output_base',
    'offline-slides': 'RevealHelpTransformer.url_prefix',
    'slide-notes': 'RevealHelpTransformer.speaker_notes'
})

nbconvert_flags = {}
nbconvert_flags.update(base_flags)
nbconvert_flags.update({
    'stdout' : (
        {'NbConvertApp' : {'writer_class' : "StdoutWriter"}},
        "Write notebook output to stdout instead of files."
        )
})

class NbConvertApp(BaseIPythonApplication):
    """Application used to convert to and from notebook file type (*.ipynb)"""

    name = 'ipython-nbconvert'
    aliases = nbconvert_aliases
    flags = nbconvert_flags

    def _log_level_default(self):
        return logging.INFO

    def _classes_default(self):
        classes = [NbConvertBase]
        for pkg in (exporters, preprocessors, writers):
            for name in dir(pkg):
                cls = getattr(pkg, name)
                if isinstance(cls, type) and issubclass(cls, Configurable):
                    classes.append(cls)
        return classes

    description = Unicode(
        u"""This application is used to convert notebook files (*.ipynb)
        to various other formats.

        WARNING: THE COMMANDLINE INTERFACE MAY CHANGE IN FUTURE RELEASES.""")

    output_base = Unicode('', config=True, help='''overwrite base name use for output files.
            can only  be use when converting one notebook at a time.
            ''')

    examples = Unicode(u"""
        The simplest way to use nbconvert is

        > ipython nbconvert mynotebook.ipynb

        which will convert mynotebook.ipynb to the default format (probably HTML).

        You can specify the export format with `--to`.
        Options include {0}

        > ipython nbconvert --to latex mynotebook.ipnynb

        Both HTML and LaTeX support multiple output templates. LaTeX includes
        'basic', 'book', and 'article'.  HTML includes 'basic' and 'full'.  You
        can specify the flavor of the format used.

        > ipython nbconvert --to html --template basic mynotebook.ipynb

        You can also pipe the output to stdout, rather than a file

        > ipython nbconvert mynotebook.ipynb --stdout

        A post-processor can be used to compile a PDF

        > ipython nbconvert mynotebook.ipynb --to latex --post PDF

        You can get (and serve) a Reveal.js-powered slideshow

        > ipython nbconvert myslides.ipynb --to slides --post serve

        Multiple notebooks can be given at the command line in a couple of
        different ways:

        > ipython nbconvert notebook*.ipynb
        > ipython nbconvert notebook1.ipynb notebook2.ipynb

        or you can specify the notebooks list in a config file, containing::

            c.NbConvertApp.notebooks = ["my_notebook.ipynb"]

        > ipython nbconvert --config mycfg.py
        """.format(get_export_names()))

    # Writer specific variables
    writer = Instance('IPython.nbconvert.writers.base.WriterBase',
                      help="""Instance of the writer class used to write the
                      results of the conversion.""")
    writer_class = DottedObjectName('FilesWriter', config=True,
                                    help="""Writer class used to write the
                                    results of the conversion""")
    writer_aliases = {'fileswriter': 'IPython.nbconvert.writers.files.FilesWriter',
                      'debugwriter': 'IPython.nbconvert.writers.debug.DebugWriter',
                      'stdoutwriter': 'IPython.nbconvert.writers.stdout.StdoutWriter'}
    writer_factory = Type()

    def _writer_class_changed(self, name, old, new):
        if new.lower() in self.writer_aliases:
            new = self.writer_aliases[new.lower()]
        self.writer_factory = import_item(new)

    # Post-processor specific variables
    postprocessor = Instance('IPython.nbconvert.postprocessors.base.PostProcessorBase',
                      help="""Instance of the PostProcessor class used to write the
                      results of the conversion.""")

    postprocessor_class = DottedOrNone(config=True,
                                    help="""PostProcessor class used to write the
                                    results of the conversion""")
    postprocessor_aliases = {'pdf': 'IPython.nbconvert.postprocessors.pdf.PDFPostProcessor',
                              'serve': 'IPython.nbconvert.postprocessors.serve.ServePostProcessor'}
    postprocessor_factory = Type()

    def _postprocessor_class_changed(self, name, old, new):
        if new.lower() in self.postprocessor_aliases:
            new = self.postprocessor_aliases[new.lower()]
        if new:
            self.postprocessor_factory = import_item(new)

    # Other configurable variables
    export_format = CaselessStrEnum(get_export_names(),
        default_value="html",
        config=True,
        help="""The export format to be used."""
    )

    notebooks = List([], config=True, help="""List of notebooks to convert.
                     Wildcards are supported.
                     Filenames passed positionally will be added to the list.
                     """)

    @catch_config_error
    def initialize(self, argv=None):
        super(NbConvertApp, self).initialize(argv)
        self.init_syspath()
        self.init_notebooks()
        self.init_writer()
        self.init_postprocessor()

    def init_syspath(self):
        """
        Add the cwd to the sys.path ($PYTHONPATH)
        """
        sys.path.insert(0, os.getcwd())

    def init_notebooks(self):
        """Construct the list of notebooks.
        If notebooks are passed on the command-line,
        they override notebooks specified in config files.
        Glob each notebook to replace notebook patterns with filenames.
        """

        # Specifying notebooks on the command-line overrides (rather than adds)
        # the notebook list
        if self.extra_args:
            patterns = self.extra_args
        else:
            patterns = self.notebooks

        # Use glob to replace all the notebook patterns with filenames.
        filenames = []
        for pattern in patterns:

            # Use glob to find matching filenames.  Allow the user to convert
            # notebooks without having to type the extension.
            globbed_files = glob.glob(pattern)
            globbed_files.extend(glob.glob(pattern + '.ipynb'))
            if not globbed_files:
                self.log.warn("pattern %r matched no files", pattern)

            for filename in globbed_files:
                if not filename in filenames:
                    filenames.append(filename)
        self.notebooks = filenames

    def init_writer(self):
        """
        Initialize the writer (which is stateless)
        """
        self._writer_class_changed(None, self.writer_class, self.writer_class)
        self.writer = self.writer_factory(parent=self)

    def init_postprocessor(self):
        """
        Initialize the postprocessor (which is stateless)
        """
        self._postprocessor_class_changed(None, self.postprocessor_class,
            self.postprocessor_class)
        if self.postprocessor_factory:
            self.postprocessor = self.postprocessor_factory(parent=self)

    def start(self):
        """
        Ran after initialization completed
        """
        super(NbConvertApp, self).start()
        self.convert_notebooks()

    def convert_notebooks(self):
        """
        Convert the notebooks in the self.notebook traitlet
        """
        # Export each notebook
        conversion_success = 0

        if self.output_base != '' and len(self.notebooks) > 1:
            self.log.error(
            """UsageError: --output flag or `NbConvertApp.output_base` config option
            cannot be used when converting multiple notebooks.
            """)
            self.exit(1)

        exporter = exporter_map[self.export_format](config=self.config)

        for notebook_filename in self.notebooks:
            self.log.info("Converting notebook %s to %s", notebook_filename, self.export_format)

            # Get a unique key for the notebook and set it in the resources object.
            basename = os.path.basename(notebook_filename)
            notebook_name = basename[:basename.rfind('.')]
            if self.output_base:
                notebook_name = self.output_base
            resources = {}
            resources['unique_key'] = notebook_name
            resources['output_files_dir'] = '%s_files' % notebook_name
            self.log.info("Support files will be in %s", os.path.join(resources['output_files_dir'], ''))

            # Try to export
            try:
                output, resources = exporter.from_filename(notebook_filename, resources=resources)
            except ConversionException as e:
                self.log.error("Error while converting '%s'", notebook_filename,
                      exc_info=True)
                self.exit(1)
            else:
                write_resultes = self.writer.write(output, resources, notebook_name=notebook_name)

                #Post-process if post processor has been defined.
                if hasattr(self, 'postprocessor') and self.postprocessor:
                    self.postprocessor(write_resultes)
                conversion_success += 1

        # If nothing was converted successfully, help the user.
        if conversion_success == 0:
            self.print_help()
            sys.exit(-1)

#-----------------------------------------------------------------------------
# Main entry point
#-----------------------------------------------------------------------------

launch_new_instance = NbConvertApp.launch_instance
<|endoftext|>


<commit_before>"""
A salt cloud provider that lets you use virtualbox on your machine and act as a cloud.
It can be used in co

Followed https://docs.saltstack.com/en/latest/topics/cloud/cloud.html#non-libcloud-based-modules
to create this.

Dicts provided by salt:
    __opts__ : contains the options used to run Salt Cloud, as well as a set of configuration and environment variables
"""

# Import python libs
import copy
import logging
import pprint
import time
import yaml

# Import salt libs
import salt.config as config
from salt.exceptions import SaltCloudSystemExit
import salt.utils.cloud

# Import virtualbox libs
HAS_LIBS = False
try:
    # This code assumes vboxapi.py from VirtualBox distribution
    # being in PYTHONPATH, or installed system-wide
    from vboxapi import VirtualBoxManager

    HAS_LIBS = True

except ImportError:
    pass

log = logging.getLogger(__name__)

def __virtual__():
    """
    This function determines whether or not to make this cloud module available upon execution.
    Most often, it uses get_configured_provider() to determine
     if the necessary configuration has been set up.
    It may also check for necessary imports, to decide whether to load the module.
    In most cases, it will return a True or False value.
    If the name of the driver used does not match the filename,
     then that name should be returned instead of True.

    @return True|False|str
    """

    if not HAS_LIBS:
        return False

    if get_configured_provider() is False:
        return False

    # If the name of the driver used does not match the filename,
    #  then that name should be returned instead of True.
    # return __virtualname__
    return True

def get_configured_provider():
    '''
    Return the first configured instance.
    '''
    return config.is_provider_configured(
        __opts__,
        __active_provider_name__ or __virtualname__,
        ('subscription_id', 'certificate_path')
    )

def create(vm_info):
    """
    Creates a virtual machine from the given VM information.
    This is what is used to request a virtual machine to be created by the cloud provider, wait for it to become available, and then (optionally) log in and install Salt on it.

    Fires:
        "starting create" : This event is tagged salt/cloud/<vm name>/creating. The payload contains the names of the VM, profile and provider.

    @param vm_info {dict}
            {
                name: <str>
                profile: <dict>
                provider: <provider>
            }
    @return dict of resulting vm. !!!Passwords cand and should be included!!!
    """
    salt.utils.cloud.fire_event(
        'event',
        'starting create',
        'salt/cloud/{0}/creating'.format(vm_info['name']),
        {
            'name': vm_info['name'],
            'profile': vm_info['profile'],
            'provider': vm_info['provider'],
        },
        transport=__opts__['transport']
    )

    #TODO Calculate kwargs with parameters required by virtualbox to create the virtual machine.
    request_kwargs = {
        'name': vm_info['name'],
        'profile': vm_info['profile']
    }

    salt.utils.cloud.fire_event(
        'event',
        'requesting instance',
        'salt/cloud/{0}/requesting'.format(vm_info['name']),
        request_kwargs,
        transport=__opts__['transport']
    )
    # TODO request a new VM!
    vm_result = vb_create_vm(**request_kwargs)

    #TODO Prepare deployment of salt on the vm
    #  Any private data, including passwords and keys (including public keys) should be stripped from the deploy kwargs before the event is fired.
    deploy_kwargs = {
    }

    salt.utils.cloud.fire_event(
        'event',
        'deploying salt',
        'salt/cloud/{0}/deploying'.format(vm_info['name']),
        deploy_kwargs,
        transport=__opts__['transport']
    )

    deploy_kwargs = deploy_kwargs.update({
        # TODO Add private data
    })

    # TODO wait for target machine to become available
    # TODO deploy!
    # Do we have to call this?
    salt.utils.cloud.deploy_script(**deploy_kwargs)

    salt.utils.cloud.fire_event(
        'event',
        'created machine',
        'salt/cloud/{0}/created'.format(vm_info['name']),
        vm_result,
        transport=__opts__['transport']
    )

    # Passwords should be included in this object!!
    return vm_result

# -----------------------------
# Virtualbox method
# -----------------------------
def vb_create_vm(**kwargs):
    """
    Tells virtualbox to create a VM

    @return dict of resulting VM
    """
    pass

def vb_start_vm(**kwargs):
    """
    Tells Virtualbox to start up a VM.
    Blocking function!

    @return dict of started VM, contains IP addresses and what not
    """
<commit_msg>A little more documentation Also get rid of keys we don't need in the provider config<commit_after>"""
A salt cloud provider that lets you use virtualbox on your machine and act as a cloud.
It can be used in co

Followed https://docs.saltstack.com/en/latest/topics/cloud/cloud.html#non-libcloud-based-modules
to create this.

Dicts provided by salt:
    __opts__ : contains the options used to run Salt Cloud, as well as a set of configuration and environment variables
"""

# Import python libs
import copy
import logging
import pprint
import time
import yaml

# Import salt libs
import salt.config as config
from salt.exceptions import SaltCloudSystemExit
import salt.utils.cloud

# Import virtualbox libs
HAS_LIBS = False
try:
    # This code assumes vboxapi.py from VirtualBox distribution
    # being in PYTHONPATH, or installed system-wide
    from vboxapi import VirtualBoxManager

    HAS_LIBS = True

except ImportError:
    pass

log = logging.getLogger(__name__)

def __virtual__():
    """
    This function determines whether or not to make this cloud module available upon execution.
    Most often, it uses get_configured_provider() to determine
     if the necessary configuration has been set up.
    It may also check for necessary imports, to decide whether to load the module.
    In most cases, it will return a True or False value.
    If the name of the driver used does not match the filename,
     then that name should be returned instead of True.

    @return True|False|str
    """

    if not HAS_LIBS:
        return False

    if get_configured_provider() is False:
        return False

    # If the name of the driver used does not match the filename,
    #  then that name should be returned instead of True.
    # return __virtualname__
    return True

def get_configured_provider():
    '''
    Return the first configured instance.
    '''
    return config.is_provider_configured(
        __opts__,
        __active_provider_name__ or __virtualname__,
        (,) # keys we need from the provider configuration
    )

def create(vm_info):
    """
    Creates a virtual machine from the given VM information.
    This is what is used to request a virtual machine to be created by the cloud provider, wait for it to become available, and then (optionally) log in and install Salt on it.

    Fires:
        "starting create" : This event is tagged salt/cloud/<vm name>/creating. The payload contains the names of the VM, profile and provider.

    @param vm_info {dict}
            {
                name: <str>
                profile: <dict>
                provider: <provider>
            }
    @return dict of resulting vm. !!!Passwords cand and should be included!!!
    """
    salt.utils.cloud.fire_event(
        'event',
        'starting create',
        'salt/cloud/{0}/creating'.format(vm_info['name']),
        {
            'name': vm_info['name'],
            'profile': vm_info['profile'],
            'provider': vm_info['provider'],
        },
        transport=__opts__['transport']
    )

    #TODO Calculate kwargs with parameters required by virtualbox to create the virtual machine.
    request_kwargs = {
        'name': vm_info['name'],
        'profile': vm_info['profile']
    }

    salt.utils.cloud.fire_event(
        'event',
        'requesting instance',
        'salt/cloud/{0}/requesting'.format(vm_info['name']),
        request_kwargs,
        transport=__opts__['transport']
    )
    # TODO request a new VM!
    vm_result = vb_create_vm(**request_kwargs)

    #TODO Prepare deployment of salt on the vm
    #  Any private data, including passwords and keys (including public keys) should be stripped from the deploy kwargs before the event is fired.
    deploy_kwargs = {
    }

    salt.utils.cloud.fire_event(
        'event',
        'deploying salt',
        'salt/cloud/{0}/deploying'.format(vm_info['name']),
        deploy_kwargs,
        transport=__opts__['transport']
    )

    deploy_kwargs = deploy_kwargs.update({
        # TODO Add private data
    })

    # TODO wait for target machine to become available
    # TODO deploy!
    # Do we have to call this?
    salt.utils.cloud.deploy_script(**deploy_kwargs)

    salt.utils.cloud.fire_event(
        'event',
        'created machine',
        'salt/cloud/{0}/created'.format(vm_info['name']),
        vm_result,
        transport=__opts__['transport']
    )

    # Passwords should be included in this object!!
    return vm_result

# -----------------------------
# Virtualbox method
# -----------------------------
def vb_create_vm(**kwargs):
    """
    Tells virtualbox to create a VM

    @return dict of resulting VM
    """
    pass

def vb_start_vm(**kwargs):
    """
    Tells Virtualbox to start up a VM.
    Blocking function!

    @return dict of started VM, contains IP addresses and what not
    """
<|endoftext|>