<commit_before>import datetime
import numpy

import grocsvs

VCF_HEADER = \
"""##fileformat=VCFv4.3
##fileDate={date}
##source=grocsvs_v{version}
##reference=file://{reference}
{chromosomes}
##INFO=<ID=ASSEMBLED,Number=0,Type=Flag,Description="Event is supported by a read-cloud-based sequence assembly across the breakpoints">
##INFO=<ID=MATEID,Number=1,Type=String,Description="ID of mate breakend">
##INFO=<ID=SVTYPE,Number=1,Type=String,Description="Type of structural variant">
##FILTER=<ID=NOLONGFRAGS,Description="No samples found with long fragments supporting event (likely segdup)">
##FILTER=<ID=NEARBYSNVS,Description="Very high number of possible SNVs detected in vicinity of breakpoint (likely segdup)">
##FILTER=<ID=ASSEMBLYGAP,Description="Nearby gap in the reference assembly found; these gaps can appear to mimic SVs">
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=BS,Number=1,Type=int,Description="Number of barcodes supporting the event">
##FORMAT=<ID=BT,Number=1,Type=int,Description="Total number of barcodes, calculated as the union of barcodes at each site">
##FORMAT=<ID=F50,Number=1,Type=Float,Description="Median fragment length supporting the breakpoint; estimated as the total across all supporting segments">
##FORMAT=<ID=F90,Number=1,Type=Float,Description="90th percentile of fragment lengths supporting the breakpoint; estimated as the total across all supporting segments">
##FORMAT=<ID=PR,Number=1,Type=Float,Description="p-value for the event, calculated by resampling from the number of supporting and total barcodes">
##FORMAT=<ID=H1x,Number=1,Type=String,Description="Number of supporting barcodes assigned to haplotype 1 (left side of breakpoint)">
##FORMAT=<ID=H2x,Number=1,Type=String,Description="Number of supporting barcodes assigned to haplotype 2 (left side of breakpoint)">
##FORMAT=<ID=H1y,Number=1,Type=String,Description="Number of supporting barcodes assigned to haplotype 1 (right side of breakpoint)">
##FORMAT=<ID=H2y,Number=1,Type=String,Description="Number of supporting barcodes assigned to haplotype 2 (right side of breakpoint)">
{samples}
#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{sample_columns}
"""

CHROMOSOME = "##contig=<ID={name},length={length}>"
SAMPLE = "##SAMPLE=<ID={name},URL=file://{path}>"
FORMAT = ["GT", "BS", "BT", "F50", "F90", "PR", "H1x", "H2x", "H1y", "H2y"]

def get_header(options):
    chromosomes = []
    for chrom, length in options.reference.chrom_lengths.items():
        chromosomes.append(CHROMOSOME.format(name=chrom, length=length))
    chromosomes = "\n".join(chromosomes)

    samples = []
    sample_columns = []
    for sample_name, sample in options.samples.items():
        samples.append(SAMPLE.format(name=sample_name, path=sample.get_10x_dataset().bam))
        sample_columns.append(sample_name)
    samples = "\n".join(samples)
    sample_columns = "\t".join(sample_columns)

    header = VCF_HEADER.format(
            date=datetime.datetime.now().strftime("%Y%m%d"),
            version=grocsvs.__version__,
            reference=options.reference.path,
            chromosomes=chromosomes,
            samples=samples,
            sample_columns=sample_columns
        )

    return header

def get_alt_breakend(options, chromx, x, chromy, y, orientation):
    # note that vcf uses one-based coords so we add 1 to all positions

    ref = options.reference.fasta[chromx][x].upper()
    if orientation == "+-":
        alt = "{}[{}:{}[".format(ref, chromy, y+1)
    elif orientation == "++":
        alt = "{}]{}:{}]".format(ref, chromy, y+1)
    elif orientation == "-+":
        alt = "]{}:{}]{}".format(chromy, y+1, ref)
    elif orientation == "--":
        alt = "[{}:{}[{}".format(chromy, y+1, ref)

    return ref, alt

def get_sample_info(sample, event):
    sample_info = {}
    sample_info["BS"] = event["{}_shared".format(sample.name)]
    sample_info["BT"] = event["{}_total".format(sample.name)]
    if sample_info["BS"] > 5:
        sample_info["GT"] = "0/1"
    else:
        sample_info["GT"] = "0/0"

    if "{}_lengths_50".format(sample.name) in event:
        sample_info["F50"] = event["{}_lengths_50".format(sample.name)]
        sample_info["F90"] = event["{}_lengths_90".format(sample.name)]
    else:
        sample_info["F50"] = "."
        sample_info["F90"] = "."

    sample_info["PR"] = event["{}_p_resampling".format(sample.name)]

    sample_info["H1x"] = event["{}_x_hap0".format(sample.name)]
    sample_info["H2x"] = event["{}_x_hap1".format(sample.name)]
    sample_info["H1y"] = event["{}_y_hap0".format(sample.name)]
    sample_info["H2y"] = event["{}_y_hap1".format(sample.name)]

    sample_info = [sample_info[key] for key in FORMAT]

    return ":".join(map(str, sample_info))

def get_filters(event):
    filters = []

    try:
        if "n=" in event["blacklist"]:
            filters.append("ASSEMBLYGAP")
    except TypeError:
        pass

    if event["nearby_snvs"] >= 15:
        filters.append("NEARBYSNVS")

    if "frag_length_passes" in event and not event["frag_length_passes"]:
        filters.append("NOLONGFRAGS")

    if len(filters) == 0:
        return "PASS"

    return ";".join(filters)

def get_info(event, mateid=None):
    info = []

    info.append("SVTYPE=BND")

    if mateid is not None:
        info.append("MATEID={}".format(mateid))

    if event["assembled"]:
        info.append("ASSEMBLED")

    if len(info) == 0:
        return "."

    return ";".join(info)

def convert_event(options, event_id, event):
    chromx, x, chromy, y, orientation = event["chromx"], event["x"], event["chromy"], event["y"], event["orientation"]

    for i in [0,1]:
        if i == 1:
            chromx, x, chromy, y = chromy, y, chromx, x
            orientation = orientation[1]+orientation[0]

        this_id = "{}:{}".format(event_id, i)
        other_id = "{}:{}".format(event_id, 1-i)

        ref, alt_breakend = get_alt_breakend(options, chromx, x, chromy, y, orientation)
        info = get_info(event, mateid=other_id)
        filters = get_filters(event)

        fields = [chromx, x+1, this_id, ref, alt_breakend, 0, filters, info, ":".join(FORMAT)]

        for sample_name, sample in options.samples.items():
            fields.append(get_sample_info(sample, event))

        yield "\t".join(map(str, fields))

def convert_to_vcf(options, results):
    header = get_header(options)

    for line in header.splitlines():
        yield line

    for event_id, result in results.iterrows():
        for line in convert_event(options, event_id, result):
            yield line
<commit_msg>clarification for genotype field in vcf<commit_after>import datetime
import numpy

import grocsvs

VCF_HEADER = \
"""##fileformat=VCFv4.3
##fileDate={date}
##source=grocsvs_v{version}
##reference=file://{reference}
{chromosomes}
##INFO=<ID=ASSEMBLED,Number=0,Type=Flag,Description="Event is supported by a read-cloud-based sequence assembly across the breakpoints">
##INFO=<ID=MATEID,Number=1,Type=String,Description="ID of mate breakend">
##INFO=<ID=SVTYPE,Number=1,Type=String,Description="Type of structural variant">
##FILTER=<ID=NOLONGFRAGS,Description="No samples found with long fragments supporting event (likely segdup)">
##FILTER=<ID=NEARBYSNVS,Description="Very high number of possible SNVs detected in vicinity of breakpoint (likely segdup)">
##FILTER=<ID=ASSEMBLYGAP,Description="Nearby gap in the reference assembly found; these gaps can appear to mimic SVs">
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype (currently either 0/1 when breakpoint is present in sample or 0/0 if not)">
##FORMAT=<ID=BS,Number=1,Type=int,Description="Number of barcodes supporting the event">
##FORMAT=<ID=BT,Number=1,Type=int,Description="Total number of barcodes, calculated as the union of barcodes at each site">
##FORMAT=<ID=F50,Number=1,Type=Float,Description="Median fragment length supporting the breakpoint; estimated as the total across all supporting segments">
##FORMAT=<ID=F90,Number=1,Type=Float,Description="90th percentile of fragment lengths supporting the breakpoint; estimated as the total across all supporting segments">
##FORMAT=<ID=PR,Number=1,Type=Float,Description="p-value for the event, calculated by resampling from the number of supporting and total barcodes">
##FORMAT=<ID=H1x,Number=1,Type=String,Description="Number of supporting barcodes assigned to haplotype 1 (left side of breakpoint)">
##FORMAT=<ID=H2x,Number=1,Type=String,Description="Number of supporting barcodes assigned to haplotype 2 (left side of breakpoint)">
##FORMAT=<ID=H1y,Number=1,Type=String,Description="Number of supporting barcodes assigned to haplotype 1 (right side of breakpoint)">
##FORMAT=<ID=H2y,Number=1,Type=String,Description="Number of supporting barcodes assigned to haplotype 2 (right side of breakpoint)">
{samples}
#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{sample_columns}
"""

CHROMOSOME = "##contig=<ID={name},length={length}>"
SAMPLE = "##SAMPLE=<ID={name},URL=file://{path}>"
FORMAT = ["GT", "BS", "BT", "F50", "F90", "PR", "H1x", "H2x", "H1y", "H2y"]

def get_header(options):
    chromosomes = []
    for chrom, length in options.reference.chrom_lengths.items():
        chromosomes.append(CHROMOSOME.format(name=chrom, length=length))
    chromosomes = "\n".join(chromosomes)

    samples = []
    sample_columns = []
    for sample_name, sample in options.samples.items():
        samples.append(SAMPLE.format(name=sample_name, path=sample.get_10x_dataset().bam))
        sample_columns.append(sample_name)
    samples = "\n".join(samples)
    sample_columns = "\t".join(sample_columns)

    header = VCF_HEADER.format(
            date=datetime.datetime.now().strftime("%Y%m%d"),
            version=grocsvs.__version__,
            reference=options.reference.path,
            chromosomes=chromosomes,
            samples=samples,
            sample_columns=sample_columns
        )

    return header

def get_alt_breakend(options, chromx, x, chromy, y, orientation):
    # note that vcf uses one-based coords so we add 1 to all positions

    ref = options.reference.fasta[chromx][x].upper()
    if orientation == "+-":
        alt = "{}[{}:{}[".format(ref, chromy, y+1)
    elif orientation == "++":
        alt = "{}]{}:{}]".format(ref, chromy, y+1)
    elif orientation == "-+":
        alt = "]{}:{}]{}".format(chromy, y+1, ref)
    elif orientation == "--":
        alt = "[{}:{}[{}".format(chromy, y+1, ref)

    return ref, alt

def get_sample_info(sample, event):
    sample_info = {}
    sample_info["BS"] = event["{}_shared".format(sample.name)]
    sample_info["BT"] = event["{}_total".format(sample.name)]
    if sample_info["BS"] > 5:
        sample_info["GT"] = "0/1"
    else:
        sample_info["GT"] = "0/0"

    if "{}_lengths_50".format(sample.name) in event:
        sample_info["F50"] = event["{}_lengths_50".format(sample.name)]
        sample_info["F90"] = event["{}_lengths_90".format(sample.name)]
    else:
        sample_info["F50"] = "."
        sample_info["F90"] = "."

    sample_info["PR"] = event["{}_p_resampling".format(sample.name)]

    sample_info["H1x"] = event["{}_x_hap0".format(sample.name)]
    sample_info["H2x"] = event["{}_x_hap1".format(sample.name)]
    sample_info["H1y"] = event["{}_y_hap0".format(sample.name)]
    sample_info["H2y"] = event["{}_y_hap1".format(sample.name)]

    sample_info = [sample_info[key] for key in FORMAT]

    return ":".join(map(str, sample_info))

def get_filters(event):
    filters = []

    try:
        if "n=" in event["blacklist"]:
            filters.append("ASSEMBLYGAP")
    except TypeError:
        pass

    if event["nearby_snvs"] >= 15:
        filters.append("NEARBYSNVS")

    if "frag_length_passes" in event and not event["frag_length_passes"]:
        filters.append("NOLONGFRAGS")

    if len(filters) == 0:
        return "PASS"

    return ";".join(filters)

def get_info(event, mateid=None):
    info = []

    info.append("SVTYPE=BND")

    if mateid is not None:
        info.append("MATEID={}".format(mateid))

    if event["assembled"]:
        info.append("ASSEMBLED")

    if len(info) == 0:
        return "."

    return ";".join(info)

def convert_event(options, event_id, event):
    chromx, x, chromy, y, orientation = event["chromx"], event["x"], event["chromy"], event["y"], event["orientation"]

    for i in [0,1]:
        if i == 1:
            chromx, x, chromy, y = chromy, y, chromx, x
            orientation = orientation[1]+orientation[0]

        this_id = "{}:{}".format(event_id, i)
        other_id = "{}:{}".format(event_id, 1-i)

        ref, alt_breakend = get_alt_breakend(options, chromx, x, chromy, y, orientation)
        info = get_info(event, mateid=other_id)
        filters = get_filters(event)

        fields = [chromx, x+1, this_id, ref, alt_breakend, 0, filters, info, ":".join(FORMAT)]

        for sample_name, sample in options.samples.items():
            fields.append(get_sample_info(sample, event))

        yield "\t".join(map(str, fields))

def convert_to_vcf(options, results):
    header = get_header(options)

    for line in header.splitlines():
        yield line

    for event_id, result in results.iterrows():
        for line in convert_event(options, event_id, result):
            yield line
<|endoftext|>


<commit_before>##
# See the file COPYRIGHT for copyright information.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##

"""
Tests for :mod:`ranger-ims-server.model._entry`
"""

from .datetimes import dt1, dt2
from .rangers import rangerBucket, rangerHubcap
from .._entry import ReportEntry
from ...ext.trial import TestCase

__all__ = ()

class ReportEntryTests(TestCase):
    """
    Tests for :class:`ReportEntry`
    """

    text = "Hello"

    def test_ordering_created(self) -> None:
        """
        Report entry ordering sorts by created.
        """
        b = ReportEntry(
            created=dt2,
            author=rangerHubcap,
            automatic=False,
            text=self.text,
        )

        a = ReportEntry(
            created=dt1,
            author=rangerHubcap,
            automatic=False,
            text=self.text,
        )

        self.assertLess(a, b)

    def test_ordering_author(self) -> None:
        """
        Report entry ordering with same created sorts by author.
        """
        b = ReportEntry(
            created=dt1,
            author=rangerHubcap,
            automatic=False,
            text=self.text,
        )

        a = ReportEntry(
            created=dt1,
            author=rangerBucket,
            automatic=False,
            text=self.text,
        )

        self.assertLess(a, b)

    def test_ordering_automatic(self) -> None:
        """
        Report entry ordering with same created and author sorts by automatic,
        where :obj:`True` comes before :obj:`False`.
        """
        b = ReportEntry(
            created=dt1,
            author=rangerHubcap,
            automatic=False,
            text=self.text,
        )

        a = ReportEntry(
            created=dt1,
            author=rangerHubcap,
            automatic=True,
            text=self.text,
        )

        self.assertLess(a, b)
<commit_msg>coverage<commit_after>##
# See the file COPYRIGHT for copyright information.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##

"""
Tests for :mod:`ranger-ims-server.model._entry`
"""

from .datetimes import dt1, dt2
from .rangers import rangerBucket, rangerHubcap
from .._entry import ReportEntry
from ...ext.trial import TestCase

__all__ = ()

class ReportEntryTests(TestCase):
    """
    Tests for :class:`ReportEntry`
    """

    text = "Hello"

    def test_ordering_created(self) -> None:
        """
        Report entry ordering sorts by created.
        """
        b = ReportEntry(
            created=dt2,
            author=rangerHubcap,
            automatic=False,
            text=self.text,
        )

        a = ReportEntry(
            created=dt1,
            author=rangerHubcap,
            automatic=False,
            text=self.text,
        )

        self.assertLess(a, b)
        self.assertLessEqual(a, b)
        self.assertNotEqual(a, b)
        self.assertGreater(b, a)
        self.assertGreaterEqual(b, a)

    def test_ordering_author(self) -> None:
        """
        Report entry ordering with same created sorts by author.
        """
        b = ReportEntry(
            created=dt1,
            author=rangerHubcap,
            automatic=False,
            text=self.text,
        )

        a = ReportEntry(
            created=dt1,
            author=rangerBucket,
            automatic=False,
            text=self.text,
        )

        self.assertLess(a, b)
        self.assertLessEqual(a, b)
        self.assertNotEqual(a, b)
        self.assertGreater(b, a)
        self.assertGreaterEqual(b, a)

    def test_ordering_automatic(self) -> None:
        """
        Report entry ordering with same created and author sorts by automatic,
        where :obj:`True` comes before :obj:`False`.
        """
        b = ReportEntry(
            created=dt1,
            author=rangerHubcap,
            automatic=False,
            text=self.text,
        )

        a = ReportEntry(
            created=dt1,
            author=rangerHubcap,
            automatic=True,
            text=self.text,
        )

        self.assertLess(a, b)
        self.assertLessEqual(a, b)
        self.assertNotEqual(a, b)
        self.assertGreater(b, a)
        self.assertGreaterEqual(b, a)

    def test_eq(self) -> None:
        """
        Test equality.
        """
        a = ReportEntry(
            created=dt1,
            author=rangerHubcap,
            automatic=True,
            text=self.text,
        )

        b = ReportEntry(
            created=dt1,
            author=rangerHubcap,
            automatic=True,
            text=self.text,
        )

        self.assertEqual(a, b)

    def test_eq_otherType(self) -> None:
        """
        Test equality with another type.
        """
        a = ReportEntry(
            created=dt1,
            author=rangerHubcap,
            automatic=True,
            text=self.text,
        )

        self.assertNotEqual(a, object())
<|endoftext|>


<commit_before># Python Classes/Functions used to Analyze Tycho's Output

# ------------------------------------- #
#        Python Package Importing       #
# ------------------------------------- #

# Importing Necessary System Packages
import math
import numpy as np
import time as tp
import matplotlib as plt
import random as rp

# Import the Amuse Base Packages
from amuse import datamodel
from amuse.units import nbody_system
from amuse.units import units
from amuse.units import constants
from amuse.datamodel import particle_attributes
from amuse.io import *
from amuse.lab import *

# Import the Amuse Stellar Packages
from amuse.ic.kingmodel import new_king_model
from amuse.ic.kroupa import new_kroupa_mass_distribution

# Import cPickle/Pickle
try:
   import cPickle as pickle
except:
   import pickle

# ------------------------------------- #
#           Defining Functions          #
# ------------------------------------- #
<commit_msg>Delete analysis.py<commit_after><|endoftext|>


<commit_before>
def getUniqueFilename(dir = None, base = None):
  """
    DESCRP: Generate a filename in the directory <dir> which is
            unique (i.e. not in use at the moment)
    PARAMS: dir  -- the directory to look in. If None, use CWD
            base -- use this as the base name for the filename
    RETURN: string -- the filename generated
  """
  while True :
    fn = str(random.randint(0,100000)) + ".tmp"
    if not os.path.exists(fn) : break
  return fn

def linesInFile(fn):
  f = open(fn)
  t = sum(1 for line in f)
  f.close()
  return t<commit_msg>updating file size method to handle different file descriptors<commit_after>
def getUniqueFilename(dir = None, base = None):
  """
    DESCRP: Generate a filename in the directory <dir> which is
            unique (i.e. not in use at the moment)
    PARAMS: dir  -- the directory to look in. If None, use CWD
            base -- use this as the base name for the filename
    RETURN: string -- the filename generated
  """
  while True :
    fn = str(random.randint(0,100000)) + ".tmp"
    if not os.path.exists(fn) : break
  return fn

def linesInFile(fd):
  if type(fd).__name__ == "str" : f = open(fd)
  else : f = fd
  t = sum(1 for line in f)
  f.close()
  return t<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -
import logging
import requests.exceptions
from datetime import datetime, timedelta
from sqlalchemy import desc
from werkzeug.exceptions import BadRequest

from flask import Blueprint, g, render_template, flash, \
                  redirect, request, url_for
from flask.ext.babel import gettext
from flask.ext.login import login_required

import conf
from web import utils
from web.lib.view_utils import etag_match
from web.lib.feed_utils import construct_feed_from
from web.forms import AddFeedForm
from web.controllers import (CategoryController, FeedController,
                                      ArticleController)

logger = logging.getLogger(__name__)
feeds_bp = Blueprint('feeds', __name__, url_prefix='/feeds')
feed_bp = Blueprint('feed', __name__, url_prefix='/feed')

@feeds_bp.route('/', methods=['GET'])
@login_required
@etag_match
def feeds():
    "Lists the subscribed  feeds in a table."
    art_contr = ArticleController(g.user.id)
    return render_template('feeds.html',
            feeds=FeedController(g.user.id).read(),
            unread_article_count=art_contr.count_by_feed(readed=False),
            article_count=art_contr.count_by_feed())

@feed_bp.route('/<int:feed_id>', methods=['GET'])
@login_required
@etag_match
def feed(feed_id=None):
    "Presents detailed information about a feed."
    feed = FeedController(g.user.id).get(id=feed_id)
    word_size = 6
    category = None
    if feed.category_id:
        category = CategoryController(g.user.id).get(id=feed.category_id)
    articles = ArticleController(g.user.id) \
            .read(feed_id=feed_id) \
            .order_by(desc("date")).all()
    top_words = utils.top_words(articles, n=50, size=int(word_size))
    tag_cloud = utils.tag_cloud(top_words)

    today = datetime.now()
    try:
        last_article = articles[0].date
        first_article = articles[-1].date
        delta = last_article - first_article
        average = round(float(len(articles)) / abs(delta.days), 2)
    except:
        last_article = datetime.fromtimestamp(0)
        first_article = datetime.fromtimestamp(0)
        delta = last_article - first_article
        average = 0
    elapsed = today - last_article

    return render_template('feed.html',
                           head_titles=[utils.clear_string(feed.title)],
                           feed=feed, tag_cloud=tag_cloud,
                           first_post_date=first_article,
                           end_post_date=last_article, category=category,
                           average=average, delta=delta, elapsed=elapsed)

@feed_bp.route('/delete/<feed_id>', methods=['GET'])
@login_required
def delete(feed_id=None):
    feed_contr = FeedController(g.user.id)
    feed = feed_contr.get(id=feed_id)
    feed_contr.delete(feed_id)
    flash(gettext("Feed %(feed_title)s successfully deleted.",
                  feed_title=feed.title), 'success')
    return redirect(url_for('home'))

@feed_bp.route('/reset_errors/<int:feed_id>', methods=['GET', 'POST'])
@login_required
def reset_errors(feed_id):
    feed_contr = FeedController(g.user.id)
    feed = feed_contr.get(id=feed_id)
    feed_contr.update({'id': feed_id}, {'error_count': 0, 'last_error': ''})
    flash(gettext('Feed %(feed_title)r successfully updated.',
                  feed_title=feed.title), 'success')
    return redirect(request.referrer or url_for('home'))

@feed_bp.route('/bookmarklet', methods=['GET', 'POST'])
@login_required
def bookmarklet():
    feed_contr = FeedController(g.user.id)
    url = (request.args if request.method == 'GET' else request.form)\
            .get('url', None)
    if not url:
        flash(gettext("Couldn't add feed: url missing."), "error")
        raise BadRequest("url is missing")

    feed_exists = list(feed_contr.read(__or__={'link': url, 'site_link': url}))
    if feed_exists:
        flash(gettext("Couldn't add feed: feed already exists."),
                "warning")
        return redirect(url_for('feed.form', feed_id=feed_exists[0].id))

    try:
        feed = construct_feed_from(url)
    except requests.exceptions.ConnectionError:
        flash(gettext("Impossible to connect to the address: {}.".format(url)),
              "danger")
        return redirect(url_for('home'))
    except Exception:
        logger.exception('something bad happened when fetching %r', url)
        return redirect(url_for('home'))
    if not feed.get('link'):
        feed['enabled'] = False
        flash(gettext("Couldn't find a feed url, you'll need to find a Atom or"
                      " RSS link manually and reactivate this feed"),
              'warning')
    feed = feed_contr.create(**feed)
    flash(gettext('Feed was successfully created.'), 'success')
    if feed.enabled and conf.CRAWLING_METHOD == "classic":
        utils.fetch(g.user.id, feed.id)
        flash(gettext("Downloading articles for the new feed..."), 'info')
    return redirect(url_for('feed.form', feed_id=feed.id))

@feed_bp.route('/update/<action>/<int:feed_id>', methods=['GET', 'POST'])
@feeds_bp.route('/update/<action>', methods=['GET', 'POST'])
@login_required
def update(action, feed_id=None):
    readed = action == 'read'
    filters = {'readed__ne': readed}

    nb_days = request.args.get('nb_days', 0, type=int)
    if nb_days != 0:
        filters['date__lt'] = datetime.now() - timedelta(days=nb_days)

    if feed_id:
        filters['feed_id'] = feed_id
    ArticleController(g.user.id).update(filters, {'readed': readed})
    flash(gettext('Feed successfully updated.'), 'success')
    return redirect(request.referrer or url_for('home'))

@feed_bp.route('/create', methods=['GET'])
@feed_bp.route('/edit/<int:feed_id>', methods=['GET'])
@login_required
@etag_match
def form(feed_id=None):
    action = gettext("Add a feed")
    categories = CategoryController(g.user.id).read()
    head_titles = [action]
    if feed_id is None:
        form = AddFeedForm()
        form.set_category_choices(categories)
        return render_template('edit_feed.html', action=action,
                               head_titles=head_titles, form=form)
    feed = FeedController(g.user.id).get(id=feed_id)
    form = AddFeedForm(obj=feed)
    form.set_category_choices(categories)
    action = gettext('Edit feed')
    head_titles = [action]
    if feed.title:
        head_titles.append(feed.title)
    return render_template('edit_feed.html', action=action,
                           head_titles=head_titles, categories=categories,
                           form=form, feed=feed)

@feed_bp.route('/create', methods=['POST'])
@feed_bp.route('/edit/<int:feed_id>', methods=['POST'])
@login_required
def process_form(feed_id=None):
    form = AddFeedForm()
    feed_contr = FeedController(g.user.id)
    form.set_category_choices(CategoryController(g.user.id).read())

    if not form.validate():
        return render_template('edit_feed.html', form=form)
    existing_feeds = list(feed_contr.read(link=form.link.data))
    if existing_feeds and feed_id is None:
        flash(gettext("Couldn't add feed: feed already exists."), "warning")
        return redirect(url_for('feed.form', feed_id=existing_feeds[0].id))
    # Edit an existing feed
    feed_attr = {'title': form.title.data, 'enabled': form.enabled.data,
                 'link': form.link.data, 'site_link': form.site_link.data,
                 'filters': [], 'category_id': form.category_id.data}
    if not feed_attr['category_id']:
        del feed_attr['category_id']

    for filter_attr in ('type', 'pattern', 'action on', 'action'):
        for i, value in enumerate(
                request.form.getlist(filter_attr.replace(' ', '_'))):
            if i >= len(feed_attr['filters']):
                feed_attr['filters'].append({})
            feed_attr['filters'][i][filter_attr] = value

    if feed_id is not None:
        feed_contr.update({'id': feed_id}, feed_attr)
        flash(gettext('Feed %(feed_title)r successfully updated.',
                      feed_title=feed_attr['title']), 'success')
        return redirect(url_for('feed.form', feed_id=feed_id))

    # Create a new feed
    new_feed = feed_contr.create(**feed_attr)

    flash(gettext('Feed %(feed_title)r successfully created.',
                  feed_title=new_feed.title), 'success')

    if conf.CRAWLING_METHOD == "classic":
        utils.fetch(g.user.id, new_feed.id)
        flash(gettext("Downloading articles for the new feed..."), 'info')

    return redirect(url_for('feed.form', feed_id=new_feed.id))

@feeds_bp.route('/inactives', methods=['GET'])
@login_required
def inactives():
    """
    List of inactive feeds.
    """
    nb_days = int(request.args.get('nb_days', 365))
    inactives = FeedController(g.user.id).get_inactives(nb_days)
    return render_template('inactives.html',
                           inactives=inactives, nb_days=nb_days)

@feed_bp.route('/duplicates/<int:feed_id>', methods=['GET'])
@login_required
def duplicates(feed_id):
    """
    Return duplicates article for a feed.
    """
    feed, duplicates = FeedController(g.user.id).get_duplicates(feed_id)
    if len(duplicates) == 0:
        flash(gettext('No duplicates in the feed "{}".').format(feed.title),
                'info')
        return redirect('home')
    return render_template('duplicates.html', duplicates=duplicates, feed=feed)
<commit_msg>Fixed an error when redirecting to home from the 'duplicates' page.<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -
import logging
import requests.exceptions
from datetime import datetime, timedelta
from sqlalchemy import desc
from werkzeug.exceptions import BadRequest

from flask import Blueprint, g, render_template, flash, \
                  redirect, request, url_for
from flask.ext.babel import gettext
from flask.ext.login import login_required

import conf
from web import utils
from web.lib.view_utils import etag_match
from web.lib.feed_utils import construct_feed_from
from web.forms import AddFeedForm
from web.controllers import (CategoryController, FeedController,
                                      ArticleController)

logger = logging.getLogger(__name__)
feeds_bp = Blueprint('feeds', __name__, url_prefix='/feeds')
feed_bp = Blueprint('feed', __name__, url_prefix='/feed')

@feeds_bp.route('/', methods=['GET'])
@login_required
@etag_match
def feeds():
    "Lists the subscribed  feeds in a table."
    art_contr = ArticleController(g.user.id)
    return render_template('feeds.html',
            feeds=FeedController(g.user.id).read(),
            unread_article_count=art_contr.count_by_feed(readed=False),
            article_count=art_contr.count_by_feed())

@feed_bp.route('/<int:feed_id>', methods=['GET'])
@login_required
@etag_match
def feed(feed_id=None):
    "Presents detailed information about a feed."
    feed = FeedController(g.user.id).get(id=feed_id)
    word_size = 6
    category = None
    if feed.category_id:
        category = CategoryController(g.user.id).get(id=feed.category_id)
    articles = ArticleController(g.user.id) \
            .read(feed_id=feed_id) \
            .order_by(desc("date")).all()
    top_words = utils.top_words(articles, n=50, size=int(word_size))
    tag_cloud = utils.tag_cloud(top_words)

    today = datetime.now()
    try:
        last_article = articles[0].date
        first_article = articles[-1].date
        delta = last_article - first_article
        average = round(float(len(articles)) / abs(delta.days), 2)
    except:
        last_article = datetime.fromtimestamp(0)
        first_article = datetime.fromtimestamp(0)
        delta = last_article - first_article
        average = 0
    elapsed = today - last_article

    return render_template('feed.html',
                           head_titles=[utils.clear_string(feed.title)],
                           feed=feed, tag_cloud=tag_cloud,
                           first_post_date=first_article,
                           end_post_date=last_article, category=category,
                           average=average, delta=delta, elapsed=elapsed)

@feed_bp.route('/delete/<feed_id>', methods=['GET'])
@login_required
def delete(feed_id=None):
    feed_contr = FeedController(g.user.id)
    feed = feed_contr.get(id=feed_id)
    feed_contr.delete(feed_id)
    flash(gettext("Feed %(feed_title)s successfully deleted.",
                  feed_title=feed.title), 'success')
    return redirect(url_for('home'))

@feed_bp.route('/reset_errors/<int:feed_id>', methods=['GET', 'POST'])
@login_required
def reset_errors(feed_id):
    feed_contr = FeedController(g.user.id)
    feed = feed_contr.get(id=feed_id)
    feed_contr.update({'id': feed_id}, {'error_count': 0, 'last_error': ''})
    flash(gettext('Feed %(feed_title)r successfully updated.',
                  feed_title=feed.title), 'success')
    return redirect(request.referrer or url_for('home'))

@feed_bp.route('/bookmarklet', methods=['GET', 'POST'])
@login_required
def bookmarklet():
    feed_contr = FeedController(g.user.id)
    url = (request.args if request.method == 'GET' else request.form)\
            .get('url', None)
    if not url:
        flash(gettext("Couldn't add feed: url missing."), "error")
        raise BadRequest("url is missing")

    feed_exists = list(feed_contr.read(__or__={'link': url, 'site_link': url}))
    if feed_exists:
        flash(gettext("Couldn't add feed: feed already exists."),
                "warning")
        return redirect(url_for('feed.form', feed_id=feed_exists[0].id))

    try:
        feed = construct_feed_from(url)
    except requests.exceptions.ConnectionError:
        flash(gettext("Impossible to connect to the address: {}.".format(url)),
              "danger")
        return redirect(url_for('home'))
    except Exception:
        logger.exception('something bad happened when fetching %r', url)
        return redirect(url_for('home'))
    if not feed.get('link'):
        feed['enabled'] = False
        flash(gettext("Couldn't find a feed url, you'll need to find a Atom or"
                      " RSS link manually and reactivate this feed"),
              'warning')
    feed = feed_contr.create(**feed)
    flash(gettext('Feed was successfully created.'), 'success')
    if feed.enabled and conf.CRAWLING_METHOD == "classic":
        utils.fetch(g.user.id, feed.id)
        flash(gettext("Downloading articles for the new feed..."), 'info')
    return redirect(url_for('feed.form', feed_id=feed.id))

@feed_bp.route('/update/<action>/<int:feed_id>', methods=['GET', 'POST'])
@feeds_bp.route('/update/<action>', methods=['GET', 'POST'])
@login_required
def update(action, feed_id=None):
    readed = action == 'read'
    filters = {'readed__ne': readed}

    nb_days = request.args.get('nb_days', 0, type=int)
    if nb_days != 0:
        filters['date__lt'] = datetime.now() - timedelta(days=nb_days)

    if feed_id:
        filters['feed_id'] = feed_id
    ArticleController(g.user.id).update(filters, {'readed': readed})
    flash(gettext('Feed successfully updated.'), 'success')
    return redirect(request.referrer or url_for('home'))

@feed_bp.route('/create', methods=['GET'])
@feed_bp.route('/edit/<int:feed_id>', methods=['GET'])
@login_required
@etag_match
def form(feed_id=None):
    action = gettext("Add a feed")
    categories = CategoryController(g.user.id).read()
    head_titles = [action]
    if feed_id is None:
        form = AddFeedForm()
        form.set_category_choices(categories)
        return render_template('edit_feed.html', action=action,
                               head_titles=head_titles, form=form)
    feed = FeedController(g.user.id).get(id=feed_id)
    form = AddFeedForm(obj=feed)
    form.set_category_choices(categories)
    action = gettext('Edit feed')
    head_titles = [action]
    if feed.title:
        head_titles.append(feed.title)
    return render_template('edit_feed.html', action=action,
                           head_titles=head_titles, categories=categories,
                           form=form, feed=feed)

@feed_bp.route('/create', methods=['POST'])
@feed_bp.route('/edit/<int:feed_id>', methods=['POST'])
@login_required
def process_form(feed_id=None):
    form = AddFeedForm()
    feed_contr = FeedController(g.user.id)
    form.set_category_choices(CategoryController(g.user.id).read())

    if not form.validate():
        return render_template('edit_feed.html', form=form)
    existing_feeds = list(feed_contr.read(link=form.link.data))
    if existing_feeds and feed_id is None:
        flash(gettext("Couldn't add feed: feed already exists."), "warning")
        return redirect(url_for('feed.form', feed_id=existing_feeds[0].id))
    # Edit an existing feed
    feed_attr = {'title': form.title.data, 'enabled': form.enabled.data,
                 'link': form.link.data, 'site_link': form.site_link.data,
                 'filters': [], 'category_id': form.category_id.data}
    if not feed_attr['category_id']:
        del feed_attr['category_id']

    for filter_attr in ('type', 'pattern', 'action on', 'action'):
        for i, value in enumerate(
                request.form.getlist(filter_attr.replace(' ', '_'))):
            if i >= len(feed_attr['filters']):
                feed_attr['filters'].append({})
            feed_attr['filters'][i][filter_attr] = value

    if feed_id is not None:
        feed_contr.update({'id': feed_id}, feed_attr)
        flash(gettext('Feed %(feed_title)r successfully updated.',
                      feed_title=feed_attr['title']), 'success')
        return redirect(url_for('feed.form', feed_id=feed_id))

    # Create a new feed
    new_feed = feed_contr.create(**feed_attr)

    flash(gettext('Feed %(feed_title)r successfully created.',
                  feed_title=new_feed.title), 'success')

    if conf.CRAWLING_METHOD == "classic":
        utils.fetch(g.user.id, new_feed.id)
        flash(gettext("Downloading articles for the new feed..."), 'info')

    return redirect(url_for('feed.form', feed_id=new_feed.id))

@feeds_bp.route('/inactives', methods=['GET'])
@login_required
def inactives():
    """
    List of inactive feeds.
    """
    nb_days = int(request.args.get('nb_days', 365))
    inactives = FeedController(g.user.id).get_inactives(nb_days)
    return render_template('inactives.html',
                           inactives=inactives, nb_days=nb_days)

@feed_bp.route('/duplicates/<int:feed_id>', methods=['GET'])
@login_required
def duplicates(feed_id):
    """
    Return duplicates article for a feed.
    """
    feed, duplicates = FeedController(g.user.id).get_duplicates(feed_id)
    if len(duplicates) == 0:
        flash(gettext('No duplicates in the feed "{}".').format(feed.title),
                'info')
        return redirect(url_for('home'))
    return render_template('duplicates.html', duplicates=duplicates, feed=feed)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import datetime
import logging
import math
import os
import random
import re
import signal
import subprocess
import sys
import telnetlib
import time

import IPy

import vrnetlab

def handle_SIGCHLD(signal, frame):
    os.waitpid(-1, os.WNOHANG)

def handle_SIGTERM(signal, frame):
    sys.exit(0)

signal.signal(signal.SIGINT, handle_SIGTERM)
signal.signal(signal.SIGTERM, handle_SIGTERM)
signal.signal(signal.SIGCHLD, handle_SIGCHLD)

TRACE_LEVEL_NUM = 9
logging.addLevelName(TRACE_LEVEL_NUM, "TRACE")
def trace(self, message, *args, **kws):
    # Yes, logger takes its '*args' as 'args'.
    if self.isEnabledFor(TRACE_LEVEL_NUM):
        self._log(TRACE_LEVEL_NUM, message, args, **kws)
logging.Logger.trace = trace

def mangle_uuid(uuid):
    """ Mangle the UUID to fix endianness mismatch on first part
    """
    parts = uuid.split("-")

    new_parts = [
        uuid_rev_part(parts[0]),
        uuid_rev_part(parts[1]),
        uuid_rev_part(parts[2]),
        parts[3],
        parts[4]
    ]

    return '-'.join(new_parts)

def uuid_rev_part(part):
    """ Reverse part of a UUID
    """
    res = ""
    for i in reversed(range(0, len(part), 2)):
        res += part[i]
        res += part[i+1]
    return res

class SROS_vm(vrnetlab.VM):
    def __init__(self, username, password, num=0):
        super(SROS_vm, self).__init__(username, password, disk_image = "/sros.qcow2", num=num)

        self.uuid = "00000000-0000-0000-0000-000000000000"
        self.read_license()

    def bootstrap_spin(self):
        """ This function should be called periodically to do work.
        """

        if self.spins > 60:
            # too many spins with no result, probably means SROS hasn't started
            # successfully, so we restart it
            self.logger.warning("no output from serial console, restarting VM")
            self.stop()
            self.start()
            self.spins = 0
            return

        (ridx, match, res) = self.tn.expect([b"Login:", b"^[^ ]+#"], 1)
        if match: # got a match!
            if ridx == 0: # matched login prompt, so should login
                self.logger.debug("matched login prompt")
                self.wait_write("admin", wait=None)
                self.wait_write("admin", wait="Password:")
            # run main config!
            self.bootstrap_config()
            # close telnet connection
            self.tn.close()
            # calc startup time
            startup_time = datetime.datetime.now() - self.start_time
            self.logger.info("Startup complete in: %s" % startup_time)
            self.running = True
            return

        # no match, if we saw some output from the router it's probably
        # booting, so let's give it some more time
        if res != b'':
            self.logger.trace("OUTPUT: %s" % res.decode())
            # reset spins if we saw some output
            self.spins = 0

        self.spins += 1

        return

    def read_license(self):
        """ Read the license file, if it exists, and extract the UUID and start
            time of the license
        """
        if not os.path.isfile("/tftpboot/license.txt"):
            self.logger.info("No license file found")
            return

        lic_file = open("/tftpboot/license.txt", "r")
        license = lic_file.read()
        lic_file.close()
        try:
            uuid_input = license.split(" ")[0]
            self.uuid = mangle_uuid(uuid_input)
            m = re.search("([0-9]{4}-[0-9]{2}-)([0-9]{2})", license)
            if m:
                self.fake_start_date = "%s%02d" % (m.group(1), int(m.group(2))+1)
        except:
            raise ValueError("Unable to parse license file")
        self.logger.info("License file found for UUID %s with start date %s" % (self.uuid, self.fake_start_date))

class SROS_integrated(SROS_vm):
    """ Integrated VSR-SIM
    """
    def __init__(self, username, password):
        super(SROS_integrated, self).__init__(username, password)

        self.num_nics = 5
        self.smbios = ["type=1,product=TIMOS:address=10.0.0.15/24@active license-file=tftp://10.0.0.2/license.txt slot=A chassis=SR-c12 card=cfm-xp-b mda/1=m5-1gb-sfp-b"]

    def gen_mgmt(self):
        """ Generate mgmt interface(s)

            We override the default function since we want a fake NIC in there
        """
        # call parent function to generate first mgmt interface (e1000)
        res = super(SROS_integrated, self).gen_mgmt()
        # add virtio NIC for internal control plane interface to vFPC
        res.append("-device")
        res.append("e1000,netdev=dummy0,mac=%s" % vrnetlab.gen_mac(1))
        res.append("-netdev")
        res.append("tap,ifname=dummy0,id=dummy0,script=no,downscript=no")
        return res

    def bootstrap_config(self):
        """ Do the actual bootstrap config
        """
        if self.username and self.password:
            self.wait_write("configure system security user \"%s\" password %s" % (self.username, self.password))
            self.wait_write("configure system security user \"%s\" access console netconf" % (self.username))
            self.wait_write("configure system security user \"%s\" console member \"administrative\" \"default\"" % (self.username))
        self.wait_write("configure system netconf no shutdown")
        self.wait_write("configure system security profile \"administrative\" netconf base-op-authorization lock")
        self.wait_write("configure card 1 mda 1 shutdown")
        self.wait_write("configure card 1 mda 1 no mda-type")
        self.wait_write("configure card 1 shutdown")
        self.wait_write("configure card 1 no card-type")
        self.wait_write("configure card 1 card-type iom-xp-b")
        self.wait_write("configure card 1 mcm 1 mcm-type mcm-xp")
        self.wait_write("configure card 1 mda 1 mda-type m5-1gb-sfp-b")
        self.wait_write("configure card 1 no shutdown")
        self.wait_write("admin save")
        self.wait_write("logout")

class SROS_cp(SROS_vm):
    """ Control plane for distributed VSR-SIM
    """
    def __init__(self, username, password, num_lc=1):
        super(SROS_cp, self).__init__(username, password)
        self.num_lc = num_lc

        self.num_nics = 0
        self.smbios = ["type=1,product=TIMOS:address=10.0.0.15/24@active license-file=tftp://10.0.0.2/license.txt chassis=XRS-20 slot=A card=cpm-x20"]

    def start(self):
        # use parent class start() function
        super(SROS_cp, self).start()
        # add interface to internal control plane bridge
        vrnetlab.run_command(["brctl", "addif", "int_cp", "vcp-int"])
        vrnetlab.run_command(["ip", "link", "set", "vcp-int", "up"])

    def gen_mgmt(self):
        """ Generate mgmt interface(s)

            We override the default function since we want a NIC to the vFPC
        """
        # call parent function to generate first mgmt interface (e1000)
        res = super(SROS_cp, self).gen_mgmt()
        # add virtio NIC for internal control plane interface to vFPC
        res.append("-device")
        res.append("e1000,netdev=vcp-int,mac=%s" % vrnetlab.gen_mac(1))
        res.append("-netdev")
        res.append("tap,ifname=vcp-int,id=vcp-int,script=no,downscript=no")
        return res

    def bootstrap_config(self):
        """ Do the actual bootstrap config
        """
        if self.username and self.password:
            self.wait_write("configure system security user \"%s\" password %s" % (self.username, self.password))
            self.wait_write("configure system security user \"%s\" access console netconf" % (self.username))
            self.wait_write("configure system security user \"%s\" console member \"administrative\" \"default\"" % (self.username))
        self.wait_write("configure system netconf no shutdown")
        self.wait_write("configure system security profile \"administrative\" netconf base-op-authorization lock")

        # configure SFMs
        for i in range(1, 16):
            self.wait_write("configure sfm {} sfm-type sfm-x20-b".format(i))

        # configure line card & MDAs
        for i in range(1, self.num_lc+1):
            self.wait_write("configure card {} card-type xcm-x20".format(i))
            self.wait_write("configure card {} mda 1 mda-type cx20-10g-sfp".format(i))

        self.wait_write("admin save")
        self.wait_write("logout")

class SROS_lc(SROS_vm):
    """ Line card for distributed VSR-SIM
    """
    def __init__(self, slot=1):
        super(SROS_lc, self).__init__(None, None, num=slot)
        self.slot = slot

        self.num_nics = 6
        self.smbios = ["type=1,product=TIMOS:chassis=XRS-20 slot={} card=xcm-x20 mda/1=cx20-10g-sfp".format(slot)]

    def start(self):
        # use parent class start() function
        super(SROS_lc, self).start()
        # add interface to internal control plane bridge
        vrnetlab.run_command(["brctl", "addif", "int_cp", "vfpc{}-int".format(self.slot)])
        vrnetlab.run_command(["ip", "link", "set", "vfpc{}-int".format(self.slot), "up"])

    def gen_mgmt(self):
        """ Generate mgmt interface
        """
        res = []
        # mgmt interface
        res.extend(["-device", "e1000,netdev=mgmt,mac=%s" % vrnetlab.gen_mac(0)])
        res.extend(["-netdev", "user,id=mgmt,net=10.0.0.0/24"])
        # internal control plane interface to vFPC
        res.extend(["-device", "e1000,netdev=vfpc-int,mac=%s" %
                    vrnetlab.gen_mac(0)])
        res.extend(["-netdev",
                    "tap,ifname=vfpc{}-int,id=vfpc-int,script=no,downscript=no".format(self.slot)])
        return res

    def gen_nics(self):
        """ Generate qemu args for the normal traffic carrying interface(s)
        """
        res = []
        # TODO: should this offset business be put in the common vrnetlab?
        offset = 6 * (self.slot-1)
        for j in range(0, self.num_nics):
            i = offset + j + 1
            res.append("-device")
            res.append(self.nic_type + ",netdev=p%(i)02d,mac=%(mac)s"
                       % { 'i': i, 'mac': vrnetlab.gen_mac(i) })
            res.append("-netdev")
            res.append("socket,id=p%(i)02d,listen=:100%(i)02d"
                       % { 'i': i })
        return res

    def bootstrap_spin(self):
        """ We have nothing to do for VSR-SIM line cards
        """
        self.running = True
        self.tn.close()
        return

class SROS(vrnetlab.VR):
    def __init__(self, username, password, num_nics):
        super(SROS, self).__init__(username, password)

        # move files into place
        for e in os.listdir("/"):
            if re.search("\.qcow2$", e):
                os.rename("/" + e, "/sros.qcow2")
            if re.search("\.license$", e):
                os.rename("/" + e, "/tftpboot/license.txt")

        if os.path.isfile("/tftpboot/license.txt"):
            self.logger.info("License found")
            self.license = True

        self.logger.error("Number of NICS: " + str(num_nics))
        # if we have more than 5 NICs we use distributed VSR-SIM
        if num_nics > 5:
            if not self.license:
                self.logger.error("More than 5 NICs require distributed VSR which requires a license but no license is found")
                sys.exit(1)

            num_lc = math.ceil(num_nics / 6)
            self.logger.info("Number of linecards: " + str(num_lc))
            self.vms = [ SROS_cp(username, password, num_lc=num_lc) ]
            for i in range(1, num_lc+1):
                self.vms.append(SROS_lc(i))

        else: # 5 ports or less means integrated VSR-SIM
            self.vms = [ SROS_integrated(username, password) ]

        # set up bridge for connecting CP with LCs
        vrnetlab.run_command(["brctl", "addbr", "int_cp"])
        vrnetlab.run_command(["ip", "link", "set", "int_cp", "up"])

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description='')
    parser.add_argument('--trace', action='store_true', help='enable trace level logging')
    parser.add_argument('--username', default='vrnetlab', help='Username')
    parser.add_argument('--password', default='VR-netlab9', help='Password')
    parser.add_argument('--num-nics', default=5, help='Number of NICs')
    args = parser.parse_args()

    LOG_FORMAT = "%(asctime)s: %(module)-10s %(levelname)-8s %(message)s"
    logging.basicConfig(format=LOG_FORMAT)
    logger = logging.getLogger()

    logger.setLevel(logging.DEBUG)
    if args.trace:
        logger.setLevel(1)

    ia = SROS(args.username, args.password, num_nics=int(args.num_nics))
    ia.start()
<commit_msg>sros: handle lacking license<commit_after>#!/usr/bin/env python3

import datetime
import logging
import math
import os
import random
import re
import signal
import subprocess
import sys
import telnetlib
import time

import IPy

import vrnetlab

def handle_SIGCHLD(signal, frame):
    os.waitpid(-1, os.WNOHANG)

def handle_SIGTERM(signal, frame):
    sys.exit(0)

signal.signal(signal.SIGINT, handle_SIGTERM)
signal.signal(signal.SIGTERM, handle_SIGTERM)
signal.signal(signal.SIGCHLD, handle_SIGCHLD)

TRACE_LEVEL_NUM = 9
logging.addLevelName(TRACE_LEVEL_NUM, "TRACE")
def trace(self, message, *args, **kws):
    # Yes, logger takes its '*args' as 'args'.
    if self.isEnabledFor(TRACE_LEVEL_NUM):
        self._log(TRACE_LEVEL_NUM, message, args, **kws)
logging.Logger.trace = trace

def mangle_uuid(uuid):
    """ Mangle the UUID to fix endianness mismatch on first part
    """
    parts = uuid.split("-")

    new_parts = [
        uuid_rev_part(parts[0]),
        uuid_rev_part(parts[1]),
        uuid_rev_part(parts[2]),
        parts[3],
        parts[4]
    ]

    return '-'.join(new_parts)

def uuid_rev_part(part):
    """ Reverse part of a UUID
    """
    res = ""
    for i in reversed(range(0, len(part), 2)):
        res += part[i]
        res += part[i+1]
    return res

class SROS_vm(vrnetlab.VM):
    def __init__(self, username, password, num=0):
        super(SROS_vm, self).__init__(username, password, disk_image = "/sros.qcow2", num=num)

        self.uuid = "00000000-0000-0000-0000-000000000000"
        self.read_license()

    def bootstrap_spin(self):
        """ This function should be called periodically to do work.
        """

        if self.spins > 60:
            # too many spins with no result, probably means SROS hasn't started
            # successfully, so we restart it
            self.logger.warning("no output from serial console, restarting VM")
            self.stop()
            self.start()
            self.spins = 0
            return

        (ridx, match, res) = self.tn.expect([b"Login:", b"^[^ ]+#"], 1)
        if match: # got a match!
            if ridx == 0: # matched login prompt, so should login
                self.logger.debug("matched login prompt")
                self.wait_write("admin", wait=None)
                self.wait_write("admin", wait="Password:")
            # run main config!
            self.bootstrap_config()
            # close telnet connection
            self.tn.close()
            # calc startup time
            startup_time = datetime.datetime.now() - self.start_time
            self.logger.info("Startup complete in: %s" % startup_time)
            self.running = True
            return

        # no match, if we saw some output from the router it's probably
        # booting, so let's give it some more time
        if res != b'':
            self.logger.trace("OUTPUT: %s" % res.decode())
            # reset spins if we saw some output
            self.spins = 0

        self.spins += 1

        return

    def read_license(self):
        """ Read the license file, if it exists, and extract the UUID and start
            time of the license
        """
        if not os.path.isfile("/tftpboot/license.txt"):
            self.logger.info("No license file found")
            return

        lic_file = open("/tftpboot/license.txt", "r")
        license = lic_file.read()
        lic_file.close()
        try:
            uuid_input = license.split(" ")[0]
            self.uuid = mangle_uuid(uuid_input)
            m = re.search("([0-9]{4}-[0-9]{2}-)([0-9]{2})", license)
            if m:
                self.fake_start_date = "%s%02d" % (m.group(1), int(m.group(2))+1)
        except:
            raise ValueError("Unable to parse license file")
        self.logger.info("License file found for UUID %s with start date %s" % (self.uuid, self.fake_start_date))

class SROS_integrated(SROS_vm):
    """ Integrated VSR-SIM
    """
    def __init__(self, username, password):
        super(SROS_integrated, self).__init__(username, password)

        self.num_nics = 5
        self.smbios = ["type=1,product=TIMOS:address=10.0.0.15/24@active license-file=tftp://10.0.0.2/license.txt slot=A chassis=SR-c12 card=cfm-xp-b mda/1=m5-1gb-sfp-b"]

    def gen_mgmt(self):
        """ Generate mgmt interface(s)

            We override the default function since we want a fake NIC in there
        """
        # call parent function to generate first mgmt interface (e1000)
        res = super(SROS_integrated, self).gen_mgmt()
        # add virtio NIC for internal control plane interface to vFPC
        res.append("-device")
        res.append("e1000,netdev=dummy0,mac=%s" % vrnetlab.gen_mac(1))
        res.append("-netdev")
        res.append("tap,ifname=dummy0,id=dummy0,script=no,downscript=no")
        return res

    def bootstrap_config(self):
        """ Do the actual bootstrap config
        """
        if self.username and self.password:
            self.wait_write("configure system security user \"%s\" password %s" % (self.username, self.password))
            self.wait_write("configure system security user \"%s\" access console netconf" % (self.username))
            self.wait_write("configure system security user \"%s\" console member \"administrative\" \"default\"" % (self.username))
        self.wait_write("configure system netconf no shutdown")
        self.wait_write("configure system security profile \"administrative\" netconf base-op-authorization lock")
        self.wait_write("configure card 1 mda 1 shutdown")
        self.wait_write("configure card 1 mda 1 no mda-type")
        self.wait_write("configure card 1 shutdown")
        self.wait_write("configure card 1 no card-type")
        self.wait_write("configure card 1 card-type iom-xp-b")
        self.wait_write("configure card 1 mcm 1 mcm-type mcm-xp")
        self.wait_write("configure card 1 mda 1 mda-type m5-1gb-sfp-b")
        self.wait_write("configure card 1 no shutdown")
        self.wait_write("admin save")
        self.wait_write("logout")

class SROS_cp(SROS_vm):
    """ Control plane for distributed VSR-SIM
    """
    def __init__(self, username, password, num_lc=1):
        super(SROS_cp, self).__init__(username, password)
        self.num_lc = num_lc

        self.num_nics = 0
        self.smbios = ["type=1,product=TIMOS:address=10.0.0.15/24@active license-file=tftp://10.0.0.2/license.txt chassis=XRS-20 slot=A card=cpm-x20"]

    def start(self):
        # use parent class start() function
        super(SROS_cp, self).start()
        # add interface to internal control plane bridge
        vrnetlab.run_command(["brctl", "addif", "int_cp", "vcp-int"])
        vrnetlab.run_command(["ip", "link", "set", "vcp-int", "up"])

    def gen_mgmt(self):
        """ Generate mgmt interface(s)

            We override the default function since we want a NIC to the vFPC
        """
        # call parent function to generate first mgmt interface (e1000)
        res = super(SROS_cp, self).gen_mgmt()
        # add virtio NIC for internal control plane interface to vFPC
        res.append("-device")
        res.append("e1000,netdev=vcp-int,mac=%s" % vrnetlab.gen_mac(1))
        res.append("-netdev")
        res.append("tap,ifname=vcp-int,id=vcp-int,script=no,downscript=no")
        return res

    def bootstrap_config(self):
        """ Do the actual bootstrap config
        """
        if self.username and self.password:
            self.wait_write("configure system security user \"%s\" password %s" % (self.username, self.password))
            self.wait_write("configure system security user \"%s\" access console netconf" % (self.username))
            self.wait_write("configure system security user \"%s\" console member \"administrative\" \"default\"" % (self.username))
        self.wait_write("configure system netconf no shutdown")
        self.wait_write("configure system security profile \"administrative\" netconf base-op-authorization lock")

        # configure SFMs
        for i in range(1, 16):
            self.wait_write("configure sfm {} sfm-type sfm-x20-b".format(i))

        # configure line card & MDAs
        for i in range(1, self.num_lc+1):
            self.wait_write("configure card {} card-type xcm-x20".format(i))
            self.wait_write("configure card {} mda 1 mda-type cx20-10g-sfp".format(i))

        self.wait_write("admin save")
        self.wait_write("logout")

class SROS_lc(SROS_vm):
    """ Line card for distributed VSR-SIM
    """
    def __init__(self, slot=1):
        super(SROS_lc, self).__init__(None, None, num=slot)
        self.slot = slot

        self.num_nics = 6
        self.smbios = ["type=1,product=TIMOS:chassis=XRS-20 slot={} card=xcm-x20 mda/1=cx20-10g-sfp".format(slot)]

    def start(self):
        # use parent class start() function
        super(SROS_lc, self).start()
        # add interface to internal control plane bridge
        vrnetlab.run_command(["brctl", "addif", "int_cp", "vfpc{}-int".format(self.slot)])
        vrnetlab.run_command(["ip", "link", "set", "vfpc{}-int".format(self.slot), "up"])

    def gen_mgmt(self):
        """ Generate mgmt interface
        """
        res = []
        # mgmt interface
        res.extend(["-device", "e1000,netdev=mgmt,mac=%s" % vrnetlab.gen_mac(0)])
        res.extend(["-netdev", "user,id=mgmt,net=10.0.0.0/24"])
        # internal control plane interface to vFPC
        res.extend(["-device", "e1000,netdev=vfpc-int,mac=%s" %
                    vrnetlab.gen_mac(0)])
        res.extend(["-netdev",
                    "tap,ifname=vfpc{}-int,id=vfpc-int,script=no,downscript=no".format(self.slot)])
        return res

    def gen_nics(self):
        """ Generate qemu args for the normal traffic carrying interface(s)
        """
        res = []
        # TODO: should this offset business be put in the common vrnetlab?
        offset = 6 * (self.slot-1)
        for j in range(0, self.num_nics):
            i = offset + j + 1
            res.append("-device")
            res.append(self.nic_type + ",netdev=p%(i)02d,mac=%(mac)s"
                       % { 'i': i, 'mac': vrnetlab.gen_mac(i) })
            res.append("-netdev")
            res.append("socket,id=p%(i)02d,listen=:100%(i)02d"
                       % { 'i': i })
        return res

    def bootstrap_spin(self):
        """ We have nothing to do for VSR-SIM line cards
        """
        self.running = True
        self.tn.close()
        return

class SROS(vrnetlab.VR):
    def __init__(self, username, password, num_nics):
        super(SROS, self).__init__(username, password)

        # move files into place
        for e in os.listdir("/"):
            if re.search("\.qcow2$", e):
                os.rename("/" + e, "/sros.qcow2")
            if re.search("\.license$", e):
                os.rename("/" + e, "/tftpboot/license.txt")

        self.license = False
        if os.path.isfile("/tftpboot/license.txt"):
            self.logger.info("License found")
            self.license = True

        self.logger.error("Number of NICS: " + str(num_nics))
        # if we have more than 5 NICs we use distributed VSR-SIM
        if num_nics > 5:
            if not self.license:
                self.logger.error("More than 5 NICs require distributed VSR which requires a license but no license is found")
                sys.exit(1)

            num_lc = math.ceil(num_nics / 6)
            self.logger.info("Number of linecards: " + str(num_lc))
            self.vms = [ SROS_cp(username, password, num_lc=num_lc) ]
            for i in range(1, num_lc+1):
                self.vms.append(SROS_lc(i))

        else: # 5 ports or less means integrated VSR-SIM
            self.vms = [ SROS_integrated(username, password) ]

        # set up bridge for connecting CP with LCs
        vrnetlab.run_command(["brctl", "addbr", "int_cp"])
        vrnetlab.run_command(["ip", "link", "set", "int_cp", "up"])

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description='')
    parser.add_argument('--trace', action='store_true', help='enable trace level logging')
    parser.add_argument('--username', default='vrnetlab', help='Username')
    parser.add_argument('--password', default='VR-netlab9', help='Password')
    parser.add_argument('--num-nics', default=5, help='Number of NICs')
    args = parser.parse_args()

    LOG_FORMAT = "%(asctime)s: %(module)-10s %(levelname)-8s %(message)s"
    logging.basicConfig(format=LOG_FORMAT)
    logger = logging.getLogger()

    logger.setLevel(logging.DEBUG)
    if args.trace:
        logger.setLevel(1)

    ia = SROS(args.username, args.password, num_nics=int(args.num_nics))
    ia.start()
<|endoftext|>


<commit_before>""" The 'base' cloudformation stack that builds out the virtual datacenter.

This includes the VPC, it's subnets, availability zones, etc.
"""

from collections import OrderedDict

from troposphere import Ref, Output, Join, FindInMap
from troposphere import ec2
import netaddr

from ..stack import StackTemplateBase

NAT_INSTANCE_NAME = "NatInstance%s"

class VPC(StackTemplateBase):
    PARAMETERS = {
        "InstanceType": {
            "type": "String",
            "description": "NAT EC2 instance type.",
            "default": "m3.medium"},
        "SshKeyName": {
            "type": "AWS::EC2::KeyPair::KeyName",
            "default": "default"},
        "BaseDomain": {
            "type": "String",
            "description": "Base domain for the stack."},
        "CidrBlock": {
            "type": "String",
            "description": "Base CIDR block for subnets.",
            "default": "10.128.0.0/16"},
        "ImageName": {
            "type": "String",
            "description": "The image name to use from the AMIMap (usually "
                           "found in the config file.)",
            "default": "NAT"},
    }

    def create_vpc(self):
        t = self.template
        vpc = t.add_resource(ec2.VPC(
            'VPC',
            CidrBlock=Ref("CidrBlock"), EnableDnsSupport=True,
            EnableDnsHostnames=True))

        # Just about everything needs this, so storing it on the object
        self.vpc_ref = Ref(vpc)
        t.add_output(Output("VpcId", Value=self.vpc_ref))

    def create_default_security_group(self):
        t = self.template
        self.default_sg = t.add_resource(ec2.SecurityGroup(
            'DefaultSG',
            VpcId=self.vpc_ref,
            GroupDescription='Default Security Group'))
        t.add_output(
            Output('DefaultSG',
                   Value=Ref(self.default_sg)))

    def create_dhcp_options(self):
        t = self.template
        dhcp_options = t.add_resource(ec2.DHCPOptions(
            'DHCPOptions',
            DomainName=Ref("BaseDomain"),
            DomainNameServers=['AmazonProvidedDNS', ]))
        t.add_resource(ec2.VPCDHCPOptionsAssociation(
            'DHCPAssociation',
            VpcId=self.vpc_ref,
            DhcpOptionsId=Ref(dhcp_options)))

    def create_network(self):
        t = self.template
        # First create the gateway
        gateway = t.add_resource(
            ec2.InternetGateway('InternetGateway'))
        # Make this an instance variable because a lot of other stuff
        # has to depend on this in order to ensure the stack can delete
        # properly every time.
        self.gw_attach = t.add_resource(ec2.VPCGatewayAttachment(
            'GatewayAttach',
            VpcId=self.vpc_ref,
            InternetGatewayId=Ref(gateway)))

        # Create the default ACL
        t.add_resource(ec2.NetworkAcl('DefaultACL',
                                      VpcId=self.vpc_ref))

        # Now create the subnets
        self.subnets = OrderedDict()
        self.subnets['public'] = []
        self.subnets['private'] = []
        networks = {'private': self.cidr_block.subnet(22)}
        # Give /24's to the public networks from the first /22
        networks['public'] = next(networks['private']).subnet(24)
        network_prefixes = []

        self.nat_sg = self.create_nat_security_groups()

        for _, subnet_type in enumerate(self.subnets):
            name_prefix = subnet_type.capitalize()
            for _, zone in enumerate(self.zones):
                name_suffix = zone[-1].upper()
                cidr_block = str(next(networks[subnet_type]))
                # Used by other templates to pick static IPs
                network_prefixes.append('.'.join(cidr_block.split('.')[:-1]))
                subnet = t.add_resource(ec2.Subnet(
                    '%sSubnet%s' % (name_prefix, name_suffix),
                    AvailabilityZone=zone,
                    VpcId=self.vpc_ref,
                    DependsOn=self.gw_attach.title,
                    CidrBlock=cidr_block))
                self.subnets[subnet_type].append(subnet)
                route_table = t.add_resource(ec2.RouteTable(
                    "%sRouteTable%s" % (name_prefix, name_suffix),
                    VpcId=self.vpc_ref,
                    Tags=[ec2.Tag('type', subnet_type)]))

                t.add_resource(ec2.SubnetRouteTableAssociation(
                    "%sRouteTableAssociation%s" % (name_prefix, name_suffix),
                    SubnetId=Ref(subnet),
                    RouteTableId=Ref(route_table)))

                if subnet_type == 'public':
                    # the public subnets are where the NAT instances live,
                    # so their default route needs to go to the AWS
                    # Internet Gateway
                    t.add_resource(
                        ec2.Route("%sRoute%s" % (name_prefix, name_suffix),
                                  RouteTableId=Ref(route_table),
                                  DestinationCidrBlock='0.0.0.0/0',
                                  GatewayId=Ref(gateway)))

                    self.create_nat_instance(zone, subnet)
                else:
                    # Private subnets are where actual instances will live
                    # so their gateway needs to be through the nat instances
                    nat_instance_name = NAT_INSTANCE_NAME % name_suffix
                    t.add_resource(
                        ec2.Route('%sRoute%s' % (name_prefix, name_suffix),
                                  RouteTableId=Ref(route_table),
                                  DestinationCidrBlock='0.0.0.0/0',
                                  InstanceId=Ref(nat_instance_name)))
            subnet_refs = [Ref(sn) for sn in self.subnets[subnet_type]]
            t.add_output(
                Output("%sSubnets" % name_prefix,
                       Value=Join(",", subnet_refs)))
            t.add_output(
                Output("%sNetworkPrefixes" % name_prefix,
                       Value=Join(",", network_prefixes)))

    def create_nat_security_groups(self):
        t = self.template
        # First setup the NAT Security Group Rules
        nat_private_in_all_rule = ec2.SecurityGroupRule(
            IpProtocol='-1', FromPort='-1', ToPort='-1',
            SourceSecurityGroupId=Ref(self.default_sg))

        nat_public_out_all_rule = ec2.SecurityGroupRule(
            IpProtocol='-1', FromPort='-1', ToPort='-1', CidrIp='0.0.0.0/0')

        return t.add_resource(ec2.SecurityGroup(
            'NATSG',
            VpcId=self.vpc_ref,
            GroupDescription='NAT Instance Security Group',
            SecurityGroupIngress=[nat_private_in_all_rule],
            SecurityGroupEgress=[nat_public_out_all_rule, ]))

    def create_nat_instance(self, zone, subnet):
        t = self.template
        suffix = zone[-1].upper()
        nat_instance = t.add_resource(ec2.Instance(
            NAT_INSTANCE_NAME % suffix,
            ImageId=FindInMap('AmiMap', Ref("AWS::Region"), Ref("ImageName")),
            SecurityGroupIds=[Ref(self.default_sg), Ref(self.nat_sg)],
            SubnetId=Ref(subnet),
            InstanceType=Ref('InstanceType'),
            SourceDestCheck=False,
            KeyName=Ref('SshKeyName'),
            Tags=[ec2.Tag('Name', 'nat-gw%s' % suffix.lower())],
            DependsOn=self.gw_attach.title))

        t.add_resource(ec2.EIP(
            'NATExternalIp%s' % suffix,
            Domain='vpc',
            InstanceId=Ref(nat_instance),
            DependsOn=self.gw_attach.title))
        return nat_instance

    def create_template(self):
        self.cidr_block = netaddr.IPNetwork(
            self.config.parameters['CidrBlock'])
        self.zones = self.config.parameters['Zones']
        self.template.add_output(
            Output("AvailabilityZones",
                   Value=Join(",", self.zones)))
        self.create_vpc()
        self.create_default_security_group()
        self.create_dhcp_options()
        self.create_network()
<commit_msg>Remove default SShKey<commit_after>""" The 'base' cloudformation stack that builds out the virtual datacenter.

This includes the VPC, it's subnets, availability zones, etc.
"""

from collections import OrderedDict

from troposphere import Ref, Output, Join, FindInMap
from troposphere import ec2
import netaddr

from ..stack import StackTemplateBase

NAT_INSTANCE_NAME = "NatInstance%s"

class VPC(StackTemplateBase):
    PARAMETERS = {
        "InstanceType": {
            "type": "String",
            "description": "NAT EC2 instance type.",
            "default": "m3.medium"},
        "SshKeyName": {
            "type": "AWS::EC2::KeyPair::KeyName"},
        "BaseDomain": {
            "type": "String",
            "description": "Base domain for the stack."},
        "CidrBlock": {
            "type": "String",
            "description": "Base CIDR block for subnets.",
            "default": "10.128.0.0/16"},
        "ImageName": {
            "type": "String",
            "description": "The image name to use from the AMIMap (usually "
                           "found in the config file.)",
            "default": "NAT"},
    }

    def create_vpc(self):
        t = self.template
        vpc = t.add_resource(ec2.VPC(
            'VPC',
            CidrBlock=Ref("CidrBlock"), EnableDnsSupport=True,
            EnableDnsHostnames=True))

        # Just about everything needs this, so storing it on the object
        self.vpc_ref = Ref(vpc)
        t.add_output(Output("VpcId", Value=self.vpc_ref))

    def create_default_security_group(self):
        t = self.template
        self.default_sg = t.add_resource(ec2.SecurityGroup(
            'DefaultSG',
            VpcId=self.vpc_ref,
            GroupDescription='Default Security Group'))
        t.add_output(
            Output('DefaultSG',
                   Value=Ref(self.default_sg)))

    def create_dhcp_options(self):
        t = self.template
        dhcp_options = t.add_resource(ec2.DHCPOptions(
            'DHCPOptions',
            DomainName=Ref("BaseDomain"),
            DomainNameServers=['AmazonProvidedDNS', ]))
        t.add_resource(ec2.VPCDHCPOptionsAssociation(
            'DHCPAssociation',
            VpcId=self.vpc_ref,
            DhcpOptionsId=Ref(dhcp_options)))

    def create_network(self):
        t = self.template
        # First create the gateway
        gateway = t.add_resource(
            ec2.InternetGateway('InternetGateway'))
        # Make this an instance variable because a lot of other stuff
        # has to depend on this in order to ensure the stack can delete
        # properly every time.
        self.gw_attach = t.add_resource(ec2.VPCGatewayAttachment(
            'GatewayAttach',
            VpcId=self.vpc_ref,
            InternetGatewayId=Ref(gateway)))

        # Create the default ACL
        t.add_resource(ec2.NetworkAcl('DefaultACL',
                                      VpcId=self.vpc_ref))

        # Now create the subnets
        self.subnets = OrderedDict()
        self.subnets['public'] = []
        self.subnets['private'] = []
        networks = {'private': self.cidr_block.subnet(22)}
        # Give /24's to the public networks from the first /22
        networks['public'] = next(networks['private']).subnet(24)
        network_prefixes = []

        self.nat_sg = self.create_nat_security_groups()

        for _, subnet_type in enumerate(self.subnets):
            name_prefix = subnet_type.capitalize()
            for _, zone in enumerate(self.zones):
                name_suffix = zone[-1].upper()
                cidr_block = str(next(networks[subnet_type]))
                # Used by other templates to pick static IPs
                network_prefixes.append('.'.join(cidr_block.split('.')[:-1]))
                subnet = t.add_resource(ec2.Subnet(
                    '%sSubnet%s' % (name_prefix, name_suffix),
                    AvailabilityZone=zone,
                    VpcId=self.vpc_ref,
                    DependsOn=self.gw_attach.title,
                    CidrBlock=cidr_block))
                self.subnets[subnet_type].append(subnet)
                route_table = t.add_resource(ec2.RouteTable(
                    "%sRouteTable%s" % (name_prefix, name_suffix),
                    VpcId=self.vpc_ref,
                    Tags=[ec2.Tag('type', subnet_type)]))

                t.add_resource(ec2.SubnetRouteTableAssociation(
                    "%sRouteTableAssociation%s" % (name_prefix, name_suffix),
                    SubnetId=Ref(subnet),
                    RouteTableId=Ref(route_table)))

                if subnet_type == 'public':
                    # the public subnets are where the NAT instances live,
                    # so their default route needs to go to the AWS
                    # Internet Gateway
                    t.add_resource(
                        ec2.Route("%sRoute%s" % (name_prefix, name_suffix),
                                  RouteTableId=Ref(route_table),
                                  DestinationCidrBlock='0.0.0.0/0',
                                  GatewayId=Ref(gateway)))

                    self.create_nat_instance(zone, subnet)
                else:
                    # Private subnets are where actual instances will live
                    # so their gateway needs to be through the nat instances
                    nat_instance_name = NAT_INSTANCE_NAME % name_suffix
                    t.add_resource(
                        ec2.Route('%sRoute%s' % (name_prefix, name_suffix),
                                  RouteTableId=Ref(route_table),
                                  DestinationCidrBlock='0.0.0.0/0',
                                  InstanceId=Ref(nat_instance_name)))
            subnet_refs = [Ref(sn) for sn in self.subnets[subnet_type]]
            t.add_output(
                Output("%sSubnets" % name_prefix,
                       Value=Join(",", subnet_refs)))
            t.add_output(
                Output("%sNetworkPrefixes" % name_prefix,
                       Value=Join(",", network_prefixes)))

    def create_nat_security_groups(self):
        t = self.template
        # First setup the NAT Security Group Rules
        nat_private_in_all_rule = ec2.SecurityGroupRule(
            IpProtocol='-1', FromPort='-1', ToPort='-1',
            SourceSecurityGroupId=Ref(self.default_sg))

        nat_public_out_all_rule = ec2.SecurityGroupRule(
            IpProtocol='-1', FromPort='-1', ToPort='-1', CidrIp='0.0.0.0/0')

        return t.add_resource(ec2.SecurityGroup(
            'NATSG',
            VpcId=self.vpc_ref,
            GroupDescription='NAT Instance Security Group',
            SecurityGroupIngress=[nat_private_in_all_rule],
            SecurityGroupEgress=[nat_public_out_all_rule, ]))

    def create_nat_instance(self, zone, subnet):
        t = self.template
        suffix = zone[-1].upper()
        nat_instance = t.add_resource(ec2.Instance(
            NAT_INSTANCE_NAME % suffix,
            ImageId=FindInMap('AmiMap', Ref("AWS::Region"), Ref("ImageName")),
            SecurityGroupIds=[Ref(self.default_sg), Ref(self.nat_sg)],
            SubnetId=Ref(subnet),
            InstanceType=Ref('InstanceType'),
            SourceDestCheck=False,
            KeyName=Ref('SshKeyName'),
            Tags=[ec2.Tag('Name', 'nat-gw%s' % suffix.lower())],
            DependsOn=self.gw_attach.title))

        t.add_resource(ec2.EIP(
            'NATExternalIp%s' % suffix,
            Domain='vpc',
            InstanceId=Ref(nat_instance),
            DependsOn=self.gw_attach.title))
        return nat_instance

    def create_template(self):
        self.cidr_block = netaddr.IPNetwork(
            self.config.parameters['CidrBlock'])
        self.zones = self.config.parameters['Zones']
        self.template.add_output(
            Output("AvailabilityZones",
                   Value=Join(",", self.zones)))
        self.create_vpc()
        self.create_default_security_group()
        self.create_dhcp_options()
        self.create_network()
<|endoftext|>


<commit_before>###########################################################################
#
#  Copyright 2020 Google LLC
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      https://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
###########################################################################

import os

UI_ROOT = os.environ.get('STARTHINKER_ROOT', 'MISSING RUN deploy.sh TO SET')
UI_CRON = os.environ.get('STARTHINKER_CRON', '')

DEVELOPMENT_MODE = bool(os.environ.get('STARTHINKER_DEVELOPMENT', '0') == '1')

# used to manage user credentials and UI data
UI_ZONE = os.environ.get('STARTHINKER_ZONE', 'MISSING RUN deploy.sh TO SET')
UI_PROJECT = os.environ.get('STARTHINKER_PROJECT',
                            'MISSING RUN deploy.sh TO SET')
UI_SERVICE = os.environ.get('STARTHINKER_SERVICE',
                            'MISSING RUN deploy.sh TO SET')
UI_CLIENT = os.environ.get('STARTHINKER_CLIENT', 'MISSING RUN deploy.sh TO SET')

# used to multiply all buffer sizes for scaling on larger or smaller machines, can be a float
BUFFER_SCALE = int(os.environ.get('STARTHINKER_SCALE', 1))

# used to write execution trace when debugging
TRACE_FILE = '/tmp/starthinker_trace.log'

# used for user authentication
APPLICATION_NAME = 'StarThinker Client'
APPLICATION_SCOPES = [
    'https://www.googleapis.com/auth/userinfo.profile',
    'https://www.googleapis.com/auth/userinfo.email',
    'https://www.googleapis.com/auth/gmail.readonly',
    'https://www.googleapis.com/auth/gmail.send',
    'https://www.googleapis.com/auth/doubleclickbidmanager',
    'https://www.googleapis.com/auth/cloudplatformprojects',
    'https://www.googleapis.com/auth/devstorage.full_control',
    'https://www.googleapis.com/auth/bigquery',
    'https://www.googleapis.com/auth/dfareporting',
    'https://www.googleapis.com/auth/dfatrafficking',
    'https://www.googleapis.com/auth/drive',
    'https://www.googleapis.com/auth/spreadsheets',
    'https://www.googleapis.com/auth/doubleclicksearch',
    'https://www.googleapis.com/auth/content',
    'https://www.googleapis.com/auth/ddmconversions',
    'https://www.googleapis.com/auth/datastore',
    'https://www.googleapis.com/auth/logging.write',
    'https://www.googleapis.com/auth/logging.read',
    'https://www.googleapis.com/auth/pubsub',
    'https://www.googleapis.com/auth/youtube',
    'https://www.googleapis.com/auth/analytics',
    'https://www.googleapis.com/auth/analytics.readonly',
    'https://www.googleapis.com/auth/display-video'
]
<commit_msg>Add scope for adwords.<commit_after>###########################################################################
#
#  Copyright 2020 Google LLC
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      https://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
###########################################################################

import os

UI_ROOT = os.environ.get('STARTHINKER_ROOT', 'MISSING RUN deploy.sh TO SET')
UI_CRON = os.environ.get('STARTHINKER_CRON', '')

DEVELOPMENT_MODE = bool(os.environ.get('STARTHINKER_DEVELOPMENT', '0') == '1')

# used to manage user credentials and UI data
UI_ZONE = os.environ.get('STARTHINKER_ZONE', 'MISSING RUN deploy.sh TO SET')
UI_PROJECT = os.environ.get('STARTHINKER_PROJECT',
                            'MISSING RUN deploy.sh TO SET')
UI_SERVICE = os.environ.get('STARTHINKER_SERVICE',
                            'MISSING RUN deploy.sh TO SET')
UI_CLIENT = os.environ.get('STARTHINKER_CLIENT', 'MISSING RUN deploy.sh TO SET')

# used to multiply all buffer sizes for scaling on larger or smaller machines, can be a float
BUFFER_SCALE = int(os.environ.get('STARTHINKER_SCALE', 1))

# used to write execution trace when debugging
TRACE_FILE = '/tmp/starthinker_trace.log'

# used for user authentication
APPLICATION_NAME = 'StarThinker Client'
APPLICATION_SCOPES = [
    'https://www.googleapis.com/auth/userinfo.profile',
    'https://www.googleapis.com/auth/userinfo.email',
    'https://www.googleapis.com/auth/gmail.readonly',
    'https://www.googleapis.com/auth/gmail.send',
    'https://www.googleapis.com/auth/doubleclickbidmanager',
    'https://www.googleapis.com/auth/cloudplatformprojects',
    'https://www.googleapis.com/auth/devstorage.full_control',
    'https://www.googleapis.com/auth/bigquery',
    'https://www.googleapis.com/auth/dfareporting',
    'https://www.googleapis.com/auth/dfatrafficking',
    'https://www.googleapis.com/auth/drive',
    'https://www.googleapis.com/auth/spreadsheets',
    'https://www.googleapis.com/auth/doubleclicksearch',
    'https://www.googleapis.com/auth/content',
    'https://www.googleapis.com/auth/ddmconversions',
    'https://www.googleapis.com/auth/datastore',
    'https://www.googleapis.com/auth/logging.write',
    'https://www.googleapis.com/auth/logging.read',
    'https://www.googleapis.com/auth/pubsub',
    'https://www.googleapis.com/auth/youtube',
    'https://www.googleapis.com/auth/analytics',
    'https://www.googleapis.com/auth/analytics.readonly',
    'https://www.googleapis.com/auth/display-video',
    'https://www.googleapis.com/auth/adwords'
]
<|endoftext|>