<commit_before># -*- coding: utf-8 -*-
from urlparse import urlparse
from werkzeug import url_quote
from flask import Markup

class Extension(object):

    def __init__(self, name, author, description,
                 github=None, bitbucket=None, docs=None, website=None,
                 approved=False, notes=None):
        self.name = name
        self.author = author
        self.description = Markup(description)
        self.github = github
        self.bitbucket = bitbucket
        self.docs = docs
        self.website = website
        self.approved = approved
        self.notes = notes

    def to_json(self):
        rv = vars(self).copy()
        # Keep developer notes out of JSON response.
        rv.pop('notes')
        rv['description'] = unicode(rv['description'])
        return rv

    @property
    def pypi(self):
        return 'http://pypi.python.org/pypi/%s' % url_quote(self.name)

    @property
    def docserver(self):
        if self.docs:
            return urlparse(self.docs)[1]

# This list contains all extensions that were approved as well as those which
# passed listing.
extensions = [
    Extension('Flask-OAuth', 'Armin Ronacher',
        description='''

Adds OAuth support to Flask.
        ''',
        github='mitsuhiko/flask-oauth',
        docs='http://packages.python.org/Flask-OAuth/',
        notes='''
            Short long description, missing tests.
        '''
    ),
    Extension('Flask-OpenID', 'Armin Ronacher',
        description='''

Adds OpenID support to Flask.
        ''',
        github='mitsuhiko/flask-openid',
        docs='http://packages.python.org/Flask-OpenID/',
        notes='''
            Short long description, missing tests.
        '''
    ),
    Extension('Flask-Babel', 'Armin Ronacher',
        description='''

Adds i18n/l10n support to Flask, based on
            babel and
            pytz.
        ''',
        github='mitsuhiko/flask-babel',
        docs='http://packages.python.org/Flask-Babel/',
        approved=True,
        notes='''
            How to improve: add a better long description to the next release.
        '''
    ),
    Extension('Flask-SQLAlchemy', 'Armin Ronacher',
        description='''

Adds SQLAlchemy support to Flask.  Quick and easy.
        ''',
        github='mitsuhiko/flask-sqlalchemy',
        docs='http://packages.python.org/Flask-SQLAlchemy/',
        approved=True,
        notes='''
            How to improve: add a better long description to the next release.
        '''
    ),
    Extension('Flask-XML-RPC', 'Matthew Frazier',
        description='''

Adds XML-RPC support to Flask.
        ''',
        bitbucket='leafstorm/flask-xml-rpc',
        docs='http://packages.python.org/Flask-XML-RPC/',
        approved=True
    ),
    Extension('Flask-CouchDB', 'Matthew Frazier',
        description='''

Adds CouchDB support to Flask.
        ''',
        bitbucket='leafstorm/flask-couchdb',
        docs='http://packages.python.org/Flask-CouchDB/',
        approved=True,
        notes='''
            There is also Flask-CouchDBKit.  Both are fine because they are
            doing different things, but the latter is not yet approved.
        '''
    ),
    Extension('Flask-Uploads', 'Matthew Frazier',
        description='''

Flask-Uploads allows your application to flexibly and
            efficiently handle file uploading and serving the uploaded files.
            You can create different sets of uploads - one for document
            attachments, one for photos, etc.
        ''',
        bitbucket='leafstorm/flask-uploads',
        docs='http://packages.python.org/Flask-Uploads/',
        approved=True
    ),
    Extension('Flask-Themes', 'Matthew Frazier',
        description='''

Flask-Themes makes it easy for your application to support
            a wide range of appearances.
        ''',
        bitbucket='leafstorm/flask-themes',
        docs='http://packages.python.org/Flask-Themes/',
        approved=True
    ),
    Extension('Flask-CouchDBKit', 'Kridsada Thanabulpong',
        description='''

Adds CouchDBKit support to Flask.
        ''',
        github='sirn/flask-couchdbkit',
        docs='http://packages.python.org/Flask-CouchDBKit/'
    ),
    Extension('Flask-Genshi', 'Dag Odenhall',
        description='''

Adds support for the Genshi
            templating language to Flask applications.
        ''',
        github='dag/flask-genshi',
        docs='http://packages.python.org/Flask-Genshi/',
        approved=True,
        notes='''
            This is the first template engine extension.  When others come
            around it would be a good idea to decide on a common interface.
        '''
    ),
    Extension('Flask-Mail', 'Dan Jacob',
        description='''

Makes sending mails from Flask applications very easy and
            has also support for unittesting.
        ''',
        bitbucket='danjac/flask-mail',
        docs='http://packages.python.org/flask-mail/',
        approved=True
    ),
    Extension('Flask-WTF', 'Dan Jacob',
        description='''

Flask-WTF offers simple integration with WTForms. This
            integration includes optional CSRF handling for greater security.
        ''',
        bitbucket='danjac/flask-wtf',
        docs='http://packages.python.org/Flask-WTF/',
        approved=True
    ),
    Extension('Flask-Testing', 'Dan Jacob',
        description='''

The Flask-Testing extension provides unit testing utilities for Flask.
        ''',
        bitbucket='danjac/flask-testing',
        docs='http://packages.python.org/Flask-Testing/',
        approved=True
    ),
    Extension('Flask-Script', 'Dan Jacob',
        description='''

The Flask-Script extension provides support for writing external
            scripts in Flask. It uses argparse to parse command line arguments.
        ''',
        bitbucket='danjac/flask-script',
        docs='http://packages.python.org/Flask-Script/',
        approved=True,
        notes='''
            Flask-Actions has some overlap.  Consider that when approving
            Flask-Actions or similar packages.
        '''
    ),
    Extension('flask-csrf', 'Steve Losh',
        description='''

A small Flask extension for adding
            CSRF protection.
        ''',
        docs='http://sjl.bitbucket.org/flask-csrf/',
        bitbucket='sjl/flask-csrf',
        notes='''
            Will not be approved because this is functionality that should be
            handled in the form handling systems which is for Flask-WTF already
            the case.  Also, this implementation only supports one open tab
            with forms.

            Name is not following Flask extension naming rules.

            Considered for unlisting.
        '''
    ),
    Extension('flask-lesscss', 'Steve Losh',
        description='''

              A small Flask extension that makes it easy to use
              LessCSS with your
              Flask application.
        ''',
        docs='http://sjl.bitbucket.org/flask-lesscss/',
        bitbucket='sjl/flask-lesscss',
        notes='''
            Broken package description, nonconforming package name, does not
            follow standard API rules (init_lesscss instead of lesscss).

            Considered for unlisting, improved version should release as
            "Flask-LessCSS" with a conforming API and fixed packages indices,
            as well as a testsuite.
        '''
    ),
    Extension('Flask-Creole', 'Ali Afshar',
        description='''

Creole parser filters for Flask.
        ''',
        docs='http://packages.python.org/Flask-Creole',
        bitbucket='aafshar/flask-creole-main',
        approved=True,
        notes='''
            Flask-Markdown and this should share API, consider that when
            approving Flask-Markdown
        '''
    ),
    Extension('Flask-Cache', 'Thadeus Burgess',
        description='''

Adds cache support to your Flask application.
        ''',
        docs='http://packages.python.org/Flask-Cache',
        github='thadeusb/flask-cache',
    ),
    Extension('Flask-Principal', 'Ali Afshar',
        description='''

Identity management for Flask.
        ''',
        docs='http://packages.python.org/Flask-Principal',
        bitbucket='aafshar/flask-principal-main',
        approved=False
    ),
    Extension('Flask-Zen', 'Noah Kantrowitz',
        description='''

Flask-Zen allows you to use PyZen via Flask-Script commands.
        ''',
        docs='http://packages.python.org/Flask-Zen/',
        github='coderanger/flask-zen',
        approved=False
    ),
    Extension('Flask-Assets', u'Michael Elsdörfer',
        description='''

              Integrates the webassets library with Flask, adding support for
              merging, minifying and compiling CSS and Javascript files.
        ''',
        docs='http://elsdoerfer.name/docs/flask-assets/',
        github='miracle2k/flask-assets',
        approved=False
    ),
    Extension('Flask-AutoIndex', 'Heungsub Lee',
        description='''

              An extension that generates an index page for your Flask
              application automatically
        ''',
        docs='http://packages.python.org/Flask-AutoIndex/',
        github='sublee/flask-autoindex',
        approved=False
    ),
    Extension('Flask-Celery', 'Ask Solem',
        description='''

              Celery integration for Flask
        ''',
        docs='http://ask.github.com/celery/',
        github='ask/flask-celery',
        approved=False
    ),
    Extension('Frozen-Flask', 'Simon Sapin',
        description='''

              Freezes a Flask application into a set of static files.
              The result can be hosted without any server-side software
              other than a traditional web server.
        ''',
        docs='http://packages.python.org/Frozen-Flask/',
        github='SimonSapin/Frozen-Flask',
        approved=True
    ),
    Extension('Flask-FlatPages', 'Simon Sapin',
        description='''

              Provides flat static pages to a Flask application, based on text
              files as opposed to a relational database.
        ''',
        docs='http://packages.python.org/Flask-FlatPages/',
        github='SimonSapin/Flask-FlatPages',
        approved=False
    ),
    Extension('Flask-FluidDB', 'Ali Afshar',
        description='''

              FluidDB access for Flask.
        ''',
        docs='http://packages.python.org/Flask-FluidDB/',
        bitbucket='aafshar/flask-fluiddb-main',
        approved=False
    ),
    Extension('Flask-Gravatar', 'Zelenyak Aleksandr',
        description='''

              Small extension for Flask to make using Gravatar easy.
        ''',
        docs='http://packages.python.org/Flask-Gravatar/',
        github='zzzsochi/Flask-Gravatar',
        approved=False
    ),
    Extension('Flask-HTMLBuilder', 'Zahari Petkov',
        description='''

              Flask-HTMLBuilder is an extension that allows flexible and easy
              Python-only generation of HTML snippets and full HTML documents
              using a robust syntax.
        ''',
        docs='http://majorz.github.com/flask-htmlbuilder/',
        github='majorz/flask-htmlbuilder',
        approved=False
    ),
    Extension('Flask-MongoAlchemy', 'Francisco Souza',
        description='''

              Add Flask support for MongoDB using MongoAlchemy.
        ''',
        docs='http://packages.python.org/Flask-MongoAlchemy/',
        github='cobrateam/flask-mongoalchemy',
        approved=False
    ),
    Extension('Flask-DebugToolbar', 'Michael van Tellingen',
        description='''

              A port of the Django debug toolbar to Flask
        ''',
        docs='https://github.com/mvantellingen/flask-debugtoolbar',
        github='mvantellingen/flask-debugtoolbar',
        approved=False
    ),
    Extension('Flask-Login', 'Matthew Frazier',
        description='''

              Flask-Login provides user session management for Flask. It
              handles the common tasks of logging in, logging out, and
              remembering your users' sessions over extended periods of time.
        ''',
        bitbucket='leafstorm/flask-login',
        docs='http://packages.python.org/Flask-Login/',
        approved=True
    ),
]

# This is a list of extensions that is currently rejected from listing and with
# that also not approved.  If an extension ends up here it should improved to
# be listed.
unlisted = [
    Extension('Flask-Actions', '',
        description='''

              Flask-actions provide some management comands for flask based
              project.
        ''',
        docs='http://packages.python.org/Flask-Actions/',
        bitbucket='youngking/flask-actions',
        approved=False,
        notes='''
            Rejected because of missing description in PyPI, formatting issues
            with the documentation (missing headlines, scrollbars etc.) and a
            general clash of functionality with the Flask-Script package.
            Latter should not be a problem, but the documentation should
            improve.  For listing, the extension developer should probably
            discuss the extension on the mailinglist with others.

            Futhermore it also has an egg registered with an invalid filename.
        '''
    ),
    Extension('Flask-Jinja2Extender', '',
        description='''

        ''',
        docs=None,
        github='dcolish/flask-jinja2extender',
        approved=False,
        notes='''
            Appears to be discontinued.

            Usecase not obvious, hacky implementation, does not solve a problem
            that could not be solved with Flask itself.  I suppose it is to aid
            other extensions, but that should be discussed on the mailinglist.
        '''
    ),
    Extension('Flask-Markdown', '',
        description='''

              This is a small module to a markdown processing filter into your
              flask.
        ''',
        docs='http://packages.python.org/Flask-Markdown/',
        github='dcolish/flask-markdown',
        approved=False,
        notes='''
            Would be great for enlisting but it should follow the API of
            Flask-Creole.  Besides that, the docstrings are not valid rst (run
            through rst2html to see the issue) and it is missing tests.
            Otherwise fine :)
        '''
    ),
    Extension('flask-urls', '',
        description='''

              A collection of URL-related functions for Flask applications.
        ''',
        docs='http://sjl.bitbucket.org/flask-urls/',
        bitbucket='sjl/flask-urls',
        approved=False,
        notes='''
            Broken PyPI index and non-conforming extension name.  Due to the
            small featureset this was also delisted from the list.  It was
            there previously before the approval process was introduced.
        '''
    ),
]

extensions.sort(key=lambda x: x.name.lower())
unlisted.sort(key=lambda x: x.name.lower())
<commit_msg>Add extensions recently discussed on mailing list.<commit_after># -*- coding: utf-8 -*-
from urlparse import urlparse
from werkzeug import url_quote
from flask import Markup

class Extension(object):

    def __init__(self, name, author, description,
                 github=None, bitbucket=None, docs=None, website=None,
                 approved=False, notes=None):
        self.name = name
        self.author = author
        self.description = Markup(description)
        self.github = github
        self.bitbucket = bitbucket
        self.docs = docs
        self.website = website
        self.approved = approved
        self.notes = notes

    def to_json(self):
        rv = vars(self).copy()
        # Keep developer notes out of JSON response.
        rv.pop('notes')
        rv['description'] = unicode(rv['description'])
        return rv

    @property
    def pypi(self):
        return 'http://pypi.python.org/pypi/%s' % url_quote(self.name)

    @property
    def docserver(self):
        if self.docs:
            return urlparse(self.docs)[1]

# This list contains all extensions that were approved as well as those which
# passed listing.
extensions = [
    Extension('Flask-OAuth', 'Armin Ronacher',
        description='''

Adds OAuth support to Flask.
        ''',
        github='mitsuhiko/flask-oauth',
        docs='http://packages.python.org/Flask-OAuth/',
        notes='''
            Short long description, missing tests.
        '''
    ),
    Extension('Flask-OpenID', 'Armin Ronacher',
        description='''

Adds OpenID support to Flask.
        ''',
        github='mitsuhiko/flask-openid',
        docs='http://packages.python.org/Flask-OpenID/',
        notes='''
            Short long description, missing tests.
        '''
    ),
    Extension('Flask-Babel', 'Armin Ronacher',
        description='''

Adds i18n/l10n support to Flask, based on
            babel and
            pytz.
        ''',
        github='mitsuhiko/flask-babel',
        docs='http://packages.python.org/Flask-Babel/',
        approved=True,
        notes='''
            How to improve: add a better long description to the next release.
        '''
    ),
    Extension('Flask-SQLAlchemy', 'Armin Ronacher',
        description='''

Adds SQLAlchemy support to Flask.  Quick and easy.
        ''',
        github='mitsuhiko/flask-sqlalchemy',
        docs='http://packages.python.org/Flask-SQLAlchemy/',
        approved=True,
        notes='''
            How to improve: add a better long description to the next release.
        '''
    ),
    Extension('Flask-XML-RPC', 'Matthew Frazier',
        description='''

Adds XML-RPC support to Flask.
        ''',
        bitbucket='leafstorm/flask-xml-rpc',
        docs='http://packages.python.org/Flask-XML-RPC/',
        approved=True
    ),
    Extension('Flask-CouchDB', 'Matthew Frazier',
        description='''

Adds CouchDB support to Flask.
        ''',
        bitbucket='leafstorm/flask-couchdb',
        docs='http://packages.python.org/Flask-CouchDB/',
        approved=True,
        notes='''
            There is also Flask-CouchDBKit.  Both are fine because they are
            doing different things, but the latter is not yet approved.
        '''
    ),
    Extension('Flask-Uploads', 'Matthew Frazier',
        description='''

Flask-Uploads allows your application to flexibly and
            efficiently handle file uploading and serving the uploaded files.
            You can create different sets of uploads - one for document
            attachments, one for photos, etc.
        ''',
        bitbucket='leafstorm/flask-uploads',
        docs='http://packages.python.org/Flask-Uploads/',
        approved=True
    ),
    Extension('Flask-Themes', 'Matthew Frazier',
        description='''

Flask-Themes makes it easy for your application to support
            a wide range of appearances.
        ''',
        bitbucket='leafstorm/flask-themes',
        docs='http://packages.python.org/Flask-Themes/',
        approved=True
    ),
    Extension('Flask-CouchDBKit', 'Kridsada Thanabulpong',
        description='''

Adds CouchDBKit support to Flask.
        ''',
        github='sirn/flask-couchdbkit',
        docs='http://packages.python.org/Flask-CouchDBKit/'
    ),
    Extension('Flask-Genshi', 'Dag Odenhall',
        description='''

Adds support for the Genshi
            templating language to Flask applications.
        ''',
        github='dag/flask-genshi',
        docs='http://packages.python.org/Flask-Genshi/',
        approved=True,
        notes='''
            This is the first template engine extension.  When others come
            around it would be a good idea to decide on a common interface.
        '''
    ),
    Extension('Flask-Mail', 'Dan Jacob',
        description='''

Makes sending mails from Flask applications very easy and
            has also support for unittesting.
        ''',
        bitbucket='danjac/flask-mail',
        docs='http://packages.python.org/flask-mail/',
        approved=True
    ),
    Extension('Flask-WTF', 'Dan Jacob',
        description='''

Flask-WTF offers simple integration with WTForms. This
            integration includes optional CSRF handling for greater security.
        ''',
        bitbucket='danjac/flask-wtf',
        docs='http://packages.python.org/Flask-WTF/',
        approved=True
    ),
    Extension('Flask-Testing', 'Dan Jacob',
        description='''

The Flask-Testing extension provides unit testing utilities for Flask.
        ''',
        bitbucket='danjac/flask-testing',
        docs='http://packages.python.org/Flask-Testing/',
        approved=True
    ),
    Extension('Flask-Script', 'Dan Jacob',
        description='''

The Flask-Script extension provides support for writing external
            scripts in Flask. It uses argparse to parse command line arguments.
        ''',
        bitbucket='danjac/flask-script',
        docs='http://packages.python.org/Flask-Script/',
        approved=True,
        notes='''
            Flask-Actions has some overlap.  Consider that when approving
            Flask-Actions or similar packages.
        '''
    ),
    Extension('flask-csrf', 'Steve Losh',
        description='''

A small Flask extension for adding
            CSRF protection.
        ''',
        docs='http://sjl.bitbucket.org/flask-csrf/',
        bitbucket='sjl/flask-csrf',
        notes='''
            Will not be approved because this is functionality that should be
            handled in the form handling systems which is for Flask-WTF already
            the case.  Also, this implementation only supports one open tab
            with forms.

            Name is not following Flask extension naming rules.

            Considered for unlisting.
        '''
    ),
    Extension('flask-lesscss', 'Steve Losh',
        description='''

              A small Flask extension that makes it easy to use
              LessCSS with your
              Flask application.
        ''',
        docs='http://sjl.bitbucket.org/flask-lesscss/',
        bitbucket='sjl/flask-lesscss',
        notes='''
            Broken package description, nonconforming package name, does not
            follow standard API rules (init_lesscss instead of lesscss).

            Considered for unlisting, improved version should release as
            "Flask-LessCSS" with a conforming API and fixed packages indices,
            as well as a testsuite.
        '''
    ),
    Extension('Flask-Creole', 'Ali Afshar',
        description='''

Creole parser filters for Flask.
        ''',
        docs='http://packages.python.org/Flask-Creole',
        bitbucket='aafshar/flask-creole-main',
        approved=True,
        notes='''
            Flask-Markdown and this should share API, consider that when
            approving Flask-Markdown
        '''
    ),
    Extension('Flask-Cache', 'Thadeus Burgess',
        description='''

Adds cache support to your Flask application.
        ''',
        docs='http://packages.python.org/Flask-Cache',
        github='thadeusb/flask-cache',
    ),
    Extension('Flask-Principal', 'Ali Afshar',
        description='''

Identity management for Flask.
        ''',
        docs='http://packages.python.org/Flask-Principal',
        bitbucket='aafshar/flask-principal-main',
        approved=False
    ),
    Extension('Flask-Zen', 'Noah Kantrowitz',
        description='''

Flask-Zen allows you to use PyZen via Flask-Script commands.
        ''',
        docs='http://packages.python.org/Flask-Zen/',
        github='coderanger/flask-zen',
        approved=False
    ),
    Extension('Flask-Assets', u'Michael Elsdörfer',
        description='''

              Integrates the webassets library with Flask, adding support for
              merging, minifying and compiling CSS and Javascript files.
        ''',
        docs='http://elsdoerfer.name/docs/flask-assets/',
        github='miracle2k/flask-assets',
        approved=False
    ),
    Extension('Flask-AutoIndex', 'Heungsub Lee',
        description='''

              An extension that generates an index page for your Flask
              application automatically
        ''',
        docs='http://packages.python.org/Flask-AutoIndex/',
        github='sublee/flask-autoindex',
        approved=False
    ),
    Extension('Flask-Celery', 'Ask Solem',
        description='''

              Celery integration for Flask
        ''',
        docs='http://ask.github.com/celery/',
        github='ask/flask-celery',
        approved=False
    ),
    Extension('Frozen-Flask', 'Simon Sapin',
        description='''

              Freezes a Flask application into a set of static files.
              The result can be hosted without any server-side software
              other than a traditional web server.
        ''',
        docs='http://packages.python.org/Frozen-Flask/',
        github='SimonSapin/Frozen-Flask',
        approved=True
    ),
    Extension('Flask-FlatPages', 'Simon Sapin',
        description='''

              Provides flat static pages to a Flask application, based on text
              files as opposed to a relational database.
        ''',
        docs='http://packages.python.org/Flask-FlatPages/',
        github='SimonSapin/Flask-FlatPages',
        approved=False
    ),
    Extension('Flask-FluidDB', 'Ali Afshar',
        description='''

              FluidDB access for Flask.
        ''',
        docs='http://packages.python.org/Flask-FluidDB/',
        bitbucket='aafshar/flask-fluiddb-main',
        approved=False
    ),
    Extension('Flask-Gravatar', 'Zelenyak Aleksandr',
        description='''

              Small extension for Flask to make using Gravatar easy.
        ''',
        docs='http://packages.python.org/Flask-Gravatar/',
        github='zzzsochi/Flask-Gravatar',
        approved=False
    ),
    Extension('Flask-HTMLBuilder', 'Zahari Petkov',
        description='''

              Flask-HTMLBuilder is an extension that allows flexible and easy
              Python-only generation of HTML snippets and full HTML documents
              using a robust syntax.
        ''',
        docs='http://majorz.github.com/flask-htmlbuilder/',
        github='majorz/flask-htmlbuilder',
        approved=False
    ),
    Extension('Flask-MongoAlchemy', 'Francisco Souza',
        description='''

              Add Flask support for MongoDB using MongoAlchemy.
        ''',
        docs='http://packages.python.org/Flask-MongoAlchemy/',
        github='cobrateam/flask-mongoalchemy',
        approved=False
    ),
    Extension('Flask-DebugToolbar', 'Michael van Tellingen',
        description='''

              A port of the Django debug toolbar to Flask
        ''',
        docs='https://github.com/mvantellingen/flask-debugtoolbar',
        github='mvantellingen/flask-debugtoolbar',
        approved=False
    ),
    Extension('Flask-Login', 'Matthew Frazier',
        description='''

              Flask-Login provides user session management for Flask. It
              handles the common tasks of logging in, logging out, and
              remembering your users' sessions over extended periods of time.
        ''',
        bitbucket='leafstorm/flask-login',
        docs='http://packages.python.org/Flask-Login/',
        approved=True
    ),
    Extension('Flask-Exceptional', 'Jonathan Zempel',
        description='''

              Adds Exceptional support to Flask applications
        ''',
        docs='http://packages.python.org/Flask-Exceptional/',
        github='jzempel/flask-exceptional',
    ),
    Extension('Flask-Bcrypt', 'Max Countryman',
        description='''

              Bcrypt support for hashing passwords
        ''',
        github='maxcountryman/flask-bcrypt',
    ),
]

# This is a list of extensions that is currently rejected from listing and with
# that also not approved.  If an extension ends up here it should improved to
# be listed.
unlisted = [
    Extension('Flask-Actions', 'Young King',
        description='''

              Flask-actions provide some management comands for flask based
              project.
        ''',
        docs='http://packages.python.org/Flask-Actions/',
        bitbucket='youngking/flask-actions',
        approved=False,
        notes='''
            Rejected because of missing description in PyPI, formatting issues
            with the documentation (missing headlines, scrollbars etc.) and a
            general clash of functionality with the Flask-Script package.
            Latter should not be a problem, but the documentation should
            improve.  For listing, the extension developer should probably
            discuss the extension on the mailinglist with others.

            Futhermore it also has an egg registered with an invalid filename.
        '''
    ),
    Extension('Flask-Jinja2Extender', 'Dan Colish',
        description='''

        ''',
        docs=None,
        github='dcolish/flask-jinja2extender',
        approved=False,
        notes='''
            Appears to be discontinued.

            Usecase not obvious, hacky implementation, does not solve a problem
            that could not be solved with Flask itself.  I suppose it is to aid
            other extensions, but that should be discussed on the mailinglist.
        '''
    ),
    Extension('Flask-Markdown', 'Dan Colish',
        description='''

              This is a small module to a markdown processing filter into your
              flask.
        ''',
        docs='http://packages.python.org/Flask-Markdown/',
        github='dcolish/flask-markdown',
        approved=False,
        notes='''
            Would be great for enlisting but it should follow the API of
            Flask-Creole.  Besides that, the docstrings are not valid rst (run
            through rst2html to see the issue) and it is missing tests.
            Otherwise fine :)
        '''
    ),
    Extension('flask-urls', 'Steve Losh',
        description='''

              A collection of URL-related functions for Flask applications.
        ''',
        docs='http://sjl.bitbucket.org/flask-urls/',
        bitbucket='sjl/flask-urls',
        approved=False,
        notes='''
            Broken PyPI index and non-conforming extension name.  Due to the
            small featureset this was also delisted from the list.  It was
            there previously before the approval process was introduced.
        '''
    ),
    Extension('Flask-Coffee', 'Col Wilson',
        description='''

              Automatically compile CoffeeScript files while developing with
              the Flask framework.
        ''',
        docs=None,
        approved=False,
        notes='''
            On the mailing list, author claims it's flask-lesscss with a
            different label.  No sphinx-based docs, just a blog post.  No
            publicly accessible repository -- requires login on
            bettercodes.org.
        '''
    ),
]

extensions.sort(key=lambda x: x.name.lower())
unlisted.sort(key=lambda x: x.name.lower())
<|endoftext|>


<commit_before>import logging
import datetime
from flexget.manager import Session
from flexget.plugin import register_plugin, PluginError, get_plugin_by_name, priority, register_parser_option, register_feed_phase
from flexget.manager import Base
from sqlalchemy import Column, Integer, String, DateTime, Boolean, Unicode
from flexget.utils.imdb import extract_id, ImdbSearch, ImdbParser, log as imdb_log
from flexget.utils.tools import str_to_boolean
from flexget.utils import qualities

log = logging.getLogger('imdb_queue')

class QueueError(Exception):
    """Exception raised if there is an error with a queue operation"""

    def __init__(self, message, errno=0):
        self.message = message
        self.errno = errno

class ImdbQueue(Base):

    __tablename__ = 'imdb_queue'

    id = Column(Integer, primary_key=True)
    imdb_id = Column(String)
    quality = Column(String)
    title = Column(Unicode)
    immortal = Column(Boolean)
    added = Column(DateTime)

    def __init__(self, imdb_id, quality, immortal):
        self.imdb_id = imdb_id
        self.quality = quality
        self.immortal = immortal
        self.added = datetime.datetime.now()

    def __str__(self):
        return '<ImdbQueue(imdb_id=%s,quality=%s,force=%s)>' % (self.imdb_id, self.quality, self.immortal)

class FilterImdbQueue(object):
    """
    Allows a queue of upcoming movies that will be forcibly allowed if the given quality matches

    Example:

    imdb_queue: yes
    """

    # Dict of entries accepted by this plugin {imdb_id: entry} format
    accepted_entries = {}

    def validator(self):
        from flexget import validator
        return validator.factory('boolean')

    @priority(129)
    def on_feed_filter(self, feed):
        # Doing this so that I register as a filter plugin. Just need to filter
        # after urlrewrite happens, just before download.
        # Also have to accept anything with an IMDB url that matches, even if
        # rejecting later.
        rejected = []
        for entry in feed.entries:
            # make sure the entry has IMDB fields filled
            try:
                get_plugin_by_name('imdb_lookup').instance.lookup(feed, entry)
            except PluginError:
                # no IMDB data, can't do anything
                continue

            # entry has imdb url
            if 'imdb_url' in entry:
                # get an imdb id
                if 'imdb_id' in entry and entry['imdb_id'] is not None:
                    imdb_id = entry['imdb_id']
                else:
                    imdb_id = extract_id(entry['imdb_url'])

                if not imdb_id:
                    log.warning("No imdb id could be determined for %s" % entry['title'])
                    continue

                item = feed.session.query(ImdbQueue).filter(ImdbQueue.imdb_id == imdb_id).first()

                if item:
                    entry['immortal'] = item.immortal
                    log.debug("Pre-accepting %s from queue" % (entry['title']))
                    feed.accept(entry, 'imdb-queue pre-accept')
                    # Keep track of entries we accepted, so they can be removed from queue on feed_exit if successful
                    self.accepted_entries[imdb_id] = entry
                else:
                    log.debugall("%s not in queue, skipping" % entry['title'])

    def on_feed_imdbqueue(self, feed):
        rejected = []
        for entry in feed.entries:
            if entry['url'] == '':
                feed.reject(entry, 'imdb-queue - no URL in entry: '
                            '%s' % entry['title'])
                rejected.append(entry['title'])
                continue
            # make sure the entry has IMDB fields filled
            try:
                get_plugin_by_name('imdb_lookup').instance.lookup(feed, entry)
            except PluginError:
                # no IMDB data, can't do anything
                continue

            # entry has imdb url
            if 'imdb_url' in entry:
                # get an imdb id
                if 'imdb_id' in entry and entry['imdb_id'] is not None:
                    imdb_id = entry['imdb_id']
                else:
                    imdb_id = extract_id(entry['imdb_url'])

                if not imdb_id:
                    log.warning("No imdb id could be determined for %s" % entry['title'])
                    continue

                if not 'quality' in entry:
                    log.warning('No quality found for %s, assigning unknown.' % entry['title'])
                    entry['quality'] = 'unknown'

                item = feed.session.query(ImdbQueue).filter(ImdbQueue.imdb_id == imdb_id).first()

                if item:
                    # This will return UNKNOWN quality if 'ANY' quality
                    minquality = qualities.parse_quality(item.quality)
                    if 'quality' in entry:
                        entry_quality = qualities.parse_quality(entry['quality'])
                    if entry_quality >= minquality:
                        entry['immortal'] = item.immortal
                        log.debug("found %s quality for %s. Need minimum %s" %
                                  (entry['title'], entry['quality'],
                                   item.quality))
                        log.info("Accepting %s from queue with quality %s. Force: %s" % (entry['title'], entry['quality'], entry['immortal']))
                        feed.accept(entry, 'imdb-queue - force: %s' % entry['immortal'])
                        # Keep track of entries we accepted, so they can be removed from queue on feed_exit if successful
                        self.accepted_entries[imdb_id] = entry
                    else:
                        log.debug("imdb-queue rejecting - found "
                                  "%s quality for %s. Need minimum %s" %
                                  (entry['title'], entry['quality'],
                                   item.quality))
                        # Rejecting, as imdb-queue overrides anything. Don't
                        # want to accidentally grab lower quality than desired.
                        entry['immortal'] = False
                        feed.reject(entry, 'imdb-queue quality '
                                    '%s below minimum %s for %s' %
                                    (entry_quality.name, minquality.name,
                                     entry['title']))
                else:
                    log.debugall("%s not in queue with wanted quality, skipping" % entry['title'])
        if len(rejected):
            log.info("Rejected due to no URL in entry (URLRewrite probably failed):"
                     " %s" % ', '.join(rejected))

    def on_feed_exit(self, feed):
        """
        Removes any entries that have not been rejected by another plugin or failed from the queue.
        """
        for imdb_id, entry in self.accepted_entries.iteritems():
            if entry in feed.accepted and entry not in feed.failed:
                # If entry was not rejected or failed, remove from database
                item = feed.session.query(ImdbQueue).filter(ImdbQueue.imdb_id == imdb_id).first()
                feed.session.delete(item)
                log.debug('%s was successful, removing from imdb-queue' % entry['title'])

class ImdbQueueManager(object):
    """
    Handle IMDb queue management; add, delete and list
    """

    valid_actions = ['add', 'del', 'list']

    options = {}

    @staticmethod
    def optik_imdb_queue(option, opt, value, parser):
        """
        Callback for Optik
        --imdb-queue (add|del|list) [IMDB_URL|NAME] [quality]
        """
        if not parser.rargs:
            print 'Usage: --imdb-queue (add|del|list) [IMDB_URL|NAME] [QUALITY] [FORCE]'
            # set some usage option so that feeds will be disabled later
            ImdbQueueManager.options['usage'] = True
            return

        ImdbQueueManager.options['action'] = parser.rargs[0].lower()

        if len(parser.rargs) == 1:
            return
        # 2 args is the minimum allowed (operation + item)
        if len(parser.rargs) >= 2:
            ImdbQueueManager.options['what'] = parser.rargs[1]

        # 3, quality
        if len(parser.rargs) >= 3:
            ImdbQueueManager.options['quality'] = parser.rargs[2]
        else:
            ImdbQueueManager.options['quality'] = 'ANY' # TODO: Get default from config somehow?

        # 4, force download
        if len(parser.rargs) >= 4:
            ImdbQueueManager.options['force'] = str_to_boolean(parser.rargs[3])
        else:
            ImdbQueueManager.options['force'] = True

    def parse_what(self, what):
        """Given an imdb id or movie title, looks up from imdb and returns a dict with imdb_id and title keys"""
        imdb_id = extract_id(what)
        title = what

        if imdb_id:
            # Given an imdb id, find title
            parser = ImdbParser()
            try:
                parser.parse('http://www.imdb.com/title/%s' % imdb_id)
            except Exception:
                raise QueueError('Error parsing info from imdb for %s' % imdb_id)
            if parser.name:
                title = parser.name
        else:
            # Given a title, try to do imdb search for id
            print 'Searching imdb for %s' % what
            search = ImdbSearch()
            result = search.smart_match(what)
            if not result:
                raise QueueError('ERROR: Unable to find any such movie from imdb, use imdb url instead.')
            imdb_id = extract_id(result['url'])
            title = result['name']

        self.options['imdb_id'] = imdb_id
        self.options['title'] = title
        return {'title': title, 'imdb_id': imdb_id}

    def on_process_start(self, feed):
        """
        Handle IMDb queue management
        """

        if not self.options:
            return

        feed.manager.disable_feeds()

        if 'usage' in self.options:
            return

        action = self.options['action']
        if action not in self.valid_actions:
            self.error('Invalid action, valid actions are: ' + ', '.join(self.valid_actions))
            return

        # all actions except list require imdb_url to work
        if action != 'list':
            if not self.options.get('what'):
                self.error('No URL or NAME given')
                return
            else:
                # Generate imdb_id and movie title from movie name, or imdb_url
                try:
                    what = self.parse_what(self.options['what'])
                except QueueError, e:
                    print e.message
                else:
                    self.options.update(what)

            if not self.options.get('title') or not self.options.get('imdb_id'):
                print 'could not determine movie to add' # TODO: Rethink errors
                return

        from sqlalchemy.exceptions import OperationalError
        try:
            if action == 'add':
                try:
                    added = self.queue_add(title=self.options['title'], imdb_id=self.options['imdb_id'],
                               quality=self.options['quality'], force=self.options['force'])
                except QueueError, e:
                    print e.message
                    if e.errno == 1:
                        # This is an invalid quality error, display some more info
                        print 'Recognized qualities are %s' % ', '.join([qual.name for qual in qualities.all()])
                        print 'ANY is the default and can also be used explicitly to specify that quality should be ignored.'
                else:
                    print 'Added %s to queue with quality %s' % (added['title'], added['quality'])
            elif action == 'del':
                try:
                    title = self.queue_del(self.options['imdb_id'])
                except QueueError, e:
                    print e.message
                else:
                    print '%s removed from queue.' % title
            elif action == 'list':
                self.queue_list()
        except OperationalError:
            log.critical('OperationalError')

    def error(self, msg):
        print 'IMDb Queue error: %s' % msg

    def validate_quality(self, quality):
        # Check that the quality is valid
        # Make sure quality is in the format we expect
        if quality.upper() == 'ANY':
            return 'ANY'
        elif qualities.get(quality, False):
            return qualities.common_name(quality)
        else:
            raise QueueError('ERROR! Unknown quality `%s`' % quality, errno=1)

    def queue_add(self, title=None, imdb_id=None, quality='ANY', force=True):
        """Add an item to the queue with the specified quality"""

        if not title or not imdb_id:
            # We don't have all the info we need to add movie, do a lookup for more info
            result = self.parse_what(imdb_id or title)
            title = result['title']
            imdb_id = result['imdb_id']
        quality = self.validate_quality(quality)

        session = Session()

        # check if the item is already queued
        item = session.query(ImdbQueue).filter(ImdbQueue.imdb_id == imdb_id).first()
        if not item:
            item = ImdbQueue(imdb_id, quality, force)
            item.title = title
            session.add(item)
            session.commit()
            return {'title': title, 'imdb_id': imdb_id, 'quality': quality, 'force': force}
        else:
            raise QueueError('ERROR: %s is already in the queue' % title)

    def queue_del(self, imdb_id):
        """Delete the given item from the queue"""

        session = Session()
        # check if the item is queued
        item = session.query(ImdbQueue).filter(ImdbQueue.imdb_id == imdb_id).first()
        if item:
            title = item.title
            session.delete(item)
            session.commit()
            return title
        else:
            raise QueueError('%s is not in the queue' % imdb_id)

    def queue_edit(self, imdb_id, quality):
        """Change the required quality for a movie in the queue"""
        self.validate_quality(quality)
        session = Session()
        # check if the item is queued
        item = session.query(ImdbQueue).filter(ImdbQueue.imdb_id == imdb_id).first()
        if item:
            item.quality = quality
            session.commit()
            return item.title
        else:
            raise QueueError('%s is not in the queue' % imdb_id)

    def queue_list(self):
        """List IMDb queue"""

        items = self.queue_get()
        print '-' * 79
        print '%-10s %-45s %-8s %s' % ('IMDB id', 'Title', 'Quality', 'Force')
        print '-' * 79
        for item in items:
            print '%-10s %-45s %-8s %s' % (item.imdb_id, item.title, item.quality, item.immortal)

        if not items:
            print 'IMDB queue is empty'

        print '-' * 79

    def queue_get(self):
        """Get the current IMDb queue.

        Returns:

        List of ImdbQueue objects (detached from session)
        """
        session = Session()
        try:
            items = session.query(ImdbQueue).all()
            for item in items:
                if not item.title:
                    # old database does not have title / title not retrieved
                    try:
                        item.title = self.parse_what(item.imdb_id)['title']
                    except QueueError:
                        item.title = 'N/A'
            return items
        finally:
            session.close()

register_plugin(FilterImdbQueue, 'imdb_queue')
register_plugin(ImdbQueueManager, 'imdb_queue_manager', builtin=True)
# Handle if a urlrewrite happens, need to get accurate quality.
register_feed_phase(FilterImdbQueue, 'imdbqueue', after='urlrewrite')

register_parser_option('--imdb-queue', action='callback', callback=ImdbQueueManager.optik_imdb_queue,
                       help='(add|del|list) [IMDB_URL|NAME] [QUALITY]')
<commit_msg>Fix imdb_queue for changes to how quality is stored in entries. fixes #999<commit_after>import logging
import datetime
from flexget.manager import Session
from flexget.plugin import register_plugin, PluginError, get_plugin_by_name, priority, register_parser_option, register_feed_phase
from flexget.manager import Base
from sqlalchemy import Column, Integer, String, DateTime, Boolean, Unicode
from flexget.utils.imdb import extract_id, ImdbSearch, ImdbParser, log as imdb_log
from flexget.utils.tools import str_to_boolean
from flexget.utils import qualities

log = logging.getLogger('imdb_queue')

class QueueError(Exception):
    """Exception raised if there is an error with a queue operation"""

    def __init__(self, message, errno=0):
        self.message = message
        self.errno = errno

class ImdbQueue(Base):

    __tablename__ = 'imdb_queue'

    id = Column(Integer, primary_key=True)
    imdb_id = Column(String)
    quality = Column(String)
    title = Column(Unicode)
    immortal = Column(Boolean)
    added = Column(DateTime)

    def __init__(self, imdb_id, quality, immortal):
        self.imdb_id = imdb_id
        self.quality = quality
        self.immortal = immortal
        self.added = datetime.datetime.now()

    def __str__(self):
        return '<ImdbQueue(imdb_id=%s,quality=%s,force=%s)>' % (self.imdb_id, self.quality, self.immortal)

class FilterImdbQueue(object):
    """
    Allows a queue of upcoming movies that will be forcibly allowed if the given quality matches

    Example:

    imdb_queue: yes
    """

    # Dict of entries accepted by this plugin {imdb_id: entry} format
    accepted_entries = {}

    def validator(self):
        from flexget import validator
        return validator.factory('boolean')

    @priority(129)
    def on_feed_filter(self, feed):
        # Doing this so that I register as a filter plugin. Just need to filter
        # after urlrewrite happens, just before download.
        # Also have to accept anything with an IMDB url that matches, even if
        # rejecting later.
        rejected = []
        for entry in feed.entries:
            # make sure the entry has IMDB fields filled
            try:
                get_plugin_by_name('imdb_lookup').instance.lookup(feed, entry)
            except PluginError:
                # no IMDB data, can't do anything
                continue

            # entry has imdb url
            if 'imdb_url' in entry:
                # get an imdb id
                if 'imdb_id' in entry and entry['imdb_id'] is not None:
                    imdb_id = entry['imdb_id']
                else:
                    imdb_id = extract_id(entry['imdb_url'])

                if not imdb_id:
                    log.warning("No imdb id could be determined for %s" % entry['title'])
                    continue

                item = feed.session.query(ImdbQueue).filter(ImdbQueue.imdb_id == imdb_id).first()

                if item:
                    entry['immortal'] = item.immortal
                    log.debug("Pre-accepting %s from queue" % (entry['title']))
                    feed.accept(entry, 'imdb-queue pre-accept')
                    # Keep track of entries we accepted, so they can be removed from queue on feed_exit if successful
                    self.accepted_entries[imdb_id] = entry
                else:
                    log.debugall("%s not in queue, skipping" % entry['title'])

    def on_feed_imdbqueue(self, feed):
        rejected = []
        for entry in feed.entries:
            if entry['url'] == '':
                feed.reject(entry, 'imdb-queue - no URL in entry: '
                            '%s' % entry['title'])
                rejected.append(entry['title'])
                continue
            # make sure the entry has IMDB fields filled
            try:
                get_plugin_by_name('imdb_lookup').instance.lookup(feed, entry)
            except PluginError:
                # no IMDB data, can't do anything
                continue

            # entry has imdb url
            if 'imdb_url' in entry:
                # get an imdb id
                if 'imdb_id' in entry and entry['imdb_id'] is not None:
                    imdb_id = entry['imdb_id']
                else:
                    imdb_id = extract_id(entry['imdb_url'])

                if not imdb_id:
                    log.warning("No imdb id could be determined for %s" % entry['title'])
                    continue

                if not 'quality' in entry:
                    log.warning('No quality found for %s, assigning unknown.' % entry['title'])
                    entry['quality'] = 'unknown'

                item = feed.session.query(ImdbQueue).filter(ImdbQueue.imdb_id == imdb_id).first()

                if item:
                    # This will return UNKNOWN quality if 'ANY' quality
                    minquality = qualities.parse_quality(item.quality)
                    if entry.get('quality', qualities.UNKNOWN) >= minquality:
                        entry['immortal'] = item.immortal
                        log.debug("found %s quality for %s. Need minimum %s" %
                                  (entry['title'], entry['quality'],
                                   item.quality))
                        log.info("Accepting %s from queue with quality %s. Force: %s" % (entry['title'], entry['quality'], entry['immortal']))
                        feed.accept(entry, 'imdb-queue - force: %s' % entry['immortal'])
                        # Keep track of entries we accepted, so they can be removed from queue on feed_exit if successful
                        self.accepted_entries[imdb_id] = entry
                    else:
                        log.debug("imdb-queue rejecting - found "
                                  "%s quality for %s. Need minimum %s" %
                                  (entry['title'], entry['quality'],
                                   item.quality))
                        # Rejecting, as imdb-queue overrides anything. Don't
                        # want to accidentally grab lower quality than desired.
                        entry['immortal'] = False
                        feed.reject(entry, 'imdb-queue quality %s below minimum %s for %s' %
                                           (entry.get('quality'), minquality.name, entry['title']))
                else:
                    log.debugall("%s not in queue with wanted quality, skipping" % entry['title'])
        if len(rejected):
            log.info("Rejected due to no URL in entry (URLRewrite probably failed):"
                     " %s" % ', '.join(rejected))

    def on_feed_exit(self, feed):
        """
        Removes any entries that have not been rejected by another plugin or failed from the queue.
        """
        for imdb_id, entry in self.accepted_entries.iteritems():
            if entry in feed.accepted and entry not in feed.failed:
                # If entry was not rejected or failed, remove from database
                item = feed.session.query(ImdbQueue).filter(ImdbQueue.imdb_id == imdb_id).first()
                feed.session.delete(item)
                log.debug('%s was successful, removing from imdb-queue' % entry['title'])

class ImdbQueueManager(object):
    """
    Handle IMDb queue management; add, delete and list
    """

    valid_actions = ['add', 'del', 'list']

    options = {}

    @staticmethod
    def optik_imdb_queue(option, opt, value, parser):
        """
        Callback for Optik
        --imdb-queue (add|del|list) [IMDB_URL|NAME] [quality]
        """
        if not parser.rargs:
            print 'Usage: --imdb-queue (add|del|list) [IMDB_URL|NAME] [QUALITY] [FORCE]'
            # set some usage option so that feeds will be disabled later
            ImdbQueueManager.options['usage'] = True
            return

        ImdbQueueManager.options['action'] = parser.rargs[0].lower()

        if len(parser.rargs) == 1:
            return
        # 2 args is the minimum allowed (operation + item)
        if len(parser.rargs) >= 2:
            ImdbQueueManager.options['what'] = parser.rargs[1]

        # 3, quality
        if len(parser.rargs) >= 3:
            ImdbQueueManager.options['quality'] = parser.rargs[2]
        else:
            ImdbQueueManager.options['quality'] = 'ANY' # TODO: Get default from config somehow?

        # 4, force download
        if len(parser.rargs) >= 4:
            ImdbQueueManager.options['force'] = str_to_boolean(parser.rargs[3])
        else:
            ImdbQueueManager.options['force'] = True

    def parse_what(self, what):
        """Given an imdb id or movie title, looks up from imdb and returns a dict with imdb_id and title keys"""
        imdb_id = extract_id(what)
        title = what

        if imdb_id:
            # Given an imdb id, find title
            parser = ImdbParser()
            try:
                parser.parse('http://www.imdb.com/title/%s' % imdb_id)
            except Exception:
                raise QueueError('Error parsing info from imdb for %s' % imdb_id)
            if parser.name:
                title = parser.name
        else:
            # Given a title, try to do imdb search for id
            print 'Searching imdb for %s' % what
            search = ImdbSearch()
            result = search.smart_match(what)
            if not result:
                raise QueueError('ERROR: Unable to find any such movie from imdb, use imdb url instead.')
            imdb_id = extract_id(result['url'])
            title = result['name']

        self.options['imdb_id'] = imdb_id
        self.options['title'] = title
        return {'title': title, 'imdb_id': imdb_id}

    def on_process_start(self, feed):
        """
        Handle IMDb queue management
        """

        if not self.options:
            return

        feed.manager.disable_feeds()

        if 'usage' in self.options:
            return

        action = self.options['action']
        if action not in self.valid_actions:
            self.error('Invalid action, valid actions are: ' + ', '.join(self.valid_actions))
            return

        # all actions except list require imdb_url to work
        if action != 'list':
            if not self.options.get('what'):
                self.error('No URL or NAME given')
                return
            else:
                # Generate imdb_id and movie title from movie name, or imdb_url
                try:
                    what = self.parse_what(self.options['what'])
                except QueueError, e:
                    print e.message
                else:
                    self.options.update(what)

            if not self.options.get('title') or not self.options.get('imdb_id'):
                print 'could not determine movie to add' # TODO: Rethink errors
                return

        from sqlalchemy.exceptions import OperationalError
        try:
            if action == 'add':
                try:
                    added = self.queue_add(title=self.options['title'], imdb_id=self.options['imdb_id'],
                               quality=self.options['quality'], force=self.options['force'])
                except QueueError, e:
                    print e.message
                    if e.errno == 1:
                        # This is an invalid quality error, display some more info
                        print 'Recognized qualities are %s' % ', '.join([qual.name for qual in qualities.all()])
                        print 'ANY is the default and can also be used explicitly to specify that quality should be ignored.'
                else:
                    print 'Added %s to queue with quality %s' % (added['title'], added['quality'])
            elif action == 'del':
                try:
                    title = self.queue_del(self.options['imdb_id'])
                except QueueError, e:
                    print e.message
                else:
                    print '%s removed from queue.' % title
            elif action == 'list':
                self.queue_list()
        except OperationalError:
            log.critical('OperationalError')

    def error(self, msg):
        print 'IMDb Queue error: %s' % msg

    def validate_quality(self, quality):
        # Check that the quality is valid
        # Make sure quality is in the format we expect
        if quality.upper() == 'ANY':
            return 'ANY'
        elif qualities.get(quality, False):
            return qualities.common_name(quality)
        else:
            raise QueueError('ERROR! Unknown quality `%s`' % quality, errno=1)

    def queue_add(self, title=None, imdb_id=None, quality='ANY', force=True):
        """Add an item to the queue with the specified quality"""

        if not title or not imdb_id:
            # We don't have all the info we need to add movie, do a lookup for more info
            result = self.parse_what(imdb_id or title)
            title = result['title']
            imdb_id = result['imdb_id']
        quality = self.validate_quality(quality)

        session = Session()

        # check if the item is already queued
        item = session.query(ImdbQueue).filter(ImdbQueue.imdb_id == imdb_id).first()
        if not item:
            item = ImdbQueue(imdb_id, quality, force)
            item.title = title
            session.add(item)
            session.commit()
            return {'title': title, 'imdb_id': imdb_id, 'quality': quality, 'force': force}
        else:
            raise QueueError('ERROR: %s is already in the queue' % title)

    def queue_del(self, imdb_id):
        """Delete the given item from the queue"""

        session = Session()
        # check if the item is queued
        item = session.query(ImdbQueue).filter(ImdbQueue.imdb_id == imdb_id).first()
        if item:
            title = item.title
            session.delete(item)
            session.commit()
            return title
        else:
            raise QueueError('%s is not in the queue' % imdb_id)

    def queue_edit(self, imdb_id, quality):
        """Change the required quality for a movie in the queue"""
        self.validate_quality(quality)
        session = Session()
        # check if the item is queued
        item = session.query(ImdbQueue).filter(ImdbQueue.imdb_id == imdb_id).first()
        if item:
            item.quality = quality
            session.commit()
            return item.title
        else:
            raise QueueError('%s is not in the queue' % imdb_id)

    def queue_list(self):
        """List IMDb queue"""

        items = self.queue_get()
        print '-' * 79
        print '%-10s %-45s %-8s %s' % ('IMDB id', 'Title', 'Quality', 'Force')
        print '-' * 79
        for item in items:
            print '%-10s %-45s %-8s %s' % (item.imdb_id, item.title, item.quality, item.immortal)

        if not items:
            print 'IMDB queue is empty'

        print '-' * 79

    def queue_get(self):
        """Get the current IMDb queue.

        Returns:

        List of ImdbQueue objects (detached from session)
        """
        session = Session()
        try:
            items = session.query(ImdbQueue).all()
            for item in items:
                if not item.title:
                    # old database does not have title / title not retrieved
                    try:
                        item.title = self.parse_what(item.imdb_id)['title']
                    except QueueError:
                        item.title = 'N/A'
            return items
        finally:
            session.close()

register_plugin(FilterImdbQueue, 'imdb_queue')
register_plugin(ImdbQueueManager, 'imdb_queue_manager', builtin=True)
# Handle if a urlrewrite happens, need to get accurate quality.
register_feed_phase(FilterImdbQueue, 'imdbqueue', after='urlrewrite')

register_parser_option('--imdb-queue', action='callback', callback=ImdbQueueManager.optik_imdb_queue,
                       help='(add|del|list) [IMDB_URL|NAME] [QUALITY]')
<|endoftext|>


<commit_before># gmemsess.py - memcache-backed session Class for Google Appengine
# Version 1.2
#    Copyright 2008 Greg Fawcett <greg@vig.co.nz>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import random
import pickle
from google.appengine.api import memcache

_sidChars='abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
_defaultTimeout=30*60 # 30 min
_defaultCookieName='gsessionid'

#----------------------------------------------------------------------
class Session(dict):
    """A secure lightweight memcache-backed session Class for Google Appengine."""

    #----------------------------------------------------------
    def __init__(self,rh,name=_defaultCookieName,timeout=_defaultTimeout):
        """Create a session object.

        Keyword arguments:
        rh -- the parent's request handler (usually self)
        name -- the cookie name (defaults to "gsid")
        timeout -- the number of seconds the session will last between
                   requests (defaults to 1800 secs - 30 minutes)
        """
        self.rh=rh    # request handler
        self._timeout=timeout
        self._name=name
        self._new=True
        self._invalid=False
        dict.__init__(self)

        if name in rh.request.cookies:
            self._sid=rh.request.cookies[name]
            data=memcache.get(self._sid)
            if data:
                self.update(pickle.loads(data))
                # memcache timeout is absolute, so we need to reset it on each access
                memcache.set(self._sid,data,self._timeout)
                self._new=False
                return

        # Create a new session ID
        # There are about 10^14 combinations, so guessing won't work
        self._sid=random.choice(_sidChars)+random.choice(_sidChars)+\
                        random.choice(_sidChars)+random.choice(_sidChars)+\
                        random.choice(_sidChars)+random.choice(_sidChars)+\
                        random.choice(_sidChars)+random.choice(_sidChars)+\
                        random.choice(_sidChars)+random.choice(_sidChars)
        rh.response.headers.add_header('Set-Cookie','%s=%s;'%(name,self._sid))

    #----------------------------------------------------------
    def save(self):
        """Save session data."""
        if not self._invalid:
            memcache.set(self._sid,pickle.dumps(self.copy()),self._timeout)

    #----------------------------------------------------------
    def is_new(self):
        """Returns True if session was created during this request."""
        return self._new

    #----------------------------------------------------------
    def invalidate(self):
        """Delete session data and cookie."""
        self.rh.response.headers.add_header('Set-Cookie',
                '%s=; expires=Sat, 1-Jan-2000 00:00:00 GMT;'%(self._name))
        memcache.delete(self._sid)
        self.clear()
        self._invalid=True
<commit_msg>Modified the Set-Cookie header to contain path (Scott Seeley).<commit_after># gmemsess.py - memcache-backed session Class for Google Appengine
# Version 1.2
#    Copyright 2008 Greg Fawcett <greg@vig.co.nz>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import random
import pickle
from google.appengine.api import memcache

_sidChars='abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
_defaultTimeout=30*60 # 30 min
_defaultCookieName='gsessionid'

#----------------------------------------------------------------------
class Session(dict):
    """A secure lightweight memcache-backed session Class for Google Appengine."""

    #----------------------------------------------------------
    def __init__(self,rh,name=_defaultCookieName,timeout=_defaultTimeout):
        """Create a session object.

        Keyword arguments:
        rh -- the parent's request handler (usually self)
        name -- the cookie name (defaults to "gsid")
        timeout -- the number of seconds the session will last between
                   requests (defaults to 1800 secs - 30 minutes)
        """
        self.rh=rh    # request handler
        self._timeout=timeout
        self._name=name
        self._new=True
        self._invalid=False
        dict.__init__(self)

        if name in rh.request.cookies:
            self._sid=rh.request.cookies[name]
            data=memcache.get(self._sid)
            if data:
                self.update(pickle.loads(data))
                # memcache timeout is absolute, so we need to reset it on each access
                memcache.set(self._sid,data,self._timeout)
                self._new=False
                return

        # Create a new session ID
        # There are about 10^14 combinations, so guessing won't work
        self._sid=random.choice(_sidChars)+random.choice(_sidChars)+\
                        random.choice(_sidChars)+random.choice(_sidChars)+\
                        random.choice(_sidChars)+random.choice(_sidChars)+\
                        random.choice(_sidChars)+random.choice(_sidChars)+\
                        random.choice(_sidChars)+random.choice(_sidChars)
        rh.response.headers.add_header('Set-Cookie','%s=%s; path=/;'%(name,self._sid))

    #----------------------------------------------------------
    def save(self):
        """Save session data."""
        if not self._invalid:
            memcache.set(self._sid,pickle.dumps(self.copy()),self._timeout)

    #----------------------------------------------------------
    def is_new(self):
        """Returns True if session was created during this request."""
        return self._new

    #----------------------------------------------------------
    def invalidate(self):
        """Delete session data and cookie."""
        self.rh.response.headers.add_header('Set-Cookie',
                '%s=; expires=Sat, 1-Jan-2000 00:00:00 GMT;'%(self._name))
        memcache.delete(self._sid)
        self.clear()
        self._invalid=True
<|endoftext|>


<commit_before>import boto3
import time
import sys
import datetime
from settings import *
import os
from threading import Thread
import subprocess
cwd = os.path.dirname(os.path.realpath(__file__))

ec2resource = boto3.resource('ec2')
ec2client = boto3.client('ec2')

def runTest(branch,OS,webdir,statusDir,now,instance):
    make_dir(os.path.join(webdir,OS))
    make_dir(statusDir)

    # Create hidden file for node - for status reporting.
    # print "Creating " + str(os.path.join(statusDir,OS + "." + branch + ".result"))
    with open(os.path.join(statusDir,OS + "." + branch + ".result"), 'w') as content_file:
        content_file.write("-1")

    # print  "Kickin tires"
    # Kick the tires a bit, this helps the remote host to 'wake up', and for a network path to be learned by involved routers.
    subprocess.call(timeout + " " + sshTime + " " + ssh + " -o ConnectTimeout=" + sshTimeout + " " + OS + ' "echo wakeup" > /dev/null 2>&1', shell=True)
    subprocess.call(timeout + " " + sshTime + " " + ssh + " -o ConnectTimeout=" + sshTimeout + " " + OS + ' "echo get ready" > /dev/null 2>&1', shell=True)

    # print "Scp script to remote box"
    # Scp a script onto the remote box that we will later call.
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + os.path.join(cwd,'installBranch.sh') + " " + OS + ":/root/installBranch.sh", shell=True)

    # Get starting time
    d1 = datetime.datetime.now()

    # print "Starting installer"
    # Start the fog installer.
    subprocess.call(timeout + " " + fogTimeout + " " + ssh + " -o ConnectTimeout=" + sshTimeout + " " + OS + ' "/root/./installBranch.sh ' + branch + '"', shell=True)

    # Get ending time.
    d2 = datetime.datetime.now()

    # Calculate duration.
    duration = d2 - d1
    duration = str(datetime.timedelta(seconds=duration.total_seconds()))
    # Write duration to file.
    with open(os.path.join(statusDir,OS + "." + branch + ".duration"), 'w') as content_file:
        content_file.write(duration)

    # print "Getting result file"
    # Get the result file.
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/root/result " + os.path.join(statusDir,OS + "." + branch + ".result"), shell=True)
    # This should send the result code of the attempt to something like /tmp/debian9.master.result

    # print "Getting output file"
    # Get the output file.
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/root/output " + os.path.join(webdir,OS,now + "_output.log"), shell=True)

    # print "Getting fog log file"
    # Get the fog log.
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/root/git/fogproject/bin/error_logs/fog_error* " + os.path.join(webdir,OS,now + "_install.log"), shell=True)

    # print "Getting apache logs"
    # Get the apache error logs. Can be in only two places.
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/var/log/httpd/error_log " + os.path.join(webdir,OS,now + "_apache.log") + " > /dev/null 2>&1", shell=True)
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/var/log/apache2/error.log " + os.path.join(webdir,OS,now + "_apache.log") + " > /dev/null 2>&1", shell=True)

    # print "Getting php-fpm logs"
    # Get php-fpm logs. Can be in several places...
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/var/log/php-fpm/www-error.log " + os.path.join(webdir,OS,now + "_php-fpm.log") + " > /dev/null 2>&1", shell=True)
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/var/log/php-fpm/error.log " + os.path.join(webdir,OS,now + "_php-fpm.log") + " > /dev/null 2>&1", shell=True)
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/var/log/php*-fpm.log " + os.path.join(webdir,OS,now + "_php-fpm.log") + " > /dev/null 2>&1", shell=True)

    # print "Getting commit"
    # Get the commit the remote node was using, just as a sainity check.
    subprocess.call(timeout + " " + fogTimeout + " " + ssh + " -o ConnectTimeout=" + sshTimeout + " " + OS + ' "cd /root/git/fogproject;git rev-parse HEAD > /root/commit"', shell=True)
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/root/commit " + os.path.join(statusDir,OS + "." + branch + ".commit"), shell=True)
    # This should send just the commit that was used in the test to something like /tmp/debian9.master.commit

    # print "Reading commit"
    # Read the commit.
    commit = read_file(os.path.join(statusDir,OS + "." + branch + ".commit"))

    # Kill the instance.
    instance.stop(Force=True)

    # print "Rebuilding log"
    # Rebuild the log file to have information at the top of it.
    log = "Date=" + now + "\n"
    log = log + "Branch=" + branch + "\n"
    log = log + "Commit=" + commit # The commit comes back with a line feed in it.
    log = log + "OS=" + OS + "\n"
    log = log + "##### Begin Log #####\n"
    log = log + read_file(os.path.join(webdir,OS,now + "_install.log"))

    # print "Writing log"
    # Write the new log.
    with open(os.path.join(webdir,OS,now + "_install.log"), 'w') as content_file:
        content_file.write(log)

def add_ssh_identities():
    known_hosts_content = ""
    subprocess.call("echo '' > ~/.ssh/known_hosts", shell=True)
    for address in dnsAddresses:
        # Kick the tires a bit, this helps the remote host to 'wake up', and for a network path to be learned by involved routers.
        for i in range(0,7):
            subprocess.call(timeout + " " + sshTime + " " + ssh_keyscan + " -H " + address + " > /dev/null 2>&1", shell=True)
            time.sleep(wait)

        time.sleep(wait)
        subprocess.call(timeout + " " + sshTime + " " + ssh_keyscan + " -H " + address + " >> ~/.ssh/known_hosts 2> /dev/null", shell=True)

def read_file(path):
    if os.path.isfile(path):
        with open(path, 'r') as content_file:
            try:
                return content_file.read()
            except Exception as e:
                return e
    else:
        return "The file '" + path + "' does not exist."

def make_dir(directory):
    if not os.path.exists(directory):
        os.makedirs(directory)

def complete_threads(threads):
    # Start all the threads.
    for x in threads:
        x.start()

    # Wait for all threads to exit.
    for x in threads:
        x.join()

def get_instance(name,value):
    # This assumes one match
    # Only for instances that are not terminated.
    response = ec2client.describe_instances(
       Filters=[
            {
                'Name': 'tag:' + name,
                'Values': [value]
            },
            {
                'Name':'instance-state-name',
                'Values':['pending','running','shutting-down','stopping','stopped']
            }
       ]
    )
    instance = ec2resource.Instance(response["Reservations"][0]["Instances"][0]["InstanceId"])
    return instance

def get_instance_volume(instance):
    # This assumes one volume.
    volume = ec2resource.Volume(instance.block_device_mappings[0]["Ebs"]["VolumeId"])
    return volume

def wait_until_stopped(instance):
    while True:
        instance.reload()
        if instance.state["Name"] == "stopped":
            break
        else:
            time.sleep(wait)
    return

def wait_until_running(instance):
    while True:
        instance.reload()
        if instance.state["Name"] == "running":
            break
        else:
            time.sleep(wait)
    return

def create_snapshot(volume,name_tag):
    snapshot = volume.create_snapshot()
    snapshot.create_tags(
        Tags=[
            {
                'Key': 'Name',
                'Value': name_tag
            }
        ]
    )
    while True:
        snapshot.reload()
        if snapshot.state == "completed":
            break
        else:
            time.sleep(wait)
    return snapshot

def delete_snapshots(name,value):
    # This deletes all matching snapshots.
    response = ec2client.describe_snapshots(
       Filters=[
            {
                'Name': 'tag:' + name,
                'Values': [value]
            }
       ]
    )
    for snapshotDict in response["Snapshots"]:
        snapshot = ec2resource.Snapshot(snapshotDict["SnapshotId"])
        snapshot.delete()

def get_snapshot(name,value):
    # Assumes one match.
    response = ec2client.describe_snapshots(
       Filters=[
            {
                'Name': 'tag:' + name,
                'Values': [value]
            }
       ]
    )
    snapshot = ec2resource.Snapshot(response["Snapshots"][0]["SnapshotId"])
    return snapshot

def restore_clean_snapshots():
    threads = []
    for OS in OSs:
        instance = get_instance("Name","fogtesting-" + OS)
        snapshot = get_snapshot("Name",OS + '-clean')
        if OS == "debian9":
            threads.append(Thread(target=restore_snapshot_to_instance,args=(snapshot,instance,"xvda")))
        elif OS == "centos7":
            threads.append(Thread(target=restore_snapshot_to_instance,args=(snapshot,instance,"/dev/sda1")))
        elif OS == "rhel7":
            threads.append(Thread(target=restore_snapshot_to_instance,args=(snapshot,instance,"/dev/sda1")))

    complete_threads(threads)

def restore_snapshot_to_instance(snapshot,instance,device):
    """
    Stop the instance
    detach and delete the old volume.
    Create a new volume
    Attach the new volume
    Start the instance
    """
    instance.stop(Force=True)
    wait_until_stopped(instance)
    oldVolume = get_instance_volume(instance)
    oldVolume.detach_from_instance(Force=True)
    while True:
        oldVolume.reload()
        if oldVolume.state == "available":
            break
        else:
            time.sleep(wait)

    oldVolume.delete()
    newVolume = ec2client.create_volume(SnapshotId=snapshot.id,AvailabilityZone=zone,VolumeType='standard')
    newVolume = ec2resource.Volume(newVolume["VolumeId"])
    while True:
        newVolume.reload()
        if newVolume.state == "available":
            break
        else:
            time.sleep(wait)
    instance.attach_volume(VolumeId=newVolume.id,Device=device)
    while True:
        newVolume.reload()
        if newVolume.state == "in-use":
            break
        else:
            time.sleep(wait)

    instance.modify_attribute(BlockDeviceMappings=[{'Ebs': {'DeleteOnTermination': True}, 'DeviceName': device}])

    instance.start()
    wait_until_running(instance)

<commit_msg>Correct timeout for getting commit<commit_after>import boto3
import time
import sys
import datetime
from settings import *
import os
from threading import Thread
import subprocess
cwd = os.path.dirname(os.path.realpath(__file__))

ec2resource = boto3.resource('ec2')
ec2client = boto3.client('ec2')

def runTest(branch,OS,webdir,statusDir,now,instance):
    make_dir(os.path.join(webdir,OS))
    make_dir(statusDir)

    # Create hidden file for node - for status reporting.
    # print "Creating " + str(os.path.join(statusDir,OS + "." + branch + ".result"))
    with open(os.path.join(statusDir,OS + "." + branch + ".result"), 'w') as content_file:
        content_file.write("-1")

    # print  "Kickin tires"
    # Kick the tires a bit, this helps the remote host to 'wake up', and for a network path to be learned by involved routers.
    subprocess.call(timeout + " " + sshTime + " " + ssh + " -o ConnectTimeout=" + sshTimeout + " " + OS + ' "echo wakeup" > /dev/null 2>&1', shell=True)
    subprocess.call(timeout + " " + sshTime + " " + ssh + " -o ConnectTimeout=" + sshTimeout + " " + OS + ' "echo get ready" > /dev/null 2>&1', shell=True)

    # print "Scp script to remote box"
    # Scp a script onto the remote box that we will later call.
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + os.path.join(cwd,'installBranch.sh') + " " + OS + ":/root/installBranch.sh", shell=True)

    # Get starting time
    d1 = datetime.datetime.now()

    # print "Starting installer"
    # Start the fog installer.
    subprocess.call(timeout + " " + fogTimeout + " " + ssh + " -o ConnectTimeout=" + sshTimeout + " " + OS + ' "/root/./installBranch.sh ' + branch + '"', shell=True)

    # Get ending time.
    d2 = datetime.datetime.now()

    # Calculate duration.
    duration = d2 - d1
    duration = str(datetime.timedelta(seconds=duration.total_seconds()))
    # Write duration to file.
    with open(os.path.join(statusDir,OS + "." + branch + ".duration"), 'w') as content_file:
        content_file.write(duration)

    # print "Getting result file"
    # Get the result file.
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/root/result " + os.path.join(statusDir,OS + "." + branch + ".result"), shell=True)
    # This should send the result code of the attempt to something like /tmp/debian9.master.result

    # print "Getting output file"
    # Get the output file.
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/root/output " + os.path.join(webdir,OS,now + "_output.log"), shell=True)

    # print "Getting fog log file"
    # Get the fog log.
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/root/git/fogproject/bin/error_logs/fog_error* " + os.path.join(webdir,OS,now + "_install.log"), shell=True)

    # print "Getting apache logs"
    # Get the apache error logs. Can be in only two places.
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/var/log/httpd/error_log " + os.path.join(webdir,OS,now + "_apache.log") + " > /dev/null 2>&1", shell=True)
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/var/log/apache2/error.log " + os.path.join(webdir,OS,now + "_apache.log") + " > /dev/null 2>&1", shell=True)

    # print "Getting php-fpm logs"
    # Get php-fpm logs. Can be in several places...
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/var/log/php-fpm/www-error.log " + os.path.join(webdir,OS,now + "_php-fpm.log") + " > /dev/null 2>&1", shell=True)
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/var/log/php-fpm/error.log " + os.path.join(webdir,OS,now + "_php-fpm.log") + " > /dev/null 2>&1", shell=True)
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/var/log/php*-fpm.log " + os.path.join(webdir,OS,now + "_php-fpm.log") + " > /dev/null 2>&1", shell=True)

    # print "Getting commit"
    # Get the commit the remote node was using, just as a sainity check.
    subprocess.call(timeout + " " + sshTime + " " + ssh + " -o ConnectTimeout=" + sshTimeout + " " + OS + ' "cd /root/git/fogproject;git rev-parse HEAD > /root/commit"', shell=True)
    subprocess.call(timeout + " " + sshTime + " " + scp + " -o ConnectTimeout=" + sshTimeout + " " + OS + ":/root/commit " + os.path.join(statusDir,OS + "." + branch + ".commit"), shell=True)
    # This should send just the commit that was used in the test to something like /tmp/debian9.master.commit

    # print "Reading commit"
    # Read the commit.
    commit = read_file(os.path.join(statusDir,OS + "." + branch + ".commit"))

    # Kill the instance.
    instance.stop(Force=True)

    # print "Rebuilding log"
    # Rebuild the log file to have information at the top of it.
    log = "Date=" + now + "\n"
    log = log + "Branch=" + branch + "\n"
    log = log + "Commit=" + commit # The commit comes back with a line feed in it.
    log = log + "OS=" + OS + "\n"
    log = log + "##### Begin Log #####\n"
    log = log + read_file(os.path.join(webdir,OS,now + "_install.log"))

    # print "Writing log"
    # Write the new log.
    with open(os.path.join(webdir,OS,now + "_install.log"), 'w') as content_file:
        content_file.write(log)

def add_ssh_identities():
    known_hosts_content = ""
    subprocess.call("echo '' > ~/.ssh/known_hosts", shell=True)
    for address in dnsAddresses:
        # Kick the tires a bit, this helps the remote host to 'wake up', and for a network path to be learned by involved routers.
        for i in range(0,7):
            subprocess.call(timeout + " " + sshTime + " " + ssh_keyscan + " -H " + address + " > /dev/null 2>&1", shell=True)
            time.sleep(wait)

        time.sleep(wait)
        subprocess.call(timeout + " " + sshTime + " " + ssh_keyscan + " -H " + address + " >> ~/.ssh/known_hosts 2> /dev/null", shell=True)

def read_file(path):
    if os.path.isfile(path):
        with open(path, 'r') as content_file:
            try:
                return content_file.read()
            except Exception as e:
                return e
    else:
        return "The file '" + path + "' does not exist."

def make_dir(directory):
    if not os.path.exists(directory):
        os.makedirs(directory)

def complete_threads(threads):
    # Start all the threads.
    for x in threads:
        x.start()

    # Wait for all threads to exit.
    for x in threads:
        x.join()

def get_instance(name,value):
    # This assumes one match
    # Only for instances that are not terminated.
    response = ec2client.describe_instances(
       Filters=[
            {
                'Name': 'tag:' + name,
                'Values': [value]
            },
            {
                'Name':'instance-state-name',
                'Values':['pending','running','shutting-down','stopping','stopped']
            }
       ]
    )
    instance = ec2resource.Instance(response["Reservations"][0]["Instances"][0]["InstanceId"])
    return instance

def get_instance_volume(instance):
    # This assumes one volume.
    volume = ec2resource.Volume(instance.block_device_mappings[0]["Ebs"]["VolumeId"])
    return volume

def wait_until_stopped(instance):
    while True:
        instance.reload()
        if instance.state["Name"] == "stopped":
            break
        else:
            time.sleep(wait)
    return

def wait_until_running(instance):
    while True:
        instance.reload()
        if instance.state["Name"] == "running":
            break
        else:
            time.sleep(wait)
    return

def create_snapshot(volume,name_tag):
    snapshot = volume.create_snapshot()
    snapshot.create_tags(
        Tags=[
            {
                'Key': 'Name',
                'Value': name_tag
            }
        ]
    )
    while True:
        snapshot.reload()
        if snapshot.state == "completed":
            break
        else:
            time.sleep(wait)
    return snapshot

def delete_snapshots(name,value):
    # This deletes all matching snapshots.
    response = ec2client.describe_snapshots(
       Filters=[
            {
                'Name': 'tag:' + name,
                'Values': [value]
            }
       ]
    )
    for snapshotDict in response["Snapshots"]:
        snapshot = ec2resource.Snapshot(snapshotDict["SnapshotId"])
        snapshot.delete()

def get_snapshot(name,value):
    # Assumes one match.
    response = ec2client.describe_snapshots(
       Filters=[
            {
                'Name': 'tag:' + name,
                'Values': [value]
            }
       ]
    )
    snapshot = ec2resource.Snapshot(response["Snapshots"][0]["SnapshotId"])
    return snapshot

def restore_clean_snapshots():
    threads = []
    for OS in OSs:
        instance = get_instance("Name","fogtesting-" + OS)
        snapshot = get_snapshot("Name",OS + '-clean')
        if OS == "debian9":
            threads.append(Thread(target=restore_snapshot_to_instance,args=(snapshot,instance,"xvda")))
        elif OS == "centos7":
            threads.append(Thread(target=restore_snapshot_to_instance,args=(snapshot,instance,"/dev/sda1")))
        elif OS == "rhel7":
            threads.append(Thread(target=restore_snapshot_to_instance,args=(snapshot,instance,"/dev/sda1")))

    complete_threads(threads)

def restore_snapshot_to_instance(snapshot,instance,device):
    """
    Stop the instance
    detach and delete the old volume.
    Create a new volume
    Attach the new volume
    Start the instance
    """
    instance.stop(Force=True)
    wait_until_stopped(instance)
    oldVolume = get_instance_volume(instance)
    oldVolume.detach_from_instance(Force=True)
    while True:
        oldVolume.reload()
        if oldVolume.state == "available":
            break
        else:
            time.sleep(wait)

    oldVolume.delete()
    newVolume = ec2client.create_volume(SnapshotId=snapshot.id,AvailabilityZone=zone,VolumeType='standard')
    newVolume = ec2resource.Volume(newVolume["VolumeId"])
    while True:
        newVolume.reload()
        if newVolume.state == "available":
            break
        else:
            time.sleep(wait)
    instance.attach_volume(VolumeId=newVolume.id,Device=device)
    while True:
        newVolume.reload()
        if newVolume.state == "in-use":
            break
        else:
            time.sleep(wait)

    instance.modify_attribute(BlockDeviceMappings=[{'Ebs': {'DeleteOnTermination': True}, 'DeviceName': device}])

    instance.start()
    wait_until_running(instance)

<|endoftext|>