<commit_before># -*- coding: utf-8 -*-

# Etalage -- Open Data POIs portal
# By: Emmanuel Raviart <eraviart@easter-eggs.com>
#
# Copyright (C) 2011, 2012 Easter-eggs
# http://gitorious.org/infos-pratiques/etalage
#
# This file is part of Etalage.
#
# Etalage is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Etalage is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""RAM-based database"""

import datetime
import logging
import sys

from biryani import strings
import threading2

from . import conf, conv
from .ramindexes import *

categories_slug_by_tag_slug = {}
categories_slug_by_word = {}
category_by_slug = {}
category_slug_by_pivot_code = {}
last_timestamp = None
read_write_lock = threading2.SHLock()
log = logging.getLogger(__name__)
schema_title_by_name = {}
territories_id_by_ancestor_id = {}
territories_id_by_postal_distribution = {}
territory_by_id = {}
territory_id_by_kind_code = {}

def get_territory_related_territories_id(territory):
    related_territories_id = set()
    for sub_territory_id in territories_id_by_ancestor_id[territory._id]:
        sub_territory = territory_by_id[sub_territory_id]
        for ancestor_id in sub_territory.ancestors_id:
            related_territories_id.add(ancestor_id)
    return related_territories_id

def iter_categories_slug(organism_types_only = False, tags_slug = None, term = None):
    intersected_sets = []
    if organism_types_only:
        intersected_sets.append(set(category_slug_by_pivot_code.itervalues()))
    for tag_slug in set(tags_slug or []):
        if tag_slug is not None:
            intersected_sets.append(categories_slug_by_tag_slug.get(tag_slug))
    if term:
        prefixes = strings.slugify(term).split(u'-')
        categories_slug_by_prefix = {}
        for prefix in prefixes:
            if prefix in categories_slug_by_prefix:
                # TODO? Handle categories with several words sharing the same prefix?
                continue
            categories_slug_by_prefix[prefix] = union_set(
                word_categories_slug
                for word, word_categories_slug in categories_slug_by_word.iteritems()
                if word.startswith(prefix)
                ) or set()
        intersected_sets.extend(categories_slug_by_prefix.itervalues())

    categories_slug = intersection_set(intersected_sets)
    if categories_slug is None:
        return category_by_slug.iterkeys()
    return categories_slug

def load():
    """Load MongoDB data into RAM-based database."""
    from . import model

    start_time = datetime.datetime.utcnow()
    global last_timestamp
    # Remove a few seconds, for data changes that occur during startup.
    last_timestamp = start_time - datetime.timedelta(seconds = 30)

    categories_slug_by_tag_slug.clear()
    categories_slug_by_word.clear()
    category_by_slug.clear()
    category_slug_by_pivot_code.clear()
    for category_bson in model.db[conf['categories_collection']].find(None, ['code', 'tags_code', 'title']):
        if not strings.slugify(category_bson.get('title')):
            continue
        category = model.Category.load(category_bson)
        category.index()

    for organism_type_bson in model.db[conf['organism_types_collection']].find(None, ['code', 'slug']):
        if organism_type_bson['slug'] not in category_by_slug:
            log.warning('Ignoring organism type "{0}" without matching category.'.format(organism_type_bson['code']))
            continue
        category_slug_by_pivot_code[organism_type_bson['code']] = organism_type_bson['slug']

    territories_id_by_ancestor_id.clear()
    territories_id_by_postal_distribution.clear()
    territories_query = dict(
        kind = {'$in': conf['territories_kinds']},
        )
    territory_by_id.clear()
    territory_id_by_kind_code.clear()
    for territory_bson in model.db[conf['territories_collection']].find(territories_query, [
            'ancestors_id',
            'code',
            'geo',
            'hinge_type',
            'kind',
            'main_postal_distribution',
            'name',
            ]):
        main_postal_distribution = territory_bson.get('main_postal_distribution')
        if main_postal_distribution is None:
            continue
        territory_class = model.Territory.kind_to_class(territory_bson['kind'])
        assert territory_class is not None, 'Invalid territory type name: {0}'.format(class_name)
        territory_id = territory_bson['_id']
        territory = territory_class(
            _id = territory_id,
            ancestors_id = territory_bson['ancestors_id'],
            code = territory_bson['code'],
            geo = territory_bson.get('geo'),
            hinge_type = territory_bson.get('hinge_type'),
            main_postal_distribution = main_postal_distribution,
            name = territory_bson['name'],
            )
        territory_by_id[territory_id] = territory
        for ancestor_id in territory_bson['ancestors_id']:
            territories_id_by_ancestor_id.setdefault(ancestor_id, set()).add(territory_id)
        territory_id_by_kind_code[(territory_bson['kind'], territory_bson['code'])] = territory_id
        territories_id_by_postal_distribution[(
            main_postal_distribution['postal_code'],
            main_postal_distribution['postal_routing'],
            )] = territory_id

    schema_title_by_name.clear()
    for schema in model.db.schemas.find(None, ['name', 'title']):
        schema_title_by_name[schema['name']] = schema['title']

    model.Poi.clear_indexes()
    model.Poi.load_pois()
    model.Poi.index_pois()

#    # Remove unused categories.
#    for category_slug in category_by_slug.keys():
#        if category_slug not in model.Poi.ids_by_category_slug:
#            log.warning('Ignoring category "{0}" not used by any POI.'.format(category_slug))
#            del category_by_slug[category_slug]
#    for category_slug in model.Poi.ids_by_category_slug'].keys():
#        if category_slug not in category_by_slug:
#            log.warning('Ignoring category "{0}" not defined in categories collection.'.format(category_slug))
#            del model.Poi.ids_by_category_slug[category_slug]

##    for category_slug in category_by_slug.iterkeys():
#        for word in category_slug.split(u'-'):
#            categories_slug_by_word.setdefault(word, set()).add(category_slug)

    log.info('RAM-based database loaded in {0} seconds'.format(datetime.datetime.utcnow() - start_time))

def ramdb_based(controller):
    """A decorator that allow to use ramdb data and update it regularily from MongoDB data."""
    def invoke(req):
        from . import model

        global last_timestamp
        reset_pois = False
        for data_update in model.db[conf['data_updates_collection']].find(dict(
                collection_name = {'$in': ['categories', 'pois', 'organism_types']},
                timestamp = {'$gt': last_timestamp},
                )).sort('timestamp'):
            if data_update['collection_name'] == 'categories':
                slug = data_update['document_id']
                category_bson = model.db[conf['categories_collection']].find_one(dict(code = slug),
                    ['code', 'tags_code', 'title'])
                read_write_lock.acquire()
                try:
                    # First find changes to do on indexes.
                    existing = {}
                    indexes = sys.modules[__name__]
                    find_existing(indexes, 'categories_slug_by_tag_slug', 'dict_of_sets', slug, existing)
                    find_existing(indexes, 'categories_slug_by_word', 'dict_of_sets', slug, existing)
                    find_existing(indexes, 'category_slug_by_pivot_code', 'dict_of_values', slug, existing)
                    # Then update indexes.
                    delete_remaining(indexes, existing)
                    if category_bson is None:
                        category_by_slug.pop(slug, None)
                        model.Poi.ids_by_category_slug.pop(slug, None)
                    else:
                        category = model.Category.load(category_bson)
                        category.index()
                finally:
                    read_write_lock.release()
            elif data_update['collection_name'] == 'organism_types':
                pivot_code = data_update['document_id']
                organism_type_bson = model.db[conf['organism_types_collection']].find_one(dict(code = pivot_code),
                    ['code', 'slug'])
                read_write_lock.acquire()
                try:
                    if organism_type_bson is None:
                        category_slug_by_pivot_code.pop(pivot_code, None)
                    elif organism_type_bson['slug'] not in category_by_slug:
                        log.warning('Ignoring organism type "{0}" without matching category.'.format(pivot_code))
                    else:
                        category_slug_by_pivot_code[pivot_code] = organism_type_bson['slug']
                finally:
                    read_write_lock.release()
            elif data_update['collection_name'] == 'pois':
                if conf['reset_on_poi_update']:
                    reset_pois = True
                else:
                    id = data_update['document_id']
                    poi_bson = model.Poi.get_collection().find_one(id)
                    read_write_lock.acquire()
                    try:
                        # Note: POI's whose parent_id == id are not updated here. They will be updated when publisher
                        # will publish their change.
                        # First find changes to do on indexes.
                        existing = {}
                        find_existing(model.Poi, 'ids_by_category_slug', 'dict_of_sets', id, existing)
                        find_existing(model.Poi, 'ids_by_competence_territory_id', 'dict_of_sets', id, existing)
                        find_existing(model.Poi, 'ids_by_parent_id', 'dict_of_sets', id, existing)
                        find_existing(model.Poi, 'ids_by_presence_territory_id', 'dict_of_sets', id, existing)
                        find_existing(model.Poi, 'ids_by_word', 'dict_of_sets', id, existing)
                        # Then update indexes.
                        delete_remaining(model.Poi, existing)
                        if poi_bson is None or poi_bson['metadata'].get('deleted', False):
                            model.Poi.instance_by_id.pop(id, None)
                            model.Poi.indexed_ids.discard(id)
                        else:
                            poi = model.Poi.load(poi_bson)
                            model.Poi.indexed_ids.add(poi._id)
                            poi.index(poi._id)
                            del poi.bson
                    finally:
                        read_write_lock.release()
            last_timestamp = data_update['timestamp']
        if reset_pois:
            read_write_lock.acquire()
            try:
                model.Poi.clear_indexes()
                model.Poi.load_pois()
                model.Poi.index_pois()
            finally:
                read_write_lock.release()

        # TODO: Handle schemas updates & schema_title_by_name.

        read_write_lock.acquire(shared = True)
        try:
            return controller(req)
        finally:
            read_write_lock.release()
    return invoke
<commit_msg>Limit RAM update to 5 by HTTP request.<commit_after># -*- coding: utf-8 -*-

# Etalage -- Open Data POIs portal
# By: Emmanuel Raviart <eraviart@easter-eggs.com>
#
# Copyright (C) 2011, 2012 Easter-eggs
# http://gitorious.org/infos-pratiques/etalage
#
# This file is part of Etalage.
#
# Etalage is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Etalage is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""RAM-based database"""

import datetime
import logging
import sys

from biryani import strings
import threading2

from . import conf, conv
from .ramindexes import *

categories_slug_by_tag_slug = {}
categories_slug_by_word = {}
category_by_slug = {}
category_slug_by_pivot_code = {}
last_timestamp = None
read_write_lock = threading2.SHLock()
log = logging.getLogger(__name__)
schema_title_by_name = {}
territories_id_by_ancestor_id = {}
territories_id_by_postal_distribution = {}
territory_by_id = {}
territory_id_by_kind_code = {}

def get_territory_related_territories_id(territory):
    related_territories_id = set()
    for sub_territory_id in territories_id_by_ancestor_id[territory._id]:
        sub_territory = territory_by_id[sub_territory_id]
        for ancestor_id in sub_territory.ancestors_id:
            related_territories_id.add(ancestor_id)
    return related_territories_id

def iter_categories_slug(organism_types_only = False, tags_slug = None, term = None):
    intersected_sets = []
    if organism_types_only:
        intersected_sets.append(set(category_slug_by_pivot_code.itervalues()))
    for tag_slug in set(tags_slug or []):
        if tag_slug is not None:
            intersected_sets.append(categories_slug_by_tag_slug.get(tag_slug))
    if term:
        prefixes = strings.slugify(term).split(u'-')
        categories_slug_by_prefix = {}
        for prefix in prefixes:
            if prefix in categories_slug_by_prefix:
                # TODO? Handle categories with several words sharing the same prefix?
                continue
            categories_slug_by_prefix[prefix] = union_set(
                word_categories_slug
                for word, word_categories_slug in categories_slug_by_word.iteritems()
                if word.startswith(prefix)
                ) or set()
        intersected_sets.extend(categories_slug_by_prefix.itervalues())

    categories_slug = intersection_set(intersected_sets)
    if categories_slug is None:
        return category_by_slug.iterkeys()
    return categories_slug

def load():
    """Load MongoDB data into RAM-based database."""
    from . import model

    start_time = datetime.datetime.utcnow()
    global last_timestamp
    # Remove a few seconds, for data changes that occur during startup.
    last_timestamp = start_time - datetime.timedelta(seconds = 30)

    categories_slug_by_tag_slug.clear()
    categories_slug_by_word.clear()
    category_by_slug.clear()
    category_slug_by_pivot_code.clear()
    for category_bson in model.db[conf['categories_collection']].find(None, ['code', 'tags_code', 'title']):
        if not strings.slugify(category_bson.get('title')):
            continue
        category = model.Category.load(category_bson)
        category.index()

    for organism_type_bson in model.db[conf['organism_types_collection']].find(None, ['code', 'slug']):
        if organism_type_bson['slug'] not in category_by_slug:
            log.warning('Ignoring organism type "{0}" without matching category.'.format(organism_type_bson['code']))
            continue
        category_slug_by_pivot_code[organism_type_bson['code']] = organism_type_bson['slug']

    territories_id_by_ancestor_id.clear()
    territories_id_by_postal_distribution.clear()
    territories_query = dict(
        kind = {'$in': conf['territories_kinds']},
        )
    territory_by_id.clear()
    territory_id_by_kind_code.clear()
    for territory_bson in model.db[conf['territories_collection']].find(territories_query, [
            'ancestors_id',
            'code',
            'geo',
            'hinge_type',
            'kind',
            'main_postal_distribution',
            'name',
            ]):
        main_postal_distribution = territory_bson.get('main_postal_distribution')
        if main_postal_distribution is None:
            continue
        territory_class = model.Territory.kind_to_class(territory_bson['kind'])
        assert territory_class is not None, 'Invalid territory type name: {0}'.format(class_name)
        territory_id = territory_bson['_id']
        territory = territory_class(
            _id = territory_id,
            ancestors_id = territory_bson['ancestors_id'],
            code = territory_bson['code'],
            geo = territory_bson.get('geo'),
            hinge_type = territory_bson.get('hinge_type'),
            main_postal_distribution = main_postal_distribution,
            name = territory_bson['name'],
            )
        territory_by_id[territory_id] = territory
        for ancestor_id in territory_bson['ancestors_id']:
            territories_id_by_ancestor_id.setdefault(ancestor_id, set()).add(territory_id)
        territory_id_by_kind_code[(territory_bson['kind'], territory_bson['code'])] = territory_id
        territories_id_by_postal_distribution[(
            main_postal_distribution['postal_code'],
            main_postal_distribution['postal_routing'],
            )] = territory_id

    schema_title_by_name.clear()
    for schema in model.db.schemas.find(None, ['name', 'title']):
        schema_title_by_name[schema['name']] = schema['title']

    model.Poi.clear_indexes()
    model.Poi.load_pois()
    model.Poi.index_pois()

#    # Remove unused categories.
#    for category_slug in category_by_slug.keys():
#        if category_slug not in model.Poi.ids_by_category_slug:
#            log.warning('Ignoring category "{0}" not used by any POI.'.format(category_slug))
#            del category_by_slug[category_slug]
#    for category_slug in model.Poi.ids_by_category_slug'].keys():
#        if category_slug not in category_by_slug:
#            log.warning('Ignoring category "{0}" not defined in categories collection.'.format(category_slug))
#            del model.Poi.ids_by_category_slug[category_slug]

##    for category_slug in category_by_slug.iterkeys():
#        for word in category_slug.split(u'-'):
#            categories_slug_by_word.setdefault(word, set()).add(category_slug)

    log.info('RAM-based database loaded in {0} seconds'.format(datetime.datetime.utcnow() - start_time))

def ramdb_based(controller):
    """A decorator that allow to use ramdb data and update it regularily from MongoDB data."""
    def invoke(req):
        from . import model

        global last_timestamp
        reset_pois = False
        for data_update in model.db[conf['data_updates_collection']].find(dict(
                collection_name = {'$in': ['categories', 'pois', 'organism_types']},
                timestamp = {'$gt': last_timestamp},
                )).sort('timestamp').limit(5):
            if data_update['collection_name'] == 'categories':
                slug = data_update['document_id']
                category_bson = model.db[conf['categories_collection']].find_one(dict(code = slug),
                    ['code', 'tags_code', 'title'])
                read_write_lock.acquire()
                try:
                    # First find changes to do on indexes.
                    existing = {}
                    indexes = sys.modules[__name__]
                    find_existing(indexes, 'categories_slug_by_tag_slug', 'dict_of_sets', slug, existing)
                    find_existing(indexes, 'categories_slug_by_word', 'dict_of_sets', slug, existing)
                    find_existing(indexes, 'category_slug_by_pivot_code', 'dict_of_values', slug, existing)
                    # Then update indexes.
                    delete_remaining(indexes, existing)
                    if category_bson is None:
                        category_by_slug.pop(slug, None)
                        model.Poi.ids_by_category_slug.pop(slug, None)
                    else:
                        category = model.Category.load(category_bson)
                        category.index()
                finally:
                    read_write_lock.release()
            elif data_update['collection_name'] == 'organism_types':
                pivot_code = data_update['document_id']
                organism_type_bson = model.db[conf['organism_types_collection']].find_one(dict(code = pivot_code),
                    ['code', 'slug'])
                read_write_lock.acquire()
                try:
                    if organism_type_bson is None:
                        category_slug_by_pivot_code.pop(pivot_code, None)
                    elif organism_type_bson['slug'] not in category_by_slug:
                        log.warning('Ignoring organism type "{0}" without matching category.'.format(pivot_code))
                    else:
                        category_slug_by_pivot_code[pivot_code] = organism_type_bson['slug']
                finally:
                    read_write_lock.release()
            elif data_update['collection_name'] == 'pois':
                if conf['reset_on_poi_update']:
                    reset_pois = True
                else:
                    id = data_update['document_id']
                    poi_bson = model.Poi.get_collection().find_one(id)
                    read_write_lock.acquire()
                    try:
                        # Note: POI's whose parent_id == id are not updated here. They will be updated when publisher
                        # will publish their change.
                        # First find changes to do on indexes.
                        existing = {}
                        find_existing(model.Poi, 'ids_by_category_slug', 'dict_of_sets', id, existing)
                        find_existing(model.Poi, 'ids_by_competence_territory_id', 'dict_of_sets', id, existing)
                        find_existing(model.Poi, 'ids_by_parent_id', 'dict_of_sets', id, existing)
                        find_existing(model.Poi, 'ids_by_presence_territory_id', 'dict_of_sets', id, existing)
                        find_existing(model.Poi, 'ids_by_word', 'dict_of_sets', id, existing)
                        # Then update indexes.
                        delete_remaining(model.Poi, existing)
                        if poi_bson is None or poi_bson['metadata'].get('deleted', False):
                            model.Poi.instance_by_id.pop(id, None)
                            model.Poi.indexed_ids.discard(id)
                        else:
                            poi = model.Poi.load(poi_bson)
                            model.Poi.indexed_ids.add(poi._id)
                            poi.index(poi._id)
                            del poi.bson
                    finally:
                        read_write_lock.release()
            last_timestamp = data_update['timestamp']
        if reset_pois:
            read_write_lock.acquire()
            try:
                model.Poi.clear_indexes()
                model.Poi.load_pois()
                model.Poi.index_pois()
            finally:
                read_write_lock.release()

        # TODO: Handle schemas updates & schema_title_by_name.

        read_write_lock.acquire(shared = True)
        try:
            return controller(req)
        finally:
            read_write_lock.release()
    return invoke
<|endoftext|>


<commit_before>"""cx_Freeze extension

Extends:

- the 'build' command with the 'exe-command' option to allow using a
different command from 'build_exe' to build executables from Python scripts.

- the 'install' command with the 'skip-sub-commands' option to allow not
running a set of sub commands, e.g.:

    install --skip-sub-commands=install_lib,install_scripts,install_data

- the 'bdist_msi' command to handle LaunchAfterInstall and a clean uninstall.
"""

import distutils.command.build
import sys
import os
from cx_Freeze.dist import build as cx_build
from cx_Freeze.dist import install as cx_install
from cx_Freeze.dist import setup as cx_setup
from cx_Freeze.dist import _AddCommandClass

class build(cx_build):

    cx_build.user_options.append(
        ('exe-command=', None, "Python script executables command"))

    def initialize_options(self):
        cx_build.initialize_options(self)
        self.exe_command = 'build_exe'

    def get_sub_commands(self):
        subCommands = distutils.command.build.build.get_sub_commands(self)
        if self.distribution.executables:
            subCommands.append(self.exe_command)
        return subCommands

class install(cx_install):

    cx_install.user_options.append(
        ('skip-sub-commands=', None,
         "sub commands to ignore when running command"))

    def initialize_options(self):
        cx_install.initialize_options(self)
        self.skip_sub_commands = None

    def get_sub_commands(self):
        subCommands = cx_install.get_sub_commands(self)
        if self.skip_sub_commands:
            skip_sub_commands = self.skip_sub_commands.split(',')
            for cmd in skip_sub_commands:
                if cmd in subCommands:
                    subCommands.remove(cmd)
        return subCommands

if sys.platform == 'win32':
    from cx_Freeze.windist import bdist_msi as cx_bdist_msi

    class bdist_msi(cx_bdist_msi):
        attribs = None

        def finalize_options(self):
            self.distribution.get_name()
            if self.initial_target_dir is None:
                if distutils.util.get_platform() == "win-amd64":
                    programFilesFolder = "ProgramFiles64Folder"
                else:
                    programFilesFolder = "ProgramFilesFolder"
                self.initial_target_dir = r"[%s]\%s" % (programFilesFolder,
                                            self.attribs.get_install_dir())
            # Using old style class so can't use super
            import cx_Freeze
            cx_Freeze.windist.bdist_msi.finalize_options(self)

        def get_executable(self):
            return self.attribs.get_win_targetName()

        def get_license(self):
            return self.attribs.get_licence()

        def add_licence_dialog(self):
            import msilib
            msilib.add_data(self.db, 'InstallUISequence',
                [("LicenceDialog", None, 380)])
            dialog = distutils.command.bdist_msi.PyDialog(self.db,
                                                          "LicenceDialog",
                    self.x, self.y, self.width, self.height, self.modal,
                    self.title, "Next", "Next", "Cancel")
            dialog.text("LicenseTitle", 15, 10, 320, 20, 0x3, "License")
            dialog.control("License", "ScrollableText",
                                  15, 30, 340, 200, 0x7, None,
                                    self.get_license(), None, None)
            dialog.control("LicenseAccepted", "CheckBox",
                               15, 240, 320, 20, 0x3,
                               "LicenseAccepted",
                               "I've accepted this agreement", None, None)
            button = dialog.cancel("Cancel", "Next")
            button.event("EndDialog", "Exit")
            button = dialog.next("Next", "Cancel", active=False)
            button.condition("Enable", "LicenseAccepted")
            button.condition("Disable", "not LicenseAccepted")
            button.event("EndDialog", "Return")

        def add_exit_dialog(self):
            import msilib
            if self.get_license() is not None:
                self.add_licence_dialog()
            dialog = distutils.command.bdist_msi.PyDialog(self.db,
                                                          "ExitDialog",
                    self.x, self.y, self.width, self.height, self.modal,
                    self.title, "Finish", "Finish", "Finish")
            dialog.title("Completing the [ProductName]")
            dialog.back("< Back", "Finish", active=False)
            dialog.cancel("Cancel", "Back", active=False)
            dialog.text("Description", 15, 235, 320, 20, 0x30003,
                    "Click the Finish button to exit the installer.")
            button = dialog.next("Finish", "Cancel", name="Finish")
            button.event("EndDialog", "Return")
            msilib.add_data(self.db, "Property",
                     [("StartClient", "1")])
            # Launch product checkbox
            c = dialog.control("LaunchAfterInstall", "CheckBox",
                               15, 200, 320, 20, 0x3,
                               "StartClient", "Launch [ProductName]",
                               None, None)
            c.condition("Hide", 'Progress1<>"Install"')
            # 18 is for execute a .exe from install
            msilib.add_data(self.db, "CustomAction", [("LaunchNuxeoDrive", 82,
                                                       "launcher.exe",
                                                       self.get_executable())])
            button.event("DoAction", "LaunchNuxeoDrive",
                         'StartClient=1 and Progress1="Install"')
            msilib.add_data(self.db, "CustomAction", [("NuxeoDriveCleanUp", 82,
                                                       self.get_executable(),
                                                       "uninstall")])
            # Deffered action with noImpersonate to have the correct privileges
            msilib.add_data(self.db, "CustomAction", [("NuxeoDriveFolderCleanUp", 3234,
                                                       "TARGETDIR",
                                                       "cmd.exe /C \"cd .. && rmdir /S /Q appdata\"")])
            msilib.add_data(self.db, "InstallExecuteSequence",
                            [("NuxeoDriveCleanUp",
                              'REMOVE="ALL" AND NOT UPGRADINGPRODUCTCODE',
                              1260)])
            # After InstallInitialize
            msilib.add_data(self.db, "InstallExecuteSequence",
                            [("NuxeoDriveFolderCleanUp",
                              'REMOVE="ALL" AND NOT UPGRADINGPRODUCTCODE',
                              1560)])
            # Add product icon
            icon_file = os.path.join(self.attribs.get_icons_home(), self.attribs.get_win_icon())
            if os.path.exists(icon_file):
                msilib.add_data(self.db, "Property", [("ARPPRODUCTICON", "InstallIcon")])
                msilib.add_data(self.db, "Icon", [("InstallIcon", msilib.Binary(icon_file))])
            # Allow to customize the MSI
            if getattr(self.attribs, 'customize_msi', None) is not None:
                self.attribs.customize_msi(self.db)

# Override cx_Freeze setup to override build and install commands.
def setup(**attrs):
    commandClasses = attrs.setdefault("cmdclass", {})
    _AddCommandClass(commandClasses, "build", build)
    _AddCommandClass(commandClasses, "install", install)
    if sys.platform == 'win32':
        bdist_msi.attribs = attrs.get("attribs")
        _AddCommandClass(commandClasses, "bdist_msi", bdist_msi)
    cx_setup(**attrs)
<commit_msg>NXDRIVE-455: Uninstall doesnt clean everything<commit_after>"""cx_Freeze extension

Extends:

- the 'build' command with the 'exe-command' option to allow using a
different command from 'build_exe' to build executables from Python scripts.

- the 'install' command with the 'skip-sub-commands' option to allow not
running a set of sub commands, e.g.:

    install --skip-sub-commands=install_lib,install_scripts,install_data

- the 'bdist_msi' command to handle LaunchAfterInstall and a clean uninstall.
"""

import distutils.command.build
import sys
import os
from cx_Freeze.dist import build as cx_build
from cx_Freeze.dist import install as cx_install
from cx_Freeze.dist import setup as cx_setup
from cx_Freeze.dist import _AddCommandClass

class build(cx_build):

    cx_build.user_options.append(
        ('exe-command=', None, "Python script executables command"))

    def initialize_options(self):
        cx_build.initialize_options(self)
        self.exe_command = 'build_exe'

    def get_sub_commands(self):
        subCommands = distutils.command.build.build.get_sub_commands(self)
        if self.distribution.executables:
            subCommands.append(self.exe_command)
        return subCommands

class install(cx_install):

    cx_install.user_options.append(
        ('skip-sub-commands=', None,
         "sub commands to ignore when running command"))

    def initialize_options(self):
        cx_install.initialize_options(self)
        self.skip_sub_commands = None

    def get_sub_commands(self):
        subCommands = cx_install.get_sub_commands(self)
        if self.skip_sub_commands:
            skip_sub_commands = self.skip_sub_commands.split(',')
            for cmd in skip_sub_commands:
                if cmd in subCommands:
                    subCommands.remove(cmd)
        return subCommands

if sys.platform == 'win32':
    from cx_Freeze.windist import bdist_msi as cx_bdist_msi

    class bdist_msi(cx_bdist_msi):
        attribs = None

        def finalize_options(self):
            self.distribution.get_name()
            if self.initial_target_dir is None:
                if distutils.util.get_platform() == "win-amd64":
                    programFilesFolder = "ProgramFiles64Folder"
                else:
                    programFilesFolder = "ProgramFilesFolder"
                self.initial_target_dir = r"[%s]\%s" % (programFilesFolder,
                                            self.attribs.get_install_dir())
            # Using old style class so can't use super
            import cx_Freeze
            cx_Freeze.windist.bdist_msi.finalize_options(self)

        def get_executable(self):
            return self.attribs.get_win_targetName()

        def get_license(self):
            return self.attribs.get_licence()

        def add_licence_dialog(self):
            import msilib
            msilib.add_data(self.db, 'InstallUISequence',
                [("LicenceDialog", None, 380)])
            dialog = distutils.command.bdist_msi.PyDialog(self.db,
                                                          "LicenceDialog",
                    self.x, self.y, self.width, self.height, self.modal,
                    self.title, "Next", "Next", "Cancel")
            dialog.text("LicenseTitle", 15, 10, 320, 20, 0x3, "License")
            dialog.control("License", "ScrollableText",
                                  15, 30, 340, 200, 0x7, None,
                                    self.get_license(), None, None)
            dialog.control("LicenseAccepted", "CheckBox",
                               15, 240, 320, 20, 0x3,
                               "LicenseAccepted",
                               "I've accepted this agreement", None, None)
            button = dialog.cancel("Cancel", "Next")
            button.event("EndDialog", "Exit")
            button = dialog.next("Next", "Cancel", active=False)
            button.condition("Enable", "LicenseAccepted")
            button.condition("Disable", "not LicenseAccepted")
            button.event("EndDialog", "Return")

        def add_exit_dialog(self):
            import msilib
            if self.get_license() is not None:
                self.add_licence_dialog()
            dialog = distutils.command.bdist_msi.PyDialog(self.db,
                                                          "ExitDialog",
                    self.x, self.y, self.width, self.height, self.modal,
                    self.title, "Finish", "Finish", "Finish")
            dialog.title("Completing the [ProductName]")
            dialog.back("< Back", "Finish", active=False)
            dialog.cancel("Cancel", "Back", active=False)
            dialog.text("Description", 15, 235, 320, 20, 0x30003,
                    "Click the Finish button to exit the installer.")
            button = dialog.next("Finish", "Cancel", name="Finish")
            button.event("EndDialog", "Return")
            msilib.add_data(self.db, "Property",
                     [("StartClient", "1")])
            # Launch product checkbox
            c = dialog.control("LaunchAfterInstall", "CheckBox",
                               15, 200, 320, 20, 0x3,
                               "StartClient", "Launch [ProductName]",
                               None, None)
            c.condition("Hide", 'Progress1<>"Install"')
            # 18 is for execute a .exe from install
            msilib.add_data(self.db, "CustomAction", [("LaunchNuxeoDrive", 82,
                                                       "launcher.exe",
                                                       self.get_executable())])
            button.event("DoAction", "LaunchNuxeoDrive",
                         'StartClient=1 and Progress1="Install"')
            msilib.add_data(self.db, "CustomAction", [("NuxeoDriveCleanUp", 82,
                                                       self.get_executable(),
                                                       "uninstall")])
            # Deffered action with noImpersonate to have the correct privileges
            msilib.add_data(self.db, "CustomAction", [("NuxeoDriveFolderCleanUp", 3234,
                                                       "TARGETDIR",
                                                       "cmd.exe /C \"rmdir /S /Q appdata\"")])
            msilib.add_data(self.db, "InstallExecuteSequence",
                            [("NuxeoDriveCleanUp",
                              'REMOVE="ALL" AND NOT UPGRADINGPRODUCTCODE',
                              1260)])
            # After InstallInitialize
            msilib.add_data(self.db, "InstallExecuteSequence",
                            [("NuxeoDriveFolderCleanUp",
                              'REMOVE="ALL" AND NOT UPGRADINGPRODUCTCODE',
                              1560)])
            # Add product icon
            icon_file = os.path.join(self.attribs.get_icons_home(), self.attribs.get_win_icon())
            if os.path.exists(icon_file):
                msilib.add_data(self.db, "Property", [("ARPPRODUCTICON", "InstallIcon")])
                msilib.add_data(self.db, "Icon", [("InstallIcon", msilib.Binary(icon_file))])
            # Allow to customize the MSI
            if getattr(self.attribs, 'customize_msi', None) is not None:
                self.attribs.customize_msi(self.db)

# Override cx_Freeze setup to override build and install commands.
def setup(**attrs):
    commandClasses = attrs.setdefault("cmdclass", {})
    _AddCommandClass(commandClasses, "build", build)
    _AddCommandClass(commandClasses, "install", install)
    if sys.platform == 'win32':
        bdist_msi.attribs = attrs.get("attribs")
        _AddCommandClass(commandClasses, "bdist_msi", bdist_msi)
    cx_setup(**attrs)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Going by the architecture used in Srivastava's paper, detailed here:
# https://github.com/lisa-lab/pylearn2/blob/master/pylearn2/scripts/papers/dropout/mnist_valid.yaml
#
# Unsure how to deal with input dropout; we're assuming (in the case of Wang)
# that input dropout is propagated through to be noise on the pre-nonlinearity
# activations. But then, the final noise is going to end up directly damaging
# the predictions, which makes no sense. Probably better just to ignore reason
# in this case and just write the architecture the way it's probably supposed
# to be.

import layers
import lasagne.layers
import lasagne.nonlinearities
import urllib2
import imp

def wangDropoutArchitecture(batch_size=128, input_dim=784, output_dim=10,
                            DropoutLayer=layers.WangGaussianDropout,
                            n_hidden=100):
    l_in = lasagne.layers.InputLayer((batch_size, input_dim))
    l_drop_in = DropoutLayer(l_in, p=0.2)
    l_hidden_1 = lasagne.layers.DenseLayer(l_drop_in, num_units=n_hidden,
            nonlinearity=lambda x: x)
    l_drop_1 = DropoutLayer(l_hidden_1, p=0.5,
            nonlinearity=lasagne.nonlinearities.rectify)
    l_hidden_2 = lasagne.layers.DenseLayer(l_drop_1, num_units=n_hidden,
            nonlinearity=lambda x: x)
    l_drop_2 = DropoutLayer(l_hidden_2, p=0.5,
            nonlinearity=lasagne.nonlinearities.rectify)
    l_out = lasagne.layers.DenseLayer(l_drop_2, num_units=output_dim,
            nonlinearity=lasagne.nonlinearities.softmax)
    return l_out

def srivastavaDropoutArchitecture(batch_size=128, input_dim=784, output_dim=10,
                            DropoutLayer=layers.WangGaussianDropout,
                            n_hidden=100):
    l_in = lasagne.layers.InputLayer((batch_size, input_dim))
    l_drop_in = DropoutLayer(l_in, p=0.2)
    l_hidden_1 = lasagne.layers.DenseLayer(l_drop_in, num_units=n_hidden,
            nonlinearity=lasagne.nonlinearities.rectify)
    l_drop_1 = DropoutLayer(l_hidden_1, p=0.5)
    l_hidden_2 = lasagne.layers.DenseLayer(l_drop_1, num_units=n_hidden,
            nonlinearity=lasagne.nonlinearities.rectify)
    l_drop_2 = DropoutLayer(l_hidden_2, p=0.5)
    l_out = lasagne.layers.DenseLayer(l_drop_2, num_units=output_dim,
            nonlinearity=lasagne.nonlinearities.softmax)
    return l_out

def earlystopping(loop, delta=0.01, max_N=1000, verbose=False):
    """
    Stops the expriment once the loss stops improving by delta per epoch.
    With a max_N of epochs to avoid infinite experiments.
    """

    return loop

def load_data():
    """
    Standardising data loading; all using MNIST in the usual way:
        * train: 50000
        * valid: 10000
        * test: separate 10000
    """
    # is this the laziest way to load mnist?
    mnist = imp.new_module()
    exec urllib2.urlopen("https://raw.githubusercontent.com/Lasagne/Lasagne"
            "/master/examples/mnist.py").read() in mnist.__dict__
    dataset = mnist.load_dataset()
    return dict(X_train=dataset[0].reshape(-1, 784),
                y_train=dataset[1],
                X_valid=dataset[2].reshape(-1, 784),
                y_valid=dataset[3],
                X_test=dataset[4].reshape(-1, 784),
                y_test=dataset[5])
<commit_msg>mostly done<commit_after>#!/usr/bin/env python

# Going by the architecture used in Srivastava's paper, detailed here:
# https://github.com/lisa-lab/pylearn2/blob/master/pylearn2/scripts/papers/dropout/mnist_valid.yaml
#
# Unsure how to deal with input dropout; we're assuming (in the case of Wang)
# that input dropout is propagated through to be noise on the pre-nonlinearity
# activations. But then, the final noise is going to end up directly damaging
# the predictions, which makes no sense. Probably better just to ignore reason
# in this case and just write the architecture the way it's probably supposed
# to be.

import layers
import holonets
import lasagne.layers
import lasagne.nonlinearities
import lasagne.updates
import urllib2
import imp
import itertools
from collections import OrderedDict

def wangDropoutArchitecture(batch_size=1000, input_dim=784, output_dim=10,
                            DropoutLayer=layers.WangGaussianDropout,
                            n_hidden=100):
    l_in = lasagne.layers.InputLayer((batch_size, input_dim))
    l_drop_in = DropoutLayer(l_in, p=0.2)
    l_hidden_1 = lasagne.layers.DenseLayer(l_drop_in, num_units=n_hidden,
            nonlinearity=lambda x: x)
    l_drop_1 = DropoutLayer(l_hidden_1, p=0.5,
            nonlinearity=lasagne.nonlinearities.rectify)
    l_hidden_2 = lasagne.layers.DenseLayer(l_drop_1, num_units=n_hidden,
            nonlinearity=lambda x: x)
    l_drop_2 = DropoutLayer(l_hidden_2, p=0.5,
            nonlinearity=lasagne.nonlinearities.rectify)
    l_out = lasagne.layers.DenseLayer(l_drop_2, num_units=output_dim,
            nonlinearity=lasagne.nonlinearities.softmax)
    return l_out

def srivastavaDropoutArchitecture(batch_size=1000, input_dim=784, output_dim=10,
                            DropoutLayer=layers.WangGaussianDropout,
                            n_hidden=100):
    l_in = lasagne.layers.InputLayer((batch_size, input_dim))
    l_drop_in = DropoutLayer(l_in, p=0.2)
    l_hidden_1 = lasagne.layers.DenseLayer(l_drop_in, num_units=n_hidden,
            nonlinearity=lasagne.nonlinearities.rectify)
    l_drop_1 = DropoutLayer(l_hidden_1, p=0.5)
    l_hidden_2 = lasagne.layers.DenseLayer(l_drop_1, num_units=n_hidden,
            nonlinearity=lasagne.nonlinearities.rectify)
    l_drop_2 = DropoutLayer(l_hidden_2, p=0.5)
    l_out = lasagne.layers.DenseLayer(l_drop_2, num_units=output_dim,
            nonlinearity=lasagne.nonlinearities.softmax)
    return l_out

def make_experiment(l_out, dataset,
        loss_function=lasagne.objectives.categorical_crossentropy,
        extra_loss=0.0, limit_alpha=False):
    """
    Build a loop for training a model, evaluating loss on training, validation
    and test.
    """
    expressions = holonets.monitor.Expressions(l_out, dataset, batch_size,
            N_train=50000, N_valid=10000, N_test=10000,
            update_rule=lasagne.updates.adam, loss_function=loss_function,
            loss_aggregate=T.mean, extra_loss=extra_loss, learning_rate=0.001)
    # only add channels for loss and accuracy
    for deterministic,dataset in itertools.product([True, False, False],
                                                   ["train", "valid", False]):
        expressions.add_channel(**expressions.loss(dataset, deterministic))
        expressions.add_channel(**expressions.accuracy(dataset, deterministic))
    channels = expressions.build_channels()
    if limit_alpha:
        # then add channel to reset all alphas at 1.0
        alphas = [p for p in lasagne.layers.get_all_params(l_out)
                if p.name == "alpha"]
        alpha_ceiling = theano.function([], alphas,
                updates=OrderedDict([(a, T.min([a, 1.0])) for a in alphas]))
        channels.append({'dataset': 'train',
                         'eval': lambda x: alpha_ceiling(),
                         'dimensions': ['Alpha']*len(alphas),
                         'names': ['alpha {0}'.format(i) for i in range(len(alphas))]})
    train = holonets.train.Train(channels,
            n_batches={'train': N_train//batch_size,
                       'valid':N_valid//batch_size,
                       'test':N_test//batch_size})
    loop = holonets.run.EpochLoop(train, dimensions=train.dimensions)
    return loop

def earlystopping(loop, delta=0.01, max_N=1000, verbose=False):
    """
    Stops the expriment once the loss stops improving by delta per epoch.
    With a max_N of epochs to avoid infinite experiments.
    """

    return loop

def load_data():
    """
    Standardising data loading; all using MNIST in the usual way:
        * train: 50000
        * valid: 10000
        * test: separate 10000
    """
    # is this the laziest way to load mnist?
    mnist = imp.new_module()
    exec urllib2.urlopen("https://raw.githubusercontent.com/Lasagne/Lasagne"
            "/master/examples/mnist.py").read() in mnist.__dict__
    dataset = mnist.load_dataset()
    return dict(X_train=dataset[0].reshape(-1, 784),
                y_train=dataset[1],
                X_valid=dataset[2].reshape(-1, 784),
                y_valid=dataset[3],
                X_test=dataset[4].reshape(-1, 784),
                y_test=dataset[5])

<|endoftext|>


<commit_before>"""Playing with versions and version file"""

from collections import abc
import functools
import pathlib
import shutil
import tempfile

import semver

from versionner.errors import VersionnerError

class InvalidVersionError(VersionnerError):
    """Bad version string/value error"""

@functools.total_ordering
class Version:
    """Parse and manipulate version string"""

    __slots__ = ('major', 'minor', 'patch', 'prerelease', 'build')

    VALID_FIELDS = ('major', 'minor', 'patch', 'prerelease', 'build')
    VALID_UP_FIELDS = ('major', 'minor', 'patch')

    def __init__(self, version=None):
        """Initialise object

        :param version:version data to initialize
        """

        self.major = 0
        self.minor = 0
        self.patch = 0
        self.prerelease = ''
        self.build = ''

        if version is not None:
            self._parse(version)

    def _parse_object(self, version):
        """Parse version as Version object and initialize self with data from version

        :param version:
        :return:
        """
        self.major = version.major
        self.minor = version.minor
        self.patch = version.patch
        self.prerelease = getattr(version, 'prerelease', '')
        self.build = getattr(version, 'build', '')

    def _parse_dict(self, version):
        """Parse version as dict returned from semver.parse and initialize self with data from version

        :param version:
        :return:
        """
        self.major = version['major']
        self.minor = version['minor']
        self.patch = version['patch']
        self.prerelease = version.get('prerelease', '')
        self.build = version.get('build', '')

    def _parse_str(self, version):
        """Parse version as string and initialize self with data from version

        :param version:
        :return:
        """
        version = semver.parse(version)
        self._parse_dict(version)

    def _parse(self, version):
        """Recognize version type and dispatch it to self._parse_*

        :param version:
        :return:
        """
        if all(hasattr(version, field) for field in self.VALID_UP_FIELDS):
            self._parse_object(version)
        elif isinstance(version, abc.Mapping):
            self._parse_dict(version)
        elif isinstance(version, (str, bytes)):
            if hasattr(version, 'decode'):
                version = version.decode()
            self._parse_str(version)
        else:
            raise InvalidVersionError("Unknown object type: %s" % type(version))

    # pylint: disable=invalid-name
    def up(self, field, value=None):
        """Increase version and return new instance

        :rtype : Version
        :param field:one of Version.VALID_UP_FIELDS
        :param value:int
        :return: :raise ValueError:
        """

        if field not in self.VALID_UP_FIELDS:
            raise ValueError("Invalid field type: %s" % field)

        if not value:
            value = 1

        version = Version(self)

        if field == 'major':
            version.major += value
            version.minor = 0
            version.patch = 0
        elif field == 'minor':
            version.minor += value
            version.patch = 0
        else:
            version.patch += value

        return version

    def set(self, field, value):
        """Set any field of semver to `value`

        :rtype : Version
        :param field:type of field (one of Version.VALID_FIELDS)
        :param value:
        :return: :raise ValueError:
        """

        if field not in self.VALID_FIELDS:
            raise ValueError("Invalid field type: %s" % field)

        version = Version(self)
        setattr(version, field, value)

        # validation
        try:
            semver.parse(str(version))
        except ValueError as exc:
            raise InvalidVersionError("Invalid value for field %s: %s" % (field, value)) from exc

        return version

    def __str__(self):
        """Return version as string compatible with semver

        :return:str
        """

        return semver.format_version(self.major, self.minor, self.patch, self.prerelease or None, self.build or None)

    def __repr__(self):
        data = {field: getattr(self, field) for field in self.VALID_FIELDS}
        tpl = '<Version({major: %(major)s, minor: %(minor)s, patch: %(patch)s, ' \
              'prerelease: %(prerelease)s, build: %(build)s})>'

        return tpl % data

    def _cmp(self, other):
        """Compare versions

        :rtype : any
        :param other: version as any recognizable type
        :return: int
        """
        if isinstance(other, self.__class__):
            v1, v2 = str(self), str(other)
        elif isinstance(other, dict):
            v1, v2 = str(self), str(self.__class__(other))
        elif isinstance(other, str):
            v1, v2 = str(self), other
        else:
            return NotImplemented

        return semver.compare(v1, v2)

    def __eq__(self, other):
        result = self._cmp(other)
        if result is NotImplemented:
            return result

        return result == 0

    def __lt__(self, other):
        result = self._cmp(other)
        if result is NotImplemented:
            return result

        return result == -1

class VersionFile():
    """Manipulate project version file"""

    __slots__ = ('_path', )

    def __init__(self, path):
        """Initialisation

        :param path:pathlib.Path
        """
        self._path = path

    def read(self):
        """Read version from version file

        :rtype : Version
        :return:
        """
        with self._path.open(mode='r') as fh:
            version = fh.read().strip()
        return Version(version)

    def write(self, version):
        """Save new version into self._path in safe way (using temporary file)

        :param version:Version
        """
        with tempfile.NamedTemporaryFile(mode='w', delete=False) as fh:
            fh.write(str(version))

        if self._path.exists():
            shutil.copystat(str(self._path), fh.name)

        try:
            shutil.move(fh.name, self._path)
        except OSError as exc:
            # handling situation with tmp file on another device
            if exc.errno == 18 and 'Invalid cross-device link' in exc.strerror:
                with self._path.open(mode='w') as fh:
                    fh.write(str(version))
            else:
                raise

    def __str__(self):
        return str(self._path)
<commit_msg>Remove unnecessary try catch<commit_after>"""Playing with versions and version file"""

from collections import abc
import functools
import pathlib
import shutil
import tempfile

import semver

from versionner.errors import VersionnerError

class InvalidVersionError(VersionnerError):
    """Bad version string/value error"""

@functools.total_ordering
class Version:
    """Parse and manipulate version string"""

    __slots__ = ('major', 'minor', 'patch', 'prerelease', 'build')

    VALID_FIELDS = ('major', 'minor', 'patch', 'prerelease', 'build')
    VALID_UP_FIELDS = ('major', 'minor', 'patch')

    def __init__(self, version=None):
        """Initialise object

        :param version:version data to initialize
        """

        self.major = 0
        self.minor = 0
        self.patch = 0
        self.prerelease = ''
        self.build = ''

        if version is not None:
            self._parse(version)

    def _parse_object(self, version):
        """Parse version as Version object and initialize self with data from version

        :param version:
        :return:
        """
        self.major = version.major
        self.minor = version.minor
        self.patch = version.patch
        self.prerelease = getattr(version, 'prerelease', '')
        self.build = getattr(version, 'build', '')

    def _parse_dict(self, version):
        """Parse version as dict returned from semver.parse and initialize self with data from version

        :param version:
        :return:
        """
        self.major = version['major']
        self.minor = version['minor']
        self.patch = version['patch']
        self.prerelease = version.get('prerelease', '')
        self.build = version.get('build', '')

    def _parse_str(self, version):
        """Parse version as string and initialize self with data from version

        :param version:
        :return:
        """
        version = semver.parse(version)
        self._parse_dict(version)

    def _parse(self, version):
        """Recognize version type and dispatch it to self._parse_*

        :param version:
        :return:
        """
        if all(hasattr(version, field) for field in self.VALID_UP_FIELDS):
            self._parse_object(version)
        elif isinstance(version, abc.Mapping):
            self._parse_dict(version)
        elif isinstance(version, (str, bytes)):
            if hasattr(version, 'decode'):
                version = version.decode()
            self._parse_str(version)
        else:
            raise InvalidVersionError("Unknown object type: %s" % type(version))

    # pylint: disable=invalid-name
    def up(self, field, value=None):
        """Increase version and return new instance

        :rtype : Version
        :param field:one of Version.VALID_UP_FIELDS
        :param value:int
        :return: :raise ValueError:
        """

        if field not in self.VALID_UP_FIELDS:
            raise ValueError("Invalid field type: %s" % field)

        if not value:
            value = 1

        version = Version(self)

        if field == 'major':
            version.major += value
            version.minor = 0
            version.patch = 0
        elif field == 'minor':
            version.minor += value
            version.patch = 0
        else:
            version.patch += value

        return version

    def set(self, field, value):
        """Set any field of semver to `value`

        :rtype : Version
        :param field:type of field (one of Version.VALID_FIELDS)
        :param value:
        :return: :raise ValueError:
        """

        if field not in self.VALID_FIELDS:
            raise ValueError("Invalid field type: %s" % field)

        version = Version(self)
        setattr(version, field, value)

        # validation
        try:
            semver.parse(str(version))
        except ValueError as exc:
            raise InvalidVersionError("Invalid value for field %s: %s" % (field, value)) from exc

        return version

    def __str__(self):
        """Return version as string compatible with semver

        :return:str
        """

        return semver.format_version(self.major, self.minor, self.patch, self.prerelease or None, self.build or None)

    def __repr__(self):
        data = {field: getattr(self, field) for field in self.VALID_FIELDS}
        tpl = '<Version({major: %(major)s, minor: %(minor)s, patch: %(patch)s, ' \
              'prerelease: %(prerelease)s, build: %(build)s})>'

        return tpl % data

    def _cmp(self, other):
        """Compare versions

        :rtype : any
        :param other: version as any recognizable type
        :return: int
        """
        if isinstance(other, self.__class__):
            v1, v2 = str(self), str(other)
        elif isinstance(other, dict):
            v1, v2 = str(self), str(self.__class__(other))
        elif isinstance(other, str):
            v1, v2 = str(self), other
        else:
            return NotImplemented

        return semver.compare(v1, v2)

    def __eq__(self, other):
        result = self._cmp(other)
        if result is NotImplemented:
            return result

        return result == 0

    def __lt__(self, other):
        result = self._cmp(other)
        if result is NotImplemented:
            return result

        return result == -1

class VersionFile():
    """Manipulate project version file"""

    __slots__ = ('_path', )

    def __init__(self, path):
        """Initialisation

        :param path:pathlib.Path
        """
        self._path = path

    def read(self):
        """Read version from version file

        :rtype : Version
        :return:
        """
        with self._path.open(mode='r') as fh:
            version = fh.read().strip()
        return Version(version)

    def write(self, version):
        """Save new version into self._path in safe way (using temporary file)

        :param version:Version
        """
        with tempfile.NamedTemporaryFile(mode='w', delete=False) as fh:
            fh.write(str(version))

        if self._path.exists():
            shutil.copystat(str(self._path), fh.name)

        shutil.move(fh.name, self._path)

    def __str__(self):
        return str(self._path)
<|endoftext|>


<commit_before>"""
Base classes for Custom Authenticator to use OAuth with JupyterHub

Most of the code c/o Kyle Kelley (@rgbkrk)
"""

import base64
import json
import os
from urllib.parse import quote, urlparse
import uuid

from tornado import web
from tornado.auth import OAuth2Mixin
from tornado.log import app_log

from jupyterhub.handlers import BaseHandler
from jupyterhub.auth import Authenticator
from jupyterhub.utils import url_path_join

from traitlets import Unicode, Bool, List, Dict, default, observe

def guess_callback_uri(protocol, host, hub_server_url):
    return '{proto}://{host}{path}'.format(
        proto=protocol, host=host, path=url_path_join(hub_server_url, 'oauth_callback')
    )

STATE_COOKIE_NAME = 'oauthenticator-state'

def _serialize_state(state):
    """Serialize OAuth state to a base64 string after passing through JSON"""
    json_state = json.dumps(state)
    return base64.urlsafe_b64encode(json_state.encode('utf8')).decode('ascii')

def _deserialize_state(b64_state):
    """Deserialize OAuth state as serialized in _serialize_state"""
    if isinstance(b64_state, str):
        b64_state = b64_state.encode('ascii')
    try:
        json_state = base64.urlsafe_b64decode(b64_state).decode('utf8')
    except ValueError:
        app_log.error("Failed to b64-decode state: %r", b64_state)
        return {}
    try:
        return json.loads(json_state)
    except ValueError:
        app_log.error("Failed to json-decode state: %r", json_state)
        return {}

class OAuthLoginHandler(OAuth2Mixin, BaseHandler):
    """Base class for OAuth login handler

    Typically subclasses will need
    """

    # these URLs are part of the OAuth2Mixin API
    # get them from the Authenticator object
    @property
    def _OAUTH_AUTHORIZE_URL(self):
        return self.authenticator.authorize_url

    @property
    def _OAUTH_ACCESS_TOKEN_URL(self):
        return self.authenticator.token_url

    @property
    def _OAUTH_USERINFO_URL(self):
        return self.authenticator.userdata_url

    def set_state_cookie(self, state):
        self._set_cookie(STATE_COOKIE_NAME, state, expires_days=1, httponly=True)

    _state = None

    def get_state(self):
        next_url = original_next_url = self.get_argument('next', None)
        if next_url:
            # avoid browsers treating \ as /
            next_url = next_url.replace('\\', quote('\\'))
            # disallow hostname-having urls,
            # force absolute path redirect
            urlinfo = urlparse(next_url)
            next_url = urlinfo._replace(
                scheme='', netloc='', path='/' + urlinfo.path.lstrip('/')
            ).geturl()
            if next_url != original_next_url:
                self.log.warning(
                    "Ignoring next_url %r, using %r", original_next_url, next_url
                )
        if self._state is None:
            self._state = _serialize_state(
                {'state_id': uuid.uuid4().hex, 'next_url': next_url}
            )
        return self._state

    def get(self):
        redirect_uri = self.authenticator.get_callback_url(self)
        extra_params = self.authenticator.extra_authorize_params.copy()
        self.log.info('OAuth redirect: %r', redirect_uri)
        state = self.get_state()
        self.set_state_cookie(state)
        extra_params['state'] = state
        self.authorize_redirect(
            redirect_uri=redirect_uri,
            client_id=self.authenticator.client_id,
            scope=self.authenticator.scope,
            extra_params=extra_params,
            response_type='code',
        )

class OAuthCallbackHandler(BaseHandler):
    """Basic handler for OAuth callback. Calls authenticator to verify username."""

    _state_cookie = None

    def get_state_cookie(self):
        """Get OAuth state from cookies

        To be compared with the value in redirect URL
        """
        if self._state_cookie is None:
            self._state_cookie = (
                self.get_secure_cookie(STATE_COOKIE_NAME) or b''
            ).decode('utf8', 'replace')
            self.clear_cookie(STATE_COOKIE_NAME)
        return self._state_cookie

    def get_state_url(self):
        """Get OAuth state from URL parameters

        to be compared with the value in cookies
        """
        return self.get_argument("state")

    def check_state(self):
        """Verify OAuth state

        compare value in cookie with redirect url param
        """
        cookie_state = self.get_state_cookie()
        url_state = self.get_state_url()
        if not cookie_state:
            raise web.HTTPError(400, "OAuth state missing from cookies")
        if not url_state:
            raise web.HTTPError(400, "OAuth state missing from URL")
        if cookie_state != url_state:
            self.log.warning("OAuth state mismatch: %s != %s", cookie_state, url_state)
            raise web.HTTPError(400, "OAuth state mismatch")

    def check_error(self):
        """Check the OAuth code"""
        error = self.get_argument("error", False)
        if error:
            message = self.get_argument("error_description", error)
            raise web.HTTPError(400, "OAuth error: %s" % message)

    def check_code(self):
        """Check the OAuth code"""
        if not self.get_argument("code", False):
            raise web.HTTPError(400, "OAuth callback made without a code")

    def check_arguments(self):
        """Validate the arguments of the redirect

        Default:

        - check for oauth-standard error, error_description arguments
        - check that there's a code
        - check that state matches
        """
        self.check_error()
        self.check_code()
        self.check_state()

    def get_next_url(self, user=None):
        """Get the redirect target from the state field"""
        state = self.get_state_url()
        if state:
            next_url = _deserialize_state(state).get('next_url')
            if next_url:
                return next_url
        # JupyterHub 0.8 adds default .get_next_url for a fallback
        if hasattr(BaseHandler, 'get_next_url'):
            return super().get_next_url(user)
        return url_path_join(self.hub.server.base_url, 'home')

    async def _login_user_pre_08(self):
        """login_user simplifies the login+cookie+auth_state process in JupyterHub 0.8

        _login_user_07 is for backward-compatibility with JupyterHub 0.7
        """
        user_info = await self.authenticator.get_authenticated_user(self, None)
        if user_info is None:
            return
        if isinstance(user_info, dict):
            username = user_info['name']
        else:
            username = user_info
        user = self.user_from_username(username)
        self.set_login_cookie(user)
        return user

    if not hasattr(BaseHandler, 'login_user'):
        # JupyterHub 0.7 doesn't have .login_user
        login_user = _login_user_pre_08

    async def get(self):
        self.check_arguments()
        user = await self.login_user()
        if user is None:
            # todo: custom error page?
            raise web.HTTPError(403)
        self.redirect(self.get_next_url(user))

class OAuthenticator(Authenticator):
    """Base class for OAuthenticators

    Subclasses must override:

    login_service (string identifying the service provider)
    authenticate (method takes one arg - the request handler handling the oauth callback)
    """

    login_handler = OAuthLoginHandler
    callback_handler = OAuthCallbackHandler

    authorize_url = Unicode(
        config=True, help="""The authenticate url for initiating oauth"""
    )
    @default("authorize_url")
    def _authorize_url_default(self):
        return os.environ.get("OAUTH2_AUTHORIZE_URL", "")

    token_url = Unicode(
        config=True,
        help="""The url retrieving an access token at the completion of oauth""",
    )
    @default("token_url")
    def _token_url_default(self):
        return os.environ.get("OAUTH2_TOKEN_URL", "")

    userdata_url = Unicode(
        config=True,
        help="""The url for retrieving user data with a completed access token""",
    )
    @default("userdata_url")
    def _userdata_url_default(self):
        return os.environ.get("OAUTH2_USERDATA_URL", "")

    scope = List(
        Unicode(),
        config=True,
        help="""The OAuth scopes to request.
        See the OAuth documentation of your OAuth provider for options.
        For GitHub in particular, you can see github_scopes.md in this repo.
        """,
    )

    extra_authorize_params = Dict(
        config=True,
        help="""Extra GET params to send along with the initial OAuth request
        to the OAuth provider.""",
    )

    login_service = 'override in subclass'
    oauth_callback_url = Unicode(
        os.getenv('OAUTH_CALLBACK_URL', ''),
        config=True,
        help="""Callback URL to use.
        Typically `https://{host}/hub/oauth_callback`""",
    )

    client_id_env = ''
    client_id = Unicode(config=True)

    def _client_id_default(self):
        if self.client_id_env:
            client_id = os.getenv(self.client_id_env, '')
            if client_id:
                return client_id
        return os.getenv('OAUTH_CLIENT_ID', '')

    client_secret_env = ''
    client_secret = Unicode(config=True)

    def _client_secret_default(self):
        if self.client_secret_env:
            client_secret = os.getenv(self.client_secret_env, '')
            if client_secret:
                return client_secret
        return os.getenv('OAUTH_CLIENT_SECRET', '')

    validate_server_cert_env = 'OAUTH_TLS_VERIFY'
    validate_server_cert = Bool(config=True)

    def _validate_server_cert_default(self):
        env_value = os.getenv(self.validate_server_cert_env, '')
        if env_value == '0':
            return False
        else:
            return True

    def login_url(self, base_url):
        return url_path_join(base_url, 'oauth_login')

    def get_callback_url(self, handler=None):
        """Get my OAuth redirect URL

        Either from config or guess based on the current request.
        """
        if self.oauth_callback_url:
            return self.oauth_callback_url
        elif handler:
            return guess_callback_uri(
                handler.request.protocol,
                handler.request.host,
                handler.hub.server.base_url,
            )
        else:
            raise ValueError(
                "Specify callback oauth_callback_url or give me a handler to guess with"
            )

    def get_handlers(self, app):
        return [
            (r'/oauth_login', self.login_handler),
            (r'/oauth_callback', self.callback_handler),
        ]

    async def authenticate(self, handler, data=None):
        raise NotImplementedError()

    def _deprecated_trait(self, change):
        """observer for deprecated traits"""
        old_attr = change.name
        new_attr, version = self._deprecated_aliases.get(old_attr)
        new_value = getattr(self, new_attr)
        if new_value != change.new:
            # only warn if different
            # protects backward-compatible config from warnings
            # if they set the same value under both names
            self.log.warning(
                "{cls}.{old} is deprecated in {cls} {version}, use {cls}.{new} instead".format(
                    cls=self.__class__.__name__,
                    old=old_attr,
                    new=new_attr,
                    version=version,
                )
            )
            setattr(self, new_attr, change.new)
<commit_msg>disable append_query_parameters in OAuthCallbackHandler<commit_after>"""
Base classes for Custom Authenticator to use OAuth with JupyterHub

Most of the code c/o Kyle Kelley (@rgbkrk)
"""

import base64
import json
import os
from urllib.parse import quote, urlparse
import uuid

from tornado import web
from tornado.auth import OAuth2Mixin
from tornado.log import app_log

from jupyterhub.handlers import BaseHandler
from jupyterhub.auth import Authenticator
from jupyterhub.utils import url_path_join

from traitlets import Unicode, Bool, List, Dict, default, observe

def guess_callback_uri(protocol, host, hub_server_url):
    return '{proto}://{host}{path}'.format(
        proto=protocol, host=host, path=url_path_join(hub_server_url, 'oauth_callback')
    )

STATE_COOKIE_NAME = 'oauthenticator-state'

def _serialize_state(state):
    """Serialize OAuth state to a base64 string after passing through JSON"""
    json_state = json.dumps(state)
    return base64.urlsafe_b64encode(json_state.encode('utf8')).decode('ascii')

def _deserialize_state(b64_state):
    """Deserialize OAuth state as serialized in _serialize_state"""
    if isinstance(b64_state, str):
        b64_state = b64_state.encode('ascii')
    try:
        json_state = base64.urlsafe_b64decode(b64_state).decode('utf8')
    except ValueError:
        app_log.error("Failed to b64-decode state: %r", b64_state)
        return {}
    try:
        return json.loads(json_state)
    except ValueError:
        app_log.error("Failed to json-decode state: %r", json_state)
        return {}

class OAuthLoginHandler(OAuth2Mixin, BaseHandler):
    """Base class for OAuth login handler

    Typically subclasses will need
    """

    # these URLs are part of the OAuth2Mixin API
    # get them from the Authenticator object
    @property
    def _OAUTH_AUTHORIZE_URL(self):
        return self.authenticator.authorize_url

    @property
    def _OAUTH_ACCESS_TOKEN_URL(self):
        return self.authenticator.token_url

    @property
    def _OAUTH_USERINFO_URL(self):
        return self.authenticator.userdata_url

    def set_state_cookie(self, state):
        self._set_cookie(STATE_COOKIE_NAME, state, expires_days=1, httponly=True)

    _state = None

    def get_state(self):
        next_url = original_next_url = self.get_argument('next', None)
        if next_url:
            # avoid browsers treating \ as /
            next_url = next_url.replace('\\', quote('\\'))
            # disallow hostname-having urls,
            # force absolute path redirect
            urlinfo = urlparse(next_url)
            next_url = urlinfo._replace(
                scheme='', netloc='', path='/' + urlinfo.path.lstrip('/')
            ).geturl()
            if next_url != original_next_url:
                self.log.warning(
                    "Ignoring next_url %r, using %r", original_next_url, next_url
                )
        if self._state is None:
            self._state = _serialize_state(
                {'state_id': uuid.uuid4().hex, 'next_url': next_url}
            )
        return self._state

    def get(self):
        redirect_uri = self.authenticator.get_callback_url(self)
        extra_params = self.authenticator.extra_authorize_params.copy()
        self.log.info('OAuth redirect: %r', redirect_uri)
        state = self.get_state()
        self.set_state_cookie(state)
        extra_params['state'] = state
        self.authorize_redirect(
            redirect_uri=redirect_uri,
            client_id=self.authenticator.client_id,
            scope=self.authenticator.scope,
            extra_params=extra_params,
            response_type='code',
        )

class OAuthCallbackHandler(BaseHandler):
    """Basic handler for OAuth callback. Calls authenticator to verify username."""

    _state_cookie = None

    def get_state_cookie(self):
        """Get OAuth state from cookies

        To be compared with the value in redirect URL
        """
        if self._state_cookie is None:
            self._state_cookie = (
                self.get_secure_cookie(STATE_COOKIE_NAME) or b''
            ).decode('utf8', 'replace')
            self.clear_cookie(STATE_COOKIE_NAME)
        return self._state_cookie

    def get_state_url(self):
        """Get OAuth state from URL parameters

        to be compared with the value in cookies
        """
        return self.get_argument("state")

    def check_state(self):
        """Verify OAuth state

        compare value in cookie with redirect url param
        """
        cookie_state = self.get_state_cookie()
        url_state = self.get_state_url()
        if not cookie_state:
            raise web.HTTPError(400, "OAuth state missing from cookies")
        if not url_state:
            raise web.HTTPError(400, "OAuth state missing from URL")
        if cookie_state != url_state:
            self.log.warning("OAuth state mismatch: %s != %s", cookie_state, url_state)
            raise web.HTTPError(400, "OAuth state mismatch")

    def check_error(self):
        """Check the OAuth code"""
        error = self.get_argument("error", False)
        if error:
            message = self.get_argument("error_description", error)
            raise web.HTTPError(400, "OAuth error: %s" % message)

    def check_code(self):
        """Check the OAuth code"""
        if not self.get_argument("code", False):
            raise web.HTTPError(400, "OAuth callback made without a code")

    def check_arguments(self):
        """Validate the arguments of the redirect

        Default:

        - check for oauth-standard error, error_description arguments
        - check that there's a code
        - check that state matches
        """
        self.check_error()
        self.check_code()
        self.check_state()

    def append_query_parameters(self, url, exclude=None):
        """JupyterHub 1.2 appends query parameters by default in get_next_url

        This is not appropriate for oauth callback handlers, where params are oauth state, code, etc.

        Override the method used to append parameters to next_url to not preserve any parameters
        """
        return url

    def get_next_url(self, user=None):
        """Get the redirect target from the state field"""
        state = self.get_state_url()
        if state:
            next_url = _deserialize_state(state).get('next_url')
            if next_url:
                return next_url
        # JupyterHub 0.8 adds default .get_next_url for a fallback
        if hasattr(BaseHandler, 'get_next_url'):
            return super().get_next_url(user)
        return url_path_join(self.hub.server.base_url, 'home')

    async def _login_user_pre_08(self):
        """login_user simplifies the login+cookie+auth_state process in JupyterHub 0.8

        _login_user_07 is for backward-compatibility with JupyterHub 0.7
        """
        user_info = await self.authenticator.get_authenticated_user(self, None)
        if user_info is None:
            return
        if isinstance(user_info, dict):
            username = user_info['name']
        else:
            username = user_info
        user = self.user_from_username(username)
        self.set_login_cookie(user)
        return user

    if not hasattr(BaseHandler, 'login_user'):
        # JupyterHub 0.7 doesn't have .login_user
        login_user = _login_user_pre_08

    async def get(self):
        self.check_arguments()
        user = await self.login_user()
        if user is None:
            # todo: custom error page?
            raise web.HTTPError(403)
        self.redirect(self.get_next_url(user))

class OAuthenticator(Authenticator):
    """Base class for OAuthenticators

    Subclasses must override:

    login_service (string identifying the service provider)
    authenticate (method takes one arg - the request handler handling the oauth callback)
    """

    login_handler = OAuthLoginHandler
    callback_handler = OAuthCallbackHandler

    authorize_url = Unicode(
        config=True, help="""The authenticate url for initiating oauth"""
    )
    @default("authorize_url")
    def _authorize_url_default(self):
        return os.environ.get("OAUTH2_AUTHORIZE_URL", "")

    token_url = Unicode(
        config=True,
        help="""The url retrieving an access token at the completion of oauth""",
    )
    @default("token_url")
    def _token_url_default(self):
        return os.environ.get("OAUTH2_TOKEN_URL", "")

    userdata_url = Unicode(
        config=True,
        help="""The url for retrieving user data with a completed access token""",
    )
    @default("userdata_url")
    def _userdata_url_default(self):
        return os.environ.get("OAUTH2_USERDATA_URL", "")

    scope = List(
        Unicode(),
        config=True,
        help="""The OAuth scopes to request.
        See the OAuth documentation of your OAuth provider for options.
        For GitHub in particular, you can see github_scopes.md in this repo.
        """,
    )

    extra_authorize_params = Dict(
        config=True,
        help="""Extra GET params to send along with the initial OAuth request
        to the OAuth provider.""",
    )

    login_service = 'override in subclass'
    oauth_callback_url = Unicode(
        os.getenv('OAUTH_CALLBACK_URL', ''),
        config=True,
        help="""Callback URL to use.
        Typically `https://{host}/hub/oauth_callback`""",
    )

    client_id_env = ''
    client_id = Unicode(config=True)

    def _client_id_default(self):
        if self.client_id_env:
            client_id = os.getenv(self.client_id_env, '')
            if client_id:
                return client_id
        return os.getenv('OAUTH_CLIENT_ID', '')

    client_secret_env = ''
    client_secret = Unicode(config=True)

    def _client_secret_default(self):
        if self.client_secret_env:
            client_secret = os.getenv(self.client_secret_env, '')
            if client_secret:
                return client_secret
        return os.getenv('OAUTH_CLIENT_SECRET', '')

    validate_server_cert_env = 'OAUTH_TLS_VERIFY'
    validate_server_cert = Bool(config=True)

    def _validate_server_cert_default(self):
        env_value = os.getenv(self.validate_server_cert_env, '')
        if env_value == '0':
            return False
        else:
            return True

    def login_url(self, base_url):
        return url_path_join(base_url, 'oauth_login')

    def get_callback_url(self, handler=None):
        """Get my OAuth redirect URL

        Either from config or guess based on the current request.
        """
        if self.oauth_callback_url:
            return self.oauth_callback_url
        elif handler:
            return guess_callback_uri(
                handler.request.protocol,
                handler.request.host,
                handler.hub.server.base_url,
            )
        else:
            raise ValueError(
                "Specify callback oauth_callback_url or give me a handler to guess with"
            )

    def get_handlers(self, app):
        return [
            (r'/oauth_login', self.login_handler),
            (r'/oauth_callback', self.callback_handler),
        ]

    async def authenticate(self, handler, data=None):
        raise NotImplementedError()

    def _deprecated_trait(self, change):
        """observer for deprecated traits"""
        old_attr = change.name
        new_attr, version = self._deprecated_aliases.get(old_attr)
        new_value = getattr(self, new_attr)
        if new_value != change.new:
            # only warn if different
            # protects backward-compatible config from warnings
            # if they set the same value under both names
            self.log.warning(
                "{cls}.{old} is deprecated in {cls} {version}, use {cls}.{new} instead".format(
                    cls=self.__class__.__name__,
                    old=old_attr,
                    new=new_attr,
                    version=version,
                )
            )
            setattr(self, new_attr, change.new)
<|endoftext|>


<commit_before>"""
Custom Authenticator to use Google OAuth with JupyterHub.

Derived from the GitHub OAuth authenticator.
"""

import os
import json

from base64 import b64decode, b64encode, urlsafe_b64decode
import re

from tornado             import gen, escape
from tornado.auth        import GoogleOAuth2Mixin
from tornado.web         import HTTPError

from traitlets           import Unicode

from jupyterhub.auth     import LocalAuthenticator
from jupyterhub.utils    import url_path_join

from .oauth2 import OAuthLoginHandler, OAuthCallbackHandler, OAuthenticator

from tornado.util import PY3
import functools

if PY3:
    import urllib.parse as urlparse
    import urllib.parse as urllib_parse
    long = int
else:
    import urlparse
    import urllib as urllib_parse

class AuthError(Exception):
    pass

class OpenIDOAuth2Mixin(GoogleOAuth2Mixin):
    """ An OpenID OAuth2 mixin to use GoogleLoginHandler with
    different Identity Providers using the OpenID standard. The current
    setup should work with MITREid Connect servers. In addtion to the usual
    parameters client ID and secret, the environment variable OPENID_HOST
    should be set to the URL of the OpenID provider. The API endpoints
    might have to be changed, depending on the ID provider."""

    CONNECTORS = os.environ.get('CONNECTOR_LIST')

    _OPENID_ENDPOINT = os.environ.get('OPENID_HOST')
    if _OPENID_ENDPOINT.startswith('http'):
        _OAUTH_AUTHORIZE_URL = "%s/auth" % _OPENID_ENDPOINT
        _OAUTH_ACCESS_TOKEN_URL = "%s/token" % _OPENID_ENDPOINT
        _OAUTH_USERINFO_URL = "%s/auth" % _OPENID_ENDPOINT
    else:
        _OAUTH_AUTHORIZE_URL = "https://%s/auth" % _OPENID_ENDPOINT
        _OAUTH_ACCESS_TOKEN_URL = "https://%s/token" % _OPENID_ENDPOINT
        _OAUTH_USERINFO_URL = "https://%s/auth" % _OPENID_ENDPOINT

    def get_authenticated_user(self, redirect_uri, code, callback):
        """Handles the login for the Google user, returning an access token.
        """
        http = self.get_auth_http_client()
        body = urllib_parse.urlencode({
            "redirect_uri": redirect_uri,
            "code": code,
            "client_id": self.settings[self._OAUTH_SETTINGS_KEY]['key'],
            "client_secret": self.settings[self._OAUTH_SETTINGS_KEY]['secret'],
            "grant_type": "authorization_code",
        })
        self.log.info('http req body: %r', body)
        self.log.info('acc tok url: %r', self._OAUTH_ACCESS_TOKEN_URL)
        self.log.info('callback url: %r', callback)
        http.fetch(self._OAUTH_ACCESS_TOKEN_URL,
                   functools.partial(self._on_access_token, callback),
                   method="POST", headers={'Content-Type': 'application/x-www-form-urlencoded'}, body=body, validate_cert = False)

    def _on_access_token(self, future, response):
        """Callback function for the exchange to the access token."""
        #self.log.info('response body: %r', response)
        if response.error:
            future.set_exception(AuthError('OpenID auth error: %s' % str(response)))
            return

        args = escape.json_decode(response.body)
        future.set_result(args)

class OpenIDLoginHandler(OAuthLoginHandler, OpenIDOAuth2Mixin):
    '''An OAuthLoginHandler that provides scope to GoogleOAuth2Mixin's
       authorize_redirect.'''
    scope=['openid','profile', 'email','offline_access','groups']
    def get(self):
        redirect_uri = self.authenticator.get_callback_url(self)
        state = self.get_state()
        self.set_state_cookie(state)
        self.log.info('OAuth redirect: %r', redirect_uri)
        self.authorize_redirect(
            redirect_uri=redirect_uri,
            client_id=self.authenticator.client_id,
            scope=['openid','profile', 'email','offline_access','groups'],
            extra_params={'state': state},
            response_type='code')

class OpenIDOAuthHandler(OAuthCallbackHandler, OpenIDOAuth2Mixin):
    pass

class OpenIDOAuthenticator(OAuthenticator, OpenIDOAuth2Mixin):

    login_handler = OpenIDLoginHandler
    callback_handler = OpenIDOAuthHandler

    # hosted_domain = Unicode(
    #     os.environ.get('HOSTED_DOMAIN', ''),
    #     config=True,
    #     help="""Hosted domain used to restrict sign-in, e.g. mycollege.edu"""
    # )
    login_service = Unicode(
        os.environ.get('LOGIN_SERVICE', 'Dex'),
        config=True,
        help="""Google Apps hosted domain string, e.g. My College"""
    )

    @gen.coroutine
    def authenticate(self, handler, data=None):
        code = handler.get_argument('code')
        if not code:
            raise HTTPError(400, "oauth callback made without a token")
        handler.settings['google_oauth'] = {
            'key': self.client_id,
            'secret': self.client_secret,
            'scope': ['openid','profile', 'email','offline_access','groups'],
            'response_type': 'code'
        }

        self.log.info('openid: settings: "%s"', str(handler.settings['google_oauth']))
        self.log.info('code is: {}'.format(code))
        user = yield handler.get_authenticated_user(
            redirect_uri=self.get_callback_url(handler),
            code=code)
        access_token = str(user['access_token'])
        self.log.info('token is: {}'.format(access_token))
        self.log.info('full user json is: {}'.format(user))

        http_client = handler.get_auth_http_client()

        #response = yield http_client.fetch(
        #    self._OAUTH_USERINFO_URL + '?access_token=' + access_token
        #)

        #if not response:
        #    self.clear_all_cookies()
        #    raise HTTPError(500, 'Google authentication failed')

        #body = response.body.decode()
        #self.log.debug('response.body.decode(): {}'.format(body))
        #bodyjs = json.loads(body)
        payload_encoded = user['id_token'].split('.')[1]
        payload = urlsafe_b64decode(payload_encoded + '=' * (4 - len(payload_encoded) % 4)).decode('utf8')
        self.log.info('urlsafe decoded payload is: {}'.format(payload))
        userstring = re.findall('(?<=sub":").+?(?=",)',payload)[0]
        substring = urlsafe_b64decode(userstring + '=' * (4 - len(userstring) % 4)).decode('utf8')

        substring_print = ''.join([i for i in substring if i.isprintable()])
        self.log.info('urlsafe decoded, printable substring is: {}'.format(substring_print))

        username = ''

        for connector in self.CONNECTORS.split(','):
            if connector == 'github':
                if re.findall('(?<=name":").+?(?=")', payload):
                    returned_name = re.findall('(?<=name":").+?(?=")', payload)[0]
                    username = re.sub(' ','',returned_name) + '_' + 'github'
                else:
                    pass
            elif re.findall(connector,substring_print):
                username = re.sub(connector,'_' + connector,substring_print)
            else:
                pass

        if not username:
            raise Exception('Connector error: Could not extract username from id_token, sub or name entry.')

        #if self.hosted_domain:
        #    if not username.endswith('@'+self.hosted_domain) or \
        #        bodyjs['hd'] != self.hosted_domain:
        #        raise HTTPError(403,
        #            "You are not signed in to your {} account.".format(
        #                self.hosted_domain)
        #        )
        #    else:
        #        username = username.split('@')[0]

        return username

class LocalOpenIDOAuthenticator(LocalAuthenticator, OpenIDOAuthenticator):
    """A version that mixes in local system user creation"""
    pass
<commit_msg>use general get_auth_user<commit_after>"""
Custom Authenticator to use Google OAuth with JupyterHub.

Derived from the GitHub OAuth authenticator.
"""

import os
import json

from base64 import b64decode, b64encode, urlsafe_b64decode
import re

from tornado             import gen, escape
from tornado.auth        import GoogleOAuth2Mixin
from tornado.web         import HTTPError

from traitlets           import Unicode

from jupyterhub.auth     import LocalAuthenticator
from jupyterhub.utils    import url_path_join

from .oauth2 import OAuthLoginHandler, OAuthCallbackHandler, OAuthenticator

from tornado.util import PY3
import functools

if PY3:
    import urllib.parse as urlparse
    import urllib.parse as urllib_parse
    long = int
else:
    import urlparse
    import urllib as urllib_parse

class AuthError(Exception):
    pass

class OpenIDOAuth2Mixin(GoogleOAuth2Mixin):
    """ An OpenID OAuth2 mixin to use GoogleLoginHandler with
    different Identity Providers using the OpenID standard. The current
    setup should work with MITREid Connect servers. In addtion to the usual
    parameters client ID and secret, the environment variable OPENID_HOST
    should be set to the URL of the OpenID provider. The API endpoints
    might have to be changed, depending on the ID provider."""

    CONNECTORS = os.environ.get('CONNECTOR_LIST')

    _OPENID_ENDPOINT = os.environ.get('OPENID_HOST')
    if _OPENID_ENDPOINT.startswith('http'):
        _OAUTH_AUTHORIZE_URL = "%s/auth" % _OPENID_ENDPOINT
        _OAUTH_ACCESS_TOKEN_URL = "%s/token" % _OPENID_ENDPOINT
        _OAUTH_USERINFO_URL = "%s/auth" % _OPENID_ENDPOINT
    else:
        _OAUTH_AUTHORIZE_URL = "https://%s/auth" % _OPENID_ENDPOINT
        _OAUTH_ACCESS_TOKEN_URL = "https://%s/token" % _OPENID_ENDPOINT
        _OAUTH_USERINFO_URL = "https://%s/auth" % _OPENID_ENDPOINT

    # def get_authenticated_user(self, redirect_uri, code, callback):
    #     """Handles the login for the Google user, returning an access token.
    #     """
    #     http = self.get_auth_http_client()
    #     body = urllib_parse.urlencode({
    #         "redirect_uri": redirect_uri,
    #         "code": code,
    #         "client_id": self.settings[self._OAUTH_SETTINGS_KEY]['key'],
    #         "client_secret": self.settings[self._OAUTH_SETTINGS_KEY]['secret'],
    #         "grant_type": "authorization_code",
    #     })
    #     self.log.info('http req body: %r', body)
    #     self.log.info('acc tok url: %r', self._OAUTH_ACCESS_TOKEN_URL)
    #     self.log.info('callback url: %r', callback)
    #     http.fetch(self._OAUTH_ACCESS_TOKEN_URL,
    #                functools.partial(self._on_access_token, callback),
    #                method="POST", headers={'Content-Type': 'application/x-www-form-urlencoded'}, body=body, validate_cert = False)
    #
    #
    # def _on_access_token(self, future, response):
    #     """Callback function for the exchange to the access token."""
    #     #self.log.info('response body: %r', response)
    #     if response.error:
    #         future.set_exception(AuthError('OpenID auth error: %s' % str(response)))
    #         return
    #
    #     args = escape.json_decode(response.body)
    #     future.set_result(args)

class OpenIDLoginHandler(OAuthLoginHandler, OpenIDOAuth2Mixin):
    '''An OAuthLoginHandler that provides scope to GoogleOAuth2Mixin's
       authorize_redirect.'''
    scope=['openid','profile', 'email','offline_access','groups']
    def get(self):
        redirect_uri = self.authenticator.get_callback_url(self)
        state = self.get_state()
        self.set_state_cookie(state)
        self.log.info('OAuth redirect: %r', redirect_uri)
        self.authorize_redirect(
            redirect_uri=redirect_uri,
            client_id=self.authenticator.client_id,
            scope=['openid','profile', 'email','offline_access','groups'],
            extra_params={'state': state},
            response_type='code')

class OpenIDOAuthHandler(OAuthCallbackHandler, OpenIDOAuth2Mixin):
    pass

class OpenIDOAuthenticator(OAuthenticator, OpenIDOAuth2Mixin):

    login_handler = OpenIDLoginHandler
    callback_handler = OpenIDOAuthHandler

    # hosted_domain = Unicode(
    #     os.environ.get('HOSTED_DOMAIN', ''),
    #     config=True,
    #     help="""Hosted domain used to restrict sign-in, e.g. mycollege.edu"""
    # )
    login_service = Unicode(
        os.environ.get('LOGIN_SERVICE', 'Dex'),
        config=True,
        help="""Google Apps hosted domain string, e.g. My College"""
    )

    @gen.coroutine
    def authenticate(self, handler, data=None):
        code = handler.get_argument('code')
        if not code:
            raise HTTPError(400, "oauth callback made without a token")
        handler.settings['google_oauth'] = {
            'key': self.client_id,
            'secret': self.client_secret,
            'scope': ['openid','profile', 'email','offline_access','groups'],
            'response_type': 'code'
        }

        self.log.info('openid: settings: "%s"', str(handler.settings['google_oauth']))
        self.log.info('code is: {}'.format(code))
        user = yield handler.get_authenticated_user(
            redirect_uri=self.get_callback_url(handler),
            code=code)
        access_token = str(user['access_token'])
        self.log.info('token is: {}'.format(access_token))
        self.log.info('full user json is: {}'.format(user))

        http_client = handler.get_auth_http_client()

        #response = yield http_client.fetch(
        #    self._OAUTH_USERINFO_URL + '?access_token=' + access_token
        #)

        #if not response:
        #    self.clear_all_cookies()
        #    raise HTTPError(500, 'Google authentication failed')

        #body = response.body.decode()
        #self.log.debug('response.body.decode(): {}'.format(body))
        #bodyjs = json.loads(body)
        payload_encoded = user['id_token'].split('.')[1]
        payload = urlsafe_b64decode(payload_encoded + '=' * (4 - len(payload_encoded) % 4)).decode('utf8')
        self.log.info('urlsafe decoded payload is: {}'.format(payload))
        userstring = re.findall('(?<=sub":").+?(?=",)',payload)[0]
        substring = urlsafe_b64decode(userstring + '=' * (4 - len(userstring) % 4)).decode('utf8')

        substring_print = ''.join([i for i in substring if i.isprintable()])
        self.log.info('urlsafe decoded, printable substring is: {}'.format(substring_print))

        username = ''

        for connector in self.CONNECTORS.split(','):
            if connector == 'github':
                if re.findall('(?<=name":").+?(?=")', payload):
                    returned_name = re.findall('(?<=name":").+?(?=")', payload)[0]
                    username = re.sub(' ','',returned_name) + '_' + 'github'
                else:
                    pass
            elif re.findall(connector,substring_print):
                username = re.sub(connector,'_' + connector,substring_print)
            else:
                pass

        if not username:
            raise Exception('Connector error: Could not extract username from id_token, sub or name entry.')

        #if self.hosted_domain:
        #    if not username.endswith('@'+self.hosted_domain) or \
        #        bodyjs['hd'] != self.hosted_domain:
        #        raise HTTPError(403,
        #            "You are not signed in to your {} account.".format(
        #                self.hosted_domain)
        #        )
        #    else:
        #        username = username.split('@')[0]

        return username

class LocalOpenIDOAuthenticator(LocalAuthenticator, OpenIDOAuthenticator):
    """A version that mixes in local system user creation"""
    pass
<|endoftext|>