<commit_before>''' -- imports from installed packages -- '''

from django.http import StreamingHttpResponse
from django.shortcuts import render_to_response, render
from django.template import RequestContext
from django.contrib.auth.models import User
from django.contrib.auth.decorators import user_passes_test
from django_mongokit import get_database

from gnowsys_ndf.ndf.views.methods import *
import json

db = get_database()
collection = db[File.collection_name]

@user_passes_test(lambda u: u.is_superuser)
def adminDesignerDashboardClass(request, class_name):
    '''
    fetching class's objects
    '''
    if request.method=="POST":
        search = request.POST.get("search","")
        classtype = request.POST.get("class","")
        nodes = collection.Node.find({'name':{'$regex':search},'_type':classtype})
    else :
        nodes = collection.Node.find({'_type':class_name})
    objects_details = []
    for each in nodes:
        member = []
        for members in each.member_of:
            member.append(collection.Node.one({ '_id': members}).name+" - "+str(members))
	if class_name in ("GSystem","File"):
		objects_details.append({"Id":each._id,"Title":each.name,"Type":",".join(member),"Author":User.objects.get(id=each.created_by).username,"Group":",".join(each.group_set),"Creation":each.created_at})
	else :
		objects_details.append({"Id":each._id,"Title":each.name,"Type":",".join(member),"Author":User.objects.get(id=each.created_by).username,"Creation":each.created_at})
    groups = []
    group = collection.Node.find({'_type':"Group"})
    for each in group:
        groups.append({'id':each._id,"title":each.name})
    systemtypes = []
    systemtype = collection.Node.find({'_type':"GSystemType"})
    for each in systemtype:
        systemtypes.append({'id':each._id,"title":each.name})

    template = "ndf/adminDashboard.html"
    variable = RequestContext(request, {'class_name':class_name,"nodes":objects_details,"Groups":groups,"systemtypes":systemtypes,"url":"designer"})
    return render_to_response(template, variable)

@user_passes_test(lambda u: u.is_superuser)
def adminDesignerDashboardClassCreate(request,class_name):
    '''
    delete class's objects
    '''
    definitionlist = []
    contentlist = []
    dependencylist = []
    options = []
    class_structure = eval(class_name).structure
    # if class_name == 'GSystemType':
    #     class_structure = collection.GSystemType.structure
    # if class_name == 'RelationType':
    #     class_structure = collection.RelationType.structure
    # if class_name == 'AttributeType':
    #     class_structure = collection.AttributeType.structure

    newdict = {}
    for key,value in class_structure.items():
        if value == bool:
            newdict[key] = "bool"
        elif value == unicode:
            newdict[key] = "unicode"
        elif value == list:
            newdict[key] = "list"
        elif type(value) == list:
            newdict[key] = "list"
        elif value == datetime.datetime:
            newdict[key] = "datetime"
        else:
            newdict[key] = value
    #        print value, key
    class_structure = newdict
    if class_name == "AttributeType":
        definitionlist = ['name','altnames','subject_type','applicable_node_type','data_type','member_of','verbose_name','null','blank','help_text','max_digit','decimal_places','auto_now','auto_now_add','path','verify_exist','status']
        contentlist = ['content','content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_by','created_at','start_publication','tags','url','last_update','login_required']
    elif class_name == "GSystemType":
        definitionlist = ['name','altnames','status','meta_type_set','attribute_type_set','relation_type_set','process_type_set']
        contentlist = ['content','content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_by','created_at','start_publication','tags','url','last_update','login_required']
    elif class_name == "RelationType":
        definitionlist = ['name','altnames','subject_type','applicable_node_type','data_type','verbose_name','null','blank','help_text','max_digit','decimal_places','auto_now','auto_now_add','path','verify_exist','status']
        contentlist = ['content','content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_by','created_at','start_publication','tags','url','last_update','login_required']
    else :
        definitionlist = []
        contentlist = []
        dependencylist = []
        options = []

    template = "ndf/adminDashboardCreate.html"
    variable = RequestContext(request, {'class_name':class_name,"url":"designer","class_structure":class_structure,'definitionlist':definitionlist,'contentlist':contentlist,'dependencylist':dependencylist,'options':options})
    return render_to_response(template, variable)

<commit_msg>modification in Relationtype classification of fields of designer form<commit_after>''' -- imports from installed packages -- '''

from django.http import StreamingHttpResponse
from django.shortcuts import render_to_response, render
from django.template import RequestContext
from django.contrib.auth.models import User
from django.contrib.auth.decorators import user_passes_test
from django_mongokit import get_database

from gnowsys_ndf.ndf.views.methods import *
import json

db = get_database()
collection = db[File.collection_name]

@user_passes_test(lambda u: u.is_superuser)
def adminDesignerDashboardClass(request, class_name):
    '''
    fetching class's objects
    '''
    if request.method=="POST":
        search = request.POST.get("search","")
        classtype = request.POST.get("class","")
        nodes = collection.Node.find({'name':{'$regex':search},'_type':classtype})
    else :
        nodes = collection.Node.find({'_type':class_name})
    objects_details = []
    for each in nodes:
        member = []
        for members in each.member_of:
            member.append(collection.Node.one({ '_id': members}).name+" - "+str(members))
	if class_name in ("GSystem","File"):
		objects_details.append({"Id":each._id,"Title":each.name,"Type":",".join(member),"Author":User.objects.get(id=each.created_by).username,"Group":",".join(each.group_set),"Creation":each.created_at})
	else :
		objects_details.append({"Id":each._id,"Title":each.name,"Type":",".join(member),"Author":User.objects.get(id=each.created_by).username,"Creation":each.created_at})
    groups = []
    group = collection.Node.find({'_type':"Group"})
    for each in group:
        groups.append({'id':each._id,"title":each.name})
    systemtypes = []
    systemtype = collection.Node.find({'_type':"GSystemType"})
    for each in systemtype:
        systemtypes.append({'id':each._id,"title":each.name})

    template = "ndf/adminDashboard.html"
    variable = RequestContext(request, {'class_name':class_name,"nodes":objects_details,"Groups":groups,"systemtypes":systemtypes,"url":"designer"})
    return render_to_response(template, variable)

@user_passes_test(lambda u: u.is_superuser)
def adminDesignerDashboardClassCreate(request,class_name):
    '''
    delete class's objects
    '''
    definitionlist = []
    contentlist = []
    dependencylist = []
    options = []
    class_structure = eval(class_name).structure
    # if class_name == 'GSystemType':
    #     class_structure = collection.GSystemType.structure
    # if class_name == 'RelationType':
    #     class_structure = collection.RelationType.structure
    # if class_name == 'AttributeType':
    #     class_structure = collection.AttributeType.structure

    newdict = {}
    for key,value in class_structure.items():
        if value == bool:
            newdict[key] = "bool"
        elif value == unicode:
            newdict[key] = "unicode"
        elif value == list:
            newdict[key] = "list"
        elif type(value) == list:
            newdict[key] = "list"
        elif value == datetime.datetime:
            newdict[key] = "datetime"
        else:
            newdict[key] = value
    #        print value, key
    class_structure = newdict
    if class_name == "AttributeType":
        definitionlist = ['name','altnames','subject_type','applicable_node_type','data_type','member_of','verbose_name','null','blank','help_text','max_digit','decimal_places','auto_now','auto_now_add','path','verify_exist','status']
        contentlist = ['content','content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_by','created_at','start_publication','tags','url','last_update','login_required']
    elif class_name == "GSystemType":
        definitionlist = ['name','altnames','status','meta_type_set','attribute_type_set','relation_type_set']
        contentlist = ['content','content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_by','created_at','start_publication','tags','url','last_update','login_required']
    elif class_name == "RelationType":
        definitionlist = ['name','inverse_name','altnames','subject_type','object_type','subject_cardinality','object_cardinality','subject_applicable_nodetype','object_applicable_nodetype','slug','is_symmetric','is_reflexive','is_transitive','status']
        contentlist = ['content','content_org']
        dependencylist = ['prior_node']
        options = ['featured','created_by','created_at','start_publication','tags','url','last_update','login_required']
    else :
        definitionlist = []
        contentlist = []
        dependencylist = []
        options = []

    template = "ndf/adminDashboardCreate.html"
    variable = RequestContext(request, {'class_name':class_name,"url":"designer","class_structure":class_structure,'definitionlist':definitionlist,'contentlist':contentlist,'dependencylist':dependencylist,'options':options})
    return render_to_response(template, variable)

<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
# Copyright (C) 2009-2016, Luis Pedro Coelho <luis@luispedro.org>
# vim: set ts=4 sts=4 sw=4 expandtab smartindent:
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#  THE SOFTWARE.

'''
redis_store: store based on a redis backend
'''

import re
import six
import logging
from base64 import b64encode, b64decode
from jug.backends.encode import encode, decode
from .base import base_store, base_lock

try:
    import redis
    redis_functional = True
except ImportError:
    redis = None
    redis_functional = False

def _resultname(name):
    if type(name) == type(six.u('')):
        name = name.encode('utf-8')
    return six.b('result:') + name

def _lockname(name):
    if type(name) == type(six.u('')):
        name = name.encode('utf-8')
    return six.b('lock:') + name

_LOCKED = 1

_redis_urlpat = re.compile(r'redis://(?P<host>[A-Za-z0-9\.\-]+)(\:(?P<port>[0-9]+))?/')

class redis_store(base_store):
    def __init__(self, url):
        '''
        '''
        if redis is None:
            raise IOError('jug.redis_store: redis module is not found!')
        redis_params = {}
        match = _redis_urlpat.match(url)
        if match:
            redis_params = match.groupdict()
            if redis_params['port'] == None:
                del redis_params['port']
            else:
                redis_params['port'] = int( redis_params['port'] )
        logging.info('connecting to %s' % redis_params)

        self.redis = redis.Redis(**redis_params)

    def dump(self, object, name):
        '''
        dump(object, name)
        '''
        s = encode(object)
        if s:
            s = b64encode(s)
        self.redis.set(_resultname(name), s)

    def can_load(self, name):
        '''
        can = can_load(name)
        '''
        return self.redis.exists(_resultname(name))

    def load(self, name):
        '''
        obj = load(name)

        Loads the object identified by `name`.
        '''
        s = self.redis.get(_resultname(name))
        if s:
            s = b64decode(s)
        return decode(s)

    def remove(self, name):
        '''
        was_removed = remove(name)

        Remove the entry associated with name.

        Returns whether any entry was actually removed.
        '''
        return self.redis.delete(_resultname(name))

    def cleanup(self, active):
        '''
        cleanup()

        Implement 'cleanup' command
        '''
        existing = list(self.list())
        for act in active:
            try:
                existing.remove(_resultname(act))
            except KeyError:
                pass
        for superflous in existing:
            self.redis.delete(_resultname(superflous))

    def remove_locks(self):
        locks = self.redis.keys('lock:*')
        for lk in locks:
            self.redis.delete(lk)
        return len(locks)

    def list(self):
        existing = self.redis.keys('result:*')
        for ex in existing:
            yield ex[len('result:'):]

    def listlocks(self):
        locks = self.redis.keys('lock:*')
        for lk in locks:
            yield lk[len('lock:')]

    def getlock(self, name):
        return redis_lock(self.redis, name)

    def close(self):
        # It seems some versions of the protocol are implemented differently
        # and do not have the ``disconnect`` method
        try:
            self.redis.disconnect()
        except:
            pass

class redis_lock(base_lock):
    '''
    redis_lock

    Functions:
    ----------

        * get(): acquire the lock
        * release(): release the lock
        * is_locked(): check lock state
    '''

    def __init__(self, redis, name):
        self.name = _lockname(name)
        self.redis = redis

    def get(self):
        '''
        lock.get()
        '''
        previous = self.redis.getset(self.name, _LOCKED)
        return (previous is None)

    def release(self):
        '''
        lock.release()

        Removes lock
        '''
        self.redis.delete(self.name)

    def is_locked(self):
        '''
        locked = lock.is_locked()
        '''
        status = self.redis.get(self.name)
        return status is not None and status == _LOCKED

<commit_msg>MIN Don't require a trailing slash on jugdir redis addresses<commit_after>#-*- coding: utf-8 -*-
# Copyright (C) 2009-2016, Luis Pedro Coelho <luis@luispedro.org>
# vim: set ts=4 sts=4 sw=4 expandtab smartindent:
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#  THE SOFTWARE.

'''
redis_store: store based on a redis backend
'''

import re
import six
import logging
from base64 import b64encode, b64decode
from jug.backends.encode import encode, decode
from .base import base_store, base_lock

try:
    import redis
    redis_functional = True
except ImportError:
    redis = None
    redis_functional = False

def _resultname(name):
    if type(name) == type(six.u('')):
        name = name.encode('utf-8')
    return six.b('result:') + name

def _lockname(name):
    if type(name) == type(six.u('')):
        name = name.encode('utf-8')
    return six.b('lock:') + name

_LOCKED = 1

_redis_urlpat = re.compile(r'redis://(?P<host>[A-Za-z0-9\.\-]+)(\:(?P<port>[0-9]+))?/?')

class redis_store(base_store):
    def __init__(self, url):
        '''
        '''
        if redis is None:
            raise IOError('jug.redis_store: redis module is not found!')
        redis_params = {}
        match = _redis_urlpat.match(url)
        if match:
            redis_params = match.groupdict()
            if redis_params['port'] == None:
                del redis_params['port']
            else:
                redis_params['port'] = int( redis_params['port'] )
        logging.info('connecting to %s' % redis_params)

        self.redis = redis.Redis(**redis_params)

    def dump(self, object, name):
        '''
        dump(object, name)
        '''
        s = encode(object)
        if s:
            s = b64encode(s)
        self.redis.set(_resultname(name), s)

    def can_load(self, name):
        '''
        can = can_load(name)
        '''
        return self.redis.exists(_resultname(name))

    def load(self, name):
        '''
        obj = load(name)

        Loads the object identified by `name`.
        '''
        s = self.redis.get(_resultname(name))
        if s:
            s = b64decode(s)
        return decode(s)

    def remove(self, name):
        '''
        was_removed = remove(name)

        Remove the entry associated with name.

        Returns whether any entry was actually removed.
        '''
        return self.redis.delete(_resultname(name))

    def cleanup(self, active):
        '''
        cleanup()

        Implement 'cleanup' command
        '''
        existing = list(self.list())
        for act in active:
            try:
                existing.remove(_resultname(act))
            except KeyError:
                pass
        for superflous in existing:
            self.redis.delete(_resultname(superflous))

    def remove_locks(self):
        locks = self.redis.keys('lock:*')
        for lk in locks:
            self.redis.delete(lk)
        return len(locks)

    def list(self):
        existing = self.redis.keys('result:*')
        for ex in existing:
            yield ex[len('result:'):]

    def listlocks(self):
        locks = self.redis.keys('lock:*')
        for lk in locks:
            yield lk[len('lock:')]

    def getlock(self, name):
        return redis_lock(self.redis, name)

    def close(self):
        # It seems some versions of the protocol are implemented differently
        # and do not have the ``disconnect`` method
        try:
            self.redis.disconnect()
        except:
            pass

class redis_lock(base_lock):
    '''
    redis_lock

    Functions:
    ----------

        * get(): acquire the lock
        * release(): release the lock
        * is_locked(): check lock state
    '''

    def __init__(self, redis, name):
        self.name = _lockname(name)
        self.redis = redis

    def get(self):
        '''
        lock.get()
        '''
        previous = self.redis.getset(self.name, _LOCKED)
        return (previous is None)

    def release(self):
        '''
        lock.release()

        Removes lock
        '''
        self.redis.delete(self.name)

    def is_locked(self):
        '''
        locked = lock.is_locked()
        '''
        status = self.redis.get(self.name)
        return status is not None and status == _LOCKED

<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

import logging
import shutil
from decimal import Decimal

import boto3
import os
from owlmixin import OwlMixin

from jumeaux.addons.final import FinalExecutor
from jumeaux.models import Report, OutputSummary, FinalAddOnPayload

logger = logging.getLogger(__name__)

class Config(OwlMixin):
    def __init__(self, table, bucket, cache_max_age=0, with_zip=True):
        self.table: str = table
        self.bucket: str = bucket
        self.cache_max_age: int = cache_max_age
        self.with_zip = with_zip

class Executor(FinalExecutor):
    def __init__(self, config: dict):
        self.config: Config = Config.from_dict(config or {})

    def exec(self, payload: FinalAddOnPayload) -> FinalAddOnPayload:
        report: Report = payload.report
        output_summary: OutputSummary = payload.output_summary

        # dynamo
        dynamodb = boto3.resource('dynamodb')
        table = dynamodb.Table(self.config.table)
        item = {
            "hashkey": report.key,
            "title": report.title,
            "one_host": report.summary.one.host,
            "other_host": report.summary.other.host,
            "paths": set(report.summary.paths),
            "same_count": Decimal(report.summary.status.same),
            "different_count": Decimal(report.summary.status.different),
            "failure_count": Decimal(report.summary.status.failure),
            "begin_time": report.summary.time.start,
            "end_time": report.summary.time.end,
            "with_zip": self.config.with_zip,
            "retry_hash": report.retry_hash
        }
        table.put_item(Item=item)

        # s3
        s3 = boto3.client('s3')

        def upload_responses(which: str):
            dir = f'{output_summary.response_dir}/{report.key}'
            for file in os.listdir(f'{dir}/{which}'):
                with open(f'{dir}/{which}/{file}', 'rb') as f:
                    logger.info(f'Put {dir}/{which}/{file}')
                    s3.put_object(Bucket=self.config.bucket,
                                  Key=f'{report.key}/{which}/{file}',
                                  Body=f.read(),
                                  CacheControl=f'max-age={self.config.cache_max_age}')

        # report
        s3.put_object(Bucket=self.config.bucket,
                      Key=f'{report.key}/report.json',
                      Body=report.to_json())

        # details
        upload_responses("one")
        upload_responses("other")

        # zip (${hashkey}.zip)
        if self.config.with_zip:
            base_name = f'{output_summary.response_dir}/{report.key}'
            with open(f'{base_name}/report.json', 'w', encoding=output_summary.encoding) as f:
                f.write(report.to_pretty_json())
            shutil.make_archive(base_name, 'zip', f'{output_summary.response_dir}/{report.key}')

            zip_fullpath = f'{base_name}.zip'
            with open(zip_fullpath, 'rb') as f:
                logger.info(f'Put {zip_fullpath}')
                s3.put_object(Bucket=self.config.bucket,
                              Key=f'{report.key}/{report.key[0:6]}.zip',
                              Body=f.read(),
                              CacheControl=f'max-age={self.config.cache_max_age}')
            os.remove(zip_fullpath)

        return payload
<commit_msg>:fire: Change hash length for zip file names [aws add-on]<commit_after># -*- coding:utf-8 -*-

import logging
import shutil
from decimal import Decimal

import boto3
import os
from owlmixin import OwlMixin

from jumeaux.addons.final import FinalExecutor
from jumeaux.models import Report, OutputSummary, FinalAddOnPayload

logger = logging.getLogger(__name__)

class Config(OwlMixin):
    def __init__(self, table, bucket, cache_max_age=0, with_zip=True):
        self.table: str = table
        self.bucket: str = bucket
        self.cache_max_age: int = cache_max_age
        self.with_zip = with_zip

class Executor(FinalExecutor):
    def __init__(self, config: dict):
        self.config: Config = Config.from_dict(config or {})

    def exec(self, payload: FinalAddOnPayload) -> FinalAddOnPayload:
        report: Report = payload.report
        output_summary: OutputSummary = payload.output_summary

        # dynamo
        dynamodb = boto3.resource('dynamodb')
        table = dynamodb.Table(self.config.table)
        item = {
            "hashkey": report.key,
            "title": report.title,
            "one_host": report.summary.one.host,
            "other_host": report.summary.other.host,
            "paths": set(report.summary.paths),
            "same_count": Decimal(report.summary.status.same),
            "different_count": Decimal(report.summary.status.different),
            "failure_count": Decimal(report.summary.status.failure),
            "begin_time": report.summary.time.start,
            "end_time": report.summary.time.end,
            "with_zip": self.config.with_zip,
            "retry_hash": report.retry_hash
        }
        table.put_item(Item=item)

        # s3
        s3 = boto3.client('s3')

        def upload_responses(which: str):
            dir = f'{output_summary.response_dir}/{report.key}'
            for file in os.listdir(f'{dir}/{which}'):
                with open(f'{dir}/{which}/{file}', 'rb') as f:
                    logger.info(f'Put {dir}/{which}/{file}')
                    s3.put_object(Bucket=self.config.bucket,
                                  Key=f'{report.key}/{which}/{file}',
                                  Body=f.read(),
                                  CacheControl=f'max-age={self.config.cache_max_age}')

        # report
        s3.put_object(Bucket=self.config.bucket,
                      Key=f'{report.key}/report.json',
                      Body=report.to_json())

        # details
        upload_responses("one")
        upload_responses("other")

        # zip (${hashkey}.zip)
        if self.config.with_zip:
            base_name = f'{output_summary.response_dir}/{report.key}'
            with open(f'{base_name}/report.json', 'w', encoding=output_summary.encoding) as f:
                f.write(report.to_pretty_json())
            shutil.make_archive(base_name, 'zip', f'{output_summary.response_dir}/{report.key}')

            zip_fullpath = f'{base_name}.zip'
            with open(zip_fullpath, 'rb') as f:
                logger.info(f'Put {zip_fullpath}')
                s3.put_object(Bucket=self.config.bucket,
                              Key=f'{report.key}/{report.key[0:7]}.zip',
                              Body=f.read(),
                              CacheControl=f'max-age={self.config.cache_max_age}')
            os.remove(zip_fullpath)

        return payload
<|endoftext|>


<commit_before>from jupyter_core.paths import jupyter_config_dir, jupyter_data_dir
import os
import sys

sys.path.append(os.path.join(jupyter_data_dir(), 'extensions'))

c = get_config()
c.Exporter.template_path = [os.path.join(jupyter_data_dir(), 'templates') ]

<commit_msg>Add local path to exporter template path<commit_after>from jupyter_core.paths import jupyter_config_dir, jupyter_data_dir
import os
import sys

sys.path.append(os.path.join(jupyter_data_dir(), 'extensions'))

c = get_config()
c.Exporter.template_path = [ '.', os.path.join(jupyter_data_dir(), 'templates') ]
<|endoftext|>


<commit_before>"""ContentsManager that allows to open Rmd, py, R and ipynb files as notebooks
"""
import os
from datetime import timedelta
import nbformat
import mock
from tornado.web import HTTPError
from traitlets import Unicode, Float, Bool
from traitlets.config import Configurable

# import notebook.transutils before notebook.services.contents.filemanager #75
try:
    import notebook.transutils  # noqa
except ImportError:
    pass

from notebook.services.contents.filemanager import FileContentsManager

import jupytext
from .combine import combine_inputs_with_outputs
from .formats import check_file_version, NOTEBOOK_EXTENSIONS, \
    format_name_for_ext, parse_one_format, parse_formats

def _jupytext_writes(ext, format_name):
    def _writes(nbk, version=nbformat.NO_CONVERT, **kwargs):
        return jupytext.writes(nbk, version=version, ext=ext,
                               format_name=format_name, **kwargs)

    return _writes

def _jupytext_reads(ext, rst2md):
    def _reads(text, as_version, **kwargs):
        return jupytext.reads(text, ext=ext, rst2md=rst2md,
                              as_version=as_version, **kwargs)

    return _reads

def check_formats(formats):
    """
    Parse, validate and return notebooks extensions
    :param formats: a list of lists of notebook extensions,
    or a colon separated string of extension groups, comma separated
    :return: list of lists (groups) of notebook extensions
    """

    # Parse formats represented as strings
    if not isinstance(formats, list):
        formats = [group.split(',') for group in formats.split(';')]

    expected_format = ("Notebook metadata 'jupytext_formats' should "
                       "be a list of extension groups, like 'ipynb,Rmd'.\n"
                       "Groups can be separated with colon, for instance: "
                       "'ipynb,nb.py;script.ipynb,py'")

    validated_formats = []
    for group in formats:
        if not isinstance(group, list):
            # In early versions (0.4 and below), formats could be a list of
            # extensions. We understand this as a single group
            return check_formats([formats])

        # Return ipynb on first position (save that file first, for #63)
        has_ipynb = False
        validated_group = []
        for fmt in group:
            try:
                fmt = u'' + fmt
            except UnicodeDecodeError:
                raise ValueError('Extensions should be strings among {}'
                                 ', not {}.\n{}'
                                 .format(str(formats.NOTEBOOK_EXTENSIONS),
                                         str(fmt),
                                         expected_format))
            if fmt == '':
                continue
            fmt, _ = parse_one_format(fmt)
            if not any([fmt.endswith(ext)
                        for ext in NOTEBOOK_EXTENSIONS]):
                raise ValueError('Group extension {} contains {}, '
                                 'which does not end with either {}.\n{}'
                                 .format(str(group), fmt,
                                         str(NOTEBOOK_EXTENSIONS),
                                         expected_format))
            if fmt == '.ipynb':
                has_ipynb = True
            else:
                validated_group.append(fmt)

        if has_ipynb:
            validated_group = ['.ipynb'] + validated_group

        if validated_group:
            validated_formats.append(validated_group)

    return validated_formats

def file_fmt_ext(path):
    """
    Return file name, format (possibly .nb.py) and extension (.py)
    """
    file, ext = os.path.splitext(path)
    file, intermediate_ext = os.path.splitext(file)
    return file, intermediate_ext + ext, ext

class TextFileContentsManager(FileContentsManager, Configurable):
    """
    A FileContentsManager Class that reads and stores notebooks to classical
    Jupyter notebooks (.ipynb), R Markdown notebooks (.Rmd), Julia (.jl),
    Python (.py) or R scripts (.R)
    """

    nb_extensions = [ext for ext in NOTEBOOK_EXTENSIONS if ext != '.ipynb']

    def all_nb_extensions(self):
        """
        Notebook extensions, including ipynb
        :return:
        """
        return ['.ipynb'] + self.nb_extensions

    default_jupytext_formats = Unicode(
        u'',
        help='Save notebooks to these file extensions. '
             'Can be any of ipynb,Rmd,md,jl,py,R,nb.jl,nb.py,nb.R '
             'comma separated. If you want another format than the '
             'default one, append the format name to the extension, '
             'e.g. ipynb,py:percent to save the notebook to '
             'hydrogen/spyder/vscode compatible scripts',
        config=True)

    preferred_jupytext_formats = Unicode(
        u'',
        help='Preferred format when saving notebooks as text, per extension. '
             'Use "jl:percent,py:percent,R:percent" if you want to save '
             'Julia, Python and R scripts in the double percent format and '
             'only write "jupytext_formats": "py" in the notebook metadata.',
        config=True)

    sphinx_convert_rst2md = Bool(
        False,
        help='When opening a Sphinx Gallery script, convert the '
             'reStructuredText to markdown',
        config=True)

    outdated_text_notebook_margin = Float(
        1.0,
        help='Refuse to overwrite inputs of a ipynb notebooks with those of a'
             'text notebook when the text notebook plus margin is older than'
             'the ipynb notebook',
        config=True)

    def format_group(self, fmt, nbk=None):
        """Return the group of extensions that contains 'fmt'"""
        # Backward compatibility with nbrmd
        for key in ['nbrmd_formats', 'nbrmd_format_version']:
            if nbk and key in nbk.metadata:
                nbk.metadata[key.replace('nbrmd', 'jupytext')] = \
                    nbk.metadata.pop(key)

        jupytext_formats = ((nbk.metadata.get('jupytext_formats')
                             if nbk else None) or
                            self.default_jupytext_formats)

        try:
            jupytext_formats = check_formats(jupytext_formats)
        except ValueError as err:
            raise HTTPError(400, str(err))

        # Find group that contains the current format
        for group in jupytext_formats:
            if fmt in group:
                return group

        # No such group, but 'ipynb'? Return current fmt + 'ipynb'
        if ['.ipynb'] in jupytext_formats:
            return ['.ipynb', fmt]

        return [fmt]

    def preferred_format(self, ext):
        """Returns the preferred format for that extension"""
        for fmt_ext, format_name in \
                parse_formats(self.preferred_jupytext_formats):
            if fmt_ext == ext:
                return format_name
        return None

    def _read_notebook(self, os_path, as_version=4):
        """Read a notebook from an os path."""
        _, ext = os.path.splitext(os_path)
        if ext in self.nb_extensions:
            with mock.patch('nbformat.reads',
                            _jupytext_reads(ext, self.sphinx_convert_rst2md)):
                return super(TextFileContentsManager, self) \
                    ._read_notebook(os_path, as_version)
        else:
            return super(TextFileContentsManager, self) \
                ._read_notebook(os_path, as_version)

    def _save_notebook(self, os_path, nb):
        """Save a notebook to an os_path."""
        os_file, fmt, _ = file_fmt_ext(os_path)
        for alt_fmt in self.format_group(fmt, nb):
            os_path_fmt = os_file + alt_fmt
            self.log.info("Saving %s", os.path.basename(os_path_fmt))
            alt_ext = '.' + alt_fmt.split('.')[-1]

            if alt_ext in self.nb_extensions:
                format_name = format_name_for_ext(nb.metadata, alt_ext) or \
                              self.preferred_format(alt_ext)
                with mock.patch('nbformat.writes',
                                _jupytext_writes(alt_ext, format_name)):
                    super(TextFileContentsManager, self) \
                        ._save_notebook(os_path_fmt, nb)
            else:
                super(TextFileContentsManager, self) \
                    ._save_notebook(os_path_fmt, nb)

    def get(self, path, content=True, type=None, format=None,
            load_alternative_format=True):
        """ Takes a path for an entity and returns its model"""
        path = path.strip('/')
        nb_file, fmt, ext = file_fmt_ext(path)

        if self.exists(path) and \
                (type == 'notebook' or
                 (type is None and ext in self.all_nb_extensions())):
            model = self._notebook_model(path, content=content)
            if not content:
                return model

            if not load_alternative_format:
                return model

            fmt_group = self.format_group(fmt, model['content'])

            source_format = fmt
            outputs_format = fmt

            # Source format is first non ipynb format found on disk
            if fmt.endswith('.ipynb'):
                for alt_fmt in fmt_group:
                    if not alt_fmt.endswith('.ipynb') and \
                            self.exists(nb_file + alt_fmt):
                        source_format = alt_fmt
                        break
            # Outputs taken from ipynb if in group, if file exists
            else:
                for alt_fmt in fmt_group:
                    if alt_fmt.endswith('.ipynb') and \
                            self.exists(nb_file + alt_fmt):
                        outputs_format = alt_fmt
                        break

            if source_format == outputs_format:
                return model

            if source_format != fmt:
                self.log.info(u'Reading SOURCE from {}'.format(
                    os.path.basename(nb_file + source_format)))
                model_outputs = model
                model = self.get(nb_file + source_format, content=content,
                                 type=type, format=format,
                                 load_alternative_format=False)
            elif outputs_format != fmt:
                self.log.info(u'Reading OUTPUTS from {}'.format(
                    os.path.basename(nb_file + outputs_format)))
                model_outputs = self.get(nb_file + outputs_format,
                                         content=content,
                                         type=type, format=format,
                                         load_alternative_format=False)

            try:
                check_file_version(model['content'],
                                   nb_file + source_format,
                                   nb_file + outputs_format)
            except ValueError as err:
                raise HTTPError(400, str(err))

            # Make sure we're not overwriting ipynb cells with an outdated
            # text file
            try:
                if model_outputs and model_outputs['last_modified'] > \
                        model['last_modified'] + \
                        timedelta(seconds=self.outdated_text_notebook_margin):
                    raise HTTPError(
                        400,
                        u'\n'
                        '{out} (last modified {out_last})\n'
                        'seems more recent than '
                        '{src} (last modified {src_last})\n'
                        'Please either:\n'
                        '- open {src} in a text editor, make sure it is '
                        'up to date, and save it,\n'
                        '- or delete {src} if not up to date,\n'
                        '- or increase check margin by adding, say, \n'
                        '    c.ContentsManager.'
                        'outdated_text_notebook_margin = 5 '
                        '# in seconds # or float("inf")\n'
                        'to your .jupyter/jupyter_notebook_config.py '
                        'file\n'.format(src=nb_file + source_format,
                                        src_last=model['last_modified'],
                                        out=nb_file + outputs_format,
                                        out_last=model_outputs[
                                            'last_modified']))
            except OverflowError:
                pass

            if model_outputs:
                combine_inputs_with_outputs(model['content'],
                                            model_outputs['content'])
            elif not fmt.endswith('.ipynb'):
                self.notary.sign(model['content'])
                self.mark_trusted_cells(model['content'], path)

            return model

        return super(TextFileContentsManager, self) \
            .get(path, content, type, format)

    def trust_notebook(self, path):
        """Trust the current notebook"""
        file, fmt, _ = file_fmt_ext(path)
        for alt_fmt in self.format_group(fmt):
            if alt_fmt.endswith('.ipynb'):
                super(TextFileContentsManager, self) \
                    .trust_notebook(file + alt_fmt)

    def rename_file(self, old_path, new_path):
        """Rename the current notebook, as well as its
         alternative representations"""
        old_file, org_fmt, _ = file_fmt_ext(old_path)
        new_file, new_fmt, _ = file_fmt_ext(new_path)

        if org_fmt == new_fmt:
            for alt_fmt in self.format_group(org_fmt):
                if self.file_exists(old_file + alt_fmt):
                    super(TextFileContentsManager, self) \
                        .rename_file(old_file + alt_fmt, new_file + alt_fmt)
        else:
            super(TextFileContentsManager, self) \
                .rename_file(old_path, new_path)
<commit_msg>Revert to previous state<commit_after>"""ContentsManager that allows to open Rmd, py, R and ipynb files as notebooks
"""
import os
from datetime import timedelta
import nbformat
import mock
from tornado.web import HTTPError
from traitlets import Unicode, Float, Bool
from traitlets.config import Configurable

# import notebook.transutils before notebook.services.contents.filemanager #75
try:
    import notebook.transutils  # noqa
except ImportError:
    pass

from notebook.services.contents.filemanager import FileContentsManager

import jupytext
from .combine import combine_inputs_with_outputs
from .formats import check_file_version, NOTEBOOK_EXTENSIONS, \
    format_name_for_ext, parse_one_format, parse_formats

def _jupytext_writes(ext, format_name):
    def _writes(nbk, version=nbformat.NO_CONVERT, **kwargs):
        return jupytext.writes(nbk, version=version, ext=ext,
                               format_name=format_name, **kwargs)

    return _writes

def _jupytext_reads(ext, rst2md):
    def _reads(text, as_version, **kwargs):
        return jupytext.reads(text, ext=ext, rst2md=rst2md,
                              as_version=as_version, **kwargs)

    return _reads

def check_formats(formats):
    """
    Parse, validate and return notebooks extensions
    :param formats: a list of lists of notebook extensions,
    or a colon separated string of extension groups, comma separated
    :return: list of lists (groups) of notebook extensions
    """

    # Parse formats represented as strings
    if not isinstance(formats, list):
        formats = [group.split(',') for group in formats.split(';')]

    expected_format = ("Notebook metadata 'jupytext_formats' should "
                       "be a list of extension groups, like 'ipynb,Rmd'.\n"
                       "Groups can be separated with colon, for instance: "
                       "'ipynb,nb.py;script.ipynb,py'")

    validated_formats = []
    for group in formats:
        if not isinstance(group, list):
            # In early versions (0.4 and below), formats could be a list of
            # extensions. We understand this as a single group
            return check_formats([formats])

        # Return ipynb on first position (save that file first, for #63)
        has_ipynb = False
        validated_group = []
        for fmt in group:
            try:
                fmt = u'' + fmt
            except UnicodeDecodeError:
                raise ValueError('Extensions should be strings among {}'
                                 ', not {}.\n{}'
                                 .format(str(formats.NOTEBOOK_EXTENSIONS),
                                         str(fmt),
                                         expected_format))
            if fmt == '':
                continue
            fmt, _ = parse_one_format(fmt)
            if not any([fmt.endswith(ext)
                        for ext in NOTEBOOK_EXTENSIONS]):
                raise ValueError('Group extension {} contains {}, '
                                 'which does not end with either {}.\n{}'
                                 .format(str(group), fmt,
                                         str(NOTEBOOK_EXTENSIONS),
                                         expected_format))
            if fmt == '.ipynb':
                has_ipynb = True
            else:
                validated_group.append(fmt)

        if has_ipynb:
            validated_group = ['.ipynb'] + validated_group

        if validated_group:
            validated_formats.append(validated_group)

    return validated_formats

def file_fmt_ext(path):
    """
    Return file name, format (possibly .nb.py) and extension (.py)
    """
    file, ext = os.path.splitext(path)
    file, intermediate_ext = os.path.splitext(file)
    return file, intermediate_ext + ext, ext

class TextFileContentsManager(FileContentsManager, Configurable):
    """
    A FileContentsManager Class that reads and stores notebooks to classical
    Jupyter notebooks (.ipynb), R Markdown notebooks (.Rmd), Julia (.jl),
    Python (.py) or R scripts (.R)
    """

    nb_extensions = [ext for ext in NOTEBOOK_EXTENSIONS if ext != '.ipynb']

    def all_nb_extensions(self):
        """
        Notebook extensions, including ipynb
        :return:
        """
        return ['.ipynb'] + self.nb_extensions

    default_jupytext_formats = Unicode(
        u'',
        help='Save notebooks to these file extensions. '
             'Can be any of ipynb,Rmd,md,jl,py,R,nb.jl,nb.py,nb.R '
             'comma separated. If you want another format than the '
             'default one, append the format name to the extension, '
             'e.g. ipynb,py:percent to save the notebook to '
             'hydrogen/spyder/vscode compatible scripts',
        config=True)

    preferred_jupytext_formats = Unicode(
        u'',
        help='Preferred format when saving notebooks as text, per extension. '
             'Use "jl:percent,py:percent,R:percent" if you want to save '
             'Julia, Python and R scripts in the double percent format and '
             'only write "jupytext_formats": "py" in the notebook metadata.',
        config=True)

    sphinx_convert_rst2md = Bool(
        False,
        help='When opening a Sphinx Gallery script, convert the '
             'reStructuredText to markdown',
        config=True)

    outdated_text_notebook_margin = Float(
        1.0,
        help='Refuse to overwrite inputs of a ipynb notebooks with those of a'
             'text notebook when the text notebook plus margin is older than'
             'the ipynb notebook',
        config=True)

    def format_group(self, fmt, nbk=None):
        """Return the group of extensions that contains 'fmt'"""
        # Backward compatibility with nbrmd
        for key in ['nbrmd_formats', 'nbrmd_format_version']:
            if nbk and key in nbk.metadata:
                nbk.metadata[key.replace('nbrmd', 'jupytext')] = \
                    nbk.metadata.pop(key)

        jupytext_formats = ((nbk.metadata.get('jupytext_formats')
                             if nbk else None) or
                            self.default_jupytext_formats)

        try:
            jupytext_formats = check_formats(jupytext_formats)
        except ValueError as err:
            raise HTTPError(400, str(err))

        # Find group that contains the current format
        for group in jupytext_formats:
            if fmt in group:
                return group

        # No such group, but 'ipynb'? Return current fmt + 'ipynb'
        if ['.ipynb'] in jupytext_formats:
            return ['.ipynb', fmt]

        return [fmt]

    def preferred_format(self, ext):
        """Returns the preferred format for that extension"""
        for fmt_ext, format_name in \
                parse_formats(self.preferred_jupytext_formats):
            if fmt_ext == ext:
                return format_name
        return None

    def _read_notebook(self, os_path, as_version=4):
        """Read a notebook from an os path."""
        _, ext = os.path.splitext(os_path)
        if ext in self.nb_extensions:
            with mock.patch('nbformat.reads',
                            _jupytext_reads(ext, self.sphinx_convert_rst2md)):
                return super(TextFileContentsManager, self) \
                    ._read_notebook(os_path, as_version)
        else:
            return super(TextFileContentsManager, self) \
                ._read_notebook(os_path, as_version)

    def _save_notebook(self, os_path, nb):
        """Save a notebook to an os_path."""
        os_file, fmt, _ = file_fmt_ext(os_path)
        for alt_fmt in self.format_group(fmt, nb):
            os_path_fmt = os_file + alt_fmt
            self.log.info("Saving %s", os.path.basename(os_path_fmt))
            alt_ext = '.' + alt_fmt.split('.')[-1]

            if alt_ext in self.nb_extensions:
                format_name = format_name_for_ext(nb.metadata, alt_ext) or \
                              self.preferred_format(alt_ext)
                with mock.patch('nbformat.writes',
                                _jupytext_writes(alt_ext, format_name)):
                    super(TextFileContentsManager, self) \
                        ._save_notebook(os_path_fmt, nb)
            else:
                super(TextFileContentsManager, self) \
                    ._save_notebook(os_path_fmt, nb)

    def get(self, path, content=True, type=None, format=None,
            load_alternative_format=True):
        """ Takes a path for an entity and returns its model"""
        path = path.strip('/')
        nb_file, fmt, ext = file_fmt_ext(path)

        if self.exists(path) and \
                (type == 'notebook' or
                 (type is None and ext in self.all_nb_extensions())):
            model = self._notebook_model(path, content=content)
            if not content:
                return model

            if not load_alternative_format:
                return model

            fmt_group = self.format_group(fmt, model['content'])

            source_format = fmt
            outputs_format = fmt

            # Source format is first non ipynb format found on disk
            if fmt.endswith('.ipynb'):
                for alt_fmt in fmt_group:
                    if not alt_fmt.endswith('.ipynb') and \
                            self.exists(nb_file + alt_fmt):
                        source_format = alt_fmt
                        break
            # Outputs taken from ipynb if in group, if file exists
            else:
                for alt_fmt in fmt_group:
                    if alt_fmt.endswith('.ipynb') and \
                            self.exists(nb_file + alt_fmt):
                        outputs_format = alt_fmt
                        break

            if source_format != fmt:
                self.log.info(u'Reading SOURCE from {}'.format(
                    os.path.basename(nb_file + source_format)))
                model_outputs = model
                model = self.get(nb_file + source_format, content=content,
                                 type=type, format=format,
                                 load_alternative_format=False)
            elif outputs_format != fmt:
                self.log.info(u'Reading OUTPUTS from {}'.format(
                    os.path.basename(nb_file + outputs_format)))
                model_outputs = self.get(nb_file + outputs_format,
                                         content=content,
                                         type=type, format=format,
                                         load_alternative_format=False)
            else:
                model_outputs = None

            try:
                check_file_version(model['content'],
                                   nb_file + source_format,
                                   nb_file + outputs_format)
            except ValueError as err:
                raise HTTPError(400, str(err))

            # Make sure we're not overwriting ipynb cells with an outdated
            # text file
            try:
                if model_outputs and model_outputs['last_modified'] > \
                        model['last_modified'] + \
                        timedelta(seconds=self.outdated_text_notebook_margin):
                    raise HTTPError(
                        400,
                        u'\n'
                        '{out} (last modified {out_last})\n'
                        'seems more recent than '
                        '{src} (last modified {src_last})\n'
                        'Please either:\n'
                        '- open {src} in a text editor, make sure it is '
                        'up to date, and save it,\n'
                        '- or delete {src} if not up to date,\n'
                        '- or increase check margin by adding, say, \n'
                        '    c.ContentsManager.'
                        'outdated_text_notebook_margin = 5 '
                        '# in seconds # or float("inf")\n'
                        'to your .jupyter/jupyter_notebook_config.py '
                        'file\n'.format(src=nb_file + source_format,
                                        src_last=model['last_modified'],
                                        out=nb_file + outputs_format,
                                        out_last=model_outputs[
                                            'last_modified']))
            except OverflowError:
                pass

            if model_outputs:
                combine_inputs_with_outputs(model['content'],
                                            model_outputs['content'])
            elif not fmt.endswith('.ipynb'):
                self.notary.sign(model['content'])
                self.mark_trusted_cells(model['content'], path)

            return model

        return super(TextFileContentsManager, self) \
            .get(path, content, type, format)

    def trust_notebook(self, path):
        """Trust the current notebook"""
        file, fmt, _ = file_fmt_ext(path)
        for alt_fmt in self.format_group(fmt):
            if alt_fmt.endswith('.ipynb'):
                super(TextFileContentsManager, self) \
                    .trust_notebook(file + alt_fmt)

    def rename_file(self, old_path, new_path):
        """Rename the current notebook, as well as its
         alternative representations"""
        old_file, org_fmt, _ = file_fmt_ext(old_path)
        new_file, new_fmt, _ = file_fmt_ext(new_path)

        if org_fmt == new_fmt:
            for alt_fmt in self.format_group(org_fmt):
                if self.file_exists(old_file + alt_fmt):
                    super(TextFileContentsManager, self) \
                        .rename_file(old_file + alt_fmt, new_file + alt_fmt)
        else:
            super(TextFileContentsManager, self) \
                .rename_file(old_path, new_path)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import os
import subprocess
import sys

import pyauto_functional  # Must be imported before pyauto
import pyauto
import pyauto_errors
import test_utils

sys.path.append('/usr/local')  # To make autotest libs importable.
from autotest.cros import cros_ui
from autotest.cros import cryptohome

class ChromeosLogin(pyauto.PyUITest):
  """TestCases for Logging into ChromeOS."""

  assert os.geteuid() == 0, 'Need to run this test as root'

  def ShouldAutoLogin(self):
    return False

  def setUp(self):
    # We want a clean session_manager instance for every run,
    # so restart ui now.
    cros_ui.stop(allow_fail=True)
    cryptohome.remove_all_vaults()
    cros_ui.start(wait_for_login_prompt=False)
    pyauto.PyUITest.setUp(self)

  def _ValidCredentials(self, account_type='test_google_account'):
    """Obtains a valid username and password from a data file.

    Returns:
      A dictionary with the keys 'username' and 'password'
    """
    return self.GetPrivateInfo()[account_type]

  def testExecuteJavascriptInOOBEWebUI(self):
    """Test that javascript can be executed at the login page."""
    msg = 'test success'
    ret = self.ExecuteJavascriptInOOBEWebUI(
              'window.domAutomationController.send("%s");' % msg)
    self.assertEqual(ret, msg)

  def testGoodLogin(self):
    """Test that login is successful with valid credentials."""
    credentials = self._ValidCredentials()
    self.Login(credentials['username'], credentials['password'])
    login_info = self.GetLoginInfo()
    self.assertTrue(login_info['is_logged_in'], msg='Login failed.')

  def testBadUsername(self):
    """Test that login fails when passed an invalid username."""
    self.assertRaises(
        pyauto_errors.JSONInterfaceError,
        lambda: self.Login('doesnotexist@fakedomain.org', 'badpassword'))
    login_info = self.GetLoginInfo()
    self.assertFalse(login_info['is_logged_in'],
                     msg='Login succeeded, with bad credentials.')

  def testBadPassword(self):
    """Test that login fails when passed an invalid password."""
    credentials = self._ValidCredentials()
    self.assertRaises(
        pyauto_errors.JSONInterfaceError,
        lambda: self.Login(credentials['username'], 'badpassword'))
    login_info = self.GetLoginInfo()
    self.assertFalse(login_info['is_logged_in'],
                     msg='Login succeeded, with bad credentials.')

  def testLoginAsGuest(self):
    """Test we can login with guest mode."""
    self.LoginAsGuest()
    login_info = self.GetLoginInfo()
    self.assertTrue(login_info['is_logged_in'], msg='Not logged in at all.')
    self.assertTrue(login_info['is_guest'], msg='Not logged in as guest.')

  def testLockScreenAfterLogin(self):
    """Test after logging in that the screen can be locked."""
    self.testGoodLogin()
    self.assertFalse(self.GetLoginInfo()['is_screen_locked'],
                     msg='Screen is locked, but the screen was not locked.')
    self.LockScreen()
    login_info = self.GetLoginInfo()
    self.assertTrue(login_info['is_screen_locked'], msg='The screen is not '
                    'locked after attempting to lock the screen.')

  def testLockAndUnlockScreenAfterLogin(self):
    """Test locking and unlocking the screen after logging in."""
    self.testLockScreenAfterLogin()
    self.UnlockScreen(self._ValidCredentials()['password'])
    login_info = self.GetLoginInfo()
    self.assertFalse(login_info['is_screen_locked'],
                     msg='Screen is locked, but it should have been unlocked.')

  def testLockAndUnlockScreenAfterLoginWithBadPassword(self):
    """Test locking and unlocking the screen with the wrong password."""
    self.testLockScreenAfterLogin()
    self.UnlockScreen('not_the_right_password')
    login_info = self.GetLoginInfo()
    self.assertTrue(login_info['is_screen_locked'],
                     msg='Screen is unlock, but it should have been unlocked '
                         'since we attempted to unlock with a bad password')

  def testLoginToCreateNewAccount(self):
    """Test we can login as a guest and create a new account."""
    self.ShowCreateAccountUI()
    # The login hook does not wait for the first tab to load, so we wait here.
    self.assertTrue(
      self.WaitUntil(self.GetActiveTabTitle, expect_retval='Google Accounts'),
                     msg='Could not verify that the Accounts tab was opened.')
    login_info = self.GetLoginInfo()
    self.assertTrue(login_info['is_guest'], msg='Not logged in as guest.')

  def testGoodLoginForTransitionedDomainAccount(self):
    """Test that login is successful with valid credentials for a domain.

    ChromeOS only allows GA+ accounts to login, there are also known as
    transitioned accounts.

    """
    credentials = self._ValidCredentials(account_type='test_domain_account')
    self.Login(credentials['username'], credentials['password'])
    login_info = self.GetLoginInfo()
    self.assertTrue(login_info['is_logged_in'], msg='Login failed.')

  def testCachedCredentials(self):
    """Test that we can login without connectivity if we have so before."""
    self.testGoodLogin()
    self.Logout()
    self.SetProxySettingsOnChromeOS('singlehttp', '10.10.10.10')
    self.testGoodLogin()
    # Reset back to direct proxy
    self.SetProxySettingsOnChromeOS('type', self.PROXY_TYPE_DIRECT)

  def testNavigateAfterLogin(self):
    """Test that page navigation is successful after logging in."""
    self.testGoodLogin()
    self.NavigateToURL("http://www.google.com")
    self.assertEqual(self.GetActiveTabTitle(), 'Google',
                     msg='Unable to navigate to Google and verify tab title.')

  def testSigningOutFromLockedScreen(self):
    """Test logout can be performed from the lock screen."""
    self.testLockScreenAfterLogin()
    self.SignoutInScreenLocker()
    self.assertFalse(self.GetLoginInfo()['is_logged_in'],
                     msg='Still logged in when we should be logged out.')

  def testLoginSequenceSanity(self):
    """Test that the interface can maintain a connection after multiple logins.

    This test is to verify the stability of the automation interface.

    """
    self.testGoodLogin()
    self.Logout()
    self.testBadPassword()
    self.testLoginAsGuest()
    self.Logout()
    self.testLoginToCreateNewAccount()

  def testLogoutWithNoWindows(self):
    """Verify logout when no browser windows are present."""
    self.testGoodLogin()
    for i in range(5):
      self.OpenNewBrowserWindow(True)
    for _ in range(self.GetBrowserWindowCount()):
      self.CloseBrowserWindow(0)
    self.assertEqual(0, self.GetBrowserWindowCount(),
                     msg='Could not close all browser windows')
    self.Logout()
    self.testGoodLogin()

  def testInitialLoginState(self):
    """Verify basic state of browser windows at initial login."""
    self.testGoodLogin()
    # Should have 1 browser window with 1 tab.
    info = self.GetBrowserInfo()
    self.assertEqual(1, len(info['windows']))
    self.assertFalse(info['windows'][0]['incognito'],
        msg='Did not expect incognito window after login')
    self.assertEqual(1, len(info['windows'][0]['tabs']))

    self.OpenNewBrowserWindow(True)
    # Should have 2 regular browser windows.
    info = self.GetBrowserInfo()
    self.assertEqual(2, len(info['windows']))
    self.assertFalse(info['windows'][0]['incognito'])
    self.assertFalse(info['windows'][1]['incognito'],
        msg='Expected a regular new window.')

  def testProfilePreservedBetweenLogins(self):
    """Verify that profile is preserved between two login sessions.

    Also verify Local State.
    """
    self.testGoodLogin()

    # Build up some history and setup state in "Local State".
    url = self.GetHttpURLForDataPath('title2.html')
    self.NavigateToURL(url)
    open('/home/chronos/__magic__', 'w').close()
    open('/home/chronos/user/__magic__', 'w').close()

    def _VerifyProfile():
      history = self.GetHistoryInfo().History()
      self.assertEqual(1, len(history))
      self.assertEqual(url, history[0]['url'])
      self.assertTrue(os.path.exists('/home/chronos/__magic__'),
          msg='/home/chronos/__magic__ did not persist across login sessions')
      self.assertTrue(os.path.exists('/home/chronos/user/__magic__'),
          msg='/home/chronos/user/__magic__ did not persist across '
              'login sessions')

    _VerifyProfile()
    self.Logout()
    self.testGoodLogin()  # Re-login with same account.
    _VerifyProfile()

  def testGuestCrosh(self):
    """Verify we can use crosh in guest mode."""
    self.LoginAsGuest()
    login_info = self.GetLoginInfo()
    self.assertTrue(login_info['is_logged_in'], msg='Not logged in at all.')
    self.assertTrue(login_info['is_guest'], msg='Not logged in as guest.')
    for _ in range(self.GetBrowserWindowCount()):
      self.CloseBrowserWindow(0)
    test_utils.OpenCroshVerification(self)

    # Verify crosh prompt.
    self.WaitForHtermText(text='crosh> ',
        msg='Could not find "crosh> " prompt')
    self.assertTrue(
        self.GetHtermRowsText(start=0, end=2).endswith('crosh> '),
        msg='Could not find "crosh> " prompt')

    # Run a crosh command.
    self.SendKeysToHterm('help\\n')
    self.WaitForHtermText(text='help_advanced',
        msg='Could not find "help_advanced" in help output.')

    # Exit crosh and close tab.
    self.SendKeysToHterm('exit\\n')
    self.WaitForHtermText(text='command crosh completed with exit code 0',
        msg='Could not exit crosh.')

if __name__ == '__main__':
  pyauto_functional.Main()
<commit_msg>[chromeos] Make testProfilePreservedBetweenLogins test robust<commit_after>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import os
import subprocess
import sys

import pyauto_functional  # Must be imported before pyauto
import pyauto
import pyauto_errors
import test_utils

sys.path.append('/usr/local')  # To make autotest libs importable.
from autotest.cros import cros_ui
from autotest.cros import cryptohome

class ChromeosLogin(pyauto.PyUITest):
  """TestCases for Logging into ChromeOS."""

  assert os.geteuid() == 0, 'Need to run this test as root'

  def ShouldAutoLogin(self):
    return False

  def setUp(self):
    # We want a clean session_manager instance for every run,
    # so restart ui now.
    cros_ui.stop(allow_fail=True)
    cryptohome.remove_all_vaults()
    cros_ui.start(wait_for_login_prompt=False)
    pyauto.PyUITest.setUp(self)

  def _ValidCredentials(self, account_type='test_google_account'):
    """Obtains a valid username and password from a data file.

    Returns:
      A dictionary with the keys 'username' and 'password'
    """
    return self.GetPrivateInfo()[account_type]

  def testExecuteJavascriptInOOBEWebUI(self):
    """Test that javascript can be executed at the login page."""
    msg = 'test success'
    ret = self.ExecuteJavascriptInOOBEWebUI(
              'window.domAutomationController.send("%s");' % msg)
    self.assertEqual(ret, msg)

  def testGoodLogin(self):
    """Test that login is successful with valid credentials."""
    credentials = self._ValidCredentials()
    self.Login(credentials['username'], credentials['password'])
    login_info = self.GetLoginInfo()
    self.assertTrue(login_info['is_logged_in'], msg='Login failed.')

  def testBadUsername(self):
    """Test that login fails when passed an invalid username."""
    self.assertRaises(
        pyauto_errors.JSONInterfaceError,
        lambda: self.Login('doesnotexist@fakedomain.org', 'badpassword'))
    login_info = self.GetLoginInfo()
    self.assertFalse(login_info['is_logged_in'],
                     msg='Login succeeded, with bad credentials.')

  def testBadPassword(self):
    """Test that login fails when passed an invalid password."""
    credentials = self._ValidCredentials()
    self.assertRaises(
        pyauto_errors.JSONInterfaceError,
        lambda: self.Login(credentials['username'], 'badpassword'))
    login_info = self.GetLoginInfo()
    self.assertFalse(login_info['is_logged_in'],
                     msg='Login succeeded, with bad credentials.')

  def testLoginAsGuest(self):
    """Test we can login with guest mode."""
    self.LoginAsGuest()
    login_info = self.GetLoginInfo()
    self.assertTrue(login_info['is_logged_in'], msg='Not logged in at all.')
    self.assertTrue(login_info['is_guest'], msg='Not logged in as guest.')

  def testLockScreenAfterLogin(self):
    """Test after logging in that the screen can be locked."""
    self.testGoodLogin()
    self.assertFalse(self.GetLoginInfo()['is_screen_locked'],
                     msg='Screen is locked, but the screen was not locked.')
    self.LockScreen()
    login_info = self.GetLoginInfo()
    self.assertTrue(login_info['is_screen_locked'], msg='The screen is not '
                    'locked after attempting to lock the screen.')

  def testLockAndUnlockScreenAfterLogin(self):
    """Test locking and unlocking the screen after logging in."""
    self.testLockScreenAfterLogin()
    self.UnlockScreen(self._ValidCredentials()['password'])
    login_info = self.GetLoginInfo()
    self.assertFalse(login_info['is_screen_locked'],
                     msg='Screen is locked, but it should have been unlocked.')

  def testLockAndUnlockScreenAfterLoginWithBadPassword(self):
    """Test locking and unlocking the screen with the wrong password."""
    self.testLockScreenAfterLogin()
    self.UnlockScreen('not_the_right_password')
    login_info = self.GetLoginInfo()
    self.assertTrue(login_info['is_screen_locked'],
                     msg='Screen is unlock, but it should have been unlocked '
                         'since we attempted to unlock with a bad password')

  def testLoginToCreateNewAccount(self):
    """Test we can login as a guest and create a new account."""
    self.ShowCreateAccountUI()
    # The login hook does not wait for the first tab to load, so we wait here.
    self.assertTrue(
      self.WaitUntil(self.GetActiveTabTitle, expect_retval='Google Accounts'),
                     msg='Could not verify that the Accounts tab was opened.')
    login_info = self.GetLoginInfo()
    self.assertTrue(login_info['is_guest'], msg='Not logged in as guest.')

  def testGoodLoginForTransitionedDomainAccount(self):
    """Test that login is successful with valid credentials for a domain.

    ChromeOS only allows GA+ accounts to login, there are also known as
    transitioned accounts.

    """
    credentials = self._ValidCredentials(account_type='test_domain_account')
    self.Login(credentials['username'], credentials['password'])
    login_info = self.GetLoginInfo()
    self.assertTrue(login_info['is_logged_in'], msg='Login failed.')

  def testCachedCredentials(self):
    """Test that we can login without connectivity if we have so before."""
    self.testGoodLogin()
    self.Logout()
    self.SetProxySettingsOnChromeOS('singlehttp', '10.10.10.10')
    self.testGoodLogin()
    # Reset back to direct proxy
    self.SetProxySettingsOnChromeOS('type', self.PROXY_TYPE_DIRECT)

  def testNavigateAfterLogin(self):
    """Test that page navigation is successful after logging in."""
    self.testGoodLogin()
    self.NavigateToURL("http://www.google.com")
    self.assertEqual(self.GetActiveTabTitle(), 'Google',
                     msg='Unable to navigate to Google and verify tab title.')

  def testSigningOutFromLockedScreen(self):
    """Test logout can be performed from the lock screen."""
    self.testLockScreenAfterLogin()
    self.SignoutInScreenLocker()
    self.assertFalse(self.GetLoginInfo()['is_logged_in'],
                     msg='Still logged in when we should be logged out.')

  def testLoginSequenceSanity(self):
    """Test that the interface can maintain a connection after multiple logins.

    This test is to verify the stability of the automation interface.

    """
    self.testGoodLogin()
    self.Logout()
    self.testBadPassword()
    self.testLoginAsGuest()
    self.Logout()
    self.testLoginToCreateNewAccount()

  def testLogoutWithNoWindows(self):
    """Verify logout when no browser windows are present."""
    self.testGoodLogin()
    for i in range(5):
      self.OpenNewBrowserWindow(True)
    for _ in range(self.GetBrowserWindowCount()):
      self.CloseBrowserWindow(0)
    self.assertEqual(0, self.GetBrowserWindowCount(),
                     msg='Could not close all browser windows')
    self.Logout()
    self.testGoodLogin()

  def testInitialLoginState(self):
    """Verify basic state of browser windows at initial login."""
    self.testGoodLogin()
    # Should have 1 browser window with 1 tab.
    info = self.GetBrowserInfo()
    self.assertEqual(1, len(info['windows']))
    self.assertFalse(info['windows'][0]['incognito'],
        msg='Did not expect incognito window after login')
    self.assertEqual(1, len(info['windows'][0]['tabs']))

    self.OpenNewBrowserWindow(True)
    # Should have 2 regular browser windows.
    info = self.GetBrowserInfo()
    self.assertEqual(2, len(info['windows']))
    self.assertFalse(info['windows'][0]['incognito'])
    self.assertFalse(info['windows'][1]['incognito'],
        msg='Expected a regular new window.')

  def testProfilePreservedBetweenLogins(self):
    """Verify that profile is preserved between two login sessions.

    Also verify Local State.
    """
    self.testGoodLogin()

    # Build up some history and setup state in "Local State".
    url = self.GetHttpURLForDataPath('title2.html')
    self.NavigateToURL(url)
    # chromeos often takes a while to register URLs into history.
    self.assertTrue(self.WaitUntil(lambda: self.GetHistoryInfo().History()),
                    msg='Could not open %s successfully' % url)
    open('/home/chronos/__magic__', 'w').close()
    open('/home/chronos/user/__magic__', 'w').close()

    def _VerifyProfile():
      history = self.GetHistoryInfo().History()
      self.assertEqual(1, len(history))
      self.assertEqual(url, history[0]['url'])
      self.assertTrue(os.path.exists('/home/chronos/__magic__'),
          msg='/home/chronos/__magic__ did not persist across login sessions')
      self.assertTrue(os.path.exists('/home/chronos/user/__magic__'),
          msg='/home/chronos/user/__magic__ did not persist across '
              'login sessions')

    _VerifyProfile()
    self.Logout()
    self.testGoodLogin()  # Re-login with same account.
    _VerifyProfile()

  def testGuestCrosh(self):
    """Verify we can use crosh in guest mode."""
    self.LoginAsGuest()
    login_info = self.GetLoginInfo()
    self.assertTrue(login_info['is_logged_in'], msg='Not logged in at all.')
    self.assertTrue(login_info['is_guest'], msg='Not logged in as guest.')
    for _ in range(self.GetBrowserWindowCount()):
      self.CloseBrowserWindow(0)
    test_utils.OpenCroshVerification(self)

    # Verify crosh prompt.
    self.WaitForHtermText(text='crosh> ',
        msg='Could not find "crosh> " prompt')
    self.assertTrue(
        self.GetHtermRowsText(start=0, end=2).endswith('crosh> '),
        msg='Could not find "crosh> " prompt')

    # Run a crosh command.
    self.SendKeysToHterm('help\\n')
    self.WaitForHtermText(text='help_advanced',
        msg='Could not find "help_advanced" in help output.')

    # Exit crosh and close tab.
    self.SendKeysToHterm('exit\\n')
    self.WaitForHtermText(text='command crosh completed with exit code 0',
        msg='Could not exit crosh.')

if __name__ == '__main__':
  pyauto_functional.Main()
<|endoftext|>


<commit_before>import numpy as np
from keras.layers import Convolution1D, MaxPooling1D, Flatten, Dense
from keras.models import Sequential
from nltk import word_tokenize

from utils.classification_exceptions import ModelNotTrainedException
import utils.kerasmodel_io as kerasio

# ref: https://gist.github.com/entron/b9bc61a74e7cadeb1fec
# ref: http://cs231n.github.io/convolutional-networks/

class CNNEmbeddedVecClassifier:
    def __init__(self,
                 wvmodel,
                 classdict,
                 n_gram,
                 vecsize=300,
                 nb_filters=1200,
                 maxlen=15):
        self.wvmodel = wvmodel
        self.classdict = classdict
        self.n_gram = n_gram
        self.vecsize = vecsize
        self.nb_filters = nb_filters
        self.maxlen = maxlen
        self.trained = False

    def convert_trainingdata_matrix(self):
        classlabels = self.classdict.keys()
        lblidx_dict = dict(zip(classlabels, range(len(classlabels))))

        # tokenize the words, and determine the word length
        phrases = []
        indices = []
        for label in classlabels:
            for shorttext in self.classdict[label]:
                category_bucket = [0]*len(classlabels)
                category_bucket[lblidx_dict[label]] = 1
                indices.append(category_bucket)
                phrases.append(word_tokenize(shorttext))
        maxlen = max(map(len, phrases))

        # store embedded vectors
        train_embedvec = np.zeros(shape=(len(phrases), maxlen, self.vecsize))
        for i in range(len(phrases)):
            for j in range(len(phrases[i])):
                train_embedvec[i, j] = self.word_to_embedvec(phrases[i][j])
        indices = np.array(indices, dtype=np.int)

        return classlabels, maxlen, train_embedvec, indices

    def train(self):
        # convert classdict to training input vectors
        self.classlabels, self.maxlen, train_embedvec, indices = self.convert_trainingdata_matrix()

        # build the model
        model = Sequential()
        model.add(Convolution1D(nb_filter=self.nb_filters,
                                filter_length=self.n_gram,
                                border_mode='valid',
                                activation='relu',
                                input_shape=(self.maxlen, self.vecsize)))
        model.add(MaxPooling1D(pool_length=self.maxlen-self.n_gram+1))
        model.add(Flatten())
        model.add(Dense(len(self.classlabels), activation='softmax'))
        model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

        # train the model
        # print train_embedvec.shape
        # print indices.shape
        model.fit(train_embedvec, indices)

        # flag switch
        self.model = model
        self.trained = True

    def savemodel(self, nameprefix):
        if not self.trained:
            raise ModelNotTrainedException()
        kerasio.save_model(nameprefix, self.model)

    def loadmodel(self, nameprefix):
        self.model = kerasio.load_model(nameprefix)
        self.trained = True

    def word_to_embedvec(self, word):
        return self.wvmodel[word] if word in self.wvmodel else np.zeros(self.vecsize)

    def score(self, shorttext):
        if not self.trained:
            raise ModelNotTrainedException()

        tokens = word_tokenize(shorttext)
        matrix = np.zeros((1, len(tokens), self.vecsize))
        for i in range(len(tokens)):
            matrix[0, i] = self.word_to_embedvec(tokens[i])

        predictions = self.model.predict(matrix)
        scoredict = {}
        for idx, classlabel in zip(range(len(self.classlabels)), self.classlabels):
            scoredict[classlabel] = predictions[0][idx]
        return scoredict<commit_msg>allow flexibility to handle variable length of words given a maximum short text length<commit_after>import numpy as np
from keras.layers import Convolution1D, MaxPooling1D, Flatten, Dense
from keras.models import Sequential
from nltk import word_tokenize

from utils.classification_exceptions import ModelNotTrainedException
import utils.kerasmodel_io as kerasio

# ref: https://gist.github.com/entron/b9bc61a74e7cadeb1fec
# ref: http://cs231n.github.io/convolutional-networks/

class CNNEmbeddedVecClassifier:
    def __init__(self,
                 wvmodel,
                 classdict,
                 n_gram,
                 vecsize=300,
                 nb_filters=1200,
                 maxlen=15):
        self.wvmodel = wvmodel
        self.classdict = classdict
        self.n_gram = n_gram
        self.vecsize = vecsize
        self.nb_filters = nb_filters
        self.maxlen = maxlen
        self.trained = False

    def convert_trainingdata_matrix(self):
        classlabels = self.classdict.keys()
        lblidx_dict = dict(zip(classlabels, range(len(classlabels))))

        # tokenize the words, and determine the word length
        phrases = []
        indices = []
        for label in classlabels:
            for shorttext in self.classdict[label]:
                category_bucket = [0]*len(classlabels)
                category_bucket[lblidx_dict[label]] = 1
                indices.append(category_bucket)
                phrases.append(word_tokenize(shorttext))

        # store embedded vectors
        train_embedvec = np.zeros(shape=(len(phrases), self.maxlen, self.vecsize))
        for i in range(len(phrases)):
            for j in range(min(self.maxlen, len(phrases[i]))):
                train_embedvec[i, j] = self.word_to_embedvec(phrases[i][j])
        indices = np.array(indices, dtype=np.int)

        return classlabels, train_embedvec, indices

    def train(self):
        # convert classdict to training input vectors
        self.classlabels, train_embedvec, indices = self.convert_trainingdata_matrix()

        # build the deep neural network model
        model = Sequential()
        model.add(Convolution1D(nb_filter=self.nb_filters,
                                filter_length=self.n_gram,
                                border_mode='valid',
                                activation='relu',
                                input_shape=(self.maxlen, self.vecsize)))
        model.add(MaxPooling1D(pool_length=self.maxlen-self.n_gram+1))
        model.add(Flatten())
        model.add(Dense(len(self.classlabels), activation='softmax'))
        model.compile(loss='categorical_crossentropy', optimizer='rmsprop')

        # train the model
        model.fit(train_embedvec, indices)

        # flag switch
        self.model = model
        self.trained = True

    def savemodel(self, nameprefix):
        if not self.trained:
            raise ModelNotTrainedException()
        kerasio.save_model(nameprefix, self.model)

    def loadmodel(self, nameprefix):
        self.model = kerasio.load_model(nameprefix)
        self.trained = True

    def word_to_embedvec(self, word):
        return self.wvmodel[word] if word in self.wvmodel else np.zeros(self.vecsize)

    def shorttext_to_matrix(self, shorttext):
        tokens = word_tokenize(shorttext)
        matrix = np.zeros((self.maxlen, self.vecsize))
        for i in range(min(self.maxlen, len(tokens))):
            matrix[i] = self.word_to_embedvec(tokens[i])
        return matrix

    def score(self, shorttext):
        if not self.trained:
            raise ModelNotTrainedException()

        # retrieve vector
        matrix = np.array([self.shorttext_to_matrix(shorttext)])

        # classification using the neural network
        predictions = self.model.predict(matrix)

        # wrangle output result
        scoredict = {}
        for idx, classlabel in zip(range(len(self.classlabels)), self.classlabels):
            scoredict[classlabel] = predictions[0][idx]
        return scoredict<|endoftext|>


<commit_before>"""Sublime Text plugin to open files changed in the current branch.

This plugin assumes that a sublime-project file is open, and assumes that the
root directory is the first directory in the project whose last element is
'src'.

This plugin will figure out how many revisions ahead of the upstream branch
the current branch is, then figure out which files have changed since the
revision this was branched from on the upstream branch. It will then open a new
window, and open all those files in that window.
"""

import os
import re
import sublime
import sublime_plugin
import subprocess

class OpenChangedFilesInNewWindowCommand(sublime_plugin.WindowCommand):
  """Implements the Open Changed Files In New Window command."""

  def run(self):
    files = self.get_changed_files_since_upstream_()
    self.open_in_new_window_(files)

  def get_root_directory_(self):
    for directory in self.window.folders():
      if os.path.basename(directory) == 'src':
        print('root directory: %s' % directory)
        return directory
    raise Exception('Failed to find root directory; folders = %s' %
                    self.window.folders())

  def run_command_(self, cmd):
    cwd = self.get_root_directory_()
    process = subprocess.Popen(
        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)

    (stdout, stderr) = process.communicate()
    if process.returncode != 0:
      raise Exception('subprocess exited with code %d\ncommand = %s' %
                      (process.returncode, cmd))

    return (stdout.decode('utf-8'), stderr.decode('utf-8'))

  def get_number_of_revisions_ahead_of_upstream_(self):
    (stdout, _) = self.run_command_(
        ['git', 'rev-list', '--left-right', '--count', '@{u}...HEAD'])

    before_after = re.search(r'(\d+)\s+(\d+)', stdout)
    if not before_after:
      raise Exception('re.search returned no matches for %s' % stdout)

    return int(before_after.group(2))

  def get_changed_files_since_upstream_(self):
    revs = self.get_number_of_revisions_ahead_of_upstream_()
    (stdout, _) = self.run_command_(
        ['git', 'diff', '--name-only', 'HEAD~%d' % revs])
    return stdout.strip().split('\n')

  def open_in_new_window_(self, items):
    if not items:
      print('No files to open!')
      return

    root_directory = self.get_root_directory_()
    sublime.run_command('new_window')
    for item in items:
      sublime.active_window().open_file(os.path.join(root_directory, item))
<commit_msg>Keep directory list open.<commit_after>"""Sublime Text plugin to open files changed in the current branch.

This plugin assumes that a sublime-project file is open, and assumes that the
root directory is the first directory in the project whose last element is
'src'.

This plugin will figure out how many revisions ahead of the upstream branch
the current branch is, then figure out which files have changed since the
revision this was branched from on the upstream branch. It will then open a new
window, and open all those files in that window.
"""

import os
import re
import sublime
import sublime_plugin
import subprocess

class OpenChangedFilesInNewWindowCommand(sublime_plugin.WindowCommand):
  """Implements the Open Changed Files In New Window command."""

  def run(self):
    files = self.get_changed_files_since_upstream_()
    self.open_in_new_window_(files)

  def get_root_directory_(self):
    for directory in self.window.folders():
      if os.path.basename(directory) == 'src':
        print('root directory: %s' % directory)
        return directory
    raise Exception('Failed to find root directory; folders = %s' %
                    self.window.folders())

  def run_command_(self, cmd):
    cwd = self.get_root_directory_()
    process = subprocess.Popen(
        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)

    (stdout, stderr) = process.communicate()
    if process.returncode != 0:
      raise Exception('subprocess exited with code %d\ncommand = %s' %
                      (process.returncode, cmd))

    return (stdout.decode('utf-8'), stderr.decode('utf-8'))

  def get_number_of_revisions_ahead_of_upstream_(self):
    (stdout, _) = self.run_command_(
        ['git', 'rev-list', '--left-right', '--count', '@{u}...HEAD'])

    before_after = re.search(r'(\d+)\s+(\d+)', stdout)
    if not before_after:
      raise Exception('re.search returned no matches for %s' % stdout)

    return int(before_after.group(2))

  def get_changed_files_since_upstream_(self):
    revs = self.get_number_of_revisions_ahead_of_upstream_()
    (stdout, _) = self.run_command_(
        ['git', 'diff', '--name-only', 'HEAD~%d' % revs])
    return stdout.strip().split('\n')

  def open_in_new_window_(self, items):
    if not items:
      print('No files to open!')
      return

    current_project_data = sublime.active_window().project_data()

    root_directory = self.get_root_directory_()
    sublime.run_command('new_window')
    window = sublime.active_window()
    window.set_project_data(current_project_data)
    for item in items:
      window.open_file(os.path.join(root_directory, item))
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
#
# A script intended to be useful in helping to collect signatures for a
# release.  This is a pretty rough, and patches are welcome to improve it.
#
# Some thoughts about future improvement:
#  * Store all sigs, etc, in a database
#    - This helps with idempotence
#    - Also allows display of per-file and per-release statistics
#  * A download link on the page itself
#  * Make the textarea access multiple signatures
#  * Don't require the signer to specify which file they are signing
#    - Just check the sign against all the files for that release
#  * Make use of the python-gpg package (http://code.google.com/p/python-gnupg/)
#

import cgi
import cgitb
cgitb.enable()

import sys, os, string, subprocess, re

try:
  sys.path.append(os.path.dirname(sys.argv[0]))
  import config
except:
  print 'Content-type: text/plain'
  print
  print 'Cannot find config file'
  sys.exit(1)

r = re.compile('\[GNUPG\:\] GOODSIG (\w*) (.*)')

def files():
  for f in os.listdir(config.filesdir):
    if config.version in f and (f.endswith('.tar.gz') or f.endswith('.zip') or f.endswith('.tar.bz2')):
      yield f

shell_content = '''
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

<title>Signature collection for Subversion $version</title>

This page is used to collect signatures for the proposed release of
Apache Subversion $version.

$content

'''

def default_page():
  c = '''
<form method="post">

Paste signature in the area below:

<textarea name="signature" rows="10" cols="80"></textarea>

<input type="submit" value="Submit" />
</form>
'''

  return c

def save_valid_sig(filename, signature):
  f = open(os.path.join(config.sigdir, filename + '.asc'), 'a')
  f.write(signature)

def verify_sig_for_file(signature, filename):
  args = ['gpg', '--logger-fd', '1', '--no-tty',
          '--status-fd', '2', '--verify', '-',
          os.path.join(config.filesdir, filename)]

  gpg = subprocess.Popen(args,
                         stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)

  gpg.stdin.write(signature)
  gpg.stdin.close()

  rc = gpg.wait()
  output = gpg.stdout.read()
  status = gpg.stderr.read()

  if rc:
    return (False, status + output)

  lines = status.split('\n')
  for line in lines:
    match = r.search(line)
    if match:
      keyid = match.group(1)[-8:]
      user = match.group(2)

  return (True, (filename, keyid, user))

def verify_sig(signature):
  all_failures = ""
  for filename in files():
    (verified, result) = verify_sig_for_file(signature, filename)
    if verified:
      return (verified, result)
    else:
      all_failures += "%s:\n[[[\n%s]]]\n\n" % (filename, result)
  return (False, all_failures)

def process_sig(signature):
  c_verified = '''

The signature is verified!

Filename: %s

Key ID: %s

User: %s

This signature has been saved, and will be included as part of the
    release signatures.  Please send mail to
    dev@subversion.apache.org
    acknowledging your successful signature.

'''
  c_unverified = '''

The signature was not able to be verified!

Signature:
%s

Reason:

%s

Please talk to the release manager if this is in error.

'''

  (verified, result) = verify_sig(signature)

  if verified:
    (filename, keyid, user) = result
    save_valid_sig(filename, signature)

    # TODO: record (filename, keyid) in a database
    return c_verified % (filename, keyid, user)
  else:
    return c_unverified % (signature, result)

def main():
  print "Content-Type: text/html"
  print

  form = cgi.FieldStorage()
  if 'signature' not in form:
    content = default_page()
  else:
    content = process_sig(form['signature'].value)

  # These are "global" values, not specific to our action.
  mapping = {
      'version' : config.version,
      'content' : content,
    }

  template = string.Template(shell_content)
  print template.safe_substitute(mapping)

if __name__ == '__main__':
  main()
<commit_msg>* tools/dist/collect_sigs.py:   Another improvement thought.<commit_after>#!/usr/bin/env python
#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
#
# A script intended to be useful in helping to collect signatures for a
# release.  This is a pretty rough, and patches are welcome to improve it.
#
# Some thoughts about future improvement:
#  * Store all sigs, etc, in a database
#    - This helps with idempotence
#    - Also allows display of per-file and per-release statistics
#  * A download link on the page itself
#  * Make the textarea access multiple signatures
#  * Don't require the signer to specify which file they are signing
#    - Just check the sign against all the files for that release
#  * Make use of the python-gpg package (http://code.google.com/p/python-gnupg/)
#  * Post to IRC when a new signature is collected
#    - Since we don't want to have a long running bot, perhaps we could
#      also patch wayita to accept and then echo a privmsg?
#

import cgi
import cgitb
cgitb.enable()

import sys, os, string, subprocess, re

try:
  sys.path.append(os.path.dirname(sys.argv[0]))
  import config
except:
  print 'Content-type: text/plain'
  print
  print 'Cannot find config file'
  sys.exit(1)

r = re.compile('\[GNUPG\:\] GOODSIG (\w*) (.*)')

def files():
  for f in os.listdir(config.filesdir):
    if config.version in f and (f.endswith('.tar.gz') or f.endswith('.zip') or f.endswith('.tar.bz2')):
      yield f

shell_content = '''
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

<title>Signature collection for Subversion $version</title>

This page is used to collect signatures for the proposed release of
Apache Subversion $version.

$content

'''

def default_page():
  c = '''
<form method="post">

Paste signature in the area below:

<textarea name="signature" rows="10" cols="80"></textarea>

<input type="submit" value="Submit" />
</form>
'''

  return c

def save_valid_sig(filename, signature):
  f = open(os.path.join(config.sigdir, filename + '.asc'), 'a')
  f.write(signature)

def verify_sig_for_file(signature, filename):
  args = ['gpg', '--logger-fd', '1', '--no-tty',
          '--status-fd', '2', '--verify', '-',
          os.path.join(config.filesdir, filename)]

  gpg = subprocess.Popen(args,
                         stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)

  gpg.stdin.write(signature)
  gpg.stdin.close()

  rc = gpg.wait()
  output = gpg.stdout.read()
  status = gpg.stderr.read()

  if rc:
    return (False, status + output)

  lines = status.split('\n')
  for line in lines:
    match = r.search(line)
    if match:
      keyid = match.group(1)[-8:]
      user = match.group(2)

  return (True, (filename, keyid, user))

def verify_sig(signature):
  all_failures = ""
  for filename in files():
    (verified, result) = verify_sig_for_file(signature, filename)
    if verified:
      return (verified, result)
    else:
      all_failures += "%s:\n[[[\n%s]]]\n\n" % (filename, result)
  return (False, all_failures)

def process_sig(signature):
  c_verified = '''

The signature is verified!

Filename: %s

Key ID: %s

User: %s

This signature has been saved, and will be included as part of the
    release signatures.  Please send mail to
    dev@subversion.apache.org
    acknowledging your successful signature.

'''
  c_unverified = '''

The signature was not able to be verified!

Signature:
%s

Reason:

%s

Please talk to the release manager if this is in error.

'''

  (verified, result) = verify_sig(signature)

  if verified:
    (filename, keyid, user) = result
    save_valid_sig(filename, signature)

    # TODO: record (filename, keyid) in a database
    return c_verified % (filename, keyid, user)
  else:
    return c_unverified % (signature, result)

def main():
  print "Content-Type: text/html"
  print

  form = cgi.FieldStorage()
  if 'signature' not in form:
    content = default_page()
  else:
    content = process_sig(form['signature'].value)

  # These are "global" values, not specific to our action.
  mapping = {
      'version' : config.version,
      'content' : content,
    }

  template = string.Template(shell_content)
  print template.safe_substitute(mapping)

if __name__ == '__main__':
  main()
<|endoftext|>