<commit_before>from __future__ import print_function
from pprint import pprint
from os import walk
from os.path import join, exists
import re
import logging
from django.core.management.base import BaseCommand, CommandError
from devilry.utils.command import setup_logging, get_verbosity

from devilry.utils.importutils import get_staticdir_from_appname

def get_appname_from_classpath(classpath):
    if classpath.startswith('Ext.'):
        return 'extjs4'
    else:
        return classpath.split('.')[0]

def get_extjs_sourceroot_from_appname(appname):
    appdir = get_staticdir_from_appname(appname)
    if appname == 'extjs4':
        return join(appdir, 'src')
    if exists(join(appdir, 'app')):
        return join(appdir, 'app')
    else:
        return appdir

def classpath_to_ospath(unixpath):
    from os import sep
    return unixpath.replace('.', sep) + '.js'

class SimpleJsFile(object):
    MULTILINE_COMMENT_PATT = re.compile('\/\*.*?\*\/', re.MULTILINE|re.DOTALL)
    SINGLELINE_COMMENT_PATT = re.compile('\/\/.*?$', re.MULTILINE)

    def __init__(self, classname, filepath):
        self.classname = classname
        self.filepath = filepath
        filecontent = open(self.filepath, 'rb').read()
        self.filecontent = self.remove_comments(filecontent)
        self.requires = []

    def remove_comments(self, filecontent):
        filecontent = self.MULTILINE_COMMENT_PATT.sub('', filecontent)
        #filecontent = self.SINGLELINE_COMMENT_PATT.sub('', filecontent)
        return filecontent

    def __str__(self):
        return 'SIMPLE: {filepath}'.format(**self.__dict__)

    def collect_jsfiles(self, collected_jsfiles_map):
        pass

class JsFile(SimpleJsFile):
    """
    Collects information about an ExtJS javascript file, including:

        - classname (define)
        - dependencies:
            - mixins
            - model
            - requires
    """
    DEFINEPATT = re.compile(r"^Ext\.define\(.(.+?).\s*,", re.MULTILINE)
    EXTENDPATT = re.compile(r"^\s*extend\s*:\s*['\"](.+?)['\"]", re.MULTILINE|re.DOTALL)
    MODELPATT = re.compile(r"^\s*model\s*:\s*['\"](.+?)['\"]", re.MULTILINE|re.DOTALL)
    REQUIRESPATT = re.compile(r"^\s*requires\s*:\s*\[(.*?)\]", re.MULTILINE|re.DOTALL)
    LISTMIXINSPATT = re.compile(r"^\s*mixins\s*:\s*[(.*?)]", re.MULTILINE|re.DOTALL)
    OBJECTMIXINSPATT = re.compile(r"^\s*mixins\s*:\s*{(.*?)}", re.MULTILINE|re.DOTALL)
    LISTSPLITPATT = re.compile(r'[\'"](.*?)[\'"]')
    STRINGOBJECT_VALUES_PATT = re.compile(r'[\'"]?.*?[\'"]?\s*:\s*[\'"](.*?)[\'"]')
    CONTROLLERS_PATT = re.compile(r"^\s*controllers\s*:\s*\[(.*?)\]", re.MULTILINE|re.DOTALL)
    VIEWS_PATT = re.compile(r"^\s*views\s*:\s*\[(.*?)\]", re.MULTILINE|re.DOTALL)
    MODELS_PATT = re.compile(r"^\s*models\s*:\s*\[(.*?)\]", re.MULTILINE|re.DOTALL)
    STORES_PATT = re.compile(r"^\s*stores\s*:\s*\[(.*?)\]", re.MULTILINE|re.DOTALL)
    EXTJS_ALTERNATE_NAMES = {
                             # Alternative names
                             'Ext.Element': 'Ext.dom.Element',
                             'Ext.core.Element': 'Ext.dom.Element',
                             'Ext.CompositeElement': 'Ext.dom.CompositeElement',
                             # Singleton instances
                             'Ext.TaskManager': 'Ext.util.TaskManager',
                             'Ext.Msg': 'Ext.window.MessageBox',
                             'Ext.MessageBox': 'Ext.window.MessageBox',
                             }

    #: Places where ExtJS put files in the Ext.* namespace.
    #: Relative to the root of the ExtJS sources
    EXTJS_CLASSPATH = ('src',
                       join('src', 'core', 'src'),
                       join('src', 'core', 'src', 'lang'))

    def __init__(self, classname, filepath):
        super(JsFile, self).__init__(classname, filepath)
        self.appname = self.get_appname()
        self.extend = None
        self.parse()

    def get_appname(self):
        return get_appname_from_classpath(self.classname)

    def parse(self):
        self.match_define()
        self.match_requires() # must be before extend and mixins, since they add to self.requires
        self.match_extend()
        self.match_mixins()
        self.match_model()
        #self.requires = filter(lambda r: not r.startswith('Ext.'), self.requires)
        if self.is_controller():
            self.match_controllers()
            self.match_models()
            self.match_stores()
            self.match_views()

    def match_define(self):
        m = self.DEFINEPATT.search(self.filecontent)
        if m:
            self.clsname = m.groups()[0]
        else:
            raise ValueError('{0}: no Ext.define(....) found.'.format(self.filepath))

    def match_requires(self):
        m = self.REQUIRESPATT.search(self.filecontent)
        if m:
            requiresstr = m.groups()[0]
            self.requires = self.LISTSPLITPATT.findall(requiresstr)
        else:
            self.requires = []

    def match_extend(self):
        m = self.EXTENDPATT.search(self.filecontent)
        if m:
            self.extend = m.groups()[0]
            self.requires.append(self.extend)
        else:
            self.extend = None

    def match_mixins(self):
        m = self.LISTMIXINSPATT.search(self.filecontent)
        if m:
            mixinsstr = m.groups()[0]
            self.mixins = self.LISTSPLITPATT.findall(mixinsstr)
            self.requires += self.mixins
        else:
            m = self.OBJECTMIXINSPATT.search(self.filecontent)
            if m:
                mixinsstr = m.groups()[0]
                self.mixins = self.STRINGOBJECT_VALUES_PATT.findall(mixinsstr)
                self.requires += self.mixins
            else:
                self.mixins = []

    def match_model(self):
        m = self.MODELPATT.search(self.filecontent)
        if m:
            model = m.groups()[0]
            self.requires.append(model)

    def get_classname_for_internalname(self, context, name):
        return '{0}.{1}.{2}'.format(self.appname, context, name)

    def merge_internalnames_with_requires(self, context, internalnames):
        classnames = [self.get_classname_for_internalname(context, name)
                      for name in internalnames]
        self.requires += classnames

    def match_controllers(self):
        m = self.CONTROLLERS_PATT.search(self.filecontent)
        if m:
            controllersstr = m.groups()[0]
            self.controllers = self.LISTSPLITPATT.findall(controllersstr)
            self.merge_internalnames_with_requires('controller', self.controllers)
        else:
            self.controllers = []

    def match_views(self):
        m = self.VIEWS_PATT.search(self.filecontent)
        if m:
            viewsstr = m.groups()[0]
            self.views = self.LISTSPLITPATT.findall(viewsstr)
            pprint(self.views)
            self.merge_internalnames_with_requires('view', self.views)
        else:
            self.views = []

    def match_models(self):
        m = self.MODELS_PATT.search(self.filecontent)
        if m:
            modelsstr = m.groups()[0]
            self.models = self.LISTSPLITPATT.findall(modelsstr)
            self.merge_internalnames_with_requires('model', self.models)
        else:
            self.models = []

    def match_stores(self):
        m = self.STORES_PATT.search(self.filecontent)
        if m:
            storesstr = m.groups()[0]
            self.stores = self.LISTSPLITPATT.findall(storesstr)
            self.merge_internalnames_with_requires('store', self.stores)
        else:
            self.stores = []

    def create_jsfile_for_extjsclass(self, classname):
        """
        ``create_jsfile_for_class``, but with all of the hacks required to
        handle finding stuff in the extjs sources.
        """
        if classname in self.EXTJS_ALTERNATE_NAMES:
            return self.create_jsfile_for_extjsclass(self.EXTJS_ALTERNATE_NAMES[classname])
        from os import sep
        appdir = get_staticdir_from_appname('extjs4')
        relativepath = sep.join(classname.split('.')[1:]) + '.js'
        app_path = None
        for dirname in self.EXTJS_CLASSPATH:
            path = join(appdir, dirname, relativepath)
            if exists(path):
                app_path = path
                break
        if not app_path:
            raise ValueError('Could not find any file for: {0}'.format(classname))
        try:
            return JsFile(classname, app_path)
        except ValueError:
            return SimpleJsFile(classname, app_path)

    def create_jsfile_for_class(self, classname):
        django_appname = get_appname_from_classpath(classname)
        if django_appname == 'extjs4':
            return self.create_jsfile_for_extjsclass(classname)
        from os import sep
        appdir = get_extjs_sourceroot_from_appname(django_appname)
        relativepath = sep.join(classname.split('.')[1:]) + '.js'
        app_path = join(appdir, relativepath)
        return JsFile(classname, app_path)

    def __str__(self):
        return "{0}: {1} ({2})".format(self.clsname, self.extend, ','.join(self.requires))

    def is_controller(self):
        return self.extend == 'Ext.app.Controller'

    def prettyprint(self):
        print('#################################')
        print('# {0}'.format(self))
        print('#################################')
        print('## Extend:')
        print(self.extend)
        print('## Requires:')
        pprint(self.requires)
        print('## Mixins:')
        pprint(self.mixins)
        if self.is_controller():
            print('## Controllers:')
            pprint(self.controllers)
            print('## Models:')
            pprint(self.models)
            print('## Stores:')
            pprint(self.stores)
            print('## Views:')
            pprint(self.views)

    def collect_jsfiles(self, collected_jsfiles_map):
        logging.debug('Collecting deps of {0}'.format(self.filepath))
        for classname in self.requires:
            jsfile = self.create_jsfile_for_class(classname)
            if not jsfile.classname in collected_jsfiles_map:
                collected_jsfiles_map[jsfile.classname] = jsfile
                jsfile.collect_jsfiles(collected_jsfiles_map)

class AppFile(JsFile):
    def __init__(self, appfile, appname):
        self.appname = appname
        super(AppFile, self).__init__(classname=None,
                                      filepath=appfile)

    def match_define(self):
        pass

    def get_appname(self):
        return self.appname

    def __str__(self):
        return "APP {appname}".format(**self.__dict__)

    def collect_appjsfiles(self, collected_jsfiles_map, context, names,
                           create_class=lambda classname, filepath: JsFile(classname, filepath)):
        for name in names:
            filepath = join(self.appdir, context, classpath_to_ospath(name))
            classname = self.get_classname_for_internalname(context, name)
            if not classname in collected_jsfiles_map:
                jsfile = create_class(classname, filepath)
                collected_jsfiles_map[jsfile.classname] = jsfile
                jsfile.collect_jsfiles(collected_jsfiles_map)

    #def collect_jsfiles(self, collected_jsfiles_map):
        #super(AppFile, self).collect_jsfiles(collected_jsfiles_map)
        #self.collect_appjsfiles(collected_jsfiles_map, 'controller', self.controllers,
                                #lambda classname, filepath: ControllerFile(classname, filepath, self.appname))
        #self.collect_appjsfiles(collected_jsfiles_map, 'models', self.models)
        #self.collect_appjsfiles(collected_jsfiles_map, 'stores', self.stores)
        #self.collect_appjsfiles(collected_jsfiles_map, 'views', self.views)

    def get_all_jsfiles(self):
        collected_jsfiles_map = {}
        self.collect_jsfiles(collected_jsfiles_map)
        return collected_jsfiles_map

def depsfullfilled(jsfile, jsfiles):
    for reqclass in jsfile.requires:
        if reqclass in jsfiles:
            return False
    return True

def orderJsFiles(jsfiles):
    ordered = []
    remaining = -1
    while jsfiles:
        for clsname in jsfiles.keys():
            jsfile = jsfiles[clsname]
            if depsfullfilled(jsfile, jsfiles):
                ordered.append(jsfile)
                del jsfiles[clsname]
        last_remaining = remaining
        remaining = len(jsfiles)
        if remaining == last_remaining:
            print('######################################################################')
            print('Can not fulfill all dependencies')
            print('######################################################################')
            for jsfile in jsfiles.itervalues():
                print('## ', jsfile.classname, ' requires:')
                for reqclass in jsfile.requires:
                    if reqclass in jsfiles:
                        print('- ', reqclass)
                print('')
            raise CommandError('ABORTING (see output above)')
    return ordered

#def find_all_jsfiles_in_dir(jsfiles, rootdir, verbose):
    #for root, dirs, files in walk(rootdir):
        #for filename in files:
            #if filename.endswith('.js'):
                #filepath = join(root, filename)
                #try:
                    #jsfile = JsFile(filepath)
                #except ValueError, e:
                    #if verbose:
                        #print str(e)
                #else:
                    #jsfiles[jsfile.clsname] = jsfile

#def collect_jsfiles_in_installedapps(jsfiles, verbose):
    #for app in settings.INSTALLED_APPS:
        #if not app.startswith('django.'):
            #mod = import_module(app)
            #if exists(mod.__file__) and isdir(dirname(mod.__file__)):
                #moddir = dirname(mod.__file__)
                #find_all_jsfiles_in_dir(jsfiles, moddir, verbose)

class Command(BaseCommand):
    help = 'Create new subject.'
    args = '<appname>'

    def handle(self, *args, **options):
        setup_logging(get_verbosity(options))
        if len(args) != 1:
            raise CommandError('Requires an <appname>.')
        appname = args[0]
        appdir = get_staticdir_from_appname(appname)
        appfile = join(appdir, 'app.js')

        jsfile = AppFile(appfile, appname)
        #jsfile.prettyprint()
        jsfiles = jsfile.get_all_jsfiles()
        ordered = orderJsFiles(jsfiles)

        outfile = join(appdir, 'app-all.js')
        open(outfile, 'wb').write('\n\n'.join([j.filecontent for j in ordered]))
<commit_msg>apps.developertools...dev_build_extjs: Close to working.<commit_after>from __future__ import print_function
from pprint import pprint
from os import walk
from os.path import join, exists
import re
import logging
from django.core.management.base import BaseCommand, CommandError
from devilry.utils.command import setup_logging, get_verbosity

from devilry.utils.importutils import get_staticdir_from_appname

def get_appname_from_classpath(classpath):
    if classpath.startswith('Ext.'):
        return 'extjs4'
    else:
        return classpath.split('.')[0]

def get_extjs_sourceroot_from_appname(appname):
    appdir = get_staticdir_from_appname(appname)
    if appname == 'extjs4':
        return join(appdir, 'src')
    if exists(join(appdir, 'app')):
        return join(appdir, 'app')
    else:
        return appdir

def classpath_to_ospath(unixpath):
    from os import sep
    return unixpath.replace('.', sep) + '.js'

class SimpleJsFile(object):
    MULTILINE_COMMENT_PATT = re.compile('\/\*.*?\*\/', re.MULTILINE|re.DOTALL)
    SINGLELINE_COMMENT_PATT = re.compile('\/\/.*?$', re.MULTILINE)

    def __init__(self, classname, filepath):
        self.classname = classname
        self.filepath = filepath
        filecontent = open(self.filepath, 'rb').read()
        self.filecontent = self.remove_comments(filecontent)
        self.requires = []

    def remove_comments(self, filecontent):
        filecontent = self.MULTILINE_COMMENT_PATT.sub('', filecontent)
        #filecontent = self.SINGLELINE_COMMENT_PATT.sub('', filecontent)
        return filecontent

    def __str__(self):
        return 'SIMPLE: {filepath}'.format(**self.__dict__)

    def collect_jsfiles(self, collected_jsfiles_map):
        pass

class JsFile(SimpleJsFile):
    """
    Collects information about an ExtJS javascript file, including:

        - classname (define)
        - dependencies:
            - mixins
            - model
            - requires
    """
    DEFINEPATT = re.compile(r"^Ext\.define\(.(.+?).\s*,", re.MULTILINE)
    EXTENDPATT = re.compile(r"^\s*extend\s*:\s*['\"](.+?)['\"]", re.MULTILINE|re.DOTALL)
    MODELPATT = re.compile(r"^\s*model\s*:\s*['\"](.+?)['\"]", re.MULTILINE|re.DOTALL)
    REQUIRESPATT = re.compile(r"^\s*requires\s*:\s*\[(.*?)\]", re.MULTILINE|re.DOTALL)
    LISTMIXINSPATT = re.compile(r"^\s*mixins\s*:\s*[(.*?)]", re.MULTILINE|re.DOTALL)
    OBJECTMIXINSPATT = re.compile(r"^\s*mixins\s*:\s*{(.*?)}", re.MULTILINE|re.DOTALL)
    LISTSPLITPATT = re.compile(r'[\'"](.*?)[\'"]')
    STRINGOBJECT_VALUES_PATT = re.compile(r'[\'"]?.*?[\'"]?\s*:\s*[\'"](.*?)[\'"]')
    CONTROLLERS_PATT = re.compile(r"^\s*controllers\s*:\s*\[(.*?)\]", re.MULTILINE|re.DOTALL)
    VIEWS_PATT = re.compile(r"^\s*views\s*:\s*\[(.*?)\]", re.MULTILINE|re.DOTALL)
    MODELS_PATT = re.compile(r"^\s*models\s*:\s*\[(.*?)\]", re.MULTILINE|re.DOTALL)
    STORES_PATT = re.compile(r"^\s*stores\s*:\s*\[(.*?)\]", re.MULTILINE|re.DOTALL)
    EXTJS_ALTERNATE_NAMES = {
                             # Alternative names
                             'Ext.Element': 'Ext.dom.Element',
                             'Ext.core.Element': 'Ext.dom.Element',
                             'Ext.CompositeElement': 'Ext.dom.CompositeElement',
                             # Singleton instances
                             'Ext.TaskManager': 'Ext.util.TaskManager',
                             'Ext.Msg': 'Ext.window.MessageBox',
                             'Ext.MessageBox': 'Ext.window.MessageBox',
                             }

    #: Places where ExtJS put files in the Ext.* namespace.
    #: Relative to the root of the ExtJS sources
    EXTJS_CLASSPATH = ('src',
                       join('src', 'core', 'src'),
                       join('src', 'core', 'src', 'lang'))

    def __init__(self, classname, filepath):
        super(JsFile, self).__init__(classname, filepath)
        self.appname = self.get_appname()
        self.extend = None
        self.parse()

    def get_appname(self):
        return get_appname_from_classpath(self.classname)

    def parse(self):
        self.match_define()
        self.match_requires() # must be before extend and mixins, since they add to self.requires
        self.match_extend()
        self.match_mixins()
        self.match_model()
        #self.requires = filter(lambda r: not r.startswith('Ext.'), self.requires)
        if self.is_controller():
            self.match_controllers()
            self.match_models()
            self.match_stores()
            self.match_views()

    def match_define(self):
        m = self.DEFINEPATT.search(self.filecontent)
        if m:
            self.clsname = m.groups()[0]
        else:
            raise ValueError('{0}: no Ext.define(....) found.'.format(self.filepath))

    def match_requires(self):
        m = self.REQUIRESPATT.search(self.filecontent)
        if m:
            requiresstr = m.groups()[0]
            self.requires = self.LISTSPLITPATT.findall(requiresstr)
        else:
            self.requires = []

    def match_extend(self):
        m = self.EXTENDPATT.search(self.filecontent)
        if m:
            self.extend = m.groups()[0]
            self.requires.append(self.extend)
        else:
            self.extend = None

    def match_mixins(self):
        m = self.LISTMIXINSPATT.search(self.filecontent)
        if m:
            mixinsstr = m.groups()[0]
            self.mixins = self.LISTSPLITPATT.findall(mixinsstr)
            self.requires += self.mixins
        else:
            m = self.OBJECTMIXINSPATT.search(self.filecontent)
            if m:
                mixinsstr = m.groups()[0]
                self.mixins = self.STRINGOBJECT_VALUES_PATT.findall(mixinsstr)
                self.requires += self.mixins
            else:
                self.mixins = []

    def match_model(self):
        m = self.MODELPATT.search(self.filecontent)
        if m:
            model = m.groups()[0]
            self.requires.append(model)

    def get_classname_for_internalname(self, context, name):
        return '{0}.{1}.{2}'.format(self.appname, context, name)

    def merge_internalnames_with_requires(self, context, internalnames):
        classnames = [self.get_classname_for_internalname(context, name)
                      for name in internalnames]
        self.requires += classnames

    def match_controllers(self):
        m = self.CONTROLLERS_PATT.search(self.filecontent)
        if m:
            controllersstr = m.groups()[0]
            self.controllers = self.LISTSPLITPATT.findall(controllersstr)
            self.merge_internalnames_with_requires('controller', self.controllers)
        else:
            self.controllers = []

    def match_views(self):
        m = self.VIEWS_PATT.search(self.filecontent)
        if m:
            viewsstr = m.groups()[0]
            self.views = self.LISTSPLITPATT.findall(viewsstr)
            pprint(self.views)
            self.merge_internalnames_with_requires('view', self.views)
        else:
            self.views = []

    def match_models(self):
        m = self.MODELS_PATT.search(self.filecontent)
        if m:
            modelsstr = m.groups()[0]
            self.models = self.LISTSPLITPATT.findall(modelsstr)
            self.merge_internalnames_with_requires('model', self.models)
        else:
            self.models = []

    def match_stores(self):
        m = self.STORES_PATT.search(self.filecontent)
        if m:
            storesstr = m.groups()[0]
            self.stores = self.LISTSPLITPATT.findall(storesstr)
            self.merge_internalnames_with_requires('store', self.stores)
        else:
            self.stores = []

    def create_jsfile_for_extjsclass(self, classname):
        """
        ``create_jsfile_for_class``, but with all of the hacks required to
        handle finding stuff in the extjs sources.
        """
        if classname in self.EXTJS_ALTERNATE_NAMES:
            return self.create_jsfile_for_extjsclass(self.EXTJS_ALTERNATE_NAMES[classname])
        from os import sep
        appdir = get_staticdir_from_appname('extjs4')
        relativepath = sep.join(classname.split('.')[1:]) + '.js'
        app_path = None
        for dirname in self.EXTJS_CLASSPATH:
            path = join(appdir, dirname, relativepath)
            if exists(path):
                app_path = path
                break
        if not app_path:
            raise ValueError('Could not find any file for: {0}'.format(classname))
        try:
            return JsFile(classname, app_path)
        except ValueError:
            return SimpleJsFile(classname, app_path)

    def create_jsfile_for_class(self, classname):
        django_appname = get_appname_from_classpath(classname)
        if django_appname == 'extjs4':
            return self.create_jsfile_for_extjsclass(classname)
        from os import sep
        appdir = get_extjs_sourceroot_from_appname(django_appname)
        relativepath = sep.join(classname.split('.')[1:]) + '.js'
        app_path = join(appdir, relativepath)
        return JsFile(classname, app_path)

    def __str__(self):
        return "{0}: {1} ({2})".format(self.clsname, self.extend, ','.join(self.requires))

    def is_controller(self):
        return self.extend == 'Ext.app.Controller'

    def prettyprint(self):
        print('#################################')
        print('# {0}'.format(self))
        print('#################################')
        print('## Extend:')
        print(self.extend)
        print('## Requires:')
        pprint(self.requires)
        print('## Mixins:')
        pprint(self.mixins)
        if self.is_controller():
            print('## Controllers:')
            pprint(self.controllers)
            print('## Models:')
            pprint(self.models)
            print('## Stores:')
            pprint(self.stores)
            print('## Views:')
            pprint(self.views)

    def collect_jsfiles(self, collected_jsfiles_map):
        logging.debug('Collecting deps of {0}'.format(self.filepath))
        for classname in self.requires:
            jsfile = self.create_jsfile_for_class(classname)
            if not jsfile.classname in collected_jsfiles_map:
                collected_jsfiles_map[jsfile.classname] = jsfile
                jsfile.collect_jsfiles(collected_jsfiles_map)

class AppFile(JsFile):
    def __init__(self, appfile, appname):
        self.appname = appname
        super(AppFile, self).__init__(classname=None,
                                      filepath=appfile)

    def match_define(self):
        pass

    def get_appname(self):
        return self.appname

    def __str__(self):
        return "APP {appname}".format(**self.__dict__)

    def collect_appjsfiles(self, collected_jsfiles_map, context, names,
                           create_class=lambda classname, filepath: JsFile(classname, filepath)):
        for name in names:
            filepath = join(self.appdir, context, classpath_to_ospath(name))
            classname = self.get_classname_for_internalname(context, name)
            if not classname in collected_jsfiles_map:
                jsfile = create_class(classname, filepath)
                collected_jsfiles_map[jsfile.classname] = jsfile
                jsfile.collect_jsfiles(collected_jsfiles_map)

    #def collect_jsfiles(self, collected_jsfiles_map):
        #super(AppFile, self).collect_jsfiles(collected_jsfiles_map)
        #self.collect_appjsfiles(collected_jsfiles_map, 'controller', self.controllers,
                                #lambda classname, filepath: ControllerFile(classname, filepath, self.appname))
        #self.collect_appjsfiles(collected_jsfiles_map, 'models', self.models)
        #self.collect_appjsfiles(collected_jsfiles_map, 'stores', self.stores)
        #self.collect_appjsfiles(collected_jsfiles_map, 'views', self.views)

    def get_all_jsfiles(self):
        collected_jsfiles_map = {}
        self.collect_jsfiles(collected_jsfiles_map)
        return collected_jsfiles_map

def depsfullfilled(jsfile, jsfiles):
    for reqclass in jsfile.requires:
        if reqclass in jsfiles:
            return False
    return True

def orderJsFiles(jsfiles):
    ordered = []
    remaining = -1
    while jsfiles:
        for clsname in jsfiles.keys():
            jsfile = jsfiles[clsname]
            if depsfullfilled(jsfile, jsfiles):
                ordered.append(jsfile)
                del jsfiles[clsname]
        last_remaining = remaining
        remaining = len(jsfiles)
        if remaining == last_remaining:
            print('######################################################################')
            print('Can not fulfill all dependencies')
            print('######################################################################')
            for jsfile in jsfiles.itervalues():
                print('## ', jsfile.classname, ' requires:')
                for reqclass in jsfile.requires:
                    if reqclass in jsfiles:
                        print('- ', reqclass)
                print('')
            raise CommandError('ABORTING (see output above)')
    return ordered

#def find_all_jsfiles_in_dir(jsfiles, rootdir, verbose):
    #for root, dirs, files in walk(rootdir):
        #for filename in files:
            #if filename.endswith('.js'):
                #filepath = join(root, filename)
                #try:
                    #jsfile = JsFile(filepath)
                #except ValueError, e:
                    #if verbose:
                        #print str(e)
                #else:
                    #jsfiles[jsfile.clsname] = jsfile

#def collect_jsfiles_in_installedapps(jsfiles, verbose):
    #for app in settings.INSTALLED_APPS:
        #if not app.startswith('django.'):
            #mod = import_module(app)
            #if exists(mod.__file__) and isdir(dirname(mod.__file__)):
                #moddir = dirname(mod.__file__)
                #find_all_jsfiles_in_dir(jsfiles, moddir, verbose)

class Command(BaseCommand):
    help = 'Create new subject.'
    args = '<appname>'

    def handle(self, *args, **options):
        setup_logging(get_verbosity(options))
        if len(args) != 1:
            raise CommandError('Requires an <appname>.')
        appname = args[0]
        appdir = get_staticdir_from_appname(appname)
        appfile = join(appdir, 'app.js')

        jsfile = AppFile(appfile, appname)
        #jsfile.prettyprint()
        jsfiles = jsfile.get_all_jsfiles()
        ordered = orderJsFiles(jsfiles)

        outfile = join(appdir, 'app-all.js')
        content = '\n\n'.join([j.filecontent for j in ordered])
        content += '\n\n' + jsfile.filecontent
        open(outfile, 'wb').write(content)
<|endoftext|>


<commit_before>#!/usr/bin/env python
from hashlib import md5
import sys
from clint.textui import indent, puts, colored, progress
from plumbum.cmd import git

def get_chunks(diff):
    diff = clean_diff(diff)
    chunk = []
    chunks = []
    for line in diff.split('\n'):
        if not line:
            continue
        if line.startswith('@@ '):
            if chunk:
                chunks.append('\n'.join(chunk))
            chunk = [line]
        else:
            chunk.append(line)
    if chunk:
        chunks.append('\n'.join(chunk))
    return chunks

def clean_diff(text):
    res = []
    skip = True
    for line in text.split('\n'):
        if line.startswith('diff --git'):
            skip = True
        if line.startswith('@@ '):
            skip = False
        if not skip:
            res.append(line)
    return '\n'.join(res)

def clean_chunk(chunk):
    return '\n'.join([x[1:] for x in chunk.split('\n')
                      if x and x[0] not in ('-', '@')])

def print_diff(diff):
    for line in diff.split('\n'):
        line = unicode(line).encode('utf-8')
        if line.startswith('+'):
            puts(colored.green(line))
        elif line.startswith('-'):
            puts(colored.red(line))
        else:
            puts(line)

def get_md5_files():
    res = []
    files = ['%s' % x for x in git['ls-files', '-m']().split('\n') if x]
    for fl in files:
        pymd5 = md5(unicode(git['show', '%s:%s' % (sys.argv[1], fl)]()).encode('utf-8')).hexdigest()
        res.append('%s %s' % (pymd5, fl))
    return '\n'.join(res) + '\n'

def main():
    status_files = get_md5_files()
    n_chunks = 0
    n_files = 0
    failed_files = []
    local_rev = git['rev-parse', 'HEAD']().strip()
    remote_rev = git['rev-parse', sys.argv[1]]().strip()
    for line in status_files:
        if not line:
            break
        pymd5, check_file = line.split(' ')
        if md5(open(check_file, 'r').read()).hexdigest() != pymd5:
            hcommand = git['log', '--no-merges', '--pretty=oneline',
                           '%s..%s' % (local_rev, remote_rev),
                           check_file]
            local_chunks = {}
            for lchunk in get_chunks(git['diff', check_file]()):
                local_chunks[md5(unicode(lchunk).encode('utf-8')).hexdigest()] = lchunk
            hlines = [x for x in hcommand().split('\n') if x]
            puts("File: %s modified remotely. Searching for local modifications in remote by %s commits" % (check_file, len(hlines)))
            puts("Local chunks: %s" % ', '.join(local_chunks.keys()))
            for hline in hlines:
                if not hline:
                    continue
                commit = hline.split(' ')[0]
                with indent(4):
                    puts(colored.yellow("**** DIFF %s^1..%s ****"
                                        % (commit, commit)))
                remote_chunks = [md5(unicode(x).encode('utf-8')).hexdigest()
                                 for x in get_chunks(git['diff', '%s^1..%s'% (commit, commit),
                                      check_file]())]
                with indent(4):
                    puts("Remote chunks: %s" % ', '.join(remote_chunks))
                for lchunk in local_chunks.keys():
                    if lchunk in remote_chunks:
                        with indent(4):
                            puts(colored.green("Local chunk %s found in remotes!" % lchunk))
                        del local_chunks[lchunk]
                    else:
                        # Secound round
                        rfile = git['show', '%s:%s' % (sys.argv[1], check_file)]()
                        chunk = clean_chunk(local_chunks[lchunk])
                        if rfile.find(chunk) >= 0:
                            with indent(4):
                                puts(colored.green("Local chunk %s found in remotes!" % lchunk))
                            del local_chunks[lchunk]
            if local_chunks:
                n_files += 1
                for chunk in local_chunks.values():
                    print_diff(chunk)
                    n_chunks += 1
                puts(colored.red("[x] %s %s" % (pymd5, check_file)))
            else:
                puts(colored.green("[o] %s %s" % (pymd5, check_file)))
        else:
            puts(colored.green("[o] %s" % line))
    if n_files:
        puts(colored.red("*** WARNING: %s chunks in %s files doesn't match" % (n_chunks, n_files)))

if __name__ == '__main__':
    main()
<commit_msg>Delete old script<commit_after><|endoftext|>


<commit_before>'''Models for bpz'''
from __future__ import unicode_literals

from django.contrib.gis.db import models
from django.utils.encoding import python_2_unicode_compatible

@python_2_unicode_compatible
class Case(models.Model):
    '''Planning or zoning case

    Model generated w/ ogrinspect. Omitted fields:
    SHAPE_Leng - derive from geometry
    SHAPE_Area - derive from geometry
    '''
    DOMAIN_BOA = 0
    DOMAIN_TMAPC = 1
    DOMAIN_CHOICES = (
        (DOMAIN_BOA, 'Board of Adjustment'),
        (DOMAIN_TMAPC, 'Tulsa Metropolitan Area Planning Commission'),
    )

    TYPE_CHOICES = [(x, x) for x in (
        "Board of Adjustment",
        "Corridor Minor Amendment",
        "Corridor Plan",
        "Lot Split",
        "PUD Major Amendment",
        "PUD Minor Amendment",
        "PUD Site Plan",
        "PUD",
        "Subdivision Plat",
        "Zoning",
    )]

    object_id = models.IntegerField()
    case_id = models.CharField(max_length=30)
    domain = models.CharField(max_length=10, choices=DOMAIN_CHOICES)
    status = models.CharField(
        max_length=10, choices=[(x, x) for x in (
            'Pending', 'Continued', 'Approved', 'Deny')])
    location = models.CharField(max_length=50)
    link = models.URLField()
    hearing_date = models.DateField()
    case_type = models.CharField(max_length=30, choices=TYPE_CHOICES)
    geom = models.GeometryField(srid=4326)
    objects = models.GeoManager()

    def __str__(self):
        return self.case_id

@python_2_unicode_compatible
class HomeOwnersAssociation(models.Model):
    '''Home Owners Association

    Model generated w/ ogrinspect. Omitted fields:
    Id - Always 0
    Umbrella - 'N' or null
    Shape_Leng - derive from geometry
    Spape_Area - derive from geometry
    '''

    object_id = models.IntegerField(null=True, blank=True)
    name = models.CharField(max_length=50)
    hoa_name = models.CharField(max_length=50)
    geom = models.GeometryField(srid=4326)
    objects = models.GeoManager()

    class Meta:
        verbose_name = 'Home Owners Association'

    def __str__(self):
        return self.name or '<unnamed>'
<commit_msg>Case: standardize choice declarations<commit_after>'''Models for bpz'''
from __future__ import unicode_literals

from django.contrib.gis.db import models
from django.utils.encoding import python_2_unicode_compatible

@python_2_unicode_compatible
class Case(models.Model):
    '''Planning or zoning case

    Model generated w/ ogrinspect. Omitted fields:
    SHAPE_Leng - derive from geometry
    SHAPE_Area - derive from geometry
    '''
    DOMAIN_BOA = 0
    DOMAIN_TMAPC = 1
    DOMAIN_CHOICES = (
        (DOMAIN_BOA, 'Board of Adjustment'),
        (DOMAIN_TMAPC, 'Tulsa Metropolitan Area Planning Commission'),
    )

    STATUS_CHOICES = [(x, x) for x in (
        'Pending',
        'Continued',
        'Approved',
        'Deny',
    )]

    CASE_TYPE_CHOICES = [(x, x) for x in (
        "Board of Adjustment",
        "Corridor Minor Amendment",
        "Corridor Plan",
        "Lot Split",
        "PUD Major Amendment",
        "PUD Minor Amendment",
        "PUD Site Plan",
        "PUD",
        "Subdivision Plat",
        "Zoning",
    )]

    object_id = models.IntegerField()
    case_id = models.CharField(max_length=30)
    domain = models.CharField(max_length=10, choices=DOMAIN_CHOICES)
    status = models.CharField(max_length=10, choices=STATUS_CHOICES)
    location = models.CharField(max_length=50)
    link = models.URLField()
    hearing_date = models.DateField()
    case_type = models.CharField(max_length=30, choices=CASE_TYPE_CHOICES)
    geom = models.GeometryField(srid=4326)
    objects = models.GeoManager()

    def __str__(self):
        return self.case_id

@python_2_unicode_compatible
class HomeOwnersAssociation(models.Model):
    '''Home Owners Association

    Model generated w/ ogrinspect. Omitted fields:
    Id - Always 0
    Umbrella - 'N' or null
    Shape_Leng - derive from geometry
    Spape_Area - derive from geometry
    '''

    object_id = models.IntegerField(null=True, blank=True)
    name = models.CharField(max_length=50)
    hoa_name = models.CharField(max_length=50)
    geom = models.GeometryField(srid=4326)
    objects = models.GeoManager()

    class Meta:
        verbose_name = 'Home Owners Association'

    def __str__(self):
        return self.name or '<unnamed>'
<|endoftext|>


<commit_before>import os
import sys
sys.setrecursionlimit(10000)
import datetime

import numpy as np

import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import matplotlib.ticker as ticker

from matplotlib.backends.backend_qt4agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.backends.backend_qt4agg import NavigationToolbar2QT as NavigationToolbar

from PyQt4 import QtGui
from PyQt4 import Qt

from matplotlib import cm
from scipy.io.idl import readsav
from mpl_toolkits.axes_grid1 import make_axes_locatable

first_file_name  = "big_nope"

plt.style.use('dark_background')
sys.path.append("./bq_t4_sys/")
sys.path.append("./bq_t4_sys/cstagger/")

import bifrost

def process_file_name(file_name):

   a = os.path.splitext(file_name)[0]
   a = a.split("/")[:-1]
   a.append(" ")
   where_str = "/".join(a)

   b = os.path.splitext(file_name)[0]
   b = b.split("/")[-1]
   b = b.split("_")

   base_name = b[0]
   snap_n    = b[1]

   return where_str.strip(), base_name.strip(), snap_n.strip()

def z2indx(z_vec, z):
    return np.argmax( z_vec >= z )

def get_bifrost_param(file_name, offset):
    where_str, base_name, snap_n = process_file_name(file_name)
    return bifrost.read_idl_ascii(where_str + base_name + ('_%s' % str(int(snap_n) + offset)) + '.idl')

def get_bifrost_obj(file_name, offset, with_no_aux):
    where_str, base_name, snap_n = process_file_name(file_name)
    print '[Processing] : ' ,base_name, " snap No: ", int(snap_n) + offset, " in ", where_str
    formater = "_%%0%ii" % len(str(int(snap_n) + offset))
    return bifrost.OSC_data(int(snap_n) + offset, template = where_str + base_name + formater, meshfile = where_str + base_name + ".mesh", no_aux=with_no_aux), base_name, int(snap_n) + offset

class Window(QtGui.QDialog):
    def __init__(self, parent=None):

        super(Window, self).__init__(parent)

        self.setAcceptDrops(True)
        self.figure = plt.figure()

        # --- Just for fun lets start with nice picture

        img = mpimg.imread('./bq_t4_sys/thor.png')
        ax = plt.Axes(self.figure,[0.,0.,1.,1.])
        ax.set_axis_off()
        self.figure.add_axes(ax)
        plt.imshow(img,cmap=cm.get_cmap('gray'))

        # ---------------------------------------------

        self.canvas = FigureCanvas(self.figure)

        # ---------------------------------------------

        self.toolbar = NavigationToolbar(self.canvas, self)

        # ---------------------------------------------

        self.slider = QtGui.QSlider(orientation=Qt.Qt.Horizontal)
        self.slider.setTickInterval(1)
        self.slider.valueChanged.connect(self.plot)

        # ----------------------------------------------

        self.combo = QtGui.QComboBox(self)
        self.combo.activated[str].connect(self.comboActivated)

        # -- check box

        self.hbox = QtGui.QHBoxLayout()
        # self.hbox.addStretch(1)

        self.back_button = QtGui.QPushButton('<',self)
        self.back_button.clicked.connect(self.goBack)
        self.next_button = QtGui.QPushButton('>',self)
        self.next_button.clicked.connect(self.goNext)

        self.check_abs = QtGui.QCheckBox('Absolute')
        # self.check_abs.toggle()
        self.check_abs.stateChanged.connect(self.plot)

        self.check_bw = QtGui.QCheckBox('B&W')
        # self.check_bw.toggle()
        self.check_bw.stateChanged.connect(self.plot)

        self.check_log = QtGui.QCheckBox('Log')
        # self.check_log.toggle()
        self.check_log.stateChanged.connect(self.plot)

        self.check_si = QtGui.QCheckBox('CGS Units')
        self.check_si.toggle()
        self.check_si.stateChanged.connect(self.plot)

        self.check_aux = QtGui.QCheckBox('No AUX')
        self.check_aux.toggle()
        self.check_aux.stateChanged.connect(self.reset_bifrost_obj)

        self.hbox.addWidget(self.check_si)
        self.hbox.addWidget(self.check_abs)
        self.hbox.addWidget(self.check_log)
        self.hbox.addWidget(self.check_bw)
        self.hbox.addWidget(self.check_aux)
        self.hbox.addWidget(self.back_button)
        self.hbox.addWidget(self.next_button)
        self.hbox.addWidget(self.combo)

        # set the layout
        layout = QtGui.QVBoxLayout()
        layout.addWidget(self.toolbar)
        layout.addLayout(self.hbox)
        layout.addWidget(self.canvas)
        layout.addWidget(self.slider)
        self.setLayout(layout)

        # actual data stuff
        self.fpath = first_file_name
        # self.param = get_bifrost_param(first_file_name,0)
        # self.b, self.base_name, self.snap_n = get_bifrost_obj(self.fpath,0)

        # self.data = np.einsum('ijk->kji', self.data)

        self.tag = 'r'
        # self.get_data()

    def keyPressEvent(self, event):

        key = event.key

        if key == Qt.Qt.Key_T:
            self.toolbar.show()
        elif key == Qt.Qt.Key_H:
            self.toolbar.hide()

    def goBack(self):
        self.param = get_bifrost_param(self.fpath,-1)
        self.b, self.base_name, self.snap_n = get_bifrost_obj(self.fpath,-1, self.check_aux.isChecked())
        self.get_data()
        self.plot()
        where_str, base_name, snap_n = process_file_name(self.fpath)
        snap_n = str(int(snap_n) - 1)
        self.fpath = where_str + base_name + '_' + snap_n + '.snap'
        pass

    def goNext(self):
        self.param = get_bifrost_param(self.fpath,1)
        self.b, self.base_name, self.snap_n = get_bifrost_obj(self.fpath,1,self.check_aux.isChecked())
        self.get_data()
        self.plot()
        where_str, base_name, snap_n = process_file_name(self.fpath)
        snap_n = str(int(snap_n) + 1)
        self.fpath = where_str + base_name + '_' + snap_n + '.snap'
        pass

    def setCombo(self):
        self.combo.clear()
        for tag in self.b.snapvars:
            self.combo.addItem(tag)
        if not self.check_aux.isChecked():
            for tag in self.b.auxvars:
                self.combo.addItem(tag)

    def comboActivated(self, text):
        self.tag = text
        self.data = self.b.getvar(self.tag)
        self.plot()

    def reset_bifrost_obj(self):
        self.b, self.base_name, self.snap_n = get_bifrost_obj(self.fpath,0,self.check_aux.isChecked())
        print self.b.snapvars
        self.get_data()

    def get_data(self):
        self.data = self.b.getvar(self.tag)
        self.slider.setMinimum(0)
        self.slider.setMaximum(self.data.shape[2]-1)

        if self.slider.value() >= self.data.shape[2]:
            self.slider.setValue(self.data.shape[2]-1)

        self.setCombo()

    def home(self):
        self.toolbar.home()
    def zoom(self):
        self.toolbar.zoom()
    def pan(self):
        self.toolbar.pan()

    def plot(self):
			plt.clf()
			image = self.data[:,:,self.slider.value()]
			image = np.fliplr(image)
			image = np.rot90(image,k=2)

			label = "Value"
			color = cm.get_cmap('jet')

			ax = self.figure.add_subplot(111)
			ax.set_title("[%s] %s (Snap: %s) for z = %f \n[time: %s]" % (self.tag, self.base_name, self.snap_n, self.b.z[self.slider.value()],str(datetime.timedelta(seconds=self.param['t']*self.param['u_t']))))
			ax.set_ylabel('y-direction')
			ax.set_xlabel('x-direction')

			ax.xaxis.set_major_locator(ticker.MultipleLocator(int(64)))
			ax.yaxis.set_major_locator(ticker.MultipleLocator(int(64)))

			extension = [0, self.param['mx'], 0, self.param['my']]

			if self.check_si.isChecked():
				ax.set_ylabel('y-direction [Mm]')
				ax.set_xlabel('x-direction [Mm]')

				ax.xaxis.set_major_locator(ticker.MultipleLocator(int(4)))
				ax.yaxis.set_major_locator(ticker.MultipleLocator(int(4)))

				extension = [0., self.param['dx']*self.param['mx'], 0.0, self.param['dy']*self.param['my']]
				if self.tag == 'r':
					image = image * self.param['u_r']
					unit_label = "[g/cm3]"
					label = "Value %s" % unit_label
				elif (self.tag == 'bx' or self.tag == 'by' or self.tag == 'bz'):
					image = image * self.param['u_b']
					unit_label = "[G]"
					label = "Value %s" % unit_label
				elif (self.tag == 'px' or self.tag == 'py' or self.tag == 'pz'):
					image = image * self.param['u_p']
					unit_label = "[Ba]"
					label = "Value %s" % unit_label
				elif self.tag == 'e':
					image = image * self.param['u_e']
					unit_label = "[erg]"
					label = "Value %s" % unit_label

			if self.check_abs.isChecked():
				image = np.absolute(image)
				label = "ABS( %s )" % label

			if self.check_log.isChecked():
				image = np.log10(image)
				label = "Log10( %s )" % label

			if self.check_bw.isChecked():
				color = cm.get_cmap('gist_yarg')

			im = ax.imshow(image, interpolation='none', origin='lower', cmap=color, extent=extension)
        # ax.text(0.025, 0.025, (r'$\langle  B_{z}  \rangle = %2.2e$'+'\n'+r'$\langle |B_{z}| \rangle = %2.2e$') % (np.average(img),np.average(np.absolute(img))), ha='left', va='bottom', transform=ax.transAxes)
			divider = make_axes_locatable(ax)
			cax = divider.append_axes("right", size="5%", pad=0.05)
			plt.colorbar(im, cax=cax,label=label)
			self.canvas.draw()

    def dragEnterEvent(self, event):
        if event.mimeData().hasUrls():
            event.accept()
        else:
            event.ignore()

    def dropEvent(self, event):
        for url in event.mimeData().urls():
            path = url.toLocalFile().toLocal8Bit().data()
        if os.path.isfile(path):
            self.fpath = path
            self.param = get_bifrost_param(self.fpath,0)
            self.b, self.base_name, self.snap_n = get_bifrost_obj(self.fpath,0, self.check_aux.isChecked())
            self.get_data()
            self.plot()

if __name__ == '__main__':
    app = QtGui.QApplication(sys.argv)
    main = Window()
    main.setWindowTitle('Bifrost Q(t4)uick Look App')
    main.show()

    sys.exit(app.exec_())
<commit_msg>Added different view (X-Y, X-Z and Y-Z)<commit_after>import os
import sys
import datetime

import numpy as np

import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import matplotlib.ticker as ticker

from matplotlib.backends.backend_qt4agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.backends.backend_qt4agg import NavigationToolbar2QT as NavigationToolbar

from PyQt4 import QtGui
from PyQt4 import Qt

from matplotlib import cm
from scipy.io.idl import readsav
from mpl_toolkits.axes_grid1 import make_axes_locatable
from matplotlib.image import NonUniformImage

first_file_name  = "big_nope"

plt.style.use('dark_background')
sys.path.append("./bq_t4_sys/")
sys.path.append("./bq_t4_sys/cstagger/")

import bifrost

def process_file_name(file_name):

   a = os.path.splitext(file_name)[0]
   a = a.split("/")[:-1]
   a.append(" ")
   where_str = "/".join(a)

   b = os.path.splitext(file_name)[0]
   b = b.split("/")[-1]
   b = b.split("_")

   base_name = b[0]
   snap_n    = b[1]

   return where_str.strip(), base_name.strip(), snap_n.strip()

def z2indx(z_vec, z):
    return np.argmax( z_vec >= z )

def get_bifrost_param(file_name, offset):
    where_str, base_name, snap_n = process_file_name(file_name)
    return bifrost.read_idl_ascii(where_str + base_name + ('_%s' % str(int(snap_n) + offset)) + '.idl')

def get_bifrost_obj(file_name, offset, with_no_aux):
    where_str, base_name, snap_n = process_file_name(file_name)
    print '[Processing] : ' ,base_name, " snap No: ", int(snap_n) + offset, " in ", where_str
    formater = "_%%0%ii" % len(str(int(snap_n) + offset))
    return bifrost.OSC_data(int(snap_n) + offset, template = where_str + base_name + formater, meshfile = where_str + base_name + ".mesh", no_aux=with_no_aux), base_name, int(snap_n) + offset

class Window(QtGui.QDialog):
    def __init__(self, parent=None):

        super(Window, self).__init__(parent)

        self.setAcceptDrops(True)
        self.figure = plt.figure()

        # --- Just for fun lets start with nice picture

        img = mpimg.imread('./bq_t4_sys/thor.png')
        ax = plt.Axes(self.figure,[0.,0.,1.,1.])
        ax.set_axis_off()
        self.figure.add_axes(ax)
        plt.imshow(img,cmap=cm.get_cmap('gray'))

        # ---------------------------------------------

        self.canvas = FigureCanvas(self.figure)

        # ---------------------------------------------

        self.toolbar = NavigationToolbar(self.canvas, self)

        # ---------------------------------------------

        self.slider = QtGui.QSlider(orientation=Qt.Qt.Horizontal)
        self.slider.setTickInterval(1)
        self.slider.valueChanged.connect(self.plot)

        # ----------------------------------------------

        self.view_combo = QtGui.QComboBox(self)
        self.view_combo.activated[str].connect(self.changeView)
        self.view_combo.addItem('X-Y')
        self.view_combo.addItem('X-Z')
        self.view_combo.addItem('Y-Z')
        self.view = 'X-Y'
        self.slide_dimension = 2 # will slide by default though z

        self.combo = QtGui.QComboBox(self)
        self.combo.activated[str].connect(self.comboActivated)

        # -- check box

        self.hbox = QtGui.QHBoxLayout()
        # self.hbox.addStretch(1)

        self.back_button = QtGui.QPushButton('<',self)
        self.back_button.clicked.connect(self.goBack)
        self.next_button = QtGui.QPushButton('>',self)
        self.next_button.clicked.connect(self.goNext)

        self.check_abs = QtGui.QCheckBox('Absolute')
        # self.check_abs.toggle()
        self.check_abs.stateChanged.connect(self.plot)

        self.check_bw = QtGui.QCheckBox('B&W')
        # self.check_bw.toggle()
        self.check_bw.stateChanged.connect(self.plot)

        self.check_log = QtGui.QCheckBox('Log')
        # self.check_log.toggle()
        self.check_log.stateChanged.connect(self.plot)

        self.check_si = QtGui.QCheckBox('CGS Units')
        self.check_si.toggle()
        self.check_si.stateChanged.connect(self.plot)

        self.check_aux = QtGui.QCheckBox('No AUX')
        self.check_aux.toggle()
        self.check_aux.stateChanged.connect(self.reset_bifrost_obj)

        self.hbox.addWidget(self.check_si)
        self.hbox.addWidget(self.check_abs)
        self.hbox.addWidget(self.check_log)
        self.hbox.addWidget(self.check_bw)
        self.hbox.addWidget(self.check_aux)
        self.hbox.addWidget(self.back_button)
        self.hbox.addWidget(self.next_button)
        self.hbox.addWidget(self.combo)
        self.hbox.addWidget(self.view_combo)

        # set the layout
        layout = QtGui.QVBoxLayout()
        layout.addWidget(self.toolbar)
        layout.addLayout(self.hbox)
        layout.addWidget(self.canvas)
        layout.addWidget(self.slider)
        self.setLayout(layout)

        # actual data stuff
        self.fpath = first_file_name
        # self.param = get_bifrost_param(first_file_name,0)
        # self.b, self.base_name, self.snap_n = get_bifrost_obj(self.fpath,0)

        # self.data = np.einsum('ijk->kji', self.data)

        self.tag = 'r'
        # self.get_data()

    def keyPressEvent(self, event):

        key = event.key

        if key == Qt.Qt.Key_T:
            self.toolbar.show()
        elif key == Qt.Qt.Key_H:
            self.toolbar.hide()

    def goBack(self):
        self.param = get_bifrost_param(self.fpath,-1)
        self.b, self.base_name, self.snap_n = get_bifrost_obj(self.fpath,-1, self.check_aux.isChecked())
        self.get_data()
        self.plot()
        where_str, base_name, snap_n = process_file_name(self.fpath)
        snap_n = str(int(snap_n) - 1)
        self.fpath = where_str + base_name + '_' + snap_n + '.snap'
        pass

    def goNext(self):
        self.param = get_bifrost_param(self.fpath,1)
        self.b, self.base_name, self.snap_n = get_bifrost_obj(self.fpath,1,self.check_aux.isChecked())
        self.get_data()
        self.plot()
        where_str, base_name, snap_n = process_file_name(self.fpath)
        snap_n = str(int(snap_n) + 1)
        self.fpath = where_str + base_name + '_' + snap_n + '.snap'
        pass

    def setCombo(self):
        self.combo.clear()
        for tag in self.b.snapvars:
            self.combo.addItem(tag)
        if not self.check_aux.isChecked():
            for tag in self.b.auxvars:
                self.combo.addItem(tag)

    def changeView(self, text):
        self.view = text
        self.get_data()
        self.plot()
        print 'View changed to: ', self.view

    def comboActivated(self, text):
        self.tag = text
        self.data = self.b.getvar(self.tag)
        self.plot()

    def reset_bifrost_obj(self):
        self.b, self.base_name, self.snap_n = get_bifrost_obj(self.fpath,0,self.check_aux.isChecked())
        print self.b.snapvars
        self.get_data()

    def get_data(self):
        self.data = self.b.getvar(self.tag)
        if self.view == 'X-Y':
            self.slide_dimension = 2
        elif self.view == 'X-Z':
            self.slide_dimension = 1
        elif self.view == 'Y-Z':
            self.slide_dimension =0
        self.slider.setMinimum(0)
        self.slider.setMaximum(self.data.shape[self.slide_dimension]-1)

        if self.slider.value() >= self.data.shape[self.slide_dimension]:
            self.slider.setValue(self.data.shape[self.slide_dimension]-1)

        self.setCombo()

    def home(self):
        self.toolbar.home()
    def zoom(self):
        self.toolbar.zoom()
    def pan(self):
        self.toolbar.pan()

    def plot(self):
        plt.clf()
        ax = self.figure.add_subplot(111)
        slice_str = '[?]'
        # extension = []
        if self.view == 'X-Y':
            image = self.data[:,:,self.slider.value()]
            slice_str = 'z = %f ' % self.b.z[self.slider.value()]
            ax.set_ylabel('y-direction')
            ax.set_xlabel('x-direction')
            # extension = [0, self.param['mx'], 0, self.param['my']]
        elif self.view == 'X-Z':
            image = self.data[:,self.slider.value(),:]
            slice_str = 'y = %f ' % self.b.y[self.slider.value()]
            ax.set_ylabel('z-direction')
            ax.set_xlabel('x-direction')
            # extension = [0, self.param['mx'], 0, self.param['mz']]
        elif self.view == 'Y-Z':
            image = self.data[self.slider.value(),:,:]
            slice_str = 'x = %f ' % self.b.x[self.slider.value()]
            ax.set_ylabel('z-direction')
            ax.set_xlabel('y-direction')
            # extension = [0, self.param['my'], 0, self.param['mz']]
        # image = np.fliplr(image)
        # image = np.rot90(image,k=3)

        label = "Value"
        color = cm.get_cmap('jet')

        ax.set_title("[%s] %s (Snap: %s) for %s \n[time: %s]" % (self.tag, self.base_name, self.snap_n, slice_str, str(datetime.timedelta(seconds=self.param['t']*self.param['u_t']))))
        # ax.xaxis.set_major_locator(ticker.MultipleLocator(int(64)))
        # ax.yaxis.set_major_locator(ticker.MultipleLocator(int(64)))

        if self.check_si.isChecked():

            if self.tag == 'r':
                image = image * self.param['u_r']
                unit_label = "[g/cm3]"
                label = "Value %s" % unit_label
            elif (self.tag == 'bx' or self.tag == 'by' or self.tag == 'bz'):
                image = image * self.param['u_b']
                unit_label = "[G]"
                label = "Value %s" % unit_label
            elif (self.tag == 'px' or self.tag == 'py' or self.tag == 'pz'):
                image = image * self.param['u_p']
                unit_label = "[Ba]"
                label = "Value %s" % unit_label
            elif self.tag == 'e':
                image = image * self.param['u_e']
                unit_label = "[erg]"
                label = "Value %s" % unit_label

        if self.check_abs.isChecked():
            image = np.absolute(image)
            label = "ABS( %s )" % label

        if self.check_log.isChecked():
            image = np.log10(image)
            label = "Log10( %s )" % label
        if self.check_bw.isChecked():
            color = cm.get_cmap('gist_yarg')

        if self.view == 'X-Y':
            ax.set_ylabel('y-direction [Mm]')
            ax.set_xlabel('x-direction [Mm]')
            im = NonUniformImage(ax, interpolation='bilinear', extent=(self.b.x.min(),self.b.x.max(),self.b.y.min(),self.b.y.max()), cmap=color)
            im.set_data(self.b.x, self.b.y, np.fliplr(zip(*image[::-1])))
            ax.images.append(im)
            ax.set_xlim(self.b.x.min(),self.b.x.max())
            ax.set_ylim(self.b.y.min(),self.b.y.max())
            ax.xaxis.set_major_locator(ticker.MultipleLocator(int(4)))
            ax.yaxis.set_major_locator(ticker.MultipleLocator(int(4)))
        elif self.view == 'X-Z':
            ax.set_ylabel('z-direction [Mm]')
            ax.set_xlabel('x-direction [Mm]')
            im = NonUniformImage(ax, interpolation='bilinear', extent=(self.b.x.min(),self.b.x.max(),self.b.z.min(),self.b.z.max()), cmap=color)
            im.set_data(self.b.x, self.b.z[::-1], np.flipud(np.fliplr(zip(*image[::-1]))))
            ax.images.append(im)
            ax.set_xlim(self.b.x.min(),self.b.x.max())
            ax.set_ylim(self.b.z.max(),self.b.z.min())
            ax.xaxis.set_major_locator(ticker.MultipleLocator(int(4)))
            ax.yaxis.set_major_locator(ticker.MultipleLocator(int(2)))
        elif self.view == 'Y-Z':
            ax.set_ylabel('z-direction [Mm]')
            ax.set_xlabel('y-direction [Mm]')
            im = NonUniformImage(ax, interpolation='bilinear', extent=(self.b.y.min(),self.b.y.max(),self.b.z.min(),self.b.z.max()), cmap=color)
            im.set_data(self.b.y, self.b.z[::-1], np.flipud(np.fliplr(zip(*image[::-1]))))
            ax.images.append(im)
            ax.set_xlim(self.b.y.min(),self.b.y.max())
            ax.set_ylim(self.b.z.max(),self.b.z.min())
            ax.xaxis.set_major_locator(ticker.MultipleLocator(int(4)))
            ax.yaxis.set_major_locator(ticker.MultipleLocator(int(2)))
        # im = ax.imshow(image, interpolation='none', origin='lower', cmap=color, extent=extension)
        # ax.text(0.025, 0.025, (r'$\langle  B_{z}  \rangle = %2.2e$'+'\n'+r'$\langle |B_{z}| \rangle = %2.2e$') % (np.average(img),np.average(np.absolute(img))), ha='left', va='bottom', transform=ax.transAxes)
        divider = make_axes_locatable(ax)
        cax = divider.append_axes("right", size="5%", pad=0.05)
        plt.colorbar(im, cax=cax,label=label)
        self.canvas.draw()

    def dragEnterEvent(self, event):
        if event.mimeData().hasUrls():
            event.accept()
        else:
            event.ignore()

    def dropEvent(self, event):
        for url in event.mimeData().urls():
            path = url.toLocalFile().toLocal8Bit().data()
        if os.path.isfile(path):
            self.fpath = path
            self.param = get_bifrost_param(self.fpath,0)
            self.b, self.base_name, self.snap_n = get_bifrost_obj(self.fpath,0, self.check_aux.isChecked())
            self.get_data()
            self.plot()

if __name__ == '__main__':
    app = QtGui.QApplication(sys.argv)
    main = Window()
    main.setWindowTitle('Bifrost Q(t4)uick Look App')
    main.show()

    sys.exit(app.exec_())
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import print_function

import json
import os
import platform
import subprocess
import sys

import ninjalog_uploader
import subprocess2

THIS_DIR = os.path.dirname(__file__)
UPLOADER = os.path.join(THIS_DIR, 'ninjalog_uploader.py')
CONFIG = os.path.join(THIS_DIR, 'ninjalog.cfg')
VERSION = 2

def LoadConfig():
  if os.path.isfile(CONFIG):
    with open(CONFIG, 'rb') as f:
      config = json.load(f)
      if config['version'] == VERSION:
        config['countdown'] = max(0, config['countdown'] - 1)
        return config

  return {
      'is-googler':
      ninjalog_uploader.IsGoogler('chromium-build-stats.appspot.com'),
      'countdown': 10,
      'version': VERSION,
  }

def SaveConfig(config):
  with open(CONFIG, 'wb') as f:
    json.dump(config, f)

def ShowMessage(countdown):
  whitelisted = '\n'.join(
      ['  * %s' % config for config in ninjalog_uploader.WHITELISTED_CONFIGS])
  print("""
Your ninjalog will be uploaded to build stats server. The uploaded log will be
used to analyze user side build performance.

The following information will be uploaded with ninjalog.
* OS (e.g. Win, Mac or Linux)
* number of cpu cores of building machine
* build targets (e.g. chrome, browser_tests)
* parallelism passed by -j flag
* following build configs
%s

Uploading ninjalog will be started after you run autoninja another %d time.

If you don't want to upload ninjalog, please run following command.
$ %s opt-out

If you want to allow upload ninjalog from next autoninja run, please run the
following command.
$ %s opt-in

If you have questions about this, please send mail to infra-dev@chromium.org

You can find a more detailed explanation in
%s

""" % (whitelisted, countdown, __file__, __file__,
       os.path.abspath(os.path.join(THIS_DIR, "ninjalog.README.md"))))

def main():
  config = LoadConfig()

  if len(sys.argv) == 2 and sys.argv[1] == 'opt-in':
    config['opt-in'] = True
    config['countdown'] = 0
    SaveConfig(config)
    print('ninjalog upload is opted in.')
    return 0

  if len(sys.argv) == 2 and sys.argv[1] == 'opt-out':
    config['opt-in'] = False
    SaveConfig(config)
    print('ninjalog upload is opted out.')
    return 0

  if 'opt-in' in config and not config['opt-in']:
    # Upload is opted out.
    return 0

  if not config.get("is-googler", False):
    # Not googler.
    return 0

  if config.get("countdown", 0) > 0:
    # Need to show message.
    ShowMessage(config["countdown"])
    # Only save config if something has meaningfully changed.
    SaveConfig(config)
    return 0

  if len(sys.argv) == 1:
    # dry-run for debugging.
    print("upload ninjalog dry-run")
    return 0

  # Run upload script without wait.
  devnull = open(os.devnull, "w")
  creationnflags = 0
  if platform.system() == 'Windows':
    creationnflags = subprocess.CREATE_NEW_PROCESS_GROUP
  subprocess2.Popen([sys.executable, UPLOADER] + sys.argv[1:],
                    stdout=devnull,
                    stderr=devnull,
                    creationflags=creationnflags)

if __name__ == '__main__':
  sys.exit(main())
<commit_msg>ninjalog_uploader_wrapper: do not use binary read/write for json<commit_after>#!/usr/bin/env python3
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import print_function

import json
import os
import platform
import subprocess
import sys

import ninjalog_uploader
import subprocess2

THIS_DIR = os.path.dirname(__file__)
UPLOADER = os.path.join(THIS_DIR, 'ninjalog_uploader.py')
CONFIG = os.path.join(THIS_DIR, 'ninjalog.cfg')
VERSION = 2

def LoadConfig():
  if os.path.isfile(CONFIG):
    with open(CONFIG, 'r') as f:
      config = json.load(f)
      if config['version'] == VERSION:
        config['countdown'] = max(0, config['countdown'] - 1)
        return config

  return {
      'is-googler':
      ninjalog_uploader.IsGoogler('chromium-build-stats.appspot.com'),
      'countdown': 10,
      'version': VERSION,
  }

def SaveConfig(config):
  with open(CONFIG, 'w') as f:
    json.dump(config, f)

def ShowMessage(countdown):
  whitelisted = '\n'.join(
      ['  * %s' % config for config in ninjalog_uploader.WHITELISTED_CONFIGS])
  print("""
Your ninjalog will be uploaded to build stats server. The uploaded log will be
used to analyze user side build performance.

The following information will be uploaded with ninjalog.
* OS (e.g. Win, Mac or Linux)
* number of cpu cores of building machine
* build targets (e.g. chrome, browser_tests)
* parallelism passed by -j flag
* following build configs
%s

Uploading ninjalog will be started after you run autoninja another %d time.

If you don't want to upload ninjalog, please run following command.
$ %s opt-out

If you want to allow upload ninjalog from next autoninja run, please run the
following command.
$ %s opt-in

If you have questions about this, please send mail to infra-dev@chromium.org

You can find a more detailed explanation in
%s

""" % (whitelisted, countdown, __file__, __file__,
       os.path.abspath(os.path.join(THIS_DIR, "ninjalog.README.md"))))

def main():
  config = LoadConfig()

  if len(sys.argv) == 2 and sys.argv[1] == 'opt-in':
    config['opt-in'] = True
    config['countdown'] = 0
    SaveConfig(config)
    print('ninjalog upload is opted in.')
    return 0

  if len(sys.argv) == 2 and sys.argv[1] == 'opt-out':
    config['opt-in'] = False
    SaveConfig(config)
    print('ninjalog upload is opted out.')
    return 0

  if 'opt-in' in config and not config['opt-in']:
    # Upload is opted out.
    return 0

  if not config.get("is-googler", False):
    # Not googler.
    return 0

  if config.get("countdown", 0) > 0:
    # Need to show message.
    ShowMessage(config["countdown"])
    # Only save config if something has meaningfully changed.
    SaveConfig(config)
    return 0

  if len(sys.argv) == 1:
    # dry-run for debugging.
    print("upload ninjalog dry-run")
    return 0

  # Run upload script without wait.
  devnull = open(os.devnull, "w")
  creationnflags = 0
  if platform.system() == 'Windows':
    creationnflags = subprocess.CREATE_NEW_PROCESS_GROUP
  subprocess2.Popen([sys.executable, UPLOADER] + sys.argv[1:],
                    stdout=devnull,
                    stderr=devnull,
                    creationflags=creationnflags)

if __name__ == '__main__':
  sys.exit(main())
<|endoftext|>


<commit_before>import glob
import os
import tempfile

import numpy as np
from skimage.io import imread, imsave
from skimage.transform import resize, rotate

def data_augmentation(img_name, angles=[], max_pixel=0, prefix=''):
    """Create many images from one training sample
    """
    n_images, img_list = len(angles), []
    img = imread(img_name)
    for th in angles:
        new_img = rotate(img, th, cval=1.0, resize=True)
        new_img = resize(new_img, (max_pixel, max_pixel))
        img_list.append(prefix + '_' + str(th) +  '.jpg')
        imsave(img_list[-1], new_img)
    return img_list

def folder_content1(folder, sort_flg=True):
    """Return all files on the first level of a folder
    """
    content = glob.glob(os.path.join(folder, '*'))
    if sort_flg:
        content.sort(key=str.lower)
    return content

def img_resolution_list(image_list):
   """Return a ndarray with the resolution (along axis 1) of a list of images
   """
   res_list = np.empty((len(image_list), 2), np.int32)
   for i, v in enumerate(image_list):
       try:
           img = imread(v)
       except:
           raise 'Cannot read image:', v
       res_list[i, :] = img.shape
   return res_list

def label_list(folder):
    """Return categories inside the passed folder

    Parameters
    ----------
    folder : string

    Returns
    -------
    labels : list

    Note
    ----
    It assumes that folder has a 1to1 mapping between subfolders and categories
    and there is no more files
    """
    labels = folder_content1(folder)
    for i, v in enumerate(labels):
        dummy, labels[i] = os.path.split(v)
    return labels

def root_folder():
    """Returns the absolute path of the folder where data is located
    """
    root, dummy =  os.path.split(os.path.realpath(__file__))
    return os.path.join(root, '..', 'data')

def test_folder():
    """Returns the absolute path of the train folder
    """
    root = root_folder()
    return os.path.join(root, 'test')

def test_list(folder):
    """Return a list with fullpath name of images on the testing set
    """
    return folder_content1(test_folder())

def train_folder():
    """Returns the absolute path of the test folder
    """
    root = root_folder()
    return os.path.join(root, 'train')

def train_list(folder):
    """Return two list: (1) fullpath name of images and (2) indexes of their
       categories (0-indexed)

    Parameters
    ----------
    folder : string

    Returns
    -------
    image_fullpath_names : list
    image_labels : list

    Note
    ----
    It assumes that folder has a 1to1 mapping between subfolders and categories
    and there is no more files
    """
    image_fullpath_names, image_labels = [], []
    labels = folder_content1(folder)
    for i, v in enumerate(labels):
        tmp = folder_content1(v)
        image_fullpath_names += tmp
        image_labels += [i] * len(tmp)
    return image_fullpath_names, image_labels

def samples_per_categories(argument):
   """Compute the number of samples per category
   """
   if isinstance(argument, basestring):
       samples_per_cat = []
       labels = folder_content1(argument)
       for i in labels:
           samples_per_cat.append(len(folder_content1(i, False)))
   else:
       raise 'Sorry, Im working on that'
   return samples_per_cat

def save_augmented_dataset(img_list, img_labels, outdir,
                           prm={'angles':range(0, 181, 30), 'max_pixel':60}):
    """Create a folder with the augmented training set

    Parameters
    ----------
    outdir : string
        Name of folder to allocate augmented training set
    prm : dict
        parameters for data augmentation function

    """
    try:
        os.makedirs(outdir)
    except Exception, e:
        raise e
    dataset_list, dataset_labels = [], []
    for (fullpath, label) in zip(img_list, img_labels):
        with tempfile.NamedTemporaryFile(dir=outdir) as temp:
            prefix = temp.name + '_' + str(label)
        aug_img = data_augmentation(fullpath, prefix=prefix, **prm)
        dataset_list += aug_img
        dataset_labels += [label] * len(aug_img)
    return dataset_list, dataset_labels

<commit_msg>Add function to create a balanced partition from training set<commit_after>import glob
import os
import tempfile

import numpy as np
from skimage.io import imread, imsave
from skimage.transform import resize, rotate

#TODO: Create a tuple with (img_list, img_labels) for create_val_set and
#      save_augmented_dataset

def create_partition(img_list, img_labels, pctg=0.3):
    """Create a balanced partition of the whole training set

    TODO
    ----
    1. Include argument to set seed for pseudo-random number generation
    """
    instances_per_cat = samples_per_categories(img_labels)
    instances_to_sample = pctg * min(instances_per_cat)
    assert instances_to_sample == 0, 'Insufficient data to create partition'

    labels_array, idx_train, idx_test = np.array(img_labels), [], []
    for i in range(0, len(instances_per_cat)):
        idx_cat_i = np.nonzero(labels_array == i)[0]
        idx = np.random.choice(idx_cat_i, size=instances_to_sample,
                               replace=False)
        idx_test += idx_cat_i[idx].tolist()
        idx_train += list(set(idx_cat_i.tolist()) -
                          set(idx_test[-instances_to_sample:]))
    return idx_train, idx_test

def data_augmentation(img_name, angles=[], max_pixel=0, prefix=''):
    """Create many images from one training sample
    """
    n_images, img_list = len(angles), []
    img = imread(img_name)
    for th in angles:
        new_img = rotate(img, th, cval=1.0, resize=True)
        new_img = resize(new_img, (max_pixel, max_pixel))
        img_list.append(prefix + '_' + str(th) +  '.jpg')
        imsave(img_list[-1], new_img)
    return img_list

def folder_content1(folder, sort_flg=True):
    """Return all files on the first level of a folder
    """
    content = glob.glob(os.path.join(folder, '*'))
    if sort_flg:
        content.sort(key=str.lower)
    return content

def img_resolution_list(image_list):
   """Return a ndarray with the resolution (along axis 1) of a list of images
   """
   res_list = np.empty((len(image_list), 2), np.int32)
   for i, v in enumerate(image_list):
       try:
           img = imread(v)
       except:
           raise 'Cannot read image:', v
       res_list[i, :] = img.shape
   return res_list

def label_list(folder):
    """Return categories inside the passed folder

    Parameters
    ----------
    folder : string

    Returns
    -------
    labels : list

    Note
    ----
    It assumes that folder has a 1to1 mapping between subfolders and categories
    and there is no more files
    """
    labels = folder_content1(folder)
    for i, v in enumerate(labels):
        dummy, labels[i] = os.path.split(v)
    return labels

def root_folder():
    """Returns the absolute path of the folder where data is located
    """
    root, dummy =  os.path.split(os.path.realpath(__file__))
    return os.path.join(root, '..', 'data')

def test_folder():
    """Returns the absolute path of the train folder
    """
    root = root_folder()
    return os.path.join(root, 'test')

def test_list(folder):
    """Return a list with fullpath name of images on the testing set
    """
    return folder_content1(test_folder())

def train_folder():
    """Returns the absolute path of the test folder
    """
    root = root_folder()
    return os.path.join(root, 'train')

def train_list(folder):
    """Return two list: (1) fullpath name of images and (2) indexes of their
       categories (0-indexed)

    Parameters
    ----------
    folder : string

    Returns
    -------
    image_fullpath_names : list
    image_labels : list

    Note
    ----
    It assumes that folder has a 1to1 mapping between subfolders and categories
    and there is no more files
    """
    image_fullpath_names, image_labels = [], []
    labels = folder_content1(folder)
    for i, v in enumerate(labels):
        tmp = folder_content1(v)
        image_fullpath_names += tmp
        image_labels += [i] * len(tmp)
    return image_fullpath_names, image_labels

def samples_per_categories(argument):
   """Compute the number of samples per category
   """
   if isinstance(argument, basestring):
       samples_per_cat = []
       labels = folder_content1(argument)
       for i in labels:
           samples_per_cat.append(len(folder_content1(i, False)))
   else:
       samples_per_cat = np.bincount(argument).tolist()
   return samples_per_cat

def save_augmented_dataset(img_list, img_labels, outdir,
                           prm={'angles':range(0, 181, 30), 'max_pixel':60}):
    """Create a folder with the augmented training set

    Parameters
    ----------
    outdir : string
        Name of folder to allocate augmented training set
    prm : dict
        parameters for data augmentation function

    """
    try:
        os.makedirs(outdir)
    except Exception, e:
        raise e
    dataset_list, dataset_labels = [], []
    for (fullpath, label) in zip(img_list, img_labels):
        with tempfile.NamedTemporaryFile(dir=outdir) as temp:
            prefix = temp.name + '_' + str(label)
        aug_img = data_augmentation(fullpath, prefix=prefix, **prm)
        dataset_list += aug_img
        dataset_labels += [label] * len(aug_img)
    return dataset_list, dataset_labels

<|endoftext|>


<commit_before># Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import absolute_import
import logging
import os
import re
import select
import subprocess

from telemetry.core import fuchsia_interface
from telemetry.internal.backends.chrome import chrome_browser_backend
from telemetry.internal.backends.chrome import minidump_finder
from telemetry.internal.platform import fuchsia_platform_backend as fuchsia_platform_backend_module

import py_utils

WEB_ENGINE_SHELL = 'web-engine-shell'
CAST_STREAMING_SHELL = 'cast-streaming-shell'
FUCHSIA_CHROME = 'fuchsia-chrome'

class FuchsiaBrowserBackend(chrome_browser_backend.ChromeBrowserBackend):
  def __init__(self, fuchsia_platform_backend, browser_options,
               browser_directory, profile_directory):
    assert isinstance(fuchsia_platform_backend,
                      fuchsia_platform_backend_module.FuchsiaPlatformBackend)
    super().__init__(
        fuchsia_platform_backend,
        browser_options=browser_options,
        browser_directory=browser_directory,
        profile_directory=profile_directory,
        supports_extensions=False,
        supports_tab_control=True)
    self._command_runner = fuchsia_platform_backend.command_runner
    self._browser_process = None
    self._devtools_port = None
    self._symbolizer_proc = None
    if os.environ.get('CHROMIUM_OUTPUT_DIR'):
      self._output_dir = os.environ.get('CHROMIUM_OUTPUT_DIR')
    else:
      self._output_dir = os.path.abspath(os.path.dirname(
          fuchsia_platform_backend.ssh_config))
    self._browser_log = ''
    self._browser_log_proc = None
    self._managed_repo = fuchsia_platform_backend.managed_repo

  @property
  def log_file_path(self):
    return None

  def _FindDevToolsPortAndTarget(self):
    return self._devtools_port, None

  def DumpMemory(self, timeout=None, detail_level=None):
    if detail_level is None:
      detail_level = 'light'
    return self.devtools_client.DumpMemory(timeout=timeout,
                                           detail_level=detail_level)

  def _ReadDevToolsPortFromStderr(self, search_regex):
    def TryReadingPort():
      if not self._browser_log_proc.stderr:
        return None
      line = self._browser_log_proc.stderr.readline()
      tokens = re.search(search_regex, line)
      self._browser_log += line
      return int(tokens.group(1)) if tokens else None
    return py_utils.WaitFor(TryReadingPort, timeout=60)

  def _ReadDevToolsPort(self):
    if (self.browser_type == WEB_ENGINE_SHELL or
        self.browser_type == CAST_STREAMING_SHELL):
      search_regex = r'Remote debugging port: (\d+)'
    else:
      search_regex = r'DevTools listening on ws://127.0.0.1:(\d+)/devtools.*'
    return self._ReadDevToolsPortFromStderr(search_regex)

  def _StartWebEngineShell(self, startup_args):
    browser_cmd = [
        'component',
        'run-legacy',
        'fuchsia-pkg://%s/web_engine_shell#meta/web_engine_shell.cmx' %
        self._managed_repo
    ]

    # Flags forwarded to the web_engine_shell component.
    browser_cmd.extend([
        '--',
        '--web-engine-package-name=web_engine_with_webui',
        '--remote-debugging-port=0',
        '--use-web-instance',
        '--enable-web-instance-tmp',
        '--with-webui',
        'about:blank'
    ])

    # Use flags used on WebEngine in production devices.
    browser_cmd.extend([
        '--',
        '--enable-low-end-device-mode',
        '--force-gpu-mem-available-mb=64',
        '--force-gpu-mem-discardable-limit-mb=32',
        '--force-max-texture-size=2048',
        '--gpu-rasterization-msaa-sample-count=0',
        '--min-height-for-gpu-raster-tile=128',
        '--webgl-msaa-sample-count=0',
        '--max-decoded-image-size-mb=10'
    ])
    if startup_args:
      browser_cmd.extend(startup_args)
    self._browser_process = self._command_runner.run_continuous_ffx_command(
        browser_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    self._browser_log_proc = self._browser_process

  def _StartCastStreamingShell(self, startup_args):
    browser_cmd = [
        'run',
        'fuchsia-pkg://%s/cast_streaming_shell#meta/cast_streaming_shell.cmx' %
        self._managed_repo,
        '--remote-debugging-port=0',
    ]

    # Use flags used on WebEngine in production devices.
    browser_cmd.extend([
        '--',
        '--enable-low-end-device-mode',
        '--force-gpu-mem-available-mb=64',
        '--force-gpu-mem-discardable-limit-mb=32',
        '--force-max-texture-size=2048',
        '--gpu-rasterization-msaa-sample-count=0',
        '--min-height-for-gpu-raster-tile=128',
        '--webgl-msaa-sample-count=0',
        '--max-decoded-image-size-mb=10',
    ])
    if startup_args:
      browser_cmd.extend(startup_args)
    self._browser_process = self._command_runner.RunCommandPiped(
        browser_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    self._browser_log_proc = self._browser_process

  def _StartChrome(self, startup_args):
    browser_cmd = [
        'session',
        'add',
        'fuchsia-pkg://%s/chrome#meta/chrome_v1.cmx' %
        self._managed_repo,
        '--',
        'about:blank',
        '--remote-debugging-port=0',
        '--enable-logging'
    ]
    if startup_args:
      browser_cmd.extend(startup_args)

    # Log the browser from this point on from system-logs.
    logging_cmd = [
        'log_listener',
        '--since_now'
    ]
    # Combine to STDOUT, as this is used for symbolization
    self._browser_log_proc = self._command_runner.RunCommandPiped(
        logging_cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)
    # Need stderr to replicate stdout for symbolization.
    self._browser_log_proc.stderr = self._browser_log_proc.stdout

    logging.debug('Browser command: %s', ' '.join(browser_cmd))
    self._browser_process = self._command_runner.run_continuous_ffx_command(
        browser_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

  def Start(self, startup_args):
    output_root = os.path.join(self._output_dir, 'gen', 'fuchsia_web')

    try:
      if self.browser_type == WEB_ENGINE_SHELL:
        self._StartWebEngineShell(startup_args)
        browser_id_files = [
            os.path.join(output_root, 'shell', 'web_engine_shell', 'ids.txt'),
            os.path.join(output_root, 'webengine', 'web_engine', 'ids.txt'),
        ]
      elif self.browser_type == CAST_STREAMING_SHELL:
        self._StartCastStreamingShell(startup_args)
        browser_id_files = [
            os.path.join(output_root, 'shell', 'cast_streaming_shell',
                'ids.txt'),
            os.path.join(output_root, 'webengine', 'web_engine', 'ids.txt'),
        ]
      else:
        self._StartChrome(startup_args)
        browser_id_files = [
            os.path.join(self._output_dir, 'gen', 'chrome', 'app', 'chrome',
                         'ids.txt'),
        ]

      # Symbolize stderr of browser process if possible
      self._symbolizer_proc = (
          fuchsia_interface.StartSymbolizerForProcessIfPossible(
              self._browser_log_proc.stderr, subprocess.PIPE, browser_id_files))
      if self._symbolizer_proc:
        self._browser_log_proc.stderr = self._symbolizer_proc.stdout

      self._dump_finder = minidump_finder.MinidumpFinder(
          self.browser.platform.GetOSName(),
          self.browser.platform.GetArchName())
      self._devtools_port = self._ReadDevToolsPort()
      self.BindDevToolsClient()

      # Start tracing if startup tracing attempted but did not actually start.
      # This occurs when no ChromeTraceConfig is present yet current_state is
      # non-None.
      tracing_backend = self._platform_backend.tracing_controller_backend
      current_state = tracing_backend.current_state
      if (not tracing_backend.GetChromeTraceConfig() and
          current_state is not None):
        tracing_backend.StopTracing()
        tracing_backend.StartTracing(current_state.config,
                                     current_state.timeout)

    except Exception as e:
      logging.exception(e)
      logging.error('The browser failed to start. Output of the browser: \n%s' %
                    self.GetStandardOutput())
      self.Close()
      raise

  def GetPid(self):
    # TODO(crbug.com/1297717): This does not work if the browser process is
    # kicked off via ffx session add, as that process is on the host, and
    # exits immediately.
    return self._browser_process.pid

  def Background(self):
    raise NotImplementedError

  def _CloseOnDeviceBrowsers(self):
    if (self.browser_type == WEB_ENGINE_SHELL or
        self.browser_type == CAST_STREAMING_SHELL):
      close_cmd = ['killall', 'web_instance.cmx']
    else:
      close_cmd = ['killall', 'chrome_v1.cmx']
    self._command_runner.RunCommand(
        close_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

  def Close(self):
    super().Close()

    if self._browser_process:
      logging.info('Shutting down browser process on Fuchsia')
      self._browser_process.kill()
    if self._browser_log_proc:
      self._browser_log_proc.kill()
    if self._symbolizer_proc:
      self._symbolizer_proc.kill()
    self._CloseOnDeviceBrowsers()
    self._browser_process = None
    self._browser_log_proc = None

  def IsBrowserRunning(self):
    # TODO(crbug.com/1297717): this does not capture if the process is still
    # running if its kicked off via ffx session add.
    return bool(self._browser_process)

  def GetStandardOutput(self):
    if self._browser_log_proc:
      self._browser_log_proc.terminate()
      self._CloseOnDeviceBrowsers()

      # Make sure there is something to read.
      if select.select([self._browser_log_proc.stderr], [], [], 0.0)[0]:
        self._browser_log += self._browser_log_proc.stderr.read()
    return self._browser_log

  def SymbolizeMinidump(self, minidump_path):
    logging.warning('Symbolizing Minidump not supported on Fuchsia.')

  def _GetBrowserExecutablePath(self):
    return None
<commit_msg>[gpu][telemetry] Use the CFv2 version of web_engine_shell<commit_after># Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import absolute_import
import logging
import os
import re
import select
import subprocess

from telemetry.core import fuchsia_interface
from telemetry.internal.backends.chrome import chrome_browser_backend
from telemetry.internal.backends.chrome import minidump_finder
from telemetry.internal.platform import fuchsia_platform_backend as fuchsia_platform_backend_module

import py_utils

WEB_ENGINE_SHELL = 'web-engine-shell'
CAST_STREAMING_SHELL = 'cast-streaming-shell'
FUCHSIA_CHROME = 'fuchsia-chrome'

class FuchsiaBrowserBackend(chrome_browser_backend.ChromeBrowserBackend):
  def __init__(self, fuchsia_platform_backend, browser_options,
               browser_directory, profile_directory):
    assert isinstance(fuchsia_platform_backend,
                      fuchsia_platform_backend_module.FuchsiaPlatformBackend)
    super().__init__(
        fuchsia_platform_backend,
        browser_options=browser_options,
        browser_directory=browser_directory,
        profile_directory=profile_directory,
        supports_extensions=False,
        supports_tab_control=True)
    self._command_runner = fuchsia_platform_backend.command_runner
    self._browser_process = None
    self._devtools_port = None
    self._symbolizer_proc = None
    if os.environ.get('CHROMIUM_OUTPUT_DIR'):
      self._output_dir = os.environ.get('CHROMIUM_OUTPUT_DIR')
    else:
      self._output_dir = os.path.abspath(os.path.dirname(
          fuchsia_platform_backend.ssh_config))
    self._browser_log = ''
    self._browser_log_proc = None
    self._managed_repo = fuchsia_platform_backend.managed_repo

  @property
  def log_file_path(self):
    return None

  def _FindDevToolsPortAndTarget(self):
    return self._devtools_port, None

  def DumpMemory(self, timeout=None, detail_level=None):
    if detail_level is None:
      detail_level = 'light'
    return self.devtools_client.DumpMemory(timeout=timeout,
                                           detail_level=detail_level)

  def _ReadDevToolsPortFromStdout(self, search_regex):
    def TryReadingPort():
      if not self._browser_log_proc.stderr:
        return None
      line = self._browser_log_proc.stdout.readline()
      tokens = re.search(search_regex, line)
      self._browser_log += line
      return int(tokens.group(1)) if tokens else None
    return py_utils.WaitFor(TryReadingPort, timeout=60)

  def _ReadDevToolsPortFromStderr(self, search_regex):
    def TryReadingPort():
      if not self._browser_log_proc.stderr:
        return None
      line = self._browser_log_proc.stderr.readline()
      tokens = re.search(search_regex, line)
      self._browser_log += line
      return int(tokens.group(1)) if tokens else None
    return py_utils.WaitFor(TryReadingPort, timeout=60)

  def _ReadDevToolsPort(self):
    if self.browser_type == WEB_ENGINE_SHELL:
      search_regex = r'Remote debugging port: (\d+)'
      result = self._ReadDevToolsPortFromStdout(search_regex)
    elif self.browser_type == CAST_STREAMING_SHELL:
      search_regex = r'Remote debugging port: (\d+)'
      result = self._ReadDevToolsPortFromStderr(search_regex)
    else:
      search_regex = r'DevTools listening on ws://127.0.0.1:(\d+)/devtools.*'
      result = self._ReadDevToolsPortFromStderr(search_regex)
    return result

  def _StartWebEngineShell(self, startup_args):
    browser_cmd = [
        'test',
        'run',
        'fuchsia-pkg://%s/web_engine_shell#meta/web_engine_shell.cm' %
        self._managed_repo,
    ]

    # Flags forwarded to the web_engine_shell component.
    browser_cmd.extend([
        '--',
        '--web-engine-package-name=web_engine_with_webui',
        '--remote-debugging-port=0',
        '--use-web-instance',
        '--enable-web-instance-tmp',
        '--with-webui',
        'about:blank'
    ])

    # Use flags used on WebEngine in production devices.
    browser_cmd.extend([
        '--',
        '--enable-low-end-device-mode',
        '--force-gpu-mem-available-mb=64',
        '--force-gpu-mem-discardable-limit-mb=32',
        '--force-max-texture-size=2048',
        '--gpu-rasterization-msaa-sample-count=0',
        '--min-height-for-gpu-raster-tile=128',
        '--webgl-msaa-sample-count=0',
        '--max-decoded-image-size-mb=10'
    ])
    if startup_args:
      browser_cmd.extend(startup_args)
    self._browser_process = self._command_runner.run_continuous_ffx_command(
        browser_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    self._browser_log_proc = self._browser_process

  def _StartCastStreamingShell(self, startup_args):
    browser_cmd = [
        'run',
        'fuchsia-pkg://%s/cast_streaming_shell#meta/cast_streaming_shell.cmx' %
        self._managed_repo,
        '--remote-debugging-port=0',
    ]

    # Use flags used on WebEngine in production devices.
    browser_cmd.extend([
        '--',
        '--enable-low-end-device-mode',
        '--force-gpu-mem-available-mb=64',
        '--force-gpu-mem-discardable-limit-mb=32',
        '--force-max-texture-size=2048',
        '--gpu-rasterization-msaa-sample-count=0',
        '--min-height-for-gpu-raster-tile=128',
        '--webgl-msaa-sample-count=0',
        '--max-decoded-image-size-mb=10',
    ])
    if startup_args:
      browser_cmd.extend(startup_args)
    self._browser_process = self._command_runner.RunCommandPiped(
        browser_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    self._browser_log_proc = self._browser_process

  def _StartChrome(self, startup_args):
    browser_cmd = [
        'session',
        'add',
        'fuchsia-pkg://%s/chrome#meta/chrome_v1.cmx' %
        self._managed_repo,
        '--',
        'about:blank',
        '--remote-debugging-port=0',
        '--enable-logging'
    ]
    if startup_args:
      browser_cmd.extend(startup_args)

    # Log the browser from this point on from system-logs.
    logging_cmd = [
        'log_listener',
        '--since_now'
    ]
    # Combine to STDOUT, as this is used for symbolization
    self._browser_log_proc = self._command_runner.RunCommandPiped(
        logging_cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)
    # Need stderr to replicate stdout for symbolization.
    self._browser_log_proc.stderr = self._browser_log_proc.stdout

    logging.debug('Browser command: %s', ' '.join(browser_cmd))
    self._browser_process = self._command_runner.run_continuous_ffx_command(
        browser_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

  def Start(self, startup_args):
    output_root = os.path.join(self._output_dir, 'gen', 'fuchsia_web')

    try:
      if self.browser_type == WEB_ENGINE_SHELL:
        self._StartWebEngineShell(startup_args)
        browser_id_files = [
            os.path.join(output_root, 'shell', 'web_engine_shell', 'ids.txt'),
            os.path.join(output_root, 'webengine', 'web_engine', 'ids.txt'),
        ]
      elif self.browser_type == CAST_STREAMING_SHELL:
        self._StartCastStreamingShell(startup_args)
        browser_id_files = [
            os.path.join(output_root, 'shell', 'cast_streaming_shell',
                'ids.txt'),
            os.path.join(output_root, 'webengine', 'web_engine', 'ids.txt'),
        ]
      else:
        self._StartChrome(startup_args)
        browser_id_files = [
            os.path.join(self._output_dir, 'gen', 'chrome', 'app', 'chrome',
                         'ids.txt'),
        ]

      # Symbolize stderr of browser process if possible
      self._symbolizer_proc = (
          fuchsia_interface.StartSymbolizerForProcessIfPossible(
              self._browser_log_proc.stderr, subprocess.PIPE, browser_id_files))
      if self._symbolizer_proc:
        self._browser_log_proc.stderr = self._symbolizer_proc.stdout

      self._dump_finder = minidump_finder.MinidumpFinder(
          self.browser.platform.GetOSName(),
          self.browser.platform.GetArchName())
      self._devtools_port = self._ReadDevToolsPort()
      self.BindDevToolsClient()

      # Start tracing if startup tracing attempted but did not actually start.
      # This occurs when no ChromeTraceConfig is present yet current_state is
      # non-None.
      tracing_backend = self._platform_backend.tracing_controller_backend
      current_state = tracing_backend.current_state
      if (not tracing_backend.GetChromeTraceConfig() and
          current_state is not None):
        tracing_backend.StopTracing()
        tracing_backend.StartTracing(current_state.config,
                                     current_state.timeout)

    except Exception as e:
      logging.exception(e)
      logging.error('The browser failed to start. Output of the browser: \n%s' %
                    self.GetStandardOutput())
      self.Close()
      raise

  def GetPid(self):
    # TODO(crbug.com/1297717): This does not work if the browser process is
    # kicked off via ffx session add, as that process is on the host, and
    # exits immediately.
    return self._browser_process.pid

  def Background(self):
    raise NotImplementedError

  def _CloseOnDeviceBrowsers(self):
    if (self.browser_type == WEB_ENGINE_SHELL or
        self.browser_type == CAST_STREAMING_SHELL):
      close_cmd = ['killall', 'web_instance.cmx']
    else:
      close_cmd = ['killall', 'chrome_v1.cmx']
    self._command_runner.RunCommand(
        close_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

  def Close(self):
    super().Close()

    if self._browser_process:
      logging.info('Shutting down browser process on Fuchsia')
      self._browser_process.kill()
    if self._browser_log_proc:
      self._browser_log_proc.kill()
    if self._symbolizer_proc:
      self._symbolizer_proc.kill()
    self._CloseOnDeviceBrowsers()
    self._browser_process = None
    self._browser_log_proc = None

  def IsBrowserRunning(self):
    # TODO(crbug.com/1297717): this does not capture if the process is still
    # running if its kicked off via ffx session add.
    return bool(self._browser_process)

  def GetStandardOutput(self):
    if self._browser_log_proc:
      self._browser_log_proc.terminate()
      self._CloseOnDeviceBrowsers()

      # Make sure there is something to read.
      if select.select([self._browser_log_proc.stderr], [], [], 0.0)[0]:
        self._browser_log += self._browser_log_proc.stderr.read()
    return self._browser_log

  def SymbolizeMinidump(self, minidump_path):
    logging.warning('Symbolizing Minidump not supported on Fuchsia.')

  def _GetBrowserExecutablePath(self):
    return None
<|endoftext|>


<commit_before>"""
datesvc provides web services around Date
python2 datesvc.py <port>
"""
import web
from datetime import date

urls = (
    '/now', 'Now',
)

web.config.debug=False
app = web.application(urls, globals())

class Now:
    def GET(self):
        now = date.today()
        """return 'Hello'"""
        strf = 'Today is %s ' % now.strftime("%m-%d-%y. %d %b %Y is a %A on the %d day of %B.")
        daySinceEpoch = (now - date(1970, 1, 1)).days
        stEpoch = '%s since Unix Epoch' % daySinceEpoch
        return strf + '\n' + stEpoch
        #        return """\
        #        %s
        #%s """ % (strf, stEpoch)

if __name__ == "__main__":
    app.run()
<commit_msg>clean code<commit_after>"""
datesvc provides web services around Date
python2 datesvc.py <port>
"""
import web
from datetime import date

urls = (
    '/now', 'Now',
)

web.config.debug=False
app = web.application(urls, globals())

class Now:
    def GET(self):
        now = date.today()
        strf = 'Today is %s ' % now.strftime("%m-%d-%y. %d %b %Y is a %A on the %d day of %B.")
        daySinceEpoch = (now - date(1970, 1, 1)).days
        stEpoch = '%s days since Unix Epoch' % daySinceEpoch
        return strf + '\n' + stEpoch

if __name__ == "__main__":
    app.run()
<|endoftext|>