<commit_before>#
# Jasy - Web Tooling Framework
# Copyright 2013 Sebastian Werner
#

import os, copy, fnmatch, re

import jasy.core.Console as Console

import jasy.item.Abstract

import jasy.style.Util as Util
import jasy.style.Engine as Engine

import jasy.style.tokenize.Tokenizer as Tokenizer

import jasy.style.parse.Parser as Parser
import jasy.style.parse.ScopeScanner as ScopeScanner

import jasy.style.clean.Unused as Unused

import jasy.style.process.Mixins as Mixins
import jasy.style.process.Variables as Variables
import jasy.style.process.Flatter as Flatter

import jasy.core.MetaData as MetaData
import jasy.style.output.Compressor as Compressor

def collectFields(node, keys=None, condition=False):

    if keys is None:
        keys = set()

    if getattr(node, "rel", None) == "condition":
        condition = True

    elif node.type == "command" and node.name == "field":
        keys.add(node.params[0].value)

    elif condition and node.type == "identifier":
        keys.add(node.value)

    # Process children
    for child in reversed(node):
        if child != None:
            collectFields(child, keys, condition)

    return keys

class StyleError(Exception):
    def __init__(self, inst, msg):
        self.__msg = msg
        self.__inst = inst

    def __str__(self):
        return "Error processing stylesheet %s: %s" % (self.__inst, self.__msg)

class StyleItem(jasy.item.Abstract.AbstractItem):

    kind = "style"

    def __getTree(self, context=None):
        """
        Returns the parsed tree
        """

        field = "style:tree[%s]" % self.id
        tree = self.project.getCache().read(field, self.mtime)
        if not tree:
            Console.info("Processing stylesheet %s %s...", Console.colorize(self.id, "bold"), Console.colorize("[%s]" % context, "cyan"))

            Console.indent()
            tree = Engine.getTree(self.getText(), self.id)
            Console.outdent()

            self.project.getCache().store(field, tree, self.mtime, True)

        return tree

    def __getOptimizedTree(self, permutation=None, context=None):
        """
        Returns an optimized tree with permutations applied
        """

        field = "style:opt-tree[%s]-%s" % (self.id, permutation)
        tree = self.project.getCache().read(field, self.mtime)
        if not tree:
            tree = copy.deepcopy(self.__getTree("%s:plain" % context))

            # Logging
            msg = "Processing stylesheet %s" % Console.colorize(self.id, "bold")
            if permutation:
                msg += Console.colorize(" (%s)" % permutation, "grey")
            if context:
                msg += Console.colorize(" [%s]" % context, "cyan")

            Console.info("%s..." % msg)
            Console.indent()

            # Apply permutation
            if permutation:
                Console.debug("Patching tree with permutation: %s", permutation)
                Console.indent()
                Engine.permutateTree(tree, permutation)
                Console.outdent()

            self.project.getCache().store(field, tree, self.mtime, True)
            Console.outdent()

        return tree

    def getBreaks(self, permutation=None, items=None, warnings=True):
        """
        Returns all down-priorized dependencies. This are dependencies which are required to
        make the module work, but are not required being available before the current item.
        """

        meta = self.getMetaData(permutation)
        result = set()

        for entry in meta.breaks:
            if entry == self.id:
                pass
            elif entry in items and items[entry].kind == "style":
                result.add(items[entry])
            elif "*" in entry:
                reobj = re.compile(fnmatch.translate(entry))
                for itemId in items:
                    if itemId != self.id:
                        if reobj.match(itemId):
                            result.add(items[itemId])
            elif warnings:
                Console.warn("Missing item for break command: %s in %s", entry, self.id)

        return result

    def getDependencies(self, permutation=None, items=None, fields=None, warnings=True):
        """
        Returns a set of dependencies seen through the given list of known
        items (ignoring all unknown items in original set). This method also
        makes use of the meta data.
        """

        permutation = self.filterPermutation(permutation)
        meta = self.getMetaData(permutation)
        result = set()

        # Manually defined names/classes
        for entry in meta.requires:
            if entry == self.id:
                pass
            elif entry in items and items[entry].kind == "style":
                result.add(items[entry])
            elif "*" in entry:
                reobj = re.compile(fnmatch.translate(entry))
                for itemId in items:
                    if itemId != self.id:
                        if reobj.match(itemId):
                            result.add(items[itemId])
            elif warnings:
                Console.warn("Missing item for require command: %s in %s", entry, self.id)

        return result

    def getMetaData(self, permutation=None):
        permutation = self.filterPermutation(permutation)

        field = "style:meta[%s]-%s" % (self.id, permutation)
        meta = self.project.getCache().read(field, self.mtime)
        if meta is None:
            meta = MetaData.MetaData(self.__getOptimizedTree(permutation, "meta"))
            self.project.getCache().store(field, meta, self.mtime)

        return meta

    def getFields(self):
        field = "style:fields[%s]" % (self.id)
        fields = self.project.getCache().read(field, self.mtime)
        if fields is None:
            fields = collectFields(self.__getTree(context="fields"))
            self.project.getCache().store(field, fields, self.mtime)

        return fields

    def filterPermutation(self, permutation):
        if permutation:
            fields = self.getFields()
            if fields:
                return permutation.filter(fields)

        return None

    def getMergedTree(self, permutation, session):
        """
        Returns the merged (includes resolved) and optimized (permutation values injected) tree.
        """

        def resolveIncludesRecurser(node):
            for child in node:
                if child.type == "include":
                    valueNode = child[0]
                    if valueNode.type in ("string", "identifier"):
                        includeName = valueNode.value
                    elif valueNode.type == "dot":
                        includeName = Util.assembleDot(valueNode)
                    else:
                        raise Exception("Invalid include: %s" % valueNode)

                    childStyleItem = session.getStyleByName(includeName)

                    if childStyleItem is None:
                        raise Exception("Did not find style sheet: %s" % includeName)

                    # Use merged tree for children as well...
                    childRoot = childStyleItem.getMergedTree(permutation, session)

                    # and copy it for being free to modify it
                    childRoot = copy.deepcopy(childRoot)

                    node.replace(child, childRoot)

                else:
                    resolveIncludesRecurser(child)

        # Work is on base of optimized tree
        tree = self.__getOptimizedTree(permutation, "includes")

        # Copying original tree
        tree = copy.deepcopy(tree)

        # Run the actual resolver engine
        resolveIncludesRecurser(tree)

        return tree

    def getCompressed(self, session, permutation=None, translation=None, optimization=None, formatting=None, context="compressed"):

        tree = self.getMergedTree(permutation, session)

        # Reduce tree
        Engine.reduceTree(tree)

        # Compress tree
        compressed = Compressor.Compressor(formatting).compress(tree)

        return compressed

<commit_msg>Implemented stylesheet caching (compresssed result)<commit_after>#
# Jasy - Web Tooling Framework
# Copyright 2013 Sebastian Werner
#

import os, copy, fnmatch, re

import jasy.core.Console as Console

import jasy.item.Abstract

import jasy.style.Util as Util
import jasy.style.Engine as Engine

import jasy.style.tokenize.Tokenizer as Tokenizer

import jasy.style.parse.Parser as Parser
import jasy.style.parse.ScopeScanner as ScopeScanner

import jasy.style.clean.Unused as Unused

import jasy.style.process.Mixins as Mixins
import jasy.style.process.Variables as Variables
import jasy.style.process.Flatter as Flatter

import jasy.core.MetaData as MetaData
import jasy.style.output.Compressor as Compressor

def collectFields(node, keys=None, condition=False):

    if keys is None:
        keys = set()

    if getattr(node, "rel", None) == "condition":
        condition = True

    elif node.type == "command" and node.name == "field":
        keys.add(node.params[0].value)

    elif condition and node.type == "identifier":
        keys.add(node.value)

    # Process children
    for child in reversed(node):
        if child != None:
            collectFields(child, keys, condition)

    return keys

class StyleError(Exception):
    def __init__(self, inst, msg):
        self.__msg = msg
        self.__inst = inst

    def __str__(self):
        return "Error processing stylesheet %s: %s" % (self.__inst, self.__msg)

class StyleItem(jasy.item.Abstract.AbstractItem):

    kind = "style"

    def __getTree(self, context=None):
        """
        Returns the parsed tree
        """

        field = "style:tree[%s]" % self.id
        tree = self.project.getCache().read(field, self.mtime)
        if not tree:
            Console.info("Processing stylesheet %s %s...", Console.colorize(self.id, "bold"), Console.colorize("[%s]" % context, "cyan"))

            Console.indent()
            tree = Engine.getTree(self.getText(), self.id)
            Console.outdent()

            self.project.getCache().store(field, tree, self.mtime, True)

        return tree

    def __getOptimizedTree(self, permutation=None, context=None):
        """
        Returns an optimized tree with permutations applied
        """

        field = "style:opt-tree[%s]-%s" % (self.id, permutation)
        tree = self.project.getCache().read(field, self.mtime)
        if not tree:
            tree = copy.deepcopy(self.__getTree("%s:plain" % context))

            # Logging
            msg = "Processing stylesheet %s" % Console.colorize(self.id, "bold")
            if permutation:
                msg += Console.colorize(" (%s)" % permutation, "grey")
            if context:
                msg += Console.colorize(" [%s]" % context, "cyan")

            Console.info("%s..." % msg)
            Console.indent()

            # Apply permutation
            if permutation:
                Console.debug("Patching tree with permutation: %s", permutation)
                Console.indent()
                Engine.permutateTree(tree, permutation)
                Console.outdent()

            self.project.getCache().store(field, tree, self.mtime, True)
            Console.outdent()

        return tree

    def getBreaks(self, permutation=None, items=None, warnings=True):
        """
        Returns all down-priorized dependencies. This are dependencies which are required to
        make the module work, but are not required being available before the current item.
        """

        meta = self.getMetaData(permutation)
        result = set()

        for entry in meta.breaks:
            if entry == self.id:
                pass
            elif entry in items and items[entry].kind == "style":
                result.add(items[entry])
            elif "*" in entry:
                reobj = re.compile(fnmatch.translate(entry))
                for itemId in items:
                    if itemId != self.id:
                        if reobj.match(itemId):
                            result.add(items[itemId])
            elif warnings:
                Console.warn("Missing item for break command: %s in %s", entry, self.id)

        return result

    def getDependencies(self, permutation=None, items=None, fields=None, warnings=True):
        """
        Returns a set of dependencies seen through the given list of known
        items (ignoring all unknown items in original set). This method also
        makes use of the meta data.
        """

        permutation = self.filterPermutation(permutation)
        meta = self.getMetaData(permutation)
        result = set()

        # Manually defined names/classes
        for entry in meta.requires:
            if entry == self.id:
                pass
            elif entry in items and items[entry].kind == "style":
                result.add(items[entry])
            elif "*" in entry:
                reobj = re.compile(fnmatch.translate(entry))
                for itemId in items:
                    if itemId != self.id:
                        if reobj.match(itemId):
                            result.add(items[itemId])
            elif warnings:
                Console.warn("Missing item for require command: %s in %s", entry, self.id)

        return result

    def getMetaData(self, permutation=None):
        permutation = self.filterPermutation(permutation)

        field = "style:meta[%s]-%s" % (self.id, permutation)
        meta = self.project.getCache().read(field, self.mtime)
        if meta is None:
            meta = MetaData.MetaData(self.__getOptimizedTree(permutation, "meta"))
            self.project.getCache().store(field, meta, self.mtime)

        return meta

    def getFields(self):
        field = "style:fields[%s]" % (self.id)
        fields = self.project.getCache().read(field, self.mtime)
        if fields is None:
            fields = collectFields(self.__getTree(context="fields"))
            self.project.getCache().store(field, fields, self.mtime)

        return fields

    def filterPermutation(self, permutation):
        if permutation:
            fields = self.getFields()
            if fields:
                return permutation.filter(fields)

        return None

    def getMergedTree(self, permutation, session):
        """
        Returns the merged (includes resolved) and optimized (permutation values injected) tree.
        """

        def resolveIncludesRecurser(node):
            for child in node:
                if child.type == "include":
                    valueNode = child[0]
                    if valueNode.type in ("string", "identifier"):
                        includeName = valueNode.value
                    elif valueNode.type == "dot":
                        includeName = Util.assembleDot(valueNode)
                    else:
                        raise Exception("Invalid include: %s" % valueNode)

                    childStyleItem = session.getStyleByName(includeName)

                    if childStyleItem is None:
                        raise Exception("Did not find style sheet: %s" % includeName)

                    # Use merged tree for children as well...
                    childRoot = childStyleItem.getMergedTree(permutation, session)

                    # and copy it for being free to modify it
                    childRoot = copy.deepcopy(childRoot)

                    node.replace(child, childRoot)

                else:
                    resolveIncludesRecurser(child)

        # Work is on base of optimized tree
        tree = self.__getOptimizedTree(permutation, "includes")

        # Copying original tree
        tree = copy.deepcopy(tree)

        # Run the actual resolver engine
        resolveIncludesRecurser(tree)

        return tree

    def getCompressed(self, session, permutation=None, translation=None, optimization=None, formatting=None, context="compressed"):

        # Disable translation for caching / patching when not actually used
        if translation and not self.getTranslations():
            translation = None

        field = "style:compressed[%s]-%s-%s-%s-%s" % (self.id, permutation, translation, optimization, formatting)
        compressed = self.project.getCache().read(field, self.mtime)
        if compressed == None:

            # Start with the merged tree (includes resolved)
            tree = self.getMergedTree(permutation, session)

            # Reduce tree
            Engine.reduceTree(tree)

            # Compress tree
            compressed = Compressor.Compressor(formatting).compress(tree)

            # Store in cache
            self.project.getCache().store(field, compressed, self.mtime)

        return compressed

    def getTranslations(self):
        # TODO
        return None

        field = "style:translations[%s]" % (self.id)
        result = self.project.getCache().read(field, self.mtime)
        if result is None:
            result = jasy.js.optimize.Translation.collectTranslations(self.__getTree(context="i18n"))
            self.project.getCache().store(field, result, self.mtime)

        return result

<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
from osfs import OSFS
import tempfile

from fs.errors import *

class TempFS(OSFS):

    """Create a Filesystem in a tempory directory (with tempfile.mkdtemp),
    and removes it when the TempFS object is cleaned up."""

    def __init__(self, identifier=None, temp_dir=None, dir_mode=0700, thread_synchronize=True):
        """Creates a temporary Filesystem

        identifier -- A string that is included in the name of the temporary directory,
        default uses "TempFS"

        """
        self._temp_dir = tempfile.mkdtemp(identifier or "TempFS",dir=temp_dir)
        self._cleaned = False
        OSFS.__init__(self, self._temp_dir, dir_mode=dir_mode, thread_synchronize=thread_synchronize)

    def __str__(self):
        return '<TempFS: %s>' % self._temp_dir

    __repr__ = __str__

    def __unicode__(self):
        return unicode(self.__str__())

    @convert_os_errors
    def close(self):
        """Removes the temporary directory.
        This will be called automatically when the object is cleaned up by
        Python. Note that once this method has been called, the FS object may
        no longer be used.
        """
        if not self._cleaned and self.exists("/"):
            self._lock.acquire()
            try:
                # shutil.rmtree doesn't handle long paths on win32,
                # so we walk the tree by hand.
                entries = os.walk(self.root_path,topdown=False)
                for (dir,dirnames,filenames) in entries:
                    for filename in filenames:
                        os.remove(os.path.join(dir,filename))
                    for dirname in dirnames:
                        os.rmdir(os.path.join(dir,dirname))
                os.rmdir(self.root_path)
                self._cleaned = True
            finally:
                self._lock.release()
        super(TempFS,self).close()

<commit_msg>TempFS.close(): recover from temporarily locked resources on win32<commit_after>#!/usr/bin/env python

import os
import time
import tempfile

from fs.osfs import OSFS
from fs.errors import *

class TempFS(OSFS):

    """Create a Filesystem in a tempory directory (with tempfile.mkdtemp),
    and removes it when the TempFS object is cleaned up."""

    def __init__(self, identifier=None, temp_dir=None, dir_mode=0700, thread_synchronize=True):
        """Creates a temporary Filesystem

        identifier -- A string that is included in the name of the temporary directory,
        default uses "TempFS"

        """
        self._temp_dir = tempfile.mkdtemp(identifier or "TempFS",dir=temp_dir)
        self._cleaned = False
        OSFS.__init__(self, self._temp_dir, dir_mode=dir_mode, thread_synchronize=thread_synchronize)

    def __str__(self):
        return '<TempFS: %s>' % self._temp_dir

    __repr__ = __str__

    def __unicode__(self):
        return unicode(self.__str__())

    def close(self):
        """Removes the temporary directory.

        This will be called automatically when the object is cleaned up by
        Python. Note that once this method has been called, the FS object may
        no longer be used.
        """
        try:
            self._close()
        except ResourceLockedError:
            # Give win32 a chance to clean up after itself
            time.sleep(0.5)
            self._close()

    @convert_os_errors
    def _close(self):
        """Actual implementation of close().

        This is a separate method so it can be re-tried in the face of
        transient errors.
        """
        if not self._cleaned and self.exists("/"):
            self._lock.acquire()
            try:
                # shutil.rmtree doesn't handle long paths on win32,
                # so we walk the tree by hand.
                entries = os.walk(self.root_path,topdown=False)
                for (dir,dirnames,filenames) in entries:
                    for filename in filenames:
                        os.remove(os.path.join(dir,filename))
                    for dirname in dirnames:
                        os.rmdir(os.path.join(dir,dirname))
                os.rmdir(self.root_path)
                self._cleaned = True
            finally:
                self._lock.release()
        super(TempFS,self).close()

<|endoftext|>


<commit_before>#    The MIT License (MIT)
#
#    Copyright (c) 2014 kyle kersey
#
#    Permission is hereby granted, free of charge, to any person obtaining a copy
#    of this software and associated documentation files (the "Software"), to deal
#    in the Software without restriction, including without limitation the rights
#    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#    copies of the Software, and to permit persons to whom the Software is
#    furnished to do so, subject to the following conditions:
#
#    The above copyright notice and this permission notice shall be included in all
#    copies or substantial portions of the Software.
#
#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

import urllib2
import json
import sqlite3
import time

# sqlite database file
cache_database = "movies.db"
# how many seconds before the entry expires
cache_expiration = 60 * 60  # one hour

class Cache:
    def get_conn(self):
        """
            connect with the sqlite database
        """
        conn = sqlite3.connect(cache_database)
        c = conn.cursor()
        c.execute(""" CREATE TABLE IF NOT EXISTS movies(
            search_query TEXT PRIMARY KEY,
            page_number INT,
            timestamp INTEGER,
            search_results BLOB); """)
        return conn

    def get(self, search_query, page_number):
        """
            get the search results from the database
        """
        with self.get_conn() as conn:
            c = conn.cursor()
            query = """SELECT search_results FROM movies WHERE
                search_query = ? AND
                page_number = ? AND
                (strftime('%s', 'now') - timestamp) < ?"""

            return c.execute(query, (search_query, page_number, cache_expiration)).fetchone()

    def put(self, search_query, page_number, search_results):
        """
            put the results into the database
        """
        timestamp = int(time.time())
        with self.get_conn() as conn:
            c = conn.cursor()
            insert = """INSERT OR REPLACE INTO movies
                        (search_query, page_number, timestamp, search_results)
                        VALUES (?, ?, ?, ?); """

            c.execute(insert, (search_query, page_number, timestamp, search_results,))
            conn.commit()

class RottenTomatoes:
    api_key = ""
    userAgent = "MovieInfoBot/1.0"

    def search(self, query, results_per_page=25, page_number=1):
        """
            searches for movies: movie name, result limit, page number
        """
        cache = Cache()
        result = cache.get(query, page_number)
        if result:
            # print "using cache"
            movie_json = result[0]
        else:
            # print "using web"
            #format the url
            base_url = "http://api.rottentomatoes.com/api/public/v1.0/movies.json"
            url = "{base_url}?apikey={api_key}&q={search_term}&page_limit={results_per_page}&page={page_number}"
            param = {}
            param["base_url"] = base_url
            param["api_key"] = self.api_key
            param["search_term"] = urllib2.quote(query.encode("utf8"))
            param["results_per_page"] = results_per_page
            param["page_number"] = page_number
            url = url.format(**param)
            req = urllib2.Request(url, headers={'User-Agent': self.userAgent})
            movie_json = urllib2.urlopen(req).read()

            # put the results into the movie cache
            cache.put(query, page_number, movie_json)

        movie_dict = json.loads(movie_json)
        return movie_dict
<commit_msg>use modern style classes<commit_after>#    The MIT License (MIT)
#
#    Copyright (c) 2014 kyle kersey
#
#    Permission is hereby granted, free of charge, to any person obtaining a copy
#    of this software and associated documentation files (the "Software"), to deal
#    in the Software without restriction, including without limitation the rights
#    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#    copies of the Software, and to permit persons to whom the Software is
#    furnished to do so, subject to the following conditions:
#
#    The above copyright notice and this permission notice shall be included in all
#    copies or substantial portions of the Software.
#
#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

import urllib2
import json
import sqlite3
import time

# sqlite database file
cache_database = "movies.db"
# how many seconds before the entry expires
cache_expiration = 60 * 60  # one hour

class Cache(object):
    def get_conn(self):
        """
            connect with the sqlite database
        """
        conn = sqlite3.connect(cache_database)
        c = conn.cursor()
        c.execute(""" CREATE TABLE IF NOT EXISTS movies(
            search_query TEXT PRIMARY KEY,
            page_number INT,
            timestamp INTEGER,
            search_results BLOB); """)
        return conn

    def get(self, search_query, page_number):
        """
            get the search results from the database
        """
        with self.get_conn() as conn:
            c = conn.cursor()
            query = """SELECT search_results FROM movies WHERE
                search_query = ? AND
                page_number = ? AND
                (strftime('%s', 'now') - timestamp) < ?"""

            return c.execute(query, (search_query, page_number, cache_expiration)).fetchone()

    def put(self, search_query, page_number, search_results):
        """
            put the results into the database
        """
        timestamp = int(time.time())
        with self.get_conn() as conn:
            c = conn.cursor()
            insert = """INSERT OR REPLACE INTO movies
                        (search_query, page_number, timestamp, search_results)
                        VALUES (?, ?, ?, ?); """

            c.execute(insert, (search_query, page_number, timestamp, search_results,))
            conn.commit()

class RottenTomatoes(object):
    api_key = ""
    userAgent = "MovieInfoBot/1.0"

    def search(self, query, results_per_page=25, page_number=1):
        """
            searches for movies: movie name, result limit, page number
        """
        cache = Cache()
        result = cache.get(query, page_number)
        if result:
            # print "using cache"
            movie_json = result[0]
        else:
            # print "using web"
            #format the url
            base_url = "http://api.rottentomatoes.com/api/public/v1.0/movies.json"
            url = "{base_url}?apikey={api_key}&q={search_term}&page_limit={results_per_page}&page={page_number}"
            param = {}
            param["base_url"] = base_url
            param["api_key"] = self.api_key
            param["search_term"] = urllib2.quote(query.encode("utf8"))
            param["results_per_page"] = results_per_page
            param["page_number"] = page_number
            url = url.format(**param)
            req = urllib2.Request(url, headers={'User-Agent': self.userAgent})
            movie_json = urllib2.urlopen(req).read()

            # put the results into the movie cache
            cache.put(query, page_number, movie_json)

        movie_dict = json.loads(movie_json)
        return movie_dict
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os,sys

# import all necessary stuff
import osg
import osgDB
import osgViewer
import osgART

# just a convenience function
def createImageBackground(video):
	layer = osgART.VideoLayer()
	layer.setSize(video)
	geode = osgART.VideoGeode(osgART.VideoGeode.USE_TEXTURE_2D, video)
	osgART.addTexturedQuad(geode,video.s(),video.t())
	layer.addChild(geode)
	return layer

# create a root node
root = osg.Group()

# only use Python path
osgDB.Registry.instance().setLibraryFilePathList(sys.path)

# create a viewer
viewer = osgViewer.Viewer()

# add some convenience handlers
viewer.addEventHandler(osgViewer.WindowSizeHandler())
viewer.addEventHandler(osgViewer.StatsHandler())
viewer.addEventHandler(osgViewer.ThreadingHandler())
viewer.addEventHandler(osgViewer.HelpHandler())

# set the scene root
viewer.setSceneData(root)

# preload plugins
video_id = osgART.PluginManager.getInstance().load("osgart_video_artoolkit2")
tracker_id = osgART.PluginManager.getInstance().load("osgart_tracker_artoolkit2")

# create a video source (move to settings)
video = osgART.Video.cast(osgART.PluginManager.getInstance().get(video_id))

# open the video
video.open()

# tracker
tracker = osgART.Tracker.cast(osgART.PluginManager.getInstance().get(tracker_id))

# create a calibration object
calibration = tracker.getOrCreateCalibration()

# load camera parameter
tracker.getOrCreateCalibration().load("data/camera_para.dat")

# initialize the tracker
tracker.setImage(video)

# add a tracker callback
osgART.TrackerCallback.addOrSet(root,tracker)

# create a marker
marker = tracker.addMarker("single;data/patt.hiro;80;0;0")

# set the marker active
marker.setActive(True)

# create a matrix transfrom utilised through osgART
ar_transform = osg.MatrixTransform()

# add the default chain of event handlers
osgART.attachDefaultEventCallbacks(ar_transform,marker)

# add a cube
ar_transform.addChild(osgART.testCube())

# need to set the renderbin > render this one last
ar_transform.getOrCreateStateSet().setRenderBinDetails(100, "RenderBin")

# create a video background
video_background = createImageBackground(video)

# set the renderbin on the video background > rendered first
video_background.getOrCreateStateSet().setRenderBinDetails(0, "RenderBin");

# create a camera from the calibration object
cam = calibration.createCamera();

# attach the AR scene and the video background
cam.addChild(ar_transform)
cam.addChild(video_background)

# add the camera to the root node
root.addChild(cam)

# start the video capture
video.start()

# let the viewer do the rest
viewer.run()

<commit_msg>- fixed to updated osgART 2.0 API<commit_after>#!/usr/bin/env python

import os,sys

# import all necessary stuff
import osg
import osgDB
import osgViewer
import osgART

# just a convenience function
def createImageBackground(video):
	layer = osgART.VideoLayer()
	layer.setSize(video)
	geode = osgART.VideoGeode(osgART.VideoGeode.USE_TEXTURE_2D, video)
	osgART.addTexturedQuad(geode,video.s(),video.t())
	layer.addChild(geode)
	return layer

# create a root node
root = osg.Group()

# only use Python path
osgDB.Registry.instance().setLibraryFilePathList(sys.path)

# create a viewer
viewer = osgViewer.Viewer()

# add some convenience handlers
viewer.addEventHandler(osgViewer.WindowSizeHandler())
viewer.addEventHandler(osgViewer.StatsHandler())
viewer.addEventHandler(osgViewer.ThreadingHandler())
viewer.addEventHandler(osgViewer.HelpHandler())

# set the scene root
viewer.setSceneData(root)

# preload plugins
video_id = osgART.PluginManager.instance().load("osgart_video_artoolkit2")
tracker_id = osgART.PluginManager.instance().load("osgart_tracker_artoolkit2")

# create a video source (move to settings)
video = osgART.Video.cast(osgART.PluginManager.instance().get(video_id))

# open the video
video.open()

# tracker
tracker = osgART.Tracker.cast(osgART.PluginManager.instance().get(tracker_id))

# create a calibration object
calibration = tracker.getOrCreateCalibration()

# load camera parameter
tracker.getOrCreateCalibration().load("data/camera_para.dat")

# initialize the tracker
tracker.setImage(video)

# add a tracker callback
osgART.TrackerCallback.addOrSet(root,tracker)

# create a marker
marker = tracker.addMarker("single;data/patt.hiro;80;0;0")

# set the marker active
marker.setActive(True)

# create a matrix transfrom utilised through osgART
ar_transform = osg.MatrixTransform()

# add the default chain of event handlers
osgART.attachDefaultEventCallbacks(ar_transform,marker)

# add a cube
ar_transform.addChild(osgART.testCube())

# need to set the renderbin > render this one last
ar_transform.getOrCreateStateSet().setRenderBinDetails(100, "RenderBin")

# create a video background
video_background = createImageBackground(video)

# set the renderbin on the video background > rendered first
video_background.getOrCreateStateSet().setRenderBinDetails(0, "RenderBin");

# create a camera from the calibration object
cam = calibration.createCamera();

# attach the AR scene and the video background
cam.addChild(ar_transform)
cam.addChild(video_background)

# add the camera to the root node
root.addChild(cam)

# start the video capture
video.start()

# let the viewer do the rest
viewer.run()

<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# IMPORTANT NOTE: If you make local mods to this file, you must run:
#   %  pnacl/build.sh driver
# in order for them to take effect in the scons build.  This command
# updates the copy in the toolchain/ tree.
#

from driver_tools import Run, ParseArgs
from driver_env import env

EXTRA_ENV = [ 'ARGS'          : '' ]
# just pass all args through to 'ARGS' and eventually to the underlying tool
PATTERNS = [ ( '(.*)',  "env.append('ARGS', $0)") ]

def main(argv):
  env.update(EXTRA_ENV)
  ParseArgs(argv, PATTERNS)
  Run('"${NM}" --plugin=LLVMgold ${ARGS}')
  # only reached in case of no errors
  return 0

def get_help(unused_argv):
  return """
Usage: %s [option(s)] [file(s)]
 List symbols in [file(s)] (a.out by default).
 The options are:
  -a, --debug-syms       Display debugger-only symbols
  -A, --print-file-name  Print name of the input file before every symbol
  -B                     Same as --format=bsd
  -C, --demangle[=STYLE] Decode low-level symbol names into user-level names
                          The STYLE, if specified, can be `auto' (the default),
                          `gnu', `lucid', `arm', `hp', `edg', `gnu-v3', `java'
                          or `gnat'
      --no-demangle      Do not demangle low-level symbol names
  -D, --dynamic          Display dynamic symbols instead of normal symbols
      --defined-only     Display only defined symbols
  -e                     (ignored)
  -f, --format=FORMAT    Use the output format FORMAT.  FORMAT can be `bsd',
                           `sysv' or `posix'.  The default is `bsd'
  -g, --extern-only      Display only external symbols
  -l, --line-numbers     Use debugging information to find a filename and
                           line number for each symbol
  -n, --numeric-sort     Sort symbols numerically by address
  -o                     Same as -A
  -p, --no-sort          Do not sort the symbols
  -P, --portability      Same as --format=posix
  -r, --reverse-sort     Reverse the sense of the sort
  -S, --print-size       Print size of defined symbols
  -s, --print-armap      Include index for symbols from archive members
      --size-sort        Sort symbols by size
      --special-syms     Include special symbols in the output
      --synthetic        Display synthetic symbols as well
  -t, --radix=RADIX      Use RADIX for printing symbol values
      --target=BFDNAME   Specify the target object format as BFDNAME
  -u, --undefined-only   Display only undefined symbols
  -X 32_64               (ignored)
  @FILE                  Read options from FILE
  -h, --help             Display this information
  -V, --version          Display this program's version number
""" % env.getone('SCRIPT_NAME')
<commit_msg>Fix wrong braces in pnacl-nm driver<commit_after>#!/usr/bin/python
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# IMPORTANT NOTE: If you make local mods to this file, you must run:
#   %  pnacl/build.sh driver
# in order for them to take effect in the scons build.  This command
# updates the copy in the toolchain/ tree.
#

from driver_tools import Run, ParseArgs
from driver_env import env

EXTRA_ENV = { 'ARGS'          : '' }
# just pass all args through to 'ARGS' and eventually to the underlying tool
PATTERNS = [ ( '(.*)',  "env.append('ARGS', $0)") ]

def main(argv):
  env.update(EXTRA_ENV)
  ParseArgs(argv, PATTERNS)
  Run('"${NM}" --plugin=LLVMgold ${ARGS}')
  # only reached in case of no errors
  return 0

def get_help(unused_argv):
  return """
Usage: %s [option(s)] [file(s)]
 List symbols in [file(s)] (a.out by default).
 The options are:
  -a, --debug-syms       Display debugger-only symbols
  -A, --print-file-name  Print name of the input file before every symbol
  -B                     Same as --format=bsd
  -C, --demangle[=STYLE] Decode low-level symbol names into user-level names
                          The STYLE, if specified, can be `auto' (the default),
                          `gnu', `lucid', `arm', `hp', `edg', `gnu-v3', `java'
                          or `gnat'
      --no-demangle      Do not demangle low-level symbol names
  -D, --dynamic          Display dynamic symbols instead of normal symbols
      --defined-only     Display only defined symbols
  -e                     (ignored)
  -f, --format=FORMAT    Use the output format FORMAT.  FORMAT can be `bsd',
                           `sysv' or `posix'.  The default is `bsd'
  -g, --extern-only      Display only external symbols
  -l, --line-numbers     Use debugging information to find a filename and
                           line number for each symbol
  -n, --numeric-sort     Sort symbols numerically by address
  -o                     Same as -A
  -p, --no-sort          Do not sort the symbols
  -P, --portability      Same as --format=posix
  -r, --reverse-sort     Reverse the sense of the sort
  -S, --print-size       Print size of defined symbols
  -s, --print-armap      Include index for symbols from archive members
      --size-sort        Sort symbols by size
      --special-syms     Include special symbols in the output
      --synthetic        Display synthetic symbols as well
  -t, --radix=RADIX      Use RADIX for printing symbol values
      --target=BFDNAME   Specify the target object format as BFDNAME
  -u, --undefined-only   Display only undefined symbols
  -X 32_64               (ignored)
  @FILE                  Read options from FILE
  -h, --help             Display this information
  -V, --version          Display this program's version number
""" % env.getone('SCRIPT_NAME')
<|endoftext|>


<commit_before># coding: utf-8

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Third-party
import numpy as np
from scipy.signal import argrelmax, argrelmin

# Project
import gary.dynamics as gd
import gary.integrate as gi

__all__ = ['ptp_periods']

def ptp_periods(t, *coords):
    """
    Estimate the dominant periods of given coordinates by
    computing the mean peak-to-peak time.

    Parameters
    ----------
    t : array_like
        Array of times. Must have same shape as individual coordinates.
    *coords
        Positional arguments allow specifying coordinates to compute the
        periods of. For example, these could simply be x,y,z, or
        could be R,phi,z for cylindrical coordinates.

    """

    freqs = []
    for x in coords:
        # first compute mean peak-to-peak
        ix = argrelmax(x, mode='wrap')[0]
        f_max = np.mean(t[ix[1:]] - t[ix[:-1]])

        # now compute mean trough-to-trough
        ix = argrelmin(x, mode='wrap')[0]
        f_min = np.mean(t[ix[1:]] - t[ix[:-1]])

        # then take the mean of these two
        freqs.append(np.mean([f_max, f_min]))

    return np.array(freqs)

def estimate_max_period(t, w):
    """
    Given an array of times and an orbit, estimate the longest period
    in the orbit. We will then use this to figure out how long to
    integrate for when frequency mapping.

    Parameters
    ----------
    t : array_like
        Array of times.
    w : array_like
        Single orbit -- should have shape (len(t), 6)

    """

    # circulation
    circ = gd.classify_orbit(w)

    if np.any(circ):  # TUBE ORBIT - pass R,φ,z
        # flip coords
        new_w = gd.align_circulation_with_z(w, circ[0])[:,0]

        # convert to cylindrical
        R = np.sqrt(new_w[:,0]**2 + new_w[:,1]**2)
        phi = np.arctan2(new_w[:,1], new_w[:,0])
        z = new_w[:,2]

        T = ptp_periods(t, R, phi, z)

    else:  # BOX ORBIT - pass x,y,z
        T = ptp_periods(t, *w[:,:3].T)

    return T.max()

def estimate_dt_nsteps(potential, w0, nperiods=200, nsteps_per_period=100):
    """
    Estimate the timestep and number of steps to integrate for given a potential
    and set of initial conditions.

    Parameters
    ----------
    potential : :class:`~gary.potential.Potential`
    w0 : array_like
    nperiods : int (optional)
        Number of (max) periods to integrate.
    nsteps_per_period : int (optional)
        Number of steps to take per (max) orbital period.

    """

    # integrate orbit
    t,w = potential.integrate_orbit(w0, dt=2., nsteps=25000,
                                    Integrator=gi.DOPRI853Integrator)

    # estimate the maximum period
    max_T = estimate_max_period(t, w)

    # timestep from number of steps per period
    dt = float(max_T) / float(nsteps_per_period)

    # integrate for nperiods times the max period
    nsteps = int(round(nperiods * nsteps_per_period, -4))

    if dt == 0.:
        raise ValueError("Timestep is zero!")

    return dt, nsteps
<commit_msg>add function names to all<commit_after># coding: utf-8

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Third-party
import numpy as np
from scipy.signal import argrelmax, argrelmin

# Project
import gary.dynamics as gd
import gary.integrate as gi

__all__ = ['ptp_periods', 'estimate_max_period', 'estimate_dt_nsteps']

def ptp_periods(t, *coords):
    """
    Estimate the dominant periods of given coordinates by
    computing the mean peak-to-peak time.

    Parameters
    ----------
    t : array_like
        Array of times. Must have same shape as individual coordinates.
    *coords
        Positional arguments allow specifying coordinates to compute the
        periods of. For example, these could simply be x,y,z, or
        could be R,phi,z for cylindrical coordinates.

    """

    freqs = []
    for x in coords:
        # first compute mean peak-to-peak
        ix = argrelmax(x, mode='wrap')[0]
        f_max = np.mean(t[ix[1:]] - t[ix[:-1]])

        # now compute mean trough-to-trough
        ix = argrelmin(x, mode='wrap')[0]
        f_min = np.mean(t[ix[1:]] - t[ix[:-1]])

        # then take the mean of these two
        freqs.append(np.mean([f_max, f_min]))

    return np.array(freqs)

def estimate_max_period(t, w):
    """
    Given an array of times and an orbit, estimate the longest period
    in the orbit. We will then use this to figure out how long to
    integrate for when frequency mapping.

    Parameters
    ----------
    t : array_like
        Array of times.
    w : array_like
        Single orbit -- should have shape (len(t), 6)

    """

    # circulation
    circ = gd.classify_orbit(w)

    if np.any(circ):  # TUBE ORBIT - pass R,φ,z
        # flip coords
        new_w = gd.align_circulation_with_z(w, circ[0])[:,0]

        # convert to cylindrical
        R = np.sqrt(new_w[:,0]**2 + new_w[:,1]**2)
        phi = np.arctan2(new_w[:,1], new_w[:,0])
        z = new_w[:,2]

        T = ptp_periods(t, R, phi, z)

    else:  # BOX ORBIT - pass x,y,z
        T = ptp_periods(t, *w[:,:3].T)

    return T.max()

def estimate_dt_nsteps(potential, w0, nperiods=200, nsteps_per_period=100):
    """
    Estimate the timestep and number of steps to integrate for given a potential
    and set of initial conditions.

    Parameters
    ----------
    potential : :class:`~gary.potential.Potential`
    w0 : array_like
    nperiods : int (optional)
        Number of (max) periods to integrate.
    nsteps_per_period : int (optional)
        Number of steps to take per (max) orbital period.

    """

    # integrate orbit
    t,w = potential.integrate_orbit(w0, dt=2., nsteps=25000,
                                    Integrator=gi.DOPRI853Integrator)

    # estimate the maximum period
    max_T = estimate_max_period(t, w)

    # timestep from number of steps per period
    dt = float(max_T) / float(nsteps_per_period)

    # integrate for nperiods times the max period
    nsteps = int(round(nperiods * nsteps_per_period, -4))

    if dt == 0.:
        raise ValueError("Timestep is zero!")

    return dt, nsteps
<|endoftext|>


<commit_before>load("@io_bazel_rules_go//go:def.bzl", "go_test")

"""Macros for running acceptance tests."""

def acceptance_test_suite(name, srcs, testdata, builder, structure_test_config = ":config.yaml", args = None, deps = None, **kwargs):
    """Macro to define an acceptance test.

    Args:
      name: the name of the test
      srcs: the test source files
      testdata: a build target for a directory containing sample test applications
      builder: a build target for a builder.tar to test
      structure_test_config: a build target for the structured container test's config file
      args: additional arguments to be passed to the test binary beyond ones corresponding to the arguments to this function
      deps: additional test dependencies beyond the acceptance package
      **kwargs: this argument captures all additional arguments and forwards them to the generated go_test rule
    """

    go_test(
        name = name,
        size = "enormous",
        srcs = srcs,
        args = _build_args(args, name, testdata, builder, structure_test_config),
        data = [
            structure_test_config,
            builder,
            testdata,
        ],
        tags = [
            "local",
        ],
        deps = _build_deps(deps),
        **kwargs
    )

def _build_args(args, name, testdata, builder, structure_test_config):
    short_name = _remove_suffix(name, "_test")
    builder_name = _extract_builder_name(builder)

    if args == None:
        args = []
    args.append("-test-data=$(location " + testdata + ")")
    args.append("-structure-test-config=$(location " + structure_test_config + ")")
    args.append("-builder-source=$(location " + builder + ")")
    args.append("-builder-prefix=" + builder_name + "-" + short_name + "-acceptance-test-")
    return args

def _extract_builder_name(builder):
    # A builder target is a full google3 path, the name of the builder, and then :builder.tar, the following
    # extracts the name of the builder.
    builder_name = _remove_suffix(builder, ":builder.tar")
    builder_name = builder_name[builder_name.rindex("/") + 1:]
    return builder_name

def _build_deps(deps):
    if deps == None:
        deps = []
    deps.append("//internal/acceptance")
    return deps

# Once bazel supports python 3.9, this function can be replaced with `value.removesuffix(suffix)`:
def _remove_suffix(value, suffix):
    if value.endswith(suffix):
        value = value[:-len(suffix)]
    return value
<commit_msg>Internal change.<commit_after>load("@io_bazel_rules_go//go:def.bzl", "go_test")

"""Macros for running acceptance tests."""

def acceptance_test_suite(name, srcs, testdata, builder, structure_test_config = ":config.yaml", args = None, deps = None, **kwargs):
    """Macro to define an acceptance test.

    Args:
      name: the name of the test
      srcs: the test source files
      testdata: a build target for a directory containing sample test applications
      builder: a build target for a builder.tar to test
      structure_test_config: a build target for the structured container test's config file
      args: additional arguments to be passed to the test binary beyond ones corresponding to the arguments to this function
      deps: additional test dependencies beyond the acceptance package
      **kwargs: this argument captures all additional arguments and forwards them to the generated go_test rule
    """

    go_test(
        name = name,
        size = "enormous",
        srcs = srcs,
        args = _build_args(args, name, testdata, builder, structure_test_config),
        data = [
            structure_test_config,
            builder,
            testdata,
        ],
        tags = [
            "local",
        ],
        gc_linkopts = [
            "-I",
            "/lib64/ld-linux-x86-64.so.2",
            "-extldflags=\"-Wl,--dynamic-linker,/lib64/ld-linux-x86-64.so.2\"",
        ],
        deps = _build_deps(deps),
        **kwargs
    )

def _build_args(args, name, testdata, builder, structure_test_config):
    short_name = _remove_suffix(name, "_test")
    builder_name = _extract_builder_name(builder)

    if args == None:
        args = []
    args.append("-test-data=$(location " + testdata + ")")
    args.append("-structure-test-config=$(location " + structure_test_config + ")")
    args.append("-builder-source=$(location " + builder + ")")
    args.append("-builder-prefix=" + builder_name + "-" + short_name + "-acceptance-test-")
    return args

def _extract_builder_name(builder):
    # A builder target is a full google3 path, the name of the builder, and then :builder.tar, the following
    # extracts the name of the builder.
    builder_name = _remove_suffix(builder, ":builder.tar")
    builder_name = builder_name[builder_name.rindex("/") + 1:]
    return builder_name

def _build_deps(deps):
    if deps == None:
        deps = []
    deps.append("//internal/acceptance")
    return deps

# Once bazel supports python 3.9, this function can be replaced with `value.removesuffix(suffix)`:
def _remove_suffix(value, suffix):
    if value.endswith(suffix):
        value = value[:-len(suffix)]
    return value
<|endoftext|>


<commit_before>"""Macros for building Objective-C libraries with strict warnings."""

# This list is derived from
# http://programmers.stackexchange.com/questions/122608/clang-warning-flags-for-objective-c-development
# with a few turned off because the warning code is widely present in Material
# Components and seems innocuous.
COMMON_COPTS = [
    "-Wall",  # Standard known-to-be-bugs warnings.
    "-Wcast-align",  # Casting a pointer such that alignment is broken.
    "-Wconversion",  # Numeric conversion warnings.
    "-Wdocumentation",  # Documentation checks.
    "-Werror",  # All warnings as errors.
    "-Wextra",  # Many useful extra warnings.
    "-Wdocumentation",  # Warn when documentation is out of date.
    "-Wimplicit-atomic-properties",  # Dynamic properties should be non-atomic.
    "-Wmissing-prototypes",  # Global function is defined without a previous prototype.
    "-Wnewline-eof",  # No newline at the end of the file.
    "-Wno-error=deprecated",  # Deprecation warnings are never errors.
    "-Wno-error=deprecated-implementations",  # Deprecation warnings are never errors.
    "-Wno-partial-availability",
    "-Wno-sign-conversion",  # Do not warn on sign conversions.
    "-Wno-unused-parameter",  # Do not warn on unused parameters.
    "-Woverlength-strings",  # Strings longer than the C maximum.
    "-Wshadow",  # Local variable shadows another variable, parameter, etc.
    "-Wstrict-selector-match",  # Compiler can't figure out the right selector.
    "-Wstrict-prototypes",
    "-Wundeclared-selector",  # Compiler doesn't see a selector.
    "-Wunreachable-code",  # Code will never be reached.
    "-Wunused-comparison",
    "-Wunused-const-variable",
    "-Wunused-exception-parameter",
    "-Wunused-function",
    "-Wunused-label",
    "-Wunused-member-function",
    "-Wunused-private-field",
    "-Wunused-property-ivar",
    "-Wunused-result",
    "-Wunused-value",
    "-Wunused-variable",
    "-Wunused-volatile-lvalue",
    "-Wused-but-marked-unused",
]

def strict_warnings_objc_library(
    name,
    copts = [],
    **kwargs):
  """Declare an Objective-C library with strict warning flags."""
  native.objc_library(
      name = name,
      copts = copts + COMMON_COPTS,
      **kwargs)
<commit_msg>Disable `unused-parameter` warnings. (#4)<commit_after>"""Macros for building Objective-C libraries with strict warnings."""

# This list is derived from
# http://programmers.stackexchange.com/questions/122608/clang-warning-flags-for-objective-c-development
# with a few turned off because the warning code is widely present in Material
# Components and seems innocuous.
COMMON_COPTS = [
    "-Wall",  # Standard known-to-be-bugs warnings.
    "-Wcast-align",  # Casting a pointer such that alignment is broken.
    "-Wconversion",  # Numeric conversion warnings.
    "-Wdocumentation",  # Documentation checks.
    "-Werror",  # All warnings as errors.
    "-Wextra",  # Many useful extra warnings.
    "-Wdocumentation",  # Warn when documentation is out of date.
    "-Wimplicit-atomic-properties",  # Dynamic properties should be non-atomic.
    "-Wmissing-prototypes",  # Global function is defined without a previous prototype.
    "-Wnewline-eof",  # No newline at the end of the file.
    "-Wno-error=deprecated",  # Deprecation warnings are never errors.
    "-Wno-error=deprecated-implementations",  # Deprecation warnings are never errors.
    "-Wno-partial-availability",
    "-Wno-sign-conversion",  # Do not warn on sign conversions.
    "-Wno-unused-parameter",  # Do not warn on unused parameters.
    "-Woverlength-strings",  # Strings longer than the C maximum.
    "-Wshadow",  # Local variable shadows another variable, parameter, etc.
    "-Wstrict-selector-match",  # Compiler can't figure out the right selector.
    "-Wstrict-prototypes",
    "-Wundeclared-selector",  # Compiler doesn't see a selector.
    "-Wunreachable-code",  # Code will never be reached.
    "-Wunused-comparison",
    "-Wunused-const-variable",
    "-Wunused-exception-parameter",
    "-Wunused-function",
    "-Wunused-label",
    "-Wunused-member-function",
    "-Wunused-private-field",
    "-Wunused-property-ivar",
    "-Wunused-result",
    "-Wunused-value",
    "-Wunused-variable",
    "-Wunused-volatile-lvalue",
    "-Wused-but-marked-unused",
    "-Wno-unused-parameter",  # Enabled by -Wall via -Wunused, problems with params used in asserts.
]

def strict_warnings_objc_library(
    name,
    copts = [],
    **kwargs):
  """Declare an Objective-C library with strict warning flags."""
  native.objc_library(
      name = name,
      copts = copts + COMMON_COPTS,
      **kwargs)
<|endoftext|>


<commit_before># -*- coding: utf8 - *-
"""Unihan file parsing, importing and codec handling.

cihai.unihan
~~~~~~~~~~~~

:copyright: Copyright 2013 Tony Narlock.
:license: BSD, see LICENSE for details

"""

from __future__ import absolute_import, division, print_function, \
    with_statement, unicode_literals

import os
import zipfile
import csv
import logging
import hashlib

from sqlalchemy import create_engine, MetaData, Table, String, Column, \
    Integer, Index

from . import conversion
from .cihai import cihai_config, cihai_db, engine, CihaiDatabase
from .util import get_datafile
from ._compat import PY2, text_type, configparser

log = logging.getLogger(__name__)

UNIHAN_FILENAMES = [
    'Unihan_DictionaryIndices.txt',
    'Unihan_DictionaryLikeData.txt',
    'Unihan_IRGSources.txt',
    'Unihan_NumericValues.txt',
    'Unihan_OtherMappings.txt',
    'Unihan_RadicalStrokeCounts.txt',
    'Unihan_Readings.txt',
    'Unihan_Variants.txt'
]

class Unihan(CihaiDatabase):

    def install_raw_csv(self, csv_filename=None):
        """Install the raw csv information into CSV, return table.

        :param csv_filename: (optional, default=None) filename in /data dir.
        :type csv_filename: string
        :rtype: :class:`sqlalchemy.schema.Table`

        """

        if not csv_filename:
            # not os.path.exists(cihai_config)
            return self.install_raw_csv(UNIHAN_FILENAMES)
        elif isinstance(csv_filename, list):
            for csv_filename in csv_filename:
                return self.install_raw_csv(csv_filename)
        else:
            table_name = csv_filename.split('.')[0]

            # config.read(cihai_config)

            # for csv_filename in UNIHAN_FILENAMES:
                # if not config.has_section(csv_filename):
                    # self.install_raw_csv(csv_filename)

            if not self.table_exists(table_name):
                table = self.import_csv_to_table(
                    engine=engine,
                    csv_filename=csv_filename,
                    table_name=table_name,
                )
            else:
                log.debug('{0} already installed.'.format(table_name))
                table = self.get_table(table_name)
            return table

    def import_csv_to_table(self, engine, csv_filename, table_name):
        """Import CSV to table.

        :param engine: sqlalchemy engine
        :type engine: :class:`sqlalchemy.engine.Engine`
        :param csv_filename: csv file name inside data, e.g. ``Unihan_Readings.txt``.
        :type csv_filename: string
        :param table_name: name of table
        :type table_name: string
        :rtype: :class:`sqlalchemy.schema.Table`

        """

        table = self.create_table(table_name)
        unihan_csv = get_datafile(csv_filename)

        with open(unihan_csv, 'r') as csv_file:
            csv_md5 = hashlib.sha256(unihan_csv.encode('utf-8')).hexdigest()
            csv_data = filter(lambda row: row[0] != '#', csv_file)
            delim = b'\t' if PY2 else '\t'

            config = configparser.ConfigParser()
            config.read(cihai_config)
            if not config.has_section(csv_filename):
                config.add_section(csv_filename)

            if (
                not os.path.exists(cihai_config) or
                not config.has_option(csv_filename, 'csv_rowcount') or
                (
                    config.has_option(csv_filename, 'csv_rowcount') and
                    table.select().count().execute().scalar() != config.getint(csv_filename, 'csv_rowcount')
                )
            ):

                r = RawReader(
                    csv_data,
                    fieldnames=['char', 'field', 'value'],
                    delimiter=delim
                )
                r = list(r)

                results = engine.execute(table.insert(), r)
                config.set(csv_filename, 'csv_rowcount', text_type(len(r)))
            else:
                log.debug('Rows populated, all is well!')

            config.set(csv_filename, 'csv_md5', csv_md5)
            config_file = open(cihai_config, 'w+')
            config.write(config_file)
            config_file.close()
        return table

    def create_table(self, table_name):
        """Create table and return  :sqlalchemy:class:`sqlalchemy.Table`.

        :param table_name: name of table to create
        :type table_name: string
        :returns: Newly created table with columns and index.
        :rtype: :class:`sqlalchemy.schema.Table`

        """
        table = Table(table_name, self.metadata)
        fields = [
            ('char', String(12)),
            ('field', String(36)),
            ('value', String(256)),
        ]

        col = Column('id', Integer, primary_key=True)
        table.append_column(col)

        field_names = [field for (field, t) in fields]

        for (field, type_) in fields:
            col = Column(field, type_)
            table.append_column(col)

        Index('%s_unique_char_field_value' % table_name, table.c.char, table.c.field, table.c.value, unique=True)
        Index('%s_unique_char_field' % table_name, table.c.char, table.c.field, unique=True)
        Index('%s_field' % table_name, table.c.field)

        if not table.exists():
            self.metadata.create_all()

        return table

class RawReader(csv.DictReader):
    """Read from Unihan CSV resource."""
    def __init__(self, *args, **kwargs):
        csv.DictReader.__init__(self, *args, **kwargs)

    def __next__(self):
        row = csv.DictReader.__next__(self)

        return self.row(row)

    def next(self):
        row = csv.DictReader.next(self)

        return self.row(row)

    def row(self, row):
        for key in row.keys():
            if not isinstance(row[key], text_type):
                row[key] = text_type(row[key].decode('utf-8'))

        return row
<commit_msg>latest cihai documentation.<commit_after># -*- coding: utf8 - *-
"""Unihan file parsing, importing and codec handling.

cihai.unihan
~~~~~~~~~~~~

:copyright: Copyright 2013 Tony Narlock.
:license: BSD, see LICENSE for details

"""

from __future__ import absolute_import, division, print_function, \
    with_statement, unicode_literals

import os
import zipfile
import csv
import logging
import hashlib

from sqlalchemy import create_engine, MetaData, Table, String, Column, \
    Integer, Index

from . import conversion
from .cihai import cihai_config, cihai_db, engine, CihaiDatabase
from .util import get_datafile
from ._compat import PY2, text_type, configparser

log = logging.getLogger(__name__)

UNIHAN_FILENAMES = [
    'Unihan_DictionaryIndices.txt',
    'Unihan_DictionaryLikeData.txt',
    'Unihan_IRGSources.txt',
    'Unihan_NumericValues.txt',
    'Unihan_OtherMappings.txt',
    'Unihan_RadicalStrokeCounts.txt',
    'Unihan_Readings.txt',
    'Unihan_Variants.txt'
]

class Unihan(CihaiDatabase):

    """Cihai dataset for `Unihan`_ database.

    :meth:`~.install_raw_csv` creates the tables, :meth:`~.import_csv_to_table`
    dumps csv to database.

    .. _Unihan: http://www.unicode.org/reports/tr38/

    """

    def install_raw_csv(self, csv_filename=None):
        """Install the raw csv information into CSV, return table.

        :param csv_filename: (optional, default=None) filename in /data dir.
        :type csv_filename: string
        :rtype: :class:`sqlalchemy.schema.Table`

        """

        if not csv_filename:
            # not os.path.exists(cihai_config)
            return self.install_raw_csv(UNIHAN_FILENAMES)
        elif isinstance(csv_filename, list):
            for csv_filename in csv_filename:
                return self.install_raw_csv(csv_filename)
        else:
            table_name = csv_filename.split('.')[0]

            # config.read(cihai_config)

            # for csv_filename in UNIHAN_FILENAMES:
                # if not config.has_section(csv_filename):
                    # self.install_raw_csv(csv_filename)

            if not self.table_exists(table_name):
                table = self.import_csv_to_table(
                    engine=engine,
                    csv_filename=csv_filename,
                    table_name=table_name,
                )
            else:
                log.debug('{0} already installed.'.format(table_name))
                table = self.get_table(table_name)
            return table

    def import_csv_to_table(self, engine, csv_filename, table_name):
        """Import CSV to table.

        :param engine: sqlalchemy engine
        :type engine: :class:`sqlalchemy.engine.Engine`
        :param csv_filename: csv file name inside data, e.g. ``Unihan_Readings.txt``.
        :type csv_filename: string
        :param table_name: name of table
        :type table_name: string
        :rtype: :class:`sqlalchemy.schema.Table`

        """

        table = self.create_table(table_name)
        unihan_csv = get_datafile(csv_filename)

        with open(unihan_csv, 'r') as csv_file:
            csv_md5 = hashlib.sha256(unihan_csv.encode('utf-8')).hexdigest()
            csv_data = filter(lambda row: row[0] != '#', csv_file)
            delim = b'\t' if PY2 else '\t'

            config = configparser.ConfigParser()
            config.read(cihai_config)
            if not config.has_section(csv_filename):
                config.add_section(csv_filename)

            if (
                not os.path.exists(cihai_config) or
                not config.has_option(csv_filename, 'csv_rowcount') or
                (
                    config.has_option(csv_filename, 'csv_rowcount') and
                    table.select().count().execute().scalar() != config.getint(csv_filename, 'csv_rowcount')
                )
            ):

                r = RawReader(
                    csv_data,
                    fieldnames=['char', 'field', 'value'],
                    delimiter=delim
                )
                r = list(r)

                results = engine.execute(table.insert(), r)
                config.set(csv_filename, 'csv_rowcount', text_type(len(r)))
            else:
                log.debug('Rows populated, all is well!')

            config.set(csv_filename, 'csv_md5', csv_md5)
            config_file = open(cihai_config, 'w+')
            config.write(config_file)
            config_file.close()
        return table

    def create_table(self, table_name):
        """Create table and return  :sqlalchemy:class:`sqlalchemy.Table`.

        :param table_name: name of table to create
        :type table_name: string
        :returns: Newly created table with columns and index.
        :rtype: :class:`sqlalchemy.schema.Table`

        """
        table = Table(table_name, self.metadata)
        fields = [
            ('char', String(12)),
            ('field', String(36)),
            ('value', String(256)),
        ]

        col = Column('id', Integer, primary_key=True)
        table.append_column(col)

        field_names = [field for (field, t) in fields]

        for (field, type_) in fields:
            col = Column(field, type_)
            table.append_column(col)

        Index('%s_unique_char_field_value' % table_name, table.c.char, table.c.field, table.c.value, unique=True)
        Index('%s_unique_char_field' % table_name, table.c.char, table.c.field, unique=True)
        Index('%s_field' % table_name, table.c.field)

        if not table.exists():
            self.metadata.create_all()

        return table

class RawReader(csv.DictReader):
    """Read from Unihan CSV into Unicode."""
    def __init__(self, *args, **kwargs):
        csv.DictReader.__init__(self, *args, **kwargs)

    def __next__(self):
        row = csv.DictReader.__next__(self)

        return self.row(row)

    def next(self):
        row = csv.DictReader.next(self)

        return self.row(row)

    def row(self, row):
        for key in row.keys():
            if not isinstance(row[key], text_type):
                row[key] = text_type(row[key].decode('utf-8'))

        return row
<|endoftext|>


<commit_before>
###############################################################################
#   Copyright 2012-2014 The University of Texas at Austin                     #
#                                                                             #
#   Licensed under the Apache License, Version 2.0 (the "License");           #
#   you may not use this file except in compliance with the License.          #
#   You may obtain a copy of the License at                                   #
#                                                                             #
#       http://www.apache.org/licenses/LICENSE-2.0                            #
#                                                                             #
#   Unless required by applicable law or agreed to in writing, software       #
#   distributed under the License is distributed on an "AS IS" BASIS,         #
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  #
#   See the License for the specific language governing permissions and       #
#   limitations under the License.                                            #
###############################################################################

import commands
import datetime
import os
import re
import pprint

from ipf.dt import localtzoffset
from ipf.error import StepError
from ipf.log import LogDirectoryWatcher

from . import computing_manager
from . import service
#from . import computing_share
from . import execution_environment
from ipf.step import Step
import json
#from xml.dom.minidom import getDOMImplementation

from ipf.data import Data, Representation

from .entity import *
from .service import *
from .endpoint import *
from ipf.sysinfo import ResourceName

#######################################################################################################################
class AbstractService(Data):
    def __init__(self, id):
        Data.__init__(self,id)
        self.services = []
        self.handles = []

    def add(self, serv):
        self.services.append(serv)

class AbstractServiceStep(Step):

    def __init__(self):
        Step.__init__(self)
        self.requires = [ResourceName]
        self.produces = [AbstractService]
        self.services = []

    def run(self):
        self.resource_name = self._getInput(ResourceName).resource_name
        servlist = AbstractService(self.resource_name)
        service_paths = []
        try:
            paths = os.environ["SERVICEPATH"]
            service_paths.extend(paths.split(":"))
        except KeyError:
            raise StepError("didn't find environment variable SERVICEPATH")

        for path in service_paths:
            try:
                packages = os.listdir(path)
            except OSError:
                continue
            for name in packages:
                #print("name of package is" +name)
                #print("path is " +path)
                if name.startswith("."):
                    continue
                if name.endswith("~"):
                    continue
                if name.endswith(".lua"):
                    self._addService(os.path.join(path,name),path,serv)
                else:
                    self.info("calling addmodule w/ version")
                    #print("calling addmodule w/ version")
                    serv = service.Service()
                    self._addService(os.path.join(path,name),path,servlist)

        self._output(servlist)

    def _addService(self, path, name, servlist):

        serv = service.Service()
        ServiceType = ""
        try:
            file = open(path)
        except IOError, e:
            self.warning("%s" % e)
            return
        text = file.read()
        file.close()
        #print("in correct _addService")
        m = re.search("Name = ([^\ ]+)\n",text)
        if m is not None:
            serv.Name = m.group(1).strip()
            #print(serv.Name)
        else:
            self.debug("no name in "+path)
            #print("no name in "+path)
        m = re.search("Name = ([^\ ]+)\n",text)
        if m is not None:
            serv.Type = m.group(1).strip()
            #print(serv.Type)
        else:
            self.debug("no type in "+path)
            #print("no type in "+path)
        m = re.search("Version = ([^\ ]+)\n",text)
        if m is not None:
            serv.Version = m.group(1).strip()
            #print(serv.Version)
        else:
            self.debug("no Version in "+path)
            #print("no Version in "+path)
        m = re.search("Endpoint = ([^\ ]+)\ *\n",text)
        if m is not None:
            serv.Endpoint = m.group(1).strip()
            #print("SERV ENDPOINT IS " +serv.Endpoint)
        else:
            self.debug("no endpoint in "+path)
            serv.Endpoint = ""
            #print("no endpoint in "+path)
        m = re.findall("Capability = ([^\ ]+)\n",text)
        if m is not None:
            if serv.Capability is not None:
                serv.Capability.append(m.group(1).strip())
            else:
                serv.Capability = m
        else:
            self.debug("no Capability in "+path)
            #print("no capability in "+path)
        m = re.search("SupportStatus = ([^\ ]+)\n",text)
        if m is not None:
            serv.QualityLevel = m.group(1).strip()
        else:
            self.debug("no support status in "+path)
            #print("no support status in "+path)
        m = re.search("QualityLevel = ([^\ ]+)\n",text)
        if m is not None:
            serv.QualityLevel = m.group(1).strip()
        else:
            self.debug("no qualitylevel in "+path)
            #print("no qualitylevel in "+path)
        m = re.search("Keywords = ([^\ ]+)\n",text)
        if m is not None:
            serv.Extension["Keywords"] = map(str.strip,m.group(1).split(","))
        else:
            self.debug("no keywords in "+path)
            #print("no keywords in "+path)
        st = serv.Capability[0].split(".")
        #print("st is %s", st)
        if st[0] == "data":
            ServiceType = "StorageService"
        else:
            if st[0] == "information":
                ServiceType = "InformationService"
            else:
                if st[0] == "executionmanagement":
                    ServiceType = "ComputingService"
                else:
                    if st[0] == "information":
                        ServiceType = "InformationService"
                    else:
                        ServiceType = "LoginService"
        serv.resource_name = self.resource_name
        serv.ID = "urn:glue2:%s:%s-%s" % (ServiceType,serv.Name,self.resource_name)
        serv.ServiceType = ServiceType
        servlist.add(serv)

#######################################################################################################################

class AbstractServiceOgfJson(EntityOgfJson):
    data_cls = Service

    def __init__(self, data):
        EntityOgfJson.__init__(self,data)

    def get(self):
        return json.dumps(self.toJson(),sort_keys=True,indent=4)

    def toJson(self):
        doc = {}
        doc = EntityOgfJson.toJson(self)

        #print("in AbstractServiceOgfJson toJson")
        if len(self.data.Capability) > 0:
            doc["Capability"] = self.data.Capability
        if self.data.Type is not None:
            doc["Type"] = self.data.Type
        if self.data.QualityLevel is not None:
            doc["QualityLevel"] = self.data.QualityLevel
        if len(self.data.StatusInfo) > 0:
            doc["StatusInfo"] = self.data.StatusInfo
        if self.data.Complexity is not None:
            doc["Complexity"] = self.data.Complexity

        associations = {}
        if len(self.data.EndpointID) > 0:
            associations["EndpointID"] = self.data.EndpointID
        if len(self.data.ShareID) > 0:
            associations["ShareID"] = self.data.ShareID
        if len(self.data.ManagerID) > 0:
            associations["ManagerID"] = self.data.ManagerID
            associations["ContactID"] = self.data.ContactID
            associations["LocationID"] = self.data.LocationID
            associations["ServiceID"] = self.data.ServiceID
        doc["Associations"] = associations

        return doc

class ASOgfJson(Representation):
    data_cls = AbstractService

    def __init__(self, data):
        Representation.__init__(self,Representation.MIME_APPLICATION_JSON,data)

    def get(self):
        return json.dumps(self.toJson(),sort_keys=True,indent=4)

    def toJson(self):
        doc = {}
        doc["StorageService"] = []
        doc["ComputingService"] = []
        doc["LoginService"] = []
        doc["InformationService"] = []
        doc["Endpoint"] = []
        for serv in self.data.services:
            if serv is not None:
                endpoint = Endpoint()
                endpoint.URL = serv.Endpoint
                endpoint.InterfaceName = serv.Type
                endpoint.InterfaceVersion = serv.Version
                endpoint.Name = serv.Name
                endpoint.ID = "urn:glue2:Endpoint:%s-%s-%s" % (serv.Version, serv.Name, serv.resource_name)
                endpoint.ServiceID = serv.ID
                endpoint.QualityLevel = serv.QualityLevel
                serv.EndpointID = endpoint.ID
                doc[serv.ServiceType].append(AbstractServiceOgfJson(serv).toJson())
                doc["Endpoint"].append(EndpointOgfJson(endpoint).toJson())
        return doc
<commit_msg>Fixed publishing of extensions (important for Globus Transfer names/descriptions of GridFTP services)<commit_after>
###############################################################################
#   Copyright 2012-2014 The University of Texas at Austin                     #
#                                                                             #
#   Licensed under the Apache License, Version 2.0 (the "License");           #
#   you may not use this file except in compliance with the License.          #
#   You may obtain a copy of the License at                                   #
#                                                                             #
#       http://www.apache.org/licenses/LICENSE-2.0                            #
#                                                                             #
#   Unless required by applicable law or agreed to in writing, software       #
#   distributed under the License is distributed on an "AS IS" BASIS,         #
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  #
#   See the License for the specific language governing permissions and       #
#   limitations under the License.                                            #
###############################################################################

import commands
import datetime
import os
import re
import pprint

from ipf.dt import localtzoffset
from ipf.error import StepError
from ipf.log import LogDirectoryWatcher

from . import computing_manager
from . import service
#from . import computing_share
from . import execution_environment
from ipf.step import Step
import json
#from xml.dom.minidom import getDOMImplementation

from ipf.data import Data, Representation

from .entity import *
from .service import *
from .endpoint import *
from ipf.sysinfo import ResourceName

#######################################################################################################################
class AbstractService(Data):
    def __init__(self, id):
        Data.__init__(self,id)
        self.services = []
        self.handles = []

    def add(self, serv):
        self.services.append(serv)

class AbstractServiceStep(Step):

    def __init__(self):
        Step.__init__(self)
        self.requires = [ResourceName]
        self.produces = [AbstractService]
        self.services = []

    def run(self):
        self.resource_name = self._getInput(ResourceName).resource_name
        servlist = AbstractService(self.resource_name)
        service_paths = []
        try:
            paths = os.environ["SERVICEPATH"]
            service_paths.extend(paths.split(":"))
        except KeyError:
            raise StepError("didn't find environment variable SERVICEPATH")

        for path in service_paths:
            try:
                packages = os.listdir(path)
            except OSError:
                continue
            for name in packages:
                #print("name of package is" +name)
                #print("path is " +path)
                if name.startswith("."):
                    continue
                if name.endswith("~"):
                    continue
                if name.endswith(".lua"):
                    self._addService(os.path.join(path,name),path,serv)
                else:
                    self.info("calling addmodule w/ version")
                    #print("calling addmodule w/ version")
                    serv = service.Service()
                    self._addService(os.path.join(path,name),path,servlist)

        self._output(servlist)

    def _addService(self, path, name, servlist):

        serv = service.Service()
        ServiceType = ""
        try:
            file = open(path)
        except IOError, e:
            self.warning("%s" % e)
            return
        text = file.read()
        file.close()
        #print("in correct _addService")
        m = re.search("Name = ([^\ ]+)\n",text)
        if m is not None:
            serv.Name = m.group(1).strip()
            #print(serv.Name)
        else:
            self.debug("no name in "+path)
            #print("no name in "+path)
        m = re.search("Name = ([^\ ]+)\n",text)
        if m is not None:
            serv.Type = m.group(1).strip()
            #print(serv.Type)
        else:
            self.debug("no type in "+path)
            #print("no type in "+path)
        m = re.search("Version = ([^\ ]+)\n",text)
        if m is not None:
            serv.Version = m.group(1).strip()
            #print(serv.Version)
        else:
            self.debug("no Version in "+path)
            #print("no Version in "+path)
        m = re.search("Endpoint = ([^\ ]+)\ *\n",text)
        if m is not None:
            serv.Endpoint = m.group(1).strip()
            #print("SERV ENDPOINT IS " +serv.Endpoint)
        else:
            self.debug("no endpoint in "+path)
            serv.Endpoint = ""
            #print("no endpoint in "+path)
        m = re.findall("Capability = ([^\ ]+)\n",text)
        if m is not None:
            if serv.Capability is not None:
                serv.Capability.append(m.group(1).strip())
            else:
                serv.Capability = m
        else:
            self.debug("no Capability in "+path)
            #print("no capability in "+path)
        m = re.search("SupportStatus = ([^\ ]+)\n",text)
        if m is not None:
            serv.QualityLevel = m.group(1).strip()
        else:
            self.debug("no support status in "+path)
            #print("no support status in "+path)
        m = re.search("QualityLevel = ([^\ ]+)\n",text)
        if m is not None:
            serv.QualityLevel = m.group(1).strip()
        else:
            self.debug("no qualitylevel in "+path)
            #print("no qualitylevel in "+path)
        m = re.search("Keywords = ([^\ ]+)\n",text)
        if m is not None:
            serv.Extension["Keywords"] = map(str.strip,m.group(1).split(","))
        else:
            self.debug("no keywords in "+path)
            #print("no keywords in "+path)
        #n = re.finditer("Extensions.(.*?) = ([^\ ]+)\n",text)
        n = re.finditer("Extensions.(.*?) = (.*?)\n",text)
	    for match in n:
            serv.Extension[match.group(1).strip()] = match.group(2).strip()

        st = serv.Capability[0].split(".")
        #print("st is %s", st)
        if st[0] == "data":
            ServiceType = "StorageService"
        else:
            if st[0] == "information":
                ServiceType = "InformationService"
            else:
                if st[0] == "executionmanagement":
                    ServiceType = "ComputingService"
                else:
                    if st[0] == "information":
                        ServiceType = "InformationService"
                    else:
                        ServiceType = "LoginService"
        serv.resource_name = self.resource_name
        serv.ID = "urn:glue2:%s:%s-%s" % (ServiceType,serv.Name,self.resource_name)
        serv.ServiceType = ServiceType
        servlist.add(serv)

#######################################################################################################################

class AbstractServiceOgfJson(EntityOgfJson):
    data_cls = Service

    def __init__(self, data):
        EntityOgfJson.__init__(self,data)

    def get(self):
        return json.dumps(self.toJson(),sort_keys=True,indent=4)

    def toJson(self):
        doc = {}
        doc = EntityOgfJson.toJson(self)

        #print("in AbstractServiceOgfJson toJson")
        if len(self.data.Capability) > 0:
            doc["Capability"] = self.data.Capability
        if self.data.Type is not None:
            doc["Type"] = self.data.Type
        if self.data.QualityLevel is not None:
            doc["QualityLevel"] = self.data.QualityLevel
        if len(self.data.StatusInfo) > 0:
            doc["StatusInfo"] = self.data.StatusInfo
        if self.data.Complexity is not None:
            doc["Complexity"] = self.data.Complexity

        associations = {}
        if len(self.data.EndpointID) > 0:
            associations["EndpointID"] = self.data.EndpointID
        if len(self.data.ShareID) > 0:
            associations["ShareID"] = self.data.ShareID
        if len(self.data.ManagerID) > 0:
            associations["ManagerID"] = self.data.ManagerID
            associations["ContactID"] = self.data.ContactID
            associations["LocationID"] = self.data.LocationID
            associations["ServiceID"] = self.data.ServiceID
        doc["Associations"] = associations

        return doc

class ASOgfJson(Representation):
    data_cls = AbstractService

    def __init__(self, data):
        Representation.__init__(self,Representation.MIME_APPLICATION_JSON,data)

    def get(self):
        return json.dumps(self.toJson(),sort_keys=True,indent=4)

    def toJson(self):
        doc = {}
        doc["StorageService"] = []
        doc["ComputingService"] = []
        doc["LoginService"] = []
        doc["InformationService"] = []
        doc["Endpoint"] = []
        for serv in self.data.services:
            if serv is not None:
                endpoint = Endpoint()
                endpoint.URL = serv.Endpoint
                endpoint.InterfaceName = serv.Type
                endpoint.InterfaceVersion = serv.Version
                endpoint.Name = serv.Name
                endpoint.ID = "urn:glue2:Endpoint:%s-%s-%s" % (serv.Version, serv.Name, serv.resource_name)
                endpoint.ServiceID = serv.ID
                endpoint.QualityLevel = serv.QualityLevel
                serv.EndpointID = endpoint.ID
                doc[serv.ServiceType].append(AbstractServiceOgfJson(serv).toJson())
                doc["Endpoint"].append(EndpointOgfJson(endpoint).toJson())
        return doc
<|endoftext|>


<commit_before># coding: utf-8
from flask_assets import Environment, Bundle
from webassets.filter import Filter, register_filter

class FixedLibSass(Filter):
    """
    This filter based on Jesús Jerez <jerezmoreno@gmail.com> code [1].

    [1] https://bitbucket.org/jhuss/webassets-libsass

    Converts `Sass <http://sass-lang.com/>`_ markup to real CSS.

    Requires the ``libsass`` package (https://pypi.python.org/pypi/libsass)::

        pip install libsass

    `libsass <http://dahlia.kr/libsass-python>`_ is binding to C/C++
    implementation of a Sass compiler `Libsass
    <https://github.com/hcatlin/libsass>`_

    *Configuration options:*

    LIBSASS_STYLE (style)
        an optional coding style of the compiled result. choose one of:
        `nested` (default), `expanded`, `compact`, `compressed`

    LIBSASS_INCLUDES (includes)
        an optional list of paths to find @imported SASS/CSS source files

    See libsass documentation for full documentation about these configuration
    options:

        http://hongminhee.org/libsass-python/sass.html#sass.compile

    Copied until https://github.com/miracle2k/webassets/pull/435 is fixed.
    """
    name = 'fixedlibsass'
    options = {
        'style': 'LIBSASS_STYLE',
        'includes': 'LIBSASS_INCLUDES',
    }
    max_debug_level = None

    def setup(self):
        super(FixedLibSass, self).setup()

        try:
            import sass
        except ImportError:
            raise EnvironmentError('The "libsass" package is not installed.')
        else:
            self.sass = sass

        if not self.style:
            self.style = 'nested'

    def input(self, _in, out, **kwargs):
        source_path = kwargs['source_path']

        out.write(
            # http://hongminhee.org/libsass-python/sass.html#sass.compile
            self.sass.compile(
                filename=source_path,
                output_style=self.style,
                include_paths=(self.includes if self.includes else []),
            )
        )

register_filter(FixedLibSass)

assets = Environment(cache=False)

styles = Bundle(
    'scss/styles.scss',
    'octicons/octicons.css',
    filters='fixedlibsass,datauri',
    output='css/styles.%(version)s.css',
    depends=('**/*.scss'),
)

assets.register('styles', styles)
<commit_msg>Revert this.<commit_after># coding: utf-8
from flask_assets import Environment, Bundle
from webassets.filter import Filter, register_filter

class FixedLibSass(Filter):
    """
    This filter based on Jesús Jerez <jerezmoreno@gmail.com> code [1].

    [1] https://bitbucket.org/jhuss/webassets-libsass

    Converts `Sass <http://sass-lang.com/>`_ markup to real CSS.

    Requires the ``libsass`` package (https://pypi.python.org/pypi/libsass)::

        pip install libsass

    `libsass <http://dahlia.kr/libsass-python>`_ is binding to C/C++
    implementation of a Sass compiler `Libsass
    <https://github.com/hcatlin/libsass>`_

    *Configuration options:*

    LIBSASS_STYLE (style)
        an optional coding style of the compiled result. choose one of:
        `nested` (default), `expanded`, `compact`, `compressed`

    LIBSASS_INCLUDES (includes)
        an optional list of paths to find @imported SASS/CSS source files

    See libsass documentation for full documentation about these configuration
    options:

        http://hongminhee.org/libsass-python/sass.html#sass.compile

    Copied until https://github.com/miracle2k/webassets/pull/435 is fixed.
    """
    name = 'fixedlibsass'
    options = {
        'style': 'LIBSASS_STYLE',
        'includes': 'LIBSASS_INCLUDES',
    }
    max_debug_level = None

    def setup(self):
        super(FixedLibSass, self).setup()

        try:
            import sass
        except ImportError:
            raise EnvironmentError('The "libsass" package is not installed.')
        else:
            self.sass = sass

        if not self.style:
            self.style = 'nested'

    def input(self, _in, out, **kwargs):
        source_path = kwargs['source_path']

        out.write(
            # http://hongminhee.org/libsass-python/sass.html#sass.compile
            self.sass.compile(
                filename=source_path,
                output_style=self.style,
                include_paths=(self.includes if self.includes else []),
            )
        )

register_filter(FixedLibSass)

assets = Environment()

styles = Bundle(
    'scss/styles.scss',
    'octicons/octicons.css',
    filters='fixedlibsass,datauri',
    output='css/styles.%(version)s.css',
    depends=('**/*.scss'),
)

assets.register('styles', styles)
<|endoftext|>