<commit_before>
from discord.ext import commands

import clients
import credentials
from utilities import checks

def setup(bot):
	bot.add_cog(Words(bot))

class Words:

	def __init__(self, bot):
		self.bot = bot

	@commands.command(aliases = ["antonyms"])
	@checks.not_forbidden()
	async def antonym(self, ctx, word : str):
		'''Antonyms of a word'''
		antonyms = self.bot.wordnik_word_api.getRelatedWords(word, relationshipTypes = "antonym",
																useCanonical = "true", limitPerRelationshipType = 100)
		if not antonyms:
			return await ctx.embed_reply(":no_entry: Word or antonyms not found")
		await ctx.embed_reply(", ".join(antonyms[0].words), title = f"Antonyms of {word.capitalize()}")

	@commands.command(aliases = ["dictionary"])
	@checks.not_forbidden()
	async def define(self, ctx, word : str):
		'''Define a word'''
		definition = self.bot.wordnik_word_api.getDefinitions(word, limit = 1)  # useCanonical = True ?
		if not definition:
			return await ctx.embed_reply(":no_entry: Definition not found")
		await ctx.embed_reply(definition[0].text, title = definition[0].word.capitalize(),
								footer_text = definition[0].attributionText)

	@commands.command(aliases = ["audiodefine", "pronounce"])
	@checks.not_forbidden()
	async def pronunciation(self, ctx, word : str):
		'''Pronunciation of a word'''
		pronunciation = self.bot.wordnik_word_api.getTextPronunciations(word, limit = 1)
		description = pronunciation[0].raw.strip("()") if pronunciation else "Audio File Link"
		audio_file = self.bot.wordnik_word_api.getAudio(word, limit = 1)
		if audio_file:
			description = f"[{description}]({audio_file[0].fileUrl})"
		elif not pronunciation:
			return await ctx.embed_reply(":no_entry: Word or pronunciation not found")
		await ctx.embed_reply(description, title = f"Pronunciation of {word.capitalize()}")

	@commands.command(aliases = ["rhymes"])
	@checks.not_forbidden()
	async def rhyme(self, ctx, word : str):
		'''Rhymes of a word'''
		rhymes = self.bot.wordnik_word_api.getRelatedWords(word, relationshipTypes = "rhyme",
															limitPerRelationshipType = 100)
		if not rhymes:
			return await ctx.embed_reply(":no_entry: Word or rhymes not found")
		await ctx.embed_reply(", ".join(rhymes[0].words),
								title = f"Words that rhyme with {word.capitalize()}")

	@commands.command()
	@checks.not_forbidden()
	async def spellcheck(self, ctx, *, words : str):
		'''Spell check words'''
		url = "https://api.cognitive.microsoft.com/bing/v5.0/spellcheck"
		headers = {"Ocp-Apim-Subscription-Key" : ctx.bot.BING_SPELL_CHECK_API_KEY}
		params = {"Text": words.replace(' ', '+')}  # replace necessary?
		async with clients.aiohttp_session.post(url, headers = headers, params = params) as resp:
			data = await resp.json()
		corrections = data["flaggedTokens"]
		corrected = words
		offset = 0
		for correction in corrections:
			offset += correction["offset"]
			suggestion = correction["suggestions"][0]["suggestion"]
			corrected = corrected[:offset] + suggestion + corrected[offset + len(correction["token"]):]
			offset += (len(suggestion) - len(correction["token"])) - correction["offset"]
		await ctx.embed_reply(corrected)

	@commands.command(aliases = ["synonyms"])
	@checks.not_forbidden()
	async def synonym(self, ctx, word : str):
		'''Synonyms of a word'''
		synonyms = self.bot.wordnik_word_api.getRelatedWords(word, relationshipTypes = "synonym",
																useCanonical = "true", limitPerRelationshipType = 100)
		if not synonyms:
			return await ctx.embed_reply(":no_entry: Word or synonyms not found")
		await ctx.embed_reply(", ".join(synonyms[0].words), title = f"Synonyms of {word.capitalize()}")

	@commands.group(description = "[Language Codes](https://tech.yandex.com/translate/doc/dg/concepts/api-overview-docpage/#languages)\n"
	"Powered by [Yandex.Translate](http://translate.yandex.com/)", invoke_without_command = True)
	@checks.not_forbidden()
	async def translate(self, ctx, *, text : str):
		'''Translate to English'''
		# TODO: From and to language code options?
		await self.process_translate(ctx, text, "en")

	@translate.command(name = "from")
	@checks.not_forbidden()
	async def translate_from(self, ctx, from_language_code : str, to_language_code : str, *, text : str):
		'''
		Translate from a specific language to another
		[Language Codes](https://tech.yandex.com/translate/doc/dg/concepts/api-overview-docpage/#languages)
		Powered by [Yandex.Translate](http://translate.yandex.com/)
		'''
		# TODO: Default to_language_code?
		await self.process_translate(ctx, text, to_language_code, from_language_code)

	@translate.command(name = "languages", aliases = ["codes", "language_codes"])
	@checks.not_forbidden()
	async def translate_languages(self, ctx, language_code : str = "en"):
		'''Language Codes'''
		url = "https://translate.yandex.net/api/v1.5/tr.json/getLangs"
		params = {"ui": language_code, "key": ctx.bot.YANDEX_TRANSLATE_API_KEY}
		async with clients.aiohttp_session.get(url, params = params) as resp:
			data = await resp.json()
		if "langs" not in data:
			return await ctx.embed_reply(":no_entry: Error: Invalid Language Code")
		await ctx.embed_reply(", ".join(sorted(f"{language} ({code})" for code, language in data["langs"].items())))

	@translate.command(name = "to")
	@checks.not_forbidden()
	async def translate_to(self, ctx, language_code : str, *, text : str):
		'''
		Translate to a specific language
		[Language Codes](https://tech.yandex.com/translate/doc/dg/concepts/api-overview-docpage/#languages)
		Powered by [Yandex.Translate](http://translate.yandex.com/)
		'''
		await self.process_translate(ctx, text, language_code)

	async def process_translate(self, ctx, text, to_language_code, from_language_code = None):
		url = "https://translate.yandex.net/api/v1.5/tr.json/translate"
		params = {"key": ctx.bot.YANDEX_TRANSLATE_API_KEY, "lang": to_language_code if not from_language_code else f"{from_language_code}-{to_language_code}", "text": text.replace(' ', '+'), "options": 1}
		async with clients.aiohttp_session.get(url, params = params) as resp:
			if resp.status == 400:  # Bad Request
				return await ctx.embed_reply(":no_entry: Error")
			data = await resp.json()
		if data["code"] != 200:
			return await ctx.embed_reply(f":no_entry: Error: {data['message']}")
		await ctx.embed_reply(data["text"][0], footer_text = f"""{f"Detected Language Code: {data['detected']['lang']} | " if not from_language_code else ""}Powered by Yandex.Translate""")

<commit_msg>[Discord] Improve process translate function formatting and clarity<commit_after>
from discord.ext import commands

import clients
import credentials
from utilities import checks

def setup(bot):
	bot.add_cog(Words(bot))

class Words:

	def __init__(self, bot):
		self.bot = bot

	@commands.command(aliases = ["antonyms"])
	@checks.not_forbidden()
	async def antonym(self, ctx, word : str):
		'''Antonyms of a word'''
		antonyms = self.bot.wordnik_word_api.getRelatedWords(word, relationshipTypes = "antonym",
																useCanonical = "true", limitPerRelationshipType = 100)
		if not antonyms:
			return await ctx.embed_reply(":no_entry: Word or antonyms not found")
		await ctx.embed_reply(", ".join(antonyms[0].words), title = f"Antonyms of {word.capitalize()}")

	@commands.command(aliases = ["dictionary"])
	@checks.not_forbidden()
	async def define(self, ctx, word : str):
		'''Define a word'''
		definition = self.bot.wordnik_word_api.getDefinitions(word, limit = 1)  # useCanonical = True ?
		if not definition:
			return await ctx.embed_reply(":no_entry: Definition not found")
		await ctx.embed_reply(definition[0].text, title = definition[0].word.capitalize(),
								footer_text = definition[0].attributionText)

	@commands.command(aliases = ["audiodefine", "pronounce"])
	@checks.not_forbidden()
	async def pronunciation(self, ctx, word : str):
		'''Pronunciation of a word'''
		pronunciation = self.bot.wordnik_word_api.getTextPronunciations(word, limit = 1)
		description = pronunciation[0].raw.strip("()") if pronunciation else "Audio File Link"
		audio_file = self.bot.wordnik_word_api.getAudio(word, limit = 1)
		if audio_file:
			description = f"[{description}]({audio_file[0].fileUrl})"
		elif not pronunciation:
			return await ctx.embed_reply(":no_entry: Word or pronunciation not found")
		await ctx.embed_reply(description, title = f"Pronunciation of {word.capitalize()}")

	@commands.command(aliases = ["rhymes"])
	@checks.not_forbidden()
	async def rhyme(self, ctx, word : str):
		'''Rhymes of a word'''
		rhymes = self.bot.wordnik_word_api.getRelatedWords(word, relationshipTypes = "rhyme",
															limitPerRelationshipType = 100)
		if not rhymes:
			return await ctx.embed_reply(":no_entry: Word or rhymes not found")
		await ctx.embed_reply(", ".join(rhymes[0].words),
								title = f"Words that rhyme with {word.capitalize()}")

	@commands.command()
	@checks.not_forbidden()
	async def spellcheck(self, ctx, *, words : str):
		'''Spell check words'''
		url = "https://api.cognitive.microsoft.com/bing/v5.0/spellcheck"
		headers = {"Ocp-Apim-Subscription-Key" : ctx.bot.BING_SPELL_CHECK_API_KEY}
		params = {"Text": words.replace(' ', '+')}  # replace necessary?
		async with clients.aiohttp_session.post(url, headers = headers, params = params) as resp:
			data = await resp.json()
		corrections = data["flaggedTokens"]
		corrected = words
		offset = 0
		for correction in corrections:
			offset += correction["offset"]
			suggestion = correction["suggestions"][0]["suggestion"]
			corrected = corrected[:offset] + suggestion + corrected[offset + len(correction["token"]):]
			offset += (len(suggestion) - len(correction["token"])) - correction["offset"]
		await ctx.embed_reply(corrected)

	@commands.command(aliases = ["synonyms"])
	@checks.not_forbidden()
	async def synonym(self, ctx, word : str):
		'''Synonyms of a word'''
		synonyms = self.bot.wordnik_word_api.getRelatedWords(word, relationshipTypes = "synonym",
																useCanonical = "true", limitPerRelationshipType = 100)
		if not synonyms:
			return await ctx.embed_reply(":no_entry: Word or synonyms not found")
		await ctx.embed_reply(", ".join(synonyms[0].words), title = f"Synonyms of {word.capitalize()}")

	@commands.group(description = "[Language Codes](https://tech.yandex.com/translate/doc/dg/concepts/api-overview-docpage/#languages)\n"
	"Powered by [Yandex.Translate](http://translate.yandex.com/)", invoke_without_command = True)
	@checks.not_forbidden()
	async def translate(self, ctx, *, text : str):
		'''Translate to English'''
		# TODO: From and to language code options?
		await self.process_translate(ctx, text, "en")

	@translate.command(name = "from")
	@checks.not_forbidden()
	async def translate_from(self, ctx, from_language_code : str, to_language_code : str, *, text : str):
		'''
		Translate from a specific language to another
		[Language Codes](https://tech.yandex.com/translate/doc/dg/concepts/api-overview-docpage/#languages)
		Powered by [Yandex.Translate](http://translate.yandex.com/)
		'''
		# TODO: Default to_language_code?
		await self.process_translate(ctx, text, to_language_code, from_language_code)

	@translate.command(name = "languages", aliases = ["codes", "language_codes"])
	@checks.not_forbidden()
	async def translate_languages(self, ctx, language_code : str = "en"):
		'''Language Codes'''
		url = "https://translate.yandex.net/api/v1.5/tr.json/getLangs"
		params = {"ui": language_code, "key": ctx.bot.YANDEX_TRANSLATE_API_KEY}
		async with clients.aiohttp_session.get(url, params = params) as resp:
			data = await resp.json()
		if "langs" not in data:
			return await ctx.embed_reply(":no_entry: Error: Invalid Language Code")
		await ctx.embed_reply(", ".join(sorted(f"{language} ({code})" for code, language in data["langs"].items())))

	@translate.command(name = "to")
	@checks.not_forbidden()
	async def translate_to(self, ctx, language_code : str, *, text : str):
		'''
		Translate to a specific language
		[Language Codes](https://tech.yandex.com/translate/doc/dg/concepts/api-overview-docpage/#languages)
		Powered by [Yandex.Translate](http://translate.yandex.com/)
		'''
		await self.process_translate(ctx, text, language_code)

	async def process_translate(self, ctx, text, to_language_code, from_language_code = None):
		url = "https://translate.yandex.net/api/v1.5/tr.json/translate"
		params = {"key": ctx.bot.YANDEX_TRANSLATE_API_KEY,
					"lang": to_language_code if not from_language_code else f"{from_language_code}-{to_language_code}",
					"text": text.replace(' ', '+'), "options": 1}
		async with clients.aiohttp_session.get(url, params = params) as resp:
			if resp.status == 400:  # Bad Request
				return await ctx.embed_reply(":no_entry: Error")
			data = await resp.json()
		if data["code"] != 200:
			return await ctx.embed_reply(f":no_entry: Error: {data['message']}")
		footer_text = "Powered by Yandex.Translate"
		if not from_language_code:
			footer_text = f"Detected Language Code: {data['detected']['lang']} | " + footer_text
		await ctx.embed_reply(data["text"][0], footer_text = footer_text)

<|endoftext|>


<commit_before>from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from past.builtins import basestring
from collections import namedtuple
import re

# export resolve_lookups at this level
from .registry import resolve_lookups  # NOQA
from .registry import register_lookup_handler  # NOQA

# TODO: we can remove the optionality of of the type in a later release, it
#       is only included to allow for an error to be thrown while people are
#       converting their configuration files to 1.0

LOOKUP_REGEX = re.compile("""
\$\{                                   # opening brace for the lookup
((?P<type>[._\-a-zA-Z0-9]*(?=\s))      # type of lookup, must be followed by a
                                       # space
?\s*                                   # any number of spaces separating the
                                       # type from the input
(?P<input>[@\+\/,\._\-a-zA-Z0-9\:\s=\[\]\*]+) # the input value to the lookup
)\}                                    # closing brace of the lookup
""", re.VERBOSE)

Lookup = namedtuple("Lookup", ("type", "input", "raw"))

def extract_lookups_from_string(value):
    """Extract any lookups within a string.

    Args:
        value (str): string value we're extracting lookups from

    Returns:
        list: list of :class:`stacker.lookups.Lookup` if any

    """
    lookups = set()
    for match in LOOKUP_REGEX.finditer(value):
        groupdict = match.groupdict()
        raw = match.groups()[0]
        lookup_type = groupdict["type"]
        lookup_input = groupdict["input"]
        lookups.add(Lookup(lookup_type, lookup_input, raw))
    return lookups

def extract_lookups(value):
    """Recursively extracts any stack lookups within the data structure.

    Args:
        value (one of str, list, dict): a structure that contains lookups to
            output values

    Returns:
        list: list of lookups if any

    """
    lookups = set()
    if isinstance(value, basestring):
        lookups = lookups.union(extract_lookups_from_string(value))
    elif isinstance(value, list):
        for v in value:
            lookups = lookups.union(extract_lookups(v))
    elif isinstance(value, dict):
        for v in value.values():
            lookups = lookups.union(extract_lookups(v))
    return lookups
<commit_msg>add '?' to lookup regex (#676)<commit_after>from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from past.builtins import basestring
from collections import namedtuple
import re

# export resolve_lookups at this level
from .registry import resolve_lookups  # NOQA
from .registry import register_lookup_handler  # NOQA

# TODO: we can remove the optionality of of the type in a later release, it
#       is only included to allow for an error to be thrown while people are
#       converting their configuration files to 1.0

LOOKUP_REGEX = re.compile("""
\$\{                                   # opening brace for the lookup
((?P<type>[._\-a-zA-Z0-9]*(?=\s))      # type of lookup, must be followed by a
                                       # space
?\s*                                   # any number of spaces separating the
                                       # type from the input
(?P<input>[@\+\/,\.\?_\-a-zA-Z0-9\:\s=\[\]\*]+) # the input value to the lookup
)\}                                    # closing brace of the lookup
""", re.VERBOSE)

Lookup = namedtuple("Lookup", ("type", "input", "raw"))

def extract_lookups_from_string(value):
    """Extract any lookups within a string.

    Args:
        value (str): string value we're extracting lookups from

    Returns:
        list: list of :class:`stacker.lookups.Lookup` if any

    """
    lookups = set()
    for match in LOOKUP_REGEX.finditer(value):
        groupdict = match.groupdict()
        raw = match.groups()[0]
        lookup_type = groupdict["type"]
        lookup_input = groupdict["input"]
        lookups.add(Lookup(lookup_type, lookup_input, raw))
    return lookups

def extract_lookups(value):
    """Recursively extracts any stack lookups within the data structure.

    Args:
        value (one of str, list, dict): a structure that contains lookups to
            output values

    Returns:
        list: list of lookups if any

    """
    lookups = set()
    if isinstance(value, basestring):
        lookups = lookups.union(extract_lookups_from_string(value))
    elif isinstance(value, list):
        for v in value:
            lookups = lookups.union(extract_lookups(v))
    elif isinstance(value, dict):
        for v in value.values():
            lookups = lookups.union(extract_lookups(v))
    return lookups
<|endoftext|>


<commit_before>import ddapp
import os
import sys
import argparse

class DRCArgParser(object):

    def __init__(self):
        self._args = None
        self._parser = None
        self.strict = False

    def getArgs(self):
        if self._args is None:
            self.parseArgs()
        return self._args

    def getParser(self):
        if self._parser is None:
            self._parser = argparse.ArgumentParser()
            self.addDefaultArgs(self._parser)
        return self._parser

    def parseArgs(self):
        parser = self.getParser()
        sys.argv = [str(v) for v in sys.argv]

        if not self.strict:
            self._args, unknown = parser.parse_known_args()
        else:
            self._args = parser.parse_args()

    def getDefaultBotConfigFile(self):
        return os.path.join(ddapp.getDRCBaseDir(), 'software/config/drc_robot.cfg')

    def getDefaultDirectorConfigFile(self):
        return self.getDefaultAtlasV4DirectorConfigFile();

    def getDefaultAtlasV3DirectorConfigFile(self):
        return os.path.join(ddapp.getDRCBaseDir(),
                            'software/models/atlas_v3/director_config.json')

    def getDefaultAtlasV4DirectorConfigFile(self):
        return os.path.join(ddapp.getDRCBaseDir(),
                            'software/models/atlas_v4/director_config.json')

    def getDefaultAtlasV5DirectorConfigFile(self):
        return os.path.join(ddapp.getDRCBaseDir(),
                            'software/models/atlas_v5/director_config.json')

    def getDefaultValkyrieDirectorConfigFile(self):
        return os.path.join(ddapp.getDRCBaseDir(),
                            'software/models/valkyrie/director_config.json')

    def addDefaultArgs(self, parser):
        parser.add_argument('-c', '--config_file', type=str, help='Robot cfg file',
                            default=self.getDefaultBotConfigFile())

        parser.add_argument('--matlab-host', type=str, help='Hostname to use external matlab server',
                            default=None)

        directorConfig = parser.add_mutually_exclusive_group(required=False)
        directorConfig.add_argument('-v3', '--atlas_v3', dest='directorConfigFile',
                            action='store_const',
                            const=self.getDefaultAtlasV3DirectorConfigFile(),
                            help='Use Atlas V3')

        directorConfig.add_argument('-v4', '--atlas_v4', dest='directorConfigFile',
                            action='store_const',
                            const=self.getDefaultAtlasV4DirectorConfigFile(),
                            help='Use Atlas V4')

        directorConfig.add_argument('-v5', '--atlas_v5', dest='directorConfigFile',
                            action='store_const',
                            const=self.getDefaultAtlasV5DirectorConfigFile(),
                            help='Use Atlas V5')

        directorConfig.add_argument('-val', '--valkyrie', dest='directorConfigFile',
                            action='store_const',
                            const=self.getDefaultValkyrieDirectorConfigFile(),
                            help='Use Valkyrie')

        directorConfig.add_argument('--director_config', dest='directorConfigFile',
                            type=str,
                            help='JSON file specifying which urdfs to use')

        parser.set_defaults(directorConfigFile=self.getDefaultDirectorConfigFile())

        parser.add_argument('data_files', type=str, nargs='*',
                            help='data files to load at startup')

_argParser = None
def getGlobalArgParser():
    global _argParser
    if not _argParser:
        _argParser = DRCArgParser()
    return _argParser

def requireStrict():
    global _argParser
    _argParser = None
    getGlobalArgParser().strict = True

def args():
    return getGlobalArgParser().getArgs()
<commit_msg>add drcargs.getDirectorConfig method to parse and return json dictionary<commit_after>import ddapp
import os
import sys
import argparse
import json

class DRCArgParser(object):

    def __init__(self):
        self._args = None
        self._parser = None
        self.strict = False

    def getArgs(self):
        if self._args is None:
            self.parseArgs()
        return self._args

    def getParser(self):
        if self._parser is None:
            self._parser = argparse.ArgumentParser()
            self.addDefaultArgs(self._parser)
        return self._parser

    def parseArgs(self):
        parser = self.getParser()
        sys.argv = [str(v) for v in sys.argv]

        if not self.strict:
            self._args, unknown = parser.parse_known_args()
        else:
            self._args = parser.parse_args()

    def getDefaultBotConfigFile(self):
        return os.path.join(ddapp.getDRCBaseDir(), 'software/config/drc_robot.cfg')

    def getDefaultDirectorConfigFile(self):
        return self.getDefaultAtlasV4DirectorConfigFile();

    def getDefaultAtlasV3DirectorConfigFile(self):
        return os.path.join(ddapp.getDRCBaseDir(),
                            'software/models/atlas_v3/director_config.json')

    def getDefaultAtlasV4DirectorConfigFile(self):
        return os.path.join(ddapp.getDRCBaseDir(),
                            'software/models/atlas_v4/director_config.json')

    def getDefaultAtlasV5DirectorConfigFile(self):
        return os.path.join(ddapp.getDRCBaseDir(),
                            'software/models/atlas_v5/director_config.json')

    def getDefaultValkyrieDirectorConfigFile(self):
        return os.path.join(ddapp.getDRCBaseDir(),
                            'software/models/valkyrie/director_config.json')

    def addDefaultArgs(self, parser):
        parser.add_argument('-c', '--config_file', type=str, help='Robot cfg file',
                            default=self.getDefaultBotConfigFile())

        parser.add_argument('--matlab-host', type=str, help='Hostname to use external matlab server',
                            default=None)

        directorConfig = parser.add_mutually_exclusive_group(required=False)
        directorConfig.add_argument('-v3', '--atlas_v3', dest='directorConfigFile',
                            action='store_const',
                            const=self.getDefaultAtlasV3DirectorConfigFile(),
                            help='Use Atlas V3')

        directorConfig.add_argument('-v4', '--atlas_v4', dest='directorConfigFile',
                            action='store_const',
                            const=self.getDefaultAtlasV4DirectorConfigFile(),
                            help='Use Atlas V4')

        directorConfig.add_argument('-v5', '--atlas_v5', dest='directorConfigFile',
                            action='store_const',
                            const=self.getDefaultAtlasV5DirectorConfigFile(),
                            help='Use Atlas V5')

        directorConfig.add_argument('-val', '--valkyrie', dest='directorConfigFile',
                            action='store_const',
                            const=self.getDefaultValkyrieDirectorConfigFile(),
                            help='Use Valkyrie')

        directorConfig.add_argument('--director_config', dest='directorConfigFile',
                            type=str,
                            help='JSON file specifying which urdfs to use')

        parser.set_defaults(directorConfigFile=self.getDefaultDirectorConfigFile())

        parser.add_argument('data_files', type=str, nargs='*',
                            help='data files to load at startup')

_argParser = None
def getGlobalArgParser():
    global _argParser
    if not _argParser:
        _argParser = DRCArgParser()
    return _argParser

def requireStrict():
    global _argParser
    _argParser = None
    getGlobalArgParser().strict = True

def args():
    return getGlobalArgParser().getArgs()

_directorConfig = None
def getDirectorConfig():
    global _directorConfig
    if not _directorConfig:
        with open(args().directorConfigFile) as directorConfigFile:
            _directorConfig = json.load(directorConfigFile)
    return _directorConfig
<|endoftext|>


<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

import six

from st2common import log as logging
from st2common.constants.meta import ALLOWED_EXTS
from st2common.bootstrap.base import ResourceRegistrar
from st2common.models.api.rule import RuleAPI
from st2common.models.system.common import ResourceReference
from st2common.persistence.reactor import Rule
import st2common.content.utils as content_utils

__all__ = [
    'RulesRegistrar',
    'register_rules'
]

LOG = logging.getLogger(__name__)

class RulesRegistrar(ResourceRegistrar):
    ALLOWED_EXTENSIONS = ALLOWED_EXTS

    def register_rules_from_packs(self, base_dirs):
        """
        :return: Number of rules registered.
        :rtype: ``int``
        """
        registered_count = 0

        content = self._pack_loader.get_content(base_dirs=base_dirs,
                                                content_type='rules')
        for pack, rules_dir in six.iteritems(content):
            try:
                LOG.debug('Registering rules from pack: %s', pack)
                rules = self._get_rules_from_pack(rules_dir)
                count = self._register_rules_from_pack(pack, rules)
                registered_count += count
            except:
                LOG.exception('Failed registering all rules from pack: %s', rules_dir)

        return registered_count

    def register_rules_from_pack(self, pack_dir):
        """
        Register all the rules from the provided pack.

        :return: Number of rules registered.
        :rtype: ``int``
        """
        pack_dir = pack_dir[:-1] if pack_dir.endswith('/') else pack_dir
        _, pack = os.path.split(pack_dir)
        rules_dir = self._pack_loader.get_content_from_pack(pack_dir=pack_dir,
                                                            content_type='rules')

        registered_count = 0
        if not rules_dir:
            return registered_count

        LOG.debug('Registering rules from pack %s:, dir: %s', pack, rules_dir)

        try:
            rules = self._get_rules_from_pack(rules_dir=rules_dir)
            registered_count = self._register_rules_from_pack(pack=pack, rules=rules)
        except:
            LOG.exception('Failed registering all rules from pack: %s', rules_dir)

        return registered_count

    def _get_rules_from_pack(self, rules_dir):
        return self._get_resources_from_pack(resources_dir=rules_dir)

    def _register_rules_from_pack(self, pack, rules):
        registered_count = 0

        for rule in rules:
            LOG.debug('Loading rule from %s.', rule)
            try:
                content = self._meta_loader.load(rule)
                pack_field = content.get('pack', None)
                if not pack_field:
                    content['pack'] = pack
                    pack_field = pack
                if pack_field != pack:
                    raise Exception('Model is in pack "%s" but field "pack" is different: %s' %
                                    (pack, pack_field))
                rule_api = RuleAPI(**content)
                rule_api.validate()
                rule_db = RuleAPI.to_model(rule_api)

                try:
                    rule_ref = ResourceReference.to_string_reference(name=content['name'],
                                                                     pack=content['pack'])
                    existing = Rule.get_by_ref(rule_ref)
                    if existing:
                        rule_db.id = existing.id
                except ValueError:
                    LOG.debug('Rule %s not found. Creating new one.', rule)

                try:
                    rule_db = Rule.add_or_update(rule_db)
                    extra = {'rule_db': rule_db}
                    LOG.audit('Rule updated. Rule %s from %s.', rule_db, rule, extra=extra)
                except Exception:
                    LOG.exception('Failed to create rule %s.', rule_api.name)
            except:
                LOG.exception('Failed registering rule from %s.', rule)
            else:
                registered_count += 1

        return registered_count

def register_rules(packs_base_paths=None, pack_dir=None):
    if packs_base_paths:
        assert(isinstance(packs_base_paths, list))

    if not packs_base_paths:
        packs_base_paths = content_utils.get_packs_base_paths()

    registrar = RulesRegistrar()

    if pack_dir:
        result = registrar.register_rules_from_pack(pack_dir=pack_dir)
    else:
        result = registrar.register_rules_from_packs(base_dirs=packs_base_paths)

    return result
<commit_msg>Changes to rulesregistrar<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

import six

from st2common import log as logging
from st2common.constants.meta import ALLOWED_EXTS
from st2common.constants.pack import DEFAULT_PACK_NAME
from st2common.bootstrap.base import ResourceRegistrar
from st2common.models.api.rule import RuleAPI
from st2common.models.system.common import ResourceReference
from st2common.persistence.reactor import Rule
import st2common.content.utils as content_utils

__all__ = [
    'RulesRegistrar',
    'register_rules'
]

LOG = logging.getLogger(__name__)

class RulesRegistrar(ResourceRegistrar):
    ALLOWED_EXTENSIONS = ALLOWED_EXTS

    def register_rules_from_packs(self, base_dirs):
        """
        :return: Number of rules registered.
        :rtype: ``int``
        """
        registered_count = 0

        content = self._pack_loader.get_content(base_dirs=base_dirs,
                                                content_type='rules')
        for pack, rules_dir in six.iteritems(content):
            try:
                LOG.debug('Registering rules from pack: %s', pack)
                rules = self._get_rules_from_pack(rules_dir)
                count = self._register_rules_from_pack(pack, rules)
                registered_count += count
            except:
                LOG.exception('Failed registering all rules from pack: %s', rules_dir)

        return registered_count

    def register_rules_from_pack(self, pack_dir):
        """
        Register all the rules from the provided pack.

        :return: Number of rules registered.
        :rtype: ``int``
        """
        pack_dir = pack_dir[:-1] if pack_dir.endswith('/') else pack_dir
        _, pack = os.path.split(pack_dir)
        rules_dir = self._pack_loader.get_content_from_pack(pack_dir=pack_dir,
                                                            content_type='rules')

        registered_count = 0
        if not rules_dir:
            return registered_count

        LOG.debug('Registering rules from pack %s:, dir: %s', pack, rules_dir)

        try:
            rules = self._get_rules_from_pack(rules_dir=rules_dir)
            registered_count = self._register_rules_from_pack(pack=pack, rules=rules)
        except:
            LOG.exception('Failed registering all rules from pack: %s', rules_dir)

        return registered_count

    def _get_rules_from_pack(self, rules_dir):
        return self._get_resources_from_pack(resources_dir=rules_dir)

    def _register_rules_from_pack(self, pack, rules):
        registered_count = 0

        for rule in rules:
            LOG.debug('Loading rule from %s.', rule)
            try:
                content = self._meta_loader.load(rule)
                pack_field = content.get('pack', None)
                if not pack_field:
                    content['pack'] = pack
                    pack_field = pack
                if pack_field != pack:
                    raise Exception('Model is in pack "%s" but field "pack" is different: %s' %
                                    (pack, pack_field))
                rule_api = RuleAPI(**content)
                rule_api.validate()
                rule_db = RuleAPI.to_model(rule_api)

                # Migration from rule without pack to rule with pack.
                # There might be a rule with same name but in pack `default`
                # generated in migration script. In this case, we want to
                # delete so we don't have duplicates.
                if pack_field != DEFAULT_PACK_NAME:
                    try:
                        rule_ref = ResourceReference.to_string_reference(name=content['name'],
                                                                         pack=DEFAULT_PACK_NAME)
                        LOG.debug('Looking for rule %s in pack %s', content['name'],
                                  DEFAULT_PACK_NAME)
                        existing = Rule.get_by_ref(rule_ref)
                        LOG.debug('Existing = %s', existing)
                        if existing:
                            LOG.debug('Found rule in pack default: %s; Deleting.', rule_ref)
                            Rule.delete(existing)
                    except:
                        LOG.exception('Exception deleting rule from %s pack.', DEFAULT_PACK_NAME)

                try:
                    rule_ref = ResourceReference.to_string_reference(name=content['name'],
                                                                     pack=content['pack'])
                    existing = Rule.get_by_ref(rule_ref)
                    if existing:
                        rule_db.id = existing.id
                        LOG.debug('Found existing rule: %s with id: %s', rule_ref, existing.id)
                except ValueError:
                    LOG.debug('Rule %s not found. Creating new one.', rule)

                try:
                    rule_db = Rule.add_or_update(rule_db)
                    extra = {'rule_db': rule_db}
                    LOG.audit('Rule updated. Rule %s from %s.', rule_db, rule, extra=extra)
                except Exception:
                    LOG.exception('Failed to create rule %s.', rule_api.name)
            except:
                LOG.exception('Failed registering rule from %s.', rule)
            else:
                registered_count += 1

        return registered_count

def register_rules(packs_base_paths=None, pack_dir=None):
    if packs_base_paths:
        assert(isinstance(packs_base_paths, list))

    if not packs_base_paths:
        packs_base_paths = content_utils.get_packs_base_paths()

    registrar = RulesRegistrar()

    if pack_dir:
        result = registrar.register_rules_from_pack(pack_dir=pack_dir)
    else:
        result = registrar.register_rules_from_packs(base_dirs=packs_base_paths)

    return result
<|endoftext|>


<commit_before>"""
TypedMatrix.py - read/write TypedMatrix format
"""
import StringIO
import struct
import json
import calendar
from datetime import datetime

# Header Structure
# {
#   length: count of rows of data
#   cols: array of column definitions
#       [{}
#       ]
# }

typemap = {
    int: 'Int32',
    float: 'Float32',
    datetime: 'Float32',
}
typeformatmap = {
    'Int32': 'i',
    'Float32': 'f',
}
typedefaultmap = {
    'Int32': 0,
    'Float32': 0.0,
}

def get_columns(data):
    """
    Gets the column definitions implicit in a dict or list of dicts.
    If any field which has a datatype that is not in typemap, thows ???
    """
    # make data iterable
    if type(data) is dict:
        data = [data]

    cols = {}
    for i, d in enumerate(data):
        for key, value in d.iteritems():
            t = type(value)
            if t not in typemap:
                raise TypeError ('TypedMatrix: "%s" is not a supported type in field "%s"' % (type(value), key))
            if key not in cols:
                cols[key] = {'name': key, 'type': typemap[t]}
    cols = cols.values()
    cols.sort(lambda a, b: cmp(a['name'], b['name']))
    return cols

def _datetime2timestamp(dt):
    return calendar.timegm(dt.utctimetuple()) * 1000.0

def conv(data, t, default):
    if 'Int32' == t:
        assert type(data) is not datetime, 'Cannot convert datetime to int32.'
        fn = int
    elif 'Float32' == t:
        if type(data) is datetime:
            fn = _datetime2timestamp
        else:
            fn = float
    else:
        assert False, 'Unknown conversion type %s' % t

    # noinspection PyBroadException
    try:
        return fn(data)
    except:
        return default

def row_fmt(columns):
    return '<' + ''.join(typeformatmap[col['type']] for col in columns)

def pack(data, extra_header_fields=None, columns=None):
    """
    Pack a dict or list of dicts into a TypedMatrix binary packed string
    If a list of columns is not given, the columns are derived from the data using get_columns()
    extra_header_fields can supply an optional dict with additional fields to be included in the packed header
    """

    # make data iterable
    if type(data) is dict:
        data = [data]

    if extra_header_fields:
        header = dict(extra_header_fields)
    else:
        header = dict()

    header['length'] = len(data)
    if not columns:
        columns = get_columns(data)
    header['cols'] = columns

    f = StringIO.StringIO()
    headerstr = json.dumps(header)

    #TODO: #100 write "magic" file format token at the start
    # f.write("tmtx")

    f.write(struct.pack("<i", len(headerstr)))
    f.write(headerstr)

    formatmap = row_fmt(columns)
    #    inverse_typemap = {v: k for k, v in typemap.iteritems()}
    colspecs = [{'name': col['name'], 'type': col['type'], 'default': typedefaultmap[col['type']]} for col in columns]

    for d in data:
        f.write(struct.pack(
            formatmap,
            *[conv(d[colspec['name']], colspec['type'], colspec['default'])
              for colspec in colspecs]))

    return f.getvalue()

def unpack(packed_str):
    """
    Unpack a binary packed string containing a TypedMatrix.   Returns a tuple of header and data
    header is a dict and data is a list of dicts
    """
    f = StringIO.StringIO(packed_str)

    #TODO: #100 read "magic" file format token
    # token = f.read(4)
    # assert(token == 'tmtx')

    header_len = struct.unpack('<i', f.read(struct.calcsize('<i')))[0]
    header = json.loads(f.read(header_len))

    fmt = row_fmt(header['cols'])
    data = []
    col_names = [col['name'] for col in header['cols']]
    for i in range(0, header['length']):
        data.append(dict(zip(col_names, struct.unpack(fmt, f.read(struct.calcsize(fmt))))))

    return header, data

def get_packed_float_value(f):
    """
    Get a float value that has been packed using Float32.   This is useful to get the actual precision used
    in the packed array for a float column, since Float32 precision is limited to about 9 digits and
    there is no native Float32 in python.
    """
    return struct.unpack('<f', struct.pack("<f", float(f)))[0]

def get_utc_timestamp(dt=datetime.now()):
    """
    Create a timestamp in milliseconds as a Float32 from a datetime
    """
    return get_packed_float_value(_datetime2timestamp(dt))
<commit_msg>Fix comment<commit_after>"""
TypedMatrix.py - read/write TypedMatrix format
"""
import StringIO
import struct
import json
import calendar
from datetime import datetime

# Header Structure
# {
#   length: count of rows of data
#   cols: array of column definitions
#       [{}
#       ]
# }

typemap = {
    int: 'Int32',
    float: 'Float32',
    datetime: 'Float32',
}
typeformatmap = {
    'Int32': 'i',
    'Float32': 'f',
}
typedefaultmap = {
    'Int32': 0,
    'Float32': 0.0,
}

def get_columns(data):
    """
    Gets the column definitions implicit in a dict or list of dicts.
    If any field which has a datatype that is not in typemap, thows TypeException
    """
    # make data iterable
    if type(data) is dict:
        data = [data]

    cols = {}
    for i, d in enumerate(data):
        for key, value in d.iteritems():
            t = type(value)
            if t not in typemap:
                raise TypeError ('TypedMatrix: "%s" is not a supported type in field "%s"' % (type(value), key))
            if key not in cols:
                cols[key] = {'name': key, 'type': typemap[t]}
    cols = cols.values()
    cols.sort(lambda a, b: cmp(a['name'], b['name']))
    return cols

def _datetime2timestamp(dt):
    return calendar.timegm(dt.utctimetuple()) * 1000.0

def conv(data, t, default):
    if 'Int32' == t:
        assert type(data) is not datetime, 'Cannot convert datetime to int32.'
        fn = int
    elif 'Float32' == t:
        if type(data) is datetime:
            fn = _datetime2timestamp
        else:
            fn = float
    else:
        assert False, 'Unknown conversion type %s' % t

    # noinspection PyBroadException
    try:
        return fn(data)
    except:
        return default

def row_fmt(columns):
    return '<' + ''.join(typeformatmap[col['type']] for col in columns)

def pack(data, extra_header_fields=None, columns=None):
    """
    Pack a dict or list of dicts into a TypedMatrix binary packed string
    If a list of columns is not given, the columns are derived from the data using get_columns()
    extra_header_fields can supply an optional dict with additional fields to be included in the packed header
    """

    # make data iterable
    if type(data) is dict:
        data = [data]

    if extra_header_fields:
        header = dict(extra_header_fields)
    else:
        header = dict()

    header['length'] = len(data)
    if not columns:
        columns = get_columns(data)
    header['cols'] = columns

    f = StringIO.StringIO()
    headerstr = json.dumps(header)

    #TODO: #100 write "magic" file format token at the start
    # f.write("tmtx")

    f.write(struct.pack("<i", len(headerstr)))
    f.write(headerstr)

    formatmap = row_fmt(columns)
    #    inverse_typemap = {v: k for k, v in typemap.iteritems()}
    colspecs = [{'name': col['name'], 'type': col['type'], 'default': typedefaultmap[col['type']]} for col in columns]

    for d in data:
        f.write(struct.pack(
            formatmap,
            *[conv(d[colspec['name']], colspec['type'], colspec['default'])
              for colspec in colspecs]))

    return f.getvalue()

def unpack(packed_str):
    """
    Unpack a binary packed string containing a TypedMatrix.   Returns a tuple of header and data
    header is a dict and data is a list of dicts
    """
    f = StringIO.StringIO(packed_str)

    #TODO: #100 read "magic" file format token
    # token = f.read(4)
    # assert(token == 'tmtx')

    header_len = struct.unpack('<i', f.read(struct.calcsize('<i')))[0]
    header = json.loads(f.read(header_len))

    fmt = row_fmt(header['cols'])
    data = []
    col_names = [col['name'] for col in header['cols']]
    for i in range(0, header['length']):
        data.append(dict(zip(col_names, struct.unpack(fmt, f.read(struct.calcsize(fmt))))))

    return header, data

def get_packed_float_value(f):
    """
    Get a float value that has been packed using Float32.   This is useful to get the actual precision used
    in the packed array for a float column, since Float32 precision is limited to about 9 digits and
    there is no native Float32 in python.
    """
    return struct.unpack('<f', struct.pack("<f", float(f)))[0]

def get_utc_timestamp(dt=datetime.now()):
    """
    Create a timestamp in milliseconds as a Float32 from a datetime
    """
    return get_packed_float_value(_datetime2timestamp(dt))
<|endoftext|>


<commit_before>#!/usr/bin/python

import shutil
import os
import sys
import random
import string
from copy import copy # don't want pass by reference sometimes

TARGET_DIRECTORY = "/home/himanshu/Downloads/hide-your-file"
MAX_LEVELS = 0
WORDS_LIST = []
FILE_EXTENSIONS = [ 'txt', 'mp3', 'odt', 'ods', 'jpeg', 'png', \
					'gif', 'mkv','avi','mp4', 'm4a', 'tiff', 'ppt' \
					'doc', 'docx', 'sh', 'cpp', 'c', 'py', 'java' ]

def random_extension():
	return random.choice(FILE_EXTENSIONS)

# def random_word(length):
# 	return ''.join(random.choice(string.lowercase) for i in range(length))

# this function works on linux only
# for windows, and random strings, uncomment above function
def read_words():
	dictionary_file = "/usr/share/dict/words"
	global WORDS_LIST
	with open(dictionary_file, 'r') as f:
		WORDS_LIST = f.readlines()
	WORDS_LIST = [ w.strip().replace('\'', '').replace('\\','') for w in WORDS_LIST if len(w) > 10 ]

def random_word(length):
	return random.choice(WORDS_LIST)

def random_filename():
	return (random_word(random.randint(20, 30)) + '.' + random_extension())

def random_dirname():
	return random_word(random.randint(20, 30))

def make_directories(target, dmin, dmax, fmin, fmax, size_min, size_max, level):
	if (level >= MAX_LEVELS):
		print 'Level exceeded at', target
		return
	# number of directories to make
	dir_count = random.randint(dmin, dmax+1)
	print "LEVEL", level, "Creating", dir_count, "directories at", target
	while (dir_count > 0):
		try:
			new_path = os.path.join(target, random_dirname())
			os.makedirs(new_path)
			# make files
			file_count = random.randint(fmin, fmax+1)
			print "Creating", file_count, "files at", new_path
			while (file_count > 0):
				with (open(os.path.join(new_path, random_filename()), 'w')) as f:
						f.write(random_word(random.randint(size_min, size_max)))
				file_count -= 1

			make_directories(new_path, dmin, dmax, fmin, fmax, size_min, size_max, copy(level+1))
			dir_count -= 1

		except OSError as e:
			print "Error", e
			pass

def main():
	try:
		os.mkdir(TARGET_DIRECTORY)
	except OSError:
		if os.path.isdir(TARGET_DIRECTORY):
			choice = raw_input("Directory with same name exists, remove that (y/n) ? ")
			print choice
			if choice == 'y':
				shutil.rmtree(TARGET_DIRECTORY)
				os.mkdir(TARGET_DIRECTORY)
			else:
				sys.exit()

	global MAX_LEVELS
	MAX_LEVELS = int(raw_input("Enter maximum depth to reach : "))
	fmin = int(raw_input("Enter number of files per directory (min) : "))
	fmax = int(raw_input("Enter number of files per directory (max) : "))
	dmin = int(raw_input("Enter number of directories per directory (min) : "))
	dmax = int(raw_input("Enter number of directories per directory (max) : "))
	size_min = int(raw_input("Minimum size of file (in characters) : "))
	size_max = int(raw_input("Maximum size of file (in characters) : "))

	read_words()
	make_directories(TARGET_DIRECTORY, dmin, dmax, fmin, fmax, size_min, size_max, 1)
	# creating a zip archive of the entire mess created
	zipfile = shutil.make_archive(TARGET_DIRECTORY, 'zip', TARGET_DIRECTORY)
	print
	print
	print 'Files are created at', TARGET_DIRECTORY
	print 'A zipfile has been made', zipfile
	print 'Ready to hide the needle in the haystack'

if __name__ == "__main__":
	print "Keep the depth as low as possible"
	print "Maximum 2^depth directories are created"
	main()<commit_msg>removed user name from default directory<commit_after>#!/usr/bin/python

import shutil
import os
import sys
import random
import string
from copy import copy # don't want pass by reference sometimes

TARGET_DIRECTORY = "~/Downloads/hide-your-file"
MAX_LEVELS = 0
WORDS_LIST = []
FILE_EXTENSIONS = [ 'txt', 'mp3', 'odt', 'ods', 'jpeg', 'png', \
					'gif', 'mkv','avi','mp4', 'm4a', 'tiff', 'ppt' \
					'doc', 'docx', 'sh', 'cpp', 'c', 'py', 'java' ]

def random_extension():
	return random.choice(FILE_EXTENSIONS)

# def random_word(length):
# 	return ''.join(random.choice(string.lowercase) for i in range(length))

# this function works on linux only
# for windows, and random strings, uncomment above function
def read_words():
	dictionary_file = "/usr/share/dict/words"
	global WORDS_LIST
	with open(dictionary_file, 'r') as f:
		WORDS_LIST = f.readlines()
	WORDS_LIST = [ w.strip().replace('\'', '').replace('\\','') for w in WORDS_LIST if len(w) > 10 ]

def random_word(length):
	return random.choice(WORDS_LIST)

def random_filename():
	return (random_word(random.randint(20, 30)) + '.' + random_extension())

def random_dirname():
	return random_word(random.randint(20, 30))

def make_directories(target, dmin, dmax, fmin, fmax, size_min, size_max, level):
	if (level >= MAX_LEVELS):
		print 'Level exceeded at', target
		return
	# number of directories to make
	dir_count = random.randint(dmin, dmax+1)
	print "LEVEL", level, "Creating", dir_count, "directories at", target
	while (dir_count > 0):
		try:
			new_path = os.path.join(target, random_dirname())
			os.makedirs(new_path)
			# make files
			file_count = random.randint(fmin, fmax+1)
			print "Creating", file_count, "files at", new_path
			while (file_count > 0):
				with (open(os.path.join(new_path, random_filename()), 'w')) as f:
						f.write(random_word(random.randint(size_min, size_max)))
				file_count -= 1

			make_directories(new_path, dmin, dmax, fmin, fmax, size_min, size_max, copy(level+1))
			dir_count -= 1

		except OSError as e:
			print "Error", e
			pass

def main():
	try:
		os.mkdir(TARGET_DIRECTORY)
	except OSError:
		if os.path.isdir(TARGET_DIRECTORY):
			choice = raw_input("Directory with same name exists, remove that (y/n) ? ")
			print choice
			if choice == 'y':
				shutil.rmtree(TARGET_DIRECTORY)
				os.mkdir(TARGET_DIRECTORY)
			else:
				sys.exit()

	global MAX_LEVELS
	MAX_LEVELS = int(raw_input("Enter maximum depth to reach : "))
	fmin = int(raw_input("Enter number of files per directory (min) : "))
	fmax = int(raw_input("Enter number of files per directory (max) : "))
	dmin = int(raw_input("Enter number of directories per directory (min) : "))
	dmax = int(raw_input("Enter number of directories per directory (max) : "))
	size_min = int(raw_input("Minimum size of file (in characters) : "))
	size_max = int(raw_input("Maximum size of file (in characters) : "))

	read_words()
	make_directories(TARGET_DIRECTORY, dmin, dmax, fmin, fmax, size_min, size_max, 1)
	# creating a zip archive of the entire mess created
	zipfile = shutil.make_archive(TARGET_DIRECTORY, 'zip', TARGET_DIRECTORY)
	print
	print
	print 'Files are created at', TARGET_DIRECTORY
	print 'A zipfile has been made', zipfile
	print 'Ready to hide the needle in the haystack'

if __name__ == "__main__":
	print "Keep the depth as low as possible"
	print "Maximum 2^depth directories are created"
	main()
<|endoftext|>


<commit_before>import numpy as np

import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.nn.functional as F

USE_CUDA = False

def set_cuda(val):
    global USE_CUDA
    USE_CUDA = val

def sequence_mask(sequence_length, max_len=None):
    if max_len is None:
        max_len = sequence_length.data.max()
    batch_size = sequence_length.size(0)
    seq_range = torch.arange(0, max_len).long()
    seq_range_expand = seq_range.unsqueeze(0).expand(batch_size, max_len)
    seq_range_expand = Variable(seq_range_expand)
    # if sequence_length.is_cuda:
    if USE_CUDA:
        seq_range_expand = seq_range_expand.cuda()
    seq_length_expand = (sequence_length.unsqueeze(1)
                         .expand_as(seq_range_expand))
    return seq_range_expand < seq_length_expand

def masked_cross_entropy(logits, target, length, gamma=0, eps=1e-7):
    length = Variable(torch.LongTensor(length))
    if USE_CUDA:
        length = length.cuda()

    """
    Args:
        logits: A Variable containing a FloatTensor of size
            (batch, max_len, num_classes) which contains the
            unnormalized probability for each class.
        target: A Variable containing a LongTensor of size
            (batch, max_len) which contains the index of the true
            class for each corresponding step.
        length: A Variable containing a LongTensor of size (batch,)
            which contains the length of each data in a batch.

    Returns:
        loss: An average loss value masked by the length.
    """

    # logits_flat: (batch * max_len, num_classes)
    logits_flat = logits.view(-1, logits.size(-1))
    # target_flat: (batch * max_len, 1)
    target_flat = target.view(-1, 1)
    # probs_flat: (batch * max_len, 1)
    probs_flat = torch.gather(F.softmax(logits_flat), dim=1, index=target_flat)
    probs_flat = probs_flat.clamp(eps, 1. - eps)   # prob: [0, 1] -> [eps, 1 - eps]
    # losses_flat: (batch * max_len, 1)
    losses_flat = -torch.log(probs_flat) * (1 - probs_flat) ** gamma  # focal loss
    # losses: (batch, max_len)
    losses = losses_flat.view(*target.size())
    # mask: (batch, max_len)
    mask = sequence_mask(sequence_length=length, max_len=target.size(1))
    losses = losses * mask.float()
    loss = losses.sum() / length.float().sum()
    return loss

class EncoderRNN(nn.Module):
    def __init__(self, input_size, hidden_size, n_layers=1, dropout=0.1):
        super(EncoderRNN, self).__init__()

        self.input_size = input_size
        self.hidden_size = hidden_size
        self.n_layers = n_layers
        self.dropout = dropout
        self.gru = nn.GRU(input_size, hidden_size, n_layers, dropout=self.dropout, bidirectional=True)

    def forward(self, input_seqs, input_lengths, hidden=None):
        # Note: we run this all at once (over multiple batches of multiple sequences)
        packed = torch.nn.utils.rnn.pack_padded_sequence(input_seqs, input_lengths)
        outputs, hidden = self.gru(packed, hidden)
        outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(outputs)  # unpack (back to padded)
        outputs = outputs[:, :, :self.hidden_size] + outputs[:, :, self.hidden_size:]  # Sum bidirectional outputs
        return outputs, hidden

class CNN(nn.Module):
    def __init__(self, input_size):
        super(CNN, self).__init__()

        inp_size = input_size

        self.features = nn.Sequential(
            nn.Conv2d(1, 10, kernel_size=(2, inp_size - 1)),
            nn.LeakyReLU(inplace=True),
            nn.MaxPool2d((2, 1)),
            nn.Conv2d(10, 10, kernel_size=(2, 1)),
            nn.LeakyReLU(inplace=True),
            nn.MaxPool2d((2, 1)),
            nn.Conv2d(10, 10, kernel_size=(2, 1)),
            nn.LeakyReLU(inplace=True),
            nn.MaxPool2d((2, 1)),
        )

        self.n_layers = 3

    def forward(self, x):
        out = self.features(x)
        out = out.view(out.size(2), out.size(0), out.size(1) * out.size(3))
        return out

class EncoderCNN(nn.Module):
    def __init__(self, input_size, hidden_size, n_layers=1, dropout=0.1):
        super(EncoderCNN, self).__init__()

        self.input_size = input_size
        self.hidden_size = hidden_size
        self.n_layers = n_layers
        self.dropout = dropout
        self.cnn = CNN(input_size)
        self.gru = nn.GRU(input_size, hidden_size, n_layers, dropout=self.dropout, bidirectional=True)

    def forward(self, input_seqs, input_lengths, hidden=None):
        input_features = self.cnn(input_seqs.transpose(0, 1).unsqueeze(1))
        features_length = [(l//(2 ** self.cnn.n_layers)) - 1 for l in input_lengths]
        print(input_seqs.size())
        print(input_lengths)
        print(input_features.size())
        print(features_length)
        # Note: we run this all at once (over multiple batches of multiple sequences)
        # packed = torch.nn.utils.rnn.pack_padded_sequence(input_seqs, input_lengths)
        packed = torch.nn.utils.rnn.pack_padded_sequence(input_features, features_length)
        outputs, hidden = self.gru(packed, hidden)
        outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(outputs)  # unpack (back to padded)
        outputs = outputs[:, :, :self.hidden_size] + outputs[:, :, self.hidden_size:]  # Sum bidirectional outputs
        return outputs, hidden

class Attn(nn.Module):
    def __init__(self, method, hidden_size):
        super(Attn, self).__init__()

        self.method = method
        self.hidden_size = hidden_size

        if self.method == 'general':
            self.attn = nn.Linear(self.hidden_size, hidden_size)

        elif self.method == 'concat':
            self.attn = nn.Linear(self.hidden_size * 2, hidden_size)
            self.v = nn.Parameter(torch.FloatTensor(1, hidden_size))

    def forward(self, hidden, encoder_outputs):
        max_len = encoder_outputs.size(0)
        this_batch_size = encoder_outputs.size(1)

        # Create variable to store attention energies
        attn_energies = Variable(torch.zeros(this_batch_size, max_len))  # B x S

        if USE_CUDA:
            attn_energies = attn_energies.cuda()

        # For each batch of encoder outputs
        for b in range(this_batch_size):
            # Calculate energy for each encoder output
            for i in range(max_len):
                attn_energies[b, i] = self.score(hidden[:, b], encoder_outputs[i, b].unsqueeze(0))

        # Normalize energies to weights in range 0 to 1, resize to 1 x B x S
        return F.softmax(attn_energies).unsqueeze(1)

    def score(self, hidden, encoder_output):

        if self.method == 'dot':
            energy = torch.dot(hidden.view(-1), encoder_output.view(-1))
            return energy

        elif self.method == 'general':
            energy = self.attn(encoder_output)
            energy = torch.dot(hidden.view(-1), energy.view(-1))
            return energy

        elif self.method == 'concat':
            energy = self.attn(torch.cat((hidden, encoder_output), 1))
            energy = self.v.dot(energy)
            return energy

class LuongAttnDecoderRNN(nn.Module):
    def __init__(self, attn_model, hidden_size, output_size, n_layers=1, dropout=0.1, embedding=None):
        super(LuongAttnDecoderRNN, self).__init__()

        # Keep for reference
        self.attn_model = attn_model
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.n_layers = n_layers
        self.dropout = dropout

        # Define layers
        if np.any(embedding):
            self.embedding_size = embedding_size = embedding.shape[1]
            self.embedding = nn.Embedding(output_size, embedding_size)
            self.embedding.weight = nn.Parameter(torch.from_numpy(embedding).float())
            self.embedding.requires_grad = True
        else:
            self.embedding_size = embedding_size = hidden_size
            self.embedding = nn.Embedding(output_size, embedding_size)

        self.embedding_dropout = nn.Dropout(dropout)
        self.gru = nn.GRU(embedding_size, hidden_size, n_layers, dropout=dropout)
        self.concat = nn.Linear(hidden_size * 2, hidden_size)
        self.out = nn.Linear(hidden_size, output_size)

        # Choose attention model
        if attn_model != 'none':
            self.attn = Attn(attn_model, hidden_size)

    def forward(self, input_seq, last_hidden, encoder_outputs):
        # Note: we run this one step at a time

        # Get the embedding of the current input word (last output word)
        batch_size = input_seq.size(0)
        embedded = self.embedding(input_seq)
        embedded = self.embedding_dropout(embedded)
        embedded = embedded.view(1, batch_size, -1)  # S=1 x B x N

        # Get current hidden state from input word and last hidden state
        rnn_output, hidden = self.gru(embedded, last_hidden)

        # Calculate attention from current RNN state and all encoder outputs;
        # apply to encoder outputs to get weighted average
        attn_weights = self.attn(rnn_output, encoder_outputs)
        context = attn_weights.bmm(encoder_outputs.transpose(0, 1))  # B x S=1 x N

        # Attentional vector using the RNN hidden state and context vector
        # concatenated together (Luong eq. 5)
        rnn_output = rnn_output.squeeze(0)  # S=1 x B x N -> B x N
        context = context.squeeze(1)  # B x S=1 x N -> B x N
        concat_input = torch.cat((rnn_output, context), 1)
        concat_output = F.tanh(self.concat(concat_input))

        # Finally predict next token (Luong eq. 6, without softmax)
        output = self.out(concat_output)

        # Return final output, hidden state, and attention weights (for visualization)
        return output, hidden, attn_weights

# https://github.com/DingKe/pytorch_workplace/blob/master/focalloss/loss.py
def one_hot(index, classes):
    size = index.size() + (classes,)
    view = index.size() + (1,)

    mask = torch.Tensor(*size).fill_(0)
    index = index.view(*view)
    ones = 1.

    if isinstance(index, Variable):
        ones = Variable(torch.Tensor(index.size()).fill_(1))
        mask = Variable(mask, volatile=index.volatile)

    return mask.scatter_(1, index, ones)

class FocalLoss(nn.Module):

    def __init__(self, gamma=0, eps=1e-7):
        super(FocalLoss, self).__init__()
        self.gamma = gamma
        self.eps = eps

    def forward(self, input, target):
        y = one_hot(target, input.size(-1))
        logit = F.softmax(input)
        logit = logit.clamp(self.eps, 1. - self.eps)

        loss = -1 * y * torch.log(logit) # cross entropy
        loss = loss * (1 - logit) ** self.gamma # focal loss

        return loss.sum()
<commit_msg>Fixed a bug to CNN model architecture.<commit_after>import numpy as np

import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.nn.functional as F

USE_CUDA = False

def set_cuda(val):
    global USE_CUDA
    USE_CUDA = val

def sequence_mask(sequence_length, max_len=None):
    if max_len is None:
        max_len = sequence_length.data.max()
    batch_size = sequence_length.size(0)
    seq_range = torch.arange(0, max_len).long()
    seq_range_expand = seq_range.unsqueeze(0).expand(batch_size, max_len)
    seq_range_expand = Variable(seq_range_expand)
    # if sequence_length.is_cuda:
    if USE_CUDA:
        seq_range_expand = seq_range_expand.cuda()
    seq_length_expand = (sequence_length.unsqueeze(1)
                         .expand_as(seq_range_expand))
    return seq_range_expand < seq_length_expand

def masked_cross_entropy(logits, target, length, gamma=0, eps=1e-7):
    length = Variable(torch.LongTensor(length))
    if USE_CUDA:
        length = length.cuda()

    """
    Args:
        logits: A Variable containing a FloatTensor of size
            (batch, max_len, num_classes) which contains the
            unnormalized probability for each class.
        target: A Variable containing a LongTensor of size
            (batch, max_len) which contains the index of the true
            class for each corresponding step.
        length: A Variable containing a LongTensor of size (batch,)
            which contains the length of each data in a batch.

    Returns:
        loss: An average loss value masked by the length.
    """

    # logits_flat: (batch * max_len, num_classes)
    logits_flat = logits.view(-1, logits.size(-1))
    # target_flat: (batch * max_len, 1)
    target_flat = target.view(-1, 1)
    # probs_flat: (batch * max_len, 1)
    probs_flat = torch.gather(F.softmax(logits_flat), dim=1, index=target_flat)
    probs_flat = probs_flat.clamp(eps, 1. - eps)   # prob: [0, 1] -> [eps, 1 - eps]
    # losses_flat: (batch * max_len, 1)
    losses_flat = -torch.log(probs_flat) * (1 - probs_flat) ** gamma  # focal loss
    # losses: (batch, max_len)
    losses = losses_flat.view(*target.size())
    # mask: (batch, max_len)
    mask = sequence_mask(sequence_length=length, max_len=target.size(1))
    losses = losses * mask.float()
    loss = losses.sum() / length.float().sum()
    return loss

class EncoderRNN(nn.Module):
    def __init__(self, input_size, hidden_size, n_layers=1, dropout=0.1):
        super(EncoderRNN, self).__init__()

        self.input_size = input_size
        self.hidden_size = hidden_size
        self.n_layers = n_layers
        self.dropout = dropout
        self.gru = nn.GRU(input_size, hidden_size, n_layers, dropout=self.dropout, bidirectional=True)

    def forward(self, input_seqs, input_lengths, hidden=None):
        # Note: we run this all at once (over multiple batches of multiple sequences)
        packed = torch.nn.utils.rnn.pack_padded_sequence(input_seqs, input_lengths)
        outputs, hidden = self.gru(packed, hidden)
        outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(outputs)  # unpack (back to padded)
        outputs = outputs[:, :, :self.hidden_size] + outputs[:, :, self.hidden_size:]  # Sum bidirectional outputs
        return outputs, hidden

class CNN(nn.Module):
    def __init__(self, input_size):
        super(CNN, self).__init__()

        inp_size = input_size

        self.features = nn.Sequential(
            nn.Conv2d(1, 10, kernel_size=(2, inp_size - 1)),
            nn.LeakyReLU(inplace=True),
            nn.MaxPool2d((2, 1)),
            nn.Conv2d(10, 10, kernel_size=(2, 1)),
            nn.LeakyReLU(inplace=True),
            nn.MaxPool2d((2, 1)),
            nn.Conv2d(10, 20, kernel_size=(2, 1)),
            nn.LeakyReLU(inplace=True),
            nn.MaxPool2d((2, 1)),
        )

        self.n_layers = 3

    def forward(self, x):
        out = self.features(x)
        out = out.view(out.size(2), out.size(0), out.size(1) * out.size(3))
        return out

class EncoderCNN(nn.Module):
    def __init__(self, input_size, hidden_size, n_layers=1, dropout=0.1):
        super(EncoderCNN, self).__init__()

        self.input_size = input_size
        self.hidden_size = hidden_size
        self.n_layers = n_layers
        self.dropout = dropout
        self.cnn = CNN(input_size)
        self.gru = nn.GRU(input_size, hidden_size, n_layers, dropout=self.dropout, bidirectional=True)

    def forward(self, input_seqs, input_lengths, hidden=None):
        input_features = self.cnn(input_seqs.transpose(0, 1).unsqueeze(1))
        features_length = [(l//(2 ** self.cnn.n_layers)) - 1 for l in input_lengths]
        print(input_seqs.size())
        print(input_lengths)
        print(input_features.size())
        print(features_length)
        # Note: we run this all at once (over multiple batches of multiple sequences)
        # packed = torch.nn.utils.rnn.pack_padded_sequence(input_seqs, input_lengths)
        packed = torch.nn.utils.rnn.pack_padded_sequence(input_features, features_length)
        outputs, hidden = self.gru(packed, hidden)
        outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(outputs)  # unpack (back to padded)
        outputs = outputs[:, :, :self.hidden_size] + outputs[:, :, self.hidden_size:]  # Sum bidirectional outputs
        return outputs, hidden

class Attn(nn.Module):
    def __init__(self, method, hidden_size):
        super(Attn, self).__init__()

        self.method = method
        self.hidden_size = hidden_size

        if self.method == 'general':
            self.attn = nn.Linear(self.hidden_size, hidden_size)

        elif self.method == 'concat':
            self.attn = nn.Linear(self.hidden_size * 2, hidden_size)
            self.v = nn.Parameter(torch.FloatTensor(1, hidden_size))

    def forward(self, hidden, encoder_outputs):
        max_len = encoder_outputs.size(0)
        this_batch_size = encoder_outputs.size(1)

        # Create variable to store attention energies
        attn_energies = Variable(torch.zeros(this_batch_size, max_len))  # B x S

        if USE_CUDA:
            attn_energies = attn_energies.cuda()

        # For each batch of encoder outputs
        for b in range(this_batch_size):
            # Calculate energy for each encoder output
            for i in range(max_len):
                attn_energies[b, i] = self.score(hidden[:, b], encoder_outputs[i, b].unsqueeze(0))

        # Normalize energies to weights in range 0 to 1, resize to 1 x B x S
        return F.softmax(attn_energies).unsqueeze(1)

    def score(self, hidden, encoder_output):

        if self.method == 'dot':
            energy = torch.dot(hidden.view(-1), encoder_output.view(-1))
            return energy

        elif self.method == 'general':
            energy = self.attn(encoder_output)
            energy = torch.dot(hidden.view(-1), energy.view(-1))
            return energy

        elif self.method == 'concat':
            energy = self.attn(torch.cat((hidden, encoder_output), 1))
            energy = self.v.dot(energy)
            return energy

class LuongAttnDecoderRNN(nn.Module):
    def __init__(self, attn_model, hidden_size, output_size, n_layers=1, dropout=0.1, embedding=None):
        super(LuongAttnDecoderRNN, self).__init__()

        # Keep for reference
        self.attn_model = attn_model
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.n_layers = n_layers
        self.dropout = dropout

        # Define layers
        if np.any(embedding):
            self.embedding_size = embedding_size = embedding.shape[1]
            self.embedding = nn.Embedding(output_size, embedding_size)
            self.embedding.weight = nn.Parameter(torch.from_numpy(embedding).float())
            self.embedding.requires_grad = True
        else:
            self.embedding_size = embedding_size = hidden_size
            self.embedding = nn.Embedding(output_size, embedding_size)

        self.embedding_dropout = nn.Dropout(dropout)
        self.gru = nn.GRU(embedding_size, hidden_size, n_layers, dropout=dropout)
        self.concat = nn.Linear(hidden_size * 2, hidden_size)
        self.out = nn.Linear(hidden_size, output_size)

        # Choose attention model
        if attn_model != 'none':
            self.attn = Attn(attn_model, hidden_size)

    def forward(self, input_seq, last_hidden, encoder_outputs):
        # Note: we run this one step at a time

        # Get the embedding of the current input word (last output word)
        batch_size = input_seq.size(0)
        embedded = self.embedding(input_seq)
        embedded = self.embedding_dropout(embedded)
        embedded = embedded.view(1, batch_size, -1)  # S=1 x B x N

        # Get current hidden state from input word and last hidden state
        rnn_output, hidden = self.gru(embedded, last_hidden)

        # Calculate attention from current RNN state and all encoder outputs;
        # apply to encoder outputs to get weighted average
        attn_weights = self.attn(rnn_output, encoder_outputs)
        context = attn_weights.bmm(encoder_outputs.transpose(0, 1))  # B x S=1 x N

        # Attentional vector using the RNN hidden state and context vector
        # concatenated together (Luong eq. 5)
        rnn_output = rnn_output.squeeze(0)  # S=1 x B x N -> B x N
        context = context.squeeze(1)  # B x S=1 x N -> B x N
        concat_input = torch.cat((rnn_output, context), 1)
        concat_output = F.tanh(self.concat(concat_input))

        # Finally predict next token (Luong eq. 6, without softmax)
        output = self.out(concat_output)

        # Return final output, hidden state, and attention weights (for visualization)
        return output, hidden, attn_weights

# https://github.com/DingKe/pytorch_workplace/blob/master/focalloss/loss.py
def one_hot(index, classes):
    size = index.size() + (classes,)
    view = index.size() + (1,)

    mask = torch.Tensor(*size).fill_(0)
    index = index.view(*view)
    ones = 1.

    if isinstance(index, Variable):
        ones = Variable(torch.Tensor(index.size()).fill_(1))
        mask = Variable(mask, volatile=index.volatile)

    return mask.scatter_(1, index, ones)

class FocalLoss(nn.Module):

    def __init__(self, gamma=0, eps=1e-7):
        super(FocalLoss, self).__init__()
        self.gamma = gamma
        self.eps = eps

    def forward(self, input, target):
        y = one_hot(target, input.size(-1))
        logit = F.softmax(input)
        logit = logit.clamp(self.eps, 1. - self.eps)

        loss = -1 * y * torch.log(logit) # cross entropy
        loss = loss * (1 - logit) ** self.gamma # focal loss

        return loss.sum()
<|endoftext|>


<commit_before>"""Test calling functions in static methods."""

import os, time
import unittest2
import lldb
import lldbutil
from lldbtest import *

class TestObjCStaticMethod(TestBase):

    mydir = os.path.join("lang", "objc", "objc-static-method")

    @unittest2.skipUnless(sys.platform.startswith("darwin"), "requires Darwin")
    @python_api_test
    def test_with_dsym_and_python_api(self):
        """Test calling functions in static methods."""
        self.buildDsym()
        self.objc_static_method()

    @python_api_test
    def test_with_dwarf_and_python_api(self):
        """Test calling functions in static methods."""
        self.buildDwarf()
        self.objc_static_method()

    def setUp(self):
        # Call super's setUp().
        TestBase.setUp(self)
        # Find the line numbers to break inside main().
        self.main_source = "static.m"
        self.break_line = line_number(self.main_source, '// Set breakpoint here.')

    @unittest2.expectedFailure
    #rdar://problem/9745789 "expression" can't call functions in class methods
    def objc_static_method(self):
        """Test calling functions in static methods."""
        exe = os.path.join(os.getcwd(), "a.out")

        target = self.dbg.CreateTarget(exe)
        self.assertTrue(target, VALID_TARGET)

        bpt = target.BreakpointCreateByLocation(self.main_source, self.break_line)
        self.assertTrue(bpt, VALID_BREAKPOINT)

        # Now launch the process, and do not stop at entry point.
        process = target.LaunchSimple (None, None, os.getcwd())

        self.assertTrue(process, PROCESS_IS_VALID)

        # The stop reason of the thread should be breakpoint.
        thread_list = lldbutil.get_threads_stopped_at_breakpoint (process, bpt)

        # Make sure we stopped at the first breakpoint.
        self.assertTrue (len(thread_list) != 0, "No thread stopped at our breakpoint.")
        self.assertTrue (len(thread_list) == 1, "More than one thread stopped at our breakpoint.")

        # Now make sure we can call a function in the static method we've stopped in.
        frame = thread_list[0].GetFrameAtIndex(0)
        self.assertTrue (frame, "Got a valid frame 0 frame.")

        cmd_value = frame.EvaluateExpression ("(char *) sel_getName (_cmd)")
        self.assertTrue (cmd_value.IsValid())
        sel_name = cmd_value.GetSummary()
        self.assertTrue (sel_name == "doSomethingWithString:", "Got the right value for the selector as string.")

if __name__ == '__main__':
    import atexit
    lldb.SBDebugger.Initialize()
    atexit.register(lambda: lldb.SBDebugger.Terminate())
    unittest2.main()
<commit_msg>Mark the test with expectedFailure for now.<commit_after>"""Test calling functions in static methods."""

import os, time
import unittest2
import lldb
import lldbutil
from lldbtest import *

class TestObjCStaticMethod(TestBase):

    mydir = os.path.join("lang", "objc", "objc-static-method")

    @unittest2.skipUnless(sys.platform.startswith("darwin"), "requires Darwin")
    @python_api_test

    #<rdar://problem/9745789> "expression" can't call functions in class methods
    @unittest2.expectedFailure
    def test_with_dsym_and_python_api(self):
        """Test calling functions in static methods."""
        self.buildDsym()
        self.objc_static_method()

    @python_api_test
    #<rdar://problem/9745789> "expression" can't call functions in class methods
    @unittest2.expectedFailure
    def test_with_dwarf_and_python_api(self):
        """Test calling functions in static methods."""
        self.buildDwarf()
        self.objc_static_method()

    def setUp(self):
        # Call super's setUp().
        TestBase.setUp(self)
        # Find the line numbers to break inside main().
        self.main_source = "static.m"
        self.break_line = line_number(self.main_source, '// Set breakpoint here.')

    @unittest2.expectedFailure
    #rdar://problem/9745789 "expression" can't call functions in class methods
    def objc_static_method(self):
        """Test calling functions in static methods."""
        exe = os.path.join(os.getcwd(), "a.out")

        target = self.dbg.CreateTarget(exe)
        self.assertTrue(target, VALID_TARGET)

        bpt = target.BreakpointCreateByLocation(self.main_source, self.break_line)
        self.assertTrue(bpt, VALID_BREAKPOINT)

        # Now launch the process, and do not stop at entry point.
        process = target.LaunchSimple (None, None, os.getcwd())

        self.assertTrue(process, PROCESS_IS_VALID)

        # The stop reason of the thread should be breakpoint.
        thread_list = lldbutil.get_threads_stopped_at_breakpoint (process, bpt)

        # Make sure we stopped at the first breakpoint.
        self.assertTrue (len(thread_list) != 0, "No thread stopped at our breakpoint.")
        self.assertTrue (len(thread_list) == 1, "More than one thread stopped at our breakpoint.")

        # Now make sure we can call a function in the static method we've stopped in.
        frame = thread_list[0].GetFrameAtIndex(0)
        self.assertTrue (frame, "Got a valid frame 0 frame.")

        cmd_value = frame.EvaluateExpression ("(char *) sel_getName (_cmd)")
        self.assertTrue (cmd_value.IsValid())
        sel_name = cmd_value.GetSummary()
        self.assertTrue (sel_name == "doSomethingWithString:", "Got the right value for the selector as string.")

if __name__ == '__main__':
    import atexit
    lldb.SBDebugger.Initialize()
    atexit.register(lambda: lldb.SBDebugger.Terminate())
    unittest2.main()
<|endoftext|>


<commit_before># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <gogogo.vm@gmail.com>
"""

from __future__ import absolute_import
from __future__ import unicode_literals

import abc

from .._typecode import Typecode
from ._interface import TypeCheckerInterface

class CheckerCreator(object):

    @property
    def min_strict_level(self):
        return min(self.__checker_mapping)

    @property
    def max_strict_level(self):
        return max(self.__checker_mapping)

    def __init__(self, type_object, value, checker_mapping):
        self.__type_object = type_object
        self.__value = value
        self.__checker_mapping = checker_mapping

    def create(self, strict_level=None):
        if strict_level is None:
            strict_level = self.max_strict_level
        elif strict_level < self.min_strict_level:
            strict_level = self.min_strict_level
        elif strict_level > self.max_strict_level:
            strict_level = self.max_strict_level

        return self.__checker_mapping.get(strict_level)(
            self.__type_object, self.__value)

class TypeCheckerStrictLevel(TypeCheckerInterface):
    __slots__ = ("_value", )

    def __init__(self, type_object, value):
        self.__type_object = type_object
        self._value = value

    @abc.abstractmethod
    def is_instance(self):
        pass

    def is_type(self):
        return all([
            self.is_instance(),
            not self.is_exclude_instance()
        ])

    def validate(self):
        """
        :raises TypeError:
            If the value is not matched the type to be expected.
        """

        if self.is_type():
            return

        raise TypeError(
            "invalid value type: expected={}, actual={}".format(
                Typecode.get_typename(self.typecode), type(self._value)))

    def is_exclude_instance(self):
        return False

    def is_valid_after_convert(self, converted_value):
        return True

class TypeChecker(TypeCheckerInterface):

    def __init__(self, value, checker_mapping, strict_level):
        self.__checker = CheckerCreator(
            self,
            value=value,
            checker_mapping=checker_mapping).create(strict_level)

    def is_type(self):
        return self.__checker.is_type()

    def is_valid_after_convert(self, value):
        return self.__checker.is_valid_after_convert(value)

    def is_instance(self):
        return self.__checker.is_instance()

    def is_exclude_instance(self):
        return self.__checker.is_exclude_instance()

    def validate(self):
        self.__checker.validate()
<commit_msg>Change to pre-calculate the property values to improve performance<commit_after># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <gogogo.vm@gmail.com>
"""

from __future__ import absolute_import
from __future__ import unicode_literals

import abc

from .._typecode import Typecode
from ._interface import TypeCheckerInterface

class CheckerCreator(object):

    @property
    def min_strict_level(self):
        return self.__min_strict_level

    @property
    def max_strict_level(self):
        return self.__max_strict_level

    def __init__(self, type_object, value, checker_mapping):
        self.__type_object = type_object
        self.__value = value
        self.__checker_mapping = checker_mapping

        self.__min_strict_level = min(checker_mapping)
        self.__max_strict_level = max(checker_mapping)

    def create(self, strict_level=None):
        if strict_level is None:
            strict_level = self.max_strict_level
        elif strict_level < self.min_strict_level:
            strict_level = self.min_strict_level
        elif strict_level > self.max_strict_level:
            strict_level = self.max_strict_level

        return self.__checker_mapping.get(strict_level)(
            self.__type_object, self.__value)

class TypeCheckerStrictLevel(TypeCheckerInterface):
    __slots__ = ("_value", )

    def __init__(self, type_object, value):
        self.__type_object = type_object
        self._value = value

    @abc.abstractmethod
    def is_instance(self):
        pass

    def is_type(self):
        return all([
            self.is_instance(),
            not self.is_exclude_instance()
        ])

    def validate(self):
        """
        :raises TypeError:
            If the value is not matched the type to be expected.
        """

        if self.is_type():
            return

        raise TypeError(
            "invalid value type: expected={}, actual={}".format(
                Typecode.get_typename(self.typecode), type(self._value)))

    def is_exclude_instance(self):
        return False

    def is_valid_after_convert(self, converted_value):
        return True

class TypeChecker(TypeCheckerInterface):

    def __init__(self, value, checker_mapping, strict_level):
        self.__checker = CheckerCreator(
            self,
            value=value,
            checker_mapping=checker_mapping).create(strict_level)

    def is_type(self):
        return self.__checker.is_type()

    def is_valid_after_convert(self, value):
        return self.__checker.is_valid_after_convert(value)

    def is_instance(self):
        return self.__checker.is_instance()

    def is_exclude_instance(self):
        return self.__checker.is_exclude_instance()

    def validate(self):
        self.__checker.validate()
<|endoftext|>


<commit_before>

#  filestuff.py
#  Playing around with f.tell(), f.seek() and so on.

# Open a file
myfile = open("myfile.txt", 'r')

# A string to store data in.
stuff = ""

# Print the current pointer position
print myfile.tell()

# Advance the pointer 3 bytes from the start of the file.
myfile.seek(3, 0)

# Print the current pointer position
print myfile.tell()

# Read a few bytes from the file into our string.
stuff = myfile.read(7)

# Print the data
print stuff

# Close the file
myfile.close()

<commit_msg>More work.<commit_after>

#  filestuff.py
#  Playing around with f.tell(), f.seek() and so on.
#  With f.seek(x, n) - n=0 measures from the start of the file.
#  n=1 measures from the current file position. n=2 measures from the
#  end of the file.

# Open a file
myfile = open("myfile.txt", 'r')

# A string to store data in.
stuff = ""

# Print the current pointer position
print myfile.tell()

# Advance the pointer 3 bytes from the start of the file.
myfile.seek(3, 0)

# Print the current pointer position
print myfile.tell()

# Read a few bytes from the file into our string.
stuff = myfile.read(7)

# Print the data
print stuff

# Reset "stuff" to missing.
stuff = ""

# Move the pointer to the start of the file.
myfile.seek(0, 0)

# Read a few bytes from the file into our string.
stuff = myfile.read(10)

# Print the data
print stuff

# Close the file
myfile.close()

<|endoftext|>