<commit_before># This is how we intend to use it
import sys
import os
import json
import codecs
import fnmatch
from multiprocessing.dummy import Pool as ThreadPool
from jsonpath_rw import parse, jsonpath
sys.path.insert(1, os.path.join(sys.path[0], '..'))
import time
import etk

tk = etk.init()
tk.load_dictionaries()

def load_json_file(file_name):
    rules = json.load(codecs.open(file_name, 'r', 'utf-8'))
    return rules

def jl_file_iterator(file):
    with codecs.open(file, 'r', 'utf-8') as f:
        for line in f:
            document = json.loads(line)
            yield document

def jl_path_iterator(file_path):
    abs_file_path = os.path.abspath(file_path)
    if os.path.isdir(abs_file_path):
        for file in os.listdir(abs_file_path):
            if fnmatch.fnmatch(file, '*.jl'):
                yield os.path.join(abs_file_path, file)
    else:
        yield abs_file_path

def buildContent(jl, extractors):
    extractors['content_relaxed'] = tk.extract_readability(jl['raw_content'], {'recall_priority': True})
    extractors['content_strict'] = tk.extract_readability(jl['raw_content'], {'recall_priority': False})
    extractors['tables'] = tk.extract_table(jl['raw_content'])

def buildTokensAndData(jl, extractors):
    # for readability and title
    if not config['extractions']:
        raise "No extractions mentioned in config file"

    for extractions in config['extractions']:
        for path in extractions['input']:
            jsonpath_expr = parse(path)
            try:
                matches = jsonpath_expr.find(extractors)
            except Exception:
                print "No matches for ", path
                matches = []
            for match in matches:
                processDataMatch(match, extractions)

def annotateTokenToExtractions(tokens, extractions):
    for extractor, extractions in extractions.iteritems():
        if extractor in ['phone']:
            " ignoring phone annotation.."
            continue
        for extraction in extractions:
            input_type = extraction['input_type']
            if 'text' in input_type:
                # build text tokens
                pass

            if 'tokens' not in input_type:
                print "ignoring ", extractor, " as tokens not dependant.."
                continue
            data = extraction['result']
            for values in data:
                start = values['context']['start']
                end = values['context']['end']
                offset = 0
                for i in range(start, end):
                    if 'semantic_type' not in tokens[i].keys():
                        tokens[i]['semantic_type'] = []
                    temp = {}
                    temp['type'] = extractor
                    temp['offset'] = offset
                    if offset == 0:
                        temp['length'] = end - start
                    tokens[i]['semantic_type'].append(temp)
                    offset += 1
    return tokens

def processDataMatch(match, extractions):
    if 'tokens' not in match.value.keys():
        match.value['tokens'] = tk.extract_crftokens(match.value['text'])
    if 'simple_tokens' not in match.value.keys():
        match.value['simple_tokens'] = tk.extract_tokens_from_crf(match.value['tokens'])

    data_extractors = {}
    for extractor in extractions['extractors']:
        extractor_info = extractions['extractors'][extractor]

        try:
            function_call = getattr(tk, extractor_info['extractor'])
        except Exception:
            print extractor_info['extractor'], " fucntion not found in etk"
            continue

        # intitalize the data extractor
        data_extractors[extractor] = {'input_type': 'tokens', 'extractor': extractor}

        if extractor_info['input_type'] == 'tokens' and 'name' in extractor_info['config'].keys() and 'ngrams' in extractor_info['config'].keys():
            data_extractors[extractor]['result'] = function_call(match.value['tokens'], name=extractor_info['config']['name'], ngrams=extractor_info['config']['ngrams'])
        elif extractor_info['input_type'] == 'tokens' and 'name' in extractor_info['config'].keys():
            data_extractors[extractor]['result'] = function_call(match.value['simple_tokens'], name=extractor_info['config']['name'])
        elif extractor_info['input_type'] == 'text':
            data_extractors[extractor]['result'] = function_call(match.value['text'])
        else:
            print "No matching call found for input type - ", extractor_info['input_type']

    #  CLEAN THE EMPTY DATA EXTRACTORS
    data_extractors = dict((key, value) for key, value in data_extractors.iteritems() if value['result'])
    if data_extractors:
        match.value['data_extractors'] = data_extractors

def processFile(jl):
    extractors = {}
    # Content extractors
    buildContent(jl, extractors)
    # tokens and data
    buildTokensAndData(jl, extractors)

    jl['extractors'] = extractors
    jl['raw_content'] = '...'
    return jl

def write_output(out, results):
    for i in range(len(results)):
        o.write(json.dumps(results[i]) + '\n')

def parallelProcess(files, outfile):
    pool = ThreadPool(threads)
    results = pool.map(processFile, files)
    pool.close()
    pool.join()
    write_output(outfile, results)

if __name__ == "__main__":
    input_path = sys.argv[1]
    output_file = sys.argv[2]
    config_file = sys.argv[3]
    if len(sys.argv) == 5:
        threads = sys.argv[4]
    else:
        threads = 5
    config = load_json_file(config_file)
    o = codecs.open(output_file, 'w', 'utf-8')

    # run in pool for extractors in batch
    i, files = 1, []
    for jl in jl_file_iterator(input_path):
        files.append(jl)
        if i % threads == 0:
            parallelProcess(files, o)
            files = []
        i += 1

    if files:
        parallelProcess(files, o)
        files = []

    o.close()
<commit_msg>with annotations<commit_after># This is how we intend to use it
import sys
import os
import json
import codecs
import fnmatch
import multiprocessing
from multiprocessing.dummy import Pool as ThreadPool
from jsonpath_rw import parse, jsonpath
sys.path.insert(1, os.path.join(sys.path[0], '..'))
import time
import etk

tk = etk.init()
tk.load_dictionaries()

def load_json_file(file_name):
    rules = json.load(codecs.open(file_name, 'r', 'utf-8'))
    return rules

def jl_file_iterator(file):
    with codecs.open(file, 'r', 'utf-8') as f:
        for line in f:
            document = json.loads(line)
            yield document

def jl_path_iterator(file_path):
    abs_file_path = os.path.abspath(file_path)
    if os.path.isdir(abs_file_path):
        for file in os.listdir(abs_file_path):
            if fnmatch.fnmatch(file, '*.jl'):
                yield os.path.join(abs_file_path, file)
    else:
        yield abs_file_path

def buildContent(jl, extractors):
    extractors['content_relaxed'] = tk.extract_readability(jl['raw_content'], {'recall_priority': True})
    extractors['content_strict'] = tk.extract_readability(jl['raw_content'], {'recall_priority': False})
    extractors['tables'] = tk.extract_table(jl['raw_content'])

def buildTokensAndData(jl, extractors):
    # for readability and title
    if not config['extractions']:
        raise "No extractions mentioned in config file"

    for extractions in config['extractions']:
        for path in extractions['input']:
            jsonpath_expr = parse(path)
            try:
                matches = jsonpath_expr.find(extractors)
            except Exception:
                print "No matches for ", path
                matches = []
            for match in matches:
                processDataMatch(match, extractions)

def annotateTokenToExtractions(tokens, extractions):
    for extractor, extraction in extractions.iteritems():
        input_type = extraction['input_type']
        if 'text' == input_type:
            # build text tokens
            continue

        data = extraction['result']
        for values in data:
            start = values['context']['start']
            end = values['context']['end']
            offset = 0
            for i in range(start, end):
                if 'semantic_type' not in tokens[i].keys():
                    tokens[i]['semantic_type'] = []
                temp = {}
                temp['type'] = extractor
                temp['offset'] = offset
                if offset == 0:
                    temp['length'] = end - start
                tokens[i]['semantic_type'].append(temp)
                offset += 1

def processDataMatch(match, extractions):
    if 'tokens' not in match.value.keys():
        match.value['tokens'] = tk.extract_crftokens(match.value['text'])
    if 'simple_tokens' not in match.value.keys():
        match.value['simple_tokens'] = tk.extract_tokens_from_crf(match.value['tokens'])

    data_extractors = {}
    for extractor in extractions['extractors']:
        extractor_info = extractions['extractors'][extractor]

        try:
            function_call = getattr(tk, extractor_info['extractor'])
        except Exception:
            print extractor_info['extractor'], " fucntion not found in etk"
            continue

        # intitalize the data extractor
        data_extractors[extractor] = {'input_type': 'tokens', 'extractor': extractor}

        if extractor_info['input_type'] == 'tokens' and 'name' in extractor_info['config'].keys() and 'ngrams' in extractor_info['config'].keys():
            data_extractors[extractor]['result'] = function_call(match.value['tokens'], name=extractor_info['config']['name'], ngrams=extractor_info['config']['ngrams'])
        elif extractor_info['input_type'] == 'tokens' and 'name' in extractor_info['config'].keys():
            data_extractors[extractor]['result'] = function_call(match.value['simple_tokens'], name=extractor_info['config']['name'])
        elif extractor_info['input_type'] == 'text':
            data_extractors[extractor]['result'] = function_call(match.value['text'])
        else:
            print "No matching call found for input type - ", extractor_info['input_type']

    #  CLEAN THE EMPTY DATA EXTRACTORS
    data_extractors = dict((key, value) for key, value in data_extractors.iteritems() if value['result'])
    if data_extractors:
        annotateTokenToExtractions(match.value['tokens'], data_extractors)
        match.value['data_extractors'] = data_extractors

def processFile(jl):
    extractors = {}
    # Content extractors
    buildContent(jl, extractors)
    # tokens and data
    buildTokensAndData(jl, extractors)

    jl['extractors'] = extractors
    jl['raw_content'] = '...'
    return jl

def write_output(out, results):
    for i in range(len(results)):
        o.write(json.dumps(results[i]) + '\n')

def parallelProcess(files, outfile):
    pool = ThreadPool(threads)
    results = pool.map(processFile, files)
    pool.close()
    pool.join()
    write_output(outfile, results)

if __name__ == "__main__":
    input_path = sys.argv[1]
    output_file = sys.argv[2]
    config_file = sys.argv[3]
    if len(sys.argv) == 5:
        threads = sys.argv[4]
    else:
        threads = multiprocessing.cpu_count()
    config = load_json_file(config_file)
    o = codecs.open(output_file, 'w', 'utf-8')

    # run in pool for extractors in batch
    i, files = 1, []
    print "Running with ", threads, 'threads'
    for jl in jl_file_iterator(input_path):
        files.append(jl)
        if i % threads == 0:
            parallelProcess(files, o)
            files = []
        i += 1

    if files:
        parallelProcess(files, o)
        files = []

    o.close()
<|endoftext|>


<commit_before># -*- coding: utf8 -*-

# Copyright (C) 2016 - Philipp Temminghoff <phil65@kodi.tv>
# This program is Free Software see LICENSE file for details

class ItemList(object):

    def __init__(self, items=None, content_type="", name=""):
        self.items = items if items else []
        self.name = name
        self.content_type = content_type
        self.sorts = []
        self.properties = []

    def __len__(self):
        return len(self.items)

    def __iter__(self):
        return iter(self.items)

    def __getitem__(self, key):
        return self.items[key]

    def __add__(self, other):
        return ItemList(items=self.items + other.items,
                        content_type=self.content_type,
                        name=self.name,
                        sorts=self.sorts,
                        properties=self.properties)

    def __iadd__(self, other):
        self.items += other.items
        return self

    def append(self, item):
        self.items.append(item)

    def remove(self, index):
        self.items.remove(index)

    def set_name(self, name):
        self.name = name

    def set_content(self, content_type):
        self.content_type = content_type

    def add_sorts(self, sorts):
        self.sorts = sorts

    def set_properties(self, properties):
        self.properties = properties

    def update_properties(self, properties):
        self.properties.update({k: v for k, v in properties.iteritems() if v})

    def set_property(self, key, value):
        self.properties[key] = value

    def get_property(self, key):
        value = self.properties.get(key)
        return value if value else ""
<commit_msg>__nonzero__ for itemlist<commit_after># -*- coding: utf8 -*-

# Copyright (C) 2016 - Philipp Temminghoff <phil65@kodi.tv>
# This program is Free Software see LICENSE file for details

class ItemList(object):

    def __init__(self, items=None, content_type="", name=""):
        self.items = items if items else []
        self.name = name
        self.content_type = content_type
        self.sorts = []
        self.properties = []

    def __len__(self):
        return len(self.items)

    def __iter__(self):
        return iter(self.items)

    def __getitem__(self, key):
        return self.items[key]

    def __nonzero__(self):
        return len(self.items) > 0

    def __add__(self, other):
        return ItemList(items=self.items + other.items,
                        content_type=self.content_type,
                        name=self.name,
                        sorts=self.sorts,
                        properties=self.properties)

    def __iadd__(self, other):
        self.items += other.items
        return self

    def append(self, item):
        self.items.append(item)

    def remove(self, index):
        self.items.remove(index)

    def set_name(self, name):
        self.name = name

    def set_content(self, content_type):
        self.content_type = content_type

    def add_sorts(self, sorts):
        self.sorts = sorts

    def set_properties(self, properties):
        self.properties = properties

    def update_properties(self, properties):
        self.properties.update({k: v for k, v in properties.iteritems() if v})

    def set_property(self, key, value):
        self.properties[key] = value

    def get_property(self, key):
        value = self.properties.get(key)
        return value if value else ""
<|endoftext|>


<commit_before>'''
	LTI authentication, based on the passthrough backend.
'''

from passthrough import PassThroughAuth

class LtiAuth(PassThroughAuth):
    name='lti'
<commit_msg>2to3 rocks<commit_after>'''
	LTI authentication, based on the passthrough backend.
'''

from .passthrough import PassThroughAuth

class LtiAuth(PassThroughAuth):
    name='lti'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import atx
import time

class YysHelper():
    def __init__(self):
        self.d = atx.connect()
        self.d.image_path = ['.', 'assets']

    def chapter(self, level):
        """
        章节循环一轮游，进章节把屏幕内的小怪打完就出来
        """
        while True:
            chapterImage = "chapter%d.1920x1080.png" % level

            ret = self.clickImage(chapterImage)

            if ret:
                self.sleep(2)

            ret = self.clickImage("explore.1920x1080.png")

            if ret:
                self.sleep(8)

            ret = self.clickImage("lock.1920x1080.png", time=5)

            fightedWithLeader = False

            while not fightedWithLeader:
                swordExists = True
                while True:
                    ret = self.clickImage("sword.1920x1080.png")
                    if not ret:
                        ret = self.clickImage("leader_sword.1920x1080.png", time=5)
                        fightedWithLeader = ret

                    if ret:
                        self.sleep(10)
                        ret = self.d.exists("friend.1920x1080.png")
                        if ret:
                            break
                    else:
                        swordExists = False
                        break

                if not swordExists:
                    break

                if ret:
                    count = 0
                    while True:
                        ret = self.clickImage("click_to_continue.1920x1080.png")
                        if not ret:
                            if count >= 3:
                                self.sleep(7)
                                break
                        else:
                            count += 1

            while fightedWithLeader:
                ret = self.clickImage("fresh.1920x1080.png")
                if ret:
                    self.clickImage("back.1920x1080.png")
                else:
                    break

            ret = self.clickImage("back.1920x1080.png")

            if ret:
                self.sleep(2)
                ret = self.clickImage("chapter_quit_confirm.1920x1080.png")
                if ret:
                    self.sleep(7)

    def clickImage(self, image, time=10):
        try:
            self.d.click_image(image, timeout=time)
            return True
        except atx.ImageNotFoundError:
            print('%s button not found' % image)
            return False

    def sleep(self, seconds, msg=""):
        if msg is not None and msg != "":
            print("sleep for " + msg)
        print("sleep : " + str(seconds))
        time.sleep(seconds)

def main():
    YysHelper().chapter(13)

if __name__ == '__main__':
    main()
<commit_msg>Fix back too much after fight with leader<commit_after># -*- coding: utf-8 -*-

import atx
import time

class YysHelper():
    def __init__(self):
        self.d = atx.connect()
        self.d.image_path = ['.', 'assets']

    def chapter(self, level):
        """
        章节循环一轮游，进章节把屏幕内的小怪打完就出来
        """
        while True:
            chapterImage = "chapter%d.1920x1080.png" % level

            ret = self.clickImage(chapterImage)

            if ret:
                self.sleep(2)

            ret = self.clickImage("explore.1920x1080.png")

            if ret:
                self.sleep(8)

            ret = self.clickImage("lock.1920x1080.png", time=5)

            fightedWithLeader = False

            while not fightedWithLeader:
                swordExists = True
                while True:
                    ret = self.clickImage("sword.1920x1080.png")
                    if not ret:
                        ret = self.clickImage("leader_sword.1920x1080.png", time=5)
                        fightedWithLeader = ret

                    if ret:
                        self.sleep(10)
                        ret = self.d.exists("friend.1920x1080.png")
                        if ret:
                            break
                    else:
                        swordExists = False
                        break

                if not swordExists:
                    break

                if ret:
                    count = 0
                    while True:
                        ret = self.clickImage("click_to_continue.1920x1080.png")
                        if not ret:
                            if count >= 3:
                                self.sleep(7)
                                break
                        else:
                            count += 1

            if fightedWithLeader:
                ret = self.clickImage("fresh.1920x1080.png")
                if ret:
                    self.clickImage("back.1920x1080.png")
                else:
                    self.sleep(7)
                    break
            else:
                ret = self.clickImage("back.1920x1080.png")

                if ret:
                    self.sleep(2)
                    ret = self.clickImage("chapter_quit_confirm.1920x1080.png")
                    if ret:
                        self.sleep(7)

    def clickImage(self, image, time=10):
        try:
            self.d.click_image(image, timeout=time)
            return True
        except atx.ImageNotFoundError:
            print('%s button not found' % image)
            return False

    def sleep(self, seconds, msg=""):
        if msg is not None and msg != "":
            print("sleep for " + msg)
        print("sleep : " + str(seconds))
        time.sleep(seconds)

def main():
    YysHelper().chapter(13)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-

import os, sys, argparse, textwrap, mutagen

from ansicolor import cyan
from ansicolor import green
from ansicolor import red
from ansicolor import yellow

#from beets.mediafile import MediaFile
from mediafile import MediaFile
from mediafile import as_string
#from beets.util.functemplate import Template
from functemplate import Template
#from beets.library import DefaultTemplateFunctions as Functions
from functions import Functions

parser = argparse.ArgumentParser(
	formatter_class=argparse.RawDescriptionHelpFormatter,
	description=textwrap.dedent('''\
Python script to rename audio files from metadata tags.

Metadata fields:

	Ordinary metadata:

		- title
		- artist
		- artist_sort:         The “sort name” of the track artist
		                       (e.g., “Beatles, The” or “White, Jack”).
		- artist_credit:       The track-specific artist credit name,
		                       which may be a variation of the artist’s
		                       “canonical” name.
		- artistsafe:          The first available value of this metatag
		                       order: “albumartist” -> “artist” ->
		                       “albumartist_credit” -> “artist_credit”
		- artistsafe_sort:     The first available value of this metatag
		                       order: “albumartist_sort” ->
		                       “artist_sort” -> “artistsafe”
		- artist_initial:      First character in lowercase of
		                        “artistsafe_sort”
		- album
		- albumartist:         The artist for the entire album, which
		                       may be different from the artists for the
		                       individual tracks.
		- albumartist_sort
		- albumartist_credit
		- album_initial:       First character in lowercase of “album”.
		- genre
		- composer
		- grouping
		- year, month, day:    The release date of the specific release.
		- original_year, original_month, original_day:
		                       The release date of the original version
		                       of the album.
		- year_safe
		- track
		- tracktotal
		- disc
		- disctotal
		- disctrack:           Combination of disc and track in the
		                       format: disk-track, e.g. 1-01, 3-099
		- lyrics
		- comments
		- bpm
		- comp:                 Compilation flag.
		- albumtype:            The MusicBrainz album type; the
		                        MusicBrainz wiki has a list of type
		                        names.
		- label
		- asin
		- catalognum
		- script
		- language
		- country
		- albumstatus
		- media
		- albumdisambig
		- disctitle
		- encoder

	Audio information:

		- length                (in seconds)
		- bitrate               (in kilobits per second, with units:
		                        e.g., “192kbps”)
		- format                (e.g., “MP3” or “FLAC”)
		- channels
		- bitdepth              (only available for some formats)
		- samplerate            (in kilohertz, with units: e.g.,
		                        “48kHz”)

	MusicBrainz and fingerprint information:

		- mb_trackid
		- mb_albumid
		- mb_artistid
		- mb_albumartistid
		- mb_releasegroupid
		- acoustid_fingerprint
		- acoustid_id

	Template Functions

		- %lower{text}:         Convert “text” to lowercase.
		- %upper{text}:         Convert “text” to UPPERCASE.
		- %title{text}:         Convert “text” to Title Case.
		- %left{text,n}:        Return the first “n” characters of “text”.
		- %right{text,n}:       Return the last “n” characters of “text”.
		- %shorten{text,n}:     Shorten “text” on word boundarys.
		- %if{condition,text} or %if{condition,truetext,falsetext}:
		                        If condition is nonempty (or nonzero,
		                        if it’s a number), then returns the
		                        second argument. Otherwise, returns the
		                        third argument if specified (or nothing
		                        if falsetext is left off).
		- %delchars{text,chars}:
		                        Delete every single character of “chars“
		                        in “text”.
		- %deldupchars{text,chars}:
		                        Search for duplicate characters and
		                        replace with only one occurrance of
		                        this characters.
		- %replchars{text,chars,replace}
		- %sanitize{text}:      Delete in most file systems not allowed
		                        characters
		- %asciify{text}:       Convert non-ASCII characters to their
		                        ASCII equivalents. For example, “café”
		                        becomes “cafe”. Uses the mapping
		                        provided by the unidecode module.
		- %time{date_time,format,curformat}:
		                        Return the date and time in any format
		                        accepted by strftime. For example, to
		                        get the year some music was added to
		                        your library, use %time{$added,%Y}.
		- %first{text}:         Returns the first item, separated by ; .
		                        You can use %first{text,count,skip},
		                        where count is the number of items
		                        (default 1) and skip is number to skip
		                        (default 0). You can also use
		                        %first{text,count,skip,sep,join} where
		                        sep is the separator, like ; or / and
		                        join is the text to concatenate the
		                        items.
		- %ifdef{field}, %ifdef{field,truetext} or %ifdef{field,truetext,falsetext}:
		                        If field exists, then return truetext or
		                        field (default). Otherwise, returns
		                        falsetext. The field should be entered
		                        without $.

	'''
	)
)

parser.add_argument('folder',
	help='A folder containing audio files or a audio file')

parser.add_argument('-f', '--format',
	help='A format string',
	default='$artist_initial/$artistsafe_sort/%shorten{${album},32}%ifdef{yea_safe,_${year_safe}}/${disctrack}_%shorten{$title,32}')

parser.add_argument('-c', '--compilation',
	help='Format string for compilations',
	default='_compilations/$album_initial/$album/${disctrack}_%shorten{$title,32}')

parser.add_argument('-s', '--singelton',
	help='A format string for singletons',
	default='$artist $track')

parser.add_argument('-S', '--shell-friendly',
	help='Rename audio files “shell friendly”, this means without whitespaces, parentheses etc.',
	action='store_true')

parser.add_argument('-d', '--dry-run',
	help='A format string for singeltons',
	action='store_true')

parser.add_argument('-D', '--debug',
	help='Show special debug informations: meta, artist, track, year',
	default=False)

parser.add_argument('-e', '--extensions',
	help='Extensions to rename',
	default='mp3')

parser.add_argument('-b', '--base-dir',
	help='Base directory',
	default='')

parser.add_argument('-a', '--folder-as-base-dir',
	help='Use specified folder as base directory',
	action='store_true')

parser.add_argument('-C', '--copy',
	help='Copy files instead of rename / move.',
	action='store_true')

args = parser.parse_args()

class Meta(object):

	def __init__(self, path):
		self.media_file = MediaFile(path)
		self.m = {}
		for key in MediaFile.readable_fields():
			value = getattr(self.media_file, key)
			if key != 'art':
				if value:
					if isinstance(value, str) or isinstance(value, unicode):
						if args.shell_friendly:
							value = Functions.tmpl_asciify(value)
							value = Functions.tmpl_delchars(value, '().,!"\'’')
							value = Functions.tmpl_replchars(value, '-', ' ')
							value = Functions.tmpl_sanitize(value)
						else:
							self.m[key] = Functions.tmpl_sanitize(value)
				else:
					value = ''
				self.m[key] = value
		self.discTrack()
		self.artistSafe()
		self.yearSafe()
		self.initials()

	def discTrack(self):
		if self.m['disctotal'] > 9:
			disk = str(self.m['disc']).zfill(2)
		else:
			disk = str(self.m['disc'])

		if self.m['tracktotal'] > 99:
			track = str(self.m['track']).zfill(3)
		else:
			track = str(self.m['track']).zfill(2)

		if self.m['disc'] and self.m['disctotal'] > 1:
			self.m['disctrack'] = disk + '-' + track
		else:
			self.m['disctrack'] = track

	def artistSafe(self):
		if not self.m['albumartist_sort'] and self.m['albumartist']:
			self.m['albumartist_sort'] = self.m['albumartist']

		if self.m['albumartist_sort']:
			self.m['artistsafe_sort'] = self.m['albumartist_sort']
		elif self.m['artist_sort']:
			self.m['artistsafe_sort'] = self.m['artist_sort']

		if self.m['albumartist']:
			self.m['artistsafe'] = self.m['albumartist']
		elif self.m['artist']:
			self.m['artistsafe'] = self.m['artist']
		elif self.m['albumartist_credit']:
			self.m['artistsafe'] = self.m['albumartist_credit']
		elif self.m['artist_credit']:
			self.m['artistsafe'] = self.m['artist_credit']

		if not 'artistsafe_sort' in self.m:
			if self.m['artistsafe']:
				self.m['artistsafe_sort'] = self.m['artistsafe']
			else:
				self.m['artistsafe_sort'] = 'Unknown'

	def yearSafe(self):
		if self.m['original_year']:
			value = self.m['original_year']
		elif self.m['year']:
			value = self.m['year']
		else:
			value = ''
		self.m['year_safe'] = value

	def initials(self):
		self.m['artist_initial'] = self.m['artistsafe_sort'][0:1].lower()
		self.m['album_initial'] = self.m['album'][0:1].lower()

	def getMeta(self):
		return self.m

class Rename(object):

	def __init__(self, file, root_path = ''):
		if root_path:
			self.old_file = os.path.join(root_path, file)
		else:
			self.old_file = file

		if args.base_dir:
			self.base_dir = args.base_dir
		else:
			self.base_dir = os.getcwd()

		if args.folder_as_base_dir:
			self.base_dir = os.path.realpath(root_path)

		self.old_path = os.path.realpath(self.old_file)
		self.extension = self.old_file.split('.')[-1]

		meta = Meta(self.old_path)
		self.meta = meta.getMeta()
		if self.meta['comp']:
			t = Template(as_string(args.compilation))
		else:
			t = Template(as_string(args.format))
		f = Functions(self.meta)
		self.new_filename = t.substitute(self.meta, f.functions())
		self.new_filename = f.tmpl_deldupchars(self.new_filename + '.' + self.extension.lower())
		self.new_path = os.path.join(self.base_dir, self.new_filename)
		self.message = red(self.old_path.decode('utf-8')) + '\n  -> ' + green(self.new_path) + '\n'

	def createDir(self, path):
		path = os.path.dirname(path)
		import errno
		try:
			os.makedirs(path)
		except OSError as exception:
			if exception.errno != errno.EEXIST:
				raise

	def dryRun(self):
		print('Dry run: ' + self.message)

	def debug(self, option):
		def p(tag):
			print(cyan(tag) + u': ' + as_string(self.meta[tag]))
		print('\n' + green('- Debug: ') + red(option, reverse=True) + green(' --------------------------------'))
		print(yellow(self.old_file))

		if option == 'artist':

			p('artist')
			p('albumartist')
			p('artistsafe')
			p('artist_sort')
			p('artist_credit')
			p('albumartist_credit')
			p('albumartist_sort')
			p('artistsafe_sort')

		elif option == 'meta':
			for key, value in self.meta.iteritems():
				if key != 'art' and value:
					print(cyan(as_string(key)) + ': ' + as_string(value))

		elif option == 'mediafile':
			m = MediaFile(self.old_file)
			for key in MediaFile.readable_fields():
				value = getattr(m, key)
				if key != 'art':
					print(cyan(key) + ': ' + as_string(value))

		elif option == 'mutagen':
			m = mutagen.File(self.old_file, easy=True)
			print(m)
			for key, value in m.iteritems():
				print(cyan(key) + ': ' + value[0])

		elif option == 'track':
			p('track')
			p('tracktotal')
			p('disc')
			p('disctotal')
			p('disctrack')

		elif option == 'year':
			p('original_year')
			p('year')
			p('date')

	def rename(self):
		print('Rename: ' + self.message)
		self.createDir(self.new_path)
		os.rename(self.old_path, self.new_path)

	def copy(self):
		print('Copy: ' + self.message)
		import shutil
		self.createDir(self.new_path)
		shutil.copy2(self.old_path, self.new_path)

def execute(path, root_path = ''):
	if path.endswith((".mp3", ".MP3", ".m4a", ".flac", ".wma")) == True:
		audio = Rename(path, root_path)
		if not audio.meta['mb_trackid']:
			print(red('☠ no musicbrainz ☠', reverse=True) + ': ' + audio.old_file)
		elif args.dry_run:
			audio.dryRun()
		elif args.debug:
			audio.debug(args.debug)
		elif args.copy:
			audio.copy()
		else:
			audio.rename()

if __name__ == '__main__':

	if os.path.isdir(args.folder):
		for root_path, subdirs, files in os.walk(args.folder):
			for file in files:
				execute(file, root_path)

	else:
		execute(args.folder)

<commit_msg>Add option skip if empty<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-

import os, sys, argparse, textwrap, mutagen

from ansicolor import cyan
from ansicolor import green
from ansicolor import red
from ansicolor import yellow

#from beets.mediafile import MediaFile
from mediafile import MediaFile
from mediafile import as_string
#from beets.util.functemplate import Template
from functemplate import Template
#from beets.library import DefaultTemplateFunctions as Functions
from functions import Functions

parser = argparse.ArgumentParser(
	formatter_class=argparse.RawDescriptionHelpFormatter,
	description=textwrap.dedent('''\
Python script to rename audio files from metadata tags.

Metadata fields:

	Ordinary metadata:

		- title
		- artist
		- artist_sort:         The “sort name” of the track artist
		                       (e.g., “Beatles, The” or “White, Jack”).
		- artist_credit:       The track-specific artist credit name,
		                       which may be a variation of the artist’s
		                       “canonical” name.
		- artistsafe:          The first available value of this metatag
		                       order: “albumartist” -> “artist” ->
		                       “albumartist_credit” -> “artist_credit”
		- artistsafe_sort:     The first available value of this metatag
		                       order: “albumartist_sort” ->
		                       “artist_sort” -> “artistsafe”
		- artist_initial:      First character in lowercase of
		                        “artistsafe_sort”
		- album
		- albumartist:         The artist for the entire album, which
		                       may be different from the artists for the
		                       individual tracks.
		- albumartist_sort
		- albumartist_credit
		- album_initial:       First character in lowercase of “album”.
		- genre
		- composer
		- grouping
		- year, month, day:    The release date of the specific release.
		- original_year, original_month, original_day:
		                       The release date of the original version
		                       of the album.
		- year_safe
		- track
		- tracktotal
		- disc
		- disctotal
		- disctrack:           Combination of disc and track in the
		                       format: disk-track, e.g. 1-01, 3-099
		- lyrics
		- comments
		- bpm
		- comp:                 Compilation flag.
		- albumtype:            The MusicBrainz album type; the
		                        MusicBrainz wiki has a list of type
		                        names.
		- label
		- asin
		- catalognum
		- script
		- language
		- country
		- albumstatus
		- media
		- albumdisambig
		- disctitle
		- encoder

	Audio information:

		- length                (in seconds)
		- bitrate               (in kilobits per second, with units:
		                        e.g., “192kbps”)
		- format                (e.g., “MP3” or “FLAC”)
		- channels
		- bitdepth              (only available for some formats)
		- samplerate            (in kilohertz, with units: e.g.,
		                        “48kHz”)

	MusicBrainz and fingerprint information:

		- mb_trackid
		- mb_albumid
		- mb_artistid
		- mb_albumartistid
		- mb_releasegroupid
		- acoustid_fingerprint
		- acoustid_id

	Template Functions

		- %lower{text}:         Convert “text” to lowercase.
		- %upper{text}:         Convert “text” to UPPERCASE.
		- %title{text}:         Convert “text” to Title Case.
		- %left{text,n}:        Return the first “n” characters of “text”.
		- %right{text,n}:       Return the last “n” characters of “text”.
		- %shorten{text,n}:     Shorten “text” on word boundarys.
		- %if{condition,text} or %if{condition,truetext,falsetext}:
		                        If condition is nonempty (or nonzero,
		                        if it’s a number), then returns the
		                        second argument. Otherwise, returns the
		                        third argument if specified (or nothing
		                        if falsetext is left off).
		- %delchars{text,chars}:
		                        Delete every single character of “chars“
		                        in “text”.
		- %deldupchars{text,chars}:
		                        Search for duplicate characters and
		                        replace with only one occurrance of
		                        this characters.
		- %replchars{text,chars,replace}
		- %sanitize{text}:      Delete in most file systems not allowed
		                        characters
		- %asciify{text}:       Convert non-ASCII characters to their
		                        ASCII equivalents. For example, “café”
		                        becomes “cafe”. Uses the mapping
		                        provided by the unidecode module.
		- %time{date_time,format,curformat}:
		                        Return the date and time in any format
		                        accepted by strftime. For example, to
		                        get the year some music was added to
		                        your library, use %time{$added,%Y}.
		- %first{text}:         Returns the first item, separated by ; .
		                        You can use %first{text,count,skip},
		                        where count is the number of items
		                        (default 1) and skip is number to skip
		                        (default 0). You can also use
		                        %first{text,count,skip,sep,join} where
		                        sep is the separator, like ; or / and
		                        join is the text to concatenate the
		                        items.
		- %ifdef{field}, %ifdef{field,truetext} or %ifdef{field,truetext,falsetext}:
		                        If field exists, then return truetext or
		                        field (default). Otherwise, returns
		                        falsetext. The field should be entered
		                        without $.

	'''
	)
)

parser.add_argument('folder',
	help='A folder containing audio files or a audio file')

parser.add_argument('-f', '--format',
	help='A format string',
	default='$artist_initial/$artistsafe_sort/%shorten{${album},32}%ifdef{yea_safe,_${year_safe}}/${disctrack}_%shorten{$title,32}')

parser.add_argument('-c', '--compilation',
	help='Format string for compilations',
	default='_compilations/$album_initial/$album/${disctrack}_%shorten{$title,32}')

parser.add_argument('-S', '--shell-friendly',
	help='Rename audio files “shell friendly”, this means without whitespaces, parentheses etc.',
	action='store_true')

parser.add_argument('-d', '--dry-run',
	help='A format string for singeltons',
	action='store_true')

parser.add_argument('-D', '--debug',
	help='Show special debug informations: meta, artist, track, year',
	default=False)

parser.add_argument('-e', '--extensions',
	help='Extensions to rename',
	default='mp3')

parser.add_argument('-b', '--base-dir',
	help='Base directory',
	default='')

parser.add_argument('-s', '--skip-if-empty',
	help='Skip renaming of field is empty.',
	default=False)

parser.add_argument('-a', '--folder-as-base-dir',
	help='Use specified folder as base directory',
	action='store_true')

parser.add_argument('-C', '--copy',
	help='Copy files instead of rename / move.',
	action='store_true')

args = parser.parse_args()

class Meta(object):

	def __init__(self, path):
		self.media_file = MediaFile(path)
		self.m = {}
		for key in MediaFile.readable_fields():
			value = getattr(self.media_file, key)
			if key != 'art':
				if value:
					if isinstance(value, str) or isinstance(value, unicode):
						if args.shell_friendly:
							value = Functions.tmpl_asciify(value)
							value = Functions.tmpl_delchars(value, '().,!"\'’')
							value = Functions.tmpl_replchars(value, '-', ' ')
							value = Functions.tmpl_sanitize(value)
						else:
							self.m[key] = Functions.tmpl_sanitize(value)
				else:
					value = ''
				self.m[key] = value
		self.discTrack()
		self.artistSafe()
		self.yearSafe()
		self.initials()

	def discTrack(self):
		if self.m['disctotal'] > 9:
			disk = str(self.m['disc']).zfill(2)
		else:
			disk = str(self.m['disc'])

		if self.m['tracktotal'] > 99:
			track = str(self.m['track']).zfill(3)
		else:
			track = str(self.m['track']).zfill(2)

		if self.m['disc'] and self.m['disctotal'] > 1:
			self.m['disctrack'] = disk + '-' + track
		else:
			self.m['disctrack'] = track

	def artistSafe(self):
		if not self.m['albumartist_sort'] and self.m['albumartist']:
			self.m['albumartist_sort'] = self.m['albumartist']

		if self.m['albumartist_sort']:
			self.m['artistsafe_sort'] = self.m['albumartist_sort']
		elif self.m['artist_sort']:
			self.m['artistsafe_sort'] = self.m['artist_sort']

		if self.m['albumartist']:
			self.m['artistsafe'] = self.m['albumartist']
		elif self.m['artist']:
			self.m['artistsafe'] = self.m['artist']
		elif self.m['albumartist_credit']:
			self.m['artistsafe'] = self.m['albumartist_credit']
		elif self.m['artist_credit']:
			self.m['artistsafe'] = self.m['artist_credit']

		if not 'artistsafe_sort' in self.m:
			if self.m['artistsafe']:
				self.m['artistsafe_sort'] = self.m['artistsafe']
			else:
				self.m['artistsafe_sort'] = 'Unknown'

	def yearSafe(self):
		if self.m['original_year']:
			value = self.m['original_year']
		elif self.m['year']:
			value = self.m['year']
		else:
			value = ''
		self.m['year_safe'] = value

	def initials(self):
		self.m['artist_initial'] = self.m['artistsafe_sort'][0:1].lower()
		self.m['album_initial'] = self.m['album'][0:1].lower()

	def getMeta(self):
		return self.m

class Rename(object):

	def __init__(self, file, root_path = ''):
		if root_path:
			self.old_file = os.path.join(root_path, file)
		else:
			self.old_file = file

		if args.base_dir:
			self.base_dir = args.base_dir
		else:
			self.base_dir = os.getcwd()

		if args.folder_as_base_dir:
			self.base_dir = os.path.realpath(root_path)

		self.old_path = os.path.realpath(self.old_file)
		self.extension = self.old_file.split('.')[-1]

		meta = Meta(self.old_path)
		self.meta = meta.getMeta()
		if self.meta['comp']:
			t = Template(as_string(args.compilation))
		else:
			t = Template(as_string(args.format))
		f = Functions(self.meta)
		self.new_filename = t.substitute(self.meta, f.functions())
		self.new_filename = f.tmpl_deldupchars(self.new_filename + '.' + self.extension.lower())
		self.new_path = os.path.join(self.base_dir, self.new_filename)
		self.message = red(self.old_path.decode('utf-8')) + '\n  -> ' + green(self.new_path) + '\n'

	def createDir(self, path):
		path = os.path.dirname(path)
		import errno
		try:
			os.makedirs(path)
		except OSError as exception:
			if exception.errno != errno.EEXIST:
				raise

	def skipMessage(self):
		print(red('☠ no field ' + args.skip_if_empty + ' ☠', reverse=True) + ': ' + self.old_file)

	def dryRun(self):
		print('Dry run: ' + self.message)

	def debug(self, option):
		def p(tag):
			print(cyan(tag) + u': ' + as_string(self.meta[tag]))
		print('\n' + green('- Debug: ') + red(option, reverse=True) + green(' --------------------------------'))
		print(yellow(self.old_file))

		if option == 'artist':

			p('artist')
			p('albumartist')
			p('artistsafe')
			p('artist_sort')
			p('artist_credit')
			p('albumartist_credit')
			p('albumartist_sort')
			p('artistsafe_sort')

		elif option == 'meta':
			for key, value in self.meta.iteritems():
				if key != 'art' and value:
					print(cyan(as_string(key)) + ': ' + as_string(value))

		elif option == 'mediafile':
			m = MediaFile(self.old_file)
			for key in MediaFile.readable_fields():
				value = getattr(m, key)
				if key != 'art':
					print(cyan(key) + ': ' + as_string(value))

		elif option == 'mutagen':
			m = mutagen.File(self.old_file, easy=True)
			print(m)
			for key, value in m.iteritems():
				print(cyan(key) + ': ' + value[0])

		elif option == 'track':
			p('track')
			p('tracktotal')
			p('disc')
			p('disctotal')
			p('disctrack')

		elif option == 'year':
			p('original_year')
			p('year')
			p('date')

	def rename(self):
		print('Rename: ' + self.message)
		self.createDir(self.new_path)
		os.rename(self.old_path, self.new_path)

	def copy(self):
		print('Copy: ' + self.message)
		import shutil
		self.createDir(self.new_path)
		shutil.copy2(self.old_path, self.new_path)

	def execute(self):
		if args.skip_if_empty and not self.meta[args.skip_if_empty]:
			self.skipMessage()
		else:
			if args.dry_run:
				self.dryRun()
			elif args.debug:
				self.debug(args.debug)
			elif args.copy:
				self.copy()
			else:
				self.rename()

def execute(path, root_path = ''):
	if path.endswith((".mp3", ".MP3", ".m4a", ".flac", ".wma")) == True:
		audio = Rename(path, root_path)
		audio.execute()

if __name__ == '__main__':

	if os.path.isdir(args.folder):
		for root_path, subdirs, files in os.walk(args.folder):
			for file in files:
				execute(file, root_path)

	else:
		execute(args.folder)

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- encoding: utf-8 -*-

from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.conf import settings

class Group(models.Model):
    name = models.CharField(_(u'Nombre del grupo'), max_length=40, blank=True)
    adverse_reaction = models.CharField(_(u'Reacciones Adversas'),
                        max_length=255)

    def __unicode__(self):
        return u'%s' % (self.name)

class Component(models.Model):
    KIND = (
        (settings.ACTIVE_INGREDIENT, _(u'Principio Activo')),
        (settings.MEDICINE, _(u'Fármaco(nombre comercial)')),
    )

    groups = models.ManyToManyField(Group, related_name='groupcomponents')

    kind_component = models.IntegerField(_(u'Tipo de componente'),
                                        choices=KIND)
    name = models.CharField(_(u'Nombre del componente'), max_length=255,
                            blank=True)

    def __unicode__(self):
        return u'%s' % (self.name)
<commit_msg>Related_name changed<commit_after># -*- encoding: utf-8 -*-

from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.conf import settings

class Group(models.Model):
    name = models.CharField(_(u'Nombre del grupo'), max_length=40, blank=True)

    adverse_reaction = models.CharField(_(u'Reacciones Adversas'),
                        max_length=255)

    def __unicode__(self):
        return u'%s' % (self.name)

class Component(models.Model):
    KIND = (
        (settings.ACTIVE_INGREDIENT, _(u'Principio Activo')),
        (settings.MEDICINE, _(u'Fármaco(nombre comercial)')),
    )

    groups = models.ManyToManyField(Group, related_name='groups_components')

    kind_component = models.IntegerField(_(u'Tipo de componente'),
                                        choices=KIND)

    name = models.CharField(_(u'Nombre del componente'), max_length=255,
                            blank=True)

    def __unicode__(self):
        return u'%s' % (self.name)
<|endoftext|>


<commit_before># Note that nothing yet exists to check this file
# for checkable errors, such as inputs/outputs
# not matching the logical function

modules={}

### Black cell
ppa_black=dict()

ppa_black['verilog']="""
module ppa_black(gin, pin, gout, pout);

    input [1:0] gin, pin;
    output gout, pout;

    assign pout=pin[1]&pin[0];
    assign gout=gin[1]|(pin[1]&gin[0]);

endmodule
"""

ppa_black['shape']='square'
ppa_black['fillcolor']='black'

# List of inputs represented by (name, bits) tuple
ppa_black['ins']=[('gin',2),('pin',2)]

# List of outputs represented by (name, bits) tuple
ppa_black['outs']=[('gout',1),('pout',1)]

# Returns function that executes the logic of the module
ppa_black['logic'] = lambda gin,pin: [
                 gin[1]|(pin[1]&gin[0]),
                 pin[1]&pin[0]
                 ]

# What section of the tree is this cell in?
# Pre, post, or main?
ppa_black['type']='main'

# Is this cell a buffer?
ppa_black['buf']=False

# Is this just a fake, helper, cell?
ppa_black['exists']=True

# Logical effort
ppa_black['pd'] = 7.5/3
ppa_black['le'] = 5.0/3
ppa_black['diag_le'] = 6.0/3

modules['ppa_black']=ppa_black

### Grey cells
ppa_grey=dict()

ppa_grey['verilog']="""
module ppa_grey(gin, pin, gout);

    input[1:0] gin;
    input pin;
    output gout;

    assign gout=gin[1]|(pin&gin[0]);

endmodule
"""

ppa_grey['shape']='square'
ppa_grey['fillcolor']='grey'

# List of inputs represented by (name, bits) tuple
ppa_grey['ins']=[('gin',2),('pin',1)]

# List of outputs represented by (name, bits) tuple
ppa_grey['outs']=[('gout',1)]

# Returns function that executes the logic of the module
ppa_grey['logic'] = lambda gin,pin: [
                gin[1]|(pin&gin[0])
                ]

ppa_grey['pd'] = 7.5/3
ppa_grey['le'] = 2

# What section of the tree is this cell in?
# Pre, post, or main?
ppa_grey['type']='main'

# Is this cell a buffer?
ppa_grey['buf']=False

# Is this just a fake, helper, cell?
ppa_grey['exists']=True

modules['ppa_grey']=ppa_grey

### Reduced Black cell
ppaL_black=dict()

ppaL_black['verilog']="""
module ppaL_black(hout, iout, gin, pin);

    input [1:0] gin, pin;
    output hout, iout;

    assign iout=pin[1]&pin[0];
    assign hout=gin[1]|gin[0];

endmodule
"""

ppaL_black['shape']='square'
ppaL_black['fillcolor']='ppa_black'

# List of inputs represented by (name, bits) tuple
ppaL_black['ins']=[('gin',2),('pin',2)]

# List of outputs represented by (name, bits) tuple
ppaL_black['outs']=[('hout',1),('iout',1)]

# Returns function that executes the logic of the module
ppaL_black['logic'] = lambda gin,pin: [
                gin[1]|gin[0],
                pin[1]&pin[0]
                ]

ppaL_black['pd'] = 5/3
ppaL_black['le'] = 2

# What section of the tree is this cell in?
# Pre, post, or main?
ppaL_black['type']='main'

# Is this cell a buffer?
ppaL_black['buf']=False

# Is this just a fake, helper, cell?
ppaL_black['exists']=True

modules['ppaL_black']=ppaL_black

### Reduced Grey cell
ppaL_grey=dict()

ppaL_grey['verilog']="""
module ppaL_grey(hout, gin);

    input[1:0] gin;
    output hout;

    assign hout=gin[1]|gin[0];

endmodule
"""

ppaL_grey['shape']='square'
ppaL_grey['fillcolor']='ppa_grey'

# List of inputs represented by (name, bits) tuple
ppaL_grey['ins']=[('gin',2)]

# List of outputs represented by (name, bits) tuple
ppaL_grey['outs']=[('hout',1)]

# Returns function that executes the logic of the module
ppaL_grey['logic'] = lambda gin: [
                gin[1]|gin[0]
                ]

ppaL_grey['pd'] = 5/3
ppaL_grey['le'] = 2

# What section of the tree is this cell in?
# Pre, post, or main?
ppaL_grey['type']='main'

# Is this cell a buffer?
ppaL_grey['buf']=False

# Is this just a fake, helper, cell?
ppaL_grey['exists']=True

modules['ppaL_grey']=ppaL_grey

### Buffer nodes
buffer_node=dict()

buffer_node['verilog']="""
module buffer_node(pin, gin, pout, gout);

    input pin, gin;
    output pout, gout;

    assign pout=pin;
    assign gout=gin;

endmodule
"""

buffer_node['shape']='invtriangle'
buffer_node['fillcolor']='white'

buffer_node['ins']=[('gin',1),('pin',1)]
buffer_node['outs']=[('gout',1),('pout',1)]

buffer_node['logic'] = lambda pin,gin: [pin,gin]

buffer_node['pd'] = 0
buffer_node['le'] = 0

# What section of the tree is this cell in?
# Pre, post, or main?
buffer_node['type']='main'

# Is this cell a buffer?
buffer_node['buf']=True

# Is this just a fake, helper, cell?
buffer_node['exists']=True

modules['buffer_node']=buffer_node

### Invis nodes
invis_node=dict()

invis_node['verilog']=buffer_node['verilog'].replace('buffer','invis')

#invis_node['style']='invis'
invis_node['shape']='point'
invis_node['fixedsize']='shape'
invis_node['width']=0
invis_node['height']=0
invis_node['fillcolor']='black'

invis_node['ins']=[('gin',1),('pin',1)]
invis_node['outs']=[('gout',1),('pout',1)]

invis_node['logic'] = lambda pin,gin: [pin,gin]

invis_node['pd'] = buffer_node['pd']
invis_node['le'] = buffer_node['le']

# What section of the tree is this cell in?
# Pre, post, or main?
invis_node['type']='main'

# Is this cell a buffer?
invis_node['buf']=True

# Is this just a fake, helper, cell?
invis_node['exists']=False

modules['invis_node']=invis_node

### Pre-processing node
ppa_pre=dict()

ppa_pre['verilog']="""
module ppa_pre(a_in, b_in, pout, gout);

    input a_in, b_in;
    output pout, gout;

    assign pout=a_in^b_in;
    assign gout=a_in&b_in;

endmodule
"""

ppa_pre['shape']='square'
ppa_pre['fillcolor']='white'
ppa_pre['label']='pre'
ppa_pre['style']='dashed'

ppa_pre['ins']=[('a_in',1),('b_in',1)]
ppa_pre['outs']=[('pout',1),('gout',1)]

ppa_pre['logic'] = lambda a,b: [
                   a^b,
                   a&b
                   ]

ppa_pre['pd'] = 9/3
ppa_pre['le'] = 9/3

# What section of the tree is this cell in?
# Pre, post, or main?
ppa_pre['type']='pre'

# Is this cell a buffer?
ppa_pre['buf']=False

# Is this just a fake, helper, cell?
ppa_pre['exists']=True

modules['ppa_pre']=ppa_pre

### Fake pre-processing node
ppa_first_pre=dict(ppa_pre)

ppa_first_pre['verilog']="""
module ppa_first_pre(cin, pout, gout);

    input cin;
    output pout, gout;

    assign pout=1'b0;
    assign gout=cin;

endmodule
"""

ppa_first_pre['ins']=[('cin',1)]

ppa_first_pre['logic'] = lambda cin: [0,cin]

modules['ppa_first_pre']=ppa_first_pre

### XOR node

xor_node=dict()

xor_node['verilog']="""
module xor_node(pin, gin, sum);

    input pin, gin;
    output sum;

    assign sum=pin^gin;

endmodule
"""

xor_node['shape']='invtrapezium'
xor_node['fillcolor']='white'
xor_node['style']='dashed'
xor_node['label']='XOR'
xor_node['fixedsize']='shape'

xor_node['ins']=[('pin',1),('gin',1)]
xor_node['outs']=[('sum',1)]

xor_node['logic'] = lambda pin,gin: [pin^gin]

xor_node['pd'] = 9/3
xor_node['le'] = 9/3

modules['xor_node']=xor_node

### Post-processing node

ppa_post=dict(xor_node)

ppa_post['verilog']=ppa_post['verilog'].replace('xor_node','ppa_post')
ppa_post['label']='post'

# What section of the tree is this cell in?
# Pre, post, or main?
ppa_post['type']='post'

# Is this cell a buffer?
ppa_post['buf']=False

# Is this just a fake, helper, cell?
ppa_post['exists']=True

modules['ppa_post']=ppa_post

if __name__=="__main__":
    raise RuntimeError("This file is importable, but not executable")
<commit_msg>Standardized tabs across HDL<commit_after># Note that nothing yet exists to check this file
# for checkable errors, such as inputs/outputs
# not matching the logical function

modules={}

### Black cell
ppa_black=dict()

ppa_black['verilog']="""
module ppa_black(gin, pin, gout, pout);

\tinput [1:0] gin, pin;
\toutput gout, pout;

\tassign pout=pin[1]&pin[0];
\tassign gout=gin[1]|(pin[1]&gin[0]);

endmodule
"""

ppa_black['shape']='square'
ppa_black['fillcolor']='black'

# List of inputs represented by (name, bits) tuple
ppa_black['ins']=[('gin',2),('pin',2)]

# List of outputs represented by (name, bits) tuple
ppa_black['outs']=[('gout',1),('pout',1)]

# Returns function that executes the logic of the module
ppa_black['logic'] = lambda gin,pin: [
                 gin[1]|(pin[1]&gin[0]),
                 pin[1]&pin[0]
                 ]

# What section of the tree is this cell in?
# Pre, post, or main?
ppa_black['type']='main'

# Is this cell a buffer?
ppa_black['buf']=False

# Is this just a fake, helper, cell?
ppa_black['exists']=True

# Logical effort
ppa_black['pd'] = 7.5/3
ppa_black['le'] = 5.0/3
ppa_black['diag_le'] = 6.0/3

modules['ppa_black']=ppa_black

### Grey cells
ppa_grey=dict()

ppa_grey['verilog']="""
module ppa_grey(gin, pin, gout);

\tinput[1:0] gin;
\tinput pin;
\toutput gout;

\tassign gout=gin[1]|(pin&gin[0]);

endmodule
"""

ppa_grey['shape']='square'
ppa_grey['fillcolor']='grey'

# List of inputs represented by (name, bits) tuple
ppa_grey['ins']=[('gin',2),('pin',1)]

# List of outputs represented by (name, bits) tuple
ppa_grey['outs']=[('gout',1)]

# Returns function that executes the logic of the module
ppa_grey['logic'] = lambda gin,pin: [
                gin[1]|(pin&gin[0])
                ]

ppa_grey['pd'] = 7.5/3
ppa_grey['le'] = 2

# What section of the tree is this cell in?
# Pre, post, or main?
ppa_grey['type']='main'

# Is this cell a buffer?
ppa_grey['buf']=False

# Is this just a fake, helper, cell?
ppa_grey['exists']=True

modules['ppa_grey']=ppa_grey

### Reduced Black cell
ppaL_black=dict()

ppaL_black['verilog']="""
module ppaL_black(hout, iout, gin, pin);

\tinput [1:0] gin, pin;
\toutput hout, iout;

\tassign iout=pin[1]&pin[0];
\tassign hout=gin[1]|gin[0];

endmodule
"""

ppaL_black['shape']='square'
ppaL_black['fillcolor']='ppa_black'

# List of inputs represented by (name, bits) tuple
ppaL_black['ins']=[('gin',2),('pin',2)]

# List of outputs represented by (name, bits) tuple
ppaL_black['outs']=[('hout',1),('iout',1)]

# Returns function that executes the logic of the module
ppaL_black['logic'] = lambda gin,pin: [
                gin[1]|gin[0],
                pin[1]&pin[0]
                ]

ppaL_black['pd'] = 5/3
ppaL_black['le'] = 2

# What section of the tree is this cell in?
# Pre, post, or main?
ppaL_black['type']='main'

# Is this cell a buffer?
ppaL_black['buf']=False

# Is this just a fake, helper, cell?
ppaL_black['exists']=True

modules['ppaL_black']=ppaL_black

### Reduced Grey cell
ppaL_grey=dict()

ppaL_grey['verilog']="""
module ppaL_grey(hout, gin);

\tinput[1:0] gin;
\toutput hout;

\tassign hout=gin[1]|gin[0];

endmodule
"""

ppaL_grey['shape']='square'
ppaL_grey['fillcolor']='ppa_grey'

# List of inputs represented by (name, bits) tuple
ppaL_grey['ins']=[('gin',2)]

# List of outputs represented by (name, bits) tuple
ppaL_grey['outs']=[('hout',1)]

# Returns function that executes the logic of the module
ppaL_grey['logic'] = lambda gin: [
                gin[1]|gin[0]
                ]

ppaL_grey['pd'] = 5/3
ppaL_grey['le'] = 2

# What section of the tree is this cell in?
# Pre, post, or main?
ppaL_grey['type']='main'

# Is this cell a buffer?
ppaL_grey['buf']=False

# Is this just a fake, helper, cell?
ppaL_grey['exists']=True

modules['ppaL_grey']=ppaL_grey

### Buffer nodes
buffer_node=dict()

buffer_node['verilog']="""
module buffer_node(pin, gin, pout, gout);

\tinput pin, gin;
\toutput pout, gout;

\tassign pout=pin;
\tassign gout=gin;

endmodule
"""

buffer_node['shape']='invtriangle'
buffer_node['fillcolor']='white'

buffer_node['ins']=[('gin',1),('pin',1)]
buffer_node['outs']=[('gout',1),('pout',1)]

buffer_node['logic'] = lambda pin,gin: [pin,gin]

buffer_node['pd'] = 0
buffer_node['le'] = 0

# What section of the tree is this cell in?
# Pre, post, or main?
buffer_node['type']='main'

# Is this cell a buffer?
buffer_node['buf']=True

# Is this just a fake, helper, cell?
buffer_node['exists']=True

modules['buffer_node']=buffer_node

### Invis nodes
invis_node=dict()

invis_node['verilog']=buffer_node['verilog'].replace('buffer','invis')

#invis_node['style']='invis'
invis_node['shape']='point'
invis_node['fixedsize']='shape'
invis_node['width']=0
invis_node['height']=0
invis_node['fillcolor']='black'

invis_node['ins']=[('gin',1),('pin',1)]
invis_node['outs']=[('gout',1),('pout',1)]

invis_node['logic'] = lambda pin,gin: [pin,gin]

invis_node['pd'] = buffer_node['pd']
invis_node['le'] = buffer_node['le']

# What section of the tree is this cell in?
# Pre, post, or main?
invis_node['type']='main'

# Is this cell a buffer?
invis_node['buf']=True

# Is this just a fake, helper, cell?
invis_node['exists']=False

modules['invis_node']=invis_node

### Pre-processing node
ppa_pre=dict()

ppa_pre['verilog']="""
module ppa_pre(a_in, b_in, pout, gout);

\tinput a_in, b_in;
\toutput pout, gout;

\tassign pout=a_in^b_in;
\tassign gout=a_in&b_in;

endmodule
"""

ppa_pre['shape']='square'
ppa_pre['fillcolor']='white'
ppa_pre['label']='pre'
ppa_pre['style']='dashed'

ppa_pre['ins']=[('a_in',1),('b_in',1)]
ppa_pre['outs']=[('pout',1),('gout',1)]

ppa_pre['logic'] = lambda a,b: [
                   a^b,
                   a&b
                   ]

ppa_pre['pd'] = 9/3
ppa_pre['le'] = 9/3

# What section of the tree is this cell in?
# Pre, post, or main?
ppa_pre['type']='pre'

# Is this cell a buffer?
ppa_pre['buf']=False

# Is this just a fake, helper, cell?
ppa_pre['exists']=True

modules['ppa_pre']=ppa_pre

### Fake pre-processing node
ppa_first_pre=dict(ppa_pre)

ppa_first_pre['verilog']="""
module ppa_first_pre(cin, pout, gout);

\tinput cin;
\toutput pout, gout;

\tassign pout=1'b0;
\tassign gout=cin;

endmodule
"""

ppa_first_pre['ins']=[('cin',1)]

ppa_first_pre['logic'] = lambda cin: [0,cin]

modules['ppa_first_pre']=ppa_first_pre

### XOR node

xor_node=dict()

xor_node['verilog']="""
module xor_node(pin, gin, sum);

\tinput pin, gin;
\toutput sum;

\tassign sum=pin^gin;

endmodule
"""

xor_node['shape']='invtrapezium'
xor_node['fillcolor']='white'
xor_node['style']='dashed'
xor_node['label']='XOR'
xor_node['fixedsize']='shape'

xor_node['ins']=[('pin',1),('gin',1)]
xor_node['outs']=[('sum',1)]

xor_node['logic'] = lambda pin,gin: [pin^gin]

xor_node['pd'] = 9/3
xor_node['le'] = 9/3

modules['xor_node']=xor_node

### Post-processing node

ppa_post=dict(xor_node)

ppa_post['verilog']=ppa_post['verilog'].replace('xor_node','ppa_post')
ppa_post['label']='post'

# What section of the tree is this cell in?
# Pre, post, or main?
ppa_post['type']='post'

# Is this cell a buffer?
ppa_post['buf']=False

# Is this just a fake, helper, cell?
ppa_post['exists']=True

modules['ppa_post']=ppa_post

if __name__=="__main__":
    raise RuntimeError("This file is importable, but not executable")
<|endoftext|>


<commit_before>import uuid
from django.core.files.base import ContentFile

__author__ = 'zhiwehu'

import utils
from models import Bookmark, BookmarkUrl
from threading import Timer
#second
t = 300

def cronjob():
    def update_bk_image():
        bookmarks = Bookmark.objects.all()
        for bookmark in bookmarks:
            if bookmark.screen_shot:
                break
            url = bookmark.url
            image = utils.get_image_by_url(url)
            if image:
                image_file_name = str(uuid.uuid1()) + ".jpg"
                bookmark.screen_shot.save(image_file_name, ContentFile(image))
                bookmark.save()

    update_bk_image()
    Timer(t, cronjob).start()

cronjob()
<commit_msg>update cron job from 300 to 7200 seconds<commit_after>import uuid
from django.core.files.base import ContentFile

__author__ = 'zhiwehu'

import utils
from models import Bookmark, BookmarkUrl
from threading import Timer
#second
t = 7200

def cronjob():
    def update_bk_image():
        bookmarks = Bookmark.objects.all()
        for bookmark in bookmarks:
            if bookmark.screen_shot:
                break
            url = bookmark.url
            image = utils.get_image_by_url(url)
            if image:
                image_file_name = str(uuid.uuid1()) + ".jpg"
                bookmark.screen_shot.save(image_file_name, ContentFile(image))
                bookmark.save()

    update_bk_image()
    Timer(t, cronjob).start()

cronjob()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from boto.s3.connection import S3Connection
from boto.s3.key import Key

def get_connection(key, secret):
	connection = S3Connection(key, secret)
	return connection

def get_bucket(bucket_name, connection):
	bucket = connection.get_bucket(bucket_name)
	return bucket

def upload_string_data(bucket, path, string_data, public=False, mime_type=None, metadata=None):
	key = Key(bucket, path)
	if mime_type:
		key.content_type = mime_type
	if metadata:
		key.update_metadata(metadata)
	key.set_contents_from_string(string_data)
	if public:
		key.make_public()

def delete_key(bucket, path):
	key = bucket.get_key(path)
	if key:
		key.delete()

def get_key(bucket, path):
	key = bucket.get_key(path)
	return key

def get_string_data(bucket, path):
	key = bucket.get_key(path)
	if key:
		return key.get_contents_as_string()<commit_msg>converting tabs to white space apparently<commit_after># -*- coding: utf-8 -*-

from boto.s3.connection import S3Connection
from boto.s3.key import Key

def get_connection(key, secret):
    connection = S3Connection(key, secret)
    return connection

def get_bucket(bucket_name, connection):
    bucket = connection.get_bucket(bucket_name)
    return bucket

def upload_string_data(bucket, path, string_data, public=False, mime_type=None, metadata=None):
    key = Key(bucket, path)
    if mime_type:
        key.content_type = mime_type
    if metadata:
        key.update_metadata(metadata)
    key.set_contents_from_string(string_data)
    if public:
        key.make_public()

def delete_key(bucket, path):
    key = bucket.get_key(path)
    if key:
        key.delete()

def get_key(bucket, path):
    key = bucket.get_key(path)
    return key

def get_string_data(bucket, path):
    key = bucket.get_key(path)
    if key:
        return key.get_contents_as_string()
<|endoftext|>


<commit_before>from toroid.warrior import Warrior

# This is the default command line for PMARS, with the settings we've specified
PMARS_CLI = "pmars -k -p 8000 -c 80000 -p 8000 -l 100 -d 100"
# This addition to the command line makes PMARS run with just validation
PMARS_NO_GRAPHICS = " -r 0 -v 000"

class MetadataNotFoundException (Exception):
    pass

def check_for_name(source):
    # Look through the source, trying to find a ;name
    for line in source.split('\n'):
        if len(line.split()) >= 2 and line.split()[0] == ";name":
            return line.strip(";name")
    # If execution reaches this point, there is no ;name line
    raise MetadataNotFoundException

def check_for_author(source):
    # Look through the source, trying to find a ;author
    for line in source.split('\n'):
        if len(line.split()) >= 2 and line.split()[0] == ";author":
            return line.strip(";author")
    # If execution reaches this point, there is not ;author line
    raise MetadataNotFoundException

def validate(source):
    import subprocess

    program_data = Warrior("", "", source)

    # First, check if ;name, ;author, and ;assert are all present
    try:
        program_data.name = check_for_name(source)
    except MetadataNotFoundException:
        return (False, "NAME metaline not found.\nYour program needs to have" +
                       " a line that starts with ;name and includes" +
                       " a program name.",
                       program_data)
    try:
        program_data.author = check_for_author(source)
    except MetadataNotFoundException:
        return (False, "AUTHOR metaline not found.\nYour program needs to" +
                       " have a line that starts with ;author and includes" +
                       " your name.",
                       program_data)

    # If we've reached this point, the metadata is valid
    # Open a PMARS process process we can communicate with
    p = subprocess.Popen((PMARS_CLI + PMARS_NO_GRAPHICS + ' -').split(),
                         stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT)
    # Send the program we need to validate, and get the output
    out = str(p.communicate(input=bytes(source, encoding="UTF-8"))[0],
              encoding="UTF-8") + "
"

    if p.returncode == 0:
        retval = True
    else:
        retval = False

    return (retval, out, program_data)
<commit_msg>Reduce reliance on exceptions.<commit_after>from toroid.warrior import Warrior

# This is the default command line for PMARS, with the settings we've specified
PMARS_CLI = "pmars -k -p 8000 -c 80000 -p 8000 -l 100 -d 100"
# This addition to the command line makes PMARS run with just validation
PMARS_NO_GRAPHICS = " -r 0 -v 000"

def check_for_name(source):
    # Look through the source, trying to find a ;name
    for line in source.split('\n'):
        if len(line.split()) >= 2 and line.split()[0] == ";name":
            return line.strip(";name")
    # If execution reaches this point, there is no ;name line
    return False

def check_for_author(source):
    # Look through the source, trying to find a ;author
    for line in source.split('\n'):
        if len(line.split()) >= 2 and line.split()[0] == ";author":
            return line.strip(";author")
    # If execution reaches this point, there is not ;author line
    return False

def validate(source):
    import subprocess

    program_data = Warrior("", "", source)

    # First, check if ;name, ;author, and ;assert are all present

    program_data.name = check_for_name(source)
    if not program_data.name:
        return (False, "NAME metaline not found.\nYour program needs to have" +
                       " a line that starts with ;name and includes" +
                       " a program name.",
                       program_data)

    program_data.author = check_for_author(source)
    if not program_data.author:
        return (False, "AUTHOR metaline not found.\nYour program needs to" +
                       " have a line that starts with ;author and includes" +
                       " your name.",
                       program_data)

    # If we've reached this point, the metadata is valid
    # Open a PMARS process process we can communicate with
    p = subprocess.Popen((PMARS_CLI + PMARS_NO_GRAPHICS + ' -').split(),
                         stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT)
    # Send the program we need to validate, and get the output
    out = str(p.communicate(input=bytes(source, encoding="UTF-8"))[0],
              encoding="UTF-8") + "
"

    if p.returncode == 0:
        retval = True
    else:
        retval = False

    return (retval, out, program_data)
<|endoftext|>


<commit_before># Copyright 2014 Isotoma Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from touchdown.core.resource import Resource
from touchdown.core.plan import Plan
from touchdown.core import argument

from .vpc import VPC
from touchdown.core import serializers
from ..common import SimpleDescribe, SimpleApply, SimpleDestroy

class Rule(Resource):

    """
    Represents a rule in a :py:class:`NetworkACL`.

    You shouldn't create ``Rule`` resources directly, they are created
    implicitly when defining a :py:class:`NetworkACL`. For example::

        network_acl = vpc.add_network_acl(
            name='my-network-acl',
            inbound=[
                {"port": 80, "network": "0.0.0.0/0"},
                {"port": 443, "network": "0.0.0.0/0"},
            ],
        )

    This will implicitly create 2 ``Rule`` resources.
    """

    resource_name = "rule"
    dot_ignore = True

    network = argument.IPNetwork(
        field="IpRanges",
        serializer=serializers.ListOfOne(serializers.Dict(
            CidrIp=serializers.String(),
        )),
    )
    """ A network range specified in CIDR form. For example, you could specify
    ``0.0.0.0/0`` to allow the entire internet to access the specified
    port/protocol. """

    protocol = argument.String(default='tcp', choices=['tcp', 'udp', 'icmp'], field="IpProtocol")
    """ The network protocol to allow or deny. By default this is ``tcp``. It can also
    be set to ``udp`` or ``icmp``. """

    port = argument.Integer(min=-1, max=32768)
    """ A port to allow access to. """

    from_port = argument.Integer(default=lambda r: r.port, min=-1, max=32768, field="FromPort")
    """ Instead of specifying ``port``, you can specify a range of ports
    starting at ``from_port`` """

    to_port = argument.Integer(default=lambda r: r.port, min=-1, max=32768, field="ToPort")
    """ If specifying a ``from_port`` you also need to specify a
    ``to_port``. """

    action = argument.String(default="ALLOW", choices=["ALLOW", "DENY"])
    """ Whether to allow or deny matching traffic. The default value is ``ALLOW``. """

    def __str__(self):
        name = super(Rule, self).__str__()
        if self.from_port == self.to_port:
            ports = "port {}".format(self.from_port)
        else:
            ports = "ports {} to {}".format(self.from_port, self.to_port)
        return "{}: {} {} from {}".format(name, self.protocol, ports, self.network)

class NetworkACL(Resource):

    """
    Network ACL's provide network filtering at subnet level, controlling both
    inbound and outbound traffic. They are:

     * Stateless. This means that return traffic is not automatically
       allowed. This can make them more difficult to set up.
     * Attached to the subnet. So you don't have to specify them when
       starting an instance.
     * Processed in the order specified. The first match is the rule that
       applies.
     * Supports ALLOW and DENY rules.

    Any traffic that doesn't match any rule is blocked.

    You can create a NetworkACL in any VPC::

        network_acl = vpc.add_network_acl(
            name='my-network-acl',
            inbound=[dict(
                protocol='tcp',
                port=22,
                network='0.0.0.0/0',
            )],
        )
    """

    resource_name = "network_acl"

    name = argument.String(field="GroupName")
    """ The name of the security group. This field is required."""

    inbound = argument.ResourceList(Rule)
    """ Rules for traffic entering the subnet. """

    outbound = argument.ResourceList(Rule)
    """ Rules for traffic leaving the subnet. """

    tags = argument.Dict()
    """ A dictionary of tags to associate with this VPC. A common use of tags
    is to group components by environment (e.g. "dev1", "staging", etc) or to
    map components to cost centres for billing purposes. """

    vpc = argument.Resource(VPC, field="VpcId")

class Describe(SimpleDescribe, Plan):

    resource = NetworkACL
    service_name = 'ec2'
    describe_action = "describe_network_acls"
    describe_list_key = "NetworkAcls"
    key = 'NetworkAclId'

    def get_describe_filters(self):
        vpc = self.runner.get_plan(self.resource.vpc)
        return {
            "Filters": [
                {'Name': 'tag:Name', 'Values': [self.resource.name]},
                {'Name': 'vpc-id', 'Values': [vpc.resource_id or '']}
            ],
        }

class Apply(SimpleApply, Describe):

    create_action = "create_network_acl"

    def _get_local_rules(self):
        local_rules = {}
        for i, rule in enumerate(self.resource.inbound):
            rule = serializers.Resource().render(self.runner, rule)
            rule['RuleNumber'] = i
            local_rules[(False, i)] = rule

        for i, rule in enumerate(self.resource.outbound):
            rule = serializers.Resource().render(self.runner, rule)
            rule['RuleNumber'] = i
            local_rules[(True, i)] = rule
        return local_rules

    def _get_remote_rules(self):
        remote_rules = {}
        for rule in self.object.get("Entries", []):
            remote_rules[(rule['Egress'], rule['RuleNumber'])] = rule
        return remote_rules

    def update_object(self):
        for action in super(Apply, self).update_object():
            yield action

        local_rules = self._get_local_rules()
        remote_rules = self._get_remote_rules()

        for key, rule in local_rules.items():
            if key not in self.remote_rules or self.remote_rules[key] != rule:
                #FIXME: Rule stale or out of date
                print("UPSERT", rule)

        for rule in remote_rules.keys():
            if rule not in local_rules:
                #FIXME: Delete remote rule not present @ local
                print("DELETE", remote_rules[rule])

class Destroy(SimpleDestroy, Describe):

    destroy_action = "delete_network_acl"
<commit_msg>NetworkAcl: Avoid unexpected and unhandled 'InternalError'<commit_after># Copyright 2014 Isotoma Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from touchdown.core.resource import Resource
from touchdown.core.plan import Plan
from touchdown.core import argument

from .vpc import VPC
from touchdown.core import serializers
from ..common import SimpleDescribe, SimpleApply, SimpleDestroy

class Rule(Resource):

    """
    Represents a rule in a :py:class:`NetworkACL`.

    You shouldn't create ``Rule`` resources directly, they are created
    implicitly when defining a :py:class:`NetworkACL`. For example::

        network_acl = vpc.add_network_acl(
            name='my-network-acl',
            inbound=[
                {"port": 80, "network": "0.0.0.0/0"},
                {"port": 443, "network": "0.0.0.0/0"},
            ],
        )

    This will implicitly create 2 ``Rule`` resources.
    """

    resource_name = "rule"
    dot_ignore = True

    network = argument.IPNetwork(
        field="IpRanges",
        serializer=serializers.ListOfOne(serializers.Dict(
            CidrIp=serializers.String(),
        )),
    )
    """ A network range specified in CIDR form. For example, you could specify
    ``0.0.0.0/0`` to allow the entire internet to access the specified
    port/protocol. """

    protocol = argument.String(default='tcp', choices=['tcp', 'udp', 'icmp'], field="IpProtocol")
    """ The network protocol to allow or deny. By default this is ``tcp``. It can also
    be set to ``udp`` or ``icmp``. """

    port = argument.Integer(min=-1, max=32768)
    """ A port to allow access to. """

    from_port = argument.Integer(default=lambda r: r.port, min=-1, max=32768, field="FromPort")
    """ Instead of specifying ``port``, you can specify a range of ports
    starting at ``from_port`` """

    to_port = argument.Integer(default=lambda r: r.port, min=-1, max=32768, field="ToPort")
    """ If specifying a ``from_port`` you also need to specify a
    ``to_port``. """

    action = argument.String(default="ALLOW", choices=["ALLOW", "DENY"])
    """ Whether to allow or deny matching traffic. The default value is ``ALLOW``. """

    def __str__(self):
        name = super(Rule, self).__str__()
        if self.from_port == self.to_port:
            ports = "port {}".format(self.from_port)
        else:
            ports = "ports {} to {}".format(self.from_port, self.to_port)
        return "{}: {} {} from {}".format(name, self.protocol, ports, self.network)

class NetworkACL(Resource):

    """
    Network ACL's provide network filtering at subnet level, controlling both
    inbound and outbound traffic. They are:

     * Stateless. This means that return traffic is not automatically
       allowed. This can make them more difficult to set up.
     * Attached to the subnet. So you don't have to specify them when
       starting an instance.
     * Processed in the order specified. The first match is the rule that
       applies.
     * Supports ALLOW and DENY rules.

    Any traffic that doesn't match any rule is blocked.

    You can create a NetworkACL in any VPC::

        network_acl = vpc.add_network_acl(
            name='my-network-acl',
            inbound=[dict(
                protocol='tcp',
                port=22,
                network='0.0.0.0/0',
            )],
        )
    """

    resource_name = "network_acl"

    name = argument.String(field="GroupName")
    """ The name of the security group. This field is required."""

    inbound = argument.ResourceList(Rule)
    """ Rules for traffic entering the subnet. """

    outbound = argument.ResourceList(Rule)
    """ Rules for traffic leaving the subnet. """

    tags = argument.Dict()
    """ A dictionary of tags to associate with this VPC. A common use of tags
    is to group components by environment (e.g. "dev1", "staging", etc) or to
    map components to cost centres for billing purposes. """

    vpc = argument.Resource(VPC, field="VpcId")

class Describe(SimpleDescribe, Plan):

    resource = NetworkACL
    service_name = 'ec2'
    describe_action = "describe_network_acls"
    describe_list_key = "NetworkAcls"
    key = 'NetworkAclId'

    def get_describe_filters(self):
        vpc = self.runner.get_plan(self.resource.vpc)
        if not vpc.resource_id:
            return None

        return {
            "Filters": [
                {'Name': 'tag:Name', 'Values': [self.resource.name]},
                {'Name': 'vpc-id', 'Values': [vpc.resource_id]}
            ],
        }

class Apply(SimpleApply, Describe):

    create_action = "create_network_acl"

    def _get_local_rules(self):
        local_rules = {}
        for i, rule in enumerate(self.resource.inbound):
            rule = serializers.Resource().render(self.runner, rule)
            rule['RuleNumber'] = i
            local_rules[(False, i)] = rule

        for i, rule in enumerate(self.resource.outbound):
            rule = serializers.Resource().render(self.runner, rule)
            rule['RuleNumber'] = i
            local_rules[(True, i)] = rule
        return local_rules

    def _get_remote_rules(self):
        remote_rules = {}
        for rule in self.object.get("Entries", []):
            remote_rules[(rule['Egress'], rule['RuleNumber'])] = rule
        return remote_rules

    def update_object(self):
        for action in super(Apply, self).update_object():
            yield action

        local_rules = self._get_local_rules()
        remote_rules = self._get_remote_rules()

        for key, rule in local_rules.items():
            if key not in self.remote_rules or self.remote_rules[key] != rule:
                #FIXME: Rule stale or out of date
                print("UPSERT", rule)

        for rule in remote_rules.keys():
            if rule not in local_rules:
                #FIXME: Delete remote rule not present @ local
                print("DELETE", remote_rules[rule])

class Destroy(SimpleDestroy, Describe):

    destroy_action = "delete_network_acl"
<|endoftext|>


<commit_before>from touristfriend.api_keys import G_API_KEY
from touristfriend.business import Business
import requests

SEARCH_URL = 'https://maps.googleapis.com/maps/api/place/nearbysearch/json?location={},{}&radius={}&types={}&key={}'
PLACE_URL = 'https://maps.googleapis.com/maps/api/place/details/json?placeid={}&key={}'

def search(lat, lng, distance, query):
    """
    Searches the Google Places API (Max Limit = 20)

    :param lat: Latitude of the request
    :param long: Longitude of the request
    :param distance: Distance to search (meters)
    :param query: The niche, i.e. restaurants, bars, etc
    :returns: List of retrieved places
    """

    url = SEARCH_URL.format(lat, lng, distance, query, G_API_KEY)
    place_list = []

    data = requests.get(url).json()

    for i in range(0, len(data['results'])):
        try:
            result = data['results'][i]
            place = search_place(result['place_id'])
            place_list.append(place)
            if len(place_list) == 5:
                break
        except Exception:
            pass

    return place_list

def search_place(place_id):
    """
    Searches Google for a specific Place

    :param id: Google Place ID
    :returns: Business object
    """
    url = PLACE_URL.format(place_id, G_API_KEY)
    data = requests.get(url).json()
    place = data['result']
    try:
        return Business(place['name'],
                        place['formatted_address'].split(',')[0],
                        place['rating'],
                        len(place['reviews']),
                        (place["geometry"]["location"]["lat"],
                            place["geometry"]["location"]["lng"]))
    except KeyError:
        pass
<commit_msg>Avoid hotels from google maps<commit_after>from touristfriend.api_keys import G_API_KEY
from touristfriend.business import Business
import requests

SEARCH_URL = 'https://maps.googleapis.com/maps/api/place/nearbysearch/json?location={},{}&radius={}&types={}&key={}'
PLACE_URL = 'https://maps.googleapis.com/maps/api/place/details/json?placeid={}&key={}'

def search(lat, lng, distance, query):
    """
    Searches the Google Places API (Max Limit = 20)

    :param lat: Latitude of the request
    :param long: Longitude of the request
    :param distance: Distance to search (meters)
    :param query: The niche, i.e. restaurants, bars, etc
    :returns: List of retrieved places
    """

    url = SEARCH_URL.format(lat, lng, distance, query, G_API_KEY)
    place_list = []

    data = requests.get(url).json()

    for i in range(0, len(data['results'])):
        try:
            result = data['results'][i]
            place = search_place(result['place_id'])
            place_list.append(place)
            if len(place_list) == 5:
                break
        except Exception:
            pass

    return place_list

def search_place(place_id):
    """
    Searches Google for a specific Place

    :param id: Google Place ID
    :returns: Business object
    """
    url = PLACE_URL.format(place_id, G_API_KEY)
    data = requests.get(url).json()
    place = data['result']
    try:
        if not "hotel" in (place['name'].lower()):
            return Business(place['name'],
                            place['formatted_address'].split(',')[0],
                            place['rating'],
                            len(place['reviews']),
                            (place["geometry"]["location"]["lat"],
                                place["geometry"]["location"]["lng"]))
    except KeyError:
        pass
<|endoftext|>