<commit_before>from datetime import date
import tornado.escape
import tornado.ioloop
import tornado.web
import rethinkdb as r
r.connect( "localhost", 28015).repl()
import json
import urlparse
import random
import string
import itertools
from datetime import datetime
import requests
import os

def get_field(l, field):
    results = []
    for d in l:
        if field in d:
            results.append(d[field])
    return results

def handle_query(payload, run=True):
    if run:
        r.db('public').table('queries').insert({'datetime': r.expr(datetime.now(r.make_timezone('-07:00'))), 'payload': payload}).run()
    dbobj = r.db('public').table(payload['table'])
    for key in payload.keys():
        if key in ['get', 'has_fields', 'doesnt_have_fields', 'match', 'has_string', 'match_any_field', 'has_string_in_any_field']:
            if type(payload[key]) is list and key in ['has_fields']:
                dbobj = getattr(dbobj, key)(*payload[key])
            elif type(payload[key]) is list and key in ['doesnt_have_fields']:
                # 'doesnt_have_fields',
                dbobj.filter(lambda row: row.pluck(*payload[key]).eq({}))
            elif key in ['doesnt_have_fields']:
                dbobj.filter(lambda row: row.pluck(payload[key]).eq({}))
            elif key == 'match':
                dbobj = getattr(dbobj, 'filter')(lambda case: case[payload['match']['field']].match(payload['match']['value']))
            elif key == 'has_string':
                dbobj = getattr(dbobj, 'filter')(lambda case: case[payload['has_string']['field']].match('.*?'+payload['has_string']['value']+'.*?'))
            elif key == 'match_any_field':
                dbobj = getattr(dbobj, 'filter')(lambda doc: doc.coerce_to('string').match(payload['match_any_field']))
            elif key == 'has_string_in_any_field':
                dbobj = getattr(dbobj, 'filter')(lambda doc: doc.coerce_to('string').match('(?i).*?'+payload['has_string_in_any_field']+'.*?'))
            else:
                dbobj = getattr(dbobj, key)(payload[key])

    if 'pluck' in payload:
        if type(payload['pluck']) is list:
            dbobj = getattr(dbobj, 'pluck')(*payload['pluck'])
        else:
            dbobj = getattr(dbobj, 'pluck')(payload['pluck'])

    if 'order_by' in payload:
        for o in payload['order_by']:
            if o['direction'] == 'desc':
                dbobj = getattr(dbobj, 'order_by')(index=r.desc(o['field']))
            else:
                dbobj = getattr(dbobj, 'order_by')(index=o['field'])
    rows_count = dbobj.count().run()
    if 'page' in payload:
        page = int(payload['page'])
    else:
        payload['page'] = int(1)
        page = int(1)
    if 'rows_per_page' in payload:
        rows_per_page = int(payload['rows_per_page'])
    else:
        rows_per_page = int(10)
    dbobj = dbobj.slice((page - 1) * rows_per_page, page * rows_per_page)
    print list(dbobj.run())

    results_for_fields = list(dbobj.run())
    fields = [row.keys() for row in results_for_fields]
    fields = list(itertools.chain.from_iterable(fields))
    fields = sorted(list(set(fields)))

    #joined_data = list(r.db("public").table("police_internal_affairs_allegations").eq_join("organization_id", r.db("public").table("organizations")).map({"right":{
    #        "organization_id": r.row["right"]["id"],
    #        "organization_name": r.row["right"]["name"]
    #    }, "left": r.row["left"]}).zip().run())
    ids_for_other_tables = [field for field in fields if field.endswith('_id')]
    modified_joined_data = []
    special_names = {'person': 'people'}
    for field in ids_for_other_tables:

        #print field
        #print field[:-3]+'s'
        # get the fields of the table
        #results_for_fields = r.db('public').table(field[:-3]+'s').run()
        #right_fields = [row.keys() for row in results_for_fields]
        #right_fields = list(itertools.chain.from_iterable(right_fields))
        #right_fields = sorted(list(set(right_fields)))

        t = special_names[field[:-3]] if field[:-3] in special_names else field[:-3]+'s'
        if '__' in t:
            t = t.split('__')[-1]
        if not t in r.db('public').table_list().run():
            continue
        print field, t
        dbobj = dbobj.eq_join(field, r.db("public").table(t))
        #d = {"left": r.row["left"], "right": {}}
        #for right_field in right_fields:
        #    d["right"][field[:-2]+right_field] = r.row["right"][right_field]
        #dbobj = dbobj.map(d)
        dbobj = dbobj.merge(  lambda row: {'left': row['right'].coerce_to('array').map(
                      lambda pair: [r.expr(field[:-2]) + pair[0], pair[1]]
                    ).coerce_to('object')}).without({'right': True}).zip()
        #dbobj = dbobj

    if 'filter' in payload:
        key = 'filter'
        dbobj = getattr(dbobj, key)(payload[key])
    print dbobj
    #for data in joined_data:
    #    d = {}
    #    d.update(data['left'])
    #    new_right_side = data['right'].items()
    #    new_right_side = dict([('organization_'+item[0], item[1]) for item in new_right_side])
    #    d.update(new_right_side)
    #    modified_joined_data.append(d)

    if 'action' in payload:
        if payload['action'] == 'get_fields':
            results = list(dbobj.run())
            fields = [row.keys() for row in results]
            fields = list(itertools.chain.from_iterable(fields))
            results = sorted(list(set(fields)))
        elif payload['action'] == 'count':
            results = {'count': dbobj.count().run()}
        elif payload['action'] == 'percentage_simple_matching':
            if 'match' in payload:
                base = r.db('public').table(payload['table']).filter(lambda case: case[payload['match']['field']].match(payload['match']['value']))

            else:
                base = r.db('public').table(payload['table']).filter(lambda case: case[payload['has_string']['field']].match('.*?'+payload['has_string']['value']+'.*?'))
            denominator = base.count().run()

            numerator = base.filter({payload['numerator']['field']: payload['numerator']['value']}).count().run()

            if denominator:
                percentage = float(numerator)/denominator
                percentage = "{:.0%}".format(percentage)+' (%s/%s)' % (numerator, denominator)
            else:
                percentage = 'Error: No denominator'
            results = {'numerator': numerator, 'denominator': denominator, 'percentage': percentage}
        elif payload['action'] == 'get_list':
            results = [item[payload['field']] for item in list(r.db('public').table(payload['table']).pluck(payload['field']).run())]
        elif payload['action'] == 'get_set':
            results = list(set([item[payload['field']] for item in list(r.db('public').table(payload['table']).pluck(payload['field']).run())]))
        elif payload['action'] == 'do_basic_mapping':
            dbobj = getattr(dbobj, 'pluck')(payload['field_for_key'], payload['field_for_value'])
            items = dbobj.run()
            d = {}
            for item in items:
                if type(item[payload['field_for_key']]) is list:
                    for k in item[payload['field_for_key']]:
                        if not k in d:
                            d[k] = [item[payload['field_for_value']]]
                        else:
                            d[k].append(item[payload['field_for_value']])
                else:
                    if not item[payload['field_for_key']] in d:
                        d[item[payload['field_for_key']]] = [item[payload['field_for_value']]]
                    else:
                        d[item[payload['field_for_key']]].append(item[payload['field_for_value']])
            results = d
        elif payload['action'] == 'do_row_mapping':
            print 'doing ', payload['action']
            items = list(dbobj.run())
            table_fields = [row.keys() for row in items]
            table_fields = list(set(list(itertools.chain.from_iterable(table_fields))))
            d = {}
            for item in items:
                if type(item[payload['field_for_key']]) is list:
                    for k in item[payload['field_for_key']]:
                        if not k in d:
                            d[k] = [item]
                        else:
                            d[k].append(item)
                else:
                    if not item[payload['field_for_key']] in d:
                        d[item[payload['field_for_key']]] = [item]
                    else:
                        d[item[payload['field_for_key']]].append(item)
            results = {'data': d, 'table_fields': table_fields, 'keys': d.keys()}
            return results
    else:
        if not run:
            return dbobj
        # if 'pluck' in payload:
        #if type(payload['pluck']) is list:
        #    dbobj = getattr(dbobj, 'pluck')(*payload['pluck'])
        results = {}
        results['table'] = r.db('public').table('tables').get(payload['table']).run()
        if 'default_order_by' in results['table']:
            if
            dbobj = getattr(dbobj, 'order_by')(r.desc(results['table']['default_order_by']['field']))
        special_names_reversed = {value: key for key, value in special_names.items()}
        t = payload['table']
        t = special_names_reversed[t] if t in special_names_reversed else t[:-1]
        t = t + '_id'
        results['data'] = list(dbobj.run(time_format="raw"))
        if 'linked_tables' in results['table']:

            for linked_table in results['table']['linked_tables']:
                linked_table_data = list(r.db('public').table(linked_table).run(time_format="raw"))
                for i, row in enumerate(results['data']):
                    print i
                    row[linked_table] = [item for item in linked_table_data if item.get(t) == row['id']]
                    # consider using group e.g. r.db('public').table('police_internal_affairs_allegations').group('person_id').run()
                    #row[linked_table] = list(r.db('public').table(linked_table).filter({t: row['id']}).run(time_format="raw"))

        results['fields'] = [row.keys() for row in results['data']]
        results['fields'] = list(set(list(itertools.chain.from_iterable(results['fields']))))
        results['number_of_rows'] = rows_count
        results['percentages'] = []
        results['group_counts'] = []
        for field in results['fields']:
            if not field.endswith('_id'):
                continue
            t = special_names[field[:-3]] if field[:-3] in special_names else field[:-3]+'s'
            if not t in r.db('public').table_list().run():
                continue
            g = r.db('public').table(t).group('id').run()

            results['group_counts'].append([t, [[g[item[0]], item[1]] for item in list(sorted(dbobj.group(field).count().run().items(), key=lambda x:x[1], reverse=True))][:10]])
        results['group_counts'].sort(key=lambda x:x[0])

        likely_boolean_fields = [field for field in results['fields'] if field.startswith('is_')]
        # remove if the field in a filter
        if 'filter' in payload:
            for field in payload['filter'].keys():
                if field in likely_boolean_fields:
                    likely_boolean_fields.remove(field)
        for field in likely_boolean_fields:
            items_in_field = get_field(results['data'], field)
            numerator = items_in_field.count(True)
            denominator = len(items_in_field)
            percentage = float(numerator)/denominator
            percentage = "{:.0%}".format(percentage)+' (%s/%s)' % (numerator, denominator)
            row_name = payload['table'].replace('_', ' ')
            if 'has_string_in_any_field' in payload:
                sentence = '%s of %s mentioning "%s" are %s.' % (percentage, row_name, payload['has_string_in_any_field'], field[3:])
            else:
                sentence = '%s of %s are %s.' % (percentage, row_name, field[3:])
            results['percentages'].append({'field': field, 'value': True, 'percentage': percentage, 'sentence': sentence})
        results['payload'] = payload
        results['name_for_rows'] = payload['table'].split('_')[-1]
        results['field_selectors'] = []
        for field in results['fields']:
            if field == 'id':
                continue
            elif field.startswith('is_'):
                results['field_selectors'].append({'selector': 'checkbox', 'name': field, 'display_name': field[3:].capitalize()})
            else:
                items = sorted(list(set([row[field] if isinstance(row[field], basestring) else '' for row in results['data'] if field in row and len(unicode(row.get(field)).encode('utf-8')) < 50])))
                if len(items) < 100 and not items == ['']:
                    results['field_selectors'].append({'selector': 'dropdown', 'name': field, 'display_name': field.replace('_', ' ').capitalize(), 'items': items})
        if run:
            return results
        return dbobj
        <commit_msg>auto commit<commit_after>from datetime import date
import tornado.escape
import tornado.ioloop
import tornado.web
import rethinkdb as r
r.connect( "localhost", 28015).repl()
import json
import urlparse
import random
import string
import itertools
from datetime import datetime
import requests
import os

def get_field(l, field):
    results = []
    for d in l:
        if field in d:
            results.append(d[field])
    return results

def handle_query(payload, run=True):
    if run:
        r.db('public').table('queries').insert({'datetime': r.expr(datetime.now(r.make_timezone('-07:00'))), 'payload': payload}).run()
    dbobj = r.db('public').table(payload['table'])
    for key in payload.keys():
        if key in ['get', 'has_fields', 'doesnt_have_fields', 'match', 'has_string', 'match_any_field', 'has_string_in_any_field']:
            if type(payload[key]) is list and key in ['has_fields']:
                dbobj = getattr(dbobj, key)(*payload[key])
            elif type(payload[key]) is list and key in ['doesnt_have_fields']:
                # 'doesnt_have_fields',
                dbobj.filter(lambda row: row.pluck(*payload[key]).eq({}))
            elif key in ['doesnt_have_fields']:
                dbobj.filter(lambda row: row.pluck(payload[key]).eq({}))
            elif key == 'match':
                dbobj = getattr(dbobj, 'filter')(lambda case: case[payload['match']['field']].match(payload['match']['value']))
            elif key == 'has_string':
                dbobj = getattr(dbobj, 'filter')(lambda case: case[payload['has_string']['field']].match('.*?'+payload['has_string']['value']+'.*?'))
            elif key == 'match_any_field':
                dbobj = getattr(dbobj, 'filter')(lambda doc: doc.coerce_to('string').match(payload['match_any_field']))
            elif key == 'has_string_in_any_field':
                dbobj = getattr(dbobj, 'filter')(lambda doc: doc.coerce_to('string').match('(?i).*?'+payload['has_string_in_any_field']+'.*?'))
            else:
                dbobj = getattr(dbobj, key)(payload[key])

    if 'pluck' in payload:
        if type(payload['pluck']) is list:
            dbobj = getattr(dbobj, 'pluck')(*payload['pluck'])
        else:
            dbobj = getattr(dbobj, 'pluck')(payload['pluck'])

    if 'order_by' in payload:
        for o in payload['order_by']:
            if o['direction'] == 'desc':
                dbobj = getattr(dbobj, 'order_by')(index=r.desc(o['field']))
            else:
                dbobj = getattr(dbobj, 'order_by')(index=o['field'])
    rows_count = dbobj.count().run()
    if 'page' in payload:
        page = int(payload['page'])
    else:
        payload['page'] = int(1)
        page = int(1)
    if 'rows_per_page' in payload:
        rows_per_page = int(payload['rows_per_page'])
    else:
        rows_per_page = int(10)
    dbobj = dbobj.slice((page - 1) * rows_per_page, page * rows_per_page)
    print list(dbobj.run())

    results_for_fields = list(dbobj.run())
    fields = [row.keys() for row in results_for_fields]
    fields = list(itertools.chain.from_iterable(fields))
    fields = sorted(list(set(fields)))

    #joined_data = list(r.db("public").table("police_internal_affairs_allegations").eq_join("organization_id", r.db("public").table("organizations")).map({"right":{
    #        "organization_id": r.row["right"]["id"],
    #        "organization_name": r.row["right"]["name"]
    #    }, "left": r.row["left"]}).zip().run())
    ids_for_other_tables = [field for field in fields if field.endswith('_id')]
    modified_joined_data = []
    special_names = {'person': 'people'}
    for field in ids_for_other_tables:

        #print field
        #print field[:-3]+'s'
        # get the fields of the table
        #results_for_fields = r.db('public').table(field[:-3]+'s').run()
        #right_fields = [row.keys() for row in results_for_fields]
        #right_fields = list(itertools.chain.from_iterable(right_fields))
        #right_fields = sorted(list(set(right_fields)))

        t = special_names[field[:-3]] if field[:-3] in special_names else field[:-3]+'s'
        if '__' in t:
            t = t.split('__')[-1]
        if not t in r.db('public').table_list().run():
            continue
        print field, t
        dbobj = dbobj.eq_join(field, r.db("public").table(t))
        #d = {"left": r.row["left"], "right": {}}
        #for right_field in right_fields:
        #    d["right"][field[:-2]+right_field] = r.row["right"][right_field]
        #dbobj = dbobj.map(d)
        dbobj = dbobj.merge(  lambda row: {'left': row['right'].coerce_to('array').map(
                      lambda pair: [r.expr(field[:-2]) + pair[0], pair[1]]
                    ).coerce_to('object')}).without({'right': True}).zip()
        #dbobj = dbobj

    if 'filter' in payload:
        key = 'filter'
        dbobj = getattr(dbobj, key)(payload[key])
    print dbobj
    #for data in joined_data:
    #    d = {}
    #    d.update(data['left'])
    #    new_right_side = data['right'].items()
    #    new_right_side = dict([('organization_'+item[0], item[1]) for item in new_right_side])
    #    d.update(new_right_side)
    #    modified_joined_data.append(d)

    if 'action' in payload:
        if payload['action'] == 'get_fields':
            results = list(dbobj.run())
            fields = [row.keys() for row in results]
            fields = list(itertools.chain.from_iterable(fields))
            results = sorted(list(set(fields)))
        elif payload['action'] == 'count':
            results = {'count': dbobj.count().run()}
        elif payload['action'] == 'percentage_simple_matching':
            if 'match' in payload:
                base = r.db('public').table(payload['table']).filter(lambda case: case[payload['match']['field']].match(payload['match']['value']))

            else:
                base = r.db('public').table(payload['table']).filter(lambda case: case[payload['has_string']['field']].match('.*?'+payload['has_string']['value']+'.*?'))
            denominator = base.count().run()

            numerator = base.filter({payload['numerator']['field']: payload['numerator']['value']}).count().run()

            if denominator:
                percentage = float(numerator)/denominator
                percentage = "{:.0%}".format(percentage)+' (%s/%s)' % (numerator, denominator)
            else:
                percentage = 'Error: No denominator'
            results = {'numerator': numerator, 'denominator': denominator, 'percentage': percentage}
        elif payload['action'] == 'get_list':
            results = [item[payload['field']] for item in list(r.db('public').table(payload['table']).pluck(payload['field']).run())]
        elif payload['action'] == 'get_set':
            results = list(set([item[payload['field']] for item in list(r.db('public').table(payload['table']).pluck(payload['field']).run())]))
        elif payload['action'] == 'do_basic_mapping':
            dbobj = getattr(dbobj, 'pluck')(payload['field_for_key'], payload['field_for_value'])
            items = dbobj.run()
            d = {}
            for item in items:
                if type(item[payload['field_for_key']]) is list:
                    for k in item[payload['field_for_key']]:
                        if not k in d:
                            d[k] = [item[payload['field_for_value']]]
                        else:
                            d[k].append(item[payload['field_for_value']])
                else:
                    if not item[payload['field_for_key']] in d:
                        d[item[payload['field_for_key']]] = [item[payload['field_for_value']]]
                    else:
                        d[item[payload['field_for_key']]].append(item[payload['field_for_value']])
            results = d
        elif payload['action'] == 'do_row_mapping':
            print 'doing ', payload['action']
            items = list(dbobj.run())
            table_fields = [row.keys() for row in items]
            table_fields = list(set(list(itertools.chain.from_iterable(table_fields))))
            d = {}
            for item in items:
                if type(item[payload['field_for_key']]) is list:
                    for k in item[payload['field_for_key']]:
                        if not k in d:
                            d[k] = [item]
                        else:
                            d[k].append(item)
                else:
                    if not item[payload['field_for_key']] in d:
                        d[item[payload['field_for_key']]] = [item]
                    else:
                        d[item[payload['field_for_key']]].append(item)
            results = {'data': d, 'table_fields': table_fields, 'keys': d.keys()}
            return results
    else:
        if not run:
            return dbobj
        # if 'pluck' in payload:
        #if type(payload['pluck']) is list:
        #    dbobj = getattr(dbobj, 'pluck')(*payload['pluck'])
        results = {}
        results['table'] = r.db('public').table('tables').get(payload['table']).run()
        if 'default_order_by' in results['table']:
            field = results['table']['default_order_by']['field']
            if field in r.db
            dbobj = getattr(dbobj, 'order_by')(r.desc(results['table']['default_order_by']['field']))
        special_names_reversed = {value: key for key, value in special_names.items()}
        t = payload['table']
        t = special_names_reversed[t] if t in special_names_reversed else t[:-1]
        t = t + '_id'
        results['data'] = list(dbobj.run(time_format="raw"))
        if 'linked_tables' in results['table']:

            for linked_table in results['table']['linked_tables']:
                linked_table_data = list(r.db('public').table(linked_table).run(time_format="raw"))
                for i, row in enumerate(results['data']):
                    print i
                    row[linked_table] = [item for item in linked_table_data if item.get(t) == row['id']]
                    # consider using group e.g. r.db('public').table('police_internal_affairs_allegations').group('person_id').run()
                    #row[linked_table] = list(r.db('public').table(linked_table).filter({t: row['id']}).run(time_format="raw"))

        results['fields'] = [row.keys() for row in results['data']]
        results['fields'] = list(set(list(itertools.chain.from_iterable(results['fields']))))
        results['number_of_rows'] = rows_count
        results['percentages'] = []
        results['group_counts'] = []
        for field in results['fields']:
            if not field.endswith('_id'):
                continue
            t = special_names[field[:-3]] if field[:-3] in special_names else field[:-3]+'s'
            if not t in r.db('public').table_list().run():
                continue
            g = r.db('public').table(t).group('id').run()

            results['group_counts'].append([t, [[g[item[0]], item[1]] for item in list(sorted(dbobj.group(field).count().run().items(), key=lambda x:x[1], reverse=True))][:10]])
        results['group_counts'].sort(key=lambda x:x[0])

        likely_boolean_fields = [field for field in results['fields'] if field.startswith('is_')]
        # remove if the field in a filter
        if 'filter' in payload:
            for field in payload['filter'].keys():
                if field in likely_boolean_fields:
                    likely_boolean_fields.remove(field)
        for field in likely_boolean_fields:
            items_in_field = get_field(results['data'], field)
            numerator = items_in_field.count(True)
            denominator = len(items_in_field)
            percentage = float(numerator)/denominator
            percentage = "{:.0%}".format(percentage)+' (%s/%s)' % (numerator, denominator)
            row_name = payload['table'].replace('_', ' ')
            if 'has_string_in_any_field' in payload:
                sentence = '%s of %s mentioning "%s" are %s.' % (percentage, row_name, payload['has_string_in_any_field'], field[3:])
            else:
                sentence = '%s of %s are %s.' % (percentage, row_name, field[3:])
            results['percentages'].append({'field': field, 'value': True, 'percentage': percentage, 'sentence': sentence})
        results['payload'] = payload
        results['name_for_rows'] = payload['table'].split('_')[-1]
        results['field_selectors'] = []
        for field in results['fields']:
            if field == 'id':
                continue
            elif field.startswith('is_'):
                results['field_selectors'].append({'selector': 'checkbox', 'name': field, 'display_name': field[3:].capitalize()})
            else:
                items = sorted(list(set([row[field] if isinstance(row[field], basestring) else '' for row in results['data'] if field in row and len(unicode(row.get(field)).encode('utf-8')) < 50])))
                if len(items) < 100 and not items == ['']:
                    results['field_selectors'].append({'selector': 'dropdown', 'name': field, 'display_name': field.replace('_', ' ').capitalize(), 'items': items})
        if run:
            return results
        return dbobj
        <|endoftext|>


<commit_before>import sublime, sublime_plugin, os, plistlib

global_id = 0

xml_location = "Packages/julooLanguageInjectorCache/"

def get_new_id():
	global global_id
	global_id = global_id + 1
	return global_id

def get_xml_path(id):
	return xml_location + str(id) +".tmLanguage"

def get_full_path(p):
	return os.path.join(sublime.packages_path(), os.path.normpath(p[9:]))

class LanguageInjectorListener(sublime_plugin.EventListener):

	def on_close(self, view):
		view.run_command("language_injector_update", {"action": "destroy"})

	def on_load_async(self, view):
		view.run_command("language_injector_update", {"action": "update"})

class LanguageInjectorUpdateCommand(sublime_plugin.TextCommand):

	xml_id = 0
	xml_pos = ""
	last_syntax = ""

	def override_syntax(self):
		if self.view.settings().has("old_syntax"):
			syntax_file = self.view.settings().get("old_syntax")
		else:
			syntax_file = self.view.settings().get("syntax")
			self.view.settings().set("old_syntax", syntax_file)
		plist = plistlib.readPlistFromBytes(sublime.load_resource(syntax_file).encode("UTF-8"))
		plist["name"] = plist["name"] + " Injected"
		plist["fileTypes"] = []
		settings = sublime.load_settings("LanguageInjector.sublime-settings")
		patterns = settings.get("patterns", {})
		for reg in patterns.keys():
			p = {"match": reg}
			valid = True
			if type(patterns[reg]) is str:
				p["name"] = patterns[reg]
			else:
				captures = {}
				for param in patterns[reg].keys():
					if param == "parent":
						valid = False
						for parent in patterns[reg]["parent"]:
							if self.view.match_selector(0, parent):
								valid = True
					elif param == "scope":
						p["name"] = patterns[reg]["scope"]
					elif param.isdigit():
						captures[param] = {"name": patterns[reg][param]}
				if len(captures.keys()) > 0:
					p["captures"] = captures
			if valid:
				plist["patterns"].append(p)
		if not os.path.exists(get_full_path(xml_location)):
			os.mkdir(get_full_path(xml_location))
		self.xml_pos = get_xml_path(self.xml_id)
		self.last_syntax = self.xml_pos
		with open(get_full_path(self.xml_pos), "wb") as f:
			plistlib.writePlist(plist, f)
		self.view.set_syntax_file(self.xml_pos)
		self.view.settings().add_on_change("language_injector_syntax_listener", self.syntax_change)

	def syntax_change(self):
		syntax = self.view.settings().get("syntax")
		if syntax != self.last_syntax and syntax != self.view.settings().get("old_syntax"):
			self.last_syntax = syntax
			self.view.settings().set("old_syntax", syntax)
			self.view.run_command("language_injector_update", {"action": "update"})

	def run(self, edit, **args):
		self.view.settings().clear_on_change("language_injector_syntax_listener")
		if self.xml_pos != "" and os.path.exists(get_full_path(self.xml_pos)):
			os.remove(get_full_path(self.xml_pos))
		if args["action"] == "update":
			self.xml_id = get_new_id()
			self.override_syntax()
		elif args["action"] == "destroy":
			if self.view.settings().has("old_syntax"):
				self.view.set_syntax_file(self.view.settings().get("old_syntax"))
				self.view.settings().erase("old_syntax")
<commit_msg>Fuck the sublime text cache<commit_after>import sublime, sublime_plugin, os, plistlib, random

global_id = 0

xml_location = "Packages/julooLanguageInjectorCache/"

def get_new_id():
	global global_id
	global_id = global_id + 1
	return str(global_id) + "".join(random.choice('abcdefghijklmnopqrstuvwxyz0123456789') for i in range(7))

def get_xml_path(id):
	return xml_location + id +".tmLanguage"

def get_full_path(p):
	return os.path.join(sublime.packages_path(), os.path.normpath(p[9:]))

class LanguageInjectorListener(sublime_plugin.EventListener):

	def on_close(self, view):
		view.run_command("language_injector_update", {"action": "destroy"})

	def on_load_async(self, view):
		view.run_command("language_injector_update", {"action": "update"})

class LanguageInjectorUpdateCommand(sublime_plugin.TextCommand):

	xml_id = "0"
	xml_pos = ""
	last_syntax = ""

	def override_syntax(self):
		if self.view.settings().has("old_syntax"):
			syntax_file = self.view.settings().get("old_syntax")
		else:
			syntax_file = self.view.settings().get("syntax")
			self.view.settings().set("old_syntax", syntax_file)
		plist = plistlib.readPlistFromBytes(sublime.load_resource(syntax_file).encode("UTF-8"))
		plist["name"] = plist["name"] + " Injected"
		plist["fileTypes"] = []
		settings = sublime.load_settings("LanguageInjector.sublime-settings")
		patterns = settings.get("patterns", {})
		for reg in patterns.keys():
			p = {"match": reg}
			valid = True
			if type(patterns[reg]) is str:
				p["name"] = patterns[reg]
			else:
				captures = {}
				for param in patterns[reg].keys():
					if param == "parent":
						valid = False
						for parent in patterns[reg]["parent"]:
							if self.view.match_selector(0, parent):
								valid = True
					elif param == "scope":
						p["name"] = patterns[reg]["scope"]
					elif param.isdigit():
						captures[param] = {"name": patterns[reg][param]}
				if len(captures.keys()) > 0:
					p["captures"] = captures
			if valid:
				plist["patterns"].append(p)
		if not os.path.exists(get_full_path(xml_location)):
			os.mkdir(get_full_path(xml_location))
		self.xml_pos = get_xml_path(self.xml_id)
		self.last_syntax = self.xml_pos
		with open(get_full_path(self.xml_pos), "wb") as f:
			plistlib.writePlist(plist, f)
		self.view.set_syntax_file(self.xml_pos)
		self.view.settings().add_on_change("language_injector_syntax_listener", self.syntax_change)

	def syntax_change(self):
		syntax = self.view.settings().get("syntax")
		if syntax != self.last_syntax and syntax != self.view.settings().get("old_syntax"):
			self.last_syntax = syntax
			self.view.settings().set("old_syntax", syntax)
			self.view.run_command("language_injector_update", {"action": "update"})

	def run(self, edit, **args):
		self.view.settings().clear_on_change("language_injector_syntax_listener")
		if self.xml_pos != "" and os.path.exists(get_full_path(self.xml_pos)):
			os.remove(get_full_path(self.xml_pos))
		if args["action"] == "update":
			self.xml_id = get_new_id()
			self.override_syntax()
		elif args["action"] == "destroy":
			if self.view.settings().has("old_syntax"):
				self.view.set_syntax_file(self.view.settings().get("old_syntax"))
				self.view.settings().erase("old_syntax")
<|endoftext|>


<commit_before>import sys, os, _tkinter

ver = str(_tkinter.TCL_VERSION)
for t in "tcl", "tk", "tix":
    key = t.upper() + "_LIBRARY"
    try:
        v = os.environ[key]
    except KeyError:
        v = os.path.join(sys.prefix, "tcl", t+ver)
        if os.path.exists(os.path.join(v, "tclIndex")):
            os.environ[key] = v
<commit_msg>Set TCL_LIBRARY before import _tkinter. Suggested by Kirill Simonov. Fixes #418173 and #219960. 2.2.1 candidate.<commit_after>import sys, os

# Delay import _tkinter until we have set TCL_LIBRARY,
# so that Tcl_FindExecutable has a chance to locate its
# encoding directory.

# Unfortunately, we cannot know the TCL_LIBRARY directory
# if we don't know the tcl version, which we cannot find out
# without import Tcl. Fortunately, Tcl will itself look in
# <TCL_LIBRARY>\..\tcl<TCL_VERSION>, so anything close to
# the real Tcl library will do.

prefix = os.path.join(sys.prefix,"tcl")
# if this does not exist, no further search is needed
if os.path.exists(prefix):
    if not os.environ.has_key("TCL_LIBRARY"):
        for name in os.listdir(prefix):
            if name.startswith("tcl"):
                tcldir = os.path.join(prefix,name)
                if os.path.isdir(tcldir):
                    os.environ["TCL_LIBRARY"] = tcldir
    # Now set the other variables accordingly
    import _tkinter
    ver = str(_tkinter.TCL_VERSION)
    for t in "tk", "tix":
        key = t.upper() + "_LIBRARY"
        try:
            v = os.environ[key]
        except KeyError:
            v = os.path.join(sys.prefix, "tcl", t+ver)
            if os.path.exists(os.path.join(v, "tclIndex")):
                os.environ[key] = v
<|endoftext|>


<commit_before>from UM.Qt.QtApplication import QtApplication
from UM.Scene.SceneNode import SceneNode
from UM.Scene.Camera import Camera
from UM.Scene.Platform import Platform
from UM.Math.Vector import Vector
from UM.Math.Matrix import Matrix
from UM.Resources import Resources

from UM.Scene.BoxRenderer import BoxRenderer
from UM.Scene.Selection import Selection

from PlatformPhysics import PlatformPhysics
from BuildVolume import BuildVolume

from PyQt5.QtCore import pyqtSlot, QUrl

import os.path

class PrinterApplication(QtApplication):
    def __init__(self):
        super().__init__(name = 'Cura')
        self.setRequiredPlugins(['CuraEngineBackend', 'MeshView', 'LayerView', 'STLReader','SelectionTool','CameraTool'])
        self._physics = None

    def _loadPlugins(self):
        self._plugin_registry.loadPlugins({ "type": "Logger"})
        self._plugin_registry.loadPlugins({ "type": "StorageDevice" })
        self._plugin_registry.loadPlugins({ "type": "View" })
        self._plugin_registry.loadPlugins({ "type": "MeshHandler" })
        self._plugin_registry.loadPlugins({ "type": "Tool" })

        self._plugin_registry.loadPlugin('CuraEngineBackend')

    def run(self):
        controller = self.getController()

        controller.setActiveView("MeshView")
        controller.setCameraTool("CameraTool")
        controller.setSelectionTool("SelectionTool")

        t = controller.getTool('TranslateTool')
        if t:
            t.setYRange(0.0, 0.0)

        Selection.selectionChanged.connect(self.onSelectionChanged)

        self._physics = PlatformPhysics(controller)

        root = controller.getScene().getRoot()
        platform = Platform(root)

        volume = BuildVolume(root)
        volume.translate(Vector(0, 73, 0))
        volume.scale(150.0)

        self.getRenderer().setLightPosition(Vector(0, 150, 0))

        camera = Camera('3d', root)
        camera.translate(Vector(0, 150, 150))
        proj = Matrix()
        proj.setPerspective(85, 640/480, 1, 500)
        camera.setProjectionMatrix(proj)
        camera.setPerspective(True)
        camera.lookAt(Vector(0, 0, 0), Vector(0, 1, 0))

        camera = Camera('left', root)
        camera.translate(Vector(-150, 50, 0))
        proj = Matrix()
        proj.setOrtho(-200, 200, -200, 200, 1, 500)
        camera.setProjectionMatrix(proj)
        camera.lookAt(Vector(0, 50, 0), Vector(0, 1, 0))
        camera.setLocked(True)

        camera = Camera('top', root)
        camera.translate(Vector(0, 150, 0))
        proj = Matrix()
        proj.setOrtho(-200, 200, -200, 200, 1, 500)
        camera.setProjectionMatrix(proj)
        camera.lookAt(Vector(0, 0, 0), Vector(0, 0, -1))
        camera.setLocked(True)

        camera = Camera('front', root)
        camera.translate(Vector(0, 50, 150))
        proj = Matrix()
        proj.setOrtho(-200, 200, -200, 200, 1, 500)
        camera.setProjectionMatrix(proj)
        camera.lookAt(Vector(0, 50, 0), Vector(0, 1, 0))
        camera.setLocked(True)

        controller.getScene().setActiveCamera('3d')

        self.setMainQml(os.path.dirname(__file__) + "/Printer.qml")
        self.initializeEngine()

        if self._engine.rootObjects:
            self.exec_()

        self.saveMachines()

    def registerObjects(self, engine):
        engine.rootContext().setContextProperty('Printer', self)

    def onSelectionChanged(self):
        if Selection.getCount() > 0:
            if not self.getController().getActiveTool():
                self.getController().setActiveTool('TranslateTool')

            self.getController().getTool('CameraTool').setOrigin(Selection.getSelectedObject(0).getGlobalPosition())
        else:
            if self.getController().getActiveTool() and self.getController().getActiveTool().getName() == 'TranslateTool':
                self.getController().setActiveTool(None)

    @pyqtSlot(QUrl)
    def saveGCode(self, file):
        try:
            gcode = self.getController().getScene().gcode
        except AttributeError:
            return

        with open(file.toLocalFile(), 'w') as f:
            f.write(gcode)
<commit_msg>Update the build volume based on machine<commit_after>from UM.Qt.QtApplication import QtApplication
from UM.Scene.SceneNode import SceneNode
from UM.Scene.Camera import Camera
from UM.Scene.Platform import Platform
from UM.Math.Vector import Vector
from UM.Math.Matrix import Matrix
from UM.Resources import Resources

from UM.Scene.BoxRenderer import BoxRenderer
from UM.Scene.Selection import Selection

from PlatformPhysics import PlatformPhysics
from BuildVolume import BuildVolume

from PyQt5.QtCore import pyqtSlot, QUrl
from PyQt5.QtGui import QColor

import os.path

class PrinterApplication(QtApplication):
    def __init__(self):
        super().__init__(name = 'Cura')
        self.setRequiredPlugins(['CuraEngineBackend', 'MeshView', 'LayerView', 'STLReader','SelectionTool','CameraTool'])
        self._physics = None
        self._volume = None
        self.activeMachineChanged.connect(self._onActiveMachineChanged)

    def _loadPlugins(self):
        self._plugin_registry.loadPlugins({ "type": "Logger"})
        self._plugin_registry.loadPlugins({ "type": "StorageDevice" })
        self._plugin_registry.loadPlugins({ "type": "View" })
        self._plugin_registry.loadPlugins({ "type": "MeshHandler" })
        self._plugin_registry.loadPlugins({ "type": "Tool" })

        self._plugin_registry.loadPlugin('CuraEngineBackend')

    def run(self):
        controller = self.getController()

        controller.setActiveView("MeshView")
        controller.setCameraTool("CameraTool")
        controller.setSelectionTool("SelectionTool")

        t = controller.getTool('TranslateTool')
        if t:
            t.setYRange(0.0, 0.0)

        Selection.selectionChanged.connect(self.onSelectionChanged)

        self._physics = PlatformPhysics(controller)

        root = controller.getScene().getRoot()
        platform = Platform(root)

        self._volume = BuildVolume(root)

        self.getRenderer().setLightPosition(Vector(0, 150, 0))
        self.getRenderer().setBackgroundColor(QColor(200, 200, 200))

        camera = Camera('3d', root)
        camera.translate(Vector(0, 150, 150))
        proj = Matrix()
        proj.setPerspective(85, 640/480, 1, 500)
        camera.setProjectionMatrix(proj)
        camera.setPerspective(True)
        camera.lookAt(Vector(0, 0, 0), Vector(0, 1, 0))

        camera = Camera('left', root)
        camera.translate(Vector(-150, 50, 0))
        proj = Matrix()
        proj.setOrtho(-200, 200, -200, 200, 1, 500)
        camera.setProjectionMatrix(proj)
        camera.lookAt(Vector(0, 50, 0), Vector(0, 1, 0))
        camera.setLocked(True)

        camera = Camera('top', root)
        camera.translate(Vector(0, 150, 0))
        proj = Matrix()
        proj.setOrtho(-200, 200, -200, 200, 1, 500)
        camera.setProjectionMatrix(proj)
        camera.lookAt(Vector(0, 0, 0), Vector(0, 0, -1))
        camera.setLocked(True)

        camera = Camera('front', root)
        camera.translate(Vector(0, 50, 150))
        proj = Matrix()
        proj.setOrtho(-200, 200, -200, 200, 1, 500)
        camera.setProjectionMatrix(proj)
        camera.lookAt(Vector(0, 50, 0), Vector(0, 1, 0))
        camera.setLocked(True)

        controller.getScene().setActiveCamera('3d')

        self.setActiveMachine(self.getMachines()[0])

        self.setMainQml(os.path.dirname(__file__) + "/Printer.qml")
        self.initializeEngine()

        if self._engine.rootObjects:
            self.exec_()

        self.saveMachines()

    def registerObjects(self, engine):
        engine.rootContext().setContextProperty('Printer', self)

    def onSelectionChanged(self):
        if Selection.getCount() > 0:
            if not self.getController().getActiveTool():
                self.getController().setActiveTool('TranslateTool')

            self.getController().getTool('CameraTool').setOrigin(Selection.getSelectedObject(0).getGlobalPosition())
        else:
            if self.getController().getActiveTool() and self.getController().getActiveTool().getName() == 'TranslateTool':
                self.getController().setActiveTool(None)

    @pyqtSlot(QUrl)
    def saveGCode(self, file):
        try:
            gcode = self.getController().getScene().gcode
        except AttributeError:
            return

        with open(file.toLocalFile(), 'w') as f:
            f.write(gcode)

    def _onActiveMachineChanged(self):
        machine = self.getActiveMachine()
        if machine:
            self._volume.setWidth(machine.getSettingValueByKey('machine_width'))
            self._volume.setHeight(machine.getSettingValueByKey('machine_height'))
            self._volume.setDepth(machine.getSettingValueByKey('machine_depth'))
            self._volume.rebuild()
<|endoftext|>


<commit_before># Functions which need the PIL

import types
import numpy as Numeric

from numpy import exp, amin, amax, ravel, asarray, cast, arange, \
     ones, NewAxis, transpose, mgrid, iscomplexobj, sum, zeros

import Image
import ImageFilter

__all__ = ['fromimage','toimage','imsave','imread','bytescale',
           'imrotate','imresize','imshow','imfilter','radon']

_UInt8 = Numeric.UnsignedInt8

# Returns a byte-scaled image
def bytescale(data, cmin=None, cmax=None, high=255, low=0):
    if data.dtype.char == _UInt8:
        return data
    high = high - low
    if cmin is None:
        cmin = amin(ravel(data))
    if cmax is None:
        cmax = amax(ravel(data))
    scale = high *1.0 / (cmax-cmin or 1)
    bytedata = ((data*1.0-cmin)*scale + 0.4999).astype(_UInt8)
    return bytedata + cast[_UInt8](low)

def imread(name,flatten=0):
    """Read an image file from a filename.

    Optional arguments:

     - flatten (0): if true, the image is flattened by calling convert('F') on
     the resulting image object.  This flattens the color layers into a single
     grayscale layer.
    """

    im = Image.open(name)
    return fromimage(im,flatten=flatten)

def imsave(name, arr):
    """Save an array to an image file.
    """
    im = toimage(arr)
    im.save(name)
    return

def fromimage(im, flatten=0):
    """Takes a PIL image and returns a copy of the image in a Numeric container.
    If the image is RGB returns a 3-dimensional array:  arr[:,:,n] is each channel

    Optional arguments:

    - flatten (0): if true, the image is flattened by calling convert('F') on
    the image object before extracting the numerical data.  This flattens the
    color layers into a single grayscale layer.  Note that the supplied image
    object is NOT modified.
    """
    assert Image.isImageType(im), "Not a PIL image."
    if flatten:
        im = im.convert('F')
    mode = im.mode
    adjust = 0
    if mode == '1':
        im = im.convert(mode='L')
        mode = 'L'
        adjust = 1
    str = im.tostring()
    type = 'B'
    if mode == 'F':
        type = 'f'
    if mode == 'I':
        type = 'I'
    arr = Numeric.fromstring(str,type)
    shape = list(im.size)
    shape.reverse()
    if mode == 'P':
        arr.shape = shape
        if im.palette.rawmode != 'RGB':
            print "Warning: Image has invalid palette."
            return arr
        pal = Numeric.fromstring(im.palette.data,type)
        N = len(pal)
        pal.shape = (int(N/3.0),3)
        return arr, pal
    if mode in ['RGB','YCbCr']:
        shape += [3]
    elif mode in ['CMYK','RGBA']:
        shape += [4]
    arr.shape = shape
    if adjust:
        arr = (arr != 0)
    return arr

_errstr = "Mode is unknown or incompatible with input array shape."
def toimage(arr,high=255,low=0,cmin=None,cmax=None,pal=None,
            mode=None,channel_axis=None):
    """Takes a Numeric array and returns a PIL image.  The mode of the
    PIL image depends on the array shape, the pal keyword, and the mode
    keyword.

    For 2-D arrays, if pal is a valid (N,3) byte-array giving the RGB values
    (from 0 to 255) then mode='P', otherwise mode='L', unless mode is given
    as 'F' or 'I' in which case a float and/or integer array is made

    For 3-D arrays, the channel_axis argument tells which dimension of the
      array holds the channel data.
    For 3-D arrays if one of the dimensions is 3, the mode is 'RGB'
      by default or 'YCbCr' if selected.
    if the

    The Numeric array must be either 2 dimensional or 3 dimensional.
    """
    data = asarray(arr)
    if iscomplexobj(data):
        raise ValueError, "Cannot convert a complex-valued array."
    shape = list(data.shape)
    valid = len(shape)==2 or ((len(shape)==3) and \
                              ((3 in shape) or (4 in shape)))
    assert valid, "Not a suitable array shape for any mode."
    if len(shape) == 2:
        shape = (shape[1],shape[0]) # columns show up first
        if mode == 'F':
            image = Image.fromstring(mode,shape,data.astype('f').tostring())
            return image
        if mode in [None, 'L', 'P']:
            bytedata = bytescale(data,high=high,low=low,cmin=cmin,cmax=cmax)
            image = Image.fromstring('L',shape,bytedata.tostring())
            if pal is not None:
                image.putpalette(asarray(pal,dtype=_UInt8).tostring())
                # Becomes a mode='P' automagically.
            elif mode == 'P':  # default gray-scale
                pal = arange(0,256,1,dtype='B')[:,NewAxis] * \
                      ones((3,),dtype='B')[NewAxis,:]
                image.putpalette(asarray(pal,dtype=_UInt8).tostring())
            return image
        if mode == '1':  # high input gives threshold for 1
            bytedata = (data > high)
            image = Image.fromstring('1',shape,bytedata.tostring())
            return image
        if cmin is None:
            cmin = amin(ravel(data))
        if cmax is None:
            cmax = amax(ravel(data))
        data = (data*1.0 - cmin)*(high-low)/(cmax-cmin) + low
        if mode == 'I':
            image = Image.fromstring(mode,shape,data.astype('i').tostring())
        else:
            raise ValueError, _errstr
        return image

    # if here then 3-d array with a 3 or a 4 in the shape length.
    # Check for 3 in datacube shape --- 'RGB' or 'YCbCr'
    if channel_axis is None:
        if (3 in shape):
            ca = Numeric.nonzero(asarray(shape) == 3)[0]
        else:
            ca = Numeric.nonzero(asarray(shape) == 4)
            if len(ca):
                ca = ca[0]
            else:
                raise ValueError, "Could not find channel dimension."
    else:
        ca = channel_axis

    numch = shape[ca]
    if numch not in [3,4]:
        raise ValueError, "Channel axis dimension is not valid."

    bytedata = bytescale(data,high=high,low=low,cmin=cmin,cmax=cmax)
    if ca == 2:
        strdata = bytedata.tostring()
        shape = (shape[1],shape[0])
    elif ca == 1:
        strdata = transpose(bytedata,(0,2,1)).tostring()
        shape = (shape[2],shape[0])
    elif ca == 0:
        strdata = transpose(bytedata,(1,2,0)).tostring()
        shape = (shape[2],shape[1])
    if mode is None:
        if numch == 3: mode = 'RGB'
        else: mode = 'RGBA'

    if mode not in ['RGB','RGBA','YCbCr','CMYK']:
        raise ValueError, _errstr

    if mode in ['RGB', 'YCbCr']:
        assert numch == 3, "Invalid array shape for mode."
    if mode in ['RGBA', 'CMYK']:
        assert numch == 4, "Invalid array shape for mode."

    # Here we know data and mode is coorect
    image = Image.fromstring(mode, shape, strdata)
    return image

def imrotate(arr,angle,interp='bilinear'):
    """Rotate an image counter-clockwise by angle degrees.

    Interpolation methods can be:
        'nearest' :  for nearest neighbor
        'bilinear' : for bilinear
        'cubic' or 'bicubic' : for bicubic
    """
    arr = asarray(arr)
    func = {'nearest':0,'bilinear':2,'bicubic':3,'cubic':3}
    im = toimage(arr)
    im = im.rotate(angle,resample=func[interp])
    return fromimage(im)

def imresize(arr,newsize,interp='bilinear',mode=None):
    newsize=list(newsize)
    newsize.reverse()
    newsize = tuple(newsize)
    arr = asarray(arr)
    func = {'nearest':0,'bilinear':2,'bicubic':3,'cubic':3}
    im = toimage(arr,mode=mode)
    im = im.resize(newsize,resample=func[interp])
    return fromimage(im)

def imshow(arr):
    """Simple showing of an image through an external viewer.
    """
    im = toimage(arr)
    if (len(arr.shape) == 3) and (arr.shape[2] == 4):
        try:
            import os
            im.save('/tmp/scipy_imshow.png')
            if os.system("(xv /tmp/scipy_imshow.png; rm -f /tmp/scipy_imshow.png)&"):
                raise RuntimeError
            return
        except:
            print "Warning: Alpha channel may not be handled correctly."

    im.show()
    return

def imresize(arr,size):
    """Resize an image.

    If size is an integer it is a percentage of current size.
    If size is a float it is a fraction of current size.
    If size is a tuple it is the size of the output image.
    """
    im = toimage(arr)
    ts = type(size)
    if ts is types.IntType:
        size = size / 100.0
    if type(size) is types.FloatType:
        size = (im.size[0]*size,im.size[1]*size)
    else:
        size = (size[1],size[0])
    imnew = im.resize(size)
    return fromimage(imnew)

def imfilter(arr,ftype):
    """Simple filtering of an image.

    type can be:
            'blur', 'contour', 'detail', 'edge_enhance', 'edge_enhance_more',
            'emboss', 'find_edges', 'smooth', 'smooth_more', 'sharpen'
    """
    _tdict = {'blur':ImageFilter.BLUR,
              'contour':ImageFilter.CONTOUR,
              'detail':ImageFilter.DETAIL,
              'edge_enhance':ImageFilter.EDGE_ENHANCE,
              'edge_enhance_more':ImageFilter.EDGE_ENHANCE_MORE,
              'emboss':ImageFilter.EMBOSS,
              'find_edges':ImageFilter.FIND_EDGES,
              'smooth':ImageFilter.SMOOTH,
              'smooth_more':ImageFilter.SMOOTH_MORE,
              'sharpen':ImageFilter.SHARPEN
              }

    im = toimage(arr)
    if ftype not in _tdict.keys():
        raise ValueError, "Unknown filter type."
    return fromimage(im.filter(_tdict[ftype]))

def radon(arr,theta=None):
    if theta is None:
        theta = mgrid[0:180]
    s = zeros((arr.shape[1],len(theta)),'d')
    k = 0
    for th in theta:
        im = imrotate(arr,-th)
        s[:,k] = sum(im,axis=0)
        k += 1
    return s

<commit_msg>remove unused imports and alias of Numeric -> numpy<commit_after># Functions which need the PIL

import types
import numpy

from numpy import amin, amax, ravel, asarray, cast, arange, \
     ones, NewAxis, transpose, mgrid, iscomplexobj, sum, zeros

import Image
import ImageFilter

__all__ = ['fromimage','toimage','imsave','imread','bytescale',
           'imrotate','imresize','imshow','imfilter','radon']

_UInt8 = numpy.UnsignedInt8

# Returns a byte-scaled image
def bytescale(data, cmin=None, cmax=None, high=255, low=0):
    if data.dtype.char == _UInt8:
        return data
    high = high - low
    if cmin is None:
        cmin = amin(ravel(data))
    if cmax is None:
        cmax = amax(ravel(data))
    scale = high *1.0 / (cmax-cmin or 1)
    bytedata = ((data*1.0-cmin)*scale + 0.4999).astype(_UInt8)
    return bytedata + cast[_UInt8](low)

def imread(name,flatten=0):
    """Read an image file from a filename.

    Optional arguments:

     - flatten (0): if true, the image is flattened by calling convert('F') on
     the resulting image object.  This flattens the color layers into a single
     grayscale layer.
    """

    im = Image.open(name)
    return fromimage(im,flatten=flatten)

def imsave(name, arr):
    """Save an array to an image file.
    """
    im = toimage(arr)
    im.save(name)
    return

def fromimage(im, flatten=0):
    """Takes a PIL image and returns a copy of the image in a numpy container.
    If the image is RGB returns a 3-dimensional array:  arr[:,:,n] is each channel

    Optional arguments:

    - flatten (0): if true, the image is flattened by calling convert('F') on
    the image object before extracting the numerical data.  This flattens the
    color layers into a single grayscale layer.  Note that the supplied image
    object is NOT modified.
    """
    assert Image.isImageType(im), "Not a PIL image."
    if flatten:
        im = im.convert('F')
    mode = im.mode
    adjust = 0
    if mode == '1':
        im = im.convert(mode='L')
        mode = 'L'
        adjust = 1
    str = im.tostring()
    type = 'B'
    if mode == 'F':
        type = 'f'
    if mode == 'I':
        type = 'I'
    arr = numpy.fromstring(str,type)
    shape = list(im.size)
    shape.reverse()
    if mode == 'P':
        arr.shape = shape
        if im.palette.rawmode != 'RGB':
            print "Warning: Image has invalid palette."
            return arr
        pal = numpy.fromstring(im.palette.data,type)
        N = len(pal)
        pal.shape = (int(N/3.0),3)
        return arr, pal
    if mode in ['RGB','YCbCr']:
        shape += [3]
    elif mode in ['CMYK','RGBA']:
        shape += [4]
    arr.shape = shape
    if adjust:
        arr = (arr != 0)
    return arr

_errstr = "Mode is unknown or incompatible with input array shape."
def toimage(arr,high=255,low=0,cmin=None,cmax=None,pal=None,
            mode=None,channel_axis=None):
    """Takes a numpy array and returns a PIL image.  The mode of the
    PIL image depends on the array shape, the pal keyword, and the mode
    keyword.

    For 2-D arrays, if pal is a valid (N,3) byte-array giving the RGB values
    (from 0 to 255) then mode='P', otherwise mode='L', unless mode is given
    as 'F' or 'I' in which case a float and/or integer array is made

    For 3-D arrays, the channel_axis argument tells which dimension of the
      array holds the channel data.
    For 3-D arrays if one of the dimensions is 3, the mode is 'RGB'
      by default or 'YCbCr' if selected.
    if the

    The numpy array must be either 2 dimensional or 3 dimensional.
    """
    data = asarray(arr)
    if iscomplexobj(data):
        raise ValueError, "Cannot convert a complex-valued array."
    shape = list(data.shape)
    valid = len(shape)==2 or ((len(shape)==3) and \
                              ((3 in shape) or (4 in shape)))
    assert valid, "Not a suitable array shape for any mode."
    if len(shape) == 2:
        shape = (shape[1],shape[0]) # columns show up first
        if mode == 'F':
            image = Image.fromstring(mode,shape,data.astype('f').tostring())
            return image
        if mode in [None, 'L', 'P']:
            bytedata = bytescale(data,high=high,low=low,cmin=cmin,cmax=cmax)
            image = Image.fromstring('L',shape,bytedata.tostring())
            if pal is not None:
                image.putpalette(asarray(pal,dtype=_UInt8).tostring())
                # Becomes a mode='P' automagically.
            elif mode == 'P':  # default gray-scale
                pal = arange(0,256,1,dtype='B')[:,NewAxis] * \
                      ones((3,),dtype='B')[NewAxis,:]
                image.putpalette(asarray(pal,dtype=_UInt8).tostring())
            return image
        if mode == '1':  # high input gives threshold for 1
            bytedata = (data > high)
            image = Image.fromstring('1',shape,bytedata.tostring())
            return image
        if cmin is None:
            cmin = amin(ravel(data))
        if cmax is None:
            cmax = amax(ravel(data))
        data = (data*1.0 - cmin)*(high-low)/(cmax-cmin) + low
        if mode == 'I':
            image = Image.fromstring(mode,shape,data.astype('i').tostring())
        else:
            raise ValueError, _errstr
        return image

    # if here then 3-d array with a 3 or a 4 in the shape length.
    # Check for 3 in datacube shape --- 'RGB' or 'YCbCr'
    if channel_axis is None:
        if (3 in shape):
            ca = numpy.nonzero(asarray(shape) == 3)[0]
        else:
            ca = numpy.nonzero(asarray(shape) == 4)
            if len(ca):
                ca = ca[0]
            else:
                raise ValueError, "Could not find channel dimension."
    else:
        ca = channel_axis

    numch = shape[ca]
    if numch not in [3,4]:
        raise ValueError, "Channel axis dimension is not valid."

    bytedata = bytescale(data,high=high,low=low,cmin=cmin,cmax=cmax)
    if ca == 2:
        strdata = bytedata.tostring()
        shape = (shape[1],shape[0])
    elif ca == 1:
        strdata = transpose(bytedata,(0,2,1)).tostring()
        shape = (shape[2],shape[0])
    elif ca == 0:
        strdata = transpose(bytedata,(1,2,0)).tostring()
        shape = (shape[2],shape[1])
    if mode is None:
        if numch == 3: mode = 'RGB'
        else: mode = 'RGBA'

    if mode not in ['RGB','RGBA','YCbCr','CMYK']:
        raise ValueError, _errstr

    if mode in ['RGB', 'YCbCr']:
        assert numch == 3, "Invalid array shape for mode."
    if mode in ['RGBA', 'CMYK']:
        assert numch == 4, "Invalid array shape for mode."

    # Here we know data and mode is coorect
    image = Image.fromstring(mode, shape, strdata)
    return image

def imrotate(arr,angle,interp='bilinear'):
    """Rotate an image counter-clockwise by angle degrees.

    Interpolation methods can be:
        'nearest' :  for nearest neighbor
        'bilinear' : for bilinear
        'cubic' or 'bicubic' : for bicubic
    """
    arr = asarray(arr)
    func = {'nearest':0,'bilinear':2,'bicubic':3,'cubic':3}
    im = toimage(arr)
    im = im.rotate(angle,resample=func[interp])
    return fromimage(im)

def imresize(arr,newsize,interp='bilinear',mode=None):
    newsize=list(newsize)
    newsize.reverse()
    newsize = tuple(newsize)
    arr = asarray(arr)
    func = {'nearest':0,'bilinear':2,'bicubic':3,'cubic':3}
    im = toimage(arr,mode=mode)
    im = im.resize(newsize,resample=func[interp])
    return fromimage(im)

def imshow(arr):
    """Simple showing of an image through an external viewer.
    """
    im = toimage(arr)
    if (len(arr.shape) == 3) and (arr.shape[2] == 4):
        try:
            import os
            im.save('/tmp/scipy_imshow.png')
            if os.system("(xv /tmp/scipy_imshow.png; rm -f /tmp/scipy_imshow.png)&"):
                raise RuntimeError
            return
        except:
            print "Warning: Alpha channel may not be handled correctly."

    im.show()
    return

def imresize(arr,size):
    """Resize an image.

    If size is an integer it is a percentage of current size.
    If size is a float it is a fraction of current size.
    If size is a tuple it is the size of the output image.
    """
    im = toimage(arr)
    ts = type(size)
    if ts is types.IntType:
        size = size / 100.0
    if type(size) is types.FloatType:
        size = (im.size[0]*size,im.size[1]*size)
    else:
        size = (size[1],size[0])
    imnew = im.resize(size)
    return fromimage(imnew)

def imfilter(arr,ftype):
    """Simple filtering of an image.

    type can be:
            'blur', 'contour', 'detail', 'edge_enhance', 'edge_enhance_more',
            'emboss', 'find_edges', 'smooth', 'smooth_more', 'sharpen'
    """
    _tdict = {'blur':ImageFilter.BLUR,
              'contour':ImageFilter.CONTOUR,
              'detail':ImageFilter.DETAIL,
              'edge_enhance':ImageFilter.EDGE_ENHANCE,
              'edge_enhance_more':ImageFilter.EDGE_ENHANCE_MORE,
              'emboss':ImageFilter.EMBOSS,
              'find_edges':ImageFilter.FIND_EDGES,
              'smooth':ImageFilter.SMOOTH,
              'smooth_more':ImageFilter.SMOOTH_MORE,
              'sharpen':ImageFilter.SHARPEN
              }

    im = toimage(arr)
    if ftype not in _tdict.keys():
        raise ValueError, "Unknown filter type."
    return fromimage(im.filter(_tdict[ftype]))

def radon(arr,theta=None):
    if theta is None:
        theta = mgrid[0:180]
    s = zeros((arr.shape[1],len(theta)),'d')
    k = 0
    for th in theta:
        im = imrotate(arr,-th)
        s[:,k] = sum(im,axis=0)
        k += 1
    return s

<|endoftext|>


<commit_before>from Cura.Wx.WxApplication import WxApplication
from Cura.Wx.MainWindow import MainWindow

class PrinterApplication(WxApplication):
    def __init__(self):
        super(PrinterApplication, self).__init__()

    def run(self):
        self._plugin_registry.loadPlugins({ "type": "StorageDevice" })

        window = MainWindow("Cura Printer")
        window.Show()
        super(PrinterApplication, self).run()
<commit_msg>Load view plugins in PrinterApplication and set a default view<commit_after>from Cura.Wx.WxApplication import WxApplication
from Cura.Wx.MainWindow import MainWindow

class PrinterApplication(WxApplication):
    def __init__(self):
        super(PrinterApplication, self).__init__()

    def run(self):
        self._plugin_registry.loadPlugins({ "type": "StorageDevice" })
        self._plugin_registry.loadPlugins({ "type": "View" })

        self.getController().setActiveView("MeshView")

        window = MainWindow("Cura Printer", self)
        window.Show()
        super(PrinterApplication, self).run()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import freedom_editor

drawing_area = None
level_display_trans_x = 0.0
level_display_trans_y = 0.0

cursor_x_pos = cursor_y_pos = None

"""
Mouse dragging state.  This is entirely a function of the first button the user
held down when he began dragging the mouse cursor.

This can be either nothing (STATE_NORM), panning the view (STATE_PAN)
or moving an object (STATE_DRAG_OBJ)
"""
STATE_NORM = 0
STATE_PAN = 1
STATE_DRAG_OBJ = 2
mouse_state = STATE_NORM

def set_builder(builder):
    global drawing_area, level_display_trans_X, level_display_trans_y

    drawing_area = builder.get_object("level_display")
    level_display_trans_x = 0.0
    level_display_trans_y = 0.0

def on_click(widget, event):
    """
    called when the user clicks on the level_display drawing area
    """
    global cursor_x_pos, cursor_y_pos, mouse_state

    if mouse_state == STATE_NORM:
        # XXX Gtk probably has constants for mouse buttons, I just don't know
        #     what they are called.
        if event.button == 1 or event.button == 3:
            # left click
            cursor_x_pos = event.x
            cursor_y_pos = event.y

        if event.button == 1:
            mouse_state = STATE_PAN
        elif event.button == 3:
            mouse_state = STATE_DRAG_OBJ

            # need to find the object under the cursor.  This involves
            # converting the window-coordinates to world-coordinates
            world_cursor_x = cursor_x_pos - level_display_trans_x
            world_cursor_y = cursor_y_pos - level_display_trans_y

            world_cursor = (world_cursor_x, world_cursor_y)

            obj_idx = freedom_editor.get_object_at_pos(world_cursor)

            freedom_editor.select_object(obj_idx)

def on_unclick(widget, event):
    """
    called when the user releases a previously held-down moust button.
    """
    global mouse_state

    if mouse_state == STATE_PAN and event.button == 1:
        mouse_state = STATE_NORM
    elif mouse_state == STATE_DRAG_OBJ and event.button == 3:
        mouse_state = STATE_NORM

def on_mouse_motion(widget, event):
    """
    called when the user drags the level_display drawing area
    """
    global cursor_x_pos, cursor_y_pos, \
        level_display_trans_x, level_display_trans_y

    if cursor_x_pos is None:
        rel_x = rel_y = 0
    else:
        rel_x = event.x - cursor_x_pos
        rel_y = event.y - cursor_y_pos

    if mouse_state == STATE_PAN:
        cursor_x_pos = event.x
        cursor_y_pos = event.y

        level_display_trans_x += rel_x
        level_display_trans_y += rel_y

        invalidate()
    elif mouse_state == STATE_DRAG_OBJ:
        cursor_x_pos = event.x
        cursor_y_pos = event.y

        selected_obj_idx = freedom_editor.get_selected_object()
        if selected_obj_idx is not None:
            obj = freedom_editor.get_object_by_index(selected_obj_idx)
            if obj is not None:
                freedom_editor.set_obj_pos(obj, (obj.pos_x + rel_x,
                                                 obj.pos_y + rel_y))

def invalidate():
    drawing_area.queue_draw()

def on_draw(widget, cr):
    cr.translate(level_display_trans_x, level_display_trans_y)
    if freedom_editor.cur_frame is not None:
        freedom_editor.cur_frame.draw(cr)
        cr.paint()
<commit_msg>Add alignment grid<commit_after>#!/usr/bin/env python

import math

import freedom_editor

drawing_area = None
level_display_trans_x = 0.0
level_display_trans_y = 0.0

cursor_x_pos = cursor_y_pos = None

"""
Mouse dragging state.  This is entirely a function of the first button the user
held down when he began dragging the mouse cursor.

This can be either nothing (STATE_NORM), panning the view (STATE_PAN)
or moving an object (STATE_DRAG_OBJ)
"""
STATE_NORM = 0
STATE_PAN = 1
STATE_DRAG_OBJ = 2
mouse_state = STATE_NORM

def set_builder(builder):
    global drawing_area, level_display_trans_X, level_display_trans_y

    drawing_area = builder.get_object("level_display")
    level_display_trans_x = 0.0
    level_display_trans_y = 0.0

def on_click(widget, event):
    """
    called when the user clicks on the level_display drawing area
    """
    global cursor_x_pos, cursor_y_pos, mouse_state

    if mouse_state == STATE_NORM:
        # XXX Gtk probably has constants for mouse buttons, I just don't know
        #     what they are called.
        if event.button == 1 or event.button == 3:
            # left click
            cursor_x_pos = event.x
            cursor_y_pos = event.y

        if event.button == 1:
            mouse_state = STATE_PAN
        elif event.button == 3:
            mouse_state = STATE_DRAG_OBJ

            # need to find the object under the cursor.  This involves
            # converting the window-coordinates to world-coordinates
            world_cursor_x = cursor_x_pos - level_display_trans_x
            world_cursor_y = cursor_y_pos - level_display_trans_y

            world_cursor = (world_cursor_x, world_cursor_y)

            obj_idx = freedom_editor.get_object_at_pos(world_cursor)

            freedom_editor.select_object(obj_idx)

def on_unclick(widget, event):
    """
    called when the user releases a previously held-down moust button.
    """
    global mouse_state

    if mouse_state == STATE_PAN and event.button == 1:
        mouse_state = STATE_NORM
    elif mouse_state == STATE_DRAG_OBJ and event.button == 3:
        mouse_state = STATE_NORM

def on_mouse_motion(widget, event):
    """
    called when the user drags the level_display drawing area
    """
    global cursor_x_pos, cursor_y_pos, \
        level_display_trans_x, level_display_trans_y

    if cursor_x_pos is None:
        rel_x = rel_y = 0
    else:
        rel_x = event.x - cursor_x_pos
        rel_y = event.y - cursor_y_pos

    if mouse_state == STATE_PAN:
        cursor_x_pos = event.x
        cursor_y_pos = event.y

        level_display_trans_x += rel_x
        level_display_trans_y += rel_y

        invalidate()
    elif mouse_state == STATE_DRAG_OBJ:
        cursor_x_pos = event.x
        cursor_y_pos = event.y

        selected_obj_idx = freedom_editor.get_selected_object()
        if selected_obj_idx is not None:
            obj = freedom_editor.get_object_by_index(selected_obj_idx)
            if obj is not None:
                freedom_editor.set_obj_pos(obj, (obj.pos_x + rel_x,
                                                 obj.pos_y + rel_y))

def invalidate():
    drawing_area.queue_draw()

def on_draw(widget, cr):
    cr.translate(level_display_trans_x, level_display_trans_y)
    draw_grid(cr, widget.get_allocation().width, widget.get_allocation().height)
    if freedom_editor.cur_frame is not None:
        freedom_editor.cur_frame.draw(cr)
        cr.paint()

GRID_WIDTH = 32
GRID_HEIGHT = 32

def draw_grid(cr, width, height):
    """
    draw an infinite grid to help players align their creations.
    cr - cairo context
    width - width of the level_display drawing area
    height - height of the level_display drawing area
    """
    cr.set_source_rgb(0, 0, 255)
    cr.set_line_width(0.5)

    x_start = (int(-level_display_trans_x) / GRID_WIDTH) * GRID_WIDTH
    x_end = x_start + (int(width) / GRID_WIDTH) * GRID_WIDTH + GRID_WIDTH

    y_start = (int(-level_display_trans_y) / GRID_HEIGHT) * GRID_HEIGHT
    y_end = y_start + (int(width) / GRID_HEIGHT) * GRID_HEIGHT + GRID_HEIGHT

    # draw vertical lines
    for col in range(int(x_start), x_end, GRID_WIDTH):
        cr.move_to(col, y_start)
        cr.line_to(col, y_end)

    # draw horizontal lines
    for row in range(int(y_start), y_end, GRID_HEIGHT):
        cr.move_to(x_start, row)
        cr.line_to(x_end, row)

    cr.stroke()
<|endoftext|>


<commit_before>import re
import json
import enum
from typing import Optional, List, Callable, Iterable
import configargparse
import requests
from bs4 import BeautifulSoup
from yume_tagger import iqdb
from yume_tagger import util
from yume_tagger.api import Api, ApiError, Post
from yume_tagger.autotag_settings import AutoTagSettings
from yume_tagger.commands.base import BaseCommand

CACHE_PATH = util.DB_DIR.joinpath('cache')
TAGS_TO_REMOVE = ['tagme']

class Force(enum.IntEnum):
    NoForce = enum.auto()
    TagAgain = enum.auto()
    DownloadAgain = enum.auto()

class AutoTagError(RuntimeError):
    pass

class TooBigError(AutoTagError):
    pass

class _ThirdPartyTag:
    def __init__(self, name: str, category: str) -> None:
        self.name = name
        self.category = category
        # TODO: info about source

class _SyncInfo:
    def __init__(self) -> None:
        self.source_tags: List[str] = []
        self.target_tags: List[str] = []
        self.tags_to_create: List[_ThirdPartyTag] = []

def _get_tags_from_gelbooru(url: str) -> Iterable[_ThirdPartyTag]:
    if 'gelbooru' not in url:
        raise ValueError('Not a valid Gelbooru URL')
    session = requests.Session()
    response = session.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')
    list_items = soup.select('#tag-sidebar li')
    for list_item in list_items:
        try:
            category = list_item['class'][0].replace('tag-type-', '')
            name = list_item.select('a')[1].text.replace(' ', '_')
            yield _ThirdPartyTag(name, category)
        except IndexError:
            pass

def _get_tags_from_danbooru(url: str) -> Iterable[_ThirdPartyTag]:
    match = re.search(r'danbooru.*?(\d+)', url)
    if not match:
        raise ValueError('Not a valid Danbooru URL')
    post_id = int(match.group(1))
    session = requests.Session()
    response = session.get(
        'http://danbooru.donmai.us/posts/{}.json'.format(post_id))
    post = json.loads(response.text)
    for category in ['artist', 'character', 'copyright', 'general']:
        for name in post['tag_string_' + category].split(' '):
            if name:
                yield _ThirdPartyTag(name, category)

def _get_third_party_tags(source_urls: List[str]) -> List[_ThirdPartyTag]:
    handlers: List[Callable[[str], Iterable[_ThirdPartyTag]]] = [
        _get_tags_from_danbooru,
        _get_tags_from_gelbooru,
    ]
    ret: List[_ThirdPartyTag] = []
    for handler in handlers:
        for source_url in source_urls:
            try:
                ret.extend(list(handler(source_url)))
            except (ValueError, NotImplementedError):
                pass
    return ret

def _get_third_party_tag_sources(post_url: str) -> List[str]:
    try:
        return [
            result.url
            for result in iqdb.search(post_url)
            if result.similarity >= 0.90]
    except iqdb.NothingFoundIqdbError:
        return []
    except iqdb.UploadTooBigIqdbError:
        raise TooBigError('Post is too big to search for')

def _get_post(
        api: Api,
        autotag_settings: AutoTagSettings,
        post_id: int,
        skip_history: bool) -> Optional[Post]:
    if post_id in autotag_settings.get_processed_post_ids() \
            and not skip_history:
        print('Skipping post {} - already processed'.format(post_id))
        return None

    print('Retrieving post {}...'.format(post_id))
    post = api.get_post(post_id)
    print(post['contentUrl'])
    print()
    return post

def _collect_third_party_tags(
        post: Post,
        source: Optional[str],
        skip_cache: bool) -> List[_ThirdPartyTag]:
    print('Collecting third party tags...')

    cache_path = CACHE_PATH.joinpath('autotagger-{}.dat'.format(post['id']))
    if cache_path.exists() and not skip_cache:
        result = json.loads(cache_path.read_text())
        sources = result['sources']
        third_party_tags = [
            _ThirdPartyTag(name=tag['name'], category=tag['category'])
            for tag in result['tags']
        ]
    else:
        if source:
            sources = [source]
        else:
            try:
                sources = _get_third_party_tag_sources(post['contentUrl'])
            except TooBigError:
                sources = _get_third_party_tag_sources(post['thumbnailUrl'])
        third_party_tags = _get_third_party_tags(sources)

        cache_path.parent.mkdir(parents=True, exist_ok=True)
        cache_path.write_text(json.dumps({
            'sources': sources,
            'tags': [
                {'name': tag.name, 'category': tag.category}
                for tag in third_party_tags],
        }))

    if sources:
        print('Sources:')
        for source in sorted(sources):
            print('- {}'.format(source))
        if not third_party_tags:
            raise AutoTagError('Tag sources found, but not on any known site')
    else:
        raise AutoTagError('No tag sources')

    print('Tags:')
    for tag in third_party_tags:
        print('- {}'.format(tag.name))
    print()
    return third_party_tags

def _sanitize_third_party_tags(
        api: Api,
        autotag_settings: AutoTagSettings,
        third_party_tags: List[_ThirdPartyTag]) -> Iterable[_ThirdPartyTag]:
    for tag in third_party_tags:
        if autotag_settings.is_tag_banned(tag.name):
            print('Ignored tag {}'.format(tag.name))
            continue

        translated_tag = autotag_settings.translate_tag(tag.name)
        if tag.name != translated_tag:
            print('Translated tag {} → {}'.format(tag.name, translated_tag))

        sanitized_name = util.sanitize_tag(translated_tag)
        if sanitized_name != tag.name:
            print('Sanitized tag {} → {}'.format(tag.name, sanitized_name))

        try:
            upstream_tag = api.get_tag(sanitized_name)
        except ApiError:
            upstream_tag = None
        if upstream_tag:
            sanitized_name = upstream_tag['names'][0]

        yield _ThirdPartyTag(
            name=sanitized_name,
            category=autotag_settings.translate_tag_category(tag.category))

def _get_sync_info(
        api: Api,
        autotag_settings: AutoTagSettings,
        post: Post,
        third_party_tags: List[_ThirdPartyTag]) -> _SyncInfo:
    sync_info = _SyncInfo()
    sync_info.source_tags = [tag.lower() for tag in post['tags']]
    sync_info.target_tags = sync_info.source_tags[:]

    for third_party_tag in _sanitize_third_party_tags(
            api, autotag_settings, third_party_tags):
        if third_party_tag.name.lower() in sync_info.source_tags:
            continue

        sync_info.target_tags.append(third_party_tag.name)

        try:
            api.get_tag(third_party_tag.name)
        except ApiError as ex:
            if 'not found' in str(ex).lower():
                sync_info.tags_to_create.append(third_party_tag)
            else:
                raise

        for implication in api.get_tag_implications(third_party_tag.name):
            implication = implication.lower()
            if implication not in sync_info.target_tags:
                sync_info.target_tags.append(implication)

    for tag_name in TAGS_TO_REMOVE:
        if tag_name.lower() in sync_info.source_tags:
            sync_info.target_tags.remove(tag_name)

    return sync_info

def _create_new_tags(
        api: Optional[Api],
        tags_to_create: List[_ThirdPartyTag]) -> None:
    print('Creating new tags...')
    if not tags_to_create:
        print('(no tags to create.)')
    else:
        for new_tag in tags_to_create:
            if new_tag.category in ('artist', 'copyright', 'character'):
                new_tag.name = util.capitalize(new_tag.name)
            print('- {}'.format(new_tag.name))
            if api:
                api.create_tag({
                    'names': [new_tag.name],
                    'category': new_tag.category,
                })
        print('Created {} new tags.'.format(len(tags_to_create)))
    print()

def _update_post_tags(
        api: Optional[Api],
        post: Post,
        source_tags: List[str],
        target_tags: List[str]) -> None:
    print('Updating post tags...')
    if target_tags == source_tags:
        print('(nothing to be done.)')
    else:
        for tag_name in target_tags:
            print('- {}{}'.format(
                '(new) ' if tag_name not in source_tags else '',
                tag_name))
        if api:
            api.update_post(
                post['id'],
                {
                    'tags': [tag_name for tag_name in target_tags],
                    'version': post['version'],
                })
        print('Added {} new tags.'.format(len(target_tags) - len(source_tags)))
    print()

def _sync(
        api: Api,
        autotag_settings: AutoTagSettings,
        post_id: int,
        source: Optional[str],
        force: Force,
        dry_run: bool) -> None:
    post = _get_post(api, autotag_settings, post_id, force >= Force.TagAgain)
    if not post:
        return
    third_party_tags = _collect_third_party_tags(
        post, source, force >= Force.DownloadAgain)
    sync_info = _get_sync_info(api, autotag_settings, post, third_party_tags)
    _create_new_tags(None if dry_run else api, sync_info.tags_to_create)
    _update_post_tags(
        None if dry_run else api,
        post,
        sync_info.source_tags,
        sync_info.target_tags)

class AutoTagChosenPostCommand(BaseCommand):
    def run(self, args: configargparse.Namespace) -> None:
        source: Optional[str] = args.source
        force: Force = args.force
        dry_run: bool = args.dry_run

        for post_id in args.post_ids:
            try:
                _sync(
                    self._api,
                    self._autotag_settings,
                    post_id,
                    source,
                    force,
                    dry_run)
                if not dry_run:
                    self._autotag_settings.mark_as_tagged(post_id)
            except Exception as ex:
                if not dry_run:
                    self._autotag_settings.mark_as_untagged(post_id)
                raise

    @staticmethod
    def _create_parser(
            parent_parser: configargparse.ArgumentParser
    ) -> configargparse.ArgumentParser:
        parser = parent_parser.add_parser(
            'autotag-chosen', help='fetch tags for chosen posts')
        parser.set_defaults(force=Force.NoForce)
        parser.add_argument(
            metavar='POST_ID', dest='post_ids', nargs='*', type=int,
            help='ID of the post to edit the tags for.')
        parser.add_argument(
            '-f', dest='force',
            action='store_const', const=Force.TagAgain, help=(
                'Force tagging even if the post was processed earlier.'))
        parser.add_argument(
            '-ff', dest='force',
            action='store_const', const=Force.DownloadAgain, help=(
                'Same as -f, but ignores IQDB cache.'))
        parser.add_argument(
            '--source', help='Source URL where to get tags from.')
        parser.add_argument(
            '--dry-run', action='store_true', help='Don\'t do anything.')
        return parser

class AutoTagNewestPostCommand(BaseCommand):
    def run(self, args: configargparse.Namespace) -> None:
        post_id = self._get_last_post_id()
        dry_run: bool = args.dry_run

        if not post_id:
            print('Everything up to date')
            return
        try:
            _sync(
                self._api,
                self._autotag_settings,
                post_id,
                None,
                Force.NoForce,
                dry_run)
            if not dry_run:
                self._autotag_settings.mark_as_tagged(post_id)
        except Exception as ex:
            if not dry_run:
                self._autotag_settings.mark_as_untagged(post_id)
            raise

    @staticmethod
    def _create_parser(
            parent_parser: configargparse.ArgumentParser
    ) -> configargparse.ArgumentParser:
        parser = parent_parser.add_parser(
            'autotag-newest', help='fetch tags for newest posts')
        parser.add_argument(
            '--dry-run', action='store_true', help='Don\'t do anything.')
        return parser

    def _get_last_post_id(self) -> Optional[int]:
        latest_id = self._api.get_last_post_id()
        if latest_id is None:
            return None
        processed_post_ids = self._autotag_settings.get_processed_post_ids()
        for post_id in range(1, latest_id + 1):
            if post_id not in processed_post_ids:
                return post_id
        return None
<commit_msg>opt/yume-tagger: allow only a single post id<commit_after>import re
import json
import enum
from typing import Optional, List, Callable, Iterable
import configargparse
import requests
from bs4 import BeautifulSoup
from yume_tagger import iqdb
from yume_tagger import util
from yume_tagger.api import Api, ApiError, Post
from yume_tagger.autotag_settings import AutoTagSettings
from yume_tagger.commands.base import BaseCommand

CACHE_PATH = util.DB_DIR.joinpath('cache')
TAGS_TO_REMOVE = ['tagme']

class Force(enum.IntEnum):
    NoForce = enum.auto()
    TagAgain = enum.auto()
    DownloadAgain = enum.auto()

class AutoTagError(RuntimeError):
    pass

class TooBigError(AutoTagError):
    pass

class _ThirdPartyTag:
    def __init__(self, name: str, category: str) -> None:
        self.name = name
        self.category = category
        # TODO: info about source

class _SyncInfo:
    def __init__(self) -> None:
        self.source_tags: List[str] = []
        self.target_tags: List[str] = []
        self.tags_to_create: List[_ThirdPartyTag] = []

def _get_tags_from_gelbooru(url: str) -> Iterable[_ThirdPartyTag]:
    if 'gelbooru' not in url:
        raise ValueError('Not a valid Gelbooru URL')
    session = requests.Session()
    response = session.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')
    list_items = soup.select('#tag-sidebar li')
    for list_item in list_items:
        try:
            category = list_item['class'][0].replace('tag-type-', '')
            name = list_item.select('a')[1].text.replace(' ', '_')
            yield _ThirdPartyTag(name, category)
        except IndexError:
            pass

def _get_tags_from_danbooru(url: str) -> Iterable[_ThirdPartyTag]:
    match = re.search(r'danbooru.*?(\d+)', url)
    if not match:
        raise ValueError('Not a valid Danbooru URL')
    post_id = int(match.group(1))
    session = requests.Session()
    response = session.get(
        'http://danbooru.donmai.us/posts/{}.json'.format(post_id))
    post = json.loads(response.text)
    for category in ['artist', 'character', 'copyright', 'general']:
        for name in post['tag_string_' + category].split(' '):
            if name:
                yield _ThirdPartyTag(name, category)

def _get_third_party_tags(source_urls: List[str]) -> List[_ThirdPartyTag]:
    handlers: List[Callable[[str], Iterable[_ThirdPartyTag]]] = [
        _get_tags_from_danbooru,
        _get_tags_from_gelbooru,
    ]
    ret: List[_ThirdPartyTag] = []
    for handler in handlers:
        for source_url in source_urls:
            try:
                ret.extend(list(handler(source_url)))
            except (ValueError, NotImplementedError):
                pass
    return ret

def _get_third_party_tag_sources(post_url: str) -> List[str]:
    try:
        return [
            result.url
            for result in iqdb.search(post_url)
            if result.similarity >= 0.90]
    except iqdb.NothingFoundIqdbError:
        return []
    except iqdb.UploadTooBigIqdbError:
        raise TooBigError('Post is too big to search for')

def _get_post(
        api: Api,
        autotag_settings: AutoTagSettings,
        post_id: int,
        skip_history: bool) -> Optional[Post]:
    if post_id in autotag_settings.get_processed_post_ids() \
            and not skip_history:
        print('Skipping post {} - already processed'.format(post_id))
        return None

    print('Retrieving post {}...'.format(post_id))
    post = api.get_post(post_id)
    print(post['contentUrl'])
    print()
    return post

def _collect_third_party_tags(
        post: Post,
        source: Optional[str],
        skip_cache: bool) -> List[_ThirdPartyTag]:
    print('Collecting third party tags...')

    cache_path = CACHE_PATH.joinpath('autotagger-{}.dat'.format(post['id']))
    if cache_path.exists() and not skip_cache:
        result = json.loads(cache_path.read_text())
        sources = result['sources']
        third_party_tags = [
            _ThirdPartyTag(name=tag['name'], category=tag['category'])
            for tag in result['tags']
        ]
    else:
        if source:
            sources = [source]
        else:
            try:
                sources = _get_third_party_tag_sources(post['contentUrl'])
            except TooBigError:
                sources = _get_third_party_tag_sources(post['thumbnailUrl'])
        third_party_tags = _get_third_party_tags(sources)

        cache_path.parent.mkdir(parents=True, exist_ok=True)
        cache_path.write_text(json.dumps({
            'sources': sources,
            'tags': [
                {'name': tag.name, 'category': tag.category}
                for tag in third_party_tags],
        }))

    if sources:
        print('Sources:')
        for source in sorted(sources):
            print('- {}'.format(source))
        if not third_party_tags:
            raise AutoTagError('Tag sources found, but not on any known site')
    else:
        raise AutoTagError('No tag sources')

    print('Tags:')
    for tag in third_party_tags:
        print('- {}'.format(tag.name))
    print()
    return third_party_tags

def _sanitize_third_party_tags(
        api: Api,
        autotag_settings: AutoTagSettings,
        third_party_tags: List[_ThirdPartyTag]) -> Iterable[_ThirdPartyTag]:
    for tag in third_party_tags:
        if autotag_settings.is_tag_banned(tag.name):
            print('Ignored tag {}'.format(tag.name))
            continue

        translated_tag = autotag_settings.translate_tag(tag.name)
        if tag.name != translated_tag:
            print('Translated tag {} → {}'.format(tag.name, translated_tag))

        sanitized_name = util.sanitize_tag(translated_tag)
        if sanitized_name != tag.name:
            print('Sanitized tag {} → {}'.format(tag.name, sanitized_name))

        try:
            upstream_tag = api.get_tag(sanitized_name)
        except ApiError:
            upstream_tag = None
        if upstream_tag:
            sanitized_name = upstream_tag['names'][0]

        yield _ThirdPartyTag(
            name=sanitized_name,
            category=autotag_settings.translate_tag_category(tag.category))

def _get_sync_info(
        api: Api,
        autotag_settings: AutoTagSettings,
        post: Post,
        third_party_tags: List[_ThirdPartyTag]) -> _SyncInfo:
    sync_info = _SyncInfo()
    sync_info.source_tags = [tag.lower() for tag in post['tags']]
    sync_info.target_tags = sync_info.source_tags[:]

    for third_party_tag in _sanitize_third_party_tags(
            api, autotag_settings, third_party_tags):
        if third_party_tag.name.lower() in sync_info.source_tags:
            continue

        sync_info.target_tags.append(third_party_tag.name)

        try:
            api.get_tag(third_party_tag.name)
        except ApiError as ex:
            if 'not found' in str(ex).lower():
                sync_info.tags_to_create.append(third_party_tag)
            else:
                raise

        for implication in api.get_tag_implications(third_party_tag.name):
            implication = implication.lower()
            if implication not in sync_info.target_tags:
                sync_info.target_tags.append(implication)

    for tag_name in TAGS_TO_REMOVE:
        if tag_name.lower() in sync_info.source_tags:
            sync_info.target_tags.remove(tag_name)

    return sync_info

def _create_new_tags(
        api: Optional[Api],
        tags_to_create: List[_ThirdPartyTag]) -> None:
    print('Creating new tags...')
    if not tags_to_create:
        print('(no tags to create.)')
    else:
        for new_tag in tags_to_create:
            if new_tag.category in ('artist', 'copyright', 'character'):
                new_tag.name = util.capitalize(new_tag.name)
            print('- {}'.format(new_tag.name))
            if api:
                api.create_tag({
                    'names': [new_tag.name],
                    'category': new_tag.category,
                })
        print('Created {} new tags.'.format(len(tags_to_create)))
    print()

def _update_post_tags(
        api: Optional[Api],
        post: Post,
        source_tags: List[str],
        target_tags: List[str]) -> None:
    print('Updating post tags...')
    if target_tags == source_tags:
        print('(nothing to be done.)')
    else:
        for tag_name in target_tags:
            print('- {}{}'.format(
                '(new) ' if tag_name not in source_tags else '',
                tag_name))
        if api:
            api.update_post(
                post['id'],
                {
                    'tags': [tag_name for tag_name in target_tags],
                    'version': post['version'],
                })
        print('Added {} new tags.'.format(len(target_tags) - len(source_tags)))
    print()

def _sync(
        api: Api,
        autotag_settings: AutoTagSettings,
        post_id: int,
        source: Optional[str],
        force: Force,
        dry_run: bool) -> None:
    post = _get_post(api, autotag_settings, post_id, force >= Force.TagAgain)
    if not post:
        return
    third_party_tags = _collect_third_party_tags(
        post, source, force >= Force.DownloadAgain)
    sync_info = _get_sync_info(api, autotag_settings, post, third_party_tags)
    _create_new_tags(None if dry_run else api, sync_info.tags_to_create)
    _update_post_tags(
        None if dry_run else api,
        post,
        sync_info.source_tags,
        sync_info.target_tags)

class AutoTagChosenPostCommand(BaseCommand):
    def run(self, args: configargparse.Namespace) -> None:
        post_id: int = args.post_id
        source: Optional[str] = args.source
        force: Force = args.force
        dry_run: bool = args.dry_run

        try:
            _sync(
                self._api,
                self._autotag_settings,
                post_id,
                source,
                force,
                dry_run)
            if not dry_run:
                self._autotag_settings.mark_as_tagged(post_id)
        except Exception as ex:
            if not dry_run:
                self._autotag_settings.mark_as_untagged(post_id)
            raise

    @staticmethod
    def _create_parser(
            parent_parser: configargparse.ArgumentParser
    ) -> configargparse.ArgumentParser:
        parser = parent_parser.add_parser(
            'autotag-chosen', help='fetch tags for chosen posts')
        parser.set_defaults(force=Force.NoForce)
        parser.add_argument(
            'post_id', type=int, help='ID of the post to edit the tags for.')
        parser.add_argument(
            '-f', dest='force',
            action='store_const', const=Force.TagAgain, help=(
                'Force tagging even if the post was processed earlier.'))
        parser.add_argument(
            '-ff', dest='force',
            action='store_const', const=Force.DownloadAgain, help=(
                'Same as -f, but ignores IQDB cache.'))
        parser.add_argument(
            '--source', help='Source URL where to get tags from.')
        parser.add_argument(
            '--dry-run', action='store_true', help='Don\'t do anything.')
        return parser

class AutoTagNewestPostCommand(BaseCommand):
    def run(self, args: configargparse.Namespace) -> None:
        post_id = self._get_last_post_id()
        dry_run: bool = args.dry_run

        if not post_id:
            print('Everything up to date')
            return
        try:
            _sync(
                self._api,
                self._autotag_settings,
                post_id,
                None,
                Force.NoForce,
                dry_run)
            if not dry_run:
                self._autotag_settings.mark_as_tagged(post_id)
        except Exception as ex:
            if not dry_run:
                self._autotag_settings.mark_as_untagged(post_id)
            raise

    @staticmethod
    def _create_parser(
            parent_parser: configargparse.ArgumentParser
    ) -> configargparse.ArgumentParser:
        parser = parent_parser.add_parser(
            'autotag-newest', help='fetch tags for newest posts')
        parser.add_argument(
            '--dry-run', action='store_true', help='Don\'t do anything.')
        return parser

    def _get_last_post_id(self) -> Optional[int]:
        latest_id = self._api.get_last_post_id()
        if latest_id is None:
            return None
        processed_post_ids = self._autotag_settings.get_processed_post_ids()
        for post_id in range(1, latest_id + 1):
            if post_id not in processed_post_ids:
                return post_id
        return None
<|endoftext|>