<commit_before># -*- coding: utf-8 -*-

#       Licensed to the Apache Software Foundation (ASF) under one
#       or more contributor license agreements.  See the NOTICE file
#       distributed with this work for additional information
#       regarding copyright ownership.  The ASF licenses this file
#       to you under the Apache License, Version 2.0 (the
#       "License"); you may not use this file except in compliance
#       with the License.  You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#       Unless required by applicable law or agreed to in writing,
#       software distributed under the License is distributed on an
#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#       KIND, either express or implied.  See the License for the
#       specific language governing permissions and limitations
#       under the License.

"""
Functions to syntax-validate output content
"""
from os import path, environ, getcwd
import os
import sys
import logging
import tempfile
import subprocess
import json
import urllib2
import re

import tg
import mock
import beaker.session
from paste.deploy import loadapp
from paste.script.appinstall import SetupCommand
from pylons import tmpl_context as c, app_globals as g
from pylons import url, request, response, session
import webtest
from webtest import TestApp
from webob import Request, Response
from nose.tools import ok_, assert_true, assert_false
from poster.encode import multipart_encode
from poster.streaminghttp import register_openers
from ming.utils import LazyProperty

from allura.lib import utils

log = logging.getLogger(__name__)

class Config(object):
    "Config to encapsulate flexible/complex test enabled/disabled rules."
    _instance = None

    def __init__(self):
        self.ini_config = None
        pass

    @classmethod
    def instance(cls):
        if not cls._instance:
            cls._instance = cls()
        return cls._instance

    @LazyProperty
    def test_ini(self):
        if not self.ini_config:
            from . import controller
            import ConfigParser
            conf = ConfigParser.ConfigParser({'validate_html5': 'false', 'validate_inlinejs': 'false'})
            conf.read(controller.get_config_file())
            self.ini_config = conf
        return self.ini_config

    @LazyProperty
    def hostname(self):
        if os.path.exists('/etc/soghost'):
            with open('/etc/soghost') as fp:
                return fp.read().strip()

    def validation_enabled(self, val_type):
        env_var = os.getenv('ALLURA_VALIDATION')
        if env_var == 'all':
            return True
        elif env_var == 'none':
            return False
        elif env_var is not None:
            return val_type in env_var.split(',')

        enabled = self.test_ini.getboolean('validation', 'validate_' + val_type)
        return enabled

    def fail_on_validation(self, val_type):
        env_var = os.getenv('ALLURA_VALIDATION')
        if env_var == 'all':
            return True
        return False

def report_validation_error(val_name, filename, message):
    message = '%s Validation errors (%s):\n%s\n' % (val_name, filename, message)
    if Config.instance().fail_on_validation(val_name):
        ok_(False, message)
    else:
        sys.stderr.write('=' * 40 + '\n' + message)

def dump_to_file(prefix, html):
    f = tempfile.NamedTemporaryFile(prefix=prefix, delete=False)
    f.write(html)
    f.close()
    return f.name

def validate_html(html_or_response):
    if hasattr(html_or_response, 'body'):
        html = html_or_response.body
    else:
        html = html_or_response

    html = html.lstrip()

    if html.startswith('<!DOCTYPE html>'):
        return validate_html5(html)
    else:
        assert False, 'Non-valid HTML: ' + html[:100] + '...'

def validate_json(json_or_response):
    if hasattr(json_or_response, 'body'):
        j = json_or_response.body
    else:
        j = json_or_response

    try:
        obj = json.loads(j)
    except Exception, e:
        ok_(False, "Couldn't validate JSON: " + str(e) + ':' + j[:100] + '...')

    return obj

def validate_html5(html_or_response):
    if hasattr(html_or_response, 'body'):
        html = html_or_response.body
    else:
        html = html_or_response
    register_openers()
    params = [("out","text"),("content",html)]
    datagen, headers = multipart_encode(params)
    request = urllib2.Request("http://html5.validator.nu/", datagen, headers)
    count = 3
    while True:
        try:
            resp = urllib2.urlopen(request, timeout=3).read()
            break
        except:
            resp = "Couldn't connect to validation service to check the HTML"
            count -= 1
            if count == 0:
                sys.stderr.write('WARNING: ' + resp + '\n')
                break

    resp = resp.replace('“','"').replace('”','"').replace('–','-')

    ignored_errors = [
        'Required attributes missing on element "object"',
        'Stray end tag "embed".',
        'Stray end tag "param".',
        r'Bad value .+? for attribute "onclick" on element "input": invalid return',
    ]
    for ignore in ignored_errors:
        resp = re.sub('Error: ' + ignore, 'Ignoring: ' + ignore, resp)

    if 'Error:' in resp:
        fname = dump_to_file('html5-', html)
        message = resp.decode('ascii','ignore')
        report_validation_error('html5', fname, message)

def validate_html5_chunk(html):
    """ When you don't have a html & body tags - this adds it"""
    # WebTest doesn't like HTML fragments without doctype,
    # so we output them sometimes for fragments, which is hack.
    # Unhack it here.
    doctype = '<!DOCTYPE html>'
    if html.startswith(doctype):
        html = html[len(doctype):]

    html = '''<!DOCTYPE html>

    <title>Not empty</title>

    %s
    ''' % html
    return validate_html5(html)

def validate_js(html_or_response):
    if hasattr(html_or_response, 'body'):
        if html_or_response.status_int != 200:
            return
        html = html_or_response.body
    else:
        html = html_or_response
    basedir = path.dirname(path.abspath(__file__))
    jslint_dir = basedir + '/../jslint'
    fname = dump_to_file('jslint-', html)
    cmd = 'java -jar ' + jslint_dir + '/js.jar '+ jslint_dir +'/jslint.js ' + fname
    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    stdout, stderr = p.communicate(html)
    if stdout.startswith('jslint: No problems found'):
        os.unlink(fname)
        return
    stdout = stdout.decode('UTF-8', 'replace')
    msg = '\n'.join(repr(s) for s in stdout.split('\n') if s)
    report_validation_error('js', fname, msg)

def validate_page(html_or_response):
    if Config.instance().validation_enabled('html5'):
        validate_html(html_or_response)
    if Config.instance().validation_enabled('inlinejs'):
        validate_js(html_or_response)

class AntiSpamTestApp(TestApp):

    def post(self, *args, **kwargs):
        if kwargs.pop('antispam', False):
            antispam = utils.AntiSpam()
            params = {
                'timestamp': antispam.timestamp_text,
                'spinner': antispam.spinner_text,
                antispam.enc('honey0'): '',
                antispam.enc('honey1'): '',
            }
            for k, v in kwargs['params'].iteritems():
                params[antispam.enc(k)] = v
            kwargs['params'] = params
        return super(AntiSpamTestApp, self).post(*args, **kwargs)

class PostParamCheckingTestApp(AntiSpamTestApp):

    def _validate_params(self, params, method):
        if not params:
            return
        # params can be a list or a dict
        if hasattr(params, 'items'):
            params = params.items()
        for k, v in params:
            if not isinstance(k, basestring):
                raise TypeError('%s key %s is %s, not str' % (method, k, type(k)))
            if not isinstance(v, (basestring, webtest.app.File)):
                raise TypeError('%s key %s has value %s of type %s, not str. ' % (method, k, v, type(v)))

    def get(self, *args, **kwargs):
        self._validate_params(kwargs.get('params'), 'get')
        return super(PostParamCheckingTestApp, self).get(*args, **kwargs)

    def post(self, *args, **kwargs):
        self._validate_params(kwargs.get('params'), 'post')
        return super(PostParamCheckingTestApp, self).post(*args, **kwargs)

class ValidatingTestApp(PostParamCheckingTestApp):

    # Subclasses may set this to True to skip validation altogether
    validate_skip = False

    def _validate(self, resp, method, val_params):
        """Perform validation on webapp response. This handles responses of
        various types and forms."""
        if resp.status_int != 200:
            return

        content = resp.body
        content_type = resp.headers['Content-Type']
        if content_type.startswith('text/html'):
            if val_params['validate_chunk']:
                validate_html5_chunk(content)
            else:
                validate_page(resp)
        elif content_type.split(';', 1)[0] in ('text/plain', 'text/x-python', 'application/octet-stream'):
            pass
        elif content_type.startswith('application/json'):
            validate_json(content)
        elif content_type.startswith(('application/x-javascript','application/javascript', 'text/javascript')):
            validate_js(content)
        elif content_type.startswith('application/xml'):
            import feedparser
            d = feedparser.parse(content)
            assert d.bozo == 0, 'Non-wellformed feed'
        elif content_type.startswith('image/'):
            pass
        else:
            assert False, 'Unexpected output content type: ' + content_type

    def _get_validation_params(self, kw):
        "Separate validation params from normal TestApp methods params."
        params = {}
        for k in ('validate_skip', 'validate_chunk'):
            params[k] = kw.pop(k, False)
        return params, kw

    def get(self, *args, **kw):
        val_params, kw = self._get_validation_params(kw)
        resp = super(ValidatingTestApp, self).get(*args, **kw)
        if not self.validate_skip and not val_params['validate_skip']:
            self._validate(resp, 'get', val_params)
        return resp

    def post(self, *args, **kw):
        val_params, kw = self._get_validation_params(kw)
        resp = super(ValidatingTestApp, self).post(*args, **kw)
        if not self.validate_skip and not val_params['validate_skip']:
            self._validate(resp, 'post', val_params)
        return resp
<commit_msg>Check if html5 validation is enabled for partial content too<commit_after># -*- coding: utf-8 -*-

#       Licensed to the Apache Software Foundation (ASF) under one
#       or more contributor license agreements.  See the NOTICE file
#       distributed with this work for additional information
#       regarding copyright ownership.  The ASF licenses this file
#       to you under the Apache License, Version 2.0 (the
#       "License"); you may not use this file except in compliance
#       with the License.  You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#       Unless required by applicable law or agreed to in writing,
#       software distributed under the License is distributed on an
#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#       KIND, either express or implied.  See the License for the
#       specific language governing permissions and limitations
#       under the License.

"""
Functions to syntax-validate output content
"""
from os import path, environ, getcwd
import os
import sys
import logging
import tempfile
import subprocess
import json
import urllib2
import re

import tg
import mock
import beaker.session
from paste.deploy import loadapp
from paste.script.appinstall import SetupCommand
from pylons import tmpl_context as c, app_globals as g
from pylons import url, request, response, session
import webtest
from webtest import TestApp
from webob import Request, Response
from nose.tools import ok_, assert_true, assert_false
from poster.encode import multipart_encode
from poster.streaminghttp import register_openers
from ming.utils import LazyProperty

from allura.lib import utils

log = logging.getLogger(__name__)

class Config(object):
    "Config to encapsulate flexible/complex test enabled/disabled rules."
    _instance = None

    def __init__(self):
        self.ini_config = None
        pass

    @classmethod
    def instance(cls):
        if not cls._instance:
            cls._instance = cls()
        return cls._instance

    @LazyProperty
    def test_ini(self):
        if not self.ini_config:
            from . import controller
            import ConfigParser
            conf = ConfigParser.ConfigParser({'validate_html5': 'false', 'validate_inlinejs': 'false'})
            conf.read(controller.get_config_file())
            self.ini_config = conf
        return self.ini_config

    @LazyProperty
    def hostname(self):
        if os.path.exists('/etc/soghost'):
            with open('/etc/soghost') as fp:
                return fp.read().strip()

    def validation_enabled(self, val_type):
        env_var = os.getenv('ALLURA_VALIDATION')
        if env_var == 'all':
            return True
        elif env_var == 'none':
            return False
        elif env_var is not None:
            return val_type in env_var.split(',')

        enabled = self.test_ini.getboolean('validation', 'validate_' + val_type)
        return enabled

    def fail_on_validation(self, val_type):
        env_var = os.getenv('ALLURA_VALIDATION')
        if env_var == 'all':
            return True
        return False

def report_validation_error(val_name, filename, message):
    message = '%s Validation errors (%s):\n%s\n' % (val_name, filename, message)
    if Config.instance().fail_on_validation(val_name):
        ok_(False, message)
    else:
        sys.stderr.write('=' * 40 + '\n' + message)

def dump_to_file(prefix, html):
    f = tempfile.NamedTemporaryFile(prefix=prefix, delete=False)
    f.write(html)
    f.close()
    return f.name

def validate_html(html_or_response):
    if hasattr(html_or_response, 'body'):
        html = html_or_response.body
    else:
        html = html_or_response

    html = html.lstrip()

    if html.startswith('<!DOCTYPE html>'):
        return validate_html5(html)
    else:
        assert False, 'Non-valid HTML: ' + html[:100] + '...'

def validate_json(json_or_response):
    if hasattr(json_or_response, 'body'):
        j = json_or_response.body
    else:
        j = json_or_response

    try:
        obj = json.loads(j)
    except Exception, e:
        ok_(False, "Couldn't validate JSON: " + str(e) + ':' + j[:100] + '...')

    return obj

def validate_html5(html_or_response):
    if hasattr(html_or_response, 'body'):
        html = html_or_response.body
    else:
        html = html_or_response
    register_openers()
    params = [("out","text"),("content",html)]
    datagen, headers = multipart_encode(params)
    request = urllib2.Request("http://html5.validator.nu/", datagen, headers)
    count = 3
    while True:
        try:
            resp = urllib2.urlopen(request, timeout=3).read()
            break
        except:
            resp = "Couldn't connect to validation service to check the HTML"
            count -= 1
            if count == 0:
                sys.stderr.write('WARNING: ' + resp + '\n')
                break

    resp = resp.replace('“','"').replace('”','"').replace('–','-')

    ignored_errors = [
        'Required attributes missing on element "object"',
        'Stray end tag "embed".',
        'Stray end tag "param".',
        r'Bad value .+? for attribute "onclick" on element "input": invalid return',
    ]
    for ignore in ignored_errors:
        resp = re.sub('Error: ' + ignore, 'Ignoring: ' + ignore, resp)

    if 'Error:' in resp:
        fname = dump_to_file('html5-', html)
        message = resp.decode('ascii','ignore')
        report_validation_error('html5', fname, message)

def validate_html5_chunk(html):
    """ When you don't have a html & body tags - this adds it"""
    # WebTest doesn't like HTML fragments without doctype,
    # so we output them sometimes for fragments, which is hack.
    # Unhack it here.
    doctype = '<!DOCTYPE html>'
    if html.startswith(doctype):
        html = html[len(doctype):]

    html = '''<!DOCTYPE html>

    <title>Not empty</title>

    %s
    ''' % html
    return validate_html5(html)

def validate_js(html_or_response):
    if hasattr(html_or_response, 'body'):
        if html_or_response.status_int != 200:
            return
        html = html_or_response.body
    else:
        html = html_or_response
    basedir = path.dirname(path.abspath(__file__))
    jslint_dir = basedir + '/../jslint'
    fname = dump_to_file('jslint-', html)
    cmd = 'java -jar ' + jslint_dir + '/js.jar '+ jslint_dir +'/jslint.js ' + fname
    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    stdout, stderr = p.communicate(html)
    if stdout.startswith('jslint: No problems found'):
        os.unlink(fname)
        return
    stdout = stdout.decode('UTF-8', 'replace')
    msg = '\n'.join(repr(s) for s in stdout.split('\n') if s)
    report_validation_error('js', fname, msg)

def validate_page(html_or_response):
    if Config.instance().validation_enabled('html5'):
        validate_html(html_or_response)
    if Config.instance().validation_enabled('inlinejs'):
        validate_js(html_or_response)

class AntiSpamTestApp(TestApp):

    def post(self, *args, **kwargs):
        if kwargs.pop('antispam', False):
            antispam = utils.AntiSpam()
            params = {
                'timestamp': antispam.timestamp_text,
                'spinner': antispam.spinner_text,
                antispam.enc('honey0'): '',
                antispam.enc('honey1'): '',
            }
            for k, v in kwargs['params'].iteritems():
                params[antispam.enc(k)] = v
            kwargs['params'] = params
        return super(AntiSpamTestApp, self).post(*args, **kwargs)

class PostParamCheckingTestApp(AntiSpamTestApp):

    def _validate_params(self, params, method):
        if not params:
            return
        # params can be a list or a dict
        if hasattr(params, 'items'):
            params = params.items()
        for k, v in params:
            if not isinstance(k, basestring):
                raise TypeError('%s key %s is %s, not str' % (method, k, type(k)))
            if not isinstance(v, (basestring, webtest.app.File)):
                raise TypeError('%s key %s has value %s of type %s, not str. ' % (method, k, v, type(v)))

    def get(self, *args, **kwargs):
        self._validate_params(kwargs.get('params'), 'get')
        return super(PostParamCheckingTestApp, self).get(*args, **kwargs)

    def post(self, *args, **kwargs):
        self._validate_params(kwargs.get('params'), 'post')
        return super(PostParamCheckingTestApp, self).post(*args, **kwargs)

class ValidatingTestApp(PostParamCheckingTestApp):

    # Subclasses may set this to True to skip validation altogether
    validate_skip = False

    def _validate(self, resp, method, val_params):
        """Perform validation on webapp response. This handles responses of
        various types and forms."""
        if resp.status_int != 200:
            return

        content = resp.body
        content_type = resp.headers['Content-Type']
        if content_type.startswith('text/html'):
            if val_params['validate_chunk']:
                if Config.instance().validation_enabled('html5'):
                    validate_html5_chunk(content)
            else:
                validate_page(resp)
        elif content_type.split(';', 1)[0] in ('text/plain', 'text/x-python', 'application/octet-stream'):
            pass
        elif content_type.startswith('application/json'):
            validate_json(content)
        elif content_type.startswith(('application/x-javascript','application/javascript', 'text/javascript')):
            validate_js(content)
        elif content_type.startswith('application/xml'):
            import feedparser
            d = feedparser.parse(content)
            assert d.bozo == 0, 'Non-wellformed feed'
        elif content_type.startswith('image/'):
            pass
        else:
            assert False, 'Unexpected output content type: ' + content_type

    def _get_validation_params(self, kw):
        "Separate validation params from normal TestApp methods params."
        params = {}
        for k in ('validate_skip', 'validate_chunk'):
            params[k] = kw.pop(k, False)
        return params, kw

    def get(self, *args, **kw):
        val_params, kw = self._get_validation_params(kw)
        resp = super(ValidatingTestApp, self).get(*args, **kw)
        if not self.validate_skip and not val_params['validate_skip']:
            self._validate(resp, 'get', val_params)
        return resp

    def post(self, *args, **kw):
        val_params, kw = self._get_validation_params(kw)
        resp = super(ValidatingTestApp, self).post(*args, **kw)
        if not self.validate_skip and not val_params['validate_skip']:
            self._validate(resp, 'post', val_params)
        return resp
<|endoftext|>


<commit_before># Tai Sakuma <tai.sakuma@cern.ch>
from ..mkdir_p import mkdir_p
import os
import ROOT

##__________________________________________________________________||
def IsROOTNullPointer(tobject):
    try:
        tobject.GetName()
        return False
    except ReferenceError:
        return True

##__________________________________________________________________||
class TblBranch(object):
    def __init__(self, outPath, analyzerName, fileName, treeName):
        self.outPath = outPath
        self.analyzerName = analyzerName
        self.fileName = fileName
        self.treeName = treeName
        self._done = False

    def begin(self): pass

    def read(self, component):
        if self._done: return
        self._done = True

        inputPath = os.path.join(getattr(component, self.analyzerName).path, self.fileName)
        file = ROOT.TFile.Open(inputPath)
        tree = file.Get(self.treeName)

        results = [ ]
        results.append(('name', 'type', 'isarray', 'countname'))
        for leaf in tree.GetListOfLeaves():
            leafcount = leaf.GetLeafCount()
            isArray = not IsROOTNullPointer(leafcount)
            row = [ ]
            row.append(leaf.GetName())
            row.append(leaf.GetTypeName())
            row.append('1' if isArray else '0')
            row.append(leafcount.GetName() if isArray else None)
            results.append(row)

        transposed = [[r[i] for r in results] for i in range(len(results[0]))]
        transposed = [[str(e) for e in r] for r in transposed]
        columnWidths = [max([len(e) for e in r]) for r in transposed]
        format = " {:>" + "s} {:>".join([str(e) for e in columnWidths]) + "s}"

        f = self._open(self.outPath)
        for row in zip(*transposed):
            f.write(format.format(*row))
            f.write("\n")
        self._close(f)

    def end(self): pass

    def _open(self, path):
        mkdir_p(os.path.dirname(path))
        return open(path, 'w')

    def _close(self, file): file.close()

##__________________________________________________________________||
<commit_msg>use listToAlignedText in TblBranch.py<commit_after># Tai Sakuma <tai.sakuma@cern.ch>
from ..mkdir_p import mkdir_p
from ..listToAlignedText import listToAlignedText
import os
import ROOT

##__________________________________________________________________||
def IsROOTNullPointer(tobject):
    try:
        tobject.GetName()
        return False
    except ReferenceError:
        return True

##__________________________________________________________________||
class TblBranch(object):
    def __init__(self, outPath, analyzerName, fileName, treeName):
        self.outPath = outPath
        self.analyzerName = analyzerName
        self.fileName = fileName
        self.treeName = treeName
        self._done = False

    def begin(self): pass

    def read(self, component):
        if self._done: return
        self._done = True

        inputPath = os.path.join(getattr(component, self.analyzerName).path, self.fileName)
        file = ROOT.TFile.Open(inputPath)
        tree = file.Get(self.treeName)

        results = [ ]
        results.append(('name', 'type', 'isarray', 'countname'))
        for leaf in tree.GetListOfLeaves():
            leafcount = leaf.GetLeafCount()
            isArray = not IsROOTNullPointer(leafcount)
            row = [ ]
            row.append(leaf.GetName())
            row.append(leaf.GetTypeName())
            row.append('1' if isArray else '0')
            row.append(leafcount.GetName() if isArray else None)
            results.append(row)

        f = self._open(self.outPath)
        f.write(listToAlignedText(results))
        self._close(f)

    def end(self): pass

    def _open(self, path):
        mkdir_p(os.path.dirname(path))
        return open(path, 'w')

    def _close(self, file): file.close()

##__________________________________________________________________||
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# _*_ coding: utf-8 _*_

###
# Project          : SubLime
# FileName         : test_subtitle.py
# ------------------------------------------------------------------------------
# Author           : sham
# E-Mail           : mauricesham@gmail.com
# ------------------------------------------------------------------------------
# Creation date    : 30/08/2013
##

import unittest

from sublime.subtitle import LanguageInfo
from sublime.subtitle import LanguageManager

# ------------------------------------------------------------------------------
#
# LanguagesTestCase class
#
# ------------------------------------------------------------------------------
class LanguagesTestCase(unittest.TestCase):
    """ Tests Languages module. """

    def test_LanguageInfo_creation(self):
        """ Tests creation of a languageinfo object. """
        lang_info = LanguageInfo("yor", None, "yo", "Yoruba")

        assert lang_info.long_code is "yor"
        assert lang_info.long_code_alt is None
        assert lang_info.short_code is "yo"
        assert lang_info.name is "Yoruba"

    def test_LanguageManager_as_singleton(self):
        """ Tests that a LanguageManager object is a singleton. """
        lang_manager01 = LanguageManager()
        lang_manager02 = LanguageManager()

        assert id(lang_manager01) == id(lang_manager02)

    def test_LanguageManager_get_all_language_codes(self):
        """ Tests getting all language code as a list. """
        lang_manager = LanguageManager()

        expecting_codes = [
            "iku", "ile", "ilo", "ina", "inc",
            "ind", "ine", "inh", "ipk", "ira",
            "iro", "ita", "jav", "jbo", "jpn",
            "jpr", "jrb", "kaa", "kab", "kac",
            "kal", "kam", "kan", "kar", "kas",
            "kau", "kaw", "kaz", "kbd", "kha",
            "khi", "khm", "kho", "kik", "kin",
            "kir", "kmb", "kok", "kom", "kon",
            "kor", "kos", "kpe", "krc", "mg", "mt"
        ]

        lang_codes_list = lang_manager.get_all_language_codes()

        for code in expecting_codes:
            assert code in lang_codes_list
        assert lang_codes_list.index("iku") < lang_codes_list.index("ile")

    def test_LanguageManager_get_language_info(self):
        """ Tests getting information about one language. """
        lang_manager = LanguageManager()

        result_lang = lang_manager.get_language_info("zun")

        assert result_lang.long_code == "zun"
        assert result_lang.name == "Zuni"

        result_lang = lang_manager.get_language_info("DUMMY")

        assert result_lang is None

if __name__ == "__main__":
    unittest.main()

# EOF
<commit_msg>Rewrite assertions by using assert methods.<commit_after>#!/usr/bin/env python3
# _*_ coding: utf-8 _*_

###
# Project          : SubLime
# FileName         : test_subtitle.py
# ------------------------------------------------------------------------------
# Author           : sham
# E-Mail           : mauricesham@gmail.com
# ------------------------------------------------------------------------------
# Creation date    : 30/08/2013
##

import unittest

from sublime.subtitle import LanguageInfo
from sublime.subtitle import LanguageManager

# ------------------------------------------------------------------------------
#
# LanguagesTestCase class
#
# ------------------------------------------------------------------------------
class LanguagesTestCase(unittest.TestCase):
    """ Tests Languages module. """

    def test_LanguageInfo_creation(self):
        """ Tests creation of a languageinfo object. """
        lang_info = LanguageInfo("yor", None, "yo", "Yoruba")

        self.assertEqual(lang_info.long_code, "yor")
        self.assertIsNone(lang_info.long_code_alt)
        self.assertEqual(lang_info.short_code, "yo")
        self.assertEqual(lang_info.name, "Yoruba")

    def test_LanguageManager_as_singleton(self):
        """ Tests that a LanguageManager object is a singleton. """
        lang_manager01 = LanguageManager()
        lang_manager02 = LanguageManager()

        self.assertEqual(id(lang_manager01), id(lang_manager02))

    def test_LanguageManager_get_all_language_codes(self):
        """ Tests getting all language code as a list. """
        lang_manager = LanguageManager()

        expecting_codes = [
            "iku", "ile", "ilo", "ina", "inc",
            "ind", "ine", "inh", "ipk", "ira",
            "iro", "ita", "jav", "jbo", "jpn",
            "jpr", "jrb", "kaa", "kab", "kac",
            "kal", "kam", "kan", "kar", "kas",
            "kau", "kaw", "kaz", "kbd", "kha",
            "khi", "khm", "kho", "kik", "kin",
            "kir", "kmb", "kok", "kom", "kon",
            "kor", "kos", "kpe", "krc", "mg", "mt"
        ]

        lang_codes_list = lang_manager.get_all_language_codes()

        for code in expecting_codes:
            self.assertIn(code, lang_codes_list)
        self.assertLess(lang_codes_list.index("iku"), lang_codes_list.index("ile"))

    def test_LanguageManager_get_language_info(self):
        """ Tests getting information about one language. """
        lang_manager = LanguageManager()

        result_lang = lang_manager.get_language_info("zun")

        self.assertEqual(result_lang.long_code, "zun")
        self.assertEqual(result_lang.name, "Zuni")

        result_lang = lang_manager.get_language_info("DUMMY")

        self.assertIsNone(result_lang)

if __name__ == "__main__":
    unittest.main()

# EOF
<|endoftext|>


<commit_before>from AppKit import NSObject, NSThread

import os

settingsIdentifier = "com.typemytype.toolbox"

class Report(object):

    def __init__(self):
        self._data = []

    def write(self, value):
        self._data.append(value)

    def writeTitle(self, value, underline="-"):
        self.write(value)
        self.write(underline * len(value))

    def writeItems(self, items):
        for i in items:
            if i:
                self.write(i)

    def newLine(self):
        self._data.append("")

    def writeDict(self, d):
        maxLength = 0
        for key in d:
            l = len(key)
            if l > maxLength:
                maxLength = l

        for key in sorted(d):
            value = d[key]
            t = "%s = %s" % (key.ljust(maxLength), value)
            self.write(t)

    def writeList(self, l):
        for i in l:
            self.write(str(i))

    def save(self, path):
        f = file(path, "w")
        f.write("\n".join(self._data))
        f.close()

def updateWithDefaultValues(data, defaults):
    for key, value in defaults.items():
        if key in data:
            continue
        data[key] = value

def buildTree(path):
    if not os.path.exists(path):
        os.makedirs(path)

class TaskRunner(NSObject):

    def __new__(cls, *args, **kwargs):
        return cls.alloc().init()

    def __init__(self, callback, threaded, progress, kwargs=dict()):
        self._callback = callback
        self._kwargs = kwargs
        self._progress = progress

        if threaded:
            self._thread = NSThread.alloc().initWithTarget_selector_object_(self, "runTask:", None)
            self._thread.start()
        else:
            self.runTask_(None)

    def runTask_(self, sender):
        try:
            self._callback(progress=self._progress, **self._kwargs)
        except:
            import traceback
            errorMessage = [
                    "*"*30,
                    traceback.format_exc(5),
                    "*"*30
                    ]
            print "\n".join(errorMessage)
        self._progress.close()
<commit_msg>add some usefull Report methods<commit_after>from AppKit import NSObject, NSThread

import os

settingsIdentifier = "com.typemytype.toolbox"

class Report(object):

    INDENT = "    "

    def __init__(self):
        self._data = []
        self._indent = 0

    def indent(self, value=None):
        if value is None:
            self._indent += 1
        else:
            self._indent = value

    def dedent(self):
        self._indent -= 1
        if self._indent < 0:
            self._indent = 0

    def write(self, value):
        indent = self.INDENT * self._indent
        value = value.replace("\n", "\n%s" % indent)
        self._data.append("%s%s" % (indent, value))

    def writeTitle(self, value, underline="*"):
        self.write(value)
        self.write(underline * len(value))

    def writeItems(self, items):
        for i in items:
            if i:
                self.write(i)

    def newLine(self):
        self._data.append("")

    def writeDict(self, d):
        maxLength = 0
        for key in d:
            l = len(key)
            if l > maxLength:
                maxLength = l

        for key in sorted(d):
            value = d[key]
            t = "%s = %s" % (key.ljust(maxLength), value)
            self.write(t)

    def writeList(self, l):
        for i in l:
            self.write(str(i))

    def save(self, path):
        f = file(path, "w")
        f.write(self.get())
        f.close()

    def get(self):
        return "\n".join(self._data)

def updateWithDefaultValues(data, defaults):
    for key, value in defaults.items():
        if key in data:
            continue
        data[key] = value

def buildTree(path):
    if not os.path.exists(path):
        os.makedirs(path)

class TaskRunner(NSObject):

    def __new__(cls, *args, **kwargs):
        return cls.alloc().init()

    def __init__(self, callback, threaded, progress, kwargs=dict()):
        self._callback = callback
        self._kwargs = kwargs
        self._progress = progress

        if threaded:
            self._thread = NSThread.alloc().initWithTarget_selector_object_(self, "runTask:", None)
            self._thread.start()
        else:
            self.runTask_(None)

    def runTask_(self, sender):
        try:
            self._callback(progress=self._progress, **self._kwargs)
        except:
            import traceback
            errorMessage = [
                    "*"*30,
                    traceback.format_exc(5),
                    "*"*30
                    ]
            print "\n".join(errorMessage)
        self._progress.close()
<|endoftext|>


<commit_before>"""
Functions for interacting with the database server.
"""
from server_common.utilities import dehex_and_decompress, print_and_log
from server_common.channel_access import ChannelAccess
from server_common.pv_names import DatabasePVNames
import json
import traceback

def get_iocs(prefix):
    """
    Get the list of available IOCs from DatabaseServer.

    Args:
        prefix : The PV prefix for this instrument.

    Returns:
        A list of the names of available IOCs.
    """
    #
    try:
        rawjson = dehex_and_decompress(bytes(ChannelAccess.caget(prefix + DatabasePVNames.IOCS), encoding="utf-8")).decode("utf-8")
        return json.loads(rawjson).keys()
    except Exception:
        print_and_log(f"Could not retrieve IOC list: {traceback.format_exc()}", "MAJOR")
        return []
<commit_msg>reverting byte cast for ioc list<commit_after>"""
Functions for interacting with the database server.
"""
from server_common.utilities import dehex_and_decompress, print_and_log
from server_common.channel_access import ChannelAccess
from server_common.pv_names import DatabasePVNames
import json
import traceback

def get_iocs(prefix):
    """
    Get the list of available IOCs from DatabaseServer.

    Args:
        prefix : The PV prefix for this instrument.

    Returns:
        A list of the names of available IOCs.
    """
    #
    try:
        rawjson = dehex_and_decompress(ChannelAccess.caget(prefix + DatabasePVNames.IOCS)).decode("utf-8")
        return json.loads(rawjson).keys()
    except Exception:
        print_and_log(f"Could not retrieve IOC list: {traceback.format_exc()}", "MAJOR")
        return []
<|endoftext|>


<commit_before># coding: utf8

# Copyright 2013-2015 Vincent Jacques <vincent@vincent-jacques.net>

import optparse

import cairo

import turkshead

parser = optparse.OptionParser()
parser.add_option("--width", type="int", default=800, help="Width")
parser.add_option("--height", type="int", default=600, help="Height")
parser.add_option("--bights", type="int", default=4, help="Bights")
parser.add_option("--leads", type="int", default=3, help="Leads")
parser.add_option("--radius-variation", type="int", default=200, help="Radius variation")
parser.add_option("--line-width", type="int", default=50, help="Radius variation")
parser.add_option("--output", type="string", default="turkshead.png", help="Output file")
options, arguments = parser.parse_args()

width = options.width
height = options.height
outerRadius = min(width, height) / 2 - 10
innerRadius = outerRadius - options.radius_variation

img = cairo.ImageSurface(cairo.FORMAT_RGB24, width, height)
ctx = cairo.Context(img)
ctx.set_source_rgb(1, 1, 0xBF / 255.)
ctx.paint()
ctx.translate(width / 2, height / 2)
ctx.scale(1, -1)

turkshead.TurksHead(options.leads, options.bights, innerRadius, outerRadius, options.line_width).draw(ctx)

img.write_to_png(options.output)
<commit_msg>Fix import<commit_after># coding: utf8

# Copyright 2013-2015 Vincent Jacques <vincent@vincent-jacques.net>

import optparse

import cairo

from . import TurksHead

parser = optparse.OptionParser()
parser.add_option("--width", type="int", default=800, help="Width")
parser.add_option("--height", type="int", default=600, help="Height")
parser.add_option("--bights", type="int", default=4, help="Bights")
parser.add_option("--leads", type="int", default=3, help="Leads")
parser.add_option("--radius-variation", type="int", default=200, help="Radius variation")
parser.add_option("--line-width", type="int", default=50, help="Radius variation")
parser.add_option("--output", type="string", default="turkshead.png", help="Output file")
options, arguments = parser.parse_args()

width = options.width
height = options.height
outerRadius = min(width, height) / 2 - 10
innerRadius = outerRadius - options.radius_variation

img = cairo.ImageSurface(cairo.FORMAT_RGB24, width, height)
ctx = cairo.Context(img)
ctx.set_source_rgb(1, 1, 0xBF / 255.)
ctx.paint()
ctx.translate(width / 2, height / 2)
ctx.scale(1, -1)

TurksHead(options.leads, options.bights, innerRadius, outerRadius, options.line_width).draw(ctx)

img.write_to_png(options.output)
<|endoftext|>


<commit_before>from enum import Enum

class InstructType(Enum):
    # Basic instruct
    ALLOCATE = "alloca"
    STORE = "store"
    LOAD = "load"
    RETURN = "ret"

    # Operator
    ADD = "add"
    SUB = "sub"
    MUL = "mul"
    DIV = "sdiv"

    # Support big number
    SEXT = "sext"
    TRUNC = "trunc"

    # Piecewise
    LABEL = "preds"  # This is very dangerous since this key word may be used any where
    BR = "br"
    CMP = "icmp"

class Instruction(object):
    def refresh_variable(self, variable_map):
        None

class InstructionFactory(object):
    @staticmethod
    def parse(instruct):
        # Purify the instruct line
        instruct = instruct.strip()
        # Remove debug info
        try:
            last_comma_index = instruct.rindex("!dbg")
            instruct = instruct[:last_comma_index]
        except ValueError:
            # Nope, it is ok to have no comma
            None
        # Remove excess commas
        instruct = instruct.replace(",", "")

        parts = instruct.strip().split()
        for i in InstructType:
            if i.value in parts:
                if i is InstructType.ALLOCATE:
                    return AllocateInstruction(parts)
                elif i is InstructType.LOAD:
                    return LoadInstruction(parts)
                elif i is InstructType.STORE:
                    return StoreInstruction(parts)
                elif i is InstructType.RETURN:
                    return ReturnInstruction(parts)
                elif i is InstructType.ADD:
                    return ADDInstruction(parts)
                elif i is InstructType.SUB:
                    return SUBInstruction(parts)
                elif i is InstructType.MUL:
                    return MULInstruction(parts)
                elif i is InstructType.DIV:
                    return DIVInstruction(parts)
                elif i is InstructType.SEXT:
                    return SEXTInstruction(parts)
                elif i is InstructType.TRUNC:
                    return TRUNCInstruction(parts)
                elif i is InstructType.LABEL:
                    return LABELInstruction(parts)
                elif i is InstructType.BR:
                    return BRInstruction(parts)
                elif i is InstructType.CMP:
                    return CMPInstruction(parts)

class AllocateInstruction(Instruction):
    # Example: %1 = alloca i32 align 4

    def __init__(self, values):
        self.target = values[0]
        self.type = values[3]
        self.align_width = values[-1]

    def refresh_variable(self, variable_map):
        variable_map[self.target] = self.target

    def __str__(self):
        description = "allocate " + self.target
        return description

class StoreInstruction(Instruction):
    # Example: store i32 %x i32* %1 align 4

    def __init__(self, values):
        self.source = values[2]
        self.source_type = values[1]
        self.target = values[4]
        self.target_type = values[3]
        self.align_width = values[-1]

    def refresh_variable(self, variable_map):
        r = self.source if self.source not in variable_map else variable_map[self.source]
        # variable_map[self.target] = variable_map[self.target].replace(self.target, r)
        # TODO recheck if it is safe
        variable_map[self.target] = r

    def __str__(self):
        description = "store " + self.source + " to " + self.target
        return description

class LoadInstruction(Instruction):
    # Example: %2 = load i32* %1 align 4

    def __init__(self, values):
        self.target = values[0]
        self.source = values[4]
        self.source_type = values[3]
        self.aligh_width = values[-1]

    def refresh_variable(self, variable_map):
        variable_map[self.target] = variable_map[self.source]

    def __str__(self):
        description = "load " + self.target + " from " + self.source
        return description

class ReturnInstruction(Instruction):
    # Example: ret i32 %4

    def __init__(self, values):
        self.target = values[-1]
        self.type = values[1]
        self.result = ""

    def refresh_variable(self, variable_map):
        description = "return " + self.target
        expression = variable_map[self.target].replace("%", "")
        if expression[0] is "(" and expression[-1] is ")":
            expression = expression[1:-1]
        self.result = "y = " + expression

    def __str__(self):
        return self.result

class ADDInstruction(Instruction):
    #  Example: %3 = add nsw i32 %2 10

    def __init__(self, values):
        self.target = values[0]
        self.left = values[-2]
        self.right = values[-1]

    def refresh_variable(self, variable_map):
        l = self.left if self.left not in variable_map else variable_map[self.left]
        r = self.right if self.right not in variable_map else variable_map[self.right]
        variable_map[self.target] = "(" + l + "+" + r + ")"

    def __str__(self):
        description = "(" + self.left + "+" + self.right + ")"
        return description

class SUBInstruction(Instruction):
    # Example: %3 = sub nsw i32 %2 12

    def __init__(self, values):
        self.target = values[0]
        self.left = values[-2]
        self.right = values[-1]

    def refresh_variable(self, variable_map):
        l = self.left if self.left not in variable_map else variable_map[self.left]
        r = self.right if self.right not in variable_map else variable_map[self.right]
        variable_map[self.target] = "(" + l + "-" + r + ")"

    def __str__(self):
        description = "(" + self.left + "-" + self.right + ")"
        return description

class MULInstruction(Instruction):
    # Example:   %4 = mul nsw i64 %3 123333333333

    def __init__(self, values):
        self.target = values[0]
        self.left = values[-2]
        self.right = values[-1]

    def refresh_variable(self, variable_map):
        l = self.left if self.left not in variable_map else variable_map[self.left]
        r = self.right if self.right not in variable_map else variable_map[self.right]
        variable_map[self.target] = l + "*" + r

    def __str__(self):
        description = "(" + self.left + "*" + self.right + ")"
        return description

class DIVInstruction(Instruction):
    # Example: %3 = sdiv i32 %2 12

    def __init__(self, values):
        self.target = values[0]
        self.left = values[-2]
        self.right = values[-1]

    def refresh_variable(self, variable_map):
        l = self.left if self.left not in variable_map else variable_map[self.left]
        r = self.right if self.right not in variable_map else variable_map[self.right]
        variable_map[self.target] = "(" + l + "/" + r + ")"

    def __str__(self):
        description = "(" + self.left + "/" + self.right + ")"
        return description

class SEXTInstruction(Instruction):
    # Example: %3 = sext i32 %2 to i64

    def __init__(self, values):
        self.target = values[0]
        self.source = values[4]

    def refresh_variable(self, variable_map):
        variable_map[self.target] = variable_map[self.source]

    def __str__(self):
        description = "sext from " + self.source + " to " + self.target
        return description

class TRUNCInstruction(Instruction):
    # Example: %5 = trunc i64 %4 to i32

    def __init__(self, values):
        self.target = values[0]
        self.source = values[4]

    def refresh_variable(self, variable_map):
        variable_map[self.target] = variable_map[self.source]

    def __str__(self):
        description = "truncate from " + self.source + " to " + self.target
        return description

class LABELInstruction(Instruction):
    # Example: ; <label>:11                                      ; preds = %9, %7

    def __init__(self, values):
        self.num = 0
        self.preds = []

        self.num = values[1].split(":")[-1]
        for i in range(values.index("=") + 1, len(values)):
            self.preds.append(values[i][1:])

    def __str__(self):
        description = "label " + self.num + " with preds " + ",".join(self.preds)
        return description

class BRInstruction(Instruction):
    # Example1: br i1 %5 label %7 label %9
    # Example2: br label %11

    # The shortest br instruction is like "br label num"
    SHORT_LENGTH = 3

    # If the br instruction just for jumping to a label
    def isdirect(self):
        return self.is_direct

    def __init__(self, values):
        if len(values) == self.SHORT_LENGTH:
            self.is_direct = True
            self.target = values[-1][1:]
        else:
            self.is_direct = False
            self.cmp = values[2]
            self.first_label = values[4][1:]
            self.second_label = values[-1][1:]

    def refresh_variable(self, variable_map):
        if not self.isdirect():
            self.cmp = self.cmp if self.cmp not in variable_map else variable_map[self.cmp].replace("%", "")

    def __str__(self):
        if self.isdirect():
            description = "this is  a direct br instruction to label " + self.target
        else:
            description = "this is a br instruction to label " + self.first_label + " or label " + self.second_label + " based on " + self.cmp
        return description

CMP_translate_map = {"slt": "<", "sgt": ">","sle":"<=","sge":">=","eq":"==","ne":"!="}

class CMPInstruction(Instruction):
    # Example: %5 = icmp slt i32 %4 %5

    def __init__(self, values):
        self.target = values[0]
        self.type = values[3]
        self.left = values[-2]
        self.right = values[-1]

    def refresh_variable(self, variable_map):
        self.left = self.left if self.left not in variable_map else variable_map[self.left]
        self.right = self.right if self.right not in variable_map else variable_map[self.right]

        variable_map[self.target] = self.left + CMP_translate_map[self.type] + self.right

    def __str__(self):
        description = " ".join(["check if", self.left, CMP_translate_map[self.type], self.right]).replace("%", "")
        return description
<commit_msg>Add bit operator support.<commit_after>from enum import Enum

class InstructType(Enum):
    # Basic instruct
    ALLOCATE = "alloca"
    STORE = "store"
    LOAD = "load"
    RETURN = "ret"

    # Operator
    ADD = "add"
    SUB = "sub"
    MUL = "mul"
    DIV = "sdiv"
    ASHR = "ashr"
    SHL = "shl"

    # Support big number
    SEXT = "sext"
    TRUNC = "trunc"

    # Piecewise
    LABEL = "preds"  # This is very dangerous since this key word may be used any where
    BR = "br"
    CMP = "icmp"

class Instruction(object):
    def refresh_variable(self, variable_map):
        None

class InstructionFactory(object):
    @staticmethod
    def parse(instruct):
        # Purify the instruct line
        instruct = instruct.strip()
        # Remove debug info
        try:
            last_comma_index = instruct.rindex("!dbg")
            instruct = instruct[:last_comma_index]
        except ValueError:
            # Nope, it is ok to have no comma
            None
        # Remove excess commas
        instruct = instruct.replace(",", "")

        parts = instruct.strip().split()
        for i in InstructType:
            if i.value in parts:
                if i is InstructType.ALLOCATE:
                    return AllocateInstruction(parts)
                elif i is InstructType.LOAD:
                    return LoadInstruction(parts)
                elif i is InstructType.STORE:
                    return StoreInstruction(parts)
                elif i is InstructType.RETURN:
                    return ReturnInstruction(parts)
                elif i is InstructType.ADD:
                    return ADDInstruction(parts)
                elif i is InstructType.SUB:
                    return SUBInstruction(parts)
                elif i is InstructType.MUL:
                    return MULInstruction(parts)
                elif i is InstructType.DIV:
                    return DIVInstruction(parts)
                elif i is InstructType.SEXT:
                    return SEXTInstruction(parts)
                elif i is InstructType.TRUNC:
                    return TRUNCInstruction(parts)
                elif i is InstructType.LABEL:
                    return LABELInstruction(parts)
                elif i is InstructType.BR:
                    return BRInstruction(parts)
                elif i is InstructType.CMP:
                    return CMPInstruction(parts)
                elif i is InstructType.SHL:
                    return SHLInstruction(parts)
                elif i is InstructType.ASHR:
                    return ASHRInstruction(parts)

class AllocateInstruction(Instruction):
    # Example: %1 = alloca i32 align 4

    def __init__(self, values):
        self.target = values[0]
        self.type = values[3]
        self.align_width = values[-1]

    def refresh_variable(self, variable_map):
        variable_map[self.target] = self.target

    def __str__(self):
        description = "allocate " + self.target
        return description

class StoreInstruction(Instruction):
    # Example: store i32 %x i32* %1 align 4

    def __init__(self, values):
        self.source = values[2]
        self.source_type = values[1]
        self.target = values[4]
        self.target_type = values[3]
        self.align_width = values[-1]

    def refresh_variable(self, variable_map):
        r = self.source if self.source not in variable_map else variable_map[self.source]
        # variable_map[self.target] = variable_map[self.target].replace(self.target, r)
        # TODO recheck if it is safe
        variable_map[self.target] = r

    def __str__(self):
        description = "store " + self.source + " to " + self.target
        return description

class LoadInstruction(Instruction):
    # Example: %2 = load i32* %1 align 4

    def __init__(self, values):
        self.target = values[0]
        self.source = values[4]
        self.source_type = values[3]
        self.aligh_width = values[-1]

    def refresh_variable(self, variable_map):
        variable_map[self.target] = variable_map[self.source]

    def __str__(self):
        description = "load " + self.target + " from " + self.source
        return description

class ReturnInstruction(Instruction):
    # Example: ret i32 %4

    def __init__(self, values):
        self.target = values[-1]
        self.type = values[1]
        self.result = ""

    def refresh_variable(self, variable_map):
        description = "return " + self.target
        expression = variable_map[self.target].replace("%", "")
        if expression[0] is "(" and expression[-1] is ")":
            expression = expression[1:-1]
        self.result = "y = " + expression

    def __str__(self):
        return self.result

class ADDInstruction(Instruction):
    #  Example: %3 = add nsw i32 %2 10

    def __init__(self, values):
        self.target = values[0]
        self.left = values[-2]
        self.right = values[-1]

    def refresh_variable(self, variable_map):
        l = self.left if self.left not in variable_map else variable_map[self.left]
        r = self.right if self.right not in variable_map else variable_map[self.right]
        variable_map[self.target] = "(" + l + "+" + r + ")"

    def __str__(self):
        description = "(" + self.left + "+" + self.right + ")"
        return description

class SUBInstruction(Instruction):
    # Example: %3 = sub nsw i32 %2 12

    def __init__(self, values):
        self.target = values[0]
        self.left = values[-2]
        self.right = values[-1]

    def refresh_variable(self, variable_map):
        l = self.left if self.left not in variable_map else variable_map[self.left]
        r = self.right if self.right not in variable_map else variable_map[self.right]
        variable_map[self.target] = "(" + l + "-" + r + ")"

    def __str__(self):
        description = "(" + self.left + "-" + self.right + ")"
        return description

class MULInstruction(Instruction):
    # Example:   %4 = mul nsw i64 %3 123333333333

    def __init__(self, values):
        self.target = values[0]
        self.left = values[-2]
        self.right = values[-1]

    def refresh_variable(self, variable_map):
        l = self.left if self.left not in variable_map else variable_map[self.left]
        r = self.right if self.right not in variable_map else variable_map[self.right]
        variable_map[self.target] = l + "*" + r

    def __str__(self):
        description = "(" + self.left + "*" + self.right + ")"
        return description

class DIVInstruction(Instruction):
    # Example: %3 = sdiv i32 %2 12

    def __init__(self, values):
        self.target = values[0]
        self.left = values[-2]
        self.right = values[-1]

    def refresh_variable(self, variable_map):
        l = self.left if self.left not in variable_map else variable_map[self.left]
        r = self.right if self.right not in variable_map else variable_map[self.right]
        variable_map[self.target] = "(" + l + "/" + r + ")"

    def __str__(self):
        description = "(" + self.left + "/" + self.right + ")"
        return description

class SEXTInstruction(Instruction):
    # Example: %3 = sext i32 %2 to i64

    def __init__(self, values):
        self.target = values[0]
        self.source = values[4]

    def refresh_variable(self, variable_map):
        variable_map[self.target] = variable_map[self.source]

    def __str__(self):
        description = "sext from " + self.source + " to " + self.target
        return description

class TRUNCInstruction(Instruction):
    # Example: %5 = trunc i64 %4 to i32

    def __init__(self, values):
        self.target = values[0]
        self.source = values[4]

    def refresh_variable(self, variable_map):
        variable_map[self.target] = variable_map[self.source]

    def __str__(self):
        description = "truncate from " + self.source + " to " + self.target
        return description

class LABELInstruction(Instruction):
    # Example: ; <label>:11                                      ; preds = %9, %7

    def __init__(self, values):
        self.num = 0
        self.preds = []

        self.num = values[1].split(":")[-1]
        for i in range(values.index("=") + 1, len(values)):
            self.preds.append(values[i][1:])

    def __str__(self):
        description = "label " + self.num + " with preds " + ",".join(self.preds)
        return description

class BRInstruction(Instruction):
    # Example1: br i1 %5 label %7 label %9
    # Example2: br label %11

    # The shortest br instruction is like "br label num"
    SHORT_LENGTH = 3

    # If the br instruction just for jumping to a label
    def isdirect(self):
        return self.is_direct

    def __init__(self, values):
        if len(values) == self.SHORT_LENGTH:
            self.is_direct = True
            self.target = values[-1][1:]
        else:
            self.is_direct = False
            self.cmp = values[2]
            self.first_label = values[4][1:]
            self.second_label = values[-1][1:]

    def refresh_variable(self, variable_map):
        if not self.isdirect():
            self.cmp = self.cmp if self.cmp not in variable_map else variable_map[self.cmp].replace("%", "")

    def __str__(self):
        if self.isdirect():
            description = "this is  a direct br instruction to label " + self.target
        else:
            description = "this is a br instruction to label " + self.first_label + " or label " + self.second_label + " based on " + self.cmp
        return description

CMP_translate_map = {"slt": "<", "sgt": ">", "sle": "<=", "sge": ">=", "eq": "==", "ne": "!="}

class CMPInstruction(Instruction):
    # Example: %5 = icmp slt i32 %4 %5

    def __init__(self, values):
        self.target = values[0]
        self.type = values[3]
        self.left = values[-2]
        self.right = values[-1]

    def refresh_variable(self, variable_map):
        self.left = self.left if self.left not in variable_map else variable_map[self.left]
        self.right = self.right if self.right not in variable_map else variable_map[self.right]

        variable_map[self.target] = self.left + CMP_translate_map[self.type] + self.right

    def __str__(self):
        description = " ".join(["check if", self.left, CMP_translate_map[self.type], self.right]).replace("%", "")
        return description

class ASHRInstruction(Instruction):
    # Example: %3 = ashr i32 %2 5

    def __init__(self, values):
        self.target = values[0]
        self.left = values[-2]
        self.right = values[-1]

    def refresh_variable(self, variable_map):
        self.left = self.left if self.left not in variable_map else variable_map[self.left]
        self.right = self.right if self.right not in variable_map else variable_map[self.right]

        variable_map[self.target] = "(" + self.left + ">>" + self.right + ")"

    def __str__(self):
        description = self.left + ">>" + self.right
        return description

class SHLInstruction(Instruction):
    # Example: %8 = shl i32 %7 3

    def __init__(self, values):
        self.target = values[0]
        self.left = values[-2]
        self.right = values[-1]

    def refresh_variable(self, variable_map):
        self.left = self.left if self.left not in variable_map else variable_map[self.left]
        self.right = self.right if self.right not in variable_map else variable_map[self.right]

        variable_map[self.target] = "(" + self.left + "<<" + self.right + ")"

    def __str__(self):
        description = self.left + "<<" + self.right
        return description
<|endoftext|>


<commit_before>import numpy as np

assoc_name = ''
prefix = 'tweak_galpy_potential_test'
config = {
    # 'datafile':'',
    'results_dir':'../results/{}'.format(prefix),
    'data_loadfile': '../data/synth_data_for_marusa_from_paper_1/same_centroid_synth_measurements.fits',
    'data_savefile': '../results/{}/same_centroid_synth_measurements_output.fits'.format(prefix), #,#''../results/{}/{}_subset.fit'.format(assoc_name, assoc_name), # Chronostar adds XYZUVW columns and
                                        # if you don't want to override the original file then save into data_savefile.
    'plot_it':True, # What kind of plots?
    # 'background_overlaps_file':'',
    'include_background_distribution':True,
    'kernel_density_input_datafile':'/home/tcrun/chronostar/data/gaia_cartesian_full_6d_table.fits', # absolute path as I ran out of space
                                                    # Cartesian data of all Gaia DR2 stars
                                                    # e.g. ../data/gaia_dr2_mean_xyzuvw.npy
    'run_with_mpi':False,       # not yet inpmlemented
    'convert_to_cartesian':True,        # whehter need to convert data from astrometry to cartesian
    'overwrite_datafile':False,         # whether to store results in same talbe and rewrite to file
    'cartesian_savefile':'',
    'save_cartesian_data':True,         #
    'overwrite_prev_run':True,          # explores provided results directorty and sees if results already
                                        # exist, and if so picks up from where left off
    'dummy_trace_orbit_function':False,  # For testing, simple function to skip computation
    'pickup_prev_run':True,             # Pick up where left off if possible

    'banyan_assoc_name': '',
}

# synth = None
synth = {
   # 'pars':np.array([
   #     [ 50., 0.,10., 0., 0., 3., 5., 2., 1e-10],
   #     [-50., 0.,20., 0., 5., 2., 5., 2., 1e-10],
   #     [  0.,50.,30., 0., 0., 1., 5., 2., 1e-10],
   # ]),
   # 'starcounts':[100,50,50]
}

data_bound = None

astro_colnames = {
    # 'main_colnames':None,     # list of names
    # 'error_colnames':None,
    # 'corr_colnames':None,
}

cart_colnames = {
    # 'main_colnames':None,
    # 'error_colnames':None,
    # 'corr_colnames':None,
}

from galpy.potential import MiyamotoNagaiPotential, MWPotential2014
orbit = {
    # 'potential': None,
    'potential': MWPotential2014,
    # 'potential': MiyamotoNagaiPotential(a=0.5,b=0.0375,amp=1.,normalize=1.), # Params from the example webpage. No idea if that's good or not.
}

special = {
    'component':'sphere',       # parameterisation for the origin
}

advanced = {
    'burnin_steps':1000,        # emcee parameters, number of steps for each burnin iteraton
    'sampling_steps':1000,
}
<commit_msg>prefix change<commit_after>import numpy as np

assoc_name = ''
prefix = 'tweak_galpy_potential_friday'
config = {
    # 'datafile':'',
    'results_dir':'../results/{}'.format(prefix),
    'data_loadfile': '../data/synth_data_for_marusa_from_paper_1/same_centroid_synth_measurements.fits',
    'data_savefile': '../results/{}/same_centroid_synth_measurements_output.fits'.format(prefix), #,#''../results/{}/{}_subset.fit'.format(assoc_name, assoc_name), # Chronostar adds XYZUVW columns and
                                        # if you don't want to override the original file then save into data_savefile.
    'plot_it':True, # What kind of plots?
    # 'background_overlaps_file':'',
    'include_background_distribution':True,
    'kernel_density_input_datafile':'/home/tcrun/chronostar/data/gaia_cartesian_full_6d_table.fits', # absolute path as I ran out of space
                                                    # Cartesian data of all Gaia DR2 stars
                                                    # e.g. ../data/gaia_dr2_mean_xyzuvw.npy
    'run_with_mpi':False,       # not yet inpmlemented
    'convert_to_cartesian':True,        # whehter need to convert data from astrometry to cartesian
    'overwrite_datafile':False,         # whether to store results in same talbe and rewrite to file
    'cartesian_savefile':'',
    'save_cartesian_data':True,         #
    'overwrite_prev_run':True,          # explores provided results directorty and sees if results already
                                        # exist, and if so picks up from where left off
    'dummy_trace_orbit_function':False,  # For testing, simple function to skip computation
    'pickup_prev_run':True,             # Pick up where left off if possible

    'banyan_assoc_name': '',
}

# synth = None
synth = {
   # 'pars':np.array([
   #     [ 50., 0.,10., 0., 0., 3., 5., 2., 1e-10],
   #     [-50., 0.,20., 0., 5., 2., 5., 2., 1e-10],
   #     [  0.,50.,30., 0., 0., 1., 5., 2., 1e-10],
   # ]),
   # 'starcounts':[100,50,50]
}

data_bound = None

astro_colnames = {
    # 'main_colnames':None,     # list of names
    # 'error_colnames':None,
    # 'corr_colnames':None,
}

cart_colnames = {
    # 'main_colnames':None,
    # 'error_colnames':None,
    # 'corr_colnames':None,
}

from galpy.potential import MiyamotoNagaiPotential, MWPotential2014
orbit = {
    # 'potential': None,
    'potential': MWPotential2014,
    # 'potential': MiyamotoNagaiPotential(a=0.5,b=0.0375,amp=1.,normalize=1.), # Params from the example webpage. No idea if that's good or not.
}

special = {
    'component':'sphere',       # parameterisation for the origin
}

advanced = {
    'burnin_steps':1000,        # emcee parameters, number of steps for each burnin iteraton
    'sampling_steps':1000,
}
<|endoftext|>


<commit_before>import logging
logger = logging.getLogger("Kafka")

import os
from serverbase import BaseServer

import time #Allows timeout for connection
from kafka import KafkaClient

class Kafka(BaseServer):
    """
    Given a folder in which a database is/should be located, it starts a kafka server rooted at that location,
    and connects to it. Once close is called, it closes the connection and kills the server.
    """

    def __init__(self,chost,hostname,port=None,dbpath="./kafka",jardir="./bin/jar/"):
        BaseServer.__init__(self,"kafka",chost,dbpath,logger,hostname,port)
        self.connect()

        #Create base path for the kafka server (it uses zookeeper)
        self.connection.zoo.ensure_path("/kafka")

        #Create the command line
        cmd = ["java","-Xmx256M","-server","-XX:+UseCompressedOops","-XX:+UseParNewGC",
               "-XX:+UseConcMarkSweepGC","-XX:+CMSClassUnloadingEnabled","-XX:+CMSScavengeBeforeRemark",
               "-XX:+DisableExplicitGC","-Djava.awt.headless=true",
               "-Dlog4j.configuration=file:"+os.path.join(self.folder,"log4j.properties"),
                "-cp",self.classpath(jardir),"kafka.Kafka",os.path.join(self.folder,"server.properties")]

        logproperties = self.configDefaults["log4j.properties"]
        def addAppender(app):
            logproperties[app] = "org.apache.log4j.ConsoleAppender"
            logproperties[app+".layout"] = "org.apache.log4j.PatternLayout"
            logproperties[app+".layout.ConversionPattern"]="[%d] %p %m (%c)%n"
        addAppender("log4j.appender.kafkaAppender")
        addAppender("log4j.appender.stateChangeAppender")
        addAppender("log4j.appender.requestAppender")
        addAppender("log4j.appender.cleanerAppender")
        addAppender("log4j.appender.controllerAppender")

        self.addConfig({"cmd": cmd,"log4j.properties": logproperties,
            "server.properties": {
                    "broker.id": 0, #TODO: Set this to randint
                    "port": str(self.port),
                    "host.name": hostname,
                    "zookeeper.connect": chost+"/kafka", #The zookeeper host
                    "log.dirs":self.dbpath,
                    "zookeeper.connection.timeout.ms":"1000000",
                    #"num.network.threads": "2",
                    #"num.io.threads":"8",
                    "socket.send.buffer.bytes":"1048576",
                    "socket.receive.buffer.bytes":"1048576",
                    "socket.request.max.bytes":"104857600",
                    "num.partitions":"2",

                    # The minimum age of a log file to be eligible for deletion
                    "log.retention.hours":"168",

                    # A size-based retention policy for logs. Segments are pruned from the log as long as the remaining
                    # segments don't drop below log.retention.bytes.
                    #log.retention.bytes=1073741824

                    # The maximum size of a log segment file. When this size is reached a new log segment will be created.
                    "log.segment.bytes":"536870912",

                    # The interval at which log segments are checked to see if they can be deleted according
                    # to the retention policies
                    "log.retention.check.interval.ms":"60000",

                    # By default the log cleaner is disabled and the log retention policy will default to just delete segments after their retention expires.
                    # If log.cleaner.enable=true is set the cleaner will be enabled and individual logs can then be marked for log compaction.
                    "log.cleaner.enable":"false",

                    "auto.create.topics.enable":"true"
                }
                        })

        #Gets the configuration for the server
        self.addConfig(self.connection.config)

        self.writeConfig()
        self.runServer()

        #Starts the client - and gives it 2 minutes to figure out whether it is going to connect or not.
        #This is dependent on whether the database daemon is successfully starting up in the background.
        #The extremely long wait time is because some old laptops can take very long to create a database.
        #The wait time is not an issue, since we check if mongoDB crashed if we can't connect - so in effect
        #the actual wait time is at most a couple seconds if the database actually fails to start.
        t = time.time()
        while (time.time() - t < 120.0 and self.client is None):
            try:
                self.client = KafkaClient(self.host)
            except:
                time.sleep(1)
                #If the process crashed for some reason, don't continue waiting like an idiot
                if (self.server.poll() is not None):
                    logger.error("MongoDB did not start correctly.")
                    self.server = None
                    break
        if (self.client==None):
            self.close()
            raise Exception("Could not connect to database")

        #The server connection is no longer necessary
        self.client.close()
        self.client = None

        #Register the client as ready
        self.connection.registerme()

        logger.info("server running...")<commit_msg>Changed default partitions to 1<commit_after>import logging
logger = logging.getLogger("Kafka")

import os
from serverbase import BaseServer

import time #Allows timeout for connection
from kafka import KafkaClient

class Kafka(BaseServer):
    """
    Given a folder in which a database is/should be located, it starts a kafka server rooted at that location,
    and connects to it. Once close is called, it closes the connection and kills the server.
    """

    def __init__(self,chost,hostname,port=None,dbpath="./kafka",jardir="./bin/jar/"):
        BaseServer.__init__(self,"kafka",chost,dbpath,logger,hostname,port)
        self.connect()

        #Create base path for the kafka server (it uses zookeeper)
        self.connection.zoo.ensure_path("/kafka")

        #Create the command line
        cmd = ["java","-Xmx256M","-server","-XX:+UseCompressedOops","-XX:+UseParNewGC",
               "-XX:+UseConcMarkSweepGC","-XX:+CMSClassUnloadingEnabled","-XX:+CMSScavengeBeforeRemark",
               "-XX:+DisableExplicitGC","-Djava.awt.headless=true",
               "-Dlog4j.configuration=file:"+os.path.join(self.folder,"log4j.properties"),
                "-cp",self.classpath(jardir),"kafka.Kafka",os.path.join(self.folder,"server.properties")]

        logproperties = self.configDefaults["log4j.properties"]
        def addAppender(app):
            logproperties[app] = "org.apache.log4j.ConsoleAppender"
            logproperties[app+".layout"] = "org.apache.log4j.PatternLayout"
            logproperties[app+".layout.ConversionPattern"]="[%d] %p %m (%c)%n"
        addAppender("log4j.appender.kafkaAppender")
        addAppender("log4j.appender.stateChangeAppender")
        addAppender("log4j.appender.requestAppender")
        addAppender("log4j.appender.cleanerAppender")
        addAppender("log4j.appender.controllerAppender")

        self.addConfig({"cmd": cmd,"log4j.properties": logproperties,
            "server.properties": {
                    "broker.id": 0, #TODO: Set this to randint
                    "port": str(self.port),
                    "host.name": hostname,
                    "zookeeper.connect": chost+"/kafka", #The zookeeper host
                    "log.dirs":self.dbpath,
                    "zookeeper.connection.timeout.ms":"1000000",
                    #"num.network.threads": "2",
                    #"num.io.threads":"8",
                    "socket.send.buffer.bytes":"1048576",
                    "socket.receive.buffer.bytes":"1048576",
                    "socket.request.max.bytes":"104857600",
                    "num.partitions":"1",

                    # The minimum age of a log file to be eligible for deletion
                    "log.retention.hours":"168",

                    # A size-based retention policy for logs. Segments are pruned from the log as long as the remaining
                    # segments don't drop below log.retention.bytes.
                    #log.retention.bytes=1073741824

                    # The maximum size of a log segment file. When this size is reached a new log segment will be created.
                    "log.segment.bytes":"536870912",

                    # The interval at which log segments are checked to see if they can be deleted according
                    # to the retention policies
                    "log.retention.check.interval.ms":"60000",

                    # By default the log cleaner is disabled and the log retention policy will default to just delete segments after their retention expires.
                    # If log.cleaner.enable=true is set the cleaner will be enabled and individual logs can then be marked for log compaction.
                    "log.cleaner.enable":"false",

                    "auto.create.topics.enable":"true"
                }
                        })

        #Gets the configuration for the server
        self.addConfig(self.connection.config)

        self.writeConfig()
        self.runServer()

        #Starts the client - and gives it 2 minutes to figure out whether it is going to connect or not.
        #This is dependent on whether the database daemon is successfully starting up in the background.
        #The extremely long wait time is because some old laptops can take very long to create a database.
        #The wait time is not an issue, since we check if mongoDB crashed if we can't connect - so in effect
        #the actual wait time is at most a couple seconds if the database actually fails to start.
        t = time.time()
        while (time.time() - t < 120.0 and self.client is None):
            try:
                self.client = KafkaClient(self.host)
            except:
                time.sleep(1)
                #If the process crashed for some reason, don't continue waiting like an idiot
                if (self.server.poll() is not None):
                    logger.error("MongoDB did not start correctly.")
                    self.server = None
                    break
        if (self.client==None):
            self.close()
            raise Exception("Could not connect to database")

        #The server connection is no longer necessary
        self.client.close()
        self.client = None

        #Register the client as ready
        self.connection.registerme()

        logger.info("server running...")<|endoftext|>


<commit_before>import os, sublime, sublime_plugin

class ConEmuOpenCommand():
    def get_project(self):
        has_project = 0
        if self.window.project_file_name(): # try project file name first
            project_dir, project_name = os.path.split(self.window.project_file_name())
            project_name, ext = os.path.splitext('st:' + project_name)
            project_dir = self.window.folders()[0]
            has_project = 1
        elif self.window.folders(): # if no project, use the top folder
            project_dir = self.window.folders()[0]
            project_name = project_dir
        elif self.window.active_view().file_name(): # if no folder, use current file's folder
            project_dir, _ = os.path.split(self.window.active_view().file_name())
            project_name = project_dir
        else: #then exit
            return (None, None, 0)
        return (project_name, project_dir, has_project)

    def open_conemu(self, dirname, title):
        command= "start conemu.exe /Single /Dir \""+dirname+"\" /cmdlist cmd -new_console:t:\""+title+"\""
        os.system(command)

# open project folder
class OpenConemuProjectCommand(sublime_plugin.WindowCommand, ConEmuOpenCommand):
    def run(self):
        project_name, project_dir, has_project = self.get_project()
        if not project_name:
            return

        self.open_conemu(project_dir, project_name)

# open "here" folder
class OpenConemuHereCommand(sublime_plugin.WindowCommand, ConEmuOpenCommand):
    def run(self, paths=[]):
        project_name, project_dir, has_project = self.get_project()
        if not project_name:
            return

        if paths:
            heredir = paths[0]
            if os.path.isfile(heredir):
                heredir = os.path.dirname(heredir)
        elif self.window.active_view().file_name():
            heredir = os.path.dirname(self.window.active_view().file_name())
        else: # if no active file open, then try to open project folder
            heredir = project_dir

        # get the tab title
        title = heredir
        if has_project:
            rel_path = os.path.relpath(heredir, project_dir)
            if rel_path == '.':
                title = project_name
            elif rel_path[:2] == '..': # not in project, show the whole path
                title = heredir
            else:
                title = project_name + ': ' + rel_path

        self.open_conemu(heredir, title)<commit_msg>Change cmd to powershell<commit_after>import os, sublime, sublime_plugin

class ConEmuOpenCommand():
    def get_project(self):
        has_project = 0
        if self.window.project_file_name(): # try project file name first
            project_dir, project_name = os.path.split(self.window.project_file_name())
            project_name, ext = os.path.splitext('st:' + project_name)
            project_dir = self.window.folders()[0]
            has_project = 1
        elif self.window.folders(): # if no project, use the top folder
            project_dir = self.window.folders()[0]
            project_name = project_dir
        elif self.window.active_view().file_name(): # if no folder, use current file's folder
            project_dir, _ = os.path.split(self.window.active_view().file_name())
            project_name = project_dir
        else: #then exit
            return (None, None, 0)
        return (project_name, project_dir, has_project)

    def open_conemu(self, dirname, title):
        command= "start conemu.exe /Single /Dir \""+dirname+"\" /cmdlist powershell -new_console:t:\""+title+"\""
        os.system(command)

# open project folder
class OpenConemuProjectCommand(sublime_plugin.WindowCommand, ConEmuOpenCommand):
    def run(self):
        project_name, project_dir, has_project = self.get_project()
        if not project_name:
            return

        self.open_conemu(project_dir, project_name)

# open "here" folder
class OpenConemuHereCommand(sublime_plugin.WindowCommand, ConEmuOpenCommand):
    def run(self, paths=[]):
        project_name, project_dir, has_project = self.get_project()
        if not project_name:
            return

        if paths:
            heredir = paths[0]
            if os.path.isfile(heredir):
                heredir = os.path.dirname(heredir)
        elif self.window.active_view().file_name():
            heredir = os.path.dirname(self.window.active_view().file_name())
        else: # if no active file open, then try to open project folder
            heredir = project_dir

        # get the tab title
        title = heredir
        if has_project:
            rel_path = os.path.relpath(heredir, project_dir)
            if rel_path == '.':
                title = project_name
            elif rel_path[:2] == '..': # not in project, show the whole path
                title = heredir
            else:
                title = project_name + ': ' + rel_path

        self.open_conemu(heredir, title)<|endoftext|>


<commit_before>from subprocess import call
import sys
import os
from PIL import Image
import numpy as np
from scipy import ndimage
from subprocess import call
import sys
import os
from PIL import Image
from datetime import datetime
import cv2

from object_detection.utils import visualization_utils as vis_util
from object_detection.utils import label_map_util

# intiate tensorflow and load custom model
import tensorflow as tf
import numpy as np

CWD_PATH = os.getcwd()

# Path to frozen detection graph. This is the actual model that is used for the object detection.
PATH_TO_CKPT = '<CONFIGURE_YOUR_PATH_HERE>_output_inference_graph_21.pb/frozen_inference_graph.pb'

# List of the strings that is used to add correct label for each box.
PATH_TO_LABELS = '<CONFIGURE_YOUR_PATH_HERE>e/data/label_map.pbtxt'

NUM_CLASSES = 5

# Loading label map
label_map = label_map_util.load_labelmap(PATH_TO_LABELS)
categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES,
                                                            use_display_name=True)
category_index = label_map_util.create_category_index(categories)

detection_graph = tf.Graph()
with detection_graph.as_default():
    od_graph_def = tf.GraphDef()
    with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid:
        serialized_graph = fid.read()
        od_graph_def.ParseFromString(serialized_graph)
        tf.import_graph_def(od_graph_def, name='')

def detect_objects(image_np, sess, detection_graph):
    # Expand dimensions since the model expects images to have shape: [1, None, None, 3]
    image_np_expanded = np.expand_dims(image_np, axis=0)
    image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')

    # Each box represents a part of the image where a particular object was detected.
    boxes = detection_graph.get_tensor_by_name('detection_boxes:0')

    # Each score represent how level of confidence for each of the objects.
    # Score is shown on the result image, together with the class label.
    scores = detection_graph.get_tensor_by_name('detection_scores:0')
    classes = detection_graph.get_tensor_by_name('detection_classes:0')
    num_detections = detection_graph.get_tensor_by_name('num_detections:0')

    # Actual detection.
    (boxes, scores, classes, num_detections) = sess.run(
        [boxes, scores, classes, num_detections],
        feed_dict={image_tensor: image_np_expanded})

    # the 0.8 is the confidence threshold. This script sorts the detected classes in descending confidence level and the code below checks to see if any of the top five detected objects match our target class with a threshold confidence greater than 80%
    if ((np.squeeze(scores)[0] > 0.8) and (np.squeeze(classes)[0] == 1)) \
            or ((np.squeeze(scores)[1] > 0.4) and (np.squeeze(classes)[1] == 1)) \
            or ((np.squeeze(scores)[2] > 0.4) and (np.squeeze(classes)[2] == 1)) \
            or ((np.squeeze(scores)[3] > 0.4) and (np.squeeze(classes)[3] == 1)) \
            or ((np.squeeze(scores)[4] > 0.4) and (np.squeeze(classes)[4] == 1)):

        # Visualization of the results of a detection.
        vis_util.visualize_boxes_and_labels_on_image_array(
            image_np,
            np.squeeze(boxes),
            np.squeeze(classes).astype(np.int32),
            np.squeeze(scores),
            category_index,
            use_normalized_coordinates=True,
            line_thickness=8)

        img = Image.fromarray(image_np, 'RGB')
        # print(os.path.splitext(segmentFileName)[0])
        img.save("<CONFIGURE_PATH_TO_SAVING_THE_IMAGE_SEGMENTS_WITH_BOUNDINGBOXES>" + "_detected.jpg")
        # img.show()

    pageHeight = 3300
    pageWidth = 5100

    cropHeight = int(pageHeight / 30)
    cropWidth = int(pageWidth / 50)

    pageIndex = 1

    with tf.Session(graph=detection_graph) as sess:
        startTime = datetime.now()

    imageNumpyArray = ndimage.imread("<CONFIGURE_PATH_TO_JPG_FILE_TO_CONDUCT_OBJECT_DETECTION_ON>")

    overlapWidth = 10

    #code below loads the jpg image into a single numpy array and extracts shorter segments to feed them individualy as input tensors into tensor_flow model

    segmentIndex = 0
    while segmentIndex <= 1499:
        if (segmentIndex == 0):
            cropArray = imageNumpyArray[0:cropHeight + overlapWidth, 0:cropWidth + overlapWidth, :]

        # catch top right corner tile
        elif (segmentIndex == 49):
            cropArray = imageNumpyArray[0:cropHeight + overlapWidth,
                    segmentIndex * cropWidth - overlapWidth:segmentIndex * cropWidth + cropWidth, :]

            # catch bottom left corner tile
        elif (segmentIndex == 1450):
            cropArray = imageNumpyArray[
                        cropHeight * segmentIndex // 50 - overlapWidth:cropHeight * segmentIndex // 50 + cropHeight,
                        0:cropWidth + overlapWidth, :]
            # catch bottom right corner tile
        elif (segmentIndex == 1499):
            cropArray = imageNumpyArray[
                        cropHeight * segmentIndex // 50 - cropHeight - overlapWidth:cropHeight * segmentIndex // 50,
                        segmentIndex % 50 * cropWidth - overlapWidth:segmentIndex % 50 * cropWidth + cropWidth, :]
            # catch right edge tiles so no overlap on left
        elif (segmentIndex % 50 == 0):
            # print(cropHeight*segmentIndex//50)
            cropArray = imageNumpyArray[cropHeight * (segmentIndex // 50):cropHeight * (segmentIndex // 50) + cropHeight,
                        0:cropWidth + overlapWidth, :]
             # catch top edge tiles so no overlap on top
        elif (segmentIndex <= 48):
            # print(segmentIndex*cropWidth)
            cropArray = imageNumpyArray[0:cropHeight + overlapWidth,
                        segmentIndex * cropWidth:segmentIndex * cropWidth + cropWidth, :]
            # catch left edge tiles so no overlap on left
        elif (segmentIndex + 1) % 50 == 0:
            # print(segmentIndex * cropWidth)
            cropArray = imageNumpyArray[((segmentIndex + 1) // 50) * cropHeight - overlapWidth:((
                                                                                segmentIndex + 1) // 50) * cropHeight + cropHeight + overlapWidth,
                        (segmentIndex) % 50 * cropWidth - overlapWidth:(segmentIndex) % 50 * cropWidth + cropWidth, :]
            # catch bottom edge tiles so no overlap on top
        elif (segmentIndex > 1450):
            # print(segmentIndex * cropWidth)
            cropArray = imageNumpyArray[((segmentIndex + 1) // 50) * cropHeight:((
                                                                                segmentIndex + 1) // 50) * cropHeight + cropHeight + overlapWidth,
                        (segmentIndex) % 50 * cropWidth - overlapWidth:(segmentIndex) % 50 * cropWidth + cropWidth, :]
         else:
            cropArray = imageNumpyArray[(segmentIndex // 50) * cropHeight - overlapWidth: (
                                                                                segmentIndex // 50) * cropHeight + cropHeight + overlapWidth,
                        (segmentIndex) % 50 * cropWidth - overlapWidth:(
                                                                       segmentIndex) % 50 * cropWidth + cropWidth + overlapWidth,
                        :]
        detect_objects(cropArray, sess, detection_graph)
        if segmentIndex % 150 == 0:
            print(str(segmentIndex // 150 * 10) + " percent complete")
    segmentIndex += 1

<commit_msg>Fix indentation in DetectPSVs.py<commit_after>from subprocess import call
import sys
import os
from PIL import Image
import numpy as np
from scipy import ndimage
from subprocess import call
import sys
import os
from PIL import Image
from datetime import datetime
import cv2

from object_detection.utils import visualization_utils as vis_util
from object_detection.utils import label_map_util

# intiate tensorflow and load custom model
import tensorflow as tf
import numpy as np

CWD_PATH = os.getcwd()

# Path to frozen detection graph. This is the actual model that is used for the object detection.
PATH_TO_CKPT = '<CONFIGURE_YOUR_PATH_HERE>_output_inference_graph_21.pb/frozen_inference_graph.pb'

# List of the strings that is used to add correct label for each box.
PATH_TO_LABELS = '<CONFIGURE_YOUR_PATH_HERE>e/data/label_map.pbtxt'

NUM_CLASSES = 5

# Loading label map
label_map = label_map_util.load_labelmap(PATH_TO_LABELS)
categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES,
                                                            use_display_name=True)
category_index = label_map_util.create_category_index(categories)

detection_graph = tf.Graph()
with detection_graph.as_default():
    od_graph_def = tf.GraphDef()
    with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid:
        serialized_graph = fid.read()
        od_graph_def.ParseFromString(serialized_graph)
        tf.import_graph_def(od_graph_def, name='')

def detect_objects(image_np, sess, detection_graph):
    # Expand dimensions since the model expects images to have shape: [1, None, None, 3]
    image_np_expanded = np.expand_dims(image_np, axis=0)
    image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')

    # Each box represents a part of the image where a particular object was detected.
    boxes = detection_graph.get_tensor_by_name('detection_boxes:0')

    # Each score represent how level of confidence for each of the objects.
    # Score is shown on the result image, together with the class label.
    scores = detection_graph.get_tensor_by_name('detection_scores:0')
    classes = detection_graph.get_tensor_by_name('detection_classes:0')
    num_detections = detection_graph.get_tensor_by_name('num_detections:0')

    # Actual detection.
    (boxes, scores, classes, num_detections) = sess.run(
        [boxes, scores, classes, num_detections],
        feed_dict={image_tensor: image_np_expanded})

    # the 0.8 is the confidence threshold. This script sorts the detected classes in descending confidence level and the code below checks to see if any of the top five detected objects match our target class with a threshold confidence greater than 80%
    if ((np.squeeze(scores)[0] > 0.8) and (np.squeeze(classes)[0] == 1)) \
            or ((np.squeeze(scores)[1] > 0.4) and (np.squeeze(classes)[1] == 1)) \
            or ((np.squeeze(scores)[2] > 0.4) and (np.squeeze(classes)[2] == 1)) \
            or ((np.squeeze(scores)[3] > 0.4) and (np.squeeze(classes)[3] == 1)) \
            or ((np.squeeze(scores)[4] > 0.4) and (np.squeeze(classes)[4] == 1)):

        # Visualization of the results of a detection.
        vis_util.visualize_boxes_and_labels_on_image_array(
            image_np,
            np.squeeze(boxes),
            np.squeeze(classes).astype(np.int32),
            np.squeeze(scores),
            category_index,
            use_normalized_coordinates=True,
            line_thickness=8)

        img = Image.fromarray(image_np, 'RGB')
        # print(os.path.splitext(segmentFileName)[0])
        img.save("<CONFIGURE_PATH_TO_SAVING_THE_IMAGE_SEGMENTS_WITH_BOUNDINGBOXES>" + "_detected.jpg")
        # img.show()

  pageHeight = 3300
  pageWidth = 5100

  cropHeight = int(pageHeight / 30)
  cropWidth = int(pageWidth / 50)

  pageIndex = 1

  with tf.Session(graph=detection_graph) as sess:
      startTime = datetime.now()

  imageNumpyArray = ndimage.imread("<CONFIGURE_PATH_TO_JPG_FILE_TO_CONDUCT_OBJECT_DETECTION_ON>")

  overlapWidth = 10

  #code below loads the jpg image into a single numpy array and extracts shorter segments to feed them individualy as input tensors into tensor_flow model

  segmentIndex = 0
  while segmentIndex <= 1499:
      if (segmentIndex == 0):
          cropArray = imageNumpyArray[0:cropHeight + overlapWidth, 0:cropWidth + overlapWidth, :]

      # catch top right corner tile
      elif (segmentIndex == 49):
          cropArray = imageNumpyArray[0:cropHeight + overlapWidth,
                  segmentIndex * cropWidth - overlapWidth:segmentIndex * cropWidth + cropWidth, :]

          # catch bottom left corner tile
      elif (segmentIndex == 1450):
          cropArray = imageNumpyArray[
                      cropHeight * segmentIndex // 50 - overlapWidth:cropHeight * segmentIndex // 50 + cropHeight,
                      0:cropWidth + overlapWidth, :]
          # catch bottom right corner tile
      elif (segmentIndex == 1499):
          cropArray = imageNumpyArray[
                      cropHeight * segmentIndex // 50 - cropHeight - overlapWidth:cropHeight * segmentIndex // 50,
                      segmentIndex % 50 * cropWidth - overlapWidth:segmentIndex % 50 * cropWidth + cropWidth, :]
          # catch right edge tiles so no overlap on left
      elif (segmentIndex % 50 == 0):
          # print(cropHeight*segmentIndex//50)
          cropArray = imageNumpyArray[cropHeight * (segmentIndex // 50):cropHeight * (segmentIndex // 50) + cropHeight,
                      0:cropWidth + overlapWidth, :]
           # catch top edge tiles so no overlap on top
      elif (segmentIndex <= 48):
          # print(segmentIndex*cropWidth)
          cropArray = imageNumpyArray[0:cropHeight + overlapWidth,
                      segmentIndex * cropWidth:segmentIndex * cropWidth + cropWidth, :]
          # catch left edge tiles so no overlap on left
      elif (segmentIndex + 1) % 50 == 0:
          # print(segmentIndex * cropWidth)
          cropArray = imageNumpyArray[((segmentIndex + 1) // 50) * cropHeight - overlapWidth:((
                                                                                segmentIndex + 1) // 50) * cropHeight + cropHeight + overlapWidth,
                      (segmentIndex) % 50 * cropWidth - overlapWidth:(segmentIndex) % 50 * cropWidth + cropWidth, :]
          # catch bottom edge tiles so no overlap on top
      elif (segmentIndex > 1450):
          # print(segmentIndex * cropWidth)
          cropArray = imageNumpyArray[((segmentIndex + 1) // 50) * cropHeight:((
                                                                               segmentIndex + 1) // 50) * cropHeight + cropHeight + overlapWidth,
                      (segmentIndex) % 50 * cropWidth - overlapWidth:(segmentIndex) % 50 * cropWidth + cropWidth, :]
       else:
          cropArray = imageNumpyArray[(segmentIndex // 50) * cropHeight - overlapWidth: (
                                                                                segmentIndex // 50) * cropHeight + cropHeight + overlapWidth,
                      (segmentIndex) % 50 * cropWidth - overlapWidth:(
                                                                     segmentIndex) % 50 * cropWidth + cropWidth + overlapWidth,
                      :]
      detect_objects(cropArray, sess, detection_graph)
      if segmentIndex % 150 == 0:
          print(str(segmentIndex // 150 * 10) + " percent complete")
  segmentIndex += 1

<|endoftext|>