<commit_before>"""
    SPF Flattener script

    WHAT IT DOES
        1. DNS Lookups on your specified records, returns their ipv4 addresses
        2. Recursively finds 'includes' and other hostnames, returns their ipv4 addresses
        2. Dedupes network blocks
        3. Generates new records, each below 512 bytes
        4. Saves the result, plus a hash of the result, in a file
        5. Will compare with the previous records on the next run, and email if they are different

    INSTRUCTIONS
        Fill out the JSON file with your domains, with record:type values.
        Also add some resolvers. Try resolvers that the public might use.
"""
# builtins
import re
import json
import hashlib
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from difflib import HtmlDiff

# noinspection PyUnresolvedReferences, PyPackageRequirements
# 3rd party imports
from dns import resolver  # dnspython/3
from netaddr import IPSet

def hash_seq(iterable):
    """ Acts as a centrifuge for our SPF records and returns a hash """
    return hashlib.sha256(''.join(sorted(''.join(sorted(iterable)))).encode()).hexdigest()

def bytelength(addresses):
    return len(bytes(('v=spf1 ' + ' '.join(addresses)).encode()))

def recurse_lookup(resourcerecord, resourcetype):
    """
        Finds IPv4 Addresses, plus hostnames.

        If there are hostnames, it calls itself to extract the IPv4 addresses from them
        It then returns a collection of all of the found addresses.
    """
    try:
        # This query takes up 85% of execution time
        result = nameserver.query(resourcerecord, resourcetype).response.to_text()
    except Exception as err:
        print(repr(err), resourcerecord, resourcetype)
    else:
        match = dns_answer.search(result)
        answers = match.group('answers')
        if resourcetype == 'a':
            addresses = a_record.findall(answers)
        else:
            addresses = ip_address.findall(answers)
        includes = spf_include.findall(answers)

        for ip in addresses:
            if resourcetype == 'a' and '/' not in ip:
                ip += '/32'
            yield ip

        if includes:
            for includetype, hostname in includes:
                includetype = includetype.lower().strip(' 1234567890')  # Remove priority info from mx records
                includetype = includetype.replace('include', 'txt')
                includetype = includetype.replace('ptr', 'cname')
                for ip in recurse_lookup(hostname, includetype):
                    yield ip

def flatten(records, domain):
    ips = set()
    for rrecord, rdtype in records.items():
        for ip in recurse_lookup(rrecord, rdtype):
            ips.add(ip)
    ips = [str(s) for s in IPSet(ips).iter_cidrs()]
    ips = ['ip6:' + ip if ':' in ip else
           'ip4:' + ip.replace('/32', '')
           for ip in ips]
    ipv4blocks, last_record = separate_into_512bytes(ips)
    return [record for record in wrap_in_spf_tokens(domain, ipv4blocks, last_record)]

def wrap_in_spf_tokens(domain, ipv4blocks, last_record):
    for spf_num, spf_set in enumerate(ipv4blocks):
        spf_set = ' '.join(spf_set)
        if spf_num == last_record:
            spfrecord = 'v=spf1 {0} -all'.format(spf_set)
        else:
            spfrecord = 'v=spf1 {0} include:spf{1}.{2} -all'.format(spf_set, spf_num + 1, domain)
        yield spfrecord

def separate_into_512bytes(ips):
    ipv4blocks = [ips]
    for index, addresses in enumerate(ipv4blocks):
        while bytelength(addresses) >= 485:  # 485 allows for spf prefix/suffixes, such as includes
            overflow = ipv4blocks[index].pop()
            try:
                ipv4blocks[index + 1]
            except IndexError:
                ipv4blocks.append(set())
            finally:
                ipv4blocks[index + 1].add(overflow)
    last_record = len(ipv4blocks) - 1
    return ipv4blocks, last_record

def output_bind_compatible(spfrec):
    spfrec = spfrec.split()
    ret = list()
    while spfrec:
        line, end = '"', '"'
        if not len(ret):
            line = '( "'
        try:
            for i in range(4):
                line += spfrec.pop(0) + ' '
        except IndexError:
            end = '" )'
        finally:
            ret.append(line + end)
    return ret

def email_changes(prev_addrs, curr_addrs):
    bindformat = list()
    for record in curr_addrs:
        for string in output_bind_compatible(record):
            bindformat.append(string)
    bindformat = '

BIND compatible format:
' + '
'.join(bindformat) + '
'

    prev_addrs = ' '.join(prev_addrs)
    curr_addrs = ' '.join(curr_addrs)
    prev = sorted([s for s in prev_addrs.split() if not re.search(pattern=r'(include|spf|all)', string=s)])
    curr = sorted([s for s in curr_addrs.split() if not re.search(pattern=r'(include|spf|all)', string=s)])

    diff = HtmlDiff()
    table = diff.make_table(
        fromlines=prev,
        tolines=curr,
        fromdesc='Old records',
        todesc='New records'
    )

    header = '
Diff
'
    html = style + bindformat + header + table
    html = MIMEText(html, 'html')
    msg_template = MIMEMultipart('alternative')
    msg_template['Subject'] = subject.format(zone=zone)
    email = msg_template
    email.attach(html)

    try:
        mailserver = smtplib.SMTP()
        mailserver.connect(settings['email']['server'])
        mailserver.sendmail(fromaddr, toaddr, email.as_string())
    except Exception as e:
        print('Email failed: ' + str(e))
        with open('result.html', 'w+') as f:
            f.write(html.as_string())

if __name__ == "__main__":
    with open('settings.json') as y:
        settings = json.load(y)

    #
    # Nameserver setup
    #
    nameserver = resolver.Resolver()
    nameserver.nameservers = [resolver for resolver in settings['resolvers']]

    #
    # Various regexes
    #
    dns_answer = re.compile(r'ANSWER\n(?P<answers>[^;]+)')
    ip_address = re.compile(r'(?<=ip[46]:)\S+')
    a_record = re.compile(r'((?:\d{1,3}\.){3}\d{1,3})')
    spf_include = re.compile(r'(?P<type>include|a|mx(?: \d+)? ?|ptr|cname ?)[:](?P<hostname>[^\s\'\"]+\w)',
                             flags=re.IGNORECASE)
    #
    # Email setup
    #
    fromaddr = settings['email']['from']
    toaddr = settings['email']['to']
    subject = settings['email']['subject']
    sending_domains = settings['sending domains'].items()
    style = '''

    '''

    #
    # Begin resolving and flattening
    #
    curr_spf = dict()
    try:
        with open(settings['output']) as prev_hashes:
            prev_spf = json.load(prev_hashes)
    except Exception as e:
        print(e)
        for zone, domains in sending_domains:
            spfrecords = flatten(domains, zone)
            shasum = hash_seq(spfrecords)
            curr_spf[zone] = {
                'sum': shasum,
                'records': spfrecords
            }
    else:
        for zone, domains in sending_domains:
            spfrecords = flatten(domains, zone)
            shasum = hash_seq(spfrecords)
            curr_spf[zone] = {
                'sum': shasum,
                'records': spfrecords
            }
            if prev_spf.get(zone, False) and curr_spf.get(zone, False):
                if prev_spf[zone]['sum'] != curr_spf[zone]['sum']:
                    previous = prev_spf[zone]['records']
                    current = curr_spf[zone]['records']
                    email_changes(previous, current)
    finally:
        with open(settings['output'], 'w+') as sumsfile:
            json.dump(curr_spf, sumsfile, indent=4, sort_keys=True)
<commit_msg>update list hashing function to avoid collisions<commit_after>"""
    SPF Flattener script

    WHAT IT DOES
        1. DNS Lookups on your specified records, returns their ipv4 addresses
        2. Recursively finds 'includes' and other hostnames, returns their ipv4 addresses
        2. Dedupes network blocks
        3. Generates new records, each below 512 bytes
        4. Saves the result, plus a hash of the result, in a file
        5. Will compare with the previous records on the next run, and email if they are different

    INSTRUCTIONS
        Fill out the JSON file with your domains, with record:type values.
        Also add some resolvers. Try resolvers that the public might use.
"""
# builtins
import re
import json
import hashlib
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from difflib import HtmlDiff

# noinspection PyUnresolvedReferences, PyPackageRequirements
# 3rd party imports
from dns import resolver  # dnspython/3
from netaddr import IPSet

def hash_seq(iterable):
    """ Acts as a centrifuge for our SPF records and returns a hash """
    flat_sorted_sequence = ' '.join(sorted([token for string in iterable for token in string.split()]))
    return hashlib.sha256(flat_sorted_sequence.encode()).hexdigest()

def bytelength(addresses):
    return len(bytes(('v=spf1 ' + ' '.join(addresses)).encode()))

def recurse_lookup(resourcerecord, resourcetype):
    """
        Finds IPv4 Addresses, plus hostnames.

        If there are hostnames, it calls itself to extract the IPv4 addresses from them
        It then returns a collection of all of the found addresses.
    """
    try:
        # This query takes up 85% of execution time
        result = nameserver.query(resourcerecord, resourcetype).response.to_text()
    except Exception as err:
        print(repr(err), resourcerecord, resourcetype)
    else:
        match = dns_answer.search(result)
        answers = match.group('answers')
        if resourcetype == 'a':
            addresses = a_record.findall(answers)
        else:
            addresses = ip_address.findall(answers)
        includes = spf_include.findall(answers)

        for ip in addresses:
            if resourcetype == 'a' and '/' not in ip:
                ip += '/32'
            yield ip

        if includes:
            for includetype, hostname in includes:
                includetype = includetype.lower().strip(' 1234567890')  # Remove priority info from mx records
                includetype = includetype.replace('include', 'txt')
                includetype = includetype.replace('ptr', 'cname')
                for ip in recurse_lookup(hostname, includetype):
                    yield ip

def flatten(records, domain):
    ips = set()
    for rrecord, rdtype in records.items():
        for ip in recurse_lookup(rrecord, rdtype):
            ips.add(ip)
    ips = [str(s) for s in IPSet(ips).iter_cidrs()]
    ips = ['ip6:' + ip if ':' in ip else
           'ip4:' + ip.replace('/32', '')
           for ip in ips]
    ipv4blocks, last_record = separate_into_512bytes(ips)
    return [record for record in wrap_in_spf_tokens(domain, ipv4blocks, last_record)]

def wrap_in_spf_tokens(domain, ipv4blocks, last_record):
    for spf_num, spf_set in enumerate(ipv4blocks):
        spf_set = ' '.join(spf_set)
        if spf_num == last_record:
            spfrecord = 'v=spf1 {0} -all'.format(spf_set)
        else:
            spfrecord = 'v=spf1 {0} include:spf{1}.{2} -all'.format(spf_set, spf_num + 1, domain)
        yield spfrecord

def separate_into_512bytes(ips):
    ipv4blocks = [ips]
    for index, addresses in enumerate(ipv4blocks):
        while bytelength(addresses) >= 485:  # 485 allows for spf prefix/suffixes, such as includes
            overflow = ipv4blocks[index].pop()
            try:
                ipv4blocks[index + 1]
            except IndexError:
                ipv4blocks.append(set())
            finally:
                ipv4blocks[index + 1].add(overflow)
    last_record = len(ipv4blocks) - 1
    return ipv4blocks, last_record

def output_bind_compatible(spfrec):
    spfrec = spfrec.split()
    ret = list()
    while spfrec:
        line, end = '"', '"'
        if not len(ret):
            line = '( "'
        try:
            for i in range(4):
                line += spfrec.pop(0) + ' '
        except IndexError:
            end = '" )'
        finally:
            ret.append(line + end)
    return ret

def email_changes(prev_addrs, curr_addrs):
    bindformat = list()
    for record in curr_addrs:
        for string in output_bind_compatible(record):
            bindformat.append(string)
    bindformat = '

BIND compatible format:
' + '
'.join(bindformat) + '
'

    prev_addrs = ' '.join(prev_addrs)
    curr_addrs = ' '.join(curr_addrs)
    prev = sorted([s for s in prev_addrs.split() if not re.search(pattern=r'(include|spf|all)', string=s)])
    curr = sorted([s for s in curr_addrs.split() if not re.search(pattern=r'(include|spf|all)', string=s)])

    diff = HtmlDiff()
    table = diff.make_table(
        fromlines=prev,
        tolines=curr,
        fromdesc='Old records',
        todesc='New records'
    )

    header = '
Diff
'
    html = style + bindformat + header + table
    html = MIMEText(html, 'html')
    msg_template = MIMEMultipart('alternative')
    msg_template['Subject'] = subject.format(zone=zone)
    email = msg_template
    email.attach(html)

    try:
        mailserver = smtplib.SMTP()
        mailserver.connect(settings['email']['server'])
        mailserver.sendmail(fromaddr, toaddr, email.as_string())
    except Exception as e:
        print('Email failed: ' + str(e))
        with open('result.html', 'w+') as f:
            f.write(html.as_string())

if __name__ == "__main__":
    with open('settings.json') as y:
        settings = json.load(y)

    #
    # Nameserver setup
    #
    nameserver = resolver.Resolver()
    nameserver.nameservers = [resolver for resolver in settings['resolvers']]

    #
    # Various regexes
    #
    dns_answer = re.compile(r'ANSWER\n(?P<answers>[^;]+)')
    ip_address = re.compile(r'(?<=ip[46]:)\S+')
    a_record = re.compile(r'((?:\d{1,3}\.){3}\d{1,3})')
    spf_include = re.compile(r'(?P<type>include|a|mx(?: \d+)? ?|ptr|cname ?)[:](?P<hostname>[^\s\'\"]+\w)',
                             flags=re.IGNORECASE)
    #
    # Email setup
    #
    fromaddr = settings['email']['from']
    toaddr = settings['email']['to']
    subject = settings['email']['subject']
    sending_domains = settings['sending domains'].items()
    style = '''

    '''

    #
    # Begin resolving and flattening
    #
    curr_spf = dict()
    try:
        with open(settings['output']) as prev_hashes:
            prev_spf = json.load(prev_hashes)
    except Exception as e:
        print(e)
        for zone, domains in sending_domains:
            spfrecords = flatten(domains, zone)
            shasum = hash_seq(spfrecords)
            curr_spf[zone] = {
                'sum': shasum,
                'records': spfrecords
            }
    else:
        for zone, domains in sending_domains:
            spfrecords = flatten(domains, zone)
            shasum = hash_seq(spfrecords)
            curr_spf[zone] = {
                'sum': shasum,
                'records': spfrecords
            }
            if prev_spf.get(zone, False) and curr_spf.get(zone, False):
                if prev_spf[zone]['sum'] != curr_spf[zone]['sum']:
                    previous = prev_spf[zone]['records']
                    current = curr_spf[zone]['records']
                    email_changes(previous, current)
    finally:
        with open(settings['output'], 'w+') as sumsfile:
            json.dump(curr_spf, sumsfile, indent=4, sort_keys=True)
<|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import time

import eventlet
from oslo.config import cfg
import six

from senlin.common.i18n import _LI
from senlin.db import api as db_api
from senlin.openstack.common import log as logging
from senlin.openstack.common import threadgroup

LOG = logging.getLogger(__name__)

wallclock = time.time

ACTION_CONTROL_REQUEST = (
    ACTION_CANCEL, ACTION_SUSPEND, ACTION_RESUME, ACTION_TIMEOUT,
) = (
    'cancel', 'suspend', 'resume', 'timeout',
)

class ThreadGroupManager(object):
    """
    """
    def __init__(self):
        super(ThreadGroupManager, self).__init__()
        self.threads = {}
        self.group = threadgroup.ThreadGroup()

        # Create dummy service task, because when there is nothing queued
        # on self.tg the process exits
        self.add_timer(cfg.CONF.periodic_interval, self._service_task)

    def _service_task(self):
        '''
        This is a dummy task which gets queued on the service.Service
        threadgroup.  Without this service.Service sees nothing running
        i.e has nothing to wait() on, so the process exits..
        This could also be used to trigger periodic non-cluster-specific
        housekeeping tasks

        (Yanyan)Not sure this is still necessary, just keep it temporarily.
        '''
        # TODO(Yanyan): have this task call dbapi purge events
        pass

    def start(self, func, *args, **kwargs):
        """
        Run the given method in a sub-thread.
        """
        return self.group.add_thread(func, *args, **kwargs)

    def start_action_thread(self, cnxt, action, *args, **kwargs):
        """
        Run the given action in a sub-thread and release the action lock
        when the thread finishes.

        :param cnxt: The context of rpc request
        :param action: The action to run in thread

        """
        def release(gt, cnxt, action):
            """
            Callback function that will be passed to GreenThread.link().
            """
            # Remove action thread from thread list
            self.threads.pop(action.id)

        action_proc = ActionProc(cnxt, action)
        th = self.start(action_proc, *args, **kwargs)
        self.threads[action.id] = th
        th.link(release, cnxt, action)
        return th

    def add_timer(self, interval, func, *args, **kwargs):
        """
        Define a periodic task, to be run in a separate thread, in the target
        threadgroups.  Periodicity is cfg.CONF.periodic_interval
        """
        self.group.add_timer(cfg.CONF.periodic_interval,
                             func, *args, **kwargs)

    def stop_timers(self):
        self.group.stop_timers()

    def stop(self, graceful=False):
        '''Stop any active threads belong to this threadgroup.'''
        # Try to stop all threads gracefully
        self.group.stop(graceful)
        self.group.wait()

        # Wait for link()ed functions (i.e. lock release)
        threads = self.group.threads[:]
        links_done = dict((th, False) for th in threads)

        def mark_done(gt, th):
            links_done[th] = True

        for th in threads:
            th.link(mark_done, th)

        while not all(links_done.values()):
            eventlet.sleep()

class ActionProc(object):
    """
    Wrapper for a resumable task(co-routine) for action execution
    """
    def __init__(self, cnxt, action):
        self.cnxt = cnxt
        self.action = action

    def __call__(self, wait_time=1):
        """
        Start and run action progress.

        Action progress will sleep for `wait_time` seconds between
        each step. To avoid sleeping, pass `None` for `wait_time`.
        """
        status = self.action.get_status()
        while status in (self.action.INIT, self.action.WAITING):
            # TODO(Qiming): Handle 'start_time' field of an action
            reschedule(self.action, sleep=self.wait_time)
            status = self.action.get_status()

        # Exit quickly if action has been taken care of or marked
        # completed or cancelled by other activities
        if status != self.action.READY:
            return

        # Do the first step
        LOG.debug('%s starting' % six.text_type(self))

        # Start action execute
        self.action.set_status(self.action.RUNNING)
        self.action.start_time = wallclock()

        result = self.action.execute()

        self.action.end_time = wallclock()

        if result == self.action.ERROR:
            # Error happened during the start,
            # mark entire action as failed and return
            LOG.info(_LI('Successfully run action %s.'), self.action.id)
            db_api.action_mark_failed(self.cnxt, self.action.id)
        elif result == self.action.OK:
            LOG.info(_LI('Action %s run failed.'), self.action.id)
            db_api.action_mark_succeeded(self.cnxt, self.action.id)
        else:
            # TODO(Yanyan): handle action retry scenario.
            pass

def start_action(cnxt, action_id, engine_id, tgm):
    """
    Start an action execution progress using given ThreadGroupManager

    :param cnxt: The context of rpc request
    :param action_id: The id of action to run in thread
    :param engine_id: The id of engine try to lock the action
    :param tgm: The ThreadGroupManager of the engine
    """
    action = db_api.action_start_work_on(cnxt, action_id, engine_id)
    if action:
        # Lock action successfully, start a thread to run it
        LOG.info(_LI('Successfully locked action %s.'), action_id)
        th = tgm.start_action_thread(cnxt, action)
        if th is None:
            LOG.debug('Action start failed, unlock action.')
            # TODO: need db_api support
            db_api.action_unlock(cnxt, action_id, engine_id)
        else:
            return True
    else:
        # The action has been locked which means other
        # dispatcher has start to work on it, so just return
        LOG.info(_LI('Action %s has been locked by other dispatcher'),
                 action_id)

def suspend_action(cnxt, action_id):
    """
    Try to suspend an action execution progress

    :param cnxt: The context of rpc request
    :param action_id: The id of action to run in thread
    """
    # Set action control flag to suspend
    # TODO: need db_api support
    db_api.action_control(cnxt, action_id, ACTION_SUSPEND)

def resume_action(cnxt, action_id):
    """
    Try to resume an action execution progress

    :param cnxt: The context of rpc request
    :param action_id: The id of action to run in thread
    """
    # Set action control flag to suspend
    # TODO: need db_api support
    db_api.action_control(cnxt, action_id, ACTION_RESUME)

def cancel_action(cnxt, action_id):
    """
    Try to cancel an action execution progress

    :param cnxt: The context of rpc request
    :param action_id: The id of action to run in thread
    """
    # Set action control flag to cancel
    # TODO: need db_api support
    db_api.action_control(cnxt, action_id, ACTION_CANCEL)

def action_control_flag(action):
    """
    Check whether there are some action control requests
    need to be handled.
    """
    # Check timeout first, if true, return timeout message
    if action.timeout is not None and action_timeout(action):
        LOG.debug('Action %s run timeout' % action.id)
        return ACTION_TIMEOUT

    # Check if action control flag is set
    result = db_api.action_control_check(action.context, action.id)
    LOG.debug('Action %s control flag is %s', (action.id, result))
    return result

def action_cancelled(action):
    """
    Check whether an action's control flag is set to cancel
    """
    if action_control_flag(action) == ACTION_CANCEL:
        return True
    else:
        return False

def action_suspended(action):
    """
    Check whether an action's control flag is set to suspend
    """
    if action_control_flag(action) == ACTION_SUSPEND:
        return True
    else:
        return False

def action_resumed(action):
    """
    Check whether an action's control flag is set to suspend
    """
    if action_control_flag(action) == ACTION_RESUME:
        return True
    else:
        return False

def action_timeout(action):
    """
    Return True if an action has run timeout, False otherwise.
    """
    time_lapse = wallclock() - action.start_time

    if time_lapse > action.timeout:
        return True
    else:
        return False

def reschedule(action, sleep_time=1):
    """
    Eventlet Sleep for the specified number of seconds.

    :param sleep_time: seconds to sleep; if None, no sleep;
    """
    if sleep_time is not None:
        LOG.debug('Action %s sleep for %s seconds' % (
            action.id, sleep_time))
        eventlet.sleep(sleep_time)

def action_wait(action):
    """
    Keep waiting util action resume control flag is set
    """
    while not action_resumed(action):
        reschedule(action, sleep_time=1)
        continue
<commit_msg>Add sleep method to scheduler module<commit_after># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import time

import eventlet
from oslo.config import cfg
import six

from senlin.common.i18n import _LI
from senlin.db import api as db_api
from senlin.openstack.common import log as logging
from senlin.openstack.common import threadgroup

LOG = logging.getLogger(__name__)

wallclock = time.time

ACTION_CONTROL_REQUEST = (
    ACTION_CANCEL, ACTION_SUSPEND, ACTION_RESUME, ACTION_TIMEOUT,
) = (
    'cancel', 'suspend', 'resume', 'timeout',
)

class ThreadGroupManager(object):
    """
    """
    def __init__(self):
        super(ThreadGroupManager, self).__init__()
        self.threads = {}
        self.group = threadgroup.ThreadGroup()

        # Create dummy service task, because when there is nothing queued
        # on self.tg the process exits
        self.add_timer(cfg.CONF.periodic_interval, self._service_task)

    def _service_task(self):
        '''
        This is a dummy task which gets queued on the service.Service
        threadgroup.  Without this service.Service sees nothing running
        i.e has nothing to wait() on, so the process exits..
        This could also be used to trigger periodic non-cluster-specific
        housekeeping tasks

        (Yanyan)Not sure this is still necessary, just keep it temporarily.
        '''
        # TODO(Yanyan): have this task call dbapi purge events
        pass

    def start(self, func, *args, **kwargs):
        """
        Run the given method in a sub-thread.
        """
        return self.group.add_thread(func, *args, **kwargs)

    def start_action_thread(self, cnxt, action, *args, **kwargs):
        """
        Run the given action in a sub-thread and release the action lock
        when the thread finishes.

        :param cnxt: The context of rpc request
        :param action: The action to run in thread

        """
        def release(gt, cnxt, action):
            """
            Callback function that will be passed to GreenThread.link().
            """
            # Remove action thread from thread list
            self.threads.pop(action.id)

        action_proc = ActionProc(cnxt, action)
        th = self.start(action_proc, *args, **kwargs)
        self.threads[action.id] = th
        th.link(release, cnxt, action)
        return th

    def add_timer(self, interval, func, *args, **kwargs):
        """
        Define a periodic task, to be run in a separate thread, in the target
        threadgroups.  Periodicity is cfg.CONF.periodic_interval
        """
        self.group.add_timer(cfg.CONF.periodic_interval,
                             func, *args, **kwargs)

    def stop_timers(self):
        self.group.stop_timers()

    def stop(self, graceful=False):
        '''Stop any active threads belong to this threadgroup.'''
        # Try to stop all threads gracefully
        self.group.stop(graceful)
        self.group.wait()

        # Wait for link()ed functions (i.e. lock release)
        threads = self.group.threads[:]
        links_done = dict((th, False) for th in threads)

        def mark_done(gt, th):
            links_done[th] = True

        for th in threads:
            th.link(mark_done, th)

        while not all(links_done.values()):
            eventlet.sleep()

class ActionProc(object):
    """
    Wrapper for a resumable task(co-routine) for action execution
    """
    def __init__(self, cnxt, action):
        self.cnxt = cnxt
        self.action = action

    def __call__(self, wait_time=1):
        """
        Start and run action progress.

        Action progress will sleep for `wait_time` seconds between
        each step. To avoid sleeping, pass `None` for `wait_time`.
        """
        status = self.action.get_status()
        while status in (self.action.INIT, self.action.WAITING):
            # TODO(Qiming): Handle 'start_time' field of an action
            reschedule(self.action, sleep=self.wait_time)
            status = self.action.get_status()

        # Exit quickly if action has been taken care of or marked
        # completed or cancelled by other activities
        if status != self.action.READY:
            return

        # Do the first step
        LOG.debug('%s starting' % six.text_type(self))

        # Start action execute
        self.action.set_status(self.action.RUNNING)
        self.action.start_time = wallclock()

        result = self.action.execute()

        self.action.end_time = wallclock()

        if result == self.action.ERROR:
            # Error happened during the start,
            # mark entire action as failed and return
            LOG.info(_LI('Successfully run action %s.'), self.action.id)
            db_api.action_mark_failed(self.cnxt, self.action.id)
        elif result == self.action.OK:
            LOG.info(_LI('Action %s run failed.'), self.action.id)
            db_api.action_mark_succeeded(self.cnxt, self.action.id)
        else:
            # TODO(Yanyan): handle action retry scenario.
            pass

def start_action(cnxt, action_id, engine_id, tgm):
    """
    Start an action execution progress using given ThreadGroupManager

    :param cnxt: The context of rpc request
    :param action_id: The id of action to run in thread
    :param engine_id: The id of engine try to lock the action
    :param tgm: The ThreadGroupManager of the engine
    """
    action = db_api.action_start_work_on(cnxt, action_id, engine_id)
    if action:
        # Lock action successfully, start a thread to run it
        LOG.info(_LI('Successfully locked action %s.'), action_id)
        th = tgm.start_action_thread(cnxt, action)
        if th is None:
            LOG.debug('Action start failed, unlock action.')
            # TODO: need db_api support
            db_api.action_unlock(cnxt, action_id, engine_id)
        else:
            return True
    else:
        # The action has been locked which means other
        # dispatcher has start to work on it, so just return
        LOG.info(_LI('Action %s has been locked by other dispatcher'),
                 action_id)

def suspend_action(cnxt, action_id):
    """
    Try to suspend an action execution progress

    :param cnxt: The context of rpc request
    :param action_id: The id of action to run in thread
    """
    # Set action control flag to suspend
    # TODO: need db_api support
    db_api.action_control(cnxt, action_id, ACTION_SUSPEND)

def resume_action(cnxt, action_id):
    """
    Try to resume an action execution progress

    :param cnxt: The context of rpc request
    :param action_id: The id of action to run in thread
    """
    # Set action control flag to suspend
    # TODO: need db_api support
    db_api.action_control(cnxt, action_id, ACTION_RESUME)

def cancel_action(cnxt, action_id):
    """
    Try to cancel an action execution progress

    :param cnxt: The context of rpc request
    :param action_id: The id of action to run in thread
    """
    # Set action control flag to cancel
    # TODO: need db_api support
    db_api.action_control(cnxt, action_id, ACTION_CANCEL)

def action_control_flag(action):
    """
    Check whether there are some action control requests
    need to be handled.
    """
    # Check timeout first, if true, return timeout message
    if action.timeout is not None and action_timeout(action):
        LOG.debug('Action %s run timeout' % action.id)
        return ACTION_TIMEOUT

    # Check if action control flag is set
    result = db_api.action_control_check(action.context, action.id)
    LOG.debug('Action %s control flag is %s', (action.id, result))
    return result

def action_cancelled(action):
    """
    Check whether an action's control flag is set to cancel
    """
    if action_control_flag(action) == ACTION_CANCEL:
        return True
    else:
        return False

def action_suspended(action):
    """
    Check whether an action's control flag is set to suspend
    """
    if action_control_flag(action) == ACTION_SUSPEND:
        return True
    else:
        return False

def action_resumed(action):
    """
    Check whether an action's control flag is set to suspend
    """
    if action_control_flag(action) == ACTION_RESUME:
        return True
    else:
        return False

def action_timeout(action):
    """
    Return True if an action has run timeout, False otherwise.
    """
    time_lapse = wallclock() - action.start_time

    if time_lapse > action.timeout:
        return True
    else:
        return False

def reschedule(action, sleep_time=1):
    """
    Eventlet Sleep for the specified number of seconds.

    :param sleep_time: seconds to sleep; if None, no sleep;
    """
    if sleep_time is not None:
        LOG.debug('Action %s sleep for %s seconds' % (
            action.id, sleep_time))
        eventlet.sleep(sleep_time)

def action_wait(action):
    """
    Keep waiting util action resume control flag is set
    """
    while not action_resumed(action):
        reschedule(action, sleep_time=1)
        continue

def sleep(sleep_time):
    '''
    Interface for sleeping.
    '''
    eventlet.sleep(sleep_time)
<|endoftext|>


<commit_before>#FLM: Output TrueType Hints
# coding: utf-8

__copyright__ = __license__ =  """
Copyright (c) 2013, 2015 Adobe Systems Incorporated. All rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""

__doc__ = """
Output TrueType Hints v1.3 - Mar 25 2015

This FontLab macro will write an external simple text file containing
TrueType instructions for each selected glyph. If the external file
already exists, the script will replace the existing entries and add
new ones. The hinting data can be loaded back into the font by using
the macro named "Input TrueType Hints".

The script will emit an error if there are hints attached to off-curve
points. That's the only difference from OutputTrueTypeHints.py.

==================================================
Versions:
v1.3 - Mar 25 2015 - Allow coordinate output
v1.2 - Mar 23 2015 – Allow instructions in x-direction.
v1.1 - Mar 04 2015 – change name to Output TrueType Hints to supersede
					 the old script of the same name.
v1.0 - Nov 27 2013 - Initial release
"""

import os
from FL import *

vAlignLinkTop = 1
vAlignLinkBottom = 2
hSingleLink = 3
vSingleLink = 4
hDoubleLink = 5
vDoubleLink = 6
hAlignLinkNear = 7
vAlignLinkNear = 8
hInterpolateLink = 13
vInterpolateLink = 14
hMidDelta = 20
vMidDelta = 21
hFinDelta = 22
vFinDelta = 23

deltas = [hMidDelta, hFinDelta, vMidDelta, vFinDelta]
interpolations = [hInterpolateLink, vInterpolateLink]
links = [hSingleLink, hDoubleLink, vSingleLink, vDoubleLink]
anchors = [vAlignLinkTop, vAlignLinkNear, vAlignLinkBottom, hAlignLinkNear]

listGlyphsSelected = []
def getgselectedglyphs(font, glyph, gindex):
	listGlyphsSelected.append(glyph.name)
fl.ForSelected(getgselectedglyphs)

allGlyphsHintList = ["# Glyph name\tTT hints\n"]

def readTTHintsFile(filePath):
	file = open(filePath, "r")
	data = file.read()
	file.close()
	lines = data.splitlines()

	ttHintsList = []

	for i in range(len(lines)):
		line = lines[i]
		# Skip over blank lines
		line2 = line.strip()
		if not line2:
			continue
		# Skip over comments
		if line.find('#') >= 0:
			continue
		else:
			ttHintsList.append(line)

	return ttHintsList

def writeTTHintsFile(content, filePath):
	outfile = open(filePath, 'w')
	outfile.writelines(content)
	outfile.close()

ttHintsGlyphNamesList = []
def processTTHintsFileData(ttHintsList):
	ttHintsDict = {}

	for item in ttHintsList:
		hintItems = item.split("\t")

		if len(hintItems) != 2:
			print "\tERROR: This hint definition does not have the correct format\n\t%s" % item
			continue

		gName = hintItems[0]
		ttHintsDict[gName] = item
		ttHintsGlyphNamesList.append(gName)

	return ttHintsDict

def checkForOffCurve(commands, gName):
	gIndex = fl.font.FindGlyph(gName)
	commandCode = commands[0]

	if commandCode in anchors:
		'The instruction is an Align Link (top or bottom), so only one node index is provided.'
		nodeIndexList = [commands[1]]
	elif commandCode in links:
		'The instruction is a Single Link or a Double Link, so two node indexes are provided.'
		nodeIndexList = [x for x in commands[1:3]]
	elif commandCode in interpolations:
		'The instruction is an Interpolation Link, so three node indexes are provided.'
		nodeIndexList = [x for x in commands[1:4]]
	elif commandCode in deltas:
		'The instruction is a Delta, one node index is provided.'
		nodeIndexList = [commands[1]]
	else:
		print "ERROR: Hint type not supported in %s." % gName
		nodeIndexList = []

	for hintedNodeIndex in nodeIndexList:
		node = fl.font[gIndex][hintedNodeIndex]
		try:
			node.type
		except:
			print "ERROR: Hinting problem in glyph %s." % gName
			continue

		if node.type == nOFF:
			print "ERROR: Hinted off-curve point #%d in glyph %s." % (hintedNodeIndex, gName)

def collectInstructions(tth, gName):
	commandsList = []
	for inst in tth.commands:
		command = [inst.code]
		for p in inst.params:
			command.append(p)

		command_string = ','.join(map(str, command))
		# Convert list of instructions to comma-separated string.
		commandsList.append(command_string)

		checkForOffCurve(command, gName)
	return commandsList

def analyzePoint(glyph, nodeIndex):
	point = glyph[nodeIndex]
	try: point.type

	except:
		'''
		Left or right bottom point have been hinted:
		In the hinting code, those points are stored as point indices larger than the actual glyph.
		Since those points cannot be grasped in the context of an outline,
		flags are created

		'''
		if nodeIndex == len(glyph):
			point_flag = '"BL"'
		elif nodeIndex == len(glyph) + 1:
			point_flag = '"BR"'
		else:
			'point not in glyph'
			print '\tERROR: Hinting problem in glyph %s' % glyph.name
			continue

		return point_flag

	point_coordinates = point.x, point.y

	if point.type == nOFF:
		gName = glyph.name
		print "\tERROR: Off-curve point %s hinted in glyph %s." % (point_coordinates, gName)

	return point_coordinates

def collectInstructions2(tth, gName, coord_option):
	'''

	Data structure:

	anchors:
	[command code, point index, alignment]

	single and double links:
	[command code, point index 1, point index 2, stem ID, alignment]

	interpolations:
	[command code, point index 1, point index 2, point index 3, alignment]

	deltas:
	[command code, point index, offset, point range min, point range max]

	'''

	glyph = fl.font[gName]
	coord_commandsList = []
	commandsList = []
	for inst in tth.commands:
		command = [inst.code]
		coord_command = [inst.code]

		for p in inst.params:
			command.append(p)

		if inst.code in deltas:
			'delta'
			nodeIndex = command[1]
			deltaDetails = command[2:]
			point_coordinates = analyzePoint(glyph, nodeIndex)

			coord_command.append(point_coordinates)
			coord_command.extend(deltaDetails)

		elif inst.code in links:
			'single- or double link'

			for nodeIndex in command[1:3]:
				point_coordinates = analyzePoint(glyph, nodeIndex)

				coord_command.append(point_coordinates)
			coord_command.extend(command[-2:])

		elif inst.code in anchors + interpolations:
			'anchor or interpolation'
			for nodeIndex in command[1:-1]:
				point_coordinates = analyzePoint(glyph, nodeIndex)

				coord_command.append(point_coordinates)
			coord_command.append(command[-1])

		else:
			'unknown instruction code'
			print "\tERROR: Hinting problem in glyph %s." % gName
			continue

		coord_command_string = ','.join(map(str, coord_command))
		coord_command_string = coord_command_string.replace(' ', '')
		command_string = ','.join(map(str, command))

		commandsList.append(command_string)
		coord_commandsList.append(coord_command_string)

	if coord_option:
		return coord_commandsList
	else:
		return commandsList

def processGlyphs(ttHintsDict, coord_option):
	# Iterate through all the glyphs instead of just the ones selected.
	# This way the order of the items in the output file will be constant and predictable.
	for glyph in fl.font.glyphs:
		gName = glyph.name

		if (gName in listGlyphsSelected) or (gName in ttHintsGlyphNamesList):
			if gName in listGlyphsSelected:
				tth = TTH(glyph)
				tth.LoadProgram()

				if len(tth.commands):
					gHints = "%s\t%s\n" % (gName, ';'.join(collectInstructions2(tth, gName, coord_option)))
					allGlyphsHintList.append(gHints)
				else:
					print "WARNING: The glyph %s has no TrueType hints." % gName
			else:
				allGlyphsHintList.append(ttHintsDict[gName] + '\n')

def run(parentDir, coord_option):

	if coord_option:
		kTTHintsFileName = "tthints_coords"
	else:
		kTTHintsFileName = "tthints"

	tthintsFilePath = os.path.join(parentDir, kTTHintsFileName)

	if not len(listGlyphsSelected):
		print "Select the glyph(s) to process and try again."
		return

	if os.path.exists(tthintsFilePath):
		print "WARNING: The %s file at %s will be modified." % (kTTHintsFileName, tthintsFilePath)
		ttHintsList = readTTHintsFile(tthintsFilePath)
		modFile = True
	else:
		ttHintsList = []
		modFile = False

	if len(ttHintsList):
		ttHintsDict = processTTHintsFileData(ttHintsList)
	else:
		ttHintsDict = {}

	processGlyphs(ttHintsDict, coord_option)

	if len(allGlyphsHintList) > 1:
		writeTTHintsFile(allGlyphsHintList, tthintsFilePath)
	else:
		print "The script found no TrueType hints to output."
		return

	if not modFile:
		print "File %s was saved at %s" % (kTTHintsFileName, tthintsFilePath)

	print "Done!"

def preRun(coord_option=False):
	# Reset the Output window
	fl.output = '\n'

	if fl.count == 0:
		print "Open a font first."
		return

	font = fl.font

	if len(font) == 0:
		print "The font has no glyphs."
		return

	try:
		parentDir = os.path.dirname(os.path.realpath(font.file_name))
	except AttributeError:
		print "The font has not been saved. Please save the font and try again."
		return

	run(parentDir, coord_option)

if __name__ == "__main__":
	preRun()
<commit_msg>Remove original `collectInstructions` and `checkForOffCurve` functions, since they have been superseded.<commit_after>#FLM: Output TrueType Hints
# coding: utf-8

__copyright__ = __license__ =  """
Copyright (c) 2013, 2015 Adobe Systems Incorporated. All rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""

__doc__ = """
Output TrueType Hints v1.3 - Mar 25 2015

This FontLab macro will write a simple text file containing
TrueType instructions for each selected glyph. If the external file
already exists, the script will replace the existing entries and add
new entries as needed. The hinting data can be loaded back into the
font by using the macro named "Input TrueType Hints".

The script will emit an error if there are hints attached to off-curve
points. That's the only difference from OutputTrueTypeHints.py.

==================================================
Versions:
v1.3 - Mar 25 2015 - Allow coordinate output
v1.2 - Mar 23 2015 – Allow instructions in x-direction.
v1.1 - Mar 04 2015 – change name to Output TrueType Hints to supersede
					 the old script of the same name.
v1.0 - Nov 27 2013 - Initial release
"""

import os
from FL import *

vAlignLinkTop = 1
vAlignLinkBottom = 2
hSingleLink = 3
vSingleLink = 4
hDoubleLink = 5
vDoubleLink = 6
hAlignLinkNear = 7
vAlignLinkNear = 8
hInterpolateLink = 13
vInterpolateLink = 14
hMidDelta = 20
vMidDelta = 21
hFinDelta = 22
vFinDelta = 23

deltas = [hMidDelta, hFinDelta, vMidDelta, vFinDelta]
interpolations = [hInterpolateLink, vInterpolateLink]
links = [hSingleLink, hDoubleLink, vSingleLink, vDoubleLink]
anchors = [vAlignLinkTop, vAlignLinkNear, vAlignLinkBottom, hAlignLinkNear]

listGlyphsSelected = []
def getgselectedglyphs(font, glyph, gindex):
	listGlyphsSelected.append(glyph.name)
fl.ForSelected(getgselectedglyphs)

allGlyphsHintList = ["# Glyph name\tTT hints\n"]

def readTTHintsFile(filePath):
	file = open(filePath, "r")
	data = file.read()
	file.close()
	lines = data.splitlines()

	ttHintsList = []

	for i in range(len(lines)):
		line = lines[i]
		# Skip over blank lines
		line2 = line.strip()
		if not line2:
			continue
		# Skip over comments
		if line.find('#') >= 0:
			continue
		else:
			ttHintsList.append(line)

	return ttHintsList

def writeTTHintsFile(content, filePath):
	outfile = open(filePath, 'w')
	outfile.writelines(content)
	outfile.close()

ttHintsGlyphNamesList = []
def processTTHintsFileData(ttHintsList):
	ttHintsDict = {}

	for item in ttHintsList:
		hintItems = item.split("\t")

		if len(hintItems) != 2:
			print "\tERROR: This hint definition does not have the correct format\n\t%s" % item
			continue

		gName = hintItems[0]
		ttHintsDict[gName] = item
		ttHintsGlyphNamesList.append(gName)

	return ttHintsDict

def analyzePoint(glyph, nodeIndex):
	'''Analyzes a given point for a given glyph.
	   In a normal case, this function returns a tuple of point index and point coordinates.
	   If the sidebearings have been hinted, this function returns a tuple containing the flag
	   for the appropriate sidebearing.

	'''

	point = glyph[nodeIndex]
	try: point.type

	except:
		''' This happens if left or right bottom point have been hinted.
			In the hinting code, those points are stored as point indices greater than the actual
			point count of the glyph. Since those points cannot be grasped in the context of the
			outline, flags are created:
			"BL" is bottom left
			"BR" is bottom right.

			Those flags are written into the output file as strings with quotes, so they can be parsed
			with eval() when reading the file later.'''

		if nodeIndex == len(glyph):
			point_flag = '"BL"'
		elif nodeIndex == len(glyph) + 1:
			point_flag = '"BR"'
		else:
			'point not in glyph -- which should not happen, but you never know.'
			print '\tERROR: Hinting problem in glyph %s' % glyph.name
			return '',''

		return point_flag, point_flag

	point_coordinates = point.x, point.y

	if point.type == nOFF:
		gName = glyph.name
		print "\tERROR: Off-curve point %s hinted in glyph %s." % (point_coordinates, gName)

	return nodeIndex, point_coordinates

def collectInstructions(tth, gName, coord_option):
	''' Parses tth commands (list of integers) and returns them formatted
		for writing the data into an external text file.

		tth data structure:
		-------------------

		anchors:
		[command code, point index, alignment]

		single and double links:
		[command code, point index 1, point index 2, stem ID, alignment]

		interpolations:
		[command code, point index 1, point index 2, point index 3, alignment]

		deltas:
		[command code, point index, offset, point range min, point range max]
	'''

	glyph = fl.font[gName]
	coord_commandsList = []
	index_commandsList = []

	for inst in tth.commands:
		command = [inst.code]
		coord_command = [inst.code]
		index_command = [inst.code]

		for p in inst.params:
			'create raw tth command list to parse'
			command.append(p)

		if inst.code in deltas:
			'delta'
			nodeIndex = command[1]
			deltaDetails = command[2:]
			point_index, point_coordinates = analyzePoint(glyph, nodeIndex)

			coord_command.append(point_coordinates)
			index_command.append(point_index)
			coord_command.extend(deltaDetails)
			index_command.extend(deltaDetails)

		elif inst.code in links:
			'single- or double link'
			linkDetails = command[-2:]
			for nodeIndex in command[1:3]:
				point_index, point_coordinates = analyzePoint(glyph, nodeIndex)

				coord_command.append(point_coordinates)
				index_command.append(point_index)
			coord_command.extend(linkDetails)
			index_command.extend(linkDetails)

		elif inst.code in anchors + interpolations:
			'anchor or interpolation'
			commandDetails = command[-1]
			for nodeIndex in command[1:-1]:
				point_index, point_coordinates = analyzePoint(glyph, nodeIndex)

				coord_command.append(point_coordinates)
				index_command.append(point_index)
			coord_command.append(commandDetails)
			index_command.append(commandDetails)

		else:
			'unknown instruction code'
			print "\tERROR: Hinting problem in glyph %s." % gName
			coord_command = []
			index_command = []

		coord_command_string = ','.join(map(str, coord_command))
		coord_command_string = coord_command_string.replace(' ', '')
		index_command_string = ','.join(map(str, index_command))

		index_commandsList.append(index_command_string)
		coord_commandsList.append(coord_command_string)

	if coord_option:
		return coord_commandsList
	else:
		return index_commandsList

def processGlyphs(ttHintsDict, coord_option):
	# Iterate through all the glyphs instead of just the ones selected.
	# This way the order of the items in the output file will be constant and predictable.
	for glyph in fl.font.glyphs:
		gName = glyph.name

		if (gName in listGlyphsSelected) or (gName in ttHintsGlyphNamesList):
			if gName in listGlyphsSelected:
				tth = TTH(glyph)
				tth.LoadProgram()

				if len(tth.commands):
					gHints = "%s\t%s\n" % (gName, ';'.join(collectInstructions(tth, gName, coord_option)))
					allGlyphsHintList.append(gHints)
				else:
					print "WARNING: The glyph %s has no TrueType hints." % gName
			else:
				allGlyphsHintList.append(ttHintsDict[gName] + '\n')

def run(parentDir, coord_option):

	if coord_option:
		kTTHintsFileName = "tthints_coords"
	else:
		kTTHintsFileName = "tthints"

	tthintsFilePath = os.path.join(parentDir, kTTHintsFileName)

	if not len(listGlyphsSelected):
		print "Select the glyph(s) to process and try again."
		return

	if os.path.exists(tthintsFilePath):
		print "WARNING: The %s file at %s will be modified." % (kTTHintsFileName, tthintsFilePath)
		ttHintsList = readTTHintsFile(tthintsFilePath)
		modFile = True
	else:
		ttHintsList = []
		modFile = False

	if len(ttHintsList):
		ttHintsDict = processTTHintsFileData(ttHintsList)
	else:
		ttHintsDict = {}

	processGlyphs(ttHintsDict, coord_option)

	if len(allGlyphsHintList) > 1:
		writeTTHintsFile(allGlyphsHintList, tthintsFilePath)
	else:
		print "The script found no TrueType hints to output."
		return

	if not modFile:
		print "File %s was saved at %s" % (kTTHintsFileName, tthintsFilePath)

	print "Done!"

def preRun(coord_option=False):
	# Reset the Output window
	fl.output = '\n'

	if fl.count == 0:
		print "Open a font first."
		return

	font = fl.font

	if len(font) == 0:
		print "The font has no glyphs."
		return

	try:
		parentDir = os.path.dirname(os.path.realpath(font.file_name))
	except AttributeError:
		print "The font has not been saved. Please save the font and try again."
		return

	run(parentDir, coord_option)

if __name__ == "__main__":
	preRun()
<|endoftext|>


<commit_before><commit_msg>Use a Protocol for TRANSFORMER to ensure common arg names (#4871)<commit_after><|endoftext|>


<commit_before>"""Image standardization.

This module implements the percentile landmark method described in [1]. Much of
the notation (variable names etc.) comes from this paper.

Learning:
    - Use landmark_scores() to get percentile scores for all images.
    - Use map_onto_scale() to map landmark scores to the standard scale.
    - Average mapped landmarks to get standard landmarks. TODO: This should be
      implemented here somehow.
    - Use write_std_cfg() to output configuration.

Standardizing:
    - Use read_std_cfg() to input configuration.
    - Use standardize() to standardize images according to configuration.
    - You can also use transform() to do this more free-form.

See also tools/standardize.py.

[1] Nyul et al. 2000: New variants of a method of MRI scale standardization.
"""

from __future__ import absolute_import, division, print_function
import collections

import numpy as np

import dwi.files
import dwi.util

def default_configuration():
    """Default standardization configuration."""
    return dict(
        pc=(0., 99.8),  # Min, max percentiles.
        landmarks=tuple(range(10, 100, 10)),  # Landmark percentiles.
        scale=(1, 4095),  # Min, max intensities on standard scale.
        )

def landmark_scores(img, pc1, pc2, landmarks, thresholding=True):
    """Get scores at histogram landmarks.

    Parameters
    ----------
    img : ndarray
        Model used for fitting.
    pc1, pc2 : number
        Minimum and maximum percentiles.
    landmarks : iterable of numbers
        Landmark percentiles.
    thresholding : bool, optional
        Whether to threshold by mean (default True). This includes only values
        higher than mean, which should help ignore the image background.

    Returns
    -------
    p1, p2 : float
        Minimum and maximum percentile scores.
    scores : tuple of floats
        Landmark percentile scores.
    """
    from scipy.stats import scoreatpercentile
    if thresholding:
        threshold = np.mean(img)
        img = img[img > threshold]
    p1 = scoreatpercentile(img, pc1)
    p2 = scoreatpercentile(img, pc2)
    scores = tuple(scoreatpercentile(img, i) for i in landmarks)
    return p1, p2, scores

def map_onto_scale(p1, p2, s1, s2, v):
    """Map value v from original scale [p1, p2] onto standard scale [s1, s2].

    Parameters
    ----------
    p1, p2 : number
        Minimum and maximum percentile scores.
    s1, s2 : number
        Minimum and maximum intensities on the standard scale.
    v : number
        Value to map.

    Returns
    -------
    r : float
        Mapped value.
    """
    assert p1 <= p2, (p1, p2)
    assert s1 <= s2, (s1, s2)
    if p1 == p2:
        assert s1 == s2, (s1, s2)
        return s1
    f = (v-p1) / (p2-p1)
    r = f * (s2-s1) + s1
    return r

def transform(img, p1, p2, scores, s1, s2, mapped_scores):
    """Transform image onto standard scale.

    Parameters
    ----------
    img : ndarray
        Image to transform.
    p1, p2 : number
        Minimum and maximum percentile scores.
    scores : iterable of numbers
        Landmark percentile scores.
    s1, s2 : number
        Minimum and maximum intensities on the standard scale.
    mapped_scores : iterable of numbers
        Standard landmark percentile scores on the standard scale.

    Returns
    -------
    r : ndarray of integers
        Transformed image.
    """
    scores = [p1] + list(scores) + [p2]
    mapped_scores = [s1] + list(mapped_scores) + [s2]
    r = np.zeros_like(img, dtype=np.int)
    for pos, v in np.ndenumerate(img):
        # Select slot where to map.
        slot = sum(v > s for s in scores)
        slot = np.clip(slot, 1, len(scores)-1)
        r[pos] = map_onto_scale(scores[slot-1], scores[slot],
                                mapped_scores[slot-1], mapped_scores[slot], v)
    return r

def standardize(img, cfg):
    """Transform an image based on a configuration (file).

    Parameters
    ----------
    img : ndarray
        Image to transform.
    cfg : filename (or dict)
        Standardization configuration file (or configuration already read).

    Returns
    -------
    img : ndarray of integers
        Transformed image.
    """
    if isinstance(cfg, basestring):
        cfg = read_std_cfg(cfg)
    d = cfg
    p1, p2, scores = landmark_scores(img, d['pc1'], d['pc2'], d['landmarks'])
    img = transform(img, p1, p2, scores, d['s1'], d['s2'], d['mapped_scores'])
    return img

def write_std_cfg(filename, pc1, pc2, landmarks, s1, s2, mapped_scores):
    """Write image standardization configuration file.

    Parameters
    ----------
    filename : string
        Output filename.
    pc1, pc2 : number
        Minimum and maximum percentiles.
    landmarks : iterable of numbers
        Landmark percentiles.
    s1, s2 : number
        Minimum and maximum intensities on the standard scale.
    mapped_scores : iterable of numbers
        Standard landmark percentile scores on the standard scale.
    """
    with open(filename, 'w') as f:
        f.write(dwi.files.toline((pc1, pc2)))
        f.write(dwi.files.toline(landmarks))
        f.write(dwi.files.toline((s1, s2)))
        f.write(dwi.files.toline(mapped_scores))

def read_std_cfg(filename):
    """Read image standardization configuration file.

    Parameters
    ----------
    filename : string
        Input filename.

    Returns
    -------
    d : OrderedDict
        Standardization configuration.
    """
    lines = list(dwi.files.valid_lines(filename))[:4]
    lines = [l.split() for l in lines]
    d = collections.OrderedDict()
    d['pc1'], d['pc2'] = (float(x) for x in lines[0])
    d['landmarks'] = tuple(float(x) for x in lines[1])
    d['s1'], d['s2'] = (int(x) for x in lines[2])
    d['mapped_scores'] = tuple(int(x) for x in lines[3])
    if len(d['landmarks']) != len(d['mapped_scores']):
        raise Exception('Invalid standardization file: {}'.format(filename))
    return d
<commit_msg>standardizing outputs int16 instead of int32<commit_after>"""Image standardization.

This module implements the percentile landmark method described in [1]. Much of
the notation (variable names etc.) comes from this paper.

Learning:
    - Use landmark_scores() to get percentile scores for all images.
    - Use map_onto_scale() to map landmark scores to the standard scale.
    - Average mapped landmarks to get standard landmarks. TODO: This should be
      implemented here somehow.
    - Use write_std_cfg() to output configuration.

Standardizing:
    - Use read_std_cfg() to input configuration.
    - Use standardize() to standardize images according to configuration.
    - You can also use transform() to do this more free-form.

See also tools/standardize.py.

[1] Nyul et al. 2000: New variants of a method of MRI scale standardization.
"""

from __future__ import absolute_import, division, print_function
import collections

import numpy as np

import dwi.files
import dwi.util

def default_configuration():
    """Default standardization configuration."""
    return dict(
        pc=(0., 99.8),  # Min, max percentiles.
        landmarks=tuple(range(10, 100, 10)),  # Landmark percentiles.
        scale=(1, 4095),  # Min, max intensities on standard scale.
        )

def landmark_scores(img, pc1, pc2, landmarks, thresholding=True):
    """Get scores at histogram landmarks.

    Parameters
    ----------
    img : ndarray
        Model used for fitting.
    pc1, pc2 : number
        Minimum and maximum percentiles.
    landmarks : iterable of numbers
        Landmark percentiles.
    thresholding : bool, optional
        Whether to threshold by mean (default True). This includes only values
        higher than mean, which should help ignore the image background.

    Returns
    -------
    p1, p2 : float
        Minimum and maximum percentile scores.
    scores : tuple of floats
        Landmark percentile scores.
    """
    from scipy.stats import scoreatpercentile
    if thresholding:
        threshold = np.mean(img)
        img = img[img > threshold]
    p1 = scoreatpercentile(img, pc1)
    p2 = scoreatpercentile(img, pc2)
    scores = tuple(scoreatpercentile(img, i) for i in landmarks)
    return p1, p2, scores

def map_onto_scale(p1, p2, s1, s2, v):
    """Map value v from original scale [p1, p2] onto standard scale [s1, s2].

    Parameters
    ----------
    p1, p2 : number
        Minimum and maximum percentile scores.
    s1, s2 : number
        Minimum and maximum intensities on the standard scale.
    v : number
        Value to map.

    Returns
    -------
    r : float
        Mapped value.
    """
    assert p1 <= p2, (p1, p2)
    assert s1 <= s2, (s1, s2)
    if p1 == p2:
        assert s1 == s2, (s1, s2)
        return s1
    f = (v-p1) / (p2-p1)
    r = f * (s2-s1) + s1
    return r

def transform(img, p1, p2, scores, s1, s2, mapped_scores):
    """Transform image onto standard scale.

    Parameters
    ----------
    img : ndarray
        Image to transform.
    p1, p2 : number
        Minimum and maximum percentile scores.
    scores : iterable of numbers
        Landmark percentile scores.
    s1, s2 : number
        Minimum and maximum intensities on the standard scale.
    mapped_scores : iterable of numbers
        Standard landmark percentile scores on the standard scale.

    Returns
    -------
    r : ndarray of integers
        Transformed image.
    """
    scores = [p1] + list(scores) + [p2]
    mapped_scores = [s1] + list(mapped_scores) + [s2]
    r = np.zeros_like(img, dtype=np.int16)
    for pos, v in np.ndenumerate(img):
        # Select slot where to map.
        slot = sum(v > s for s in scores)
        slot = np.clip(slot, 1, len(scores)-1)
        r[pos] = map_onto_scale(scores[slot-1], scores[slot],
                                mapped_scores[slot-1], mapped_scores[slot], v)
    return r

def standardize(img, cfg):
    """Transform an image based on a configuration (file).

    Parameters
    ----------
    img : ndarray
        Image to transform.
    cfg : filename (or dict)
        Standardization configuration file (or configuration already read).

    Returns
    -------
    img : ndarray of integers
        Transformed image.
    """
    if isinstance(cfg, basestring):
        cfg = read_std_cfg(cfg)
    d = cfg
    p1, p2, scores = landmark_scores(img, d['pc1'], d['pc2'], d['landmarks'])
    img = transform(img, p1, p2, scores, d['s1'], d['s2'], d['mapped_scores'])
    return img

def write_std_cfg(filename, pc1, pc2, landmarks, s1, s2, mapped_scores):
    """Write image standardization configuration file.

    Parameters
    ----------
    filename : string
        Output filename.
    pc1, pc2 : number
        Minimum and maximum percentiles.
    landmarks : iterable of numbers
        Landmark percentiles.
    s1, s2 : number
        Minimum and maximum intensities on the standard scale.
    mapped_scores : iterable of numbers
        Standard landmark percentile scores on the standard scale.
    """
    with open(filename, 'w') as f:
        f.write(dwi.files.toline((pc1, pc2)))
        f.write(dwi.files.toline(landmarks))
        f.write(dwi.files.toline((s1, s2)))
        f.write(dwi.files.toline(mapped_scores))

def read_std_cfg(filename):
    """Read image standardization configuration file.

    Parameters
    ----------
    filename : string
        Input filename.

    Returns
    -------
    d : OrderedDict
        Standardization configuration.
    """
    lines = list(dwi.files.valid_lines(filename))[:4]
    lines = [l.split() for l in lines]
    d = collections.OrderedDict()
    d['pc1'], d['pc2'] = (float(x) for x in lines[0])
    d['landmarks'] = tuple(float(x) for x in lines[1])
    d['s1'], d['s2'] = (int(x) for x in lines[2])
    d['mapped_scores'] = tuple(int(x) for x in lines[3])
    if len(d['landmarks']) != len(d['mapped_scores']):
        raise Exception('Invalid standardization file: {}'.format(filename))
    return d
<|endoftext|>


<commit_before>"""Image standardization.

This module implements the percentile landmark method described in [1]. Much of
the notation (variable names etc.) comes from this paper.

Learning:
    - Use landmark_scores() to get percentile scores for all images.
    - Use map_onto_scale() to map landmark scores to the standard scale.
    - Average mapped landmarks to get standard landmarks. TODO: This should be
      implemented here somehow.
    - Use write_std_cfg() to output configuration.

Standardizing:
    - Use read_std_cfg() to input configuration.
    - Use standardize() to standardize images according to configuration.
    - You can also use transform() to do this more free-form.

See also tools/standardize.py.

[1] Nyul et al. 2000: New variants of a method of MRI scale standardization.
"""

# TODO: Mention that the whole image must be included when standardizing.
# TODO: Replace scoreatpercentile() with np.percentile().

from __future__ import absolute_import, division, print_function
import collections

import numpy as np

import dwi.files
import dwi.util

def default_configuration():
    """Default standardization configuration."""
    return dict(
        pc=(0., 99.8),  # Min, max percentiles.
        landmarks=tuple(range(10, 100, 10)),  # Landmark percentiles.
        scale=(1, 4095),  # Min, max intensities on standard scale.
        )

def landmark_scores(img, pc1, pc2, landmarks, thresholding=True):
    """Get scores at histogram landmarks.

    Parameters
    ----------
    img : ndarray
        Model used for fitting.
    pc1, pc2 : number
        Minimum and maximum percentiles.
    landmarks : iterable of numbers
        Landmark percentiles.
    thresholding : bool, optional
        Whether to threshold by mean (default True). This includes only values
        higher than mean, which should help ignore the image background.

    Returns
    -------
    p1, p2 : float
        Minimum and maximum percentile scores.
    scores : tuple of floats
        Landmark percentile scores.
    """
    from scipy.stats import scoreatpercentile
    if thresholding:
        threshold = np.mean(img)
        img = img[img > threshold]
    p1 = scoreatpercentile(img, pc1)
    p2 = scoreatpercentile(img, pc2)
    scores = tuple(scoreatpercentile(img, i) for i in landmarks)
    return p1, p2, scores

def map_onto_scale(p1, p2, s1, s2, v):
    """Map value v from original scale [p1, p2] onto standard scale [s1, s2].

    Parameters
    ----------
    p1, p2 : number
        Minimum and maximum percentile scores.
    s1, s2 : number
        Minimum and maximum intensities on the standard scale.
    v : number
        Value to map.

    Returns
    -------
    r : float
        Mapped value.
    """
    assert p1 <= p2, (p1, p2)
    assert s1 <= s2, (s1, s2)
    if p1 == p2:
        assert s1 == s2, (p1, p2, s1, s2)
        return s1
    f = (v-p1) / (p2-p1)
    r = f * (s2-s1) + s1
    return r

def transform(img, p1, p2, scores, s1, s2, mapped_scores):
    """Transform image onto standard scale.

    Parameters
    ----------
    img : ndarray
        Image to transform.
    p1, p2 : number
        Minimum and maximum percentile scores.
    scores : iterable of numbers
        Landmark percentile scores.
    s1, s2 : number
        Minimum and maximum intensities on the standard scale.
    mapped_scores : iterable of numbers
        Standard landmark percentile scores on the standard scale.

    Returns
    -------
    r : ndarray of integers
        Transformed image.
    """
    scores = [p1] + list(scores) + [p2]
    mapped_scores = [s1] + list(mapped_scores) + [s2]
    r = np.zeros_like(img, dtype=np.int16)
    for pos, v in np.ndenumerate(img):
        # Select slot where to map.
        slot = sum(v > s for s in scores)
        slot = np.clip(slot, 1, len(scores)-1)
        r[pos] = map_onto_scale(scores[slot-1], scores[slot],
                                mapped_scores[slot-1], mapped_scores[slot], v)
    r = np.clip(r, s1-1, s2+1, out=r)
    return r

def standardize(img, cfg):
    """Transform an image based on a configuration (file).

    Parameters
    ----------
    img : ndarray
        Image to transform.
    cfg : filename (or dict)
        Standardization configuration file (or configuration already read).

    Returns
    -------
    img : ndarray of integers
        Transformed image.
    """
    if isinstance(cfg, basestring):
        cfg = read_std_cfg(cfg)
    d = cfg
    p1, p2, scores = landmark_scores(img, d['pc1'], d['pc2'], d['landmarks'])
    img = transform(img, p1, p2, scores, d['s1'], d['s2'], d['mapped_scores'])
    return img

def write_std_cfg(filename, pc1, pc2, landmarks, s1, s2, mapped_scores):
    """Write image standardization configuration file.

    Parameters
    ----------
    filename : string
        Output filename.
    pc1, pc2 : number
        Minimum and maximum percentiles.
    landmarks : iterable of numbers
        Landmark percentiles.
    s1, s2 : number
        Minimum and maximum intensities on the standard scale.
    mapped_scores : iterable of numbers
        Standard landmark percentile scores on the standard scale.
    """
    with open(filename, 'w') as f:
        f.write(dwi.files.toline((pc1, pc2)))
        f.write(dwi.files.toline(landmarks))
        f.write(dwi.files.toline((s1, s2)))
        f.write(dwi.files.toline(mapped_scores))

def read_std_cfg(filename):
    """Read image standardization configuration file.

    Parameters
    ----------
    filename : string
        Input filename.

    Returns
    -------
    d : OrderedDict
        Standardization configuration.
    """
    lines = list(dwi.files.valid_lines(filename))[:4]
    lines = [l.split() for l in lines]
    d = collections.OrderedDict()
    d['pc1'], d['pc2'] = (float(x) for x in lines[0])
    d['landmarks'] = tuple(float(x) for x in lines[1])
    d['s1'], d['s2'] = (int(x) for x in lines[2])
    d['mapped_scores'] = tuple(int(x) for x in lines[3])
    if len(d['landmarks']) != len(d['mapped_scores']):
        raise Exception('Invalid standardization file: {}'.format(filename))
    return d
<commit_msg>threshold by median instead of mean, seems to work better<commit_after>"""Image standardization.

This module implements the percentile landmark method described in [1]. Much of
the notation (variable names etc.) comes from this paper.

Learning:
    - Use landmark_scores() to get percentile scores for all images.
    - Use map_onto_scale() to map landmark scores to the standard scale.
    - Average mapped landmarks to get standard landmarks. TODO: This should be
      implemented here somehow.
    - Use write_std_cfg() to output configuration.

Standardizing:
    - Use read_std_cfg() to input configuration.
    - Use standardize() to standardize images according to configuration.
    - You can also use transform() to do this more free-form.

See also tools/standardize.py.

[1] Nyul et al. 2000: New variants of a method of MRI scale standardization.
"""

# TODO: Mention that the whole image must be included when standardizing.
# TODO: Replace scoreatpercentile() with np.percentile().

from __future__ import absolute_import, division, print_function
import collections

import numpy as np

import dwi.files
import dwi.util

def default_configuration():
    """Default standardization configuration."""
    return dict(
        pc=(0., 99.8),  # Min, max percentiles.
        landmarks=tuple(range(10, 100, 10)),  # Landmark percentiles.
        scale=(1, 4095),  # Min, max intensities on standard scale.
        )

def landmark_scores(img, pc1, pc2, landmarks, thresholding=True):
    """Get scores at histogram landmarks.

    Parameters
    ----------
    img : ndarray
        Model used for fitting.
    pc1, pc2 : number
        Minimum and maximum percentiles.
    landmarks : iterable of numbers
        Landmark percentiles.
    thresholding : bool, optional
        Whether to threshold by mean (default True). This includes only values
        higher than mean, which should help ignore the image background.

    Returns
    -------
    p1, p2 : float
        Minimum and maximum percentile scores.
    scores : tuple of floats
        Landmark percentile scores.
    """
    from scipy.stats import scoreatpercentile
    if thresholding:
        # threshold = np.mean(img)
        threshold = np.median(img)
        img = img[img > threshold]
    p1 = scoreatpercentile(img, pc1)
    p2 = scoreatpercentile(img, pc2)
    scores = tuple(scoreatpercentile(img, i) for i in landmarks)
    return p1, p2, scores

def map_onto_scale(p1, p2, s1, s2, v):
    """Map value v from original scale [p1, p2] onto standard scale [s1, s2].

    Parameters
    ----------
    p1, p2 : number
        Minimum and maximum percentile scores.
    s1, s2 : number
        Minimum and maximum intensities on the standard scale.
    v : number
        Value to map.

    Returns
    -------
    r : float
        Mapped value.
    """
    assert p1 <= p2, (p1, p2)
    assert s1 <= s2, (s1, s2)
    if p1 == p2:
        assert s1 == s2, (p1, p2, s1, s2)
        return s1
    f = (v-p1) / (p2-p1)
    r = f * (s2-s1) + s1
    return r

def transform(img, p1, p2, scores, s1, s2, mapped_scores):
    """Transform image onto standard scale.

    Parameters
    ----------
    img : ndarray
        Image to transform.
    p1, p2 : number
        Minimum and maximum percentile scores.
    scores : iterable of numbers
        Landmark percentile scores.
    s1, s2 : number
        Minimum and maximum intensities on the standard scale.
    mapped_scores : iterable of numbers
        Standard landmark percentile scores on the standard scale.

    Returns
    -------
    r : ndarray of integers
        Transformed image.
    """
    scores = [p1] + list(scores) + [p2]
    mapped_scores = [s1] + list(mapped_scores) + [s2]
    r = np.zeros_like(img, dtype=np.int16)
    for pos, v in np.ndenumerate(img):
        # Select slot where to map.
        slot = sum(v > s for s in scores)
        slot = np.clip(slot, 1, len(scores)-1)
        r[pos] = map_onto_scale(scores[slot-1], scores[slot],
                                mapped_scores[slot-1], mapped_scores[slot], v)
    r = np.clip(r, s1-1, s2+1, out=r)
    return r

def standardize(img, cfg):
    """Transform an image based on a configuration (file).

    Parameters
    ----------
    img : ndarray
        Image to transform.
    cfg : filename (or dict)
        Standardization configuration file (or configuration already read).

    Returns
    -------
    img : ndarray of integers
        Transformed image.
    """
    if isinstance(cfg, basestring):
        cfg = read_std_cfg(cfg)
    d = cfg
    p1, p2, scores = landmark_scores(img, d['pc1'], d['pc2'], d['landmarks'])
    img = transform(img, p1, p2, scores, d['s1'], d['s2'], d['mapped_scores'])
    return img

def write_std_cfg(filename, pc1, pc2, landmarks, s1, s2, mapped_scores):
    """Write image standardization configuration file.

    Parameters
    ----------
    filename : string
        Output filename.
    pc1, pc2 : number
        Minimum and maximum percentiles.
    landmarks : iterable of numbers
        Landmark percentiles.
    s1, s2 : number
        Minimum and maximum intensities on the standard scale.
    mapped_scores : iterable of numbers
        Standard landmark percentile scores on the standard scale.
    """
    with open(filename, 'w') as f:
        f.write(dwi.files.toline((pc1, pc2)))
        f.write(dwi.files.toline(landmarks))
        f.write(dwi.files.toline((s1, s2)))
        f.write(dwi.files.toline(mapped_scores))

def read_std_cfg(filename):
    """Read image standardization configuration file.

    Parameters
    ----------
    filename : string
        Input filename.

    Returns
    -------
    d : OrderedDict
        Standardization configuration.
    """
    lines = list(dwi.files.valid_lines(filename))[:4]
    lines = [l.split() for l in lines]
    d = collections.OrderedDict()
    d['pc1'], d['pc2'] = (float(x) for x in lines[0])
    d['landmarks'] = tuple(float(x) for x in lines[1])
    d['s1'], d['s2'] = (int(x) for x in lines[2])
    d['mapped_scores'] = tuple(int(x) for x in lines[3])
    if len(d['landmarks']) != len(d['mapped_scores']):
        raise Exception('Invalid standardization file: {}'.format(filename))
    return d
<|endoftext|>


<commit_before>import logging

class PSOutput():
    def __init__(self, config, queues):
        self.threadOps = queues["threadControl"]
        self.toPrint = queues["toPrint"]
        self.config = config

        self.logger = logging.getLogger("PrinterOutput")

        self.run()

    def run(self):
        while(True):
            jid = self.toPrint.get(block=True)
            self.logger.debug("Got print request for job %s", jid)
            self.printJob(jid)
            self.logger.debug("Printed job %s", jid)
            self.rmJob(jid)

    def printJob(self, jid):
        destPrinter = self.getDestPrinter(jid)
        printer = self.config["printers"][destPrinter]["address"]
        port =  self.config["printers"][destPrinter]["port"]
        self.logger.debug("Sending %s to %s", jid, destPrinter)

        try:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            ps = self.getPS(jid)
            s.connect((printer, port))
            #s.sendall(ps)
            s.close()
        except Exception as e:
            self.logger.critical("Encountered error while printing: %s", e)

    def getDestPrinter(self, jid):
        f = open(jid)
        j = json.load(f)
        f.close()
        return j["destPrinter"]

    def getPS(self, jid):
        f = open(jid)
        j = json.load(f)
        f.close()
        return j["postscript"]

    def rmJob(self, jid):
        self.logger.debug("Removing spool file for %s", jid)
        os.remove(jid)
        self.logger.info("Completed handling of %s", jid)
<commit_msg>resolve dependancies with psoutput class<commit_after>import logging
import json
import os
import socket

class PSOutput():
    def __init__(self, config, queues):
        self.threadOps = queues["threadControl"]
        self.toPrint = queues["toPrint"]
        self.config = config

        self.logger = logging.getLogger("PrinterOutput")

        self.run()

    def run(self):
        while(True):
            jid = self.toPrint.get(block=True)
            self.logger.debug("Got print request for job %s", jid)
            self.printJob(jid)
            self.logger.debug("Printed job %s", jid)
            self.rmJob(jid)

    def printJob(self, jid):
        destPrinter = self.getDestPrinter(jid)
        printer = self.config["printers"][destPrinter]["address"]
        port =  self.config["printers"][destPrinter]["port"]
        self.logger.debug("Sending %s to %s", jid, destPrinter)

        try:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            ps = self.getPS(jid)
            s.connect((printer, port))
            #s.sendall(ps)
            s.close()
        except Exception as e:
            self.logger.critical("Encountered error while printing: %s", e)

    def getDestPrinter(self, jid):
        f = open(jid)
        j = json.load(f)
        f.close()
        return j["destPrinter"]

    def getPS(self, jid):
        f = open(jid)
        j = json.load(f)
        f.close()
        return j["postscript"]

    def rmJob(self, jid):
        self.logger.debug("Removing spool file for %s", jid)
        os.remove(jid)
        self.logger.info("Completed handling of %s", jid)
<|endoftext|>


<commit_before>"Build up the sphinx autodoc file for the python code"

import os
import sys

docs_folder = os.path.dirname(__file__)
pywin_folder = os.path.dirname(docs_folder)

sys.path.append(pywin_folder)
pywin_folder = os.path.join(pywin_folder, "pywinauto")

excluded_dirs = ["unittests"]
excluded_files = [
    "_menux.py",
    "__init__.py",
    "win32defines.py",
    "win32structures.py",
    "win32functions.py"]

output_folder = "code"

try:
    os.mkdir(output_folder)
except WindowsError:
    pass

module_docs = []
for root, dirs, files in os.walk(pywin_folder):

    # Skip over directories we don't want to document
    for i, d in enumerate(dirs):
        if d in excluded_dirs:
            del dirs[i]

    py_files = [f for f in files if f.endswith(".py")]

    # skip over py files we don't want to document
    for filename in py_files:
        if filename in excluded_files:
            continue

        doc_source_filename = os.path.join(
            output_folder, filename + ".txt")
        if os.path.exists(doc_source_filename):
            continue

        print filename

        filepath =  os.path.join(root, filename)

        # find the last instance of 'pywinauto'
        modulename = 'pywinauto' + filepath.rsplit("pywinauto", 1)[1]
        modulename = os.path.splitext(modulename)[0]
        modulename = modulename.replace('\\', '.')

        out = open(doc_source_filename, "w")

        out.write(modulename + "\n")
        out.write("-" * len(modulename) + "\n")
        out.write(" .. automodule:: %s\n"% modulename)
        out.write("    :members:\n")
        out.write("    :undoc-members:\n\n")
        #out.write("    :inherited-members:\n")
        #out.write(" .. autoattribute:: %s\n"% modulename)
        out.close()

        module_docs.append(doc_source_filename)

# This section needs to be updated - I should idealy parse the
# existing file to see if any new docs have been added, if not then
# I should just leave the file alone rathre than re-create.
#
#c = open(os.path.join(output_folder, "code.txt"), "w")
#c.write("Source Code\n")
#c.write("=" * 30 + "\n")
#
#c.write(".. toctree::\n")
#c.write("   :maxdepth: 3\n\n")
#for doc in module_docs:
#    c.write("   " + doc + "\n")
#
#c.close()
#        <commit_msg>Ensure the output is the ?code? directory below this module<commit_after>"Build up the sphinx autodoc file for the python code"

import os
import sys

docs_folder = os.path.dirname(__file__)
pywin_folder = os.path.dirname(docs_folder)

sys.path.append(pywin_folder)
pywin_folder = os.path.join(pywin_folder, "pywinauto")

excluded_dirs = ["unittests"]
excluded_files = [
    "_menux.py",
    "__init__.py",
    "win32defines.py",
    "win32structures.py",
    "win32functions.py"]

output_folder = os.path.join(docs_folder, "code")

try:
    os.mkdir(output_folder)
except WindowsError:
    pass

module_docs = []
for root, dirs, files in os.walk(pywin_folder):

    # Skip over directories we don't want to document
    for i, d in enumerate(dirs):
        if d in excluded_dirs:
            del dirs[i]

    py_files = [f for f in files if f.endswith(".py")]

    # skip over py files we don't want to document
    for filename in py_files:
        if filename in excluded_files:
            continue

        doc_source_filename = os.path.join(
            output_folder, filename + ".txt")
        if os.path.exists(doc_source_filename):
            continue

        print filename

        filepath =  os.path.join(root, filename)

        # find the last instance of 'pywinauto'
        modulename = 'pywinauto' + filepath.rsplit("pywinauto", 1)[1]
        modulename = os.path.splitext(modulename)[0]
        modulename = modulename.replace('\\', '.')

        out = open(doc_source_filename, "w")

        out.write(modulename + "\n")
        out.write("-" * len(modulename) + "\n")
        out.write(" .. automodule:: %s\n"% modulename)
        out.write("    :members:\n")
        out.write("    :undoc-members:\n\n")
        #out.write("    :inherited-members:\n")
        #out.write(" .. autoattribute:: %s\n"% modulename)
        out.close()

        module_docs.append(doc_source_filename)

# This section needs to be updated - I should idealy parse the
# existing file to see if any new docs have been added, if not then
# I should just leave the file alone rathre than re-create.
#
#c = open(os.path.join(output_folder, "code.txt"), "w")
#c.write("Source Code\n")
#c.write("=" * 30 + "\n")
#
#c.write(".. toctree::\n")
#c.write("   :maxdepth: 3\n\n")
#for doc in module_docs:
#    c.write("   " + doc + "\n")
#
#c.close()
#        <|endoftext|>


<commit_before>import logging
import json
import re

class PSParser():
    def __init__(self):
        self.logger = logging.getLogger("PSParser")
        self.logger.info("Loaded PostScript Parser")

    def __getPSFromJID(self, jid):
        jobFile = open(jid, 'r')
        job = json.load(jobFile)
        jobFile.close()
        return job["postscript"]

    def isDuplex(self, jid):
        ps = self.__getPSFromJID(jid)
        if("/Duplex true" in ps):
            self.logger.debug("job %s is duplex enabled", jid)
            return True
        else:
            self.logger.debug("job %s is not duplex enabled", jid)
            return False

    def pageCount(self, jid):
        ps = self.__getPSFromJID(jid)
        numPages = None
        self.logger.debug("Computing page count for %s", jid)

        rgxresult = re.search('%%Pages: [0-9]*', ps)
        logging.debug("rgxresult: {0}".format(rgxresult.group(0)))

        if(rgxresult != None) :
            numPages = int(re.search('%%Pages: [0-9]*', ps).group(0).split(" ")[1])
            self.logger.debug("File is adobe compliant, suspect to be {0} pages".format(numPages))
        else:
            self.logger.error("File is not adobe compliant, page count indeterminate.")
        return numPages

<commit_msg>fix logging context in psparser<commit_after>import logging
import json
import re

class PSParser():
    def __init__(self):
        self.logger = logging.getLogger("PSParser")
        self.logger.info("Loaded PostScript Parser")

    def __getPSFromJID(self, jid):
        jobFile = open(jid, 'r')
        job = json.load(jobFile)
        jobFile.close()
        return job["postscript"]

    def isDuplex(self, jid):
        ps = self.__getPSFromJID(jid)
        if("/Duplex true" in ps):
            self.logger.debug("job %s is duplex enabled", jid)
            return True
        else:
            self.logger.debug("job %s is not duplex enabled", jid)
            return False

    def pageCount(self, jid):
        ps = self.__getPSFromJID(jid)
        numPages = None
        self.logger.debug("Computing page count for %s", jid)

        rgxresult = re.search('%%Pages: [0-9]*', ps)
        self.logger.debug("rgxresult: {0}".format(rgxresult.group(0)))

        if(rgxresult != None) :
            numPages = int(re.search('%%Pages: [0-9]*', ps).group(0).split(" ")[1])
            self.logger.debug("File is adobe compliant, suspect to be {0} pages".format(numPages))
        else:
            self.logger.error("File is not adobe compliant, page count indeterminate.")
        return numPages

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals

'''
========================
Feedback Exception Logic
========================

Scenario "consume" all the Input in the Executable, then:

+-----------------------------------------------------------+----------+
| Executable          | Scenario    | Feedback              | Invoke   |
+ ====================+=============+=======================+==========+
| EOF                 | More Input  | ShouldInputBeforeEOF  | Internal |
+---------------------+-------------+-----------------------+----------+
| EOF                 | More Output | ShouldOutputBeforeEOF | Internal |
+---------------------+-------------+-----------------------+----------+
| Input or            | EOF         | ShouldEOF             | Internal |
| Not Exiting         |             |                       |          |
+---------------------+-------------+-----------------------+----------+
| Input or            | More Output | ShouldOutput          | Internal |
| Not Exiting         |             |                       |          |
+---------------------+-------------+-----------------------+----------+
| Signal              |             | MemoryFeedbackError   | Internal |
+---------------------+-------------+-----------------------+----------+
| Timeout             |             | TimeoutFeedbackError  | External |
+---------------------+-------------+-----------------------+----------+
| Overflow            |             | OverflowFeedbackError | External |
+---------------------+-------------+-----------------------+----------+
'''

class FeedbackException(Exception):
    def __init__(self, msg, quote=None):
        Exception.__init__(self)
        self.quote = quote
        self.feedback = msg.format(**quote)

class InternalFeedbackException(FeedbackException):
    def __init__(self, msg, quote=None):
        FeedbackException.__init__(self, msg, quote)

class ShouldOutput(InternalFeedbackException):
    '''
    The output is not just before EOF
    '''

    msg = 'הפלט {name} ({value}) לא הופיע או הופיע במקום הלא מתאים.'

    def __init__(self, quote):
        InternalFeedbackException.__init__(self, ShouldOutput.msg, quote)

class ShouldEOF(InternalFeedbackException):
    msg = 'ריצת התכנית אמורה הייתה להסתיים לאחר הקלטים והפלטים שנבדקו, אך התכנית עדיין רצה.' + '\n' + \
          'אולי התכנית מחכה לקלט נוסף שהיא לא הייתה אמורה לקלוט? אולי יש לולאה אינסופית בקוד? אולי יש getchar מיותר בסוף ה-main?'

    def __init__(self, quote):
        InternalFeedbackException.__init__(self, ShouldEOF.msg, quote)

class ShouldOutputBeforeEOF(InternalFeedbackException):
    msg = 'ריצת התכנית הסתיימה, אך הפלט {name} ({value}) לא הופיע או הופיע במקום הלא מתאים לפי כן.'

    def __init__(self, quote):
        InternalFeedbackException.__init__(self, ShouldOutputBeforeEOF.msg, quote)

class SholdNoOutputBeforeInput(InternalFeedbackException):
    '''
    only in `flow = False`
    '''

    msg = 'התכנית לא הייתה אמורה להדפיס פלט לפני קבלת הקלט {name} ({value})'

    def __init__(self, quote):
        InternalFeedbackException.__init__(self, SholdNoOutputBeforeInput.msg, quote)

class ShouldInputBeforeEOF(InternalFeedbackException):
    msg = 'ריצת התכנית הסתיימה, אך התכנית אמורה הייתה לקלוט את הקלט {name} ({value}).'

    def __init__(self, quote):
        InternalFeedbackException.__init__(self, ShouldInputBeforeEOF.msg, quote)

class MemoryFeedbackError(InternalFeedbackException):
    msg = 'התרחשה שגיאת זיכרון.'

    def __init__(self):
        InternalFeedbackException.__init__(self, MemoryFeedbackError.msg, {})

class ExternalFeedbackException(FeedbackException):
    def __init__(self, msg, quote=None):
        FeedbackException.__init__(self, msg, quote)

class TimeoutFeedbackError(ExternalFeedbackException):
    msg = 'התכנית רצה יותר מדי זמן. אולי יש לולאה אינסופית בקוד?'

    def __init__(self):
        ExternalFeedbackException.__init__(self, TimeoutFeedbackError.msg, {})

class OverflowFeedbackError(ExternalFeedbackException):
    msg = 'התכנית השתמשה ביותר מדי זיכרון. אולי התכנית מדפיסה יותר מדי פלט? אולי יש לולאה אינסופית בקוד?.'

    def __init__(self):
        ExternalFeedbackException.__init__(self, OverflowFeedbackError.msg, {})
<commit_msg>removing extra . in feedback string<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals

'''
========================
Feedback Exception Logic
========================

Scenario "consume" all the Input in the Executable, then:

+-----------------------------------------------------------+----------+
| Executable          | Scenario    | Feedback              | Invoke   |
+ ====================+=============+=======================+==========+
| EOF                 | More Input  | ShouldInputBeforeEOF  | Internal |
+---------------------+-------------+-----------------------+----------+
| EOF                 | More Output | ShouldOutputBeforeEOF | Internal |
+---------------------+-------------+-----------------------+----------+
| Input or            | EOF         | ShouldEOF             | Internal |
| Not Exiting         |             |                       |          |
+---------------------+-------------+-----------------------+----------+
| Input or            | More Output | ShouldOutput          | Internal |
| Not Exiting         |             |                       |          |
+---------------------+-------------+-----------------------+----------+
| Signal              |             | MemoryFeedbackError   | Internal |
+---------------------+-------------+-----------------------+----------+
| Timeout             |             | TimeoutFeedbackError  | External |
+---------------------+-------------+-----------------------+----------+
| Overflow            |             | OverflowFeedbackError | External |
+---------------------+-------------+-----------------------+----------+
'''

class FeedbackException(Exception):
    def __init__(self, msg, quote=None):
        Exception.__init__(self)
        self.quote = quote
        self.feedback = msg.format(**quote)

class InternalFeedbackException(FeedbackException):
    def __init__(self, msg, quote=None):
        FeedbackException.__init__(self, msg, quote)

class ShouldOutput(InternalFeedbackException):
    '''
    The output is not just before EOF
    '''

    msg = 'הפלט {name} ({value}) לא הופיע או הופיע במקום הלא מתאים.'

    def __init__(self, quote):
        InternalFeedbackException.__init__(self, ShouldOutput.msg, quote)

class ShouldEOF(InternalFeedbackException):
    msg = 'ריצת התכנית אמורה הייתה להסתיים לאחר הקלטים והפלטים שנבדקו, אך התכנית עדיין רצה.' + '\n' + \
          'אולי התכנית מחכה לקלט נוסף שהיא לא הייתה אמורה לקלוט? אולי יש לולאה אינסופית בקוד? אולי יש getchar מיותר בסוף ה-main?'

    def __init__(self, quote):
        InternalFeedbackException.__init__(self, ShouldEOF.msg, quote)

class ShouldOutputBeforeEOF(InternalFeedbackException):
    msg = 'ריצת התכנית הסתיימה, אך הפלט {name} ({value}) לא הופיע או הופיע במקום הלא מתאים לפי כן.'

    def __init__(self, quote):
        InternalFeedbackException.__init__(self, ShouldOutputBeforeEOF.msg, quote)

class SholdNoOutputBeforeInput(InternalFeedbackException):
    '''
    only in `flow = False`
    '''

    msg = 'התכנית לא הייתה אמורה להדפיס פלט לפני קבלת הקלט {name} ({value})'

    def __init__(self, quote):
        InternalFeedbackException.__init__(self, SholdNoOutputBeforeInput.msg, quote)

class ShouldInputBeforeEOF(InternalFeedbackException):
    msg = 'ריצת התכנית הסתיימה, אך התכנית אמורה הייתה לקלוט את הקלט {name} ({value}).'

    def __init__(self, quote):
        InternalFeedbackException.__init__(self, ShouldInputBeforeEOF.msg, quote)

class MemoryFeedbackError(InternalFeedbackException):
    msg = 'התרחשה שגיאת זיכרון.'

    def __init__(self):
        InternalFeedbackException.__init__(self, MemoryFeedbackError.msg, {})

class ExternalFeedbackException(FeedbackException):
    def __init__(self, msg, quote=None):
        FeedbackException.__init__(self, msg, quote)

class TimeoutFeedbackError(ExternalFeedbackException):
    msg = 'התכנית רצה יותר מדי זמן. אולי יש לולאה אינסופית בקוד?'

    def __init__(self):
        ExternalFeedbackException.__init__(self, TimeoutFeedbackError.msg, {})

class OverflowFeedbackError(ExternalFeedbackException):
    msg = 'התכנית השתמשה ביותר מדי זיכרון. אולי התכנית מדפיסה יותר מדי פלט? אולי יש לולאה אינסופית בקוד?'

    def __init__(self):
        ExternalFeedbackException.__init__(self, OverflowFeedbackError.msg, {})
<|endoftext|>


<commit_before>'''
Created on 16Nov.,2016

@author: u76345
'''
from geophys_utils._array_pieces import array_pieces
from geophys_utils._data_stats import DataStats
from geophys_utils._netcdf_grid_utils import NetCDFGridUtils
from geophys_utils._netcdf_line_utils import NetCDFLineUtils
from geophys_utils._polygon_utils import get_grid_edge_points, get_netcdf_edge_points, points2convex_hull, points2alpha_shape, netcdf2convex_hull
from geophys_utils._crs_utils import get_spatial_ref_from_crs, get_coordinate_transformation, get_utm_crs, transform_coords
from geophys_utils._gdal_grid_utils import get_gdal_wcs_dataset, get_gdal_grid_values
from geophys_utils._transect_utils import line_length, point_along_line, utm_coords, coords2distance, sample_transect<commit_msg>New classes added to geophys_utils __init__.py<commit_after>'''
Created on 16Nov.,2016

@author: u76345
'''
from geophys_utils._netcdf_utils import NetCDFUtils
from geophys_utils._netcdf_grid_utils import NetCDFGridUtils
from geophys_utils._netcdf_line_utils import NetCDFLineUtils
from geophys_utils._csw_utils import CSWUtils
from geophys_utils._array_pieces import array_pieces
from geophys_utils._data_stats import DataStats
from geophys_utils._polygon_utils import get_grid_edge_points, get_netcdf_edge_points, points2convex_hull, points2alpha_shape, netcdf2convex_hull
from geophys_utils._crs_utils import get_spatial_ref_from_crs, get_coordinate_transformation, get_utm_crs, transform_coords
from geophys_utils._gdal_grid_utils import get_gdal_wcs_dataset, get_gdal_grid_values
from geophys_utils._transect_utils import line_length, point_along_line, utm_coords, coords2distance, sample_transect<|endoftext|>