<commit_before>from __future__ import print_function

import re
import json
from dateutil.parser import parse
import string
from datetime import datetime
import pytz

import facepy
from facepy import GraphAPI

from frontend import write_html

# Put Facebook 'Access Token' in a plain text file ACCESS_TOKEN in same dir.
# To get an access token follow this SO answer:
# http://stackoverflow.com/a/16054555/1780891

with open('./ACCESS_TOKEN', 'r') as f:
    access_token = f.readline().rstrip('\n')

graph = GraphAPI(access_token)

def get_comments(post_id):
    base_query = post_id + '/comments'

    # scrape the first page
    print('scraping:', base_query)
    comments = graph.get(base_query)
    data = comments['data']
    return data

def get_picture(post_id, dir="."):
    base_query = post_id + '?fields=object_id'
    try:
        pic_id = graph.get(base_query)['object_id']
    except KeyError:
        return None

    try:
        pic = graph.get('{}/picture'.format(pic_id))

        f_name = "{}/{}.png".format(dir, pic_id)
        f_handle = open(f_name, "wb")
        f_handle.write(pic)
        f_handle.close()
        return "{}.png".format(pic_id)
    except facepy.FacebookError:
        return None

def get_link(post_id):
    base_query = post_id + '?fields=link'

    try:
        link = graph.get(base_query)['link']
    except KeyError:
        return None

    return link
def get_event(post_id) :
    event_id = post_id.split('_')[1]  #getting only the event id from the post id
    base_query = event_id
    event_dict = graph.get(base_query)
    DateTime = prettify_date([{'created_time': event_dict['start_time']}])
    message = "%s \nDate : %s\nTime : %s\nVenue : %s "  %  (event_dict['description'] ,DateTime[0]['real_time']  , DateTime[0]['real_date']  , event_dict['place']['name'])
    return message
def get_shared_post(post_id) :
    base_query = post_id + '?fields=parent_id'
    parent_id = graph.get(base_query)['parent_id']  #getting if of the original post
    query = parent_id + '?fields=message'
    original_message = graph.get(query)['message']
    return original_message

def get_feed(page_id, pages=10):
    # check last update time
    try:
        old_data = json.load(open('output/{}.json'.format(page_id), 'r'))
        last_post_time = parse(old_data[0]['created_time'])
    except FileNotFoundError:
        old_data = []
        last_post_time = parse("1950-01-01T12:05:06+0000")

    base_query = page_id + '/feed?limit=2'

    # scrape the first page
    print('scraping:', base_query)
    feed = graph.get(base_query)
    new_page_data = feed['data']

    data = []
    is_new_post = (parse(new_page_data[0]['created_time']) > last_post_time)

    if is_new_post:
        data = new_page_data

    # determine the next page
    next_page = feed['paging']['next']
    next_search = re.search('.*(\&until=[0-9]+)', next_page, re.IGNORECASE)
    if next_search:
        the_until_arg = next_search.group(1)

    pages = pages - 1

    # scrape the rest of the pages
    while (next_page is not False) and is_new_post and pages > 0:
        the_query = base_query + the_until_arg
        print('baking:', the_query)
        try:
            feed = graph.get(the_query)
            new_page_data = feed['data']
            is_new_post = (parse(new_page_data[0]['created_time']) > last_post_time)

            data.extend(new_page_data)
        except facepy.exceptions.OAuthError:
            print('start again at', the_query)
            break

        # determine the next page, until there isn't one
        try:
            next_page = feed['paging']['next']
            next_search = re.search('.*(\&until=[0-9]+)', next_page, re.IGNORECASE)
            if next_search:
                the_until_arg = next_search.group(1)
        except IndexError:
            print('last page...')
            next_page = False
        pages = pages - 1

    for post_dict in data:
        post_dict['pic'] = get_picture(post_dict['id'], dir='output')
        post_dict['link'] = get_link(post_dict['id'])

        try :  #Events and shared post have story key
            if "event" in post_dict['story'] :
                post_dict['message'] =   get_event(post_dict['id'])
            elif "shared" in post_dict['story'] :
               post_dict['message'] = '' + post_dict['story'] + '' + '\n\n' + get_shared_post(post_dict['id'])
        except KeyError :
            pass

    data.extend(old_data)

    data.sort(key=lambda x: parse(x['created_time']), reverse=True)

    json.dump(data, open('output/{}.json'.format(page_id), 'w'))

    return data

def remove_duplicates(data):

    uniq_data = []
    for i in range(0,len(data)):
        if data[i] not in uniq_data:
            uniq_data.append(data[i])

    return uniq_data

def utc_to_time(naive, timezone="Asia/Colombo"):
    return naive.replace(tzinfo=pytz.utc).astimezone(pytz.timezone(timezone))

def prettify_date(data):
  for i in range(0,len(data)):
    date = data[i]['created_time']
    p1 = date.split("T")[0]
    p2 = date.split("T")[1].split("+")[0]
    date = parse(p1 + " " + p2)
    date = utc_to_time(date,"Asia/Colombo")
    time = datetime.strptime(date.strftime("%H%M"), '%H%M').strftime('%I:%M%p').upper()
    data[i]['real_date'] = date.strftime("%d-%m-%Y ")
    data[i]['real_time'] = time
  return data

def get_aggregated_feed(pages):
    """
    Aggregates feeds give a list of pages and their ids.

    Input: A list of tuples
    Output: Combined list of posts sorted by timestamp
    """
    data = list()
    for page_name, _id in pages:
        page_data = get_feed(_id)
        for data_dict in page_data:
            data_dict['source'] = page_name
        data.extend(page_data)

    data.sort(key=lambda x: parse(x['created_time']), reverse=True)
    return data

if __name__ == "__main__":
    # Great thanks to https://gist.github.com/abelsonlive/4212647
    news_pages = [('The Scholar\'s Avenue', 'scholarsavenue'),
                   ('Awaaz IIT Kharagpur', 'awaaziitkgp'),
                   ('Technology Students Gymkhana', 'TSG.IITKharagpur'),
                   ('Technology IIT KGP', 'iitkgp.tech')  ]
    for_later = ['Cultural-IIT-Kharagpur']

    data = get_aggregated_feed(news_pages)
    data = remove_duplicates(data)
    data = prettify_date(data)

    json.dump(data, open('output/feed.json', 'w'))
    write_html(data, 'output/index.html')
<commit_msg>Updated get events to show name if description is not present<commit_after>from __future__ import print_function

import re
import json
from dateutil.parser import parse
import string
from datetime import datetime
import pytz

import facepy
from facepy import GraphAPI

from frontend import write_html

# Put Facebook 'Access Token' in a plain text file ACCESS_TOKEN in same dir.
# To get an access token follow this SO answer:
# http://stackoverflow.com/a/16054555/1780891

with open('./ACCESS_TOKEN', 'r') as f:
    access_token = f.readline().rstrip('\n')

graph = GraphAPI(access_token)

def get_comments(post_id):
    base_query = post_id + '/comments'

    # scrape the first page
    print('scraping:', base_query)
    comments = graph.get(base_query)
    data = comments['data']
    return data

def get_picture(post_id, dir="."):
    base_query = post_id + '?fields=object_id'
    try:
        pic_id = graph.get(base_query)['object_id']
    except KeyError:
        return None

    try:
        pic = graph.get('{}/picture'.format(pic_id))

        f_name = "{}/{}.png".format(dir, pic_id)
        f_handle = open(f_name, "wb")
        f_handle.write(pic)
        f_handle.close()
        return "{}.png".format(pic_id)
    except facepy.FacebookError:
        return None

def get_link(post_id):
    base_query = post_id + '?fields=link'

    try:
        link = graph.get(base_query)['link']
    except KeyError:
        return None

    return link
def get_event(post_id) :
    event_id = post_id.split('_')[1]  #getting only the event id from the post id
    base_query = event_id + '/'
    event_dict = graph.get(base_query)
    DateTime = prettify_date([{'created_time': event_dict['start_time']}])
    if 'description' in event_dict.keys() :  #checking if the event have description
    	message = "%s \nDate : %s\nTime : %s\nVenue : %s "  %  (event_dict['description'] ,DateTime[0]['real_time']  , DateTime[0]['real_date']  , event_dict['place']['name'])
    else :
    	message = "%s \nDate : %s\nTime : %s\nVenue : %s "  %  (event_dict['name'] ,DateTime[0]['real_time']  , DateTime[0]['real_date']  , event_dict['place']['name'])
    return message
def get_shared_post(post_id) :
    base_query = post_id + '?fields=parent_id'
    parent_id = graph.get(base_query)['parent_id']  #getting if of the original post
    query = parent_id + '?fields=message'
    original_message = graph.get(query)['message']
    return original_message

def get_feed(page_id, pages=10):
    # check last update time
    try:
        old_data = json.load(open('output/{}.json'.format(page_id), 'r'))
        last_post_time = parse(old_data[0]['created_time'])
    except FileNotFoundError:
        old_data = []
        last_post_time = parse("1950-01-01T12:05:06+0000")

    base_query = page_id + '/feed?limit=2'

    # scrape the first page
    print('scraping:', base_query)
    feed = graph.get(base_query)
    new_page_data = feed['data']

    data = []
    is_new_post = (parse(new_page_data[0]['created_time']) > last_post_time)

    if is_new_post:
        data = new_page_data

    # determine the next page
    next_page = feed['paging']['next']
    next_search = re.search('.*(\&until=[0-9]+)', next_page, re.IGNORECASE)
    if next_search:
        the_until_arg = next_search.group(1)

    pages = pages - 1

    # scrape the rest of the pages
    while (next_page is not False) and is_new_post and pages > 0:
        the_query = base_query + the_until_arg
        print('baking:', the_query)
        try:
            feed = graph.get(the_query)
            new_page_data = feed['data']
            is_new_post = (parse(new_page_data[0]['created_time']) > last_post_time)

            data.extend(new_page_data)
        except facepy.exceptions.OAuthError:
            print('start again at', the_query)
            break

        # determine the next page, until there isn't one
        try:
            next_page = feed['paging']['next']
            next_search = re.search('.*(\&until=[0-9]+)', next_page, re.IGNORECASE)
            if next_search:
                the_until_arg = next_search.group(1)
        except IndexError:
            print('last page...')
            next_page = False
        pages = pages - 1

    for post_dict in data:
        post_dict['pic'] = get_picture(post_dict['id'], dir='output')
        post_dict['link'] = get_link(post_dict['id'])
        try :  #Events and shared post have story key
            if "event" in post_dict['story'] :
            	post_dict['message'] = get_event(post_dict['id'])
            elif "shared" in post_dict['story'] :
                post_dict['message'] = '' + post_dict['story'] + '' + '\n\n' + get_shared_post(post_dict['id'])
        except KeyError :
            pass

    data.extend(old_data)

    data.sort(key=lambda x: parse(x['created_time']), reverse=True)

    json.dump(data, open('output/{}.json'.format(page_id), 'w'))

    return data

def remove_duplicates(data):

    uniq_data = []
    for i in range(0,len(data)):
        if data[i] not in uniq_data:
            uniq_data.append(data[i])

    return uniq_data

def utc_to_time(naive, timezone="Asia/Colombo"):
    return naive.replace(tzinfo=pytz.utc).astimezone(pytz.timezone(timezone))

def prettify_date(data):
  for i in range(0,len(data)):
    date = data[i]['created_time']
    p1 = date.split("T")[0]
    p2 = date.split("T")[1].split("+")[0]
    date = parse(p1 + " " + p2)
    date = utc_to_time(date,"Asia/Colombo")
    time = datetime.strptime(date.strftime("%H%M"), '%H%M').strftime('%I:%M%p').upper()
    data[i]['real_date'] = date.strftime("%d-%m-%Y ")
    data[i]['real_time'] = time
  return data

def get_aggregated_feed(pages):
    """
    Aggregates feeds give a list of pages and their ids.

    Input: A list of tuples
    Output: Combined list of posts sorted by timestamp
    """
    data = list()
    for page_name, _id in pages:
        page_data = get_feed(_id)
        for data_dict in page_data:
            data_dict['source'] = page_name
        data.extend(page_data)

    data.sort(key=lambda x: parse(x['created_time']), reverse=True)
    return data

if __name__ == "__main__":
    # Great thanks to https://gist.github.com/abelsonlive/4212647
    news_pages = [('The Scholar\'s Avenue', 'scholarsavenue'),
                   ('Awaaz IIT Kharagpur', 'awaaziitkgp'),
                   ('Technology Students Gymkhana', 'TSG.IITKharagpur'),
                   ('Technology IIT KGP', 'iitkgp.tech')  ]
    for_later = ['Cultural-IIT-Kharagpur']

    data = get_aggregated_feed(news_pages)
    data = remove_duplicates(data)
    data = prettify_date(data)

    json.dump(data, open('output/feed.json', 'w'))
    write_html(data, 'output/index.html')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
    Plugins engine
"""

from __future__ import print_function, unicode_literals, division

import collections
import glob
import os
import sys
import textwrap

from ff.utils import disp

class FFPluginError(Exception):
    """ Exception class for plugins.
    """
    # TODO: do not hardocde 'test'
    PREFIX = 'ffplugin_test_'

    def __init__(self, msg, name=None):
        super(FFPluginError, self).__init__(msg)
        self.plugin_name = name

    @staticmethod
    def _get_tb_filename(tb):
        return os.path.splitext(os.path.basename(tb.tb_frame.f_code.co_filename))[0]

    def get_plugin_name(self):
        if not self.plugin_name:
            tb = sys.exc_info()[2]
            i = 7
            while i > 0 and not self._get_tb_filename(tb).startswith(self.PREFIX):
                i -= 1
                tb = tb.tb_next

            self.plugin_name = self._get_tb_filename(tb)[len(self.PREFIX):]

        return self.plugin_name

class InvalidPluginsPath(FFPluginError):
    """ Exception for invalid plugins paths
    """
    def __init__(self, msg, path):
        super(InvalidPluginsPath, self).__init__(msg)
        self.path = path

class FFPlugin(object):
    """ Wrapper for custom plugin.

        Loads module, read data, bind custom argument and allow to easy run plugin.
    """

    def __init__(self, name, type_, **kw):
        """ Initializer.

            Allow to set values of instance:
                * name (required)
                * type_ (required)
                * action (optional) (will be overwrited by `FFPlugin`.`load`)
                * descr (optional) (will be overwrited by `FFPlugin`.`load`)
                * help (optional) (will be overwrited by `FFPlugin`.`load`)
                * argument (optional)
        """
        super(FFPlugin, self).__init__()

        self.name = name
        self.type = type_
        self.action = kw.get('action')
        self.descr = kw.get('descr', '')
        self.help = kw.get('help', '')
        self.argument = kw.get('argument')

        self.load()

    @staticmethod
    def _import(type_, name):
        """ Imports plugins module.

            Plugin's name is created from three parts:
            fixed prefix: 'ffplugin'
            plugin type (just 'test' right now)
            plugin name

            joined with underscore.

            Returns imported module.
        """
        _mod = __import__('_'.join(['ffplugin', type_, name]), {}, {}, [], 0)
        ## monkey patch - plugin doesn't need to import FFPluginError
        _mod.FFPluginError = FFPluginError

        return _mod

    def load(self):
        """ Load and initialize plugin with data from module.

            Set `descr`, `help` and `action`.
        """
        _module = self._import(self.type, self.name)

        self.descr = getattr(_module, 'PLUGIN_DESCR', '')
        if isinstance(self.descr, collections.Callable):
            self.descr = self.descr(self.name)

        self.help = getattr(_module, 'PLUGIN_HELP', '')
        if isinstance(self.help, collections.Callable):
            self.help = self.help(self.name)

        self.action = _module.plugin_action

    def run(self, path):
        """ Run plugins callable.

            Pass self.name, self.argument and path.
        """
        return self.action(self.name, self.argument, path)

class FFPlugins(list):
    """ List of plugins available for `ff`.

        Static fields:
            * list of paths where to search for plugins.
    """

    _paths = set()

    @staticmethod
    def _print_descr(item):
        """ Helper for FFPlugins.print_help/FFPlugins.print_list.
            Prints single plugin description
        """
        text = 'ff plugin: ' + item.name + (' - ' + textwrap.fill(item.descr) if item.descr else '')
        disp(text)

    def print_help(self):
        """ Print list of plugins with their help to STDOUT
        """
        for item in self:
            self._print_descr(item)
            if item.help:
                disp(item.help.rstrip() + os.linesep)

    def print_list(self):
        """ Print list of plugins with their short descriptions to STDOUT
        """
        for item in self:
            self._print_descr(item)

    @classmethod
    def path_add(cls, path):
        """ Append path to known plugins paths.
        """
        if path not in cls._paths:
            cls._paths.add(path)
            sys.path.append(path)

    @classmethod
    def _find_all_plugins(cls, type_):
        """ Helper for FFPlugins.find_all

            Search for every plugin in specified paths and returns it's names.
        """
        result = {}
        prefix_len = len('ffplugin_') + len(type_) + 1
        for path in cls._paths:
            if not os.path.isdir(path):
                continue

            for file_ in glob.glob(os.path.join(path, '_'.join(['ffplugin', type_, '*.py']))):
                plugin_name = os.path.basename(file_)[prefix_len:-3]
                ## paths order describe priority of plugins too (first found are most important)
                if plugin_name in result:
                    continue
                result[plugin_name] = True

        order = list(result.keys())
        order.sort()

        return order

    @classmethod
    def find_all(cls, type_):
        """ Find all plugins available for `ff` and return FFPlugins
            initialized with it.

            Every item is instance of FFPlugin.

            Uses `FFPlugins.find` to load plugins.
        """

        plugins_names = cls._find_all_plugins(type_)
        return cls.find(plugins_names, type_=type_)

    @classmethod
    def find(cls, names, type_):
        """ Load given plugins list and initialize `FFPlugins` with them.

            Returns `FFPlugins` instance.
        """
        plugins = cls()
        for plugin_name in names:
            plugins.append(FFPlugin(plugin_name, type_))

        return plugins
<commit_msg>add: parse_plugin_filename - extract name and type of plugin from filename<commit_after># -*- coding: utf-8 -*-

"""
    Plugins engine
"""

from __future__ import print_function, unicode_literals, division

import collections
import glob
import os
import sys
import textwrap

from ff.utils import disp

PluginMetaData = collections.namedtuple('PluginMetaData', ('type', 'name'))

def parse_plugin_filename(name):
    """
    Parse plugins filename and extract info from it.
    As name can be passed even full path, basename would be extracted
    and extension stripped.
    :param name:str
    :return:PluginMetaData:raise ValueError:
    """
    name = os.path.basename(name)
    name = os.path.splitext(name)[0]

    if not name.startswith('ffplugin_'):
        raise ValueError("Incorrect value: %s. Should begins with \"ffplugin_\"" % name)

    try:
        _, plugin_type, plugin_name = name.split('_', 2)
    except ValueError:
        raise ValueError("Incorrect plugin name: %s" % name)

    return PluginMetaData(plugin_type, plugin_name)

class FFPluginError(Exception):
    """ Exception class for plugins.
    """
    # TODO: do not hardocde 'test'
    PREFIX = 'ffplugin_test_'

    def __init__(self, msg, name=None):
        super(FFPluginError, self).__init__(msg)
        self.plugin_name = name

    @staticmethod
    def _get_tb_filename(tb):
        return os.path.splitext(os.path.basename(tb.tb_frame.f_code.co_filename))[0]

    def get_plugin_name(self):
        if not self.plugin_name:
            tb = sys.exc_info()[2]
            i = 7
            while i > 0 and not self._get_tb_filename(tb).startswith(self.PREFIX):
                i -= 1
                tb = tb.tb_next

            self.plugin_name = self._get_tb_filename(tb)[len(self.PREFIX):]

        return self.plugin_name

class InvalidPluginsPath(FFPluginError):
    """ Exception for invalid plugins paths
    """
    def __init__(self, msg, path):
        super(InvalidPluginsPath, self).__init__(msg)
        self.path = path

class FFPlugin(object):
    """ Wrapper for custom plugin.

        Loads module, read data, bind custom argument and allow to easy run plugin.
    """

    def __init__(self, name, type_, **kw):
        """ Initializer.

            Allow to set values of instance:
                * name (required)
                * type_ (required)
                * action (optional) (will be overwrited by `FFPlugin`.`load`)
                * descr (optional) (will be overwrited by `FFPlugin`.`load`)
                * help (optional) (will be overwrited by `FFPlugin`.`load`)
                * argument (optional)
        """
        super(FFPlugin, self).__init__()

        self.name = name
        self.type = type_
        self.action = kw.get('action')
        self.descr = kw.get('descr', '')
        self.help = kw.get('help', '')
        self.argument = kw.get('argument')

        self.load()

    @staticmethod
    def _import(type_, name):
        """ Imports plugins module.

            Plugin's name is created from three parts:
            fixed prefix: 'ffplugin'
            plugin type (just 'test' right now)
            plugin name

            joined with underscore.

            Returns imported module.
        """
        _mod = __import__('_'.join(['ffplugin', type_, name]), {}, {}, [], 0)
        ## monkey patch - plugin doesn't need to import FFPluginError
        _mod.FFPluginError = FFPluginError

        return _mod

    def load(self):
        """ Load and initialize plugin with data from module.

            Set `descr`, `help` and `action`.
        """
        _module = self._import(self.type, self.name)

        self.descr = getattr(_module, 'PLUGIN_DESCR', '')
        if isinstance(self.descr, collections.Callable):
            self.descr = self.descr(self.name)

        self.help = getattr(_module, 'PLUGIN_HELP', '')
        if isinstance(self.help, collections.Callable):
            self.help = self.help(self.name)

        self.action = _module.plugin_action

    def run(self, path):
        """ Run plugins callable.

            Pass self.name, self.argument and path.
        """
        return self.action(self.name, self.argument, path)

class FFPlugins(list):
    """ List of plugins available for `ff`.

        Static fields:
            * list of paths where to search for plugins.
    """

    _paths = set()

    @staticmethod
    def _print_descr(item):
        """ Helper for FFPlugins.print_help/FFPlugins.print_list.
            Prints single plugin description
        """
        text = 'ff plugin: ' + item.name + (' - ' + textwrap.fill(item.descr) if item.descr else '')
        disp(text)

    def print_help(self):
        """ Print list of plugins with their help to STDOUT
        """
        for item in self:
            self._print_descr(item)
            if item.help:
                disp(item.help.rstrip() + os.linesep)

    def print_list(self):
        """ Print list of plugins with their short descriptions to STDOUT
        """
        for item in self:
            self._print_descr(item)

    @classmethod
    def path_add(cls, path):
        """ Append path to known plugins paths.
        """
        if path not in cls._paths:
            cls._paths.add(path)
            sys.path.append(path)

    @classmethod
    def _find_all_plugins(cls, type_):
        """ Helper for FFPlugins.find_all

            Search for every plugin in specified paths and returns it's names.
        """
        result = {}
        prefix_len = len('ffplugin_') + len(type_) + 1
        for path in cls._paths:
            if not os.path.isdir(path):
                continue

            for file_ in glob.glob(os.path.join(path, '_'.join(['ffplugin', type_, '*.py']))):
                plugin_name = os.path.basename(file_)[prefix_len:-3]
                ## paths order describe priority of plugins too (first found are most important)
                if plugin_name in result:
                    continue
                result[plugin_name] = True

        order = list(result.keys())
        order.sort()

        return order

    @classmethod
    def find_all(cls, type_):
        """ Find all plugins available for `ff` and return FFPlugins
            initialized with it.

            Every item is instance of FFPlugin.

            Uses `FFPlugins.find` to load plugins.
        """

        plugins_names = cls._find_all_plugins(type_)
        return cls.find(plugins_names, type_=type_)

    @classmethod
    def find(cls, names, type_):
        """ Load given plugins list and initialize `FFPlugins` with them.

            Returns `FFPlugins` instance.
        """
        plugins = cls()
        for plugin_name in names:
            plugins.append(FFPlugin(plugin_name, type_))

        return plugins
<|endoftext|>


<commit_before># This file is part of RinohType, the Python document preparation system.
#
# Copyright (c) Brecht Machiels.
#
# Use of this source code is subject to the terms of the GNU Affero General
# Public License v3. See the LICENSE file or http://www.gnu.org/licenses/.

import struct

from .parse import OpenTypeTable, MultiFormatTable, Record
from .parse import fixed, int16, uint16, tag, glyph_id, offset, Packed
from .parse import array, context, context_array, indirect, indirect_array
from .layout import LayoutTable, ScriptListTable, FeatureListTable, LookupTable
from .layout import Coverage, ClassDefinition, Device
from ...util import cached_property

class ValueFormat(Packed):
    reader = uint16
    fields = [('XPlacement', 0x0001, bool),
              ('YPlacement', 0x0002, bool),
              ('XAdvance', 0x0004, bool),
              ('YAdvance', 0x0008, bool),
              ('XPlaDevice', 0x0010, bool),
              ('YPlaDevice', 0x0020, bool),
              ('XAdvDevice', 0x0040, bool),
              ('YAdvDevice', 0x0080, bool)]
    formats = {'XPlacement': 'h',
               'YPlacement': 'h',
               'XAdvance': 'h',
               'YAdvance': 'h',
               'XPlaDevice': 'H',
               'YPlaDevice': 'H',
               'XAdvDevice': 'H',
               'YAdvDevice': 'H'}

    @cached_property
    def data_format(self):
        data_format = ''
        for name, present in self.items():
            if present:
                data_format += self.formats[name]
        return data_format

    @cached_property
    def present_keys(self):
        keys = []
        for name, present in self.items():
            if present:
                keys.append(name)
        return keys

class ValueRecord(OpenTypeTable):
    formats = {'XPlacement': int16,
               'YPlacement': int16,
               'XAdvance': int16,
               'YAdvance': int16,
               'XPlaDevice': indirect(Device),
               'YPlaDevice': indirect(Device),
               'XAdvDevice': indirect(Device),
               'YAdvDevice': indirect(Device)}

    def __init__(self, file, value_format):
        super().__init__(file)
        for name, present in value_format.items():
            if present:
                self[name] = self.formats[name](file)

class Anchor(MultiFormatTable):
    entries = [('AnchorFormat', uint16),
               ('XCoordinate', int16),
               ('YCoordinate', int16)]
    formats = {2: [('AnchorPoint', uint16)],
               3: [('XDeviceTable', indirect(Device)),
                   ('YDeviceTable', indirect(Device))]}

class MarkRecord(Record):
    entries = [('Class', uint16),
               ('MarkAnchor', indirect(Anchor))]

class MarkArray(OpenTypeTable):
    entries = [('MarkCount', uint16),
               ('MarkRecord', context_array(MarkRecord, 'MarkCount'))]

class SingleAdjustmentSubtable(MultiFormatTable):
    entries = [('PosFormat', uint16),
               ('Coverage', indirect(Coverage)),
               ('ValueFormat', ValueFormat)]
    formats = {1: [('Value', context(ValueRecord, 'ValueFormat'))],
               2: [('ValueCount', uint16),
                   ('Value', context_array(ValueRecord, 'ValueCount',
                                           'ValueFormat'))]}

class PairSetTable(OpenTypeTable):
    entries = [('PairValueCount', uint16)]

    def __init__(self, file, file_offset, format_1, format_2):
        super().__init__(file, file_offset)
        record_format = format_1.data_format + format_2.data_format
        value_1_length = len(format_1)
        format_1_keys = format_1.present_keys
        format_2_keys = format_2.present_keys
        pvr_struct = struct.Struct('>H' + record_format)
        pvr_size = pvr_struct.size
        pvr_list = []
        self.by_second_glyph_id = {}
        for i in range(self['PairValueCount']):
            record_data = pvr_struct.unpack(file.read(pvr_size))
            second_glyph = record_data[0]
            value_1 = {}
            value_2 = {}
            for i, key in enumerate(format_1_keys):
                value_1[key] = record_data[1 + i]
            for i, key in enumerate(format_2_keys):
                value_2[key] = record_data[1 + value_1_length + i]
            pvr = {'Value1': value_1,
                   'Value2': value_2}
            pvr_list.append(pvr)
            self.by_second_glyph_id[second_glyph] = pvr
        self['PairValueRecord'] = pvr_list

class PairAdjustmentSubtable(MultiFormatTable):
    entries = [('PosFormat', uint16),
               ('Coverage', indirect(Coverage)),
               ('ValueFormat1', ValueFormat),
               ('ValueFormat2', ValueFormat)]
    formats = {1: [('PairSetCount', uint16),
                   ('PairSet', indirect_array(PairSetTable, 'PairSetCount',
                                              'ValueFormat1', 'ValueFormat2'))],
               2: [('ClassDef1', indirect(ClassDefinition)),
                   ('ClassDef2', indirect(ClassDefinition)),
                   ('Class1Count', uint16),
                   ('Class2Count', uint16)]}

    def __init__(self, file, file_offset=None):
        super().__init__(file, file_offset)
        format_1, format_2 = self['ValueFormat1'], self['ValueFormat2']
        if self['PosFormat'] == 2:
            record_format = format_1.data_format + format_2.data_format
            c2r_struct = struct.Struct('>' + record_format)
            c2r_size = c2r_struct.size
            value_1_length = len(format_1)
            format_1_keys = format_1.present_keys
            format_2_keys = format_2.present_keys
            class_1_record = []
            for i in range(self['Class1Count']):
                class_2_record = []
                for j in range(self['Class2Count']):
                    record_data = c2r_struct.unpack(file.read(c2r_size))
                    value_1 = {}
                    value_2 = {}
                    for i, key in enumerate(format_1_keys):
                        value_1[key] = record_data[i]
                    for i, key in enumerate(format_2_keys):
                        value_2[key] = record_data[value_1_length + i]
                    class_2_record.append({'Value1': value_1,
                                           'Value2': value_2})
                class_1_record.append(class_2_record)
            self['Class1Record'] = class_1_record

    def lookup(self, a_id, b_id):
        if self['PosFormat'] == 1:
            try:
                index = self['Coverage'].index(a_id)
            except ValueError:
                raise KeyError
            pair_value_record = self['PairSet'][index].by_second_glyph_id[b_id]
            return pair_value_record['Value1']['XAdvance']
        elif self['PosFormat'] == 2:
            a_class = self['ClassDef1'].class_number(a_id)
            b_class = self['ClassDef2'].class_number(b_id)
            class_2_record = self['Class1Record'][a_class][b_class]
            return class_2_record['Value1']['XAdvance']

class EntryExitRecord(OpenTypeTable):
    entries = [('EntryAnchor', indirect(Anchor)),
               ('ExitAnchor', indirect(Anchor, 'EntryExitCount'))]

class CursiveAttachmentSubtable(OpenTypeTable):
    entries = [('PosFormat', uint16),
               ('Coverage', indirect(Coverage)),
               ('EntryExitCount', uint16),
               ('EntryExitRecord', context_array(EntryExitRecord, 'EntryExitCount'))]

class MarkCoverage(OpenTypeTable):
    pass

class BaseCoverage(OpenTypeTable):
    pass

class Mark2Array(OpenTypeTable):
    pass

class BaseRecord(OpenTypeTable):
##    entries = [('BaseAnchor', indirect_array(Anchor, 'ClassCount'))]

    def __init__(self, file, file_offset, class_count):
        super().__init__(self, file, file_offset)
##        self['BaseAnchor'] = indirect_array(Anchor, 'ClassCount'])(file)

class BaseArray(OpenTypeTable):
    entries = [('BaseCount', uint16)]
##               ('BaseRecord', context_array(BaseRecord, 'BaseCount'))]

    def __init__(self, file, file_offset, class_count):
        super().__init__(self, file, file_offset)
        self['BaseRecord'] = array(BaseRecord, self['BaseCount'],
                                   class_count=class_count)(file)

class MarkToBaseAttachmentSubtable(OpenTypeTable):
    entries = [('PosFormat', uint16),
               ('MarkCoverage', indirect(MarkCoverage)),
               ('BaseCoverage', indirect(BaseCoverage)),
               ('ClassCount', uint16),
               ('MarkArray', indirect(MarkArray)),
               ('BaseArray', indirect(BaseArray, 'ClassCount'))]

class MarkToMarkAttachmentSubtable(OpenTypeTable):
    entries = [('PosFormat', uint16),
               ('Mark1Coverage', indirect(MarkCoverage)),
               ('Mark1Coverage', indirect(MarkCoverage)),
               ('ClassCount', uint16),
               ('Mark1Array', indirect(MarkArray)),
               ('Mark1Array', indirect(Mark2Array))]

class GposTable(LayoutTable):
    """Glyph positioning table"""
    tag = 'GPOS'
    lookup_types = {1: SingleAdjustmentSubtable,
                    2: PairAdjustmentSubtable,
                    3: CursiveAttachmentSubtable,
                    4: MarkToBaseAttachmentSubtable,
                    6: MarkToMarkAttachmentSubtable}
<commit_msg>OpenType: GPOS Extension Positioning table support<commit_after># This file is part of RinohType, the Python document preparation system.
#
# Copyright (c) Brecht Machiels.
#
# Use of this source code is subject to the terms of the GNU Affero General
# Public License v3. See the LICENSE file or http://www.gnu.org/licenses/.

import struct

from .parse import OpenTypeTable, MultiFormatTable, Record
from .parse import int16, uint16, ushort, ulong, Packed
from .parse import array, context, context_array, indirect, indirect_array
from .layout import LayoutTable, Coverage, ClassDefinition, Device
from ...util import cached_property

class ValueFormat(Packed):
    reader = uint16
    fields = [('XPlacement', 0x0001, bool),
              ('YPlacement', 0x0002, bool),
              ('XAdvance', 0x0004, bool),
              ('YAdvance', 0x0008, bool),
              ('XPlaDevice', 0x0010, bool),
              ('YPlaDevice', 0x0020, bool),
              ('XAdvDevice', 0x0040, bool),
              ('YAdvDevice', 0x0080, bool)]
    formats = {'XPlacement': 'h',
               'YPlacement': 'h',
               'XAdvance': 'h',
               'YAdvance': 'h',
               'XPlaDevice': 'H',
               'YPlaDevice': 'H',
               'XAdvDevice': 'H',
               'YAdvDevice': 'H'}

    @cached_property
    def data_format(self):
        data_format = ''
        for name, present in self.items():
            if present:
                data_format += self.formats[name]
        return data_format

    @cached_property
    def present_keys(self):
        keys = []
        for name, present in self.items():
            if present:
                keys.append(name)
        return keys

class ValueRecord(OpenTypeTable):
    formats = {'XPlacement': int16,
               'YPlacement': int16,
               'XAdvance': int16,
               'YAdvance': int16,
               'XPlaDevice': indirect(Device),
               'YPlaDevice': indirect(Device),
               'XAdvDevice': indirect(Device),
               'YAdvDevice': indirect(Device)}

    def __init__(self, file, value_format):
        super().__init__(file)
        for name, present in value_format.items():
            if present:
                self[name] = self.formats[name](file)

class Anchor(MultiFormatTable):
    entries = [('AnchorFormat', uint16),
               ('XCoordinate', int16),
               ('YCoordinate', int16)]
    formats = {2: [('AnchorPoint', uint16)],
               3: [('XDeviceTable', indirect(Device)),
                   ('YDeviceTable', indirect(Device))]}

class MarkRecord(Record):
    entries = [('Class', uint16),
               ('MarkAnchor', indirect(Anchor))]

class MarkArray(OpenTypeTable):
    entries = [('MarkCount', uint16),
               ('MarkRecord', context_array(MarkRecord, 'MarkCount'))]

class SingleAdjustmentSubtable(MultiFormatTable):
    entries = [('PosFormat', uint16),
               ('Coverage', indirect(Coverage)),
               ('ValueFormat', ValueFormat)]
    formats = {1: [('Value', context(ValueRecord, 'ValueFormat'))],
               2: [('ValueCount', uint16),
                   ('Value', context_array(ValueRecord, 'ValueCount',
                                           'ValueFormat'))]}

class PairSetTable(OpenTypeTable):
    entries = [('PairValueCount', uint16)]

    def __init__(self, file, file_offset, format_1, format_2):
        super().__init__(file, file_offset)
        record_format = format_1.data_format + format_2.data_format
        value_1_length = len(format_1)
        format_1_keys = format_1.present_keys
        format_2_keys = format_2.present_keys
        pvr_struct = struct.Struct('>H' + record_format)
        pvr_size = pvr_struct.size
        pvr_list = []
        self.by_second_glyph_id = {}
        for i in range(self['PairValueCount']):
            record_data = pvr_struct.unpack(file.read(pvr_size))
            second_glyph = record_data[0]
            value_1 = {}
            value_2 = {}
            for i, key in enumerate(format_1_keys):
                value_1[key] = record_data[1 + i]
            for i, key in enumerate(format_2_keys):
                value_2[key] = record_data[1 + value_1_length + i]
            pvr = {'Value1': value_1,
                   'Value2': value_2}
            pvr_list.append(pvr)
            self.by_second_glyph_id[second_glyph] = pvr
        self['PairValueRecord'] = pvr_list

class PairAdjustmentSubtable(MultiFormatTable):
    entries = [('PosFormat', uint16),
               ('Coverage', indirect(Coverage)),
               ('ValueFormat1', ValueFormat),
               ('ValueFormat2', ValueFormat)]
    formats = {1: [('PairSetCount', uint16),
                   ('PairSet', indirect_array(PairSetTable, 'PairSetCount',
                                              'ValueFormat1', 'ValueFormat2'))],
               2: [('ClassDef1', indirect(ClassDefinition)),
                   ('ClassDef2', indirect(ClassDefinition)),
                   ('Class1Count', uint16),
                   ('Class2Count', uint16)]}

    def __init__(self, file, file_offset=None):
        super().__init__(file, file_offset)
        format_1, format_2 = self['ValueFormat1'], self['ValueFormat2']
        if self['PosFormat'] == 2:
            record_format = format_1.data_format + format_2.data_format
            c2r_struct = struct.Struct('>' + record_format)
            c2r_size = c2r_struct.size
            value_1_length = len(format_1)
            format_1_keys = format_1.present_keys
            format_2_keys = format_2.present_keys
            class_1_record = []
            for i in range(self['Class1Count']):
                class_2_record = []
                for j in range(self['Class2Count']):
                    record_data = c2r_struct.unpack(file.read(c2r_size))
                    value_1 = {}
                    value_2 = {}
                    for i, key in enumerate(format_1_keys):
                        value_1[key] = record_data[i]
                    for i, key in enumerate(format_2_keys):
                        value_2[key] = record_data[value_1_length + i]
                    class_2_record.append({'Value1': value_1,
                                           'Value2': value_2})
                class_1_record.append(class_2_record)
            self['Class1Record'] = class_1_record

    def lookup(self, a_id, b_id):
        if self['PosFormat'] == 1:
            try:
                index = self['Coverage'].index(a_id)
            except ValueError:
                raise KeyError
            pair_value_record = self['PairSet'][index].by_second_glyph_id[b_id]
            return pair_value_record['Value1']['XAdvance']
        elif self['PosFormat'] == 2:
            a_class = self['ClassDef1'].class_number(a_id)
            b_class = self['ClassDef2'].class_number(b_id)
            class_2_record = self['Class1Record'][a_class][b_class]
            return class_2_record['Value1']['XAdvance']

class EntryExitRecord(OpenTypeTable):
    entries = [('EntryAnchor', indirect(Anchor)),
               ('ExitAnchor', indirect(Anchor, 'EntryExitCount'))]

class CursiveAttachmentSubtable(OpenTypeTable):
    entries = [('PosFormat', uint16),
               ('Coverage', indirect(Coverage)),
               ('EntryExitCount', uint16),
               ('EntryExitRecord', context_array(EntryExitRecord, 'EntryExitCount'))]

class MarkCoverage(OpenTypeTable):
    pass

class BaseCoverage(OpenTypeTable):
    pass

class Mark2Array(OpenTypeTable):
    pass

class BaseRecord(OpenTypeTable):
##    entries = [('BaseAnchor', indirect_array(Anchor, 'ClassCount'))]

    def __init__(self, file, file_offset, class_count):
        super().__init__(self, file, file_offset)
##        self['BaseAnchor'] = indirect_array(Anchor, 'ClassCount'])(file)

class BaseArray(OpenTypeTable):
    entries = [('BaseCount', uint16)]
##               ('BaseRecord', context_array(BaseRecord, 'BaseCount'))]

    def __init__(self, file, file_offset, class_count):
        super().__init__(self, file, file_offset)
        self['BaseRecord'] = array(BaseRecord, self['BaseCount'],
                                   class_count=class_count)(file)

class MarkToBaseAttachmentSubtable(OpenTypeTable):
    entries = [('PosFormat', uint16),
               ('MarkCoverage', indirect(MarkCoverage)),
               ('BaseCoverage', indirect(BaseCoverage)),
               ('ClassCount', uint16),
               ('MarkArray', indirect(MarkArray)),
               ('BaseArray', indirect(BaseArray, 'ClassCount'))]

class MarkToMarkAttachmentSubtable(OpenTypeTable):
    entries = [('PosFormat', uint16),
               ('Mark1Coverage', indirect(MarkCoverage)),
               ('Mark1Coverage', indirect(MarkCoverage)),
               ('ClassCount', uint16),
               ('Mark1Array', indirect(MarkArray)),
               ('Mark1Array', indirect(Mark2Array))]

class ExtensionPositioning(OpenTypeTable):
    entries = [('PosFormat', ushort),
               ('ExtensionLookupType', ushort),
               ('ExtensionOffset', ulong)]

    def __init__(self, file, file_offset=None):
        super().__init__(file, file_offset=file_offset)
        subtable_class = GposTable.lookup_types[self['ExtensionLookupType']]
        table_offset = file_offset + self['ExtensionOffset']
        self.subtable = subtable_class(file, table_offset)

    def lookup(self, *args, **kwargs):
        return self.subtable.lookup(*args, **kwargs)

class GposTable(LayoutTable):
    """Glyph positioning table"""
    tag = 'GPOS'
    lookup_types = {1: SingleAdjustmentSubtable,
                    2: PairAdjustmentSubtable,
                    3: CursiveAttachmentSubtable,
                    4: MarkToBaseAttachmentSubtable,
                    6: MarkToMarkAttachmentSubtable,
                    9: ExtensionPositioning}
<|endoftext|>


<commit_before>#
# Copyright (c) 2006-2007 rPath, Inc.  All Rights Reserved.
#
import errno
import fcntl
import itertools
import os
import resource
import signal
import sys
import tempfile
import time
import traceback

from conary.build import cook,macros,use
from conary.deps import deps
from conary.lib import epdb
from conary.lib import log,util
from conary.local import database
from conary import versions
from conary.deps.deps import ThawFlavor

from rmake.failure import BuildFailed, FailureReason
from rmake.lib import flavorutil
from rmake.lib import logfile
from rmake.lib import recipeutil
from rmake.lib.apiutils import thaw, freeze
from rmake.worker import resolvesource

class CookResults(object):
    def __init__(self, name, version, flavorList):
        self.name = name
        self.version = version
        if not isinstance(flavorList, (list, tuple)):
            flavorList = [flavorList]
        self.flavorList = flavorList
        self.status = ''
        self.csFile = ''
        self.pid = 0
        self.failureReason = None
        self.signal = ''

    def exitedNormally(self):
        return not self.signal

    def setExitStatus(self, status):
        self.status = status

    def setExitSignal(self, signal):
        self.signal = signal

    def getExitSignal(self):
        return self.signal

    def getExitStatus(self):
        return self.status

    def setChangeSetFile(self, csFile):
        self.csFile = csFile

    def getChangeSetFile(self):
        return self.csFile

    def setFailureReason(self, reason):
        self.failureReason = reason

    def getFailureReason(self):
        return self.failureReason

    def isBuildSuccess(self):
        return self.exitedNormally() and not self.status

    def __freeze__(self):
        d = self.__dict__.copy()
        d['pid'] = self.pid
        d['version'] = str(self.version)
        d['flavorList'] = [ x.freeze() for x in self.flavorList ]
        d['failureReason'] = freeze('FailureReason', self.failureReason)
        return d

    @staticmethod
    def __thaw__(d):
        d = d.copy()
        new = CookResults(d.pop('name'),
                          versions.VersionFromString(d.pop('version')),
                          [ ThawFlavor(x) for x in d.pop('flavorList')])
        new.__dict__.update(d)
        new.failureReason = thaw('FailureReason', new.failureReason)
        return new

def cookTrove(cfg, repos, logger, name, version, flavorList, targetLabel,
              loadSpecsList=None, builtTroves=None, logData=None,
              buildReqs=None, crossReqs=None):
    if not isinstance(flavorList, (tuple, list)):
        flavorList = [flavorList]
    util.mkdirChain(cfg.root + '/tmp')
    fd, csFile = tempfile.mkstemp(dir=cfg.root + '/tmp',
                                  prefix='rmake-%s-' % name,
                                  suffix='.ccs')
    os.chmod(csFile, 0644)
    os.close(fd)
    logPath = cfg.root + '/tmp/rmake/%s-%s.log' % (name,
                                    version.trailingRevision())
    logFile = logfile.LogFile(logPath)
    os.chmod(logPath, 0664)
    os.chmod(cfg.root + '/tmp/rmake', 0775)

    results = CookResults(name, version, flavorList)

    # ignore child output problems
    signal.signal(signal.SIGTTOU, signal.SIG_IGN)

    inF, outF = os.pipe()
    fcntl.fcntl(inF, fcntl.FD_CLOEXEC)
    fcntl.fcntl(outF, fcntl.FD_CLOEXEC)
    pid = os.fork()
    if not pid:
        try:
            try:
                signal.signal(signal.SIGTERM, signal.SIG_DFL)
                os.close(inF)
                os.setpgrp()
                # don't accidentally make world writable files
                os.umask(0022)
                # don't allow us to create core dumps
                resource.setrlimit(resource.RLIMIT_CORE, (0,0))
                #if logData:
                #    logFile.logToPort(*logData)
                #else:
                #    logFile.redirectOutput()
                log.setVerbosity(log.DEBUG)
                log.info("Cook process started (pid %s)" % os.getpid())
                _cookTrove(cfg, repos, name, version, flavorList, targetLabel,
                           loadSpecsList, builtTroves,
                           csFile, buildReqs=buildReqs, crossReqs=crossReqs,
                           failureFd=outF, logger=logger)
            except Exception, msg:
                if len(flavorList) > 1:
                    errMsg = 'Error cooking %s=%s with flavors %s: %s' % \
                        (name, version, ', '.join([str(x) for x in flavorList]),
                         str(msg))
                _buildFailed(outF, errMsg, traceback.format_exc())
                logFile.close()
                os._exit(1)
            else:
                logFile.close()
                os._exit(0)
        finally:
            logFile.close()
            # some kind of error occurred if we get here.
            os._exit(1)
    else:
        os.close(outF)
        return logPath, pid, (results, pid, inF, csFile)

def getResults(results, pid, inF, csFile):
    (gotResult, status) = os.waitpid(pid, os.WNOHANG)
    if not gotResult:
        return None

    if os.WIFSIGNALED(status):
        results.setExitSignal(os.WTERMSIG(status))
    else:
        assert(os.WIFEXITED(status))
        results.setExitStatus(os.WEXITSTATUS(status))

    if results.isBuildSuccess():
        results.setChangeSetFile(csFile)
    elif results.getExitSignal():
        results.setFailureReason(BuildFailed('Build exited with signal %s' % results.getExitSignal()))
    else:
        errReason = []
        buffer = os.read(inF, 1024)
        while buffer:
            errReason.append(buffer)
            buffer = os.read(inF, 1024)
        errReason = ''.join(errReason)
        errTag, data = errReason.split('\002', 1)
        results.setFailureReason(thaw('FailureReason', (errTag, data)))
    os.close(inF)
    return results

def stopBuild(results, pid, inF, csFile):
    log.info('killing %s' % pid)
    try:
        os.kill(-pid, signal.SIGTERM)
    except OSError, err:
        if err.errno != errno.ENOENT:
            raise
        else:
            log.warning('cooking pid %s did not exit' % pid)

    timeSlept = 0
    while timeSlept < 10:
        gotResult, status = os.waitpid(pid, os.WNOHANG)
        if gotResult:
            break
        else:
            time.sleep(.5)
            timeSlept += .5
    os.close(inF)

    if not gotResult:
        log.warning('pid %s did not respond to kill, trying SIGKILL' % pid)
        try:
            os.kill(-pid, signal.SIGKILL)
        except OSError, err:
            if err.errno != err.ESRCH:
                raise
            else:
                return

        # just hang waiting
        gotResult, status = os.waitpid(pid, 0)
    log.info('pid %s killed' % pid)

def _buildFailed(failureFd, errMsg, traceBack=''):
    log.error(errMsg)
    frz = '\002'.join(str(x) for x in freeze('FailureReason',
                                BuildFailed(errMsg, traceBack)))
    if failureFd is not None:
        os.write(failureFd, frz)
        os.close(failureFd)
    os._exit(1)

def _cookTrove(cfg, repos, name, version, flavorList, targetLabel,
               loadSpecsList, builtTroves, csFile, buildReqs, crossReqs,
               failureFd, logger):
    baseFlavor = cfg.buildFlavor
    db = database.Database(cfg.root, cfg.dbPath)
    buildLabel = version.trailingLabel()
    buildBranch = version.branch()
    binaryBranch = version.getBinaryVersion().branch()
    if targetLabel:
        source = recipeutil.RemoveHostSource(db, targetLabel.getHost())
        if version.trailingLabel() == targetLabel and version.depth() > 1:
            buildBranch = version.branch().parentBranch()
            buildLabel = buildBranch.label()
            binaryBranch = version.parentVersion().getBinaryVersion().branch()
    else:
        source = db
    loaders = []
    recipeClasses = []

    if not isinstance(flavorList, (tuple, list)):
        flavorList = [flavorList]
    if not isinstance(loadSpecsList, (tuple, list)):
        loadSpecsList = [loadSpecsList] * len(flavorList)

    for flavor, loadSpecs in itertools.izip(flavorList, loadSpecsList):
        try:
            logger.debug('Cooking %s=%s[%s] to %s (stored in %s)' % \
                         (name, version, flavor, targetLabel, csFile))
            cfg.buildFlavor = deps.overrideFlavor(baseFlavor, flavor)
            cfg.initializeFlavors()
            (loader, recipeClass, localFlags, usedFlags)  = \
                recipeutil.loadRecipeClass(repos, name, version,
                                           cfg.buildFlavor,
                                           ignoreInstalled=False, root=cfg.root,
                                           loadInstalledSource=source,
                                           overrides=loadSpecs,
                                           cfg=cfg)
            loaders.append(loader)
            recipeClasses.append(recipeClass)
            recipeClass.buildRequirementsOverride = buildReqs
            recipeClass.crossRequirementsOverride = crossReqs
        except Exception, msg:
            errMsg = 'Error loading recipe %s=%s[%s]: %s' % \
                                            (name, version, flavor, str(msg))
            _buildFailed(failureFd, errMsg, traceback.format_exc())

    try:
        # get the correct environment variables from this root
        # some packages depend on environment variables e.g. $QTDIR that
        # are set by other packages.
        setupEnvironment()

        # now override flags set in flavor
        # don't need to reset this flavor ever, because
        # we are in a fork
        flavorutil.setLocalFlags(localFlags)
        packageName = name.split(':')[0]
        # this shouldn't matter for group recipes as it will get overridden
        # by the behavior in cookGroupObject.  But it matters for some other
        # recipe types.  That should be fixed and all that code should be
        # moved inside cookObject so I could get rid of this.
        use.setBuildFlagsFromFlavor(packageName, cfg.buildFlavor, error=False)
        use.resetUsed()
        use.setUsed(usedFlags)

        # we don't want to sign packages here, if necessary, we can sign
        # them at a higher level.
        cfg.signatureKeyMap = {}
        cfg.signatureKey = None
        crossCompile = flavorutil.getCrossCompile(cfg.buildFlavor)

        # add extra buildreqs manually added for this trove
        # by the builder.  Only add them if the recipe is of the
        # right type, and the cfg file we're passed in understands them
        # (it might be a simple conary cfg file).
        if (hasattr(recipeClasses[0], 'buildRequires')
            and hasattr(cfg, 'defaultBuildReqs')):
            for recipeClass in recipeClasses:
                recipeClass.buildRequires += cfg.defaultBuildReqs

        if builtTroves:
            # FIXME: is this cached?
            builtTroves = repos.getTroves(builtTroves, withFiles=False)

            builtTroves = resolvesource.BuiltTroveSource(builtTroves, repos)
            builtTroves.searchAsRepository()
            if targetLabel:
                builtTroves = recipeutil.RemoveHostSource(builtTroves,
                                                          targetLabel.getHost())
            else:
                builtTroves = recipeutil.RemoveHostSource(builtTroves,
                                              version.trailingLabel().getHost())

            # this should only make a difference when cooking groups, redirects,
            # etc.
            oldRepos = repos
            repos = resolvesource.DepHandlerSource(builtTroves,
                                                   None,
                                                   repos)
            repos.TROVE_QUERY_ALL = oldRepos.TROVE_QUERY_ALL

        # if we're already on the target label, we'll assume no targeting
        # is necessary
        if targetLabel == version.trailingLabel():
            targetLabel = None

    except Exception, msg:
        errMsg = 'Error initializing cook environment %s=%s[%s]: %s' % \
                                            (name, version, flavor, str(msg))
        _buildFailed(failureFd, errMsg, traceback.format_exc())

    try:
        os.chdir('/tmp') # make sure we're in a directory
                         # that we can write to.  Although
                         # this _shouldn't_ be an issue,
                         # conary 1.0.{19,20} require it.
        # finally actually cook the recipe!
        groupOptions = cook.GroupCookOptions(alwaysBumpCount=False,
                                        shortenFlavors=cfg.shortenGroupFlavors,
                                        errorOnFlavorChange=False)
        m = macros.Macros()
        m._override('buildlabel', str(buildLabel))
        m._override('buildbranch', str(buildBranch))
        m._override('binarybranch', binaryBranch)
        built = cook.cookObject(repos, cfg, recipeClasses, version,
                                prep=False,
                                macros=m,
                                targetLabel=targetLabel,
                                changeSetFile=csFile,
                                alwaysBumpCount=False,
                                ignoreDeps=False,
                                logBuild=True,
                                crossCompile=crossCompile,
                                requireCleanSources=True,
                                groupOptions=groupOptions)
        if not built:
            if log.errorOccurred():
                msg = 'Check logs'
            else:
                msg = 'Unknown failure'
            errMsg = 'Error building recipe %s=%s[%s]: %s' % (name, version,
                                                              flavor, msg)
            _buildFailed(failureFd, errMsg)
    except Exception, msg:
        errMsg = 'Error building recipe %s=%s[%s]: %s' % (name, version,
                                                          flavor, str(msg))
        _buildFailed(failureFd, errMsg, traceback.format_exc())

def setupEnvironment():
    """
    Grab a fresh copy of the environment, based on the currently installed
    troves.
    """
    skipenv = set(['DISPLAY', 'STY', 'COVERAGE_DIR'])
    for key in os.environ.keys():
        if key not in skipenv:
            del os.environ[key]
    for line in os.popen('/bin/bash -l -c env'):
        key, val = line.split('=', 1)
        if key not in skipenv:
            os.environ[key] = val[:-1]
    os.environ['LANG'] = 'C'
    os.environ['HOME'] = '/tmp/rmake'
<commit_msg>This fn doesn't work<commit_after>#
# Copyright (c) 2006-2007 rPath, Inc.  All Rights Reserved.
#
import errno
import fcntl
import itertools
import os
import resource
import signal
import sys
import tempfile
import time
import traceback

from conary.build import cook,macros,use
from conary.deps import deps
from conary.lib import epdb
from conary.lib import log,util
from conary.local import database
from conary import versions
from conary.deps.deps import ThawFlavor

from rmake.failure import BuildFailed, FailureReason
from rmake.lib import flavorutil
from rmake.lib import logfile
from rmake.lib import recipeutil
from rmake.lib.apiutils import thaw, freeze
from rmake.worker import resolvesource

class CookResults(object):
    def __init__(self, name, version, flavorList):
        self.name = name
        self.version = version
        if not isinstance(flavorList, (list, tuple)):
            flavorList = [flavorList]
        self.flavorList = flavorList
        self.status = ''
        self.csFile = ''
        self.pid = 0
        self.failureReason = None
        self.signal = ''

    def exitedNormally(self):
        return not self.signal

    def setExitStatus(self, status):
        self.status = status

    def setExitSignal(self, signal):
        self.signal = signal

    def getExitSignal(self):
        return self.signal

    def getExitStatus(self):
        return self.status

    def setChangeSetFile(self, csFile):
        self.csFile = csFile

    def getChangeSetFile(self):
        return self.csFile

    def setFailureReason(self, reason):
        self.failureReason = reason

    def getFailureReason(self):
        return self.failureReason

    def isBuildSuccess(self):
        return self.exitedNormally() and not self.status

    def __freeze__(self):
        d = self.__dict__.copy()
        d['pid'] = self.pid
        d['version'] = str(self.version)
        d['flavorList'] = [ x.freeze() for x in self.flavorList ]
        d['failureReason'] = freeze('FailureReason', self.failureReason)
        return d

    @staticmethod
    def __thaw__(d):
        d = d.copy()
        new = CookResults(d.pop('name'),
                          versions.VersionFromString(d.pop('version')),
                          [ ThawFlavor(x) for x in d.pop('flavorList')])
        new.__dict__.update(d)
        new.failureReason = thaw('FailureReason', new.failureReason)
        return new

def cookTrove(cfg, repos, logger, name, version, flavorList, targetLabel,
              loadSpecsList=None, builtTroves=None, logData=None,
              buildReqs=None, crossReqs=None):
    if not isinstance(flavorList, (tuple, list)):
        flavorList = [flavorList]
    util.mkdirChain(cfg.root + '/tmp')
    fd, csFile = tempfile.mkstemp(dir=cfg.root + '/tmp',
                                  prefix='rmake-%s-' % name,
                                  suffix='.ccs')
    os.chmod(csFile, 0644)
    os.close(fd)
    logPath = cfg.root + '/tmp/rmake/%s-%s.log' % (name,
                                    version.trailingRevision())
    logFile = logfile.LogFile(logPath)
    os.chmod(logPath, 0664)
    os.chmod(cfg.root + '/tmp/rmake', 0775)

    results = CookResults(name, version, flavorList)

    # ignore child output problems
    signal.signal(signal.SIGTTOU, signal.SIG_IGN)

    inF, outF = os.pipe()
    fcntl.fcntl(inF, fcntl.FD_CLOEXEC)
    fcntl.fcntl(outF, fcntl.FD_CLOEXEC)
    pid = os.fork()
    if not pid:
        try:
            try:
                signal.signal(signal.SIGTERM, signal.SIG_DFL)
                os.close(inF)
                os.setpgrp()
                # don't accidentally make world writable files
                os.umask(0022)
                # don't allow us to create core dumps
                resource.setrlimit(resource.RLIMIT_CORE, (0,0))
                #if logData:
                #    logFile.logToPort(*logData)
                #else:
                #    logFile.redirectOutput()
                log.setVerbosity(log.DEBUG)
                log.info("Cook process started (pid %s)" % os.getpid())
                _cookTrove(cfg, repos, name, version, flavorList, targetLabel,
                           loadSpecsList, builtTroves,
                           csFile, buildReqs=buildReqs, crossReqs=crossReqs,
                           failureFd=outF, logger=logger)
            except Exception, msg:
                if len(flavorList) > 1:
                    errMsg = 'Error cooking %s=%s with flavors %s: %s' % \
                        (name, version, ', '.join([str(x) for x in flavorList]),
                         str(msg))
                _buildFailed(outF, errMsg, traceback.format_exc())
                logFile.close()
                os._exit(1)
            else:
                logFile.close()
                os._exit(0)
        finally:
            logFile.close()
            # some kind of error occurred if we get here.
            os._exit(1)
    else:
        os.close(outF)
        return logPath, pid, (results, pid, inF, csFile)

def getResults(results, pid, inF, csFile):
    (gotResult, status) = os.waitpid(pid, os.WNOHANG)
    if not gotResult:
        return None

    if os.WIFSIGNALED(status):
        results.setExitSignal(os.WTERMSIG(status))
    else:
        assert(os.WIFEXITED(status))
        results.setExitStatus(os.WEXITSTATUS(status))

    if results.isBuildSuccess():
        results.setChangeSetFile(csFile)
    elif results.getExitSignal():
        results.setFailureReason(BuildFailed('Build exited with signal %s' % results.getExitSignal()))
    else:
        errReason = []
        buffer = os.read(inF, 1024)
        while buffer:
            errReason.append(buffer)
            buffer = os.read(inF, 1024)
        errReason = ''.join(errReason)
        errTag, data = errReason.split('\002', 1)
        results.setFailureReason(thaw('FailureReason', (errTag, data)))
    os.close(inF)
    return results

def stopBuild(results, pid, inF, csFile):
    log.info('killing %s' % pid)
    try:
        os.kill(-pid, signal.SIGTERM)
    except OSError, err:
        if err.errno != errno.ENOENT:
            raise
        else:
            log.warning('cooking pid %s did not exit' % pid)

    timeSlept = 0
    while timeSlept < 10:
        gotResult, status = os.waitpid(pid, os.WNOHANG)
        if gotResult:
            break
        else:
            time.sleep(.5)
            timeSlept += .5
    os.close(inF)

    if not gotResult:
        log.warning('pid %s did not respond to kill, trying SIGKILL' % pid)
        try:
            os.kill(-pid, signal.SIGKILL)
        except OSError, err:
            if err.errno != err.ESRCH:
                raise
            else:
                return

        # just hang waiting
        gotResult, status = os.waitpid(pid, 0)
    log.info('pid %s killed' % pid)

def _buildFailed(failureFd, errMsg, traceBack=''):
    log.error(errMsg)
    frz = '\002'.join(str(x) for x in freeze('FailureReason',
                                BuildFailed(errMsg, traceBack)))
    if failureFd is not None:
        os.write(failureFd, frz)
        os.close(failureFd)
    os._exit(1)

def _cookTrove(cfg, repos, name, version, flavorList, targetLabel,
               loadSpecsList, builtTroves, csFile, buildReqs, crossReqs,
               failureFd, logger):
    baseFlavor = cfg.buildFlavor
    db = database.Database(cfg.root, cfg.dbPath)
    buildLabel = version.trailingLabel()
    buildBranch = version.branch()
    binaryBranch = version.getBinaryVersion().branch()
    if targetLabel:
        source = recipeutil.RemoveHostSource(db, targetLabel.getHost())
        if version.trailingLabel() == targetLabel and version.depth() > 1:
            buildBranch = version.branch().parentBranch()
            buildLabel = buildBranch.label()
            # FIXME: need a method that takes a branch and returns the upstream
            # binary branch.
            #binaryBranch = version.parentVersion().getBinaryVersion().branch()
    else:
        source = db
    loaders = []
    recipeClasses = []

    if not isinstance(flavorList, (tuple, list)):
        flavorList = [flavorList]
    if not isinstance(loadSpecsList, (tuple, list)):
        loadSpecsList = [loadSpecsList] * len(flavorList)

    for flavor, loadSpecs in itertools.izip(flavorList, loadSpecsList):
        try:
            logger.debug('Cooking %s=%s[%s] to %s (stored in %s)' % \
                         (name, version, flavor, targetLabel, csFile))
            cfg.buildFlavor = deps.overrideFlavor(baseFlavor, flavor)
            cfg.initializeFlavors()
            (loader, recipeClass, localFlags, usedFlags)  = \
                recipeutil.loadRecipeClass(repos, name, version,
                                           cfg.buildFlavor,
                                           ignoreInstalled=False, root=cfg.root,
                                           loadInstalledSource=source,
                                           overrides=loadSpecs,
                                           cfg=cfg)
            loaders.append(loader)
            recipeClasses.append(recipeClass)
            recipeClass.buildRequirementsOverride = buildReqs
            recipeClass.crossRequirementsOverride = crossReqs
        except Exception, msg:
            errMsg = 'Error loading recipe %s=%s[%s]: %s' % \
                                            (name, version, flavor, str(msg))
            _buildFailed(failureFd, errMsg, traceback.format_exc())

    try:
        # get the correct environment variables from this root
        # some packages depend on environment variables e.g. $QTDIR that
        # are set by other packages.
        setupEnvironment()

        # now override flags set in flavor
        # don't need to reset this flavor ever, because
        # we are in a fork
        flavorutil.setLocalFlags(localFlags)
        packageName = name.split(':')[0]
        # this shouldn't matter for group recipes as it will get overridden
        # by the behavior in cookGroupObject.  But it matters for some other
        # recipe types.  That should be fixed and all that code should be
        # moved inside cookObject so I could get rid of this.
        use.setBuildFlagsFromFlavor(packageName, cfg.buildFlavor, error=False)
        use.resetUsed()
        use.setUsed(usedFlags)

        # we don't want to sign packages here, if necessary, we can sign
        # them at a higher level.
        cfg.signatureKeyMap = {}
        cfg.signatureKey = None
        crossCompile = flavorutil.getCrossCompile(cfg.buildFlavor)

        # add extra buildreqs manually added for this trove
        # by the builder.  Only add them if the recipe is of the
        # right type, and the cfg file we're passed in understands them
        # (it might be a simple conary cfg file).
        if (hasattr(recipeClasses[0], 'buildRequires')
            and hasattr(cfg, 'defaultBuildReqs')):
            for recipeClass in recipeClasses:
                recipeClass.buildRequires += cfg.defaultBuildReqs

        if builtTroves:
            # FIXME: is this cached?
            builtTroves = repos.getTroves(builtTroves, withFiles=False)

            builtTroves = resolvesource.BuiltTroveSource(builtTroves, repos)
            builtTroves.searchAsRepository()
            if targetLabel:
                builtTroves = recipeutil.RemoveHostSource(builtTroves,
                                                          targetLabel.getHost())
            else:
                builtTroves = recipeutil.RemoveHostSource(builtTroves,
                                              version.trailingLabel().getHost())

            # this should only make a difference when cooking groups, redirects,
            # etc.
            oldRepos = repos
            repos = resolvesource.DepHandlerSource(builtTroves,
                                                   None,
                                                   repos)
            repos.TROVE_QUERY_ALL = oldRepos.TROVE_QUERY_ALL

        # if we're already on the target label, we'll assume no targeting
        # is necessary
        if targetLabel == version.trailingLabel():
            targetLabel = None

    except Exception, msg:
        errMsg = 'Error initializing cook environment %s=%s[%s]: %s' % \
                                            (name, version, flavor, str(msg))
        _buildFailed(failureFd, errMsg, traceback.format_exc())

    try:
        os.chdir('/tmp') # make sure we're in a directory
                         # that we can write to.  Although
                         # this _shouldn't_ be an issue,
                         # conary 1.0.{19,20} require it.
        # finally actually cook the recipe!
        groupOptions = cook.GroupCookOptions(alwaysBumpCount=False,
                                        shortenFlavors=cfg.shortenGroupFlavors,
                                        errorOnFlavorChange=False)
        m = macros.Macros()
        m._override('buildlabel', str(buildLabel))
        m._override('buildbranch', str(buildBranch))
        m._override('binarybranch', binaryBranch)
        built = cook.cookObject(repos, cfg, recipeClasses, version,
                                prep=False,
                                macros=m,
                                targetLabel=targetLabel,
                                changeSetFile=csFile,
                                alwaysBumpCount=False,
                                ignoreDeps=False,
                                logBuild=True,
                                crossCompile=crossCompile,
                                requireCleanSources=True,
                                groupOptions=groupOptions)
        if not built:
            if log.errorOccurred():
                msg = 'Check logs'
            else:
                msg = 'Unknown failure'
            errMsg = 'Error building recipe %s=%s[%s]: %s' % (name, version,
                                                              flavor, msg)
            _buildFailed(failureFd, errMsg)
    except Exception, msg:
        errMsg = 'Error building recipe %s=%s[%s]: %s' % (name, version,
                                                          flavor, str(msg))
        _buildFailed(failureFd, errMsg, traceback.format_exc())

def setupEnvironment():
    """
    Grab a fresh copy of the environment, based on the currently installed
    troves.
    """
    skipenv = set(['DISPLAY', 'STY', 'COVERAGE_DIR'])
    for key in os.environ.keys():
        if key not in skipenv:
            del os.environ[key]
    for line in os.popen('/bin/bash -l -c env'):
        key, val = line.split('=', 1)
        if key not in skipenv:
            os.environ[key] = val[:-1]
    os.environ['LANG'] = 'C'
    os.environ['HOME'] = '/tmp/rmake'
<|endoftext|>