<commit_before>#!/usr/bin/env python

'''
'''

__docformat__ = 'restructuredtext'
__version__ = '$Id$'

import ctypes

from pyglet.media import BasePlayer, ManagedSoundPlayerMixIn, Listener
from pyglet.media import MediaException

from pyglet.media.drivers.alsa import asound

alsa_debug = 'alsa.log'

class ALSAException(MediaException):
    pass

def check(err):
    if err < 0:
        raise ALSAException(asound.snd_strerror(err))
    return err

class Device(object):
    _ring_buffer_time = 0.3

    def __init__(self, name):
        self.name = name
        self.pcm = ctypes.POINTER(asound.snd_pcm_t)()
        self.hwparams = ctypes.POINTER(asound.snd_pcm_hw_params_t)()
        self.swparams = ctypes.POINTER(asound.snd_pcm_sw_params_t)()

        check(asound.snd_pcm_open(ctypes.byref(self.pcm),
                                  name,
                                  asound.SND_PCM_STREAM_PLAYBACK,
                                  asound.SND_PCM_NONBLOCK))
        check(asound.snd_pcm_hw_params_malloc(ctypes.byref(self.hwparams)))
        check(asound.snd_pcm_sw_params_malloc(ctypes.byref(self.swparams)))
        check(asound.snd_pcm_hw_params_any(self.pcm, self.hwparams))

        if alsa_debug:
            asound.snd_output_printf(debug_output, 'New device: %s\n' % name)

    def __del__(self):
        try:
            check(asound.snd_pcm_close(self.pcm))
        except (NameError, AttributeError):
            pass

    def prepare(self, source):
        if not source.audio_format:
            # TODO avoid creating in this case.
            return

        format = {
            8:  asound.SND_PCM_FORMAT_U8,
            16: asound.SND_PCM_FORMAT_S16,
            24: asound.SND_PCM_FORMAT_S24,  # probably won't work
            32: asound.SND_PCM_FORMAT_S32
        }.get(source.audio_format.sample_size)
        if format is None:
            raise ALSAException('Unsupported audio format.')

        check(asound.snd_pcm_hw_params_set_access(self.pcm, self.hwparams,
            asound.SND_PCM_ACCESS_RW_INTERLEAVED))
        check(asound.snd_pcm_hw_params_set_format(self.pcm, self.hwparams,
            format))
        check(asound.snd_pcm_hw_params_set_channels(self.pcm, self.hwparams,
            source.audio_format.channels))
        check(asound.snd_pcm_hw_params_set_rate(self.pcm, self.hwparams,
            source.audio_format.sample_rate, 0))
        check(asound.snd_pcm_hw_params_set_buffer_size(self.pcm, self.hwparams,
            int(self._ring_buffer_time * source.audio_format.sample_rate)))
        check(asound.snd_pcm_hw_params(self.pcm, self.hwparams))
        if alsa_debug:
            check(asound.snd_pcm_dump(self.pcm, debug_output))

        self.can_pause = asound.snd_pcm_hw_params_can_pause(self.hwparams)

class ALSAPlayer(BasePlayer):
    _min_buffer_time = 0.3
    _max_buffer_size = 65536

    def __init__(self):
        super(ALSAPlayer, self).__init__()

        self._sources = []
        self._source_read_index = 0
        self._playing = False
        self._device = None

        # timestamp of last packet buffered (must be of _source_read_index)
        self._current_buffer_time = 0.

        # cumulative duration of all sources completely buffered.
        self._cumulative_buffer_time = 0.

        # asound time for current source begin time
        self._start_time = None

        # saved timestamp when paused
        self._timestamp = 0.

        self._queue_audio_data = None

    def queue(self, source):
        source = source._get_queue_source()

        if not self._sources:
            self._source_read_index = 0
            source._init_texture(self)
        self._sources.append(source)

    def next(self):
        if self._sources:
            old_source = self._sources.pop(0)
            old_source._release_texture(self)
            old_source._stop()
            self._source_read_index -= 1

        if self._sources:
            self._sources[0]._init_texture(self)

    def dispatch_events(self):
        if not self._sources:
            return

        if not self._playing:
            # If paused, just update the video texture.
            if self._texture:
                self._sources[0]._update_texture(self, self.time)
            return

        # Create a device if there isn't one.  TODO only if source and source
        # has audio
        if not self._device:
            self._device = Device('plug:front')
            self._device.prepare(self._sources[0])

        self_time = self.time

        # Passed EOS?
        source = self._sources[0]
        while source and source.duration < self_time:
            if self._eos_action == self.EOS_NEXT:
                self.next()
            elif self._eos_action == self.EOS_STOP:
                self.stop()
                self._sources = []
            self.dispatch_event('on_eos')

            self_time -= source.duration
            self._start_time = self._get_asound_time() - self_time
            self._cumulative_buffer_time -= source.duration
            assert self._cumulative_buffer_time >= 0.
            try:
                source = self._sources[0]
            except IndexError:
                source = None
                self._start_time = None

        # Ensure device buffer is full
        try:
            source = self._sources[self._source_read_index]
        except IndexError:
            source = None
        while (source and
               self._cumulative_buffer_time + self._current_buffer_time - self_time
                  < self._min_buffer_time):
            if self._queue_audio_data:
                audio_data = self._queue_audio_data
                self._queue_audio_data = None
            else:
                max_bytes = int(
                  self._min_buffer_time * source.audio_format.bytes_per_second)
                max_bytes = min(max_bytes, self._max_buffer_size)
                audio_data = source._get_audio_data(max_bytes)

            if audio_data:
                samples = \
                    audio_data.length // source.audio_format.bytes_per_sample
                samples_out = asound.snd_pcm_writei(self._device.pcm,
                    audio_data.data, samples)
                if samples_out < 0:
                    if samples_out == -11: # EAGAIN
                        self._queue_audio_data = audio_data
                    elif samples_out == -32: # EPIPE
                        # xrun recovery
                        check(asound.snd_pcm_prepare(self._device.pcm))
                        self._queue_audio_data = audio_data
                    else:
                        raise ALSAException(asound.snd_strerror(samples_out))
                elif samples_out < samples:
                    audio_data.consume(
                        samples_out * source.audio_format.bytes_per_sample,
                        source.audio_format)
                    self._current_buffer_time = audio_data.timestamp
                    self._queue_audio_data = audio_data
                else:
                    self._current_buffer_time = \
                        audio_data.timestamp + audio_data.duration

                if self._start_time is None:
                    # XXX start playback
                    delay = asound.snd_pcm_sframes_t()
                    check(asound.snd_pcm_delay(self._device.pcm,
                                               ctypes.byref(delay)))
                    delay = delay.value / \
                        source.audio_format.bytes_per_sample / \
                        float(source.audio_format.bytes_per_second)
                    self._start_time = self._get_asound_time() - \
                        audio_data.timestamp - delay
            else:
                # EOS on read source
                self._cumulative_buffer_time += source.duration
                self._current_buffer_time = 0.
                if self._eos_action == self.EOS_NEXT:
                    self._source_read_index += 1
                    try:
                        # preroll
                        source = self._sources[self._source_read_index]
                        source._play()
                    except IndexError:
                        source = None
                elif self._eos_action == self.EOS_LOOP:
                    source._seek(0)
                elif self._eos_action == self.EOS_PAUSE:
                    source = None
                elif self._eos_action == self.EOS_STOP:
                    source = None
                else:
                    assert False, 'Invalid eos_action'
                    source = None

        # Update video texture
        if self._texture:
            self._sources[0]._update_texture(self, self_time)

    def _get_time(self):
        if self._start_time is None or not self._playing:
            return self._timestamp
        return self._get_asound_time() - self._start_time

    def _get_asound_time(self):
        status = ctypes.POINTER(asound.snd_pcm_status_t)()
        timestamp = asound.snd_timestamp_t()

        check(asound.snd_pcm_status_malloc(ctypes.byref(status)))
        check(asound.snd_pcm_status(self._device.pcm, status))
        asound.snd_pcm_status_get_tstamp(status, ctypes.byref(timestamp))
        asound.snd_pcm_status_free(status)
        return timestamp.tv_sec + timestamp.tv_usec * 0.000001

    def _clear_ring_buffer(self):
        check(asound.snd_pcm_drop(self._device.pcm))
        check(asound.snd_pcm_prepare(self._device.pcm))
        self._current_buffer_time = 0.
        self._cumulative_buffer_time = 0.
        self._source_read_index = 0
        self._queue_audio_data = None
        self._start_time = None

    def play(self):
        if self._playing:
            return

        self._playing = True

        if not self._sources:
            return

        if self._device:
            state = asound.snd_pcm_state(self._device.pcm)
            if self._device.can_pause and state == asound.SND_PCM_STATE_PAUSED:
                check(asound.snd_pcm_pause(self._device.pcm, 0))
                self._start_time = self._get_asound_time() - self._timestamp

    def pause(self):
        if not self._playing:
            return

        self._playing = False

        if not self._sources:
            return

        if self._device:
            self._timestamp = self._get_asound_time() - self._start_time
            if self._device.can_pause:
                check(asound.snd_pcm_pause(self._device.pcm, 1))
            else:
                # Hardware can't pause, so we'll just drop everything that's
                # been buffered.  Improvement could be to rebuffer data that
                # wasn't played.
                self._clear_ring_buffer()

    def seek(self, timestamp):
        if self._sources:
            self._sources[0]._seek(timestamp)
            self._timestamp = timestamp
            self._start_time = None

            if self._device:
                self._clear_ring_buffer()

    def _get_source(self):
        if self._sources:
            return self._sources[0]
        return None

    def _stop(self):
        raise RuntimeError('Invalid eos_action for this player.')

    def _set_volume(self, volume):
        self._volume = volume
        # TODO apply to device

    # All other properties are silently ignored.

    def _set_min_gain(self, min_gain):
        self._min_gain = min_gain

    def _set_max_gain(self, max_gain):
        self._max_gain = max_gain

    def _set_position(self, position):
        self._position = position

    def _set_velocity(self, velocity):
        self._velocity = velocity

    def _set_pitch(self, pitch):
        self._pitch = pitch

    def _set_cone_orientation(self, cone_orientation):
        self._cone_orientation = cone_orientation

    def _set_cone_inner_angle(self, cone_inner_angle):
        self._cone_inner_angle = cone_inner_angle

    def _set_cone_outer_gain(self, cone_outer_gain):
        self._cone_outer_gain = cone_outer_gain

class ALSAManagedSoundPlayer(ALSAPlayer, ManagedSoundPlayerMixIn):
    pass

class ALSAListener(Listener):
    def set_volume(self, volume):
        # TODO master volume
        self._volume = volume

    # All other properties are silently ignored.

    def set_position(self, position):
        self._position = position

    def set_velocity(self, velocity):
        self._velocity = velocity

    def set_forward_orientation(self, orientation):
        self._forward_orientation = orientation

    def set_up_orientation(self, orientation):
        self._up_orientation = orientation

    def set_doppler_factor(self, factor):
        self._doppler_factor = factor

    def set_speed_of_sound(self, speed_of_sound):
        self._speed_of_sound = speed_of_sound

def driver_init():
    global debug_output
    print asound.snd_asoundlib_version()
    debug_output = ctypes.POINTER(asound.snd_output_t)()
    if alsa_debug:
        check(asound.snd_output_stdio_open(ctypes.byref(debug_output),
                                           alsa_debug,
                                           'w'))

driver_listener = ALSAListener()
DriverPlayer = ALSAPlayer
DriverManagedSoundPlayer = ALSAManagedSoundPlayer
<commit_msg>ALSA: Fix performance problem by reducing min buffer size.<commit_after>#!/usr/bin/env python

'''
'''

__docformat__ = 'restructuredtext'
__version__ = '$Id$'

import ctypes

from pyglet.media import BasePlayer, ManagedSoundPlayerMixIn, Listener
from pyglet.media import MediaException

from pyglet.media.drivers.alsa import asound

alsa_debug = 'alsa.log'

class ALSAException(MediaException):
    pass

def check(err):
    if err < 0:
        raise ALSAException(asound.snd_strerror(err))
    return err

class Device(object):
    _ring_buffer_time = 0.3

    def __init__(self, name):
        self.name = name
        self.pcm = ctypes.POINTER(asound.snd_pcm_t)()
        self.hwparams = ctypes.POINTER(asound.snd_pcm_hw_params_t)()
        self.swparams = ctypes.POINTER(asound.snd_pcm_sw_params_t)()

        check(asound.snd_pcm_open(ctypes.byref(self.pcm),
                                  name,
                                  asound.SND_PCM_STREAM_PLAYBACK,
                                  asound.SND_PCM_NONBLOCK))
        check(asound.snd_pcm_hw_params_malloc(ctypes.byref(self.hwparams)))
        check(asound.snd_pcm_sw_params_malloc(ctypes.byref(self.swparams)))
        check(asound.snd_pcm_hw_params_any(self.pcm, self.hwparams))

        if alsa_debug:
            asound.snd_output_printf(debug_output, 'New device: %s\n' % name)

    def __del__(self):
        try:
            check(asound.snd_pcm_close(self.pcm))
        except (NameError, AttributeError):
            pass

    def prepare(self, source):
        if not source.audio_format:
            # TODO avoid creating in this case.
            return

        format = {
            8:  asound.SND_PCM_FORMAT_U8,
            16: asound.SND_PCM_FORMAT_S16,
            24: asound.SND_PCM_FORMAT_S24,  # probably won't work
            32: asound.SND_PCM_FORMAT_S32
        }.get(source.audio_format.sample_size)
        if format is None:
            raise ALSAException('Unsupported audio format.')

        check(asound.snd_pcm_hw_params_set_access(self.pcm, self.hwparams,
            asound.SND_PCM_ACCESS_RW_INTERLEAVED))
        check(asound.snd_pcm_hw_params_set_format(self.pcm, self.hwparams,
            format))
        check(asound.snd_pcm_hw_params_set_channels(self.pcm, self.hwparams,
            source.audio_format.channels))
        check(asound.snd_pcm_hw_params_set_rate(self.pcm, self.hwparams,
            source.audio_format.sample_rate, 0))
        check(asound.snd_pcm_hw_params_set_buffer_size(self.pcm, self.hwparams,
            int(self._ring_buffer_time * source.audio_format.sample_rate)))
        check(asound.snd_pcm_hw_params(self.pcm, self.hwparams))
        if alsa_debug:
            check(asound.snd_pcm_dump(self.pcm, debug_output))

        self.can_pause = asound.snd_pcm_hw_params_can_pause(self.hwparams)

class ALSAPlayer(BasePlayer):
    _min_buffer_time = 0.1
    _max_buffer_size = 65536

    def __init__(self):
        super(ALSAPlayer, self).__init__()

        self._sources = []
        self._source_read_index = 0
        self._playing = False
        self._device = None

        # timestamp of last packet buffered (must be of _source_read_index)
        self._current_buffer_time = 0.

        # cumulative duration of all sources completely buffered.
        self._cumulative_buffer_time = 0.

        # asound time for current source begin time
        self._start_time = None

        # saved timestamp when paused
        self._timestamp = 0.

        self._queue_audio_data = None

    def queue(self, source):
        source = source._get_queue_source()

        if not self._sources:
            self._source_read_index = 0
            source._init_texture(self)
        self._sources.append(source)

    def next(self):
        if self._sources:
            old_source = self._sources.pop(0)
            old_source._release_texture(self)
            old_source._stop()
            self._source_read_index -= 1

        if self._sources:
            self._sources[0]._init_texture(self)

    def dispatch_events(self):
        if not self._sources:
            return

        if not self._playing:
            # If paused, just update the video texture.
            if self._texture:
                self._sources[0]._update_texture(self, self.time)
            return

        # Create a device if there isn't one.  TODO only if source and source
        # has audio
        if not self._device:
            self._device = Device('plug:front')
            self._device.prepare(self._sources[0])

        self_time = self.time

        # Passed EOS?
        source = self._sources[0]
        while source and source.duration < self_time:
            if self._eos_action == self.EOS_NEXT:
                self.next()
            elif self._eos_action == self.EOS_STOP:
                self.stop()
                self._sources = []
                return
            self.dispatch_event('on_eos')

            self_time -= source.duration
            self._cumulative_buffer_time -= source.duration
            assert self._cumulative_buffer_time >= -0.001 # some float err ok
            try:
                source = self._sources[0]
                self._set_start_time(self._sources[0], self_time)
            except IndexError:
                source = None
                self._start_time = None

        # Ensure device buffer is full
        try:
            source = self._sources[self._source_read_index]
        except IndexError:
            source = None
        while (source and
               self._cumulative_buffer_time + self._current_buffer_time - self_time
                  < self._min_buffer_time):
            if self._queue_audio_data:
                audio_data = self._queue_audio_data
                self._queue_audio_data = None
            else:
                max_bytes = int(
                  self._min_buffer_time * source.audio_format.bytes_per_second)
                max_bytes = min(max_bytes, self._max_buffer_size)
                audio_data = source._get_audio_data(max_bytes)

            if audio_data:
                samples = \
                    audio_data.length // source.audio_format.bytes_per_sample
                samples_out = asound.snd_pcm_writei(self._device.pcm,
                    audio_data.data, samples)
                if samples_out < 0:
                    if samples_out == -11: # EAGAIN
                        self._queue_audio_data = audio_data
                    elif samples_out == -32: # EPIPE
                        # xrun recovery
                        check(asound.snd_pcm_prepare(self._device.pcm))
                        self._queue_audio_data = audio_data
                    else:
                        raise ALSAException(asound.snd_strerror(samples_out))
                elif samples_out < samples:
                    audio_data.consume(
                        samples_out * source.audio_format.bytes_per_sample,
                        source.audio_format)
                    self._current_buffer_time = audio_data.timestamp
                    self._queue_audio_data = audio_data
                else:
                    self._current_buffer_time = \
                        audio_data.timestamp + audio_data.duration

                if self._start_time is None:
                    # XXX start playback
                    self._set_start_time(source, audio_data.timestamp)

            else:
                # EOS on read source
                self._cumulative_buffer_time += source.duration
                self._current_buffer_time = 0.
                if self._eos_action == self.EOS_NEXT:
                    self._source_read_index += 1
                    try:
                        # preroll
                        source = self._sources[self._source_read_index]
                        source._play()
                    except IndexError:
                        source = None
                elif self._eos_action == self.EOS_LOOP:
                    source._seek(0)
                elif self._eos_action == self.EOS_PAUSE:
                    source = None
                elif self._eos_action == self.EOS_STOP:
                    source = None
                else:
                    assert False, 'Invalid eos_action'
                    source = None

        # Update video texture
        if self._texture:
            self._sources[0]._update_texture(self, self_time)

    def _get_time(self):
        if self._start_time is None or not self._playing:
            return self._timestamp
        return self._get_asound_time() - self._start_time

    def _get_asound_time(self):
        status = ctypes.POINTER(asound.snd_pcm_status_t)()
        timestamp = asound.snd_timestamp_t()

        check(asound.snd_pcm_status_malloc(ctypes.byref(status)))
        check(asound.snd_pcm_status(self._device.pcm, status))
        asound.snd_pcm_status_get_tstamp(status, ctypes.byref(timestamp))
        asound.snd_pcm_status_free(status)
        return timestamp.tv_sec + timestamp.tv_usec * 0.000001

    def _clear_ring_buffer(self):
        check(asound.snd_pcm_drop(self._device.pcm))
        check(asound.snd_pcm_prepare(self._device.pcm))
        self._current_buffer_time = 0.
        self._cumulative_buffer_time = 0.
        self._source_read_index = 0
        self._queue_audio_data = None
        self._start_time = None

    def _set_start_time(self, source, timestamp):
        delay = asound.snd_pcm_sframes_t()
        check(asound.snd_pcm_delay(self._device.pcm, ctypes.byref(delay)))
        delay = (delay.value /
                 source.audio_format.bytes_per_sample /
                 float(source.audio_format.bytes_per_second))
        self._start_time = self._get_asound_time() - timestamp - delay

    def play(self):
        if self._playing:
            return

        self._playing = True

        if not self._sources:
            return

        if self._device:
            state = asound.snd_pcm_state(self._device.pcm)
            if self._device.can_pause and state == asound.SND_PCM_STATE_PAUSED:
                check(asound.snd_pcm_pause(self._device.pcm, 0))
                self._set_start_time(self._sources[0], self._timestamp)

    def pause(self):
        if not self._playing:
            return

        self._playing = False

        if not self._sources:
            return

        if self._device:
            self._timestamp = self._get_asound_time() - self._start_time
            if self._device.can_pause:
                check(asound.snd_pcm_pause(self._device.pcm, 1))
            else:
                # Hardware can't pause, so we'll just drop everything that's
                # been buffered.  Improvement could be to rebuffer data that
                # wasn't played.
                self._clear_ring_buffer()

    def seek(self, timestamp):
        if self._sources:
            self._sources[0]._seek(timestamp)
            self._timestamp = timestamp
            self._start_time = None

            if self._device:
                self._clear_ring_buffer()

    def _get_source(self):
        if self._sources:
            return self._sources[0]
        return None

    def _stop(self):
        raise RuntimeError('Invalid eos_action for this player.')

    def _set_volume(self, volume):
        self._volume = volume
        # TODO apply to device

    # All other properties are silently ignored.

    def _set_min_gain(self, min_gain):
        self._min_gain = min_gain

    def _set_max_gain(self, max_gain):
        self._max_gain = max_gain

    def _set_position(self, position):
        self._position = position

    def _set_velocity(self, velocity):
        self._velocity = velocity

    def _set_pitch(self, pitch):
        self._pitch = pitch

    def _set_cone_orientation(self, cone_orientation):
        self._cone_orientation = cone_orientation

    def _set_cone_inner_angle(self, cone_inner_angle):
        self._cone_inner_angle = cone_inner_angle

    def _set_cone_outer_gain(self, cone_outer_gain):
        self._cone_outer_gain = cone_outer_gain

class ALSAManagedSoundPlayer(ALSAPlayer, ManagedSoundPlayerMixIn):
    pass

class ALSAListener(Listener):
    def set_volume(self, volume):
        # TODO master volume
        self._volume = volume

    # All other properties are silently ignored.

    def set_position(self, position):
        self._position = position

    def set_velocity(self, velocity):
        self._velocity = velocity

    def set_forward_orientation(self, orientation):
        self._forward_orientation = orientation

    def set_up_orientation(self, orientation):
        self._up_orientation = orientation

    def set_doppler_factor(self, factor):
        self._doppler_factor = factor

    def set_speed_of_sound(self, speed_of_sound):
        self._speed_of_sound = speed_of_sound

def driver_init():
    global debug_output
    print asound.snd_asoundlib_version()
    debug_output = ctypes.POINTER(asound.snd_output_t)()
    if alsa_debug:
        check(asound.snd_output_stdio_open(ctypes.byref(debug_output),
                                           alsa_debug,
                                           'w'))

driver_listener = ALSAListener()
DriverPlayer = ALSAPlayer
DriverManagedSoundPlayer = ALSAManagedSoundPlayer
<|endoftext|>


<commit_before>#!/usr/bin/env python
from itertools import product
from pprint import pprint
import glob
import json
import sys
import os

from Cheetah.Template import Template

class BohriumTemplate(Template):
    """This class adds a couple of functions which can be used within the template."""

    def addw(text, nw=10):
        w = ' '*nw
        return text+w[len(text):] if nw > len(text) else text

    def addsep(element, elements):
        cur_id  = element['id']
        last_id = elements[-1]['id']

        return '' if cur_id == last_id else ','

def forward_everything(opcodes, ops, opers, types, layouts):
    """This is for those functions that do not need some sort of mangling, they just want it all."""

    return {
        'opcodes':  opcodes,
        'ops':      ops,
        'opers':    opers,
        'types':    types,
        'layouts':  layouts
    }

def utils_mapping(opcodes, ops, opers, types, layouts):
    return forward_everything(opcodes, ops, opers, types, layouts)

def tac(opcodes, ops, opers, types, layouts):
    return forward_everything(opcodes, ops, opers, types, layouts)

def instrs_to_tacs(opcodes, ops, opers, types, layouts):
    """Construct the data need to create a map from bh_instruction to tac_t."""

    ewise_u     = []
    ewise_b     = []
    scans       = []
    reductions  = []
    generators  = []
    system      = []

    huh = []

    for o in opcodes:
        opcode = o['opcode']
        if o["composite"]:  # TODO: Specialize composite codes.
            continue

        if o['system_opcode']:
            system.append([opcode, 'SYSTEM', opcode.replace('BH_',''), 0])

        else:
            if 'REDUCE' in opcode:
                operator = '_'.join(opcode.split('_')[1:-1])
                reductions.append([opcode, 'REDUCE_PARTIAL', operator, 2])
            elif 'ACCUMULATE' in opcode:
                operator = '_'.join(opcode.split('_')[1:-1])
                scans.append([opcode, 'SCAN', operator, 2])
            elif 'RANDOM' in opcode:
                generators.append([opcode, 'GENERATE', 'RANDOM', 2])
            elif 'RANGE' in opcode:
                generators.append([opcode, 'GENERATE', 'RANGE', 0])
            else:
                operator = '_'.join(opcode.split('_')[1:])
                if o['nop'] == 3:
                    ewise_b.append([opcode, 'ZIP', operator, 2])
                elif o['nop'] == 2:
                    ewise_u.append([opcode, 'MAP', operator, 1])
                else:
                    huh.append([opcode, '?', operator, 0])

    if len(huh)>0:
        print "Something is weird here!", huh

    operations = [{'operations': \
        sorted(ewise_u)         +\
        sorted(ewise_b)         +\
        sorted(reductions)      +\
        sorted(scans)           +\
        sorted(generators)      +\
        sorted(system)
    }]

    return operations

def main(self):

    root    = "../../../core/codegen"
    prefix  = "./tac/"

    opcodes = json.load(open("%s%s%s.json" % (root,   os.sep, 'opcodes')))
    ops     = json.load(open("%s%s%s.json" % (prefix, os.sep, 'operations')))
    opers   = json.load(open("%s%s%s.json" % (prefix, os.sep, 'operators')))
    types   = json.load(open("%s%s%s.json" % (prefix, os.sep, 'types')))
    layouts = json.load(open("%s%s%s.json" % (prefix, os.sep, 'layouts')))

    # Map template names to mapping-functons and fill out the template
    for fn in glob.glob('templates/*.tpl'):
        fn, _ = os.path.basename(fn).split('.tpl')
        if fn in self.__dict__:
            template = BohriumTemplate(
                file = "%s%s%s.tpl" % ("templates", os.sep, fn),
                searchList=globals()[fn](opcodes, ops, opers, types, layouts)
            )
            with open('output/%s.cpp' % fn, 'w') as fd:
                fd.write(str(template))

if __name__ == "__main__":
    main(sys.modules[__name__])
<commit_msg>cpu-autogen: Added handling of composite opcodes.<commit_after>#!/usr/bin/env python
from itertools import product
from pprint import pprint
import glob
import json
import sys
import os

from Cheetah.Template import Template

class BohriumTemplate(Template):
    """This class adds a couple of functions which can be used within the template."""

    def addw(text, nw=10):
        w = ' '*nw
        return text+w[len(text):] if nw > len(text) else text

    def addsep(element, elements):
        cur_id  = element['id']
        last_id = elements[-1]['id']

        return '' if cur_id == last_id else ','

def forward_everything(opcodes, ops, opers, types, layouts):
    """This is for those functions that do not need some sort of mangling, they just want it all."""

    return {
        'opcodes':  opcodes,
        'ops':      ops,
        'opers':    opers,
        'types':    types,
        'layouts':  layouts
    }

def utils_mapping(opcodes, ops, opers, types, layouts):
    return forward_everything(opcodes, ops, opers, types, layouts)

def tac(opcodes, ops, opers, types, layouts):
    return forward_everything(opcodes, ops, opers, types, layouts)

def instrs_to_tacs(opcodes, ops, opers, types, layouts):
    """Construct the data need to create a map from bh_instruction to tac_t."""

    ewise_u     = []
    ewise_b     = []
    scans       = []
    reductions  = []
    generators  = []
    system      = []

    huh = []
    for o in opcodes:
        opcode = o['opcode']
        if o["composite"]:
            tac_name = opcode.replace("BH_", '')
            if not [tac for tac in opers if tac['name'] == tac_name]:
                print "Non-specialized composite: %s" % opcode
                continue

        if o['system_opcode']:
            system.append([opcode, 'SYSTEM', opcode.replace('BH_',''), 0])

        else:
            if 'REDUCE' in opcode:
                operator = '_'.join(opcode.split('_')[1:-1])
                reductions.append([opcode, 'REDUCE_PARTIAL', operator, 2])
            elif 'ACCUMULATE' in opcode:
                operator = '_'.join(opcode.split('_')[1:-1])
                scans.append([opcode, 'SCAN', operator, 2])
            elif 'RANDOM' in opcode:
                generators.append([opcode, 'GENERATE', 'RANDOM', 2])
            elif 'RANGE' in opcode:
                generators.append([opcode, 'GENERATE', 'RANGE', 0])
            else:
                operator = '_'.join(opcode.split('_')[1:])
                if o['nop'] == 3:
                    ewise_b.append([opcode, 'ZIP', operator, 2])
                elif o['nop'] == 2:
                    ewise_u.append([opcode, 'MAP', operator, 1])
                else:
                    huh.append([opcode, '?', operator, 0])

    if len(huh)>0:
        print "Something is weird here!", huh

    operations = [{'operations': \
        sorted(ewise_u)         +\
        sorted(ewise_b)         +\
        sorted(reductions)      +\
        sorted(scans)           +\
        sorted(generators)      +\
        sorted(system)
    }]

    return operations

def main(self):

    root    = "../../../core/codegen"
    prefix  = "./tac/"

    opcodes = json.load(open("%s%s%s.json" % (root,   os.sep, 'opcodes')))
    ops     = json.load(open("%s%s%s.json" % (prefix, os.sep, 'operations')))
    opers   = json.load(open("%s%s%s.json" % (prefix, os.sep, 'operators')))
    types   = json.load(open("%s%s%s.json" % (prefix, os.sep, 'types')))
    layouts = json.load(open("%s%s%s.json" % (prefix, os.sep, 'layouts')))

    # Map template names to mapping-functons and fill out the template
    for fn in glob.glob('templates/*.tpl'):
        fn, _ = os.path.basename(fn).split('.tpl')
        if fn in self.__dict__:
            template = BohriumTemplate(
                file = "%s%s%s.tpl" % ("templates", os.sep, fn),
                searchList=globals()[fn](opcodes, ops, opers, types, layouts)
            )
            with open('output/%s.cpp' % fn, 'w') as fd:
                fd.write(str(template))

if __name__ == "__main__":
    main(sys.modules[__name__])
<|endoftext|>


<commit_before>import os.path
import numpy as np
import pytest

from eccodes import *

SAMPLE_DATA_FOLDER = os.path.join(os.path.dirname(__file__), 'sample-data')
TEST_DATA = os.path.join(SAMPLE_DATA_FOLDER, 'era5-levels-members.grib')

# GRIB
def test_grib_read():
    gid = codes_grib_new_from_samples('regular_ll_sfc_grib1')
    assert codes_get(gid, 'Ni') == 16
    assert codes_get(gid, 'Nj') == 31
    assert codes_get(gid, 'centre', str) == 'ecmf'
    codes_release(gid)

def test_grib_write(tmpdir):
    gid = codes_grib_new_from_samples('GRIB2')
    codes_set(gid, 'backgroundProcess', 44)
    output = tmpdir.join('test_grib_write.grib')
    with open(str(output), 'wb') as fout:
        codes_write(gid, fout)
    codes_release(gid)

# BUFR
def test_bufr_read_write(tmpdir):
    bid = codes_bufr_new_from_samples('BUFR4')
    codes_set(bid, 'unpack', 1)
    assert codes_get(bid, 'typicalYear') == 2012
    assert codes_get(bid, 'centre', str) == 'ecmf'
    codes_set(bid, 'totalSunshine', 13)
    codes_set(bid, 'pack', 1)
    output = tmpdir.join('test_bufr_write.bufr')
    with open(str(output), 'wb') as fout:
        codes_write(bid, fout)
    assert codes_get(bid, 'totalSunshine') == 13
    codes_release(bid)
<commit_msg>Add more tests<commit_after>import os.path
import numpy as np
import pytest

from eccodes import *

SAMPLE_DATA_FOLDER = os.path.join(os.path.dirname(__file__), 'sample-data')
TEST_DATA = os.path.join(SAMPLE_DATA_FOLDER, 'era5-levels-members.grib')

# GRIB
def test_grib_read():
    gid = codes_grib_new_from_samples('regular_ll_sfc_grib1')
    assert codes_get(gid, 'Ni') == 16
    assert codes_get(gid, 'Nj') == 31
    assert codes_get(gid, 'const') == 1
    assert codes_get(gid, 'centre', str) == 'ecmf'
    assert codes_get(gid, 'packingType', str) == 'grid_simple'
    assert codes_get(gid, 'gridType', str) == 'regular_ll'
    codes_release(gid)
    gid = codes_grib_new_from_samples('sh_ml_grib2')
    assert codes_get(gid, 'const') == 0
    assert codes_get(gid, 'gridType', str) == 'sh'
    assert codes_get(gid, 'typeOfLevel', str) == 'hybrid'
    assert codes_get_long(gid, 'avg') == 185
    codes_release(gid)

def test_grib_write(tmpdir):
    gid = codes_grib_new_from_samples('GRIB2')
    codes_set(gid, 'backgroundProcess', 44)
    output = tmpdir.join('test_grib_write.grib')
    with open(str(output), 'wb') as fout:
        codes_write(gid, fout)
    codes_release(gid)

def test_grib_keys_iterator():
    gid = codes_grib_new_from_samples('reduced_gg_pl_1280_grib1')
    iterid = codes_keys_iterator_new(gid, 'ls')
    count = 0
    while codes_keys_iterator_next(iterid):
        keyname = codes_keys_iterator_get_name(iterid)
        keyval = codes_get_string(gid, keyname)
        count += 1
    assert count == 10
    codes_keys_iterator_delete(iterid)
    codes_release(gid)

# BUFR
def test_bufr_read_write(tmpdir):
    bid = codes_bufr_new_from_samples('BUFR4')
    codes_set(bid, 'unpack', 1)
    assert codes_get(bid, 'typicalYear') == 2012
    assert codes_get(bid, 'centre', str) == 'ecmf'
    codes_set(bid, 'totalSunshine', 13)
    codes_set(bid, 'pack', 1)
    output = tmpdir.join('test_bufr_write.bufr')
    with open(str(output), 'wb') as fout:
        codes_write(bid, fout)
    assert codes_get(bid, 'totalSunshine') == 13
    codes_release(bid)

def test_bufr_keys_iterator():
    bid = codes_bufr_new_from_samples('BUFR3_local_satellite')
    # Header keys only
    iterid = codes_bufr_keys_iterator_new(bid)
    count = 0
    while codes_bufr_keys_iterator_next(iterid):
        keyname = codes_bufr_keys_iterator_get_name(iterid)
        assert '#' not in keyname
        count += 1
    assert count == 53

    codes_set(bid, 'unpack', 1)
    codes_bufr_keys_iterator_rewind(iterid)
    count = 0
    while codes_bufr_keys_iterator_next(iterid):
        keyname = codes_bufr_keys_iterator_get_name(iterid)
        count += 1
    assert count == 156

    codes_bufr_keys_iterator_delete(iterid)
    codes_release(bid)
<|endoftext|>


<commit_before>"""
    pygments.lexers._postgres_builtins
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    Self-updating data files for PostgreSQL lexer.

    :copyright: Copyright 2011 by Daniele Varrazzo.
    :license: BSD, see LICENSE for details.
"""

import re
import urllib2

# One man's constant is another man's variable.
SOURCE_URL = 'https://github.com/postgres/postgres/raw/REL9_0_STABLE'
KEYWORDS_URL = SOURCE_URL + '/doc/src/sgml/keywords.sgml'
DATATYPES_URL = SOURCE_URL + '/doc/src/sgml/datatype.sgml'

def update_myself():
    datatypes = parse_datatypes(fetch(DATATYPES_URL))
    keywords = parse_keywords(fetch(KEYWORDS_URL))
    update_consts(__file__, 'DATATYPES', datatypes)
    update_consts(__file__, 'KEYWORDS', keywords)

def parse_keywords(f):
    kw = []
    re_entry = re.compile('\s*<entry><token>([^<]+)</token></entry>')
    for line in f:
        m = re_entry.match(line)
        if m is None:
            continue

        kw.append(m.group(1))

    kw.sort()
    return kw

def parse_datatypes(f):
    dt = set()
    re_entry = re.compile('\s*<entry><type>([^<]+)</type></entry>')
    for line in f:
        if '<sect1' in line:
            break
        if '<entry><type>' not in line:
            continue

        # Parse a string such as
        # time [ (<replaceable>p</replaceable>) ] [ without time zone ]
        # into types "time" and "without time zone"

        # remove all the tags
        line = re.sub("<replaceable>[^<]+</replaceable>", "", line)
        line = re.sub("<[^>]+>", "", line)

        # Drop the parts containing braces
        for tmp in [ t for tmp in line.split('[') for t in tmp.split(']') if "(" not in t ]:
            for t in tmp.split(','):
                t = t.strip()
                if not t: continue
                dt.add(" ".join(t.split()))

    dt = list(dt)
    dt.sort()
    return dt

def fetch(url):
    return urllib2.urlopen(url)

def update_consts(filename, constname, content):
    f = open(filename)
    lines = f.readlines()
    f.close()

    # Line to start/end inserting
    re_start = re.compile(r'^%s\s*=\s*\[\s*$' % constname)
    re_end = re.compile(r'^\s*\]\s*$')
    start = [ n for n, l in enumerate(lines) if re_start.match(l) ]
    if not start:
        raise ValueError("couldn't find line containing '%s = ['" % constname)
    if len(start) > 1:
        raise ValueError("too many lines containing '%s = ['" % constname)
    start = start[0] + 1

    end = [ n for n, l in enumerate(lines) if n >= start and re_end.match(l) ]
    if not end:
        raise ValueError("couldn't find line containing ']' after %s " % constname)
    end = end[0]

    # Pack the new content in lines not too long
    content = [repr(item) for item in content ]
    new_lines = [[]]
    for item in content:
        if sum(map(len, new_lines[-1])) + 2 * len(new_lines[-1]) + len(item) + 4 > 75:
            new_lines.append([])
        new_lines[-1].append(item)

    lines[start:end] = [ "    %s,\n" % ", ".join(items) for items in new_lines ]

    f = open(filename, 'w')
    f.write(''.join(lines))
    f.close()

# Autogenerated: please edit them if you like wasting your time.

KEYWORDS = [
    ]

DATATYPES = [
    ]

if __name__ == '__main__':
    update_myself()

<commit_msg>PostgreSQL keywords updated to version 9.0<commit_after>"""
    pygments.lexers._postgres_builtins
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    Self-updating data files for PostgreSQL lexer.

    :copyright: Copyright 2011 by Daniele Varrazzo.
    :license: BSD, see LICENSE for details.
"""

import re
import urllib2

# One man's constant is another man's variable.
SOURCE_URL = 'https://github.com/postgres/postgres/raw/REL9_0_STABLE'
KEYWORDS_URL = SOURCE_URL + '/doc/src/sgml/keywords.sgml'
DATATYPES_URL = SOURCE_URL + '/doc/src/sgml/datatype.sgml'

def update_myself():
    datatypes = parse_datatypes(fetch(DATATYPES_URL))
    keywords = parse_keywords(fetch(KEYWORDS_URL))
    update_consts(__file__, 'DATATYPES', datatypes)
    update_consts(__file__, 'KEYWORDS', keywords)

def parse_keywords(f):
    kw = []
    re_entry = re.compile('\s*<entry><token>([^<]+)</token></entry>')
    for line in f:
        m = re_entry.match(line)
        if m is None:
            continue

        kw.append(m.group(1))

    kw.sort()
    return kw

def parse_datatypes(f):
    dt = set()
    re_entry = re.compile('\s*<entry><type>([^<]+)</type></entry>')
    for line in f:
        if '<sect1' in line:
            break
        if '<entry><type>' not in line:
            continue

        # Parse a string such as
        # time [ (<replaceable>p</replaceable>) ] [ without time zone ]
        # into types "time" and "without time zone"

        # remove all the tags
        line = re.sub("<replaceable>[^<]+</replaceable>", "", line)
        line = re.sub("<[^>]+>", "", line)

        # Drop the parts containing braces
        for tmp in [ t for tmp in line.split('[') for t in tmp.split(']') if "(" not in t ]:
            for t in tmp.split(','):
                t = t.strip()
                if not t: continue
                dt.add(" ".join(t.split()))

    dt = list(dt)
    dt.sort()
    return dt

def fetch(url):
    return urllib2.urlopen(url)

def update_consts(filename, constname, content):
    f = open(filename)
    lines = f.readlines()
    f.close()

    # Line to start/end inserting
    re_start = re.compile(r'^%s\s*=\s*\[\s*$' % constname)
    re_end = re.compile(r'^\s*\]\s*$')
    start = [ n for n, l in enumerate(lines) if re_start.match(l) ]
    if not start:
        raise ValueError("couldn't find line containing '%s = ['" % constname)
    if len(start) > 1:
        raise ValueError("too many lines containing '%s = ['" % constname)
    start = start[0] + 1

    end = [ n for n, l in enumerate(lines) if n >= start and re_end.match(l) ]
    if not end:
        raise ValueError("couldn't find line containing ']' after %s " % constname)
    end = end[0]

    # Pack the new content in lines not too long
    content = [repr(item) for item in content ]
    new_lines = [[]]
    for item in content:
        if sum(map(len, new_lines[-1])) + 2 * len(new_lines[-1]) + len(item) + 4 > 75:
            new_lines.append([])
        new_lines[-1].append(item)

    lines[start:end] = [ "    %s,\n" % ", ".join(items) for items in new_lines ]

    f = open(filename, 'w')
    f.write(''.join(lines))
    f.close()

# Autogenerated: please edit them if you like wasting your time.

KEYWORDS = [
    'A', 'ABORT', 'ABS', 'ABSENT', 'ABSOLUTE', 'ACCESS', 'ACCORDING',
    'ACTION', 'ADA', 'ADD', 'ADMIN', 'AFTER', 'AGGREGATE', 'ALIAS', 'ALL',
    'ALLOCATE', 'ALSO', 'ALTER', 'ALWAYS', 'ANALYSE', 'ANALYZE', 'AND',
    'ANY', 'ARE', 'ARRAY', 'ARRAY_AGG', 'AS', 'ASC', 'ASENSITIVE',
    'ASSERTION', 'ASSIGNMENT', 'ASYMMETRIC', 'AT', 'ATOMIC', 'ATTRIBUTE',
    'ATTRIBUTES', 'AUTHORIZATION', 'AVG', 'BACKWARD', 'BASE64', 'BEFORE',
    'BEGIN', 'BERNOULLI', 'BETWEEN', 'BIGINT', 'BINARY', 'BIT', 'BITVAR',
    'BIT_LENGTH', 'BLOB', 'BLOCKED', 'BOM', 'BOOLEAN', 'BOTH', 'BREADTH',
    'BY', 'C', 'CACHE', 'CALL', 'CALLED', 'CARDINALITY', 'CASCADE',
    'CASCADED', 'CASE', 'CAST', 'CATALOG', 'CATALOG_NAME', 'CEIL',
    'CEILING', 'CHAIN', 'CHAR', 'CHARACTER', 'CHARACTERISTICS',
    'CHARACTERS', 'CHARACTER_LENGTH', 'CHARACTER_SET_CATALOG',
    'CHARACTER_SET_NAME', 'CHARACTER_SET_SCHEMA', 'CHAR_LENGTH', 'CHECK',
    'CHECKED', 'CHECKPOINT', 'CLASS', 'CLASS_ORIGIN', 'CLOB', 'CLOSE',
    'CLUSTER', 'COALESCE', 'COBOL', 'COLLATE', 'COLLATION',
    'COLLATION_CATALOG', 'COLLATION_NAME', 'COLLATION_SCHEMA', 'COLLECT',
    'COLUMN', 'COLUMNS', 'COLUMN_NAME', 'COMMAND_FUNCTION',
    'COMMAND_FUNCTION_CODE', 'COMMENT', 'COMMENTS', 'COMMIT', 'COMMITTED',
    'COMPLETION', 'CONCURRENTLY', 'CONDITION', 'CONDITION_NUMBER',
    'CONFIGURATION', 'CONNECT', 'CONNECTION', 'CONNECTION_NAME',
    'CONSTRAINT', 'CONSTRAINTS', 'CONSTRAINT_CATALOG', 'CONSTRAINT_NAME',
    'CONSTRAINT_SCHEMA', 'CONSTRUCTOR', 'CONTAINS', 'CONTENT', 'CONTINUE',
    'CONTROL', 'CONVERSION', 'CONVERT', 'COPY', 'CORR', 'CORRESPONDING',
    'COST', 'COUNT', 'COVAR_POP', 'COVAR_SAMP', 'CREATE', 'CREATEDB',
    'CREATEROLE', 'CREATEUSER', 'CROSS', 'CSV', 'CUBE', 'CUME_DIST',
    'CURRENT', 'CURRENT_CATALOG', 'CURRENT_DATE',
    'CURRENT_DEFAULT_TRANSFORM_GROUP', 'CURRENT_PATH', 'CURRENT_ROLE',
    'CURRENT_SCHEMA', 'CURRENT_TIME', 'CURRENT_TIMESTAMP',
    'CURRENT_TRANSFORM_GROUP_FOR_TYPE', 'CURRENT_USER', 'CURSOR',
    'CURSOR_NAME', 'CYCLE', 'DATA', 'DATABASE', 'DATALINK', 'DATE',
    'DATETIME_INTERVAL_CODE', 'DATETIME_INTERVAL_PRECISION', 'DAY', 'DB',
    'DEALLOCATE', 'DEC', 'DECIMAL', 'DECLARE', 'DEFAULT', 'DEFAULTS',
    'DEFERRABLE', 'DEFERRED', 'DEFINED', 'DEFINER', 'DEGREE', 'DELETE',
    'DELIMITER', 'DELIMITERS', 'DENSE_RANK', 'DEPTH', 'DEREF', 'DERIVED',
    'DESC', 'DESCRIBE', 'DESCRIPTOR', 'DESTROY', 'DESTRUCTOR',
    'DETERMINISTIC', 'DIAGNOSTICS', 'DICTIONARY', 'DISABLE', 'DISCARD',
    'DISCONNECT', 'DISPATCH', 'DISTINCT', 'DLNEWCOPY', 'DLPREVIOUSCOPY',
    'DLURLCOMPLETE', 'DLURLCOMPLETEONLY', 'DLURLCOMPLETEWRITE', 'DLURLPATH',
    'DLURLPATHONLY', 'DLURLPATHWRITE', 'DLURLSCHEME', 'DLURLSERVER',
    'DLVALUE', 'DO', 'DOCUMENT', 'DOMAIN', 'DOUBLE', 'DROP', 'DYNAMIC',
    'DYNAMIC_FUNCTION', 'DYNAMIC_FUNCTION_CODE', 'EACH', 'ELEMENT', 'ELSE',
    'EMPTY', 'ENABLE', 'ENCODING', 'ENCRYPTED', 'END', 'END-EXEC', 'ENUM',
    'EQUALS', 'ESCAPE', 'EVERY', 'EXCEPT', 'EXCEPTION', 'EXCLUDE',
    'EXCLUDING', 'EXCLUSIVE', 'EXEC', 'EXECUTE', 'EXISTING', 'EXISTS',
    'EXP', 'EXPLAIN', 'EXTERNAL', 'EXTRACT', 'FALSE', 'FAMILY', 'FETCH',
    'FILE', 'FILTER', 'FINAL', 'FIRST', 'FIRST_VALUE', 'FLAG', 'FLOAT',
    'FLOOR', 'FOLLOWING', 'FOR', 'FORCE', 'FOREIGN', 'FORTRAN', 'FORWARD',
    'FOUND', 'FREE', 'FREEZE', 'FROM', 'FS', 'FULL', 'FUNCTION',
    'FUNCTIONS', 'FUSION', 'G', 'GENERAL', 'GENERATED', 'GET', 'GLOBAL',
    'GO', 'GOTO', 'GRANT', 'GRANTED', 'GREATEST', 'GROUP', 'GROUPING',
    'HANDLER', 'HAVING', 'HEADER', 'HEX', 'HIERARCHY', 'HOLD', 'HOST',
    'HOUR', 'ID', 'IDENTITY', 'IF', 'IGNORE', 'ILIKE', 'IMMEDIATE',
    'IMMUTABLE', 'IMPLEMENTATION', 'IMPLICIT', 'IMPORT', 'IN', 'INCLUDING',
    'INCREMENT', 'INDENT', 'INDEX', 'INDEXES', 'INDICATOR', 'INFIX',
    'INHERIT', 'INHERITS', 'INITIALIZE', 'INITIALLY', 'INLINE', 'INNER',
    'INOUT', 'INPUT', 'INSENSITIVE', 'INSERT', 'INSTANCE', 'INSTANTIABLE',
    'INSTEAD', 'INT', 'INTEGER', 'INTEGRITY', 'INTERSECT', 'INTERSECTION',
    'INTERVAL', 'INTO', 'INVOKER', 'IS', 'ISNULL', 'ISOLATION', 'ITERATE',
    'JOIN', 'K', 'KEY', 'KEY_MEMBER', 'KEY_TYPE', 'LAG', 'LANGUAGE',
    'LARGE', 'LAST', 'LAST_VALUE', 'LATERAL', 'LC_COLLATE', 'LC_CTYPE',
    'LEAD', 'LEADING', 'LEAST', 'LEFT', 'LENGTH', 'LESS', 'LEVEL',
    'LIBRARY', 'LIKE', 'LIKE_REGEX', 'LIMIT', 'LINK', 'LISTEN', 'LN',
    'LOAD', 'LOCAL', 'LOCALTIME', 'LOCALTIMESTAMP', 'LOCATION', 'LOCATOR',
    'LOCK', 'LOGIN', 'LOWER', 'M', 'MAP', 'MAPPING', 'MATCH', 'MATCHED',
    'MAX', 'MAXVALUE', 'MAX_CARDINALITY', 'MEMBER', 'MERGE',
    'MESSAGE_LENGTH', 'MESSAGE_OCTET_LENGTH', 'MESSAGE_TEXT', 'METHOD',
    'MIN', 'MINUTE', 'MINVALUE', 'MOD', 'MODE', 'MODIFIES', 'MODIFY',
    'MODULE', 'MONTH', 'MORE', 'MOVE', 'MULTISET', 'MUMPS', 'NAME', 'NAMES',
    'NAMESPACE', 'NATIONAL', 'NATURAL', 'NCHAR', 'NCLOB', 'NESTING', 'NEW',
    'NEXT', 'NFC', 'NFD', 'NFKC', 'NFKD', 'NIL', 'NO', 'NOCREATEDB',
    'NOCREATEROLE', 'NOCREATEUSER', 'NOINHERIT', 'NOLOGIN', 'NONE',
    'NORMALIZE', 'NORMALIZED', 'NOSUPERUSER', 'NOT', 'NOTHING', 'NOTIFY',
    'NOTNULL', 'NOWAIT', 'NTH_VALUE', 'NTILE', 'NULL', 'NULLABLE', 'NULLIF',
    'NULLS', 'NUMBER', 'NUMERIC', 'OBJECT', 'OCCURRENCES_REGEX', 'OCTETS',
    'OCTET_LENGTH', 'OF', 'OFF', 'OFFSET', 'OIDS', 'OLD', 'ON', 'ONLY',
    'OPEN', 'OPERATION', 'OPERATOR', 'OPTION', 'OPTIONS', 'OR', 'ORDER',
    'ORDERING', 'ORDINALITY', 'OTHERS', 'OUT', 'OUTER', 'OUTPUT', 'OVER',
    'OVERLAPS', 'OVERLAY', 'OVERRIDING', 'OWNED', 'OWNER', 'P', 'PAD',
    'PARAMETER', 'PARAMETERS', 'PARAMETER_MODE', 'PARAMETER_NAME',
    'PARAMETER_ORDINAL_POSITION', 'PARAMETER_SPECIFIC_CATALOG',
    'PARAMETER_SPECIFIC_NAME', 'PARAMETER_SPECIFIC_SCHEMA', 'PARSER',
    'PARTIAL', 'PARTITION', 'PASCAL', 'PASSING', 'PASSTHROUGH', 'PASSWORD',
    'PATH', 'PERCENTILE_CONT', 'PERCENTILE_DISC', 'PERCENT_RANK',
    'PERMISSION', 'PLACING', 'PLANS', 'PLI', 'POSITION', 'POSITION_REGEX',
    'POSTFIX', 'POWER', 'PRECEDING', 'PRECISION', 'PREFIX', 'PREORDER',
    'PREPARE', 'PREPARED', 'PRESERVE', 'PRIMARY', 'PRIOR', 'PRIVILEGES',
    'PROCEDURAL', 'PROCEDURE', 'PUBLIC', 'QUOTE', 'RANGE', 'RANK', 'READ',
    'READS', 'REAL', 'REASSIGN', 'RECHECK', 'RECOVERY', 'RECURSIVE', 'REF',
    'REFERENCES', 'REFERENCING', 'REGR_AVGX', 'REGR_AVGY', 'REGR_COUNT',
    'REGR_INTERCEPT', 'REGR_R2', 'REGR_SLOPE', 'REGR_SXX', 'REGR_SXY',
    'REGR_SYY', 'REINDEX', 'RELATIVE', 'RELEASE', 'RENAME', 'REPEATABLE',
    'REPLACE', 'REPLICA', 'REQUIRING', 'RESET', 'RESPECT', 'RESTART',
    'RESTORE', 'RESTRICT', 'RESULT', 'RETURN', 'RETURNED_CARDINALITY',
    'RETURNED_LENGTH', 'RETURNED_OCTET_LENGTH', 'RETURNED_SQLSTATE',
    'RETURNING', 'RETURNS', 'REVOKE', 'RIGHT', 'ROLE', 'ROLLBACK', 'ROLLUP',
    'ROUTINE', 'ROUTINE_CATALOG', 'ROUTINE_NAME', 'ROUTINE_SCHEMA', 'ROW',
    'ROWS', 'ROW_COUNT', 'ROW_NUMBER', 'RULE', 'SAVEPOINT', 'SCALE',
    'SCHEMA', 'SCHEMA_NAME', 'SCOPE', 'SCOPE_CATALOG', 'SCOPE_NAME',
    'SCOPE_SCHEMA', 'SCROLL', 'SEARCH', 'SECOND', 'SECTION', 'SECURITY',
    'SELECT', 'SELECTIVE', 'SELF', 'SENSITIVE', 'SEQUENCE', 'SEQUENCES',
    'SERIALIZABLE', 'SERVER', 'SERVER_NAME', 'SESSION', 'SESSION_USER',
    'SET', 'SETOF', 'SETS', 'SHARE', 'SHOW', 'SIMILAR', 'SIMPLE', 'SIZE',
    'SMALLINT', 'SOME', 'SOURCE', 'SPACE', 'SPECIFIC', 'SPECIFICTYPE',
    'SPECIFIC_NAME', 'SQL', 'SQLCODE', 'SQLERROR', 'SQLEXCEPTION',
    'SQLSTATE', 'SQLWARNING', 'SQRT', 'STABLE', 'STANDALONE', 'START',
    'STATE', 'STATEMENT', 'STATIC', 'STATISTICS', 'STDDEV_POP',
    'STDDEV_SAMP', 'STDIN', 'STDOUT', 'STORAGE', 'STRICT', 'STRIP',
    'STRUCTURE', 'STYLE', 'SUBCLASS_ORIGIN', 'SUBLIST', 'SUBMULTISET',
    'SUBSTRING', 'SUBSTRING_REGEX', 'SUM', 'SUPERUSER', 'SYMMETRIC',
    'SYSID', 'SYSTEM', 'SYSTEM_USER', 'T', 'TABLE', 'TABLES', 'TABLESAMPLE',
    'TABLESPACE', 'TABLE_NAME', 'TEMP', 'TEMPLATE', 'TEMPORARY',
    'TERMINATE', 'TEXT', 'THAN', 'THEN', 'TIES', 'TIME', 'TIMESTAMP',
    'TIMEZONE_HOUR', 'TIMEZONE_MINUTE', 'TO', 'TOKEN', 'TOP_LEVEL_COUNT',
    'TRAILING', 'TRANSACTION', 'TRANSACTIONS_COMMITTED',
    'TRANSACTIONS_ROLLED_BACK', 'TRANSACTION_ACTIVE', 'TRANSFORM',
    'TRANSFORMS', 'TRANSLATE', 'TRANSLATE_REGEX', 'TRANSLATION', 'TREAT',
    'TRIGGER', 'TRIGGER_CATALOG', 'TRIGGER_NAME', 'TRIGGER_SCHEMA', 'TRIM',
    'TRIM_ARRAY', 'TRUE', 'TRUNCATE', 'TRUSTED', 'TYPE', 'UESCAPE',
    'UNBOUNDED', 'UNCOMMITTED', 'UNDER', 'UNENCRYPTED', 'UNION', 'UNIQUE',
    'UNKNOWN', 'UNLINK', 'UNLISTEN', 'UNNAMED', 'UNNEST', 'UNTIL',
    'UNTYPED', 'UPDATE', 'UPPER', 'URI', 'USAGE', 'USER',
    'USER_DEFINED_TYPE_CATALOG', 'USER_DEFINED_TYPE_CODE',
    'USER_DEFINED_TYPE_NAME', 'USER_DEFINED_TYPE_SCHEMA', 'USING', 'VACUUM',
    'VALID', 'VALIDATOR', 'VALUE', 'VALUES', 'VARBINARY', 'VARCHAR',
    'VARIABLE', 'VARIADIC', 'VARYING', 'VAR_POP', 'VAR_SAMP', 'VERBOSE',
    'VERSION', 'VIEW', 'VOLATILE', 'WHEN', 'WHENEVER', 'WHERE',
    'WHITESPACE', 'WIDTH_BUCKET', 'WINDOW', 'WITH', 'WITHIN', 'WITHOUT',
    'WORK', 'WRAPPER', 'WRITE', 'XML', 'XMLAGG', 'XMLATTRIBUTES',
    'XMLBINARY', 'XMLCAST', 'XMLCOMMENT', 'XMLCONCAT', 'XMLDECLARATION',
    'XMLDOCUMENT', 'XMLELEMENT', 'XMLEXISTS', 'XMLFOREST', 'XMLITERATE',
    'XMLNAMESPACES', 'XMLPARSE', 'XMLPI', 'XMLQUERY', 'XMLROOT',
    'XMLSCHEMA', 'XMLSERIALIZE', 'XMLTABLE', 'XMLTEXT', 'XMLVALIDATE',
    'YEAR', 'YES', 'ZONE',
    ]

DATATYPES = [
    'bigint', 'bigserial', 'bit', 'bit varying', 'bool', 'boolean', 'box',
    'bytea', 'char', 'character', 'character varying', 'cidr', 'circle',
    'date', 'decimal', 'double precision', 'float4', 'float8', 'inet',
    'int', 'int2', 'int4', 'int8', 'integer', 'interval', 'line', 'lseg',
    'macaddr', 'money', 'numeric', 'path', 'point', 'polygon', 'real',
    'serial', 'serial4', 'serial8', 'smallint', 'text', 'time', 'timestamp',
    'timestamptz', 'timetz', 'tsquery', 'tsvector', 'txid_snapshot', 'uuid',
    'varbit', 'varchar', 'with time zone', 'without time zone', 'xml',
    ]

if __name__ == '__main__':
    update_myself()

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# Copyright (c) 2014 ASMlover. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list ofconditions and the following disclaimer.
#
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materialsprovided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import socket
import time
import threading

ADDRESS = ('', 5555)
RECVBUF = 1024

running = None
def MessageHandler():
    server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    server.bind(ADDRESS)
    server.listen(5)
    running = True

    if server is None:
        print 'start server failed ...'
        running = False
        return
    print 'start server suceess ...'
    while running:
        client, addr = server.accept()
        if client is not None:
            data = client.recv(RECVBUF)
            if data is not None:
                print 'recevice from client', addr, data
                client.send(data)
            client.close()
    server.close()

def ServerMain():
    trd = threading.Thread(target=MessageHandler)
    trd.start()

    try:
        while True:
            time.sleep(0.1)
    except KeyboardInterrupt:
        running = False

if __name__ == '__main__':
    ServerMain()
<commit_msg>updated the server of python tcp<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# Copyright (c) 2014 ASMlover. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list ofconditions and the following disclaimer.
#
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materialsprovided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import socket
import time
import threading

ADDRESS = ('', 5555)
RECVBUF = 1024

running = None
def MessageHandler():
    server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    server.bind(ADDRESS)
    server.listen(5)
    running = True

    if server is None:
        print 'start server failed ...'
        running = False
        return
    print 'start server suceess ...'
    while running:
        client, addr = server.accept()
        if client is not None:
            data = client.recv(RECVBUF)
            if data is not None:
                print 'recevice from client', addr, data
                client.send(data)
            client.close()
    server.close()

def ServerMain():
    trd = threading.Thread(target=MessageHandler)
    trd.start()

    try:
        while True:
            time.sleep(0.1)
    except KeyboardInterrupt:
        running = False

if __name__ == '__main__':
    ServerMain()
<|endoftext|>


<commit_before>import os
import subprocess

from conda_concourse_ci import execute
import conda_concourse_ci
from conda_concourse_ci.utils import HashableDict

import pytest
import yaml

from .utils import test_data_dir, graph_data_dir, test_config_dir

def test_collect_tasks(mocker, testing_conda_resolve, testing_graph):
    mocker.patch.object(execute, 'Resolve')
    mocker.patch.object(execute, 'get_build_index')
    mocker.patch.object(conda_concourse_ci.compute_build_graph, '_installable')
    execute.Resolve.return_value = testing_conda_resolve
    conda_concourse_ci.compute_build_graph._installable.return_value = True
    task_graph = execute.collect_tasks(graph_data_dir, folders=['a'],
                                       matrix_base_dir=test_config_dir)
    build_platforms = os.listdir(os.path.join(test_config_dir, 'build_platforms.d'))
    # one build, one test per platform, uploads only for builds.
    n_platforms = len(build_platforms)
    # minimum args means build and test provided folders.  Two tasks.
    assert len(task_graph.nodes()) == n_platforms

boilerplate_test_vars = {'base-name': 'steve',
                         'aws-bucket': '123',
                         'aws-key-id': 'abc',
                         'aws-secret-key': 'weee',
                         'aws-region-name': 'frank'}

def test_get_build_task(testing_graph):
    # ensure that our channels make it into the args
    meta = testing_graph.node['b-on-linux']['meta']
    meta.config.channel_urls = ['conda_build_test']
    task = execute.get_build_task(base_path=graph_data_dir, graph=testing_graph,
                                node='b-on-linux', base_name="frank",
                                commit_id='abc123')
    assert task['config']['platform'] == 'linux'
    assert task['config']['inputs'] == [{'name': 'rsync-recipes'}]
    assert 'rsync-recipes/b-on-linux' in task['config']['run']['args'][-1]
    assert 'conda_build_test' in task['config']['run']['args'][-1]

def test_graph_to_plan_with_jobs(mocker, testing_graph):
    # stub out uploads, since it depends on config file stuff and we want to manipulate it
    # get_upload = mocker.patch.object(execute, "get_upload_tasks")
    # get_upload.return_value = []

    with open(os.path.join(test_config_dir, 'config.yml')) as f:
        config_vars = yaml.load(f)
    plan_dict = execute.graph_to_plan_with_jobs(graph_data_dir, testing_graph, 'abc123',
                                                test_config_dir, config_vars)
    # rsync-recipes, rsync-source, rsync-stats, and one artifact resource per build
    assert len(plan_dict['resources']) == 6
    # a, b, c
    assert len(plan_dict['jobs']) == 3

def test_resource_to_dict():
    resource = HashableDict(source=HashableDict(options=set(('a', 'b'))))
    out = execute._resource_to_dict(resource)
    assert type(out['source']['options']) == list
    assert type(out['source']) == dict

def test_submit(mocker):
    mocker.patch.object(execute, 'subprocess')
    pipeline_file = os.path.join(test_config_dir, 'plan_director.yml')
    execute.submit(pipeline_file, base_name="test", pipeline_name="test-pipeline",
                   src_dir='.', config_root_dir=os.path.join(test_data_dir, 'config-test'))

@pytest.mark.serial
def test_submit_one_off(mocker):
    check_call = mocker.patch.object(execute.subprocess, 'check_call')
    execute.submit_one_off('frank', os.path.join(test_data_dir, 'one-off-recipes'),
                           folders=('bzip2', 'pytest', 'pytest-cov'),
                           config_root_dir=test_config_dir)
    # basically what we're checking here is that the config_overrides have been passed correctly
    check_call.assert_has_calls([mocker.call(['rsync', '--delete', '-av', '-e',
                               mocker.ANY,  # ssh command that we don't care about much
                               mocker.ANY,  # temp source directory that we don't care about
                               mocker.ANY,  # -p (makes chmod flags work)
                               mocker.ANY,  # chmod flags
                               ('your-intermediate-user@your-intermediate-server:'
                                # this is what we care about.  The middle entry here
                                #    needs 'test' replaced with 'frank'.  Also, we're syncing a
                                #    plan and recipe folder, not a config folder
                                '/ci/frank/plan_and_recipes')
                               ])])

def test_bootstrap(mocker, testing_workdir):
    execute.bootstrap('frank')
    assert os.path.isfile('plan_director.yml')
    assert os.path.isdir('frank')
    assert os.path.isfile('frank/config.yml')
    assert os.path.isdir('frank/uploads.d')
    assert os.path.isdir('frank/build_platforms.d')
    assert os.path.isdir('frank/test_platforms.d')

def test_get_current_git_rev(testing_workdir):
    subprocess.check_call('git clone https://github.com/conda/conda_build_test_recipe'.split())
    git_repo = 'conda_build_test_recipe'
    assert execute._get_current_git_rev(git_repo) == '7e3525f4'
    with execute.checkout_git_rev('1.21.0', git_repo):
        assert execute._get_current_git_rev(git_repo) == '29bb0bd2'
        assert execute._get_current_git_rev(git_repo, True) == 'HEAD'

def test_compute_builds(testing_workdir, mocker, monkeypatch):
    monkeypatch.chdir(test_data_dir)
    output = os.path.join(testing_workdir, 'output')
    # neutralize git checkout so we're really testing the HEAD commit
    mocker.patch.object(execute, 'checkout_git_rev')
    execute.compute_builds('.', 'config-name', 'master',
                           folders=['python_test', 'conda_forge_style_recipe'],
                           matrix_base_dir=os.path.join(test_data_dir, 'linux-config-test'),
                           output_dir=output)
    assert os.path.isdir(output)
    files = os.listdir(output)
    assert 'plan.yml' in files

    assert os.path.isfile(os.path.join(output, 'frank-1.0-python_2.7-on-centos5-64', 'meta.yaml'))
    assert os.path.isfile(os.path.join(output, 'frank-1.0-python_2.7-on-centos5-64/', 'conda_build_config.yaml'))
    assert os.path.isfile(os.path.join(output, 'frank-1.0-python_3.6-on-centos5-64', 'meta.yaml'))
    assert os.path.isfile(os.path.join(output, 'frank-1.0-python_3.6-on-centos5-64/', 'conda_build_config.yaml'))
    assert os.path.isfile(os.path.join(output, 'dummy_conda_forge_test-1.0-on-centos5-64', 'meta.yaml'))
    with open(os.path.join(output, 'dummy_conda_forge_test-1.0-on-centos5-64/', 'conda_build_config.yaml')) as f:
        cfg = f.read()

    assert cfg is not None
    if hasattr(cfg, 'decode'):
        cfg = cfg.decode()
    assert "HashableDict" not in cfg

def test_compute_builds_intradependencies(testing_workdir, monkeypatch, mocker):
    """When we build stuff, and upstream dependencies are part of the batch, but they're
    also already installable, then we do extra work to make sure that we order our build
    so that downstream builds depend on upstream builds (and don't directly use the
    already-available packages.)"""
    monkeypatch.chdir(os.path.join(test_data_dir, 'intradependencies'))
    # neutralize git checkout so we're really testing the HEAD commit
    mocker.patch.object(execute, 'checkout_git_rev')
    output_dir = os.path.join(testing_workdir, 'output')
    execute.compute_builds('.', 'config-name', 'master',
                           folders=['zlib', 'uses_zlib'],
                           matrix_base_dir=os.path.join(test_data_dir, 'linux-config-test'),
                           output_dir=output_dir)
    assert os.path.isdir(output_dir)
    files = os.listdir(output_dir)
    assert 'plan.yml' in files
    with open(os.path.join(output_dir, 'plan.yml')) as f:
        plan = yaml.load(f)

    uses_zlib_job = [job for job in plan['jobs'] if job['name'] == 'uses_zlib-1.0-on-centos5-64'][0]
    assert any(task.get('passed') == ['zlib_wannabe-1.2.8-on-centos5-64']
               for task in uses_zlib_job['plan'])

def test_compute_builds_dies_when_no_folders_and_no_git(testing_workdir, mocker, capfd):
    changed = mocker.patch.object(execute, 'git_changed_recipes')
    changed.return_value = None
    output_dir = os.path.join(testing_workdir, 'output')
    execute.compute_builds('.', 'config-name', 'master',
                           folders=None,
                           matrix_base_dir=os.path.join(test_data_dir, 'linux-config-test'),
                           output_dir=output_dir)
    out, err = capfd.readouterr()
    assert "No folders specified to build" in out

def test_python_build_matrix_expansion(monkeypatch):
    monkeypatch.chdir(test_data_dir)
    tasks = execute.collect_tasks('.', matrix_base_dir=os.path.join(test_data_dir, 'linux-config-test'),
                                  folders=['python_test'])
    assert len(tasks.nodes()) == 2
    assert 'frank-1.0-python_2.7-on-centos5-64' in tasks.nodes()
    assert 'frank-1.0-python_3.6-on-centos5-64' in tasks.nodes()

def test_subpackage_matrix_no_subpackages(monkeypatch):
    """Subpackages should not constitute new entries in the build graph.  They should be lumped in
    with their parent recipe.  However, we have to include them initially for the sake of
    dependency ordering.  Thus we initially include them as though they were full packages, but
    then we squish them together and re-assign and dependency edges."""
    monkeypatch.chdir(test_data_dir)
    tasks = execute.collect_tasks('.', matrix_base_dir=os.path.join(test_data_dir, 'linux-config-test'),
                                  folders=['has_subpackages', 'depends_on_subpackage'])
    assert len(tasks.nodes()) == 2
    assert 'has_subpackages_toplevel-1.0-on-centos5-64' in tasks.nodes()
    assert 'depends_on_subpackage-1.0-on-centos5-64' in tasks.nodes()
    assert 'has_subpackages_subpackage-1.0-on-centos5-64' not in tasks.nodes()
    # this is the actual dependency
    assert ('depends_on_subpackage-1.0-on-centos5-64', 'has_subpackages_subpackage-1.0-on-centos5-64') not in tasks.edges()
    # this is what we remap it to
    assert ('depends_on_subpackage-1.0-on-centos5-64', 'has_subpackages_toplevel-1.0-on-centos5-64') in tasks.edges()

def test_dependency_with_selector_cross_compile(testing_conda_resolve):
    g = execute.collect_tasks(test_data_dir, ['selector_run', 'functools32-feedstock'],
                              matrix_base_dir=os.path.join(test_data_dir, 'config-win'),
                              variant_config_files=os.path.join(test_data_dir, 'conda_build_config.yaml'))
    assert len(g.nodes()) == 6
    # native edge
    assert ('test_run_deps_with_selector-1.0-python_2.7-on-win-64',
            'functools32_wannabe-3.2.3.2-python_2.7-on-win-64') in g.edges()
    # cross edge
    assert ('test_run_deps_with_selector-1.0-python_2.7-target_win-32-on-win-64',
            'functools32_wannabe-3.2.3.2-python_2.7-target_win-32-on-win-64') in g.edges()

def test_collapse_with_win_matrix_and_subpackages(monkeypatch):
    monkeypatch.chdir(test_data_dir)
    tasks = execute.collect_tasks('.', matrix_base_dir=os.path.join(test_data_dir, 'config-win'),
                                  folders=['win_split_outputs_compiler_reduction'])
    # 8 subpackages, but 4 total builds - 2 subpackages per build
    assert len(tasks.nodes()) == 4
    assert 'postgresql-split-10.1-c_compiler_vs2008-on-win-64' in tasks.nodes()
    assert 'postgresql-split-10.1-c_compiler_vs2015-on-win-64' in tasks.nodes()
    assert 'postgresql-split-10.1-c_compiler_vs2008-target_win-32-on-win-64' in tasks.nodes()
    assert 'postgresql-split-10.1-c_compiler_vs2015-target_win-32-on-win-64' in tasks.nodes()
<commit_msg>Fix test: drop base_name from call to get_build_task()<commit_after>import os
import subprocess

from conda_concourse_ci import execute
import conda_concourse_ci
from conda_concourse_ci.utils import HashableDict

import pytest
import yaml

from .utils import test_data_dir, graph_data_dir, test_config_dir

def test_collect_tasks(mocker, testing_conda_resolve, testing_graph):
    mocker.patch.object(execute, 'Resolve')
    mocker.patch.object(execute, 'get_build_index')
    mocker.patch.object(conda_concourse_ci.compute_build_graph, '_installable')
    execute.Resolve.return_value = testing_conda_resolve
    conda_concourse_ci.compute_build_graph._installable.return_value = True
    task_graph = execute.collect_tasks(graph_data_dir, folders=['a'],
                                       matrix_base_dir=test_config_dir)
    build_platforms = os.listdir(os.path.join(test_config_dir, 'build_platforms.d'))
    # one build, one test per platform, uploads only for builds.
    n_platforms = len(build_platforms)
    # minimum args means build and test provided folders.  Two tasks.
    assert len(task_graph.nodes()) == n_platforms

boilerplate_test_vars = {'base-name': 'steve',
                         'aws-bucket': '123',
                         'aws-key-id': 'abc',
                         'aws-secret-key': 'weee',
                         'aws-region-name': 'frank'}

def test_get_build_task(testing_graph):
    # ensure that our channels make it into the args
    meta = testing_graph.node['b-on-linux']['meta']
    meta.config.channel_urls = ['conda_build_test']
    task = execute.get_build_task(base_path=graph_data_dir, graph=testing_graph,
                                node='b-on-linux', commit_id='abc123')
    assert task['config']['platform'] == 'linux'
    assert task['config']['inputs'] == [{'name': 'rsync-recipes'}]
    assert 'rsync-recipes/b-on-linux' in task['config']['run']['args'][-1]
    assert 'conda_build_test' in task['config']['run']['args'][-1]

def test_graph_to_plan_with_jobs(mocker, testing_graph):
    # stub out uploads, since it depends on config file stuff and we want to manipulate it
    # get_upload = mocker.patch.object(execute, "get_upload_tasks")
    # get_upload.return_value = []

    with open(os.path.join(test_config_dir, 'config.yml')) as f:
        config_vars = yaml.load(f)
    plan_dict = execute.graph_to_plan_with_jobs(graph_data_dir, testing_graph, 'abc123',
                                                test_config_dir, config_vars)
    # rsync-recipes, rsync-source, rsync-stats, and one artifact resource per build
    assert len(plan_dict['resources']) == 6
    # a, b, c
    assert len(plan_dict['jobs']) == 3

def test_resource_to_dict():
    resource = HashableDict(source=HashableDict(options=set(('a', 'b'))))
    out = execute._resource_to_dict(resource)
    assert type(out['source']['options']) == list
    assert type(out['source']) == dict

def test_submit(mocker):
    mocker.patch.object(execute, 'subprocess')
    pipeline_file = os.path.join(test_config_dir, 'plan_director.yml')
    execute.submit(pipeline_file, base_name="test", pipeline_name="test-pipeline",
                   src_dir='.', config_root_dir=os.path.join(test_data_dir, 'config-test'))

@pytest.mark.serial
def test_submit_one_off(mocker):
    check_call = mocker.patch.object(execute.subprocess, 'check_call')
    execute.submit_one_off('frank', os.path.join(test_data_dir, 'one-off-recipes'),
                           folders=('bzip2', 'pytest', 'pytest-cov'),
                           config_root_dir=test_config_dir)
    # basically what we're checking here is that the config_overrides have been passed correctly
    check_call.assert_has_calls([mocker.call(['rsync', '--delete', '-av', '-e',
                               mocker.ANY,  # ssh command that we don't care about much
                               mocker.ANY,  # temp source directory that we don't care about
                               mocker.ANY,  # -p (makes chmod flags work)
                               mocker.ANY,  # chmod flags
                               ('your-intermediate-user@your-intermediate-server:'
                                # this is what we care about.  The middle entry here
                                #    needs 'test' replaced with 'frank'.  Also, we're syncing a
                                #    plan and recipe folder, not a config folder
                                '/ci/frank/plan_and_recipes')
                               ])])

def test_bootstrap(mocker, testing_workdir):
    execute.bootstrap('frank')
    assert os.path.isfile('plan_director.yml')
    assert os.path.isdir('frank')
    assert os.path.isfile('frank/config.yml')
    assert os.path.isdir('frank/uploads.d')
    assert os.path.isdir('frank/build_platforms.d')
    assert os.path.isdir('frank/test_platforms.d')

def test_get_current_git_rev(testing_workdir):
    subprocess.check_call('git clone https://github.com/conda/conda_build_test_recipe'.split())
    git_repo = 'conda_build_test_recipe'
    assert execute._get_current_git_rev(git_repo) == '7e3525f4'
    with execute.checkout_git_rev('1.21.0', git_repo):
        assert execute._get_current_git_rev(git_repo) == '29bb0bd2'
        assert execute._get_current_git_rev(git_repo, True) == 'HEAD'

def test_compute_builds(testing_workdir, mocker, monkeypatch):
    monkeypatch.chdir(test_data_dir)
    output = os.path.join(testing_workdir, 'output')
    # neutralize git checkout so we're really testing the HEAD commit
    mocker.patch.object(execute, 'checkout_git_rev')
    execute.compute_builds('.', 'config-name', 'master',
                           folders=['python_test', 'conda_forge_style_recipe'],
                           matrix_base_dir=os.path.join(test_data_dir, 'linux-config-test'),
                           output_dir=output)
    assert os.path.isdir(output)
    files = os.listdir(output)
    assert 'plan.yml' in files

    assert os.path.isfile(os.path.join(output, 'frank-1.0-python_2.7-on-centos5-64', 'meta.yaml'))
    assert os.path.isfile(os.path.join(output, 'frank-1.0-python_2.7-on-centos5-64/', 'conda_build_config.yaml'))
    assert os.path.isfile(os.path.join(output, 'frank-1.0-python_3.6-on-centos5-64', 'meta.yaml'))
    assert os.path.isfile(os.path.join(output, 'frank-1.0-python_3.6-on-centos5-64/', 'conda_build_config.yaml'))
    assert os.path.isfile(os.path.join(output, 'dummy_conda_forge_test-1.0-on-centos5-64', 'meta.yaml'))
    with open(os.path.join(output, 'dummy_conda_forge_test-1.0-on-centos5-64/', 'conda_build_config.yaml')) as f:
        cfg = f.read()

    assert cfg is not None
    if hasattr(cfg, 'decode'):
        cfg = cfg.decode()
    assert "HashableDict" not in cfg

def test_compute_builds_intradependencies(testing_workdir, monkeypatch, mocker):
    """When we build stuff, and upstream dependencies are part of the batch, but they're
    also already installable, then we do extra work to make sure that we order our build
    so that downstream builds depend on upstream builds (and don't directly use the
    already-available packages.)"""
    monkeypatch.chdir(os.path.join(test_data_dir, 'intradependencies'))
    # neutralize git checkout so we're really testing the HEAD commit
    mocker.patch.object(execute, 'checkout_git_rev')
    output_dir = os.path.join(testing_workdir, 'output')
    execute.compute_builds('.', 'config-name', 'master',
                           folders=['zlib', 'uses_zlib'],
                           matrix_base_dir=os.path.join(test_data_dir, 'linux-config-test'),
                           output_dir=output_dir)
    assert os.path.isdir(output_dir)
    files = os.listdir(output_dir)
    assert 'plan.yml' in files
    with open(os.path.join(output_dir, 'plan.yml')) as f:
        plan = yaml.load(f)

    uses_zlib_job = [job for job in plan['jobs'] if job['name'] == 'uses_zlib-1.0-on-centos5-64'][0]
    assert any(task.get('passed') == ['zlib_wannabe-1.2.8-on-centos5-64']
               for task in uses_zlib_job['plan'])

def test_compute_builds_dies_when_no_folders_and_no_git(testing_workdir, mocker, capfd):
    changed = mocker.patch.object(execute, 'git_changed_recipes')
    changed.return_value = None
    output_dir = os.path.join(testing_workdir, 'output')
    execute.compute_builds('.', 'config-name', 'master',
                           folders=None,
                           matrix_base_dir=os.path.join(test_data_dir, 'linux-config-test'),
                           output_dir=output_dir)
    out, err = capfd.readouterr()
    assert "No folders specified to build" in out

def test_python_build_matrix_expansion(monkeypatch):
    monkeypatch.chdir(test_data_dir)
    tasks = execute.collect_tasks('.', matrix_base_dir=os.path.join(test_data_dir, 'linux-config-test'),
                                  folders=['python_test'])
    assert len(tasks.nodes()) == 2
    assert 'frank-1.0-python_2.7-on-centos5-64' in tasks.nodes()
    assert 'frank-1.0-python_3.6-on-centos5-64' in tasks.nodes()

def test_subpackage_matrix_no_subpackages(monkeypatch):
    """Subpackages should not constitute new entries in the build graph.  They should be lumped in
    with their parent recipe.  However, we have to include them initially for the sake of
    dependency ordering.  Thus we initially include them as though they were full packages, but
    then we squish them together and re-assign and dependency edges."""
    monkeypatch.chdir(test_data_dir)
    tasks = execute.collect_tasks('.', matrix_base_dir=os.path.join(test_data_dir, 'linux-config-test'),
                                  folders=['has_subpackages', 'depends_on_subpackage'])
    assert len(tasks.nodes()) == 2
    assert 'has_subpackages_toplevel-1.0-on-centos5-64' in tasks.nodes()
    assert 'depends_on_subpackage-1.0-on-centos5-64' in tasks.nodes()
    assert 'has_subpackages_subpackage-1.0-on-centos5-64' not in tasks.nodes()
    # this is the actual dependency
    assert ('depends_on_subpackage-1.0-on-centos5-64', 'has_subpackages_subpackage-1.0-on-centos5-64') not in tasks.edges()
    # this is what we remap it to
    assert ('depends_on_subpackage-1.0-on-centos5-64', 'has_subpackages_toplevel-1.0-on-centos5-64') in tasks.edges()

def test_dependency_with_selector_cross_compile(testing_conda_resolve):
    g = execute.collect_tasks(test_data_dir, ['selector_run', 'functools32-feedstock'],
                              matrix_base_dir=os.path.join(test_data_dir, 'config-win'),
                              variant_config_files=os.path.join(test_data_dir, 'conda_build_config.yaml'))
    assert len(g.nodes()) == 6
    # native edge
    assert ('test_run_deps_with_selector-1.0-python_2.7-on-win-64',
            'functools32_wannabe-3.2.3.2-python_2.7-on-win-64') in g.edges()
    # cross edge
    assert ('test_run_deps_with_selector-1.0-python_2.7-target_win-32-on-win-64',
            'functools32_wannabe-3.2.3.2-python_2.7-target_win-32-on-win-64') in g.edges()

def test_collapse_with_win_matrix_and_subpackages(monkeypatch):
    monkeypatch.chdir(test_data_dir)
    tasks = execute.collect_tasks('.', matrix_base_dir=os.path.join(test_data_dir, 'config-win'),
                                  folders=['win_split_outputs_compiler_reduction'])
    # 8 subpackages, but 4 total builds - 2 subpackages per build
    assert len(tasks.nodes()) == 4
    assert 'postgresql-split-10.1-c_compiler_vs2008-on-win-64' in tasks.nodes()
    assert 'postgresql-split-10.1-c_compiler_vs2015-on-win-64' in tasks.nodes()
    assert 'postgresql-split-10.1-c_compiler_vs2008-target_win-32-on-win-64' in tasks.nodes()
    assert 'postgresql-split-10.1-c_compiler_vs2015-target_win-32-on-win-64' in tasks.nodes()
<|endoftext|>