<commit_before>import sublime
import sublime_plugin
import re

surround_settings = sublime.load_settings('surround.sublime-settings')

class SurroundCommand(sublime_plugin.TextCommand):
    def surround_pairs_for_addition(self, surround):
        pairs = surround_settings.get('surround_pairs_for_addition')
        return self.surround_pairs(surround, pairs)

    def surround_pairs(self, surround, pairs):
        if surround[0] in pairs:
            return pairs[surround[0]]
        else:
            return surround

    def surround_tags_for_addition(self, surround):
        matches = re.search(r"<([\S]+)[^>]*>", surround[0])
        if matches:
            return [surround[0], "</" + matches.group(1) + ">"]
        else:
            return surround

    def surround_addition(self, surround):
        surround = [surround, surround]
        surround = self.surround_pairs_for_addition(surround)
        surround = self.surround_tags_for_addition(surround)
        return surround

    def run_surround(self, caption, callback):
        window = self.view.window()
        window.show_input_panel(caption, '', callback, None, None)

class SurroundSelectionCommand(SurroundCommand):
    """
    Surround the current selection(s) with something
    """

    def run(self, edit):
        # If this is called from Vintage the selection will be reset as soon as
        # the method returns, but we need it in the callback so we copy it here.
        self.sel = [sel for sel in self.view.sel()]
        self.run_surround('Surround with:', self.surround_selection)

    def surround_selection(self, surround):
        view = self.view
        surround = self.surround_addition(surround)
        edit = view.begin_edit()
        for region in reversed(self.sel):
            view.insert(edit, region.end(), surround[1])
            view.insert(edit, region.begin(), surround[0])
        view.end_edit(edit)

class SurroundChangeCommand(SurroundCommand):
    """
    Change something surrounding the current insertion point(s) to something else
    """

    def run(self, edit):
        self.run_surround('Match:', self.replace_with)

    def replace_with(self, surround):
        self.surround = surround
        window = self.view.window()
        window.show_input_panel(
            'Replace with:', '', self.replace_surround, None, None)

    def replace_surround(self, replacement):
        search = self.surround_search_patterns(self.surround)
        replacement = self.surround_addition(replacement)

        view = self.view
        edit = view.begin_edit()
        for region in reversed(view.sel()):

            end = self.find_end(region.end(), search)
            if end:

                start = self.find_start(region.begin(), search)

                if start:
                    self.view.replace(edit, end, replacement[1])
                    self.view.replace(edit, start, replacement[0])
        view.end_edit(edit)

    def find_start(self, to_pos, search):
        previous = self.find_between(0, to_pos, search).pop()
        # balance pairs
        close_search = [search[1], search[0], search[2]]
        count_closing_pairs = len(self.find_between(previous.end(), to_pos, close_search))
        if count_closing_pairs % 2 is 0:
            return previous
        else:
            return self.find_start(previous.begin(), search)

    def find_end(self, from_pos, search):
        next = self.view.find(search[1], from_pos, search[2])
        # balance pairs
        count_opening_pairs = len(self.find_between(from_pos, next.begin(), search))
        if count_opening_pairs % 2 is 0:
            return next
        else:
            return self.find_end(next.end(), search)

    def find_between(self, from_pos, to_pos, search):
        found = []
        possible_finds = self.view.find_all(search[0], search[2])
        for possible_find in possible_finds:
            if possible_find.end() <= to_pos and possible_find.begin() >= from_pos:
                found.append(possible_find)
        return found

    def surround_search_patterns(self, surround):
        surround = [surround, surround]
        surround = self.surround_pairs_for_search(surround)
        surround = self.surround_tags_for_search(surround)

        if len(surround[0]) <= 1:
            flag = sublime.LITERAL
        else:
            flag = 0
        surround.append(flag)
        return surround

    def surround_pairs_for_search(self, surround):
        pairs = surround_settings.get('surround_pairs_for_search')
        return self.surround_pairs(surround, pairs)

    def surround_tags_for_search(self, surround):
        matches = re.search(r"<([\S]+)([^>]*)>", surround[0])
        if matches:
            attrs = matches.group(2)
            if len(attrs) == 0:
                attrs = '[^>]*'
            open_tag = unicode("<" + matches.group(1) + attrs + ">")
            close_tag = unicode("</" + matches.group(1) + ">")
            return [open_tag, close_tag]
        else:
            return surround

class SurroundDeleteCommand(SurroundChangeCommand):
    """
    Delete something surrounding current insertion point(s)
    """

    def run(self, edit):
        self.run_surround('Delete:', self.delete_surround)

    def delete_surround(self, surround):
        self.surround = surround
        self.replace_surround('')
<commit_msg>whitespace<commit_after>import sublime
import sublime_plugin
import re

surround_settings = sublime.load_settings('surround.sublime-settings')

class SurroundCommand(sublime_plugin.TextCommand):
    def surround_pairs_for_addition(self, surround):
        pairs = surround_settings.get('surround_pairs_for_addition')
        return self.surround_pairs(surround, pairs)

    def surround_pairs(self, surround, pairs):
        if surround[0] in pairs:
            return pairs[surround[0]]
        else:
            return surround

    def surround_tags_for_addition(self, surround):
        matches = re.search(r"<([\S]+)[^>]*>", surround[0])
        if matches:
            return [surround[0], "</" + matches.group(1) + ">"]
        else:
            return surround

    def surround_addition(self, surround):
        surround = [surround, surround]
        surround = self.surround_pairs_for_addition(surround)
        surround = self.surround_tags_for_addition(surround)
        return surround

    def run_surround(self, caption, callback):
        window = self.view.window()
        window.show_input_panel(caption, '', callback, None, None)

class SurroundSelectionCommand(SurroundCommand):
    """
    Surround the current selection(s) with something
    """

    def run(self, edit):
        # If this is called from Vintage the selection will be reset as soon as
        # the method returns, but we need it in the callback so we copy it here.
        self.sel = [sel for sel in self.view.sel()]
        self.run_surround('Surround with:', self.surround_selection)

    def surround_selection(self, surround):
        view = self.view
        surround = self.surround_addition(surround)
        edit = view.begin_edit()
        for region in reversed(self.sel):
            view.insert(edit, region.end(), surround[1])
            view.insert(edit, region.begin(), surround[0])
        view.end_edit(edit)

class SurroundChangeCommand(SurroundCommand):
    """
    Change something surrounding the current insertion point(s) to something else
    """

    def run(self, edit):
        self.run_surround('Match:', self.replace_with)

    def replace_with(self, surround):
        self.surround = surround
        window = self.view.window()
        window.show_input_panel(
            'Replace with:', '', self.replace_surround, None, None)

    def replace_surround(self, replacement):
        search = self.surround_search_patterns(self.surround)
        replacement = self.surround_addition(replacement)
        view = self.view
        edit = view.begin_edit()
        for region in reversed(view.sel()):
            end = self.find_end(region.end(), search)
            if end:
                start = self.find_start(region.begin(), search)
                if start:
                    self.view.replace(edit, end, replacement[1])
                    self.view.replace(edit, start, replacement[0])
        view.end_edit(edit)

    def find_start(self, to_pos, search):
        previous = self.find_between(0, to_pos, search).pop()
        # balance pairs
        close_search = [search[1], search[0], search[2]]
        count_closing_pairs = len(self.find_between(previous.end(), to_pos, close_search))
        if count_closing_pairs % 2 is 0:
            return previous
        else:
            return self.find_start(previous.begin(), search)

    def find_end(self, from_pos, search):
        next = self.view.find(search[1], from_pos, search[2])
        # balance pairs
        count_opening_pairs = len(self.find_between(from_pos, next.begin(), search))
        if count_opening_pairs % 2 is 0:
            return next
        else:
            return self.find_end(next.end(), search)

    def find_between(self, from_pos, to_pos, search):
        found = []
        possible_finds = self.view.find_all(search[0], search[2])
        for possible_find in possible_finds:
            if possible_find.end() <= to_pos and possible_find.begin() >= from_pos:
                found.append(possible_find)
        return found

    def surround_search_patterns(self, surround):
        surround = [surround, surround]
        surround = self.surround_pairs_for_search(surround)
        surround = self.surround_tags_for_search(surround)
        if len(surround[0]) <= 1:
            flag = sublime.LITERAL
        else:
            flag = 0
        surround.append(flag)
        return surround

    def surround_pairs_for_search(self, surround):
        pairs = surround_settings.get('surround_pairs_for_search')
        return self.surround_pairs(surround, pairs)

    def surround_tags_for_search(self, surround):
        matches = re.search(r"<([\S]+)([^>]*)>", surround[0])
        if matches:
            attrs = matches.group(2)
            if len(attrs) == 0:
                attrs = '[^>]*'
            open_tag = unicode("<" + matches.group(1) + attrs + ">")
            close_tag = unicode("</" + matches.group(1) + ">")
            return [open_tag, close_tag]
        else:
            return surround

class SurroundDeleteCommand(SurroundChangeCommand):
    """
    Delete something surrounding current insertion point(s)
    """

    def run(self, edit):
        self.run_surround('Delete:', self.delete_surround)

    def delete_surround(self, surround):
        self.surround = surround
        self.replace_surround('')
<|endoftext|>


<commit_before># Here's a probably-not-very-good port of the Adafruit Arduino library for
# this sensor. The Adafruit library is located here:
# https://github.com/adafruit/Adafruit_TCS34725
# This port is for the Beaglebone Black and the Adafruit_BBIO and Adafruit_I2C
# libraries. By AKA MEDIA SYSTEM

from Adafruit_I2C import Adafruit_I2C
import time

class TCS34725():

    TCS34725_ADDRESS          = 0x29

    TCS34725_COMMAND_BIT      = 0x80

    TCS34725_ENABLE           = 0x00
    TCS34725_ENABLE_AIEN      = 0x10    # RGBC Interrupt Enable
    TCS34725_ENABLE_WEN       = 0x08    # Wait enable - Writing 1 activates the wait timer
    TCS34725_ENABLE_AEN       = 0x02    # RGBC Enable - Writing 1 actives the ADC, 0 disables it
    TCS34725_ENABLE_PON       = 0x01    # Power on - Writing 1 activates the internal oscillator, 0 disables it
    TCS34725_ATIME            = 0x01    # Integration time
    TCS34725_WTIME            = 0x03    # Wait time (if TCS34725_ENABLE_WEN is asserted
    TCS34725_WTIME_2_4MS      = 0xFF    # WLONG0 = 2.4ms   WLONG1 = 0.029s
    TCS34725_WTIME_204MS      = 0xAB    # WLONG0 = 204ms   WLONG1 = 2.45s
    TCS34725_WTIME_614MS      = 0x00    # WLONG0 = 614ms   WLONG1 = 7.4s
    TCS34725_AILTL            = 0x04    # Clear channel lower interrupt threshold
    TCS34725_AILTH            = 0x05
    TCS34725_AIHTL            = 0x06    # Clear channel upper interrupt threshold
    TCS34725_AIHTH            = 0x07
    TCS34725_PERS             = 0x0C    # Persistence register - basic SW filtering mechanism for interrupts
    TCS34725_PERS_NONE        = 0b0000  # Every RGBC cycle generates an interrupt
    TCS34725_PERS_1_CYCLE     = 0b0001  # 1 clean channel value outside threshold range generates an interrupt
    TCS34725_PERS_2_CYCLE     = 0b0010  # 2 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_3_CYCLE     = 0b0011  # 3 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_5_CYCLE     = 0b0100  # 5 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_10_CYCLE    = 0b0101  # 10 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_15_CYCLE    = 0b0110  # 15 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_20_CYCLE    = 0b0111  # 20 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_25_CYCLE    = 0b1000  # 25 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_30_CYCLE    = 0b1001  # 30 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_35_CYCLE    = 0b1010  # 35 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_40_CYCLE    = 0b1011  # 40 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_45_CYCLE    = 0b1100  # 45 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_50_CYCLE    = 0b1101  # 50 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_55_CYCLE    = 0b1110  # 55 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_60_CYCLE    = 0b1111  # 60 clean channel values outside threshold range generates an interrupt
    TCS34725_CONFIG           = 0x0D
    TCS34725_CONFIG_WLONG     = 0x02    # Choose between short and long (12x wait times via TCS34725_WTIME
    TCS34725_CONTROL          = 0x0F    # Set the gain level for the sensor
    TCS34725_ID               = 0x12    # 0x44 = TCS34721/TCS34725, 0x4D = TCS34723/TCS34727
    TCS34725_STATUS           = 0x13
    TCS34725_STATUS_AINT      = 0x10    # RGBC Clean channel interrupt
    TCS34725_STATUS_AVALID    = 0x01    # Indicates that the RGBC channels have completed an integration cycle
    TCS34725_CDATAL           = 0x14    # Clear channel data
    TCS34725_CDATAH           = 0x15
    TCS34725_RDATAL           = 0x16    # Red channel data
    TCS34725_RDATAH           = 0x17
    TCS34725_GDATAL           = 0x18    # Green channel data
    TCS34725_GDATAH           = 0x19
    TCS34725_BDATAL           = 0x1A    # Blue channel data
    TCS34725_BDATAH           = 0x1B

  # TCS34725_INTEGRATIONTIME_2_4MS  = 0xFF,   /**<  2.4ms - 1 cycle    - Max Count: 1024
  # TCS34725_INTEGRATIONTIME_24MS   = 0xF6,   /**<  24ms  - 10 cycles  - Max Count: 10240
  # TCS34725_INTEGRATIONTIME_50MS   = 0xEB,   /**<  50ms  - 20 cycles  - Max Count: 20480
  # TCS34725_INTEGRATIONTIME_101MS  = 0xD5,   /**<  101ms - 42 cycles  - Max Count: 43008
  # TCS34725_INTEGRATIONTIME_154MS  = 0xC0,   /**<  154ms - 64 cycles  - Max Count: 65535
  # TCS34725_INTEGRATIONTIME_700MS  = 0x00    /**<  700ms - 256 cycles - Max Count: 65535

    _tcs34725Initialised = False
    _tcs34725Gain = 0
    _tcs34725IntegrationTime = 0x00

    def __init__(self, *args, **kwargs):
        self.i2c = Adafruit_I2C(self.TCS34725_ADDRESS)

    def begin(self):
        x = self.i2c.readU8(self.TCS34725_COMMAND_BIT | self.TCS34725_ID)
        if x != 0x44: # code I was basing this on expects 0x44, not sure why. Got 0x12
            print 'did not get the expected response from sensor: ', x
            return False
        self._tcs34725Initialised = True
        self.setIntegrationTime(self._tcs34725IntegrationTime)
        self.setGain(0)
        self.enable()
        return True

    def enable(self):
        self.i2c.write8(self.TCS34725_ENABLE, self.TCS34725_ENABLE_PON)
        time.sleep(0.003)
        self.i2c.write8(self.TCS34725_ENABLE, self.TCS34725_ENABLE_PON | self.TCS34725_ENABLE_AEN)

    def disable(self):
        reg = 0
        reg = self.i2c.readU8(self.TCS34725_ENABLE)
        self.i2c.write8(self.TCS34725_ENABLE, reg & ~(self.TCS34725_ENABLE_PON | self.TCS34725_ENABLE_AEN))

    def setIntegrationTime(self, theTime):
        if theTime not in [0xFF,0xF6,0xEB,0xD5,0xC0,0x00]:
            print 'setting integration time to 0x00, %s is illegal' % theTime
            theTime = 0x00
        self.i2c.write8(self.TCS34725_ATIME, theTime)
        self._tcs34725IntegrationTime = theTime

    def setGain(self, gain):
        # TCS34725_GAIN_1X                = 0x00,   /**<  No gain
        # TCS34725_GAIN_4X                = 0x01,   /**<  2x gain
        # TCS34725_GAIN_16X               = 0x02,   /**<  16x gain
        # TCS34725_GAIN_60X               = 0x03    /**<  60x gain
        if gain not in [0,1,2,3]:
            print 'setting gain to 0, %s is illegal' % gain
            gain = 0
        self.i2c.write8(self.TCS34725_CONTROL, gain)
        self._tcs34725Gain = gain

    def getStatus(self):
        return self.i2c.readU8(self.TCS34725_STATUS)

    def getRawData(self):
        c = self.i2c.readU16(self.TCS34725_CDATAL)
        r = self.i2c.readU16(self.TCS34725_RDATAL)
        g = self.i2c.readU16(self.TCS34725_GDATAL)
        b = self.i2c.readU16(self.TCS34725_BDATAL)
        if self._tcs34725IntegrationTime == 0xFF:
            time.sleep(0.0024)
        elif self._tcs34725IntegrationTime == 0xF6:
            time.sleep(0.024)
        elif self._tcs34725IntegrationTime == 0xEB:
            time.sleep(0.050)
        elif self._tcs34725IntegrationTime == 0xD5:
            time.sleep(0.101)
        elif self._tcs34725IntegrationTime == 0xC0:
            time.sleep(0.154)
        elif self._tcs34725IntegrationTime == 0x00:
            time.sleep(0.700)
        else:
            time.sleep(0.700)
        return c, r, g, b

    def calculateColorTemperature(self, r, g, b):
        # this is all from the Adafruit C library
        # 1. Map RGB values to their XYZ counterparts.
        # Based on 6500K fluorescent, 3000K fluorescent
        # and 60W incandescent values for a wide range.
        # Note: Y = Illuminance or lux
        X = (-0.14282 * r) + (1.54924 * g) + (-0.95641 * b);
        Y = (-0.32466 * r) + (1.57837 * g) + (-0.73191 * b);
        Z = (-0.68202 * r) + (0.77073 * g) + ( 0.56332 * b);

        # 2. Calculate the chromaticity co-ordinates
        xc = (X) / (X + Y + Z);
        yc = (Y) / (X + Y + Z);

        # 3. Use McCamy's formula to determine the CCT
        n = (xc - 0.3320) / (0.1858 - yc);

        # Calculate the final CCT
        cct = (449.0 * pow(n, 3)) + (3525.0 * pow(n, 2)) + (6823.3 * n) + 5520.33;
        return cct

    def calculateLux(self, r, g, b):
        return ((-0.32466 * r) + (1.57837 * g) + (-0.73191 * b))

    def setInterrupt(self, bool):
        r = self.i2c.readU8(self.TCS34725_ENABLE)
        if bool:
            r |= self.TCS34725_ENABLE_AIEN
        else:
            r &= ~self.TCS34725_ENABLE_AIEN
        self.i2c.write8(self.TCS34725_ENABLE, r)

    def clearInterrupt(self):
        self.i2c.write8(self.TCS34725_ADDRESS, 0x66)

    def setInterruptLimits(self, lo, hi):
        pass<commit_msg>Cleaning up typos and other dumb stuff<commit_after># Here's a probably-not-very-good port of the Adafruit Arduino library for
# this sensor. The Adafruit library is located here:
# https://github.com/adafruit/Adafruit_TCS34725
# This port is for the Beaglebone Black and the Adafruit_BBIO and Adafruit_I2C
# libraries. By AKA MEDIA SYSTEM

from Adafruit_I2C import Adafruit_I2C
import time

class TCS34725():

    TCS34725_ADDRESS          = 0x29

    TCS34725_COMMAND_BIT      = 0x80

    TCS34725_ENABLE           = 0x00
    TCS34725_ENABLE_AIEN      = 0x10    # RGBC Interrupt Enable
    TCS34725_ENABLE_WEN       = 0x08    # Wait enable - Writing 1 activates the wait timer
    TCS34725_ENABLE_AEN       = 0x02    # RGBC Enable - Writing 1 actives the ADC, 0 disables it
    TCS34725_ENABLE_PON       = 0x01    # Power on - Writing 1 activates the internal oscillator, 0 disables it
    TCS34725_ATIME            = 0x01    # Integration time
    TCS34725_WTIME            = 0x03    # Wait time (if TCS34725_ENABLE_WEN is asserted
    TCS34725_WTIME_2_4MS      = 0xFF    # WLONG0 = 2.4ms   WLONG1 = 0.029s
    TCS34725_WTIME_204MS      = 0xAB    # WLONG0 = 204ms   WLONG1 = 2.45s
    TCS34725_WTIME_614MS      = 0x00    # WLONG0 = 614ms   WLONG1 = 7.4s
    TCS34725_AILTL            = 0x04    # Clear channel lower interrupt threshold
    TCS34725_AILTH            = 0x05
    TCS34725_AIHTL            = 0x06    # Clear channel upper interrupt threshold
    TCS34725_AIHTH            = 0x07
    TCS34725_PERS             = 0x0C    # Persistence register - basic SW filtering mechanism for interrupts
    TCS34725_PERS_NONE        = 0b0000  # Every RGBC cycle generates an interrupt
    TCS34725_PERS_1_CYCLE     = 0b0001  # 1 clean channel value outside threshold range generates an interrupt
    TCS34725_PERS_2_CYCLE     = 0b0010  # 2 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_3_CYCLE     = 0b0011  # 3 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_5_CYCLE     = 0b0100  # 5 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_10_CYCLE    = 0b0101  # 10 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_15_CYCLE    = 0b0110  # 15 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_20_CYCLE    = 0b0111  # 20 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_25_CYCLE    = 0b1000  # 25 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_30_CYCLE    = 0b1001  # 30 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_35_CYCLE    = 0b1010  # 35 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_40_CYCLE    = 0b1011  # 40 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_45_CYCLE    = 0b1100  # 45 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_50_CYCLE    = 0b1101  # 50 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_55_CYCLE    = 0b1110  # 55 clean channel values outside threshold range generates an interrupt
    TCS34725_PERS_60_CYCLE    = 0b1111  # 60 clean channel values outside threshold range generates an interrupt
    TCS34725_CONFIG           = 0x0D
    TCS34725_CONFIG_WLONG     = 0x02    # Choose between short and long (12x wait times via TCS34725_WTIME
    TCS34725_CONTROL          = 0x0F    # Set the gain level for the sensor
    TCS34725_ID               = 0x12    # 0x44 = TCS34721/TCS34725, 0x4D = TCS34723/TCS34727
    TCS34725_STATUS           = 0x13
    TCS34725_STATUS_AINT      = 0x10    # RGBC Clean channel interrupt
    TCS34725_STATUS_AVALID    = 0x01    # Indicates that the RGBC channels have completed an integration cycle
    TCS34725_CDATAL           = 0x14    # Clear channel data
    TCS34725_CDATAH           = 0x15
    TCS34725_RDATAL           = 0x16    # Red channel data
    TCS34725_RDATAH           = 0x17
    TCS34725_GDATAL           = 0x18    # Green channel data
    TCS34725_GDATAH           = 0x19
    TCS34725_BDATAL           = 0x1A    # Blue channel data
    TCS34725_BDATAH           = 0x1B

  # TCS34725_INTEGRATIONTIME_2_4MS  = 0xFF,   /**<  2.4ms - 1 cycle    - Max Count: 1024
  # TCS34725_INTEGRATIONTIME_24MS   = 0xF6,   /**<  24ms  - 10 cycles  - Max Count: 10240
  # TCS34725_INTEGRATIONTIME_50MS   = 0xEB,   /**<  50ms  - 20 cycles  - Max Count: 20480
  # TCS34725_INTEGRATIONTIME_101MS  = 0xD5,   /**<  101ms - 42 cycles  - Max Count: 43008
  # TCS34725_INTEGRATIONTIME_154MS  = 0xC0,   /**<  154ms - 64 cycles  - Max Count: 65535
  # TCS34725_INTEGRATIONTIME_700MS  = 0x00    /**<  700ms - 256 cycles - Max Count: 65535

    _tcs34725Initialised = False
    _tcs34725Gain = 0
    _tcs34725IntegrationTime = 0x00

    def __init__(self, *args, **kwargs):
        self.i2c = Adafruit_I2C(self.TCS34725_ADDRESS)

    def begin(self):
        x = self.i2c.readU8(self.TCS34725_COMMAND_BIT | self.TCS34725_ID)
        if x != 0x44: # code I was basing this on expects 0x44, not sure why. Got 0x12
            print 'did not get the expected response from sensor: ', x
            return False
        self._tcs34725Initialised = True
        self.setIntegrationTime(self._tcs34725IntegrationTime)
        self.setGain(0)
        self.enable()
        return True

    def enable(self):
        self.i2c.write8(self.TCS34725_ENABLE, self.TCS34725_ENABLE_PON)
        time.sleep(0.003)
        self.i2c.write8(self.TCS34725_ENABLE, self.TCS34725_ENABLE_PON | self.TCS34725_ENABLE_AEN)

    def disable(self):
        reg = 0
        reg = self.i2c.readU8(self.TCS34725_COMMAND_BIT | self.TCS34725_ENABLE)
        self.i2c.write8(self.TCS34725_ENABLE, reg & ~(self.TCS34725_ENABLE_PON | self.TCS34725_ENABLE_AEN))

    def setIntegrationTime(self, theTime):
        if theTime not in [0xFF,0xF6,0xEB,0xD5,0xC0,0x00]:
            print 'setting integration time to 0x00, %s is illegal' % theTime
            theTime = 0x00
        self.i2c.write8(self.TCS34725_ATIME, theTime)
        self._tcs34725IntegrationTime = theTime

    def setGain(self, gain):
        # TCS34725_GAIN_1X                = 0x00,   /**<  No gain
        # TCS34725_GAIN_4X                = 0x01,   /**<  2x gain
        # TCS34725_GAIN_16X               = 0x02,   /**<  16x gain
        # TCS34725_GAIN_60X               = 0x03    /**<  60x gain
        if gain not in [0,1,2,3]:
            print 'setting gain to 0, %s is illegal' % gain
            gain = 0
        self.i2c.write8(self.TCS34725_CONTROL, gain)
        self._tcs34725Gain = gain

    def getStatus(self):
        return self.i2c.readU8(self.TCS34725_COMMAND_BIT | self.TCS34725_STATUS)

    def getRawData(self):
        c = self.i2c.readU16(self.TCS34725_COMMAND_BIT | self.TCS34725_CDATAL)
        r = self.i2c.readU16(self.TCS34725_COMMAND_BIT | self.TCS34725_RDATAL)
        g = self.i2c.readU16(self.TCS34725_COMMAND_BIT | self.TCS34725_GDATAL)
        b = self.i2c.readU16(self.TCS34725_COMMAND_BIT | self.TCS34725_BDATAL)
        if self._tcs34725IntegrationTime == 0xFF:
            time.sleep(0.0024)
        elif self._tcs34725IntegrationTime == 0xF6:
            time.sleep(0.024)
        elif self._tcs34725IntegrationTime == 0xEB:
            time.sleep(0.050)
        elif self._tcs34725IntegrationTime == 0xD5:
            time.sleep(0.101)
        elif self._tcs34725IntegrationTime == 0xC0:
            time.sleep(0.154)
        elif self._tcs34725IntegrationTime == 0x00:
            time.sleep(0.700)
        else:
            time.sleep(0.700)
        return c, r, g, b

    def calculateColorTemperature(self, r, g, b):
        # this is all from the Adafruit C library
        # 1. Map RGB values to their XYZ counterparts.
        # Based on 6500K fluorescent, 3000K fluorescent
        # and 60W incandescent values for a wide range.
        # Note: Y = Illuminance or lux
        X = (-0.14282 * r) + (1.54924 * g) + (-0.95641 * b);
        Y = (-0.32466 * r) + (1.57837 * g) + (-0.73191 * b);
        Z = (-0.68202 * r) + (0.77073 * g) + ( 0.56332 * b);

        # 2. Calculate the chromaticity co-ordinates
        xc = (X) / (X + Y + Z);
        yc = (Y) / (X + Y + Z);

        # 3. Use McCamy's formula to determine the CCT
        n = (xc - 0.3320) / (0.1858 - yc);

        # Calculate the final CCT
        cct = (449.0 * pow(n, 3)) + (3525.0 * pow(n, 2)) + (6823.3 * n) + 5520.33;
        return cct

    def calculateLux(self, r, g, b):
        return ((-0.32466 * r) + (1.57837 * g) + (-0.73191 * b))

    def setInterrupt(self, bool):
        r = self.i2c.readU8(self.TCS34725_COMMAND_BIT | self.TCS34725_ENABLE)
        if bool:
            r |= self.TCS34725_ENABLE_AIEN
        else:
            r &= ~self.TCS34725_ENABLE_AIEN
        self.i2c.write8(self.TCS34725_ENABLE, r)

    def clearInterrupt(self):
        self.i2c.write8(self.TCS34725_ADDRESS, 0x66)

    def setInterruptLimits(self, lo, hi):
        pass<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from hwt.bitmask import mask
from hwt.code import If, connect, Concat, Switch, log2ceil
from hwt.hdlObjects.typeShortcuts import vecT
from hwt.interfaces.std import Handshaked, Signal
from hwt.interfaces.utils import addClkRstn, propagateClkRstn
from hwt.synthesizer.interfaceLevel.unit import Unit
from hwt.synthesizer.param import Param
from hwtLib.amba.axis import AxiStream
from hwtLib.amba.axis_comp.builder import AxiSBuilder
from hwtLib.amba.axis_comp.fifo import AxiSFifo
from hwtLib.handshaked.fifo import HandshakedFifo
from hwtLib.handshaked.streamNode import streamSync, streamAck
from hwtLib.mem.fifo import Fifo

def strbToRem(strbBits):
    """
    :return: tuples (mask, numberofBits)
    """
    for rem in range(strbBits):
        if (rem == 0):
            strb = mask(strbBits)
        else:
            strb = mask(rem)
        yield strb, rem

class AxiS_measuringFifo(Unit):
    """
    Fifo which are counting sizes of frames and sends it over
    dedicated handshaked interface "sizes"
    """
    def _config(self):
        Fifo._config(self)
        self.SIZES_BUFF_DEPTH = Param(16)
        self.MAX_LEN = Param((2048 // 8) - 1)

    def getAlignBitsCnt(self):
        return log2ceil(self.DATA_WIDTH // 8).val

    def _declr(self):
        addClkRstn(self)
        with self._paramsShared():
            self.dataIn = AxiStream()
            self.dataOut = AxiStream()

        self.sizes = Handshaked()
        self.sizes.DATA_WIDTH.set(log2ceil(self.MAX_LEN) + 1 + self.getAlignBitsCnt())

        db = self.dataBuff = AxiSFifo(AxiStream)
        # to place fifo in bram
        db.DATA_WIDTH.set(self.DATA_WIDTH)
        db.DEPTH.set((self.MAX_LEN + 1) * 2)

        sb = self.sizesBuff = HandshakedFifo(Handshaked)
        sb.DEPTH.set(self.SIZES_BUFF_DEPTH)
        sb.DATA_WIDTH.set(self.sizes.DATA_WIDTH.get())
        self.errorAlignment = Signal()

    def _impl(self):
        propagateClkRstn(self)
        dIn = AxiSBuilder(self, self.dataIn).buff().end
        STRB_BITS = dIn.strb._dtype.bit_length()

        sb = self.sizesBuff
        db = self.dataBuff

        wordCntr = self._reg("wordCntr",
                             vecT(log2ceil(self.MAX_LEN) + 1),
                             defVal=0)
        errorAlignment = self._reg("errorAlignment_reg", defVal=0)
        self.errorAlignment ** errorAlignment

        overflow = wordCntr._eq(self.MAX_LEN)
        last = dIn.last | overflow
        If(streamAck(masters=[dIn], slaves=[sb.dataIn, db.dataIn]),
            If(last,
                wordCntr ** 0
            ).Else(
                wordCntr ** (wordCntr + 1)
            )
        )
        rem = self._sig("rem", vecT(log2ceil(STRB_BITS)))
        Switch(dIn.strb).addCases(
           [(strb, rem ** r)
            for strb, r in strbToRem(STRB_BITS)]
        ).Default(
            rem ** 0,
            If(dIn.valid,
               errorAlignment ** 1
            )
        )

        length = self._sig("length", wordCntr._dtype)
        If(dIn.strb != mask(STRB_BITS),
            length ** wordCntr
        ).Else(
            length ** (wordCntr + 1)
        )

        sb.dataIn.data ** Concat(length, rem)

        connect(dIn, db.dataIn, exclude=[dIn.valid, dIn.ready])

        streamSync(masters=[dIn],
                   slaves=[sb.dataIn, db.dataIn],
                   extraConds={
                               sb.dataIn: last
                              })

        self.sizes ** sb.dataOut
        connect(db.dataOut, self.dataOut, exclude=[db.dataOut.last])
        self.dataOut.last ** db.dataOut.last

if __name__ == "__main__":
    from hwt.synthesizer.shortcuts import toRtl
    u = AxiS_measuringFifo()
    print(toRtl(u))
<commit_msg>AxiSMeasuringFifo, optimize strb-rem, fix overflow handling<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from hwt.bitmask import mask
from hwt.code import If, connect, Concat, Switch, log2ceil, SwitchLogic
from hwt.hdlObjects.typeShortcuts import vecT
from hwt.interfaces.std import Handshaked, Signal
from hwt.interfaces.utils import addClkRstn, propagateClkRstn
from hwt.synthesizer.interfaceLevel.unit import Unit
from hwt.synthesizer.param import Param
from hwtLib.amba.axis import AxiStream
from hwtLib.amba.axis_comp.builder import AxiSBuilder
from hwtLib.amba.axis_comp.fifo import AxiSFifo
from hwtLib.handshaked.fifo import HandshakedFifo
from hwtLib.handshaked.streamNode import streamSync, streamAck
from hwtLib.mem.fifo import Fifo

class AxiS_measuringFifo(Unit):
    """
    Fifo which are counting sizes of frames and sends it over
    dedicated handshaked interface "sizes"
    """
    def _config(self):
        Fifo._config(self)
        self.SIZES_BUFF_DEPTH = Param(16)
        self.MAX_LEN = Param((2048 // 8) - 1)

    def getAlignBitsCnt(self):
        return log2ceil(self.DATA_WIDTH // 8).val

    def _declr(self):
        addClkRstn(self)
        with self._paramsShared():
            self.dataIn = AxiStream()
            self.dataOut = AxiStream()

        self.sizes = Handshaked()
        self.sizes.DATA_WIDTH.set(log2ceil(self.MAX_LEN) + 1 + self.getAlignBitsCnt())

        db = self.dataBuff = AxiSFifo(AxiStream)
        # to place fifo in bram
        db.DATA_WIDTH.set(self.DATA_WIDTH)
        db.DEPTH.set((self.MAX_LEN + 1) * 2)

        sb = self.sizesBuff = HandshakedFifo(Handshaked)
        sb.DEPTH.set(self.SIZES_BUFF_DEPTH)
        sb.DATA_WIDTH.set(self.sizes.DATA_WIDTH.get())
        self.errorAlignment = Signal()

    def _impl(self):
        propagateClkRstn(self)
        dIn = AxiSBuilder(self, self.dataIn).buff().end
        STRB_BITS = dIn.strb._dtype.bit_length()

        sb = self.sizesBuff
        db = self.dataBuff

        wordCntr = self._reg("wordCntr",
                             vecT(log2ceil(self.MAX_LEN) + 1),
                             defVal=0)
        errorAlignment = self._reg("errorAlignment_reg", defVal=0)
        self.errorAlignment ** errorAlignment

        overflow = wordCntr._eq(self.MAX_LEN)
        last = dIn.last | overflow
        If(streamAck(masters=[dIn], slaves=[sb.dataIn, db.dataIn]),
            If(last,
                wordCntr ** 0
            ).Else(
                wordCntr ** (wordCntr + 1)
            )
        )
        rem = self._sig("rem", vecT(log2ceil(STRB_BITS)))
        SwitchLogic(
            cases=[
                (dIn.strb[i], rem ** (0 if i == STRB_BITS - 1 else i + 1))
                for i in reversed(range(STRB_BITS))],
            default=[
                rem ** 0,
                If(dIn.valid,
                   errorAlignment ** 1
                )
            ]
        )

        length = self._sig("length", wordCntr._dtype)
        If(dIn.strb != mask(STRB_BITS),
            length ** wordCntr
        ).Else(
            length ** (wordCntr + 1)
        )

        sb.dataIn.data ** Concat(length, rem)

        connect(dIn, db.dataIn, exclude=[dIn.valid, dIn.ready, dIn.last])
        db.dataIn.last ** last

        streamSync(masters=[dIn],
                   slaves=[sb.dataIn, db.dataIn],
                   extraConds={
                               sb.dataIn: last
                              })

        self.sizes ** sb.dataOut
        connect(db.dataOut, self.dataOut)

if __name__ == "__main__":
    from hwt.synthesizer.shortcuts import toRtl
    u = AxiS_measuringFifo()
    print(toRtl(u))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# ###
# Copyright (c) 2013, Rice University
# This software is subject to the provisions of the GNU Affero General
# Public License version 3 (AGPLv3).
# See LICENCE.txt for details.
# ###
"""Connexions content parsers"""
import os
import lxml.etree

__all__ = ('parse_collection_xml', 'parse_module_xml',)

def _generate_xpath_func(xml_doc, default_namespace_name='base'):
    """Generates an easy to work with xpath function."""
    nsmap = xml_doc.nsmap.copy()
    try:
        nsmap[default_namespace_name] = nsmap.pop(None)
    except KeyError:
        # There isn't a default namespace.
        pass
    if "http://cnx.rice.edu/mdml/0.4" in nsmap.values():
        # Fixes an issue where the namespace is defined twice, once in the
        #   document tag and again in the metadata tag.
        nsmap['md4'] = "http://cnx.rice.edu/mdml/0.4"
        nsmap['md'] = "http://cnx.rice.edu/mdml"
    return lambda xpth: xml_doc.xpath(xpth, namespaces=nsmap)

def _parse_common_elements(xml_doc):
    """Parse the common elements between a ColXML and CnXML files."""
    xpath = _generate_xpath_func(xml_doc)

    # Pull the abstract
    try:
        abstract = xpath('//md:abstract/text()')[0]
    except IndexError:
        abstract = None

    # Pull the license
    try:
        license = xpath('//md:license/@url')[0]
    except IndexError:
        raise ValueError("Missing license metadata.")

    # Pull the collection metadata
    metadata = {
        'moduleid': xpath('//md:content-id/text()')[0],
        'version': xpath('//md:version/text()')[0],
        'name': xpath('//md:title/text()')[0],
        # FIXME Don't feel like parsing the dates at the moment.
        # 'created': ?,
        # 'revised': ?,
        'doctype': '',  # Can't be null, but appears unused.
        'submitter': '',
        'submitlog': '',
        'language': xpath('//md:language/text()')[0],
        'authors': xpath('//md:roles/md:role[type="author"]/text()')[:],
        'maintainers': xpath('//md:roles/md:role[type="maintainer"]/text()')[:],
        'licensors': xpath('//md:roles/md:role[type="licensor"]/text()')[:],
        # 'parentauthors': None,
        # 'portal_type': 'Collection' or 'Module',

        # Related on insert...
        # 'parent': 1,
        # 'stateid': 1,
        # 'licenseid': 1,
        # 'abstractid': 1,
        }

    return [abstract, license, metadata]

def parse_collection_xml(fp):
    """Parse into the file into segments that will fit into the database.
    Returns the abstract content, license url, metadata dictionary,
    and a list of content ids that are part of this collection.
    """
    # Parse the document
    tree = lxml.etree.parse(fp)
    doc = tree.getroot()
    xpath = _generate_xpath_func(doc, 'colxml')

    data = _parse_common_elements(doc)
    data[2]['portal_type'] = 'Collection'
    # Pull the linked content (modules)
    contents = xpath('//colxml:module/@document')[:]
    data.append(contents)
    return data

def parse_module_xml(fp):
    """Parse the file into segments that will fit into the database.
    This works against the index_auto_generated.cnxml
    Returns the abstract content, license url, metadata dictionary,
    and a list of resource urls that are in the content.
    """
    # Parse the document
    tree = lxml.etree.parse(fp)
    doc = tree.getroot()
    xpath = _generate_xpath_func(doc, 'cnxml')

    data = _parse_common_elements(doc)
    data[2]['portal_type'] = 'Module'
    # Pull the linked content (modules)
    resources = [(e.get('src'), e.get('mime-type'),)
                 for e in xpath('//cnxml:image')]
    filenames = [filename for filename, mimetype in resources]
    resources = [r for i, r in enumerate(resources) if r[0] not in filenames[:i]]
    data.append(resources)
    return data
<commit_msg>Fix it so that the xml metadata namespace is included in the global namespaces declaration. This is necessary when the root element doesn't define all namespaces used by the document.<commit_after># -*- coding: utf-8 -*-
# ###
# Copyright (c) 2013, Rice University
# This software is subject to the provisions of the GNU Affero General
# Public License version 3 (AGPLv3).
# See LICENCE.txt for details.
# ###
"""Connexions content parsers"""
import os
import lxml.etree

__all__ = ('parse_collection_xml', 'parse_module_xml',)

def _generate_xpath_func(xml_doc, default_namespace_name='base'):
    """Generates an easy to work with xpath function."""
    nsmap = xml_doc.nsmap.copy()
    try:
        nsmap[default_namespace_name] = nsmap.pop(None)
    except KeyError:
        # There isn't a default namespace.
        pass
    if "http://cnx.rice.edu/mdml/" not in nsmap.values() \
       or "http://cnx.rice.edu/mdml/0.4" in nsmap.values():
        # Fixes an issue where the namespace is defined twice, once in the
        #   document tag and again in the metadata tag.
        nsmap['md4'] = "http://cnx.rice.edu/mdml/0.4"
        nsmap['md'] = "http://cnx.rice.edu/mdml"
    return lambda xpth: xml_doc.xpath(xpth, namespaces=nsmap)

def _parse_common_elements(xml_doc):
    """Parse the common elements between a ColXML and CnXML files."""
    xpath = _generate_xpath_func(xml_doc)

    # Pull the abstract
    try:
        abstract = xpath('//md:abstract/text()')[0]
    except IndexError:
        abstract = None

    # Pull the license
    try:
        license = xpath('//md:license/@url')[0]
    except IndexError:
        raise ValueError("Missing license metadata.")

    # Pull the collection metadata
    metadata = {
        'moduleid': xpath('//md:content-id/text()')[0],
        'version': xpath('//md:version/text()')[0],
        'name': xpath('//md:title/text()')[0],
        # FIXME Don't feel like parsing the dates at the moment.
        # 'created': ?,
        # 'revised': ?,
        'doctype': '',  # Can't be null, but appears unused.
        'submitter': '',
        'submitlog': '',
        'language': xpath('//md:language/text()')[0],
        'authors': xpath('//md:roles/md:role[type="author"]/text()')[:],
        'maintainers': xpath('//md:roles/md:role[type="maintainer"]/text()')[:],
        'licensors': xpath('//md:roles/md:role[type="licensor"]/text()')[:],
        # 'parentauthors': None,
        # 'portal_type': 'Collection' or 'Module',

        # Related on insert...
        # 'parent': 1,
        # 'stateid': 1,
        # 'licenseid': 1,
        # 'abstractid': 1,
        }

    return [abstract, license, metadata]

def parse_collection_xml(fp):
    """Parse into the file into segments that will fit into the database.
    Returns the abstract content, license url, metadata dictionary,
    and a list of content ids that are part of this collection.
    """
    # Parse the document
    tree = lxml.etree.parse(fp)
    doc = tree.getroot()
    xpath = _generate_xpath_func(doc, 'colxml')

    data = _parse_common_elements(doc)
    data[2]['portal_type'] = 'Collection'
    # Pull the linked content (modules)
    contents = xpath('//colxml:module/@document')[:]
    data.append(contents)
    return data

def parse_module_xml(fp):
    """Parse the file into segments that will fit into the database.
    This works against the index_auto_generated.cnxml
    Returns the abstract content, license url, metadata dictionary,
    and a list of resource urls that are in the content.
    """
    # Parse the document
    tree = lxml.etree.parse(fp)
    doc = tree.getroot()
    xpath = _generate_xpath_func(doc, 'cnxml')

    data = _parse_common_elements(doc)
    data[2]['portal_type'] = 'Module'
    # Pull the linked content (modules)
    resources = [(e.get('src'), e.get('mime-type'),)
                 for e in xpath('//cnxml:image')]
    filenames = [filename for filename, mimetype in resources]
    resources = [r for i, r in enumerate(resources) if r[0] not in filenames[:i]]
    data.append(resources)
    return data
<|endoftext|>


<commit_before>from hyperadmin.indexes import Index

class ModelIndex(Index):
    @property
    def model(self):
        return self.resource.model

    def get_primary_field(self):
        return self.model._meta.pk

    def get_url_params(self, param_map={}):
        """
        returns url parts for use in the url regexp for conducting item lookups
        """
        #TODO support non integer lookups
        param_map.setdefault('pk', 'pk')
        field = self.get_primary_field()
        from django.db import models
        if isinstance(field, (models.IntegerField, models.AutoField)):
            return [r'(?P<{pk}>\d+)'.format(**param_map)]
        return [r'(?P<{pk}>[\w\d\-]+)'.format(**param_map)]

    def get_paginator_kwargs(self):
        return {'per_page':self.resource.list_per_page,}

    def get_links(self):
        links = super(ModelIndex, self).get_links()
        #links += self.getchangelist_sort_links()
        return links

    def get_changelist_sort_links(self):
        links = list()
        changelist = self.state['changelist']
        from django.contrib.admin.templatetags.admin_list import result_headers
        for header in result_headers(changelist):
            if header.get("sortable", False):
                prompt = unicode(header["text"])
                classes = ["sortby"]
                if "url" in header:
                    links.append(self.get_link(url=header["url"], prompt=prompt, classes=classes+["primary"], rel="sortby"))
                else:
                    if header["ascending"]:
                        classes.append("ascending")
                    if header["sorted"]:
                        classes.append("sorted")
                    links.append(self.get_link(url=header["url_primary"], prompt=prompt, classes=classes+["primary"], rel="sortby"))
                    links.append(self.get_link(url=header["url_remove"], prompt=prompt, classes=classes+["remove"], rel="sortby"))
                    links.append(self.get_link(url=header["url_toggle"], prompt=prompt, classes=classes+["toggle"], rel="sortby"))
        return links

class InlineIndex(Index):
    def get(self, **kwargs):
        return self.get_index_query().get(pk=kwargs['inline_pk'])
<commit_msg>match negative primary keys, fix #2<commit_after>from hyperadmin.indexes import Index

class ModelIndex(Index):
    @property
    def model(self):
        return self.resource.model

    def get_primary_field(self):
        return self.model._meta.pk

    def get_url_params(self, param_map={}):
        """
        returns url parts for use in the url regexp for conducting item lookups
        """
        #TODO support non integer lookups
        param_map.setdefault('pk', 'pk')
        field = self.get_primary_field()
        from django.db import models
        if isinstance(field, (models.IntegerField, models.AutoField)):
            #TODO detect if positive integers only
            return [r'(?P<{pk}>(-){0,1}\d+)'.format(**param_map)]
        return [r'(?P<{pk}>[\w\d\-]+)'.format(**param_map)]

    def get_paginator_kwargs(self):
        return {'per_page':self.resource.list_per_page,}

    def get_links(self):
        links = super(ModelIndex, self).get_links()
        #links += self.getchangelist_sort_links()
        return links

    def get_changelist_sort_links(self):
        links = list()
        changelist = self.state['changelist']
        from django.contrib.admin.templatetags.admin_list import result_headers
        for header in result_headers(changelist):
            if header.get("sortable", False):
                prompt = unicode(header["text"])
                classes = ["sortby"]
                if "url" in header:
                    links.append(self.get_link(url=header["url"], prompt=prompt, classes=classes+["primary"], rel="sortby"))
                else:
                    if header["ascending"]:
                        classes.append("ascending")
                    if header["sorted"]:
                        classes.append("sorted")
                    links.append(self.get_link(url=header["url_primary"], prompt=prompt, classes=classes+["primary"], rel="sortby"))
                    links.append(self.get_link(url=header["url_remove"], prompt=prompt, classes=classes+["remove"], rel="sortby"))
                    links.append(self.get_link(url=header["url_toggle"], prompt=prompt, classes=classes+["toggle"], rel="sortby"))
        return links

class InlineIndex(Index):
    def get(self, **kwargs):
        return self.get_index_query().get(pk=kwargs['inline_pk'])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Utility classes
---------------
"""

from __future__ import absolute_import
import six
from collections import namedtuple, OrderedDict, Set

__all__ = ['NameTitle', 'LabeledEnum', 'InspectableSet', 'classmethodproperty']

NameTitle = namedtuple('NameTitle', ['name', 'title'])

class _LabeledEnumMeta(type):
    """Construct labeled enumeration"""
    @classmethod
    def __prepare__(metacls, name, bases, **kwargs):  # pragma: no cover
        return OrderedDict()

    def __new__(cls, name, bases, attrs, **kwargs):
        labels = OrderedDict()
        names = OrderedDict()

        def pop_name_by_value(value):
            for k, v in list(names.items()):
                if v == value:
                    names.pop(k)
                    return k

        for key, value in tuple(attrs.items()):
            if key != '__order__' and isinstance(value, tuple):
                # value = tuple of actual value (0), label/name (1), optional title (2)
                if len(value) == 2:
                    labels[value[0]] = value[1]
                    attrs[key] = names[key] = value[0]
                elif len(value) == 3:
                    labels[value[0]] = NameTitle(value[1], value[2])
                    attrs[key] = names[key] = value[0]
                else:  # pragma: no cover
                    raise AttributeError("Unprocessed attribute %s" % key)
            elif key != '__order__' and isinstance(value, set):
                # value = set of other unprocessed values
                attrs[key] = names[key] = {v[0] if isinstance(v, tuple) else v for v in value}

        if '__order__' in attrs:
            ordered_labels = OrderedDict()
            ordered_names = OrderedDict()
            for value in attrs['__order__']:
                ordered_labels[value[0]] = labels.pop(value[0])
                attr_name = pop_name_by_value(value[0])
                if attr_name is not None:
                    ordered_names[attr_name] = value[0]
            for key, value in labels.items():  # Left over items after processing the list in __order__
                ordered_labels[key] = value
                attr_name = pop_name_by_value(value)
                if attr_name is not None:
                    ordered_names[attr_name] = value
            ordered_names.update(names)  # Left over names that don't have a label
        else:  # This enum doesn't care about ordering, or is using Py3 with __prepare__
            ordered_labels = labels
            ordered_names = names
        attrs['__labels__'] = ordered_labels
        attrs['__names__'] = ordered_names
        return type.__new__(cls, name, bases, attrs)

    def __getitem__(cls, key):
        return cls.__labels__[key]

    def __contains__(cls, key):
        return key in cls.__labels__

class LabeledEnum(six.with_metaclass(_LabeledEnumMeta)):
    """
    Labeled enumerations. Declarate an enumeration with values and labels
    (for use in UI)::

        >>> class MY_ENUM(LabeledEnum):
        ...     FIRST = (1, "First")
        ...     THIRD = (3, "Third")
        ...     SECOND = (2, "Second")

    :class:`LabeledEnum` will convert any attribute that is a 2-tuple into
    a value and label pair. Access values as direct attributes of the enumeration::

        >>> MY_ENUM.FIRST
        1
        >>> MY_ENUM.SECOND
        2
        >>> MY_ENUM.THIRD
        3

    Access labels via dictionary lookup on the enumeration::

        >>> MY_ENUM[MY_ENUM.FIRST]
        'First'
        >>> MY_ENUM[2]
        'Second'
        >>> MY_ENUM.get(3)
        'Third'
        >>> MY_ENUM.get(4) is None
        True

    Retrieve a full list of values and labels with ``.items()``. Definition order is
    preserved in Python 3.x, but not in 2.x::

        >>> sorted(MY_ENUM.items())
        [(1, 'First'), (2, 'Second'), (3, 'Third')]
        >>> sorted(MY_ENUM.keys())
        [1, 2, 3]
        >>> sorted(MY_ENUM.values())
        ['First', 'Second', 'Third']

    However, if you really want ordering in Python 2.x, add an __order__ list.
    Anything not in it will default to Python's ordering::

        >>> class RSVP(LabeledEnum):
        ...     RSVP_Y = ('Y', "Yes")
        ...     RSVP_N = ('N', "No")
        ...     RSVP_M = ('M', "Maybe")
        ...     RSVP_U = ('U', "Unknown")
        ...     RSVP_A = ('A', "Awaiting")
        ...     __order__ = (RSVP_Y, RSVP_N, RSVP_M, RSVP_A)

        >>> RSVP.items()
        [('Y', 'Yes'), ('N', 'No'), ('M', 'Maybe'), ('A', 'Awaiting'), ('U', 'Unknown')]

    Three value tuples are assumed to be (value, name, title) and the name and
    title are converted into NameTitle(name, title)::

        >>> class NAME_ENUM(LabeledEnum):
        ...     FIRST = (1, 'first', "First")
        ...     THIRD = (3, 'third', "Third")
        ...     SECOND = (2, 'second', "Second")
        ...     __order__ = (FIRST, SECOND, THIRD)

        >>> NAME_ENUM.FIRST
        1
        >>> NAME_ENUM[NAME_ENUM.FIRST]
        NameTitle(name='first', title='First')
        >>> NAME_ENUM[NAME_ENUM.SECOND].name
        'second'
        >>> NAME_ENUM[NAME_ENUM.THIRD].title
        'Third'

    To make it easier to use with forms and to hide the actual values, a list of (name, title)
    pairs is available::

        >>> NAME_ENUM.nametitles()
        [('first', 'First'), ('second', 'Second'), ('third', 'Third')]

    Given a name, the value can be looked up::

        >>> NAME_ENUM.value_for('first')
        1
        >>> NAME_ENUM.value_for('second')
        2

    Values can be grouped together using a set, for performing "in" operations.
    These do not have labels and cannot be accessed via dictionary access::

        >>> class RSVP_EXTRA(LabeledEnum):
        ...     RSVP_Y = ('Y', "Yes")
        ...     RSVP_N = ('N', "No")
        ...     RSVP_M = ('M', "Maybe")
        ...     RSVP_U = ('U', "Unknown")
        ...     RSVP_A = ('A', "Awaiting")
        ...     __order__ = (RSVP_Y, RSVP_N, RSVP_M, RSVP_U, RSVP_A)
        ...     UNCERTAIN = {RSVP_M, RSVP_U, 'A'}

        >>> isinstance(RSVP_EXTRA.UNCERTAIN, set)
        True
        >>> sorted(RSVP_EXTRA.UNCERTAIN)
        ['A', 'M', 'U']
        >>> 'N' in RSVP_EXTRA.UNCERTAIN
        False
        >>> 'M' in RSVP_EXTRA.UNCERTAIN
        True
        >>> RSVP_EXTRA.RSVP_U in RSVP_EXTRA.UNCERTAIN
        True

    Labels are stored internally in a dictionary named ``__labels__``, mapping
    the value to the label. Symbol names are stored in ``__names__``, mapping
    name to the value. The label dictionary will only contain values processed
    using the tuple syntax, which excludes grouped values, while the names
    dictionary will contain both, but will exclude anything else found in the
    class that could not be processed (use ``__dict__`` for everything)::

        >>> list(RSVP_EXTRA.__labels__.keys())
        ['Y', 'N', 'M', 'U', 'A']
        >>> list(RSVP_EXTRA.__names__.keys())
        ['RSVP_Y', 'RSVP_N', 'RSVP_M', 'RSVP_U', 'RSVP_A', 'UNCERTAIN']
    """

    @classmethod
    def get(cls, key, default=None):
        return cls.__labels__.get(key, default)

    @classmethod
    def keys(cls):
        return list(cls.__labels__.keys())

    @classmethod
    def values(cls):
        return list(cls.__labels__.values())

    @classmethod
    def items(cls):
        return list(cls.__labels__.items())

    @classmethod
    def value_for(cls, name):
        for key, value in list(cls.__labels__.items()):
            if isinstance(value, NameTitle) and value.name == name:
                return key

    @classmethod
    def nametitles(cls):
        return list((name, title) for name, title in cls.values())

class InspectableSet(Set):
    """
    Given a set, mimics a read-only dictionary where the items are keys and
    have a value of ``True``, and any other key has a value of ``False``. Also
    supports attribute access. Useful in templates to simplify membership
    inspection::

        >>> myset = InspectableSet({'member', 'other'})
        >>> 'member' in myset
        True
        >>> 'random' in myset
        False
        >>> myset.member
        True
        >>> myset.random
        False
        >>> myset['member']
        True
        >>> myset['random']
        False
        >>> joinset = myset | {'added'}
        >>> isinstance(joinset, InspectableSet)
        True
        >>> joinset = joinset | InspectableSet({'inspectable'})
        >>> isinstance(joinset, InspectableSet)
        True
        >>> 'member' in joinset
        True
        >>> 'other' in joinset
        True
        >>> 'added' in joinset
        True
        >>> 'inspectable' in joinset
        True
    """
    def __init__(self, members):
        if not isinstance(members, set):
            members = set(members)
        object.__setattr__(self, '_members', members)

    def __repr__(self):  # pragma: no cover
        return 'InspectableSet({members})'.format(members=repr(self._members))

    def __len__(self):
        return len(self._members)

    def __contains__(self, key):
        return key in self._members

    def __iter__(self):
        for key in self._members:
            yield key

    def __getitem__(self, key):
        return key in self._members  # Returns True if present, False otherwise

    def __getattr__(self, attr):
        return attr in self._members  # Returns True if present, False otherwise

    def __setattr__(self, attr, value):
        raise AttributeError(attr)

class classmethodproperty(object):
    """
    Class method decorator to make class methods behave like properties::

        >>> class Foo(object):
        ...     @classmethodproperty
        ...     def test(cls):
        ...         print(repr(cls))
        ...

    Works on classes::

        >>> Foo.test
        <class 'coaster.utils.classes.Foo'>

    Works on class instances::

        >>> Foo().test
        <class 'coaster.utils.classes.Foo'>

    Works on subclasses too::

        >>> class Bar(Foo):
        ...     pass
        ...
        >>> Bar.test
        <class 'coaster.utils.classes.Bar'>
        >>> Bar().test
        <class 'coaster.utils.classes.Bar'>

    Due to limitations in Python's descriptor API, :class:`classmethodproperty`
    can block write and delete access on an instance...

    ::

        >>> Foo().test = 'bar'
        Traceback (most recent call last):
        AttributeError: test is read-only
        >>> del Foo().test
        Traceback (most recent call last):
        AttributeError: test is read-only

    ...but not on the class itself::

        >>> Foo.test = 'bar'
        >>> Foo.test
        'bar'
    """
    def __init__(self, func):
        self.func = func

    def __get__(self, obj, cls=None):
        if cls is None:
            cls = type(obj)
        return self.func(cls)

    def __set__(self, obj, value):
        raise AttributeError("%s is read-only" % self.func.__name__)

    def __delete__(self, obj):
        raise AttributeError("%s is read-only" % self.func.__name__)
<commit_msg>Don't use print in tests<commit_after># -*- coding: utf-8 -*-

"""
Utility classes
---------------
"""

from __future__ import absolute_import
import six
from collections import namedtuple, OrderedDict, Set

__all__ = ['NameTitle', 'LabeledEnum', 'InspectableSet', 'classmethodproperty']

NameTitle = namedtuple('NameTitle', ['name', 'title'])

class _LabeledEnumMeta(type):
    """Construct labeled enumeration"""
    @classmethod
    def __prepare__(metacls, name, bases, **kwargs):  # pragma: no cover
        return OrderedDict()

    def __new__(cls, name, bases, attrs, **kwargs):
        labels = OrderedDict()
        names = OrderedDict()

        def pop_name_by_value(value):
            for k, v in list(names.items()):
                if v == value:
                    names.pop(k)
                    return k

        for key, value in tuple(attrs.items()):
            if key != '__order__' and isinstance(value, tuple):
                # value = tuple of actual value (0), label/name (1), optional title (2)
                if len(value) == 2:
                    labels[value[0]] = value[1]
                    attrs[key] = names[key] = value[0]
                elif len(value) == 3:
                    labels[value[0]] = NameTitle(value[1], value[2])
                    attrs[key] = names[key] = value[0]
                else:  # pragma: no cover
                    raise AttributeError("Unprocessed attribute %s" % key)
            elif key != '__order__' and isinstance(value, set):
                # value = set of other unprocessed values
                attrs[key] = names[key] = {v[0] if isinstance(v, tuple) else v for v in value}

        if '__order__' in attrs:
            ordered_labels = OrderedDict()
            ordered_names = OrderedDict()
            for value in attrs['__order__']:
                ordered_labels[value[0]] = labels.pop(value[0])
                attr_name = pop_name_by_value(value[0])
                if attr_name is not None:
                    ordered_names[attr_name] = value[0]
            for key, value in labels.items():  # Left over items after processing the list in __order__
                ordered_labels[key] = value
                attr_name = pop_name_by_value(value)
                if attr_name is not None:
                    ordered_names[attr_name] = value
            ordered_names.update(names)  # Left over names that don't have a label
        else:  # This enum doesn't care about ordering, or is using Py3 with __prepare__
            ordered_labels = labels
            ordered_names = names
        attrs['__labels__'] = ordered_labels
        attrs['__names__'] = ordered_names
        return type.__new__(cls, name, bases, attrs)

    def __getitem__(cls, key):
        return cls.__labels__[key]

    def __contains__(cls, key):
        return key in cls.__labels__

class LabeledEnum(six.with_metaclass(_LabeledEnumMeta)):
    """
    Labeled enumerations. Declarate an enumeration with values and labels
    (for use in UI)::

        >>> class MY_ENUM(LabeledEnum):
        ...     FIRST = (1, "First")
        ...     THIRD = (3, "Third")
        ...     SECOND = (2, "Second")

    :class:`LabeledEnum` will convert any attribute that is a 2-tuple into
    a value and label pair. Access values as direct attributes of the enumeration::

        >>> MY_ENUM.FIRST
        1
        >>> MY_ENUM.SECOND
        2
        >>> MY_ENUM.THIRD
        3

    Access labels via dictionary lookup on the enumeration::

        >>> MY_ENUM[MY_ENUM.FIRST]
        'First'
        >>> MY_ENUM[2]
        'Second'
        >>> MY_ENUM.get(3)
        'Third'
        >>> MY_ENUM.get(4) is None
        True

    Retrieve a full list of values and labels with ``.items()``. Definition order is
    preserved in Python 3.x, but not in 2.x::

        >>> sorted(MY_ENUM.items())
        [(1, 'First'), (2, 'Second'), (3, 'Third')]
        >>> sorted(MY_ENUM.keys())
        [1, 2, 3]
        >>> sorted(MY_ENUM.values())
        ['First', 'Second', 'Third']

    However, if you really want ordering in Python 2.x, add an __order__ list.
    Anything not in it will default to Python's ordering::

        >>> class RSVP(LabeledEnum):
        ...     RSVP_Y = ('Y', "Yes")
        ...     RSVP_N = ('N', "No")
        ...     RSVP_M = ('M', "Maybe")
        ...     RSVP_U = ('U', "Unknown")
        ...     RSVP_A = ('A', "Awaiting")
        ...     __order__ = (RSVP_Y, RSVP_N, RSVP_M, RSVP_A)

        >>> RSVP.items()
        [('Y', 'Yes'), ('N', 'No'), ('M', 'Maybe'), ('A', 'Awaiting'), ('U', 'Unknown')]

    Three value tuples are assumed to be (value, name, title) and the name and
    title are converted into NameTitle(name, title)::

        >>> class NAME_ENUM(LabeledEnum):
        ...     FIRST = (1, 'first', "First")
        ...     THIRD = (3, 'third', "Third")
        ...     SECOND = (2, 'second', "Second")
        ...     __order__ = (FIRST, SECOND, THIRD)

        >>> NAME_ENUM.FIRST
        1
        >>> NAME_ENUM[NAME_ENUM.FIRST]
        NameTitle(name='first', title='First')
        >>> NAME_ENUM[NAME_ENUM.SECOND].name
        'second'
        >>> NAME_ENUM[NAME_ENUM.THIRD].title
        'Third'

    To make it easier to use with forms and to hide the actual values, a list of (name, title)
    pairs is available::

        >>> NAME_ENUM.nametitles()
        [('first', 'First'), ('second', 'Second'), ('third', 'Third')]

    Given a name, the value can be looked up::

        >>> NAME_ENUM.value_for('first')
        1
        >>> NAME_ENUM.value_for('second')
        2

    Values can be grouped together using a set, for performing "in" operations.
    These do not have labels and cannot be accessed via dictionary access::

        >>> class RSVP_EXTRA(LabeledEnum):
        ...     RSVP_Y = ('Y', "Yes")
        ...     RSVP_N = ('N', "No")
        ...     RSVP_M = ('M', "Maybe")
        ...     RSVP_U = ('U', "Unknown")
        ...     RSVP_A = ('A', "Awaiting")
        ...     __order__ = (RSVP_Y, RSVP_N, RSVP_M, RSVP_U, RSVP_A)
        ...     UNCERTAIN = {RSVP_M, RSVP_U, 'A'}

        >>> isinstance(RSVP_EXTRA.UNCERTAIN, set)
        True
        >>> sorted(RSVP_EXTRA.UNCERTAIN)
        ['A', 'M', 'U']
        >>> 'N' in RSVP_EXTRA.UNCERTAIN
        False
        >>> 'M' in RSVP_EXTRA.UNCERTAIN
        True
        >>> RSVP_EXTRA.RSVP_U in RSVP_EXTRA.UNCERTAIN
        True

    Labels are stored internally in a dictionary named ``__labels__``, mapping
    the value to the label. Symbol names are stored in ``__names__``, mapping
    name to the value. The label dictionary will only contain values processed
    using the tuple syntax, which excludes grouped values, while the names
    dictionary will contain both, but will exclude anything else found in the
    class that could not be processed (use ``__dict__`` for everything)::

        >>> list(RSVP_EXTRA.__labels__.keys())
        ['Y', 'N', 'M', 'U', 'A']
        >>> list(RSVP_EXTRA.__names__.keys())
        ['RSVP_Y', 'RSVP_N', 'RSVP_M', 'RSVP_U', 'RSVP_A', 'UNCERTAIN']
    """

    @classmethod
    def get(cls, key, default=None):
        return cls.__labels__.get(key, default)

    @classmethod
    def keys(cls):
        return list(cls.__labels__.keys())

    @classmethod
    def values(cls):
        return list(cls.__labels__.values())

    @classmethod
    def items(cls):
        return list(cls.__labels__.items())

    @classmethod
    def value_for(cls, name):
        for key, value in list(cls.__labels__.items()):
            if isinstance(value, NameTitle) and value.name == name:
                return key

    @classmethod
    def nametitles(cls):
        return list((name, title) for name, title in cls.values())

class InspectableSet(Set):
    """
    Given a set, mimics a read-only dictionary where the items are keys and
    have a value of ``True``, and any other key has a value of ``False``. Also
    supports attribute access. Useful in templates to simplify membership
    inspection::

        >>> myset = InspectableSet({'member', 'other'})
        >>> 'member' in myset
        True
        >>> 'random' in myset
        False
        >>> myset.member
        True
        >>> myset.random
        False
        >>> myset['member']
        True
        >>> myset['random']
        False
        >>> joinset = myset | {'added'}
        >>> isinstance(joinset, InspectableSet)
        True
        >>> joinset = joinset | InspectableSet({'inspectable'})
        >>> isinstance(joinset, InspectableSet)
        True
        >>> 'member' in joinset
        True
        >>> 'other' in joinset
        True
        >>> 'added' in joinset
        True
        >>> 'inspectable' in joinset
        True
    """
    def __init__(self, members):
        if not isinstance(members, set):
            members = set(members)
        object.__setattr__(self, '_members', members)

    def __repr__(self):  # pragma: no cover
        return 'InspectableSet({members})'.format(members=repr(self._members))

    def __len__(self):
        return len(self._members)

    def __contains__(self, key):
        return key in self._members

    def __iter__(self):
        for key in self._members:
            yield key

    def __getitem__(self, key):
        return key in self._members  # Returns True if present, False otherwise

    def __getattr__(self, attr):
        return attr in self._members  # Returns True if present, False otherwise

    def __setattr__(self, attr, value):
        raise AttributeError(attr)

class classmethodproperty(object):
    """
    Class method decorator to make class methods behave like properties::

        >>> class Foo(object):
        ...     @classmethodproperty
        ...     def test(cls):
        ...         return repr(cls)
        ...

    Works on classes::

        >>> Foo.test
        "<class 'coaster.utils.classes.Foo'>"

    Works on class instances::

        >>> Foo().test
        "<class 'coaster.utils.classes.Foo'>"

    Works on subclasses too::

        >>> class Bar(Foo):
        ...     pass
        ...
        >>> Bar.test
        "<class 'coaster.utils.classes.Bar'>"
        >>> Bar().test
        "<class 'coaster.utils.classes.Bar'>"

    Due to limitations in Python's descriptor API, :class:`classmethodproperty`
    can block write and delete access on an instance...

    ::

        >>> Foo().test = 'bar'
        Traceback (most recent call last):
        AttributeError: test is read-only
        >>> del Foo().test
        Traceback (most recent call last):
        AttributeError: test is read-only

    ...but not on the class itself::

        >>> Foo.test = 'bar'
        >>> Foo.test
        'bar'
    """
    def __init__(self, func):
        self.func = func

    def __get__(self, obj, cls=None):
        if cls is None:
            cls = type(obj)
        return self.func(cls)

    def __set__(self, obj, value):
        raise AttributeError("%s is read-only" % self.func.__name__)

    def __delete__(self, obj):
        raise AttributeError("%s is read-only" % self.func.__name__)
<|endoftext|>


<commit_before>#coding=utf8
import csv
from datetime import datetime

from .event import events, MarketEvent
from .utils.py3 import with_metaclass
from .utils.metabase import MetaParams

class FeedBase(with_metaclass(MetaParams,object)):
    params = dict(
                  dtformat = '%Y-%m-%d %H:%M:%S',
                  tmformat = '%H:%M:%S',
                  timeindex = None)

    def __init__(self,datapath,instrument,
                        fromdate=None,todate=None,timeframe=None):
        self.live_mode = False
        self.continue_backtest = True

        self.datapath = datapath
        self.instrument = instrument
        self.fromdate = datetime.strptime(fromdate, '%Y-%m-%d') if fromdate else None         # 先将日期转化为datetime对象
        self.todate = datetime.strptime(todate, '%Y-%m-%d') if todate else None               # 先将日期转化为datetime对象
        self.timeframe = timeframe

        self.cur_bar_list = []
        self.bar_dict = {self.instrument:[]}

        self.preload_bar_list = []

        self.index_list = [i.lower() for i in next(self.load_csv())] # 全部变成小写
        self.id = self._get_index_dict()  # index_dict 索引值
        self.iteral_data = self.load_csv()
        self.iteral_data2 = self.load_csv()    # for indicator

    def set_dtformat(self,bar):
        # 目前只设置支持int和str
        date = bar['date']
        dt = "%Y-%m-%d %H:%M:%S"
        if self.p.timeindex:
            date = datetime.strptime(str(date), self.p.dtformat).strftime('%Y-%m-%d')
            return date + ' ' + bar[self.p.timeindex.lower()]
        else:
            return datetime.strptime(str(date), self.p.dtformat).strftime(dt)

    def _get_index_dict(self):
        # 生成一个字典，key为index名，value为索引
        func = lambda x: self.index_list.index(x)
        dic = {i : func(i) for i in self.index_list}
        return dic

    def _get_new_bar(self):
        def _update():
            bar = next(self.iteral_data)
            bar = {i : bar[self.id[i]] for i in self.index_list}
            bar['date'] = self.set_dtformat(bar)
            for i in self.index_list:
                try:bar[i] = float(bar[i])      # 将数值转化为float
                except ValueError:pass
            return bar

        try:
            bar = _update()
            # 日期范围判断
            dt = '%Y-%m-%d %H:%M:%S'
            if self.fromdate:
                while datetime.strptime(bar['date'], dt) < self.fromdate:
                    bar = _update()
            if self.todate:
                while datetime.strptime(bar['date'], dt) > self.todate:
                    raise StopIteration

            self.cur_bar_list.pop(0) if len(self.cur_bar_list) is 2 else None
            self.cur_bar_list.append(bar)

        except StopIteration:
            self.continue_backtest = False  # stop backtest

    def load_csv(self):
        return csv.reader(open(self.datapath))

    def run_once(self):
        next(self.iteral_data) # pass index row
        next(self.iteral_data2) # pass index row
        self._get_new_bar()
        self._preload()         # preload for indicator

    def update_bar(self, instrument):
        self.bar_dict[instrument].append(self.cur_bar_list[0])

    def _preload(self):
        """只需运行一次，先将fromdate前的数据都load到preload_bar_list"""
        """若没有fromdate，则不用load"""
        def _update():
            bar = next(self.iteral_data2)
            bar = {i : bar[self.id[i]] for i in self.index_list}
            bar['date'] = self.set_dtformat(bar)
            for i in self.index_list:
                try:bar[i] = float(bar[i])      # 将数值转化为float
                except ValueError:pass
            return bar

        try:
            bar = _update()
            # 日期范围判断
            dt = '%Y-%m-%d %H:%M:%S'
            if self.fromdate != None:
                while datetime.strptime(bar['date'], dt) < self.fromdate:
                    bar = _update()
                    self.preload_bar_list.append(bar)

                else:
                    self.preload_bar_list.pop(-1)   # 经过验证bug检查的，最后删除掉一个重复
            elif self.fromdate is None:
                pass
            else:
                raise SyntaxError('Catch a Bug!')

        except IndexError:
            pass

        except StopIteration:
            print('???')

        self.preload_bar_list.reverse()

    def start(self):
        # preload for indicator
        pass

    def prenext(self):
        self._get_new_bar()

    def next(self):
        self.update_bar(self.instrument)

        info = dict(instrument = self.instrument,
                    cur_bar_list = self.cur_bar_list,
                    bar_dict = self.bar_dict,
                    iteral_data2 = self.iteral_data2,
                    fromdate = self.fromdate,
                    dtformat = self.p.dtformat,
                    tmformat = self.p.tmformat,
                    timeindex = self.p.timeindex,
                    index_list = self.index_list,
                    preload_bar_list = self.preload_bar_list)
        events.put(MarketEvent(info))

class Forex_CSVFeed(with_metaclass(MetaParams, FeedBase)):
    '''
    如果CSV中日期和时间分为两列，即一列为2017.01.01，一列为12:00:00，
    则需要在params中注明 timeindex，以及日期和时间的格式
    '''
    params = dict(
                  dtformat = '%Y%m%d',
                  tmformat = '%H:%M:%S',
                  timeindex = 'Timestamp')

    def __init__(self,datapath,instrument,fromdate=None,
                                todate=None,timeframe=None):
        super(Forex_CSVFeed,self).__init__(datapath,instrument,
                                            fromdate,todate,timeframe)

class Tushare_CSVFeed(with_metaclass(MetaParams, FeedBase)):
    '''
    如果CSV中日期和时间分为两列，即一列为2017.01.01，一列为12:00:00，
    则需要在params中注明 timeindex，以及日期和时间的格式
    '''
    params = dict(
                  dtformat = '%Y-%m-%d',
                  tmformat = None,
                  timeindex = None)

    def __init__(self,datapath,instrument,fromdate=None,
                                todate=None,timeframe=None):
        super(Tushare_CSVFeed,self).__init__(datapath,instrument,
                                            fromdate,todate,timeframe)

class Futures_CSVFeed(with_metaclass(MetaParams, FeedBase)):
    '''
    如果CSV中日期和时间分为两列，即一列为2017.01.01，一列为12:00:00，
    则需要在params中注明 timeindex，以及日期和时间的格式
    '''
    params = dict(
                  dtformat = '%Y/%m/%d',
                  tmformat = '%H:%M:%S',
                  timeindex = 'Time')

    def __init__(self,datapath,instrument,fromdate=None,
                                todate=None,timeframe=None):
        super(Futures_CSVFeed,self).__init__(datapath,instrument,
                                            fromdate,todate,timeframe)

class GenericCSVFeed(with_metaclass(MetaParams, FeedBase)):
    '''
    如果CSV中日期和时间分为两列，即一列为2017.01.01，一列为12:00:00，
    则需要在params中注明 timeindex，以及日期和时间的格式
    '''
    params = dict(
                  dtformat = '%Y-%m-%d',
                  tmformat = '%H:%M:%S',
                  timeindex = None)

    def __init__(self,datapath,instrument,fromdate=None,todate=None,timeframe=None):
        super(GenericCSVFeed,self).__init__(datapath,instrument,fromdate,todate,timeframe)

################ Main func ##################
def run_first(feed_list):
    for feed in feed_list:
        feed.run_once()

def load_all_feed(feed_list):
    for feed in feed_list:
        feed.start()
        feed.prenext()
        feed.next()

def combine_feed():
    # 将所有feed的信息都整合在一起
    pass
<commit_msg>update<commit_after>#coding=utf8
import csv
from datetime import datetime

from .event import events, MarketEvent
from .utils.py3 import with_metaclass
from .utils.metabase import MetaParams

class FeedBase(with_metaclass(MetaParams,object)):
    params = dict(
                  dtformat = '%Y-%m-%d %H:%M:%S',
                  tmformat = '%H:%M:%S',
                  timeindex = None)

    def __init__(self,datapath,instrument,
                        fromdate=None,todate=None,timeframe=None):
        self.live_mode = False
        self.continue_backtest = True

        self.datapath = datapath
        self.instrument = instrument
        self.fromdate = datetime.strptime(fromdate, '%Y-%m-%d') if fromdate else None         # 先将日期转化为datetime对象
        self.todate = datetime.strptime(todate, '%Y-%m-%d') if todate else None               # 先将日期转化为datetime对象
        self.timeframe = timeframe

        self.cur_bar_list = []
        self.bar_dict = {self.instrument:[]}

        self.preload_bar_list = []

        self.index_list = [i.lower() for i in next(self.load_csv())] # 全部变成小写
        self.id = self._get_index_dict()  # index_dict 索引值
        self.iteral_data = self.load_csv()
        self.iteral_data2 = self.load_csv()    # for indicator

    def set_dtformat(self,bar):
        # 目前只设置支持int和str
        date = bar['date']
        dt = "%Y-%m-%d %H:%M:%S"
        if self.p.timeindex:
            date = datetime.strptime(str(date), self.p.dtformat).strftime('%Y-%m-%d')
            return date + ' ' + bar[self.p.timeindex.lower()]
        else:
            return datetime.strptime(str(date), self.p.dtformat).strftime(dt)

    def _get_index_dict(self):
        # 生成一个字典，key为index名，value为索引
        func = lambda x: self.index_list.index(x)
        dic = {i : func(i) for i in self.index_list}
        return dic

    def _get_new_bar(self):
        def _update():
            bar = next(self.iteral_data)
            bar = {i : bar[self.id[i]] for i in self.index_list}
            bar['date'] = self.set_dtformat(bar)
            for i in self.index_list:
                try:bar[i] = float(bar[i])      # 将数值转化为float
                except ValueError:pass
            return bar

        try:
            bar = _update()
            # 日期范围判断
            dt = '%Y-%m-%d %H:%M:%S'
            if self.fromdate:
                while datetime.strptime(bar['date'], dt) < self.fromdate:
                    bar = _update()
            if self.todate:
                while datetime.strptime(bar['date'], dt) > self.todate:
                    raise StopIteration

            self.cur_bar_list.pop(0) if len(self.cur_bar_list) is 2 else None
            self.cur_bar_list.append(bar)

        except StopIteration:
            self.continue_backtest = False  # stop backtest

    def load_csv(self):
        return csv.reader(open(self.datapath))

    def run_once(self):
        next(self.iteral_data) # pass index row
        next(self.iteral_data2) # pass index row
        self._get_new_bar()
        self._preload()         # preload for indicator

    def update_bar(self, instrument):
        self.bar_dict[instrument].append(self.cur_bar_list[0])

    def _preload(self):
        """只需运行一次，先将fromdate前的数据都load到preload_bar_list"""
        """若没有fromdate，则不用load"""
        def _update():
            bar = next(self.iteral_data2)
            bar = {i : bar[self.id[i]] for i in self.index_list}
            bar['date'] = self.set_dtformat(bar)
            for i in self.index_list:
                try:bar[i] = float(bar[i])      # 将数值转化为float
                except ValueError:pass
            return bar

        try:
            bar = _update()
            # 日期范围判断
            dt = '%Y-%m-%d %H:%M:%S'
            if self.fromdate:
                while datetime.strptime(bar['date'], dt) < self.fromdate:
                    bar = _update()
                    self.preload_bar_list.append(bar)

                else:
                    self.preload_bar_list.pop(-1)   # 经过验证bug检查的，最后删除掉一个重复
            elif self.fromdate is None:
                pass
            else:
                raise SyntaxError('Catch a Bug!')

        except IndexError:
            pass

        except StopIteration:
            print('???')

        self.preload_bar_list.reverse()

    def start(self):
        # preload for indicator
        pass

    def prenext(self):
        self._get_new_bar()

    def next(self):
        self.update_bar(self.instrument)

        info = dict(instrument = self.instrument,
                    cur_bar_list = self.cur_bar_list,
                    bar_dict = self.bar_dict,
                    iteral_data2 = self.iteral_data2,
                    fromdate = self.fromdate,
                    dtformat = self.p.dtformat,
                    tmformat = self.p.tmformat,
                    timeindex = self.p.timeindex,
                    index_list = self.index_list,
                    preload_bar_list = self.preload_bar_list)
        events.put(MarketEvent(info))

class Forex_CSVFeed(with_metaclass(MetaParams, FeedBase)):
    '''
    如果CSV中日期和时间分为两列，即一列为2017.01.01，一列为12:00:00，
    则需要在params中注明 timeindex，以及日期和时间的格式
    '''
    params = dict(
                  dtformat = '%Y%m%d',
                  tmformat = '%H:%M:%S',
                  timeindex = 'Timestamp')

    def __init__(self,datapath,instrument,fromdate=None,
                                todate=None,timeframe=None):
        super(Forex_CSVFeed,self).__init__(datapath,instrument,
                                            fromdate,todate,timeframe)

class Tushare_CSVFeed(with_metaclass(MetaParams, FeedBase)):
    '''
    如果CSV中日期和时间分为两列，即一列为2017.01.01，一列为12:00:00，
    则需要在params中注明 timeindex，以及日期和时间的格式
    '''
    params = dict(
                  dtformat = '%Y-%m-%d',
                  tmformat = None,
                  timeindex = None)

    def __init__(self,datapath,instrument,fromdate=None,
                                todate=None,timeframe=None):
        super(Tushare_CSVFeed,self).__init__(datapath,instrument,
                                            fromdate,todate,timeframe)

class Futures_CSVFeed(with_metaclass(MetaParams, FeedBase)):
    '''
    如果CSV中日期和时间分为两列，即一列为2017.01.01，一列为12:00:00，
    则需要在params中注明 timeindex，以及日期和时间的格式
    '''
    params = dict(
                  dtformat = '%Y/%m/%d',
                  tmformat = '%H:%M:%S',
                  timeindex = 'Time')

    def __init__(self,datapath,instrument,fromdate=None,
                                todate=None,timeframe=None):
        super(Futures_CSVFeed,self).__init__(datapath,instrument,
                                            fromdate,todate,timeframe)

class GenericCSVFeed(with_metaclass(MetaParams, FeedBase)):
    '''
    如果CSV中日期和时间分为两列，即一列为2017.01.01，一列为12:00:00，
    则需要在params中注明 timeindex，以及日期和时间的格式
    '''
    params = dict(
                  dtformat = '%Y-%m-%d',
                  tmformat = '%H:%M:%S',
                  timeindex = None)

    def __init__(self,datapath,instrument,fromdate=None,todate=None,timeframe=None):
        super(GenericCSVFeed,self).__init__(datapath,instrument,fromdate,todate,timeframe)

################ Main func ##################
def run_first(feed_list):
    for feed in feed_list:
        feed.run_once()

def load_all_feed(feed_list):
    for feed in feed_list:
        feed.start()
        feed.prenext()
        feed.next()

def combine_feed():
    # 将所有feed的信息都整合在一起
    pass
<|endoftext|>


<commit_before># Simple script that manages the creation of
# datastores in CKAN / HDX.

# Dependencies
import os
import csv
import json
import scraperwiki
import ckanapi
import urllib
import requests
import sys
import hashlib

# Collecting configuration variables
API_KEY = sys.argv[1]
FILE_PATH = sys.argv[2]

# configuring the remote CKAN instance
ckan = ckanapi.RemoteCKAN('http://data.hdx.rwlabs.org', apikey=API_KEY)

# This is where the resources are declared. For now,
# they are declared as a Python list.
# This is a skeleton of a function that
# should fetch those schemas using other
# more refined methods.
def getResources(p):
    resources = [
        {
            'resource_id': '7329f8c7-2112-499a-8b93-d6da0331d092',
            'path': p,
            'schema': {
                "fields": [
                  { "id": "code", "type": "text" },
                  { "id": "title", "type": "text" },
                  { "id": "value", "type": "text" },
                  { "id": "latest_date", "type": "datetime" },
                  { "id": "source", "type": "text" },
                  { "id": "source_link", "type": "text" },
                  { "id": "notes", "type": "text" },
                  { "id": "explore", "type": "text" },
                  { "id": "units", "type": "text" }
                ]
            },
            'indexes':["code"],
            "primary_key": "code"
        }
    ]
    return resources

# Function to download a resource from CKAN.
def downloadResource(filename, resource_id, apikey):

    # Querying
    url = 'https://data.hdx.rwlabs.org/api/action/resource_show?id=' + resource_id
    headers = { 'Authorization': apikey }
    r = requests.get(url, headers=headers)
    doc = r.json()
    fileUrl = doc["result"]["url"]

    # Downloading
    try:
        r = requests.get(fileUrl, stream=True, headers=headers)
        with open(filename, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024):
                if chunk: # filter out keep-alive new chunks
                    f.write(chunk)
                    f.flush()
    except Exception as e:
        print e
        print 'There was an error downlaoding the file.'

# Function that checks for old SHA hash
# and stores as a SW variable the new hash
# if they differ. If this function returns true,
# then the datastore is created.
def checkHash(filename, first_run, resource_id):
    hasher = hashlib.sha1()
    with open(filename, 'rb') as afile:
        buf = afile.read()
        hasher.update(buf)
        new_hash = hasher.hexdigest()

    # checking if the files are identical or if
    # they have changed
    if first_run:
        scraperwiki.sqlite.save_var(resource_id, new_hash)
        new_data = False

    else:
        old_hash = scraperwiki.sqlite.get_var(resource_id)
        scraperwiki.sqlite.save_var(resource_id, new_hash)
        new_data = old_hash != new_hash

    # returning a boolean
    return new_data

#
def updateDatastore(filename, resource_id, resource, apikey):

    # Checking if there is new data
    update_data = checkHash(filename=filename,
                            first_run = False,
                            resource_id=resource_id)
    if (update_data == False):
        print "DataStore Status: No new data. Not updating datastore."
        return

    print "DataStore Status: New data. Updating datastore."

    def upload_data_to_datastore(ckan_resource_id, resource, apikey):
        # Let's delete any existing data before we upload again
        try:
            ckan.action.datastore_delete(resource_id=ckan_resource_id, force=True, apikey=apikey)
        except:
            pass

        ckan.action.datastore_create(
                apikey=apikey,
                resource_id=ckan_resource_id,
                force=True,
                fields=resource['schema']['fields'],
                primary_key=resource['schema'].get('primary_key'),
                indexes=resource['indexes'])

        reader = csv.DictReader(open(resource['path']))
        rows = [ row for row in reader ]
        chunksize = 1000
        offset = 0
        print('Uploading data for file: %s' % resource['path'])
        while offset < len(rows):
            rowset = rows[offset:offset+chunksize]
            ckan.action.datastore_upsert(
                    resource_id=ckan_resource_id,
                    force=True,
                    method='insert',
                    records=rowset)
            offset += chunksize
            complete = str(float(offset)/len(rows) * 100)[:3] + "%"
            print('Update successful: %s completed' % complete)

    # Running the upload function
    upload_data_to_datastore(resource_id, resource, apikey)

    # updating the UI timestamp
    # updateTimestamp(resource_id, apikey)

# Wrapper call for all functions
def runEverything(p):
    # fetch the resources list
    resources = getResources(p)
    print '-------------------------------------'

    # iterating through the provided list of resources
    for i in range(0,len(resources)):
        resource = resources[i]  # getting the right resource
        resource_id = resource['resource_id']  # getting the resource_id
        print "Reading resource id: " + resource_id
        # downloadResource(p, resource_id, apikey)
        updateDatastore(p, resource_id, resource, API_KEY)
    print '-------------------------------------'
    print 'Done.'
    print '-------------------------------------'

# Error handler for running the entire script
try:
    runEverything(FILE_PATH)
    # if everything ok
    print "ScraperWiki Status: Everything seems to be just fine."
    scraperwiki.status('ok')

except Exception as e:
    print e
    scraperwiki.status('error', 'Creating datastore failed')
    os.system("mail -s 'Guinea 3W: creating datastore failed.' luiscape@gmail.com")<commit_msg>correcting the date stamp<commit_after># Simple script that manages the creation of
# datastores in CKAN / HDX.

# Dependencies
import os
import csv
import json
import scraperwiki
import ckanapi
import urllib
import requests
import sys
import hashlib

# Collecting configuration variables
API_KEY = sys.argv[1]
FILE_PATH = sys.argv[2]

# configuring the remote CKAN instance
ckan = ckanapi.RemoteCKAN('http://data.hdx.rwlabs.org', apikey=API_KEY)

# This is where the resources are declared. For now,
# they are declared as a Python list.
# This is a skeleton of a function that
# should fetch those schemas using other
# more refined methods.
def getResources(p):
    resources = [
        {
            'resource_id': '7329f8c7-2112-499a-8b93-d6da0331d092',
            'path': p,
            'schema': {
                "fields": [
                  { "id": "code", "type": "text" },
                  { "id": "title", "type": "text" },
                  { "id": "value", "type": "text" },
                  { "id": "latest_date", "type": "timestamp" },
                  { "id": "source", "type": "text" },
                  { "id": "source_link", "type": "text" },
                  { "id": "notes", "type": "text" },
                  { "id": "explore", "type": "text" },
                  { "id": "units", "type": "text" }
                ]
            },
            'indexes':["code"],
            "primary_key": "code"
        }
    ]
    return resources

# Function to download a resource from CKAN.
def downloadResource(filename, resource_id, apikey):

    # Querying
    url = 'https://data.hdx.rwlabs.org/api/action/resource_show?id=' + resource_id
    headers = { 'Authorization': apikey }
    r = requests.get(url, headers=headers)
    doc = r.json()
    fileUrl = doc["result"]["url"]

    # Downloading
    try:
        r = requests.get(fileUrl, stream=True, headers=headers)
        with open(filename, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024):
                if chunk: # filter out keep-alive new chunks
                    f.write(chunk)
                    f.flush()
    except Exception as e:
        print e
        print 'There was an error downlaoding the file.'

# Function that checks for old SHA hash
# and stores as a SW variable the new hash
# if they differ. If this function returns true,
# then the datastore is created.
def checkHash(filename, first_run, resource_id):
    hasher = hashlib.sha1()
    with open(filename, 'rb') as afile:
        buf = afile.read()
        hasher.update(buf)
        new_hash = hasher.hexdigest()

    # checking if the files are identical or if
    # they have changed
    if first_run:
        scraperwiki.sqlite.save_var(resource_id, new_hash)
        new_data = False

    else:
        old_hash = scraperwiki.sqlite.get_var(resource_id)
        scraperwiki.sqlite.save_var(resource_id, new_hash)
        new_data = old_hash != new_hash

    # returning a boolean
    return new_data

#
def updateDatastore(filename, resource_id, resource, apikey):

    # Checking if there is new data
    update_data = checkHash(filename=filename,
                            first_run = False,
                            resource_id=resource_id)
    if (update_data == False):
        print "DataStore Status: No new data. Not updating datastore."
        return

    print "DataStore Status: New data. Updating datastore."

    def upload_data_to_datastore(ckan_resource_id, resource, apikey):
        # Let's delete any existing data before we upload again
        try:
            ckan.action.datastore_delete(resource_id=ckan_resource_id, force=True, apikey=apikey)
        except:
            pass

        ckan.action.datastore_create(
                apikey=apikey,
                resource_id=ckan_resource_id,
                force=True,
                fields=resource['schema']['fields'],
                primary_key=resource['schema'].get('primary_key'),
                indexes=resource['indexes'])

        reader = csv.DictReader(open(resource['path']))
        rows = [ row for row in reader ]
        chunksize = 1000
        offset = 0
        print('Uploading data for file: %s' % resource['path'])
        while offset < len(rows):
            rowset = rows[offset:offset+chunksize]
            ckan.action.datastore_upsert(
                    resource_id=ckan_resource_id,
                    force=True,
                    method='insert',
                    records=rowset)
            offset += chunksize
            complete = str(float(offset)/len(rows) * 100)[:3] + "%"
            print('Update successful: %s completed' % complete)

    # Running the upload function
    upload_data_to_datastore(resource_id, resource, apikey)

    # updating the UI timestamp
    # updateTimestamp(resource_id, apikey)

# Wrapper call for all functions
def runEverything(p):
    # fetch the resources list
    resources = getResources(p)
    print '-------------------------------------'

    # iterating through the provided list of resources
    for i in range(0,len(resources)):
        resource = resources[i]  # getting the right resource
        resource_id = resource['resource_id']  # getting the resource_id
        print "Reading resource id: " + resource_id
        # downloadResource(p, resource_id, apikey)
        updateDatastore(p, resource_id, resource, API_KEY)
    print '-------------------------------------'
    print 'Done.'
    print '-------------------------------------'

# Error handler for running the entire script
try:
    runEverything(FILE_PATH)
    # if everything ok
    print "ScraperWiki Status: Everything seems to be just fine."
    scraperwiki.status('ok')

except Exception as e:
    print e
    scraperwiki.status('error', 'Creating datastore failed')
    os.system("mail -s 'Guinea 3W: creating datastore failed.' luiscape@gmail.com")<|endoftext|>