<commit_before>SOL_PORT                = 0xf0ba

# type names

SOL_TYPE_DISTANCE_JUDD_RS232_RAW            = 0x01
SOL_TYPE_DISTANCE_JUDD_RS232_STATS          = 0x02
SOL_TYPE_SNOW_MAXBOTIX_MB7554_RS232_RAW     = 0x03
SOL_TYPE_SNOW_MAXBOTIX_MB7554_RS232_STATS   = 0x04
SOL_TYPE_TEMPRH_SENSERION_SHT15_RS232_RAW   = 0x05
SOL_TYPE_TEMPRH_SENSERION_SHT15_RS232_STATS = 0x06
SOL_TYPE_TEMPRH_SENSERION_SHT25_RS232_RAW   = 0x07
SOL_TYPE_TEMPRH_SENSERION_SHT25_RS232_STATS = 0x08
SOL_TYPE_SOLAR_HUKSEFLUX_LP25_AV_RAW        = 0x09
SOL_TYPE_SOLAR_HUKSEFLUX_LP25_AV_STATS      = 0x0a
SOL_TYPE_SOIL_DECAGON_GS3_RS232_RAW         = 0x0b
SOL_TYPE_SOIL_DECAGON_GS3_RS232_STATS       = 0x0c
SOL_TYPE_DUST_NOTIFLOG                      = 0x0d
SOL_TYPE_DUST_NOTIFDATA                     = 0x0e
SOL_TYPE_DUST_NOTIFIPDATA                   = 0x0f
SOL_TYPE_DUST_NOTIF_HRDEVICE                = 0x10
SOL_TYPE_DUST_NOTIF_HRNEIGHBORS             = 0x11
SOL_TYPE_DUST_NOTIF_HRDISCOVERED            = 0x12
SOL_TYPE_DUST_EVENTCOMMANDFINISHED          = 0x13
SOL_TYPE_DUST_EVENTPATHCREATE               = 0x14
SOL_TYPE_DUST_EVENTPATHDELETE               = 0x15
SOL_TYPE_DUST_EVENTPING                     = 0x16
SOL_TYPE_DUST_EVENTNETWORKTIME              = 0x17
SOL_TYPE_DUST_EVENTNETWORKRESET             = 0x18
SOL_TYPE_DUST_EVENTMOTEJOIN                 = 0x19
SOL_TYPE_DUST_EVENTMOTECREATE               = 0x1a
SOL_TYPE_DUST_EVENTMOTEDELETE               = 0x1b
SOL_TYPE_DUST_EVENTMOTELOST                 = 0x1c
SOL_TYPE_DUST_EVENTMOTEOPERATIONAL          = 0x1d
SOL_TYPE_DUST_EVENTMOTERESET                = 0x1e
SOL_TYPE_DUST_EVENTPACKETSENT               = 0x1f
SOL_TYPE_DUST_SNAPSHOT                      = 0x20
SOL_TYPE_DUST_OAP_TEMPSAMPLE                = 0x27
SOL_TYPE_SOLMANAGER_STATS                   = 0x28
SOL_TYPE_SENS_MB7363_D2S2N1L1G1             = 0x29
SOL_TYPE_SENS_GS3_I1D4T4E4N1                = 0x30
SOL_TYPE_SENS_SHT25_T2N1H2N1                = 0x31
SOL_TYPE_SENS_NEOVBAT_V2N1                  = 0x32

def solTypeToTypeName(solDefinesClass,type_id):
    for n in dir(solDefinesClass):
        if n.startswith('SOL_TYPE_') and getattr(solDefinesClass,n)==type_id:
            return n
    raise ValueError("SOL type %s does not exist" % type_id)

def solStructure(type_id):
    '''
    Return the SOL structure according to the given type id
    If the element is not found, it raises a ValueError.

    :return: a dictionary that contains the following keys:
        type, description, structure, fields
    '''
    sol_item = {}
    for item in sol_types:
        if item['type'] == type_id:
            sol_item = item
    if any(sol_item):
        return sol_item
    else:
        raise ValueError("SOL structure not found for given id:%s" % type_id)

### Dust Constants
MAX_NUM_NEIGHBORS       = 100

### Header

# version

SOL_HDR_V_OFFSET        = 6
SOL_HDR_V               = 0

# Type: single or multi MTtlv

SOL_HDR_T_OFFSET        = 5
SOL_HDR_T_SINGLE        = 0
SOL_HDR_T_MULTI         = 1

# MAC

SOL_HDR_M_OFFSET        = 4
SOL_HDR_M_NOMAC         = 0
SOL_HDR_M_8BMAC         = 1

# timestamp encoding

SOL_HDR_S_OFFSET        = 3
SOL_HDR_S_EPOCH         = 0
SOL_HDR_S_ELIDED        = 1

# Type encoding

SOL_HDR_Y_OFFSET        = 2
SOL_HDR_Y_1B            = 0
SOL_HDR_Y_2B            = 1

# Length encoding

SOL_HDR_L_OFFSET        = 0
SOL_HDR_L_WK            = 0
SOL_HDR_L_1B            = 1
SOL_HDR_L_2B            = 2
SOL_HDR_L_ELIDED        = 3

### SOL Object

SOL_HEADER_SIZE         = 1
SOL_TIMESTAMP_SIZE      = 4
SOL_OBJNUMBER_SIZE      = 1

### type definitions

sol_types = [
    {
        'type':         SOL_TYPE_DISTANCE_JUDD_RS232_RAW,
        'description':  '',
        'structure':    '>HHHB',
        'fields':       ['airtemp', 'travel_time', 'distance', 'retries'],
    },
    {
        'type':         SOL_TYPE_DISTANCE_JUDD_RS232_STATS,
        'description':  '',
        'structure':    '>HHHBBI',
        'fields':       ['airtemp', 'travel_time', 'distance', 'retries', 'count', 'std'],
    },
    {
        'type':         SOL_TYPE_SNOW_MAXBOTIX_MB7554_RS232_RAW,
        'description':  '',
        'structure':    '>H',
        'fields':       ['distance'],
    },
    {
        'type':         SOL_TYPE_SNOW_MAXBOTIX_MB7554_RS232_STATS,
        'description':  '',
        'structure':    '>HBI',
        'fields':       ['distance', 'count', 'std'],
    },
    {
        'type':         SOL_TYPE_TEMPRH_SENSERION_SHT15_RS232_RAW,
        'description':  '',
        'structure':    '>II',
        'fields':       ['temp', 'rH'],
    },
    {
        'type':         SOL_TYPE_TEMPRH_SENSERION_SHT15_RS232_STATS,
        'description':  '',
        'structure':    '>IIBBII',
        'fields':       ['temp', 'rH', 'count', 'std_temp', 'std_rH'],
    },
    {
        'type':         SOL_TYPE_TEMPRH_SENSERION_SHT25_RS232_RAW,
        'description':  '',
        'structure':    '>II',
        'fields':       ['temp', 'rH'],
    },
    {
        'type':         SOL_TYPE_TEMPRH_SENSERION_SHT25_RS232_STATS,
        'description':  '',
        'structure':    '>IIBII',
        'fields':       ['temp', 'rH', 'count', 'std_temp', 'std_rH'],
    },
    {
        'type':         SOL_TYPE_SOLAR_HUKSEFLUX_LP25_AV_RAW,
        'description':  '',
        'structure':    '>I',
        'fields':       ['Vout'],
    },
    {
        'type':         SOL_TYPE_SOLAR_HUKSEFLUX_LP25_AV_STATS,
        'description':  '',
        'structure':    '>IBI',
        'fields':       ['Vout', 'count', 'std'],
    },
    {
        'type':         SOL_TYPE_SOIL_DECAGON_GS3_RS232_RAW,
        'description':  '',
        'structure':    '>III',
        'fields':       ['moisture', 'soil_temp', 'soil_ec'],
    },
    {
        'type':         SOL_TYPE_SOIL_DECAGON_GS3_RS232_STATS,
        'description':  '',
        'structure':    '>IIIBI',
        'fields':       ['moisture', 'soil_temp', 'soil_ec', 'count', 'std'],
    },
    {
        'type':         SOL_TYPE_DUST_NOTIFDATA,
        'description':  '',
        'structure':    '>HH',
        'fields':       ['srcPort', 'dstPort'],
        'extrafields':  'data',
    },
    {
        'type':         SOL_TYPE_DUST_NOTIF_HRDEVICE,
        'description':  '',
        'structure':    '>IBbHHHHHBBBIB',
        'fields':       ['charge','queueOcc','temperature','batteryVoltage','numTxOk','numTxFail','numRxOk','numRxLost','numMacDropped','numTxBad','badLinkFrameId','badLinkSlot','badLinkOffset'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTCOMMANDFINISHED,
        'description':  '',
        'structure':    '>IB',
        'fields':       ['callbackId', 'rc'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTPATHCREATE,
        'description':  '',
        'structure':    '>QQB',
        'fields':       ['source', 'dest', 'direction'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTPATHDELETE,
        'description':  '',
        'structure':    '>QQB',
        'fields':       ['source', 'dest', 'direction'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTPING,
        'description':  '',
        'structure':    '>IQIHB',
        'fields':       ['callbackId','macAddress', 'delay', 'voltage', 'temperature'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTNETWORKTIME,
        'description':  '',
        'structure':    '>IQ5pH',
        'fields':       ['uptime','utcTime', 'asn', 'asnOffset'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTNETWORKRESET,
        'description':  '',
        'structure':    '>',
        'fields':       [],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTMOTEJOIN,
        'description':  '',
        'structure':    '>Q',
        'fields':       ['macAddress'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTMOTECREATE,
        'description':  '',
        'structure':    '>QH',
        'fields':       ['macAddress', 'moteId'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTMOTEDELETE,
        'description':  '',
        'structure':    '>QH',
        'fields':       ['macAddress', 'moteId'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTMOTELOST,
        'description':  '',
        'structure':    '>Q',
        'fields':       ['macAddress'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTMOTEOPERATIONAL,
        'description':  '',
        'structure':    '>Q',
        'fields':       ['macAddress'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTMOTERESET,
        'description':  '',
        'structure':    '>Q',
        'fields':       ['macAddress'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTPACKETSENT,
        'description':  '',
        'structure':    '>IB',
        'fields':       ['callbackId', 'rc'],
    },
    {
        'type':         SOL_TYPE_DUST_OAP_TEMPSAMPLE,
        'description':  '',
        'structure':    '>h',
        'fields':       ['temperature'],
    },
    {
        'type':         SOL_TYPE_SOLMANAGER_STATS,
        'description':  '',
        'structure':    '>III',
        'fields':       ['sol_version', 'solmanager_version', 'sdk_version'],
    },
    {
        'type':         SOL_TYPE_SENS_MB7363_D2S2N1L1G1,
        'description':  'mean & stddev of Nval d2g readings',
        'structure':    '<HHBBB',
        'fields':       ['mean_d2g[mm]', 'stdev[mm]', 'Nval', 'N<min', 'N<max'],
    },
    {
        'type':         SOL_TYPE_SENS_GS3_I1D4T4E4N1,
        'description':  'sub_id indicating soil depth - to convert dielect to VWC',
        'structure':    '<BfffB',
        'fields':       ['sub_id', 'dielect', 'temp[degC]', 'eleCond[dS/m]', 'Nval'],
    },
    {
        'type':         SOL_TYPE_SENS_SHT25_T2N1H2N1,
        'description':  'temperature and humidity sensor',
        'structure':    '<HBHB',
        'fields':       ['temp_raw', 't_countV', 'rhumidity', 'rh_countV'],
        'apply':        [
                {
                    'name':     "tempPhysical",
                    'function': lambda x: -46.85 + 175.72*(float(x)/65536),
                    'args':     ['temp_raw'],
                },
                {
                    'name':     "humidRaw",
                    'function': lambda x:  -6 + 125*(float(x)/65536),
                    'args':     ['rhumidity'],
                },
            ]
    },
    {
        'type':         SOL_TYPE_SENS_NEOVBAT_V2N1,
        'description':  'raw battery voltage of Neomote - to divide by 11',
        'structure':    '<hB',
        'fields':       ['voltage', 'numReadings'],
        'apply':        [
                {
                    'name':     "vol_phys",
                    'function': lambda x: float(x)/11,
                    'args':     ['voltage'],
                }
            ]
    },
]
<commit_msg>removed units, fixed voltage function and RH namings<commit_after>SOL_PORT                = 0xf0ba

# type names

SOL_TYPE_DISTANCE_JUDD_RS232_RAW            = 0x01
SOL_TYPE_DISTANCE_JUDD_RS232_STATS          = 0x02
SOL_TYPE_SNOW_MAXBOTIX_MB7554_RS232_RAW     = 0x03
SOL_TYPE_SNOW_MAXBOTIX_MB7554_RS232_STATS   = 0x04
SOL_TYPE_TEMPRH_SENSERION_SHT15_RS232_RAW   = 0x05
SOL_TYPE_TEMPRH_SENSERION_SHT15_RS232_STATS = 0x06
SOL_TYPE_TEMPRH_SENSERION_SHT25_RS232_RAW   = 0x07
SOL_TYPE_TEMPRH_SENSERION_SHT25_RS232_STATS = 0x08
SOL_TYPE_SOLAR_HUKSEFLUX_LP25_AV_RAW        = 0x09
SOL_TYPE_SOLAR_HUKSEFLUX_LP25_AV_STATS      = 0x0a
SOL_TYPE_SOIL_DECAGON_GS3_RS232_RAW         = 0x0b
SOL_TYPE_SOIL_DECAGON_GS3_RS232_STATS       = 0x0c
SOL_TYPE_DUST_NOTIFLOG                      = 0x0d
SOL_TYPE_DUST_NOTIFDATA                     = 0x0e
SOL_TYPE_DUST_NOTIFIPDATA                   = 0x0f
SOL_TYPE_DUST_NOTIF_HRDEVICE                = 0x10
SOL_TYPE_DUST_NOTIF_HRNEIGHBORS             = 0x11
SOL_TYPE_DUST_NOTIF_HRDISCOVERED            = 0x12
SOL_TYPE_DUST_EVENTCOMMANDFINISHED          = 0x13
SOL_TYPE_DUST_EVENTPATHCREATE               = 0x14
SOL_TYPE_DUST_EVENTPATHDELETE               = 0x15
SOL_TYPE_DUST_EVENTPING                     = 0x16
SOL_TYPE_DUST_EVENTNETWORKTIME              = 0x17
SOL_TYPE_DUST_EVENTNETWORKRESET             = 0x18
SOL_TYPE_DUST_EVENTMOTEJOIN                 = 0x19
SOL_TYPE_DUST_EVENTMOTECREATE               = 0x1a
SOL_TYPE_DUST_EVENTMOTEDELETE               = 0x1b
SOL_TYPE_DUST_EVENTMOTELOST                 = 0x1c
SOL_TYPE_DUST_EVENTMOTEOPERATIONAL          = 0x1d
SOL_TYPE_DUST_EVENTMOTERESET                = 0x1e
SOL_TYPE_DUST_EVENTPACKETSENT               = 0x1f
SOL_TYPE_DUST_SNAPSHOT                      = 0x20
SOL_TYPE_DUST_OAP_TEMPSAMPLE                = 0x27
SOL_TYPE_SOLMANAGER_STATS                   = 0x28
SOL_TYPE_SENS_MB7363_D2S2N1L1G1             = 0x29
SOL_TYPE_SENS_GS3_I1D4T4E4N1                = 0x30
SOL_TYPE_SENS_SHT25_T2N1H2N1                = 0x31
SOL_TYPE_SENS_NEOVBAT_V2N1                  = 0x32

def solTypeToTypeName(solDefinesClass,type_id):
    for n in dir(solDefinesClass):
        if n.startswith('SOL_TYPE_') and getattr(solDefinesClass,n)==type_id:
            return n
    raise ValueError("SOL type %s does not exist" % type_id)

def solStructure(type_id):
    '''
    Return the SOL structure according to the given type id
    If the element is not found, it raises a ValueError.

    :return: a dictionary that contains the following keys:
        type, description, structure, fields
    '''
    sol_item = {}
    for item in sol_types:
        if item['type'] == type_id:
            sol_item = item
    if any(sol_item):
        return sol_item
    else:
        raise ValueError("SOL structure not found for given id:%s" % type_id)

### Dust Constants
MAX_NUM_NEIGHBORS       = 100

### Header

# version

SOL_HDR_V_OFFSET        = 6
SOL_HDR_V               = 0

# Type: single or multi MTtlv

SOL_HDR_T_OFFSET        = 5
SOL_HDR_T_SINGLE        = 0
SOL_HDR_T_MULTI         = 1

# MAC

SOL_HDR_M_OFFSET        = 4
SOL_HDR_M_NOMAC         = 0
SOL_HDR_M_8BMAC         = 1

# timestamp encoding

SOL_HDR_S_OFFSET        = 3
SOL_HDR_S_EPOCH         = 0
SOL_HDR_S_ELIDED        = 1

# Type encoding

SOL_HDR_Y_OFFSET        = 2
SOL_HDR_Y_1B            = 0
SOL_HDR_Y_2B            = 1

# Length encoding

SOL_HDR_L_OFFSET        = 0
SOL_HDR_L_WK            = 0
SOL_HDR_L_1B            = 1
SOL_HDR_L_2B            = 2
SOL_HDR_L_ELIDED        = 3

### SOL Object

SOL_HEADER_SIZE         = 1
SOL_TIMESTAMP_SIZE      = 4
SOL_OBJNUMBER_SIZE      = 1

### type definitions

sol_types = [
    {
        'type':         SOL_TYPE_DISTANCE_JUDD_RS232_RAW,
        'description':  '',
        'structure':    '>HHHB',
        'fields':       ['airtemp', 'travel_time', 'distance', 'retries'],
    },
    {
        'type':         SOL_TYPE_DISTANCE_JUDD_RS232_STATS,
        'description':  '',
        'structure':    '>HHHBBI',
        'fields':       ['airtemp', 'travel_time', 'distance', 'retries', 'count', 'std'],
    },
    {
        'type':         SOL_TYPE_SNOW_MAXBOTIX_MB7554_RS232_RAW,
        'description':  '',
        'structure':    '>H',
        'fields':       ['distance'],
    },
    {
        'type':         SOL_TYPE_SNOW_MAXBOTIX_MB7554_RS232_STATS,
        'description':  '',
        'structure':    '>HBI',
        'fields':       ['distance', 'count', 'std'],
    },
    {
        'type':         SOL_TYPE_TEMPRH_SENSERION_SHT15_RS232_RAW,
        'description':  '',
        'structure':    '>II',
        'fields':       ['temp', 'rH'],
    },
    {
        'type':         SOL_TYPE_TEMPRH_SENSERION_SHT15_RS232_STATS,
        'description':  '',
        'structure':    '>IIBBII',
        'fields':       ['temp', 'rH', 'count', 'std_temp', 'std_rH'],
    },
    {
        'type':         SOL_TYPE_TEMPRH_SENSERION_SHT25_RS232_RAW,
        'description':  '',
        'structure':    '>II',
        'fields':       ['temp', 'rH'],
    },
    {
        'type':         SOL_TYPE_TEMPRH_SENSERION_SHT25_RS232_STATS,
        'description':  '',
        'structure':    '>IIBII',
        'fields':       ['temp', 'rH', 'count', 'std_temp', 'std_rH'],
    },
    {
        'type':         SOL_TYPE_SOLAR_HUKSEFLUX_LP25_AV_RAW,
        'description':  '',
        'structure':    '>I',
        'fields':       ['Vout'],
    },
    {
        'type':         SOL_TYPE_SOLAR_HUKSEFLUX_LP25_AV_STATS,
        'description':  '',
        'structure':    '>IBI',
        'fields':       ['Vout', 'count', 'std'],
    },
    {
        'type':         SOL_TYPE_SOIL_DECAGON_GS3_RS232_RAW,
        'description':  '',
        'structure':    '>III',
        'fields':       ['moisture', 'soil_temp', 'soil_ec'],
    },
    {
        'type':         SOL_TYPE_SOIL_DECAGON_GS3_RS232_STATS,
        'description':  '',
        'structure':    '>IIIBI',
        'fields':       ['moisture', 'soil_temp', 'soil_ec', 'count', 'std'],
    },
    {
        'type':         SOL_TYPE_DUST_NOTIFDATA,
        'description':  '',
        'structure':    '>HH',
        'fields':       ['srcPort', 'dstPort'],
        'extrafields':  'data',
    },
    {
        'type':         SOL_TYPE_DUST_NOTIF_HRDEVICE,
        'description':  '',
        'structure':    '>IBbHHHHHBBBIB',
        'fields':       ['charge','queueOcc','temperature','batteryVoltage','numTxOk','numTxFail','numRxOk','numRxLost','numMacDropped','numTxBad','badLinkFrameId','badLinkSlot','badLinkOffset'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTCOMMANDFINISHED,
        'description':  '',
        'structure':    '>IB',
        'fields':       ['callbackId', 'rc'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTPATHCREATE,
        'description':  '',
        'structure':    '>QQB',
        'fields':       ['source', 'dest', 'direction'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTPATHDELETE,
        'description':  '',
        'structure':    '>QQB',
        'fields':       ['source', 'dest', 'direction'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTPING,
        'description':  '',
        'structure':    '>IQIHB',
        'fields':       ['callbackId','macAddress', 'delay', 'voltage', 'temperature'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTNETWORKTIME,
        'description':  '',
        'structure':    '>IQ5pH',
        'fields':       ['uptime','utcTime', 'asn', 'asnOffset'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTNETWORKRESET,
        'description':  '',
        'structure':    '>',
        'fields':       [],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTMOTEJOIN,
        'description':  '',
        'structure':    '>Q',
        'fields':       ['macAddress'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTMOTECREATE,
        'description':  '',
        'structure':    '>QH',
        'fields':       ['macAddress', 'moteId'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTMOTEDELETE,
        'description':  '',
        'structure':    '>QH',
        'fields':       ['macAddress', 'moteId'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTMOTELOST,
        'description':  '',
        'structure':    '>Q',
        'fields':       ['macAddress'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTMOTEOPERATIONAL,
        'description':  '',
        'structure':    '>Q',
        'fields':       ['macAddress'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTMOTERESET,
        'description':  '',
        'structure':    '>Q',
        'fields':       ['macAddress'],
    },
    {
        'type':         SOL_TYPE_DUST_EVENTPACKETSENT,
        'description':  '',
        'structure':    '>IB',
        'fields':       ['callbackId', 'rc'],
    },
    {
        'type':         SOL_TYPE_DUST_OAP_TEMPSAMPLE,
        'description':  '',
        'structure':    '>h',
        'fields':       ['temperature'],
    },
    {
        'type':         SOL_TYPE_SOLMANAGER_STATS,
        'description':  '',
        'structure':    '>III',
        'fields':       ['sol_version', 'solmanager_version', 'sdk_version'],
    },
    {
        'type':         SOL_TYPE_SENS_MB7363_D2S2N1L1G1,
        'description':  'mean & stddev of Nval d2g readings',
        'structure':    '<HHBBB',
        'fields':       ['mean_d2g', 'stdev', 'Nval', 'Nltm', 'NgtM'],
    },
    {
        'type':         SOL_TYPE_SENS_GS3_I1D4T4E4N1,
        'description':  'sub_id indicating soil depth - to convert dielect to VWC',
        'structure':    '<BfffB',
        'fields':       ['sub_id', 'dielect', 'temp', 'eleCond', 'Nval'],
    },
    {
        'type':         SOL_TYPE_SENS_SHT25_T2N1H2N1,
        'description':  'temperature and humidity sensor',
        'structure':    '<HBHB',
        'fields':       ['temp_raw', 't_Nval', 'rh_raw', 'rh_Nval'],
        'apply':        [
                {
                    'name':     "temp_phys",
                    'function': lambda x: -46.85 + 175.72*(float(x)/65536),
                    'args':     ['temp_raw'],
                },
                {
                    'name':     "rh_phys",
                    'function': lambda x:  -6 + 125*(float(x)/65536),
                    'args':     ['rh_raw'],
                },
            ]
    },
    {
        'type':         SOL_TYPE_SENS_NEOVBAT_V2N1,
        'description':  'raw battery voltage of Neomote',
        'structure':    '<hB',
        'fields':       ['voltage', 'N'],
        'apply':        [
                {
                    'name':     "vol_phys",
                    'function': lambda x: float(x)*1.1,
                    'args':     ['voltage'],
                }
            ]
    },
]
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright: (c) 2017, Andrew Saraceni <andrew.saraceni@gmail.com>
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

ANSIBLE_METADATA = {'metadata_version': '1.1',
                    'status': ['preview'],
                    'supported_by': 'community'}

DOCUMENTATION = r'''
---
module: win_domain_group_membership
version_added: "2.8"
short_description: Manage Windows domain group membership
description:
     - Allows the addition and removal of domain users
       and domain groups from/to a domain group.
options:
  name:
    description:
      - Name of the domain group to manage membership on.
    type: str
    required: yes
  members:
    description:
      - A list of members to ensure are present/absent from the group.
      - The given names must be a SmaAccountName of a user, group, service account, or computer
    type: list
    required: yes
  state:
    description:
      - Desired state of the members in the group.
      - When C(state) is C(pure), only the members specified will exist,
        and all other existing members not specified are removed.
    type: str
    choices: [ absent, present, pure ]
    default: present
  domain_username:
    description:
    - The username to use when interacting with AD.
    - If this is not set then the user Ansible used to log in with will be
      used instead when using CredSSP or Kerberos with credential delegation.
    type: str
  domain_password:
    description:
    - The password for I(username).
    type: str
  domain_server:
    description:
    - Specifies the Active Directory Domain Services instance to connect to.
    - Can be in the form of an FQDN or NetBIOS name.
    - If not specified then the value is based on the domain of the computer
      running PowerShell.
    type: str
seealso:
- module: win_domain_user
- module: win_domain_group
author:
    - Marius Rieder (@jiuka)
'''

EXAMPLES = r'''
- name: Add a domain user/group to a domain group
  win_domain_group_membership:
    name: Foo
    members:
      - Bar
    state: present

- name: Remove a domain user/group from a domain group
  win_domain_group_membership:
    name: Foo
    members:
      - Bar
    state: absent

- name: Ensure only a domain user/group exists in a domain group
  win_domain_group_membership:
    name: Foo
    members:
      - Bar
    state: pure

- name: Add a computer to a domain group
  win_domain_group_membership:
    name: Foo
    members:
      - DESKTOP$
    state: present
'''

RETURN = r'''
name:
    description: The name of the target domain group.
    returned: always
    type: str
    sample: Domain-Admins
added:
    description: A list of members added when C(state) is C(present) or
      C(pure); this is empty if no members are added.
    returned: success and C(state) is C(present) or C(pure)
    type: list
    sample: ["UserName", "GroupName"]
removed:
    description: A list of members removed when C(state) is C(absent) or
      C(pure); this is empty if no members are removed.
    returned: success and C(state) is C(absent) or C(pure)
    type: list
    sample: ["UserName", "GroupName"]
members:
    description: A list of all domain group members at completion; this is empty
      if the group contains no members.
    returned: success
    type: list
    sample: ["UserName", "GroupName"]
'''
<commit_msg>Update win_domain_group_membership.py (#55594)<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright: (c) 2017, Andrew Saraceni <andrew.saraceni@gmail.com>
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

ANSIBLE_METADATA = {'metadata_version': '1.1',
                    'status': ['preview'],
                    'supported_by': 'community'}

DOCUMENTATION = r'''
---
module: win_domain_group_membership
version_added: "2.8"
short_description: Manage Windows domain group membership
description:
     - Allows the addition and removal of domain users
       and domain groups from/to a domain group.
options:
  name:
    description:
      - Name of the domain group to manage membership on.
    type: str
    required: yes
  members:
    description:
      - A list of members to ensure are present/absent from the group.
      - The given names must be a SamAccountName of a user, group, service account, or computer.
      - For computers, you must add "$" after the name; for example, to add "Mycomputer" to a group, use "Mycomputer$" as the member.
    type: list
    required: yes
  state:
    description:
      - Desired state of the members in the group.
      - When C(state) is C(pure), only the members specified will exist,
        and all other existing members not specified are removed.
    type: str
    choices: [ absent, present, pure ]
    default: present
  domain_username:
    description:
    - The username to use when interacting with AD.
    - If this is not set then the user Ansible used to log in with will be
      used instead when using CredSSP or Kerberos with credential delegation.
    type: str
  domain_password:
    description:
    - The password for I(username).
    type: str
  domain_server:
    description:
    - Specifies the Active Directory Domain Services instance to connect to.
    - Can be in the form of an FQDN or NetBIOS name.
    - If not specified then the value is based on the domain of the computer
      running PowerShell.
    type: str
seealso:
- module: win_domain_user
- module: win_domain_group
author:
    - Marius Rieder (@jiuka)
'''

EXAMPLES = r'''
- name: Add a domain user/group to a domain group
  win_domain_group_membership:
    name: Foo
    members:
      - Bar
    state: present

- name: Remove a domain user/group from a domain group
  win_domain_group_membership:
    name: Foo
    members:
      - Bar
    state: absent

- name: Ensure only a domain user/group exists in a domain group
  win_domain_group_membership:
    name: Foo
    members:
      - Bar
    state: pure

- name: Add a computer to a domain group
  win_domain_group_membership:
    name: Foo
    members:
      - DESKTOP$
    state: present
'''

RETURN = r'''
name:
    description: The name of the target domain group.
    returned: always
    type: str
    sample: Domain-Admins
added:
    description: A list of members added when C(state) is C(present) or
      C(pure); this is empty if no members are added.
    returned: success and C(state) is C(present) or C(pure)
    type: list
    sample: ["UserName", "GroupName"]
removed:
    description: A list of members removed when C(state) is C(absent) or
      C(pure); this is empty if no members are removed.
    returned: success and C(state) is C(absent) or C(pure)
    type: list
    sample: ["UserName", "GroupName"]
members:
    description: A list of all domain group members at completion; this is empty
      if the group contains no members.
    returned: success
    type: list
    sample: ["UserName", "GroupName"]
'''
<|endoftext|>


<commit_before>from __future__ import print_function

import json
import xml.etree.ElementTree as xmlElementTree
import base64
import os
import sys
from os.path import isfile, join
import gzip
import datetime
import random
import traceback
import sys

import features_JSON
import features_XML

VERSION="2018-07-16"
processed = 0
skipped = 0
err_count = 0

if len(sys.argv) != 2:
    print ("Usage: python feature_usage_extractor.py PathToSessions")

path = sys.argv[1]

paths = []

def emptyFeatureUsageMap():
    return {
        "ListAtLevel": False,
        "HiddenNodes": False,
        "UpstreamHiddenNodes": False,
        "ShortestLacing": False,
        "LongestLacing": False,
        "DisabledLacing": False,
        "CrossProductLacing": False,
        "Pinned": False,
        "Frozen": False,
        "CustomFunction": False
    }

i = 0

print ('Enumerating files')
for root, subdirs, files in os.walk(path):
    for ff in files:
        i = i + 1
        if i % 1000 == 0:
            print (i)
        path = join(root,ff)
        if (not path.endswith('sorted.gz')):
            continue
        paths.append(path)

random.shuffle(paths)
print ('Paths to process: ' + str(len(paths)))

for path in paths:
    print (str(datetime.datetime.now()) + ": " + path + ": processed: " + str(processed) + ", errs: " + str(err_count) + ", results_exist: " + str(skipped) + ", total: " + str(processed + skipped) )

    out_path = path + ".features" + "." + VERSION
    # skip files that have been processed already
    if os.path.exists(out_path) and os.path.getmtime(out_path) > os.path.getmtime(path):
        skipped = skipped + 1
        continue

    try:
        f = gzip.open (path)
        fo = open(out_path , 'w')
        processed = processed + 1
        isJSON = False
        searchMap = {}
        searchNodeAdded = {}
        tags = set()
        featureUsageMap = emptyFeatureUsageMap()
        nodeUsageMap = {}
        userId = None
        version = None
        sessionStartMicroTime = 0
        sessionEndMicroTime = 0
        sessionDate = ''

        def writeDataToFile():
            if isJSON: # Don't write results for JSON files for now
                return

            print (json.dumps(
                {
                    "Searches" : searchMap,
                    "SearchesNodeAdded" : searchNodeAdded,
                    "Tags" : list(tags),
                    "FeatureUsageMap" : featureUsageMap,
                    "NodeUsageMap" : nodeUsageMap,
                    "UserID": userId,
                    "WorkspaceVersion": version,
                    "SessionDuration": sessionEndMicroTime - sessionStartMicroTime,
                    "Date": sessionDate
                }), file=fo)

        for ln in f:
            if ln.startswith("Downloading phase"):
                continue
            data = json.loads(ln)

            if sessionStartMicroTime == 0:
                sessionStartMicroTime = int(data["MicroTime"])
                sessionDate = data["DateTime"].split(" ")[0]
            if sessionDate != data["DateTime"].split(" ")[0]:
                print (path + " has session over multiple days")
                # Split the session: write session so far to file, then reset data collection.
                writeDataToFile()
                searchMap = {}
                searchNodeAdded = {}
                tags = set()
                featureUsageMap = emptyFeatureUsageMap()
                nodeUsageMap = {}
                sessionStartMicroTime = int(data["MicroTime"])
                sessionDate = data["DateTime"].split(" ")[0]
            sessionEndMicroTime = int(data["MicroTime"])

            tags.add(data["Tag"])
            b64decodedData = base64.b64decode(data["Data"])

            if data["Tag"] == "Search":
                searchMap[data["MicroTime"]] = b64decodedData

            if data["Tag"] == "Search-NodeAdded":
                searchNodeAdded[data["MicroTime"]] = b64decodedData

            if data["Tag"] == "Workspace":
                if b64decodedData == '':
                    continue

                # Select which feature extraction library to use depending on what version on the file format
                feature_lib = None
                if b64decodedData.startswith("<"):
                    feature_lib = features_XML
                else:
                    isJSON = True
                    print ("Skipping JSON based file: " + path)
                    if os.path.exists(out_path):
                        os.remove(out_path)

                    continue  # Skip JSON coded files for now
                    # feature_lib = features_JSON

                featureUsageMap["ListAtLevel"] = featureUsageMap["ListAtLevel"] or feature_lib.usesListAtLevel(b64decodedData)
                featureUsageMap["HiddenNodes"] = featureUsageMap["HiddenNodes"] or feature_lib.hasHiddenNodes(b64decodedData)
                featureUsageMap["UpstreamHiddenNodes"] = featureUsageMap["UpstreamHiddenNodes"] or feature_lib.hasUpstreamHiddenNodes(b64decodedData)
                featureUsageMap["ShortestLacing"] = featureUsageMap["ShortestLacing"] or feature_lib.hasShortestLacing(b64decodedData)
                featureUsageMap["LongestLacing"] = featureUsageMap["LongestLacing"] or feature_lib.hasLongestLacing(b64decodedData)
                featureUsageMap["DisabledLacing"] = featureUsageMap["DisabledLacing"] or feature_lib.hasDisabledLacing(b64decodedData)
                featureUsageMap["CrossProductLacing"] = featureUsageMap["CrossProductLacing"] or feature_lib.hasCrossProductLacing(b64decodedData)
                featureUsageMap["Pinned"] = featureUsageMap["Pinned"] or feature_lib.hasPinned(b64decodedData)
                featureUsageMap["Frozen"] = featureUsageMap["Frozen"] or feature_lib.hasFrozen(b64decodedData)

                # TOOD: Build the equiavalent to this for JSON
                workspaceElement = xmlElementTree.fromstring(b64decodedData)
                for element in workspaceElement.find('Elements'):
                    if (element.tag == 'Dynamo.Graph.Nodes.ZeroTouch.DSFunction'):
                        lib_name = element.attrib['assembly'].split('\\')[-1]
                        best_name = lib_name + ":" + element.attrib['function']
                        nodeUsageMap[best_name] = True
                    elif (element.tag == 'Dynamo.Graph.Nodes.ZeroTouch.DSVarArgFunction'):
                        lib_name = element.attrib['assembly'].split('\\')[-1]
                        best_name = lib_name + ":" + element.attrib['function']
                        nodeUsageMap[best_name] = True
                    elif (element.tag == 'Dynamo.Graph.Nodes.CustomNodes.Function'):
                        featureUsageMap["CustomFunction"] = True
                        custom_node_best_name = element.attrib['nickname'] + " (" + element.find('ID').attrib["value"] + ")"
                        nodeUsageMap["CustomFunction: " + custom_node_best_name] = True
                    else:
                        nodeUsageMap[element.attrib['type']] = True

                if (version == None):
                    version = feature_lib.getVersion(b64decodedData)

            if userId == None:
                userId = data["UserID"]
    except Exception as e:
        exc_type, exc_value, exc_traceback = sys.exc_info()
        print (e)
        print (path)

        traceback.print_tb(exc_traceback, file=sys.stdout)
        fo.flush()
        os.remove(out_path)
        err_count = err_count + 1
        continue

    writeDataToFile()
<commit_msg>Migrate commentary from the doc to the source<commit_after>from __future__ import print_function

import json
import xml.etree.ElementTree as xmlElementTree
import base64
import os
import sys
from os.path import isfile, join
import gzip
import datetime
import random
import traceback
import sys

import features_JSON
import features_XML

VERSION="2018-07-16"        # This is the name of the feature set, update it with any major changes

# Progress counters
processed = 0
skipped = 0
err_count = 0
i = 0

if len(sys.argv) != 2:
    print ("Usage: python feature_usage_extractor.py PathToSessions")

path = sys.argv[1]
paths = []

# A list of all the usages features that are currently extracted
def emptyFeatureUsageMap():
    return {
        "ListAtLevel": False,
        "HiddenNodes": False,
        "UpstreamHiddenNodes": False,
        "ShortestLacing": False,
        "LongestLacing": False,
        "DisabledLacing": False,
        "CrossProductLacing": False,
        "Pinned": False,
        "Frozen": False,
        "CustomFunction": False
    }

# Walk over the dataset computing a list of all the files to process
print ('Enumerating files')
for root, subdirs, files in os.walk(path):
    for ff in files:
        i = i + 1
        if i % 1000 == 0:
            print (i)
        path = join(root,ff)
        if (not path.endswith('sorted.gz')):
            continue
        paths.append(path)

# Randomise them in order to avoid collisions + leapfrogging problems with distributed workers
# If processing locally removing this may slightly improve performance
random.shuffle(paths)

print ('Paths to process: ' + str(len(paths)))
for path in paths:

    # Report progress
    print (str(datetime.datetime.now()) + ": " + path + ": processed: " + str(processed) + ", errs: " + str(err_count) + ", results_exist: " + str(skipped) + ", total: " + str(processed + skipped) )

    out_path = path + ".features" + "." + VERSION

    # skip files that have been processed already
    if os.path.exists(out_path) and os.path.getmtime(out_path) > os.path.getmtime(path):
        skipped = skipped + 1
        continue

    try:
        f = gzip.open (path)
        fo = open(out_path , 'w')
        processed = processed + 1
        isJSON = False

        # Initialise structures for this day
        searchMap = {}
        searchNodeAdded = {}
        tags = set()
        featureUsageMap = emptyFeatureUsageMap()
        nodeUsageMap = {}
        userId = None
        version = None
        sessionStartMicroTime = 0
        sessionEndMicroTime = 0
        sessionDate = ''

        # Helper function to export data so far
        # TODO: De-nest this function
        def writeDataToFile():
            if isJSON: # Don't write results for JSON files for now
                return

            print (json.dumps(
                {
                    "Searches" : searchMap,
                    "SearchesNodeAdded" : searchNodeAdded,
                    "Tags" : list(tags),
                    "FeatureUsageMap" : featureUsageMap,
                    "NodeUsageMap" : nodeUsageMap,
                    "UserID": userId,
                    "WorkspaceVersion": version,
                    "SessionDuration": sessionEndMicroTime - sessionStartMicroTime,
                    "Date": sessionDate
                }), file=fo)

        # Process each line of the session file
        for ln in f:
            if ln.startswith("Downloading phase"): # Marker from the download script, ignore
                continue
            data = json.loads(ln)

            # Compute the first day
            if sessionStartMicroTime == 0:
                sessionStartMicroTime = int(data["MicroTime"])
                sessionDate = data["DateTime"].split(" ")[0]

            # If a day has rolled over, export the data
            if sessionDate != data["DateTime"].split(" ")[0]:
                print (path + " has session over multiple days")

                # Split the session: write session so far to file, then reset data collection.
                writeDataToFile()
                searchMap = {}
                searchNodeAdded = {}
                tags = set()
                featureUsageMap = emptyFeatureUsageMap()
                nodeUsageMap = {}
                sessionStartMicroTime = int(data["MicroTime"])
                sessionDate = data["DateTime"].split(" ")[0]
            sessionEndMicroTime = int(data["MicroTime"])

            # Keep track of what we've seen in the data file
            tags.add(data["Tag"])

            # Data is in base64 to protect against special characters, unpack it
            b64decodedData = base64.b64decode(data["Data"])

            # Split what to do based on what kind of message this is
            # Text entered into the search box
            if data["Tag"] == "Search":
                searchMap[data["MicroTime"]] = b64decodedData

            # A node being added to the canvas from the search UI
            if data["Tag"] == "Search-NodeAdded":
                searchNodeAdded[data["MicroTime"]] = b64decodedData

            # A workspace being reported
            if data["Tag"] == "Workspace":
                if b64decodedData == '':    # An empty workspace, ignore
                    continue

                # Select which feature extraction library to use depending on what version on the file format
                feature_lib = None
                if b64decodedData.startswith("<"):
                    feature_lib = features_XML
                else:
                    isJSON = True
                    print ("Skipping JSON based file: " + path)
                    if os.path.exists(out_path):
                        os.remove(out_path)

                    continue  # Skip JSON coded files for now
                    # feature_lib = features_JSON

                # Extract usage features from the workspace
                featureUsageMap["ListAtLevel"] = featureUsageMap["ListAtLevel"] or feature_lib.usesListAtLevel(b64decodedData)
                featureUsageMap["HiddenNodes"] = featureUsageMap["HiddenNodes"] or feature_lib.hasHiddenNodes(b64decodedData)
                featureUsageMap["UpstreamHiddenNodes"] = featureUsageMap["UpstreamHiddenNodes"] or feature_lib.hasUpstreamHiddenNodes(b64decodedData)
                featureUsageMap["ShortestLacing"] = featureUsageMap["ShortestLacing"] or feature_lib.hasShortestLacing(b64decodedData)
                featureUsageMap["LongestLacing"] = featureUsageMap["LongestLacing"] or feature_lib.hasLongestLacing(b64decodedData)
                featureUsageMap["DisabledLacing"] = featureUsageMap["DisabledLacing"] or feature_lib.hasDisabledLacing(b64decodedData)
                featureUsageMap["CrossProductLacing"] = featureUsageMap["CrossProductLacing"] or feature_lib.hasCrossProductLacing(b64decodedData)
                featureUsageMap["Pinned"] = featureUsageMap["Pinned"] or feature_lib.hasPinned(b64decodedData)
                featureUsageMap["Frozen"] = featureUsageMap["Frozen"] or feature_lib.hasFrozen(b64decodedData)

                # TOOD: Build the equiavalent to this for JSON
                workspaceElement = xmlElementTree.fromstring(b64decodedData)
                for element in workspaceElement.find('Elements'):
                    if (element.tag == 'Dynamo.Graph.Nodes.ZeroTouch.DSFunction'):
                        lib_name = element.attrib['assembly'].split('\\')[-1]
                        best_name = lib_name + ":" + element.attrib['function']
                        nodeUsageMap[best_name] = True
                    elif (element.tag == 'Dynamo.Graph.Nodes.ZeroTouch.DSVarArgFunction'):
                        lib_name = element.attrib['assembly'].split('\\')[-1]
                        best_name = lib_name + ":" + element.attrib['function']
                        nodeUsageMap[best_name] = True
                    elif (element.tag == 'Dynamo.Graph.Nodes.CustomNodes.Function'):
                        featureUsageMap["CustomFunction"] = True
                        custom_node_best_name = element.attrib['nickname'] + " (" + element.find('ID').attrib["value"] + ")"
                        nodeUsageMap["CustomFunction: " + custom_node_best_name] = True
                    else:
                        nodeUsageMap[element.attrib['type']] = True

                # Extract version number (first time only)
                if (version == None):
                    version = feature_lib.getVersion(b64decodedData)

            # Extract user ID (first time only)
            if userId == None:
                userId = data["UserID"]
    except Exception as e:
        # If there were a problem, get the stack trace for what happeend
        exc_type, exc_value, exc_traceback = sys.exc_info()

        # Log it
        print (e)
        print (path)
        traceback.print_tb(exc_traceback, file=sys.stdout)

        # Remove partial results
        fo.flush()
        os.remove(out_path)
        err_count = err_count + 1
        continue

    # Flush any further data to the file
    writeDataToFile()
<|endoftext|>


<commit_before># coding: utf-8

import codecs
import os
import sys
import zipfile

from argparse import ArgumentParser

COLORS_FILE = 'colors.tdesktop-theme'
THEME_MAP_FILE = 'theme-map.ini'
TEMP_DIR = 'tmp'

DESCRIPTION = 'Convert Telegram Android theme to Telegram Desktop ones.'

SINGLE_COMMENT_CHARS = ['//', ';', '#']
THEME_MAP_SEPARATOR = '='
ATTHEME_SEPARATOR = '='

DESKTOP_KEYS_FILE = 'desktop.keys'
ANDROID_KEYS_FILE = 'android.keys'

def main():
    arg_parser = create_arg_parser()
    return parse_args(arg_parser)

def create_arg_parser():
    parser = ArgumentParser(prog='TDroidDesk', description=DESCRIPTION)
    parser.add_argument(dest='theme', help='theme to convert')
    return parser

def parse_args(arg_parser):
    args = arg_parser.parse_args()

    theme = args.theme
    if os.path.isfile(theme):
        attheme = open_attheme(theme)
        desktop_theme = convert_att_desktop(attheme)
        save_desktop_theme(desktop_theme, os.path.splitext(theme)[0])
    elif os.path.isdir(theme):
        arg_parser.error('{0} is a directory'.format(theme))
        return 1
    else:
        arg_parser.error('{0} does not exist'.format(theme))
        return 2
    return 0

def open_attheme(attheme_path):
    attheme = {}
    with codecs.open(attheme_path, 'r', 'cp1251') as fp:
        # TODO: read embed image
        while True:
            line = fp.readline()

            # Reached EOL
            if not line:
                break

            if 'WPS' in line:
                break

            if is_key_val_pair(line, ATTHEME_SEPARATOR):
                key, raw_color = line.strip().split(ATTHEME_SEPARATOR, 1)
                color = convert_signed_int(int(raw_color))

                attheme[key] = color

    return attheme

def save_desktop_theme(desktop_theme, filename):
    with open(COLORS_FILE, 'w') as fp:
        for key, color in desktop_theme.items():
            fp.write('{0}: #{1:x};\n'.format(key, color))

    desktop_theme_file = '{0}.tdesktop-theme'.format(filename)
    with zipfile.ZipFile(desktop_theme_file, mode='w') as zp:
        zp.write(COLORS_FILE)

    os.remove(COLORS_FILE)

def convert_att_desktop(attheme):
    theme_map = get_theme_map()
    desktop_theme = {}

    for desktop_key, att_key in theme_map.items():
        if att_key not in attheme:
            # print('Missing {0} key in source theme'.format(att_key))
            continue

        color = attheme[att_key]
        desktop_theme[desktop_key] = color

    return desktop_theme

def get_theme_map():
    theme_map = {}
    desktop_keys = get_desktop_theme_keys()
    android_keys = get_android_theme_keys()

    with open(THEME_MAP_FILE, 'r') as fp:
        for line in fp.readlines():
            if is_comment(line):
                continue

            if is_key_val_pair(line, THEME_MAP_SEPARATOR):
                desktop_key, android_key = line.strip().split('=', 1)
                if desktop_key not in desktop_keys:
                    print('Warning: unknown key: {0}'.format(desktop_key))
                if android_key not in android_keys:
                    print('Warning: unknown key: {0}'.format(android_key))

                theme_map[desktop_key] = android_key

    return theme_map

def get_android_theme_keys():
    return get_theme_keys(ANDROID_KEYS_FILE)

def get_desktop_theme_keys():
    return get_theme_keys(DESKTOP_KEYS_FILE)

def get_theme_keys(filename):
    theme_keys = []

    with open(filename, 'r') as fp:
        theme_keys = [line.strip() for line in fp if line]

    return theme_keys

def is_comment(line):
    for char in SINGLE_COMMENT_CHARS:
        if char in line:
            return True
    return False

def is_key_val_pair(line, sep):
    return sep in line

def convert_signed_int(value):
    rgb = (value + 0x100000000)

    a = (rgb & 0xFF000000) >> 24
    r = (rgb & 0x00FF0000) >> 16
    g = (rgb & 0x0000FF00) >> 8
    b = (rgb & 0x000000FF)

    return (r << 24) | (g << 16) | (b << 8) | a

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Read background from source theme<commit_after># coding: utf-8

import os
import sys
import zipfile

from argparse import ArgumentParser

BACKGROUND_FILE = 'background.jpg'
COLORS_FILE = 'colors.tdesktop-theme'
THEME_MAP_FILE = 'theme-map.ini'
TEMP_DIR = 'tmp'

THEME = 'theme'
BACKGROUND = 'background'

DESCRIPTION = 'Convert Telegram Android theme to Telegram Desktop ones.'

SINGLE_COMMENT_CHARS = ['//', ';', '#']
THEME_MAP_SEPARATOR = '='
ATTHEME_SEPARATOR = '='

DESKTOP_KEYS_FILE = 'desktop.keys'
ANDROID_KEYS_FILE = 'android.keys'

STATE_READ_THEME = 0
STATE_READ_BACKGROUND = 1

def main():
    arg_parser = create_arg_parser()
    return parse_args(arg_parser)

def create_arg_parser():
    parser = ArgumentParser(prog='TDroidDesk', description=DESCRIPTION)
    parser.add_argument(dest='theme', help='theme to convert')
    return parser

def parse_args(arg_parser):
    args = arg_parser.parse_args()

    theme = args.theme
    if os.path.isfile(theme):
        attheme = open_attheme(theme)
        desktop_theme = convert_att_desktop(attheme)
        save_desktop_theme(desktop_theme, os.path.splitext(theme)[0])
    elif os.path.isdir(theme):
        arg_parser.error('{0} is a directory'.format(theme))
        return 1
    else:
        arg_parser.error('{0} does not exist'.format(theme))
        return 2
    return 0

def open_attheme(attheme_path):
    state = STATE_READ_THEME

    attheme = get_empty_theme()
    with open(attheme_path, 'rb') as fp:
        # TODO: read embed image
        while True:
            if state == STATE_READ_THEME:
                line = fp.readline().decode('ascii')

                # Reached EOL
                if not line:
                    break

                if 'WPS' in line:
                    state = STATE_READ_BACKGROUND
                    continue

                if is_key_val_pair(line, ATTHEME_SEPARATOR):
                    key, raw_color = line.strip().split(ATTHEME_SEPARATOR, 1)
                    color = convert_signed_int(int(raw_color))

                    attheme[THEME][key] = color
            elif state == STATE_READ_BACKGROUND:
                buf = fp.read(1024)
                if buf:
                    # FIXME: Ignore WPE tag
                    # if is_end_of_background(buf):
                    #    break

                    attheme[BACKGROUND].extend(buf)
                else:
                    # malformed theme, ignore background image
                    break

    return attheme

def save_desktop_theme(desktop_theme, filename):
    with open(COLORS_FILE, 'w') as fp:
        for key, color in desktop_theme[THEME].items():
            fp.write('{0}: #{1:x};\n'.format(key, color))

    with open(BACKGROUND_FILE, 'wb') as fp:
        fp.write(desktop_theme[BACKGROUND])

    desktop_theme_file = '{0}.tdesktop-theme'.format(filename)
    with zipfile.ZipFile(desktop_theme_file, mode='w') as zp:
        zp.write(COLORS_FILE)
        zp.write(BACKGROUND_FILE)

    os.remove(COLORS_FILE)
    os.remove(BACKGROUND_FILE)

def convert_att_desktop(attheme):
    theme_map = get_theme_map()
    desktop_theme = get_empty_theme()

    for desktop_key, att_key in theme_map.items():
        if att_key not in attheme[THEME]:
            # print('Missing {0} key in source theme'.format(att_key))
            continue

        color = attheme[THEME][att_key]
        desktop_theme[THEME][desktop_key] = color

    desktop_theme[BACKGROUND] = attheme[BACKGROUND]

    return desktop_theme

def get_theme_map():
    theme_map = {}
    desktop_keys = get_desktop_theme_keys()
    android_keys = get_android_theme_keys()

    with open(THEME_MAP_FILE, 'r') as fp:
        for line in fp.readlines():
            if is_comment(line):
                continue

            if is_key_val_pair(line, THEME_MAP_SEPARATOR):
                desktop_key, android_key = line.strip().split('=', 1)
                if desktop_key not in desktop_keys:
                    print('Warning: unknown key: {0}'.format(desktop_key))
                if android_key not in android_keys:
                    print('Warning: unknown key: {0}'.format(android_key))

                theme_map[desktop_key] = android_key

    return theme_map

def get_empty_theme():
    return {
        'theme': {},
        'background': bytearray()
    }

def get_android_theme_keys():
    return get_theme_keys(ANDROID_KEYS_FILE)

def get_desktop_theme_keys():
    return get_theme_keys(DESKTOP_KEYS_FILE)

def get_theme_keys(filename):
    theme_keys = []

    with open(filename, 'r') as fp:
        theme_keys = [line.strip() for line in fp if line]

    return theme_keys

def is_comment(line):
    for char in SINGLE_COMMENT_CHARS:
        if char in line:
            return True
    return False

def is_key_val_pair(line, sep):
    return sep in line

def is_end_of_background(buf):
    return buf.endswith(b'WPE\n') or buf.endswith(b'WPE')

def convert_signed_int(value):
    rgb = (value + 0x100000000)

    a = (rgb & 0xFF000000) >> 24
    r = (rgb & 0x00FF0000) >> 16
    g = (rgb & 0x0000FF00) >> 8
    b = (rgb & 0x000000FF)

    return (r << 24) | (g << 16) | (b << 8) | a

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

"""
@version: ??
@author: liangliangyy
@license: MIT Licence
@contact: liangliangyy@gmail.com
@site: https://www.lylinux.org/
@software: PyCharm
@file: blog_tags.py
@time: 2016/11/2 下午11:10
"""

from django import template
from django.conf import settings
from django.template.defaultfilters import stringfilter
from django.utils.safestring import mark_safe
import random
from django.urls import reverse
from blog.models import Article, Category, Tag, Links, SideBar
from django.utils.encoding import force_text
from django.shortcuts import get_object_or_404
import hashlib
import urllib
from comments.models import Comment
from DjangoBlog.utils import cache_decorator
from django.contrib.auth import get_user_model
from oauth.models import OAuthUser
from django.contrib.sites.models import Site
import logging

logger = logging.getLogger(__name__)

register = template.Library()

@register.simple_tag
def timeformat(data):
    try:
        return data.strftime(settings.TIME_FORMAT)
        # print(data.strftime(settings.TIME_FORMAT))
        # return "ddd"
    except Exception as e:
        logger.error(e)
        return ""

@register.simple_tag
def datetimeformat(data):
    try:
        return data.strftime(settings.DATE_TIME_FORMAT)
    except Exception as e:
        logger.error(e)
        return ""

@register.filter(is_safe=True)
@stringfilter
def custom_markdown(content):
    from DjangoBlog.utils import CommonMarkdown
    return mark_safe(CommonMarkdown.get_markdown(content))

@register.filter(is_safe=True)
@stringfilter
def truncatechars_content(content):
    """
    获得文章内容的摘要
    :param content:
    :return:
    """
    from django.template.defaultfilters import truncatechars_html
    from DjangoBlog.utils import get_blog_setting
    blogsetting = get_blog_setting()
    return truncatechars_html(content, blogsetting.article_sub_length)

@register.filter(is_safe=True)
@stringfilter
def truncate(content):
    from django.utils.html import strip_tags

    return strip_tags(content)[:150]

@register.inclusion_tag('blog/tags/breadcrumb.html')
def load_breadcrumb(article):
    """
    获得文章面包屑
    :param article:
    :return:
    """
    names = article.get_category_tree()
    from DjangoBlog.utils import get_blog_setting
    blogsetting = get_blog_setting()
    site = Site.objects.get_current().domain
    names.append((blogsetting.sitename, site))
    names = names[::-1]

    return {
        'names': names,
        'title': article.title
    }

@register.inclusion_tag('blog/tags/article_tag_list.html')
def load_articletags(article):
    """
    文章标签
    :param article:
    :return:
    """
    tags = article.tags.all()
    tags_list = []
    for tag in tags:
        url = tag.get_absolute_url()
        count = tag.get_article_count()
        tags_list.append((
            url, count, tag, random.choice(settings.BOOTSTRAP_COLOR_TYPES)
        ))
    return {
        'article_tags_list': tags_list
    }

@register.inclusion_tag('blog/tags/sidebar.html')
def load_sidebar(user):
    """
    加载侧边栏
    :return:
    """
    logger.info('load sidebar')
    from DjangoBlog.utils import get_blog_setting
    blogsetting = get_blog_setting()
    recent_articles = Article.objects.filter(status='p')[:blogsetting.sidebar_article_count]
    sidebar_categorys = Category.objects.all()
    extra_sidebars = SideBar.objects.filter(is_enable=True).order_by('sequence')
    most_read_articles = Article.objects.filter(status='p').order_by('-views')[:blogsetting.sidebar_article_count]
    dates = Article.objects.datetimes('created_time', 'month', order='DESC')
    links = Links.objects.all()
    commment_list = Comment.objects.filter(is_enable=True).order_by('-id')[:blogsetting.sidebar_comment_count]
    # show_adsense = settings.SHOW_GOOGLE_ADSENSE
    # 标签云 计算字体大小
    # 根据总数计算出平均值 大小为 (数目/平均值)*步长
    increment = 5
    tags = Tag.objects.all()
    sidebar_tags = None
    if tags and len(tags) > 0:
        s = list(map(lambda t: (t, t.get_article_count()), tags))
        count = sum(map(lambda t: t[1], s))
        dd = 1 if count == 0 else count / len(tags)
        sidebar_tags = list(map(lambda x: (x[0], x[1], (x[1] / dd) * increment + 10), s))

    return {
        'recent_articles': recent_articles,
        'sidebar_categorys': sidebar_categorys,
        'most_read_articles': most_read_articles,
        'article_dates': dates,
        'sidabar_links': links,
        'sidebar_comments': commment_list,
        'user': user,
        'show_google_adsense': blogsetting.show_google_adsense,
        'google_adsense_codes': blogsetting.google_adsense_codes,
        'open_site_comment': blogsetting.open_site_comment,
        'show_gongan_code': blogsetting.show_gongan_code,
        'sidebar_tags': sidebar_tags,
        'extra_sidebars': extra_sidebars
    }

@register.inclusion_tag('blog/tags/article_meta_info.html')
def load_article_metas(article, user):
    """
    获得文章meta信息
    :param article:
    :return:
    """
    return {
        'article': article,
        'user': user
    }

@register.inclusion_tag('blog/tags/article_pagination.html')
def load_pagination_info(page_obj, page_type, tag_name):
    previous_url = ''
    next_url = ''
    if page_type == '':
        if page_obj.has_next():
            next_number = page_obj.next_page_number()
            next_url = reverse('blog:index_page', kwargs={'page': next_number})
        if page_obj.has_previous():
            previous_number = page_obj.previous_page_number()
            previous_url = reverse('blog:index_page', kwargs={'page': previous_number})
    if page_type == '分类标签归档':
        tag = get_object_or_404(Tag, name=tag_name)
        if page_obj.has_next():
            next_number = page_obj.next_page_number()
            next_url = reverse('blog:tag_detail_page', kwargs={'page': next_number, 'tag_name': tag.slug})
        if page_obj.has_previous():
            previous_number = page_obj.previous_page_number()
            previous_url = reverse('blog:tag_detail_page', kwargs={'page': previous_number, 'tag_name': tag.slug})
    if page_type == '作者文章归档':
        if page_obj.has_next():
            next_number = page_obj.next_page_number()
            next_url = reverse('blog:author_detail_page', kwargs={'page': next_number, 'author_name': tag_name})
        if page_obj.has_previous():
            previous_number = page_obj.previous_page_number()
            previous_url = reverse('blog:author_detail_page', kwargs={'page': previous_number, 'author_name': tag_name})

    if page_type == '分类目录归档':
        category = get_object_or_404(Category, name=tag_name)
        if page_obj.has_next():
            next_number = page_obj.next_page_number()
            next_url = reverse('blog:category_detail_page',
                               kwargs={'page': next_number, 'category_name': category.slug})
        if page_obj.has_previous():
            previous_number = page_obj.previous_page_number()
            previous_url = reverse('blog:category_detail_page',
                                   kwargs={'page': previous_number, 'category_name': category.slug})

    return {
        'previous_url': previous_url,
        'next_url': next_url,
        'page_obj': page_obj
    }

"""
@register.inclusion_tag('nav.html')
def load_nav_info():
    category_list = Category.objects.all()
    return {
        'nav_category_list': category_list
    }
"""

@register.inclusion_tag('blog/tags/article_info.html')
def load_article_detail(article, isindex, user):
    """
    加载文章详情
    :param article:
    :param isindex:是否列表页，若是列表页只显示摘要
    :return:
    """
    from DjangoBlog.utils import get_blog_setting
    blogsetting = get_blog_setting()

    return {
        'article': article,
        'isindex': isindex,
        'user': user,
        'open_site_comment': blogsetting.open_site_comment,
    }

# return only the URL of the gravatar
# TEMPLATE USE:  {{ email|gravatar_url:150 }}
@register.filter
def gravatar_url(email, size=40):
    """获得gravatar头像"""
    usermodels = OAuthUser.objects.filter(email=email)
    if usermodels:
        o = list(filter(lambda x: x.picture is not None, usermodels))
        if o:
            return o[0].picture
    email = email.encode('utf-8')

    default = "https://resource.lylinux.net/image/2017/03/26/120117.jpg".encode('utf-8')

    return "https://www.gravatar.com/avatar/%s?%s" % (
        hashlib.md5(email.lower()).hexdigest(), urllib.parse.urlencode({'d': default, 's': str(size)}))

# return an image tag with the gravatar
# TEMPLATE USE:  {{ email|gravatar:150 }}
@register.filter
def gravatar(email, size=40):
    """获得gravatar头像"""
    url = gravatar_url(email, size)
    return mark_safe('' % (url, size, size))

@register.simple_tag
def query(qs, **kwargs):
    """ template tag which allows queryset filtering. Usage:
          {% query books author=author as mybooks %}
          {% for book in mybooks %}
            ...
          {% endfor %}
    """
    return qs.filter(**kwargs)
<commit_msg>为用户头像增加缓存<commit_after>#!/usr/bin/env python
# encoding: utf-8

"""
@version: ??
@author: liangliangyy
@license: MIT Licence
@contact: liangliangyy@gmail.com
@site: https://www.lylinux.org/
@software: PyCharm
@file: blog_tags.py
@time: 2016/11/2 下午11:10
"""

from django import template
from django.conf import settings
from django.template.defaultfilters import stringfilter
from django.utils.safestring import mark_safe
import random
from django.urls import reverse
from blog.models import Article, Category, Tag, Links, SideBar
from django.utils.encoding import force_text
from django.shortcuts import get_object_or_404
import hashlib
import urllib
from comments.models import Comment
from DjangoBlog.utils import cache_decorator, cache
from django.contrib.auth import get_user_model
from oauth.models import OAuthUser
from django.contrib.sites.models import Site
import logging

logger = logging.getLogger(__name__)

register = template.Library()

@register.simple_tag
def timeformat(data):
    try:
        return data.strftime(settings.TIME_FORMAT)
        # print(data.strftime(settings.TIME_FORMAT))
        # return "ddd"
    except Exception as e:
        logger.error(e)
        return ""

@register.simple_tag
def datetimeformat(data):
    try:
        return data.strftime(settings.DATE_TIME_FORMAT)
    except Exception as e:
        logger.error(e)
        return ""

@register.filter(is_safe=True)
@stringfilter
def custom_markdown(content):
    from DjangoBlog.utils import CommonMarkdown
    return mark_safe(CommonMarkdown.get_markdown(content))

@register.filter(is_safe=True)
@stringfilter
def truncatechars_content(content):
    """
    获得文章内容的摘要
    :param content:
    :return:
    """
    from django.template.defaultfilters import truncatechars_html
    from DjangoBlog.utils import get_blog_setting
    blogsetting = get_blog_setting()
    return truncatechars_html(content, blogsetting.article_sub_length)

@register.filter(is_safe=True)
@stringfilter
def truncate(content):
    from django.utils.html import strip_tags

    return strip_tags(content)[:150]

@register.inclusion_tag('blog/tags/breadcrumb.html')
def load_breadcrumb(article):
    """
    获得文章面包屑
    :param article:
    :return:
    """
    names = article.get_category_tree()
    from DjangoBlog.utils import get_blog_setting
    blogsetting = get_blog_setting()
    site = Site.objects.get_current().domain
    names.append((blogsetting.sitename, site))
    names = names[::-1]

    return {
        'names': names,
        'title': article.title
    }

@register.inclusion_tag('blog/tags/article_tag_list.html')
def load_articletags(article):
    """
    文章标签
    :param article:
    :return:
    """
    tags = article.tags.all()
    tags_list = []
    for tag in tags:
        url = tag.get_absolute_url()
        count = tag.get_article_count()
        tags_list.append((
            url, count, tag, random.choice(settings.BOOTSTRAP_COLOR_TYPES)
        ))
    return {
        'article_tags_list': tags_list
    }

@register.inclusion_tag('blog/tags/sidebar.html')
def load_sidebar(user):
    """
    加载侧边栏
    :return:
    """
    logger.info('load sidebar')
    from DjangoBlog.utils import get_blog_setting
    blogsetting = get_blog_setting()
    recent_articles = Article.objects.filter(status='p')[:blogsetting.sidebar_article_count]
    sidebar_categorys = Category.objects.all()
    extra_sidebars = SideBar.objects.filter(is_enable=True).order_by('sequence')
    most_read_articles = Article.objects.filter(status='p').order_by('-views')[:blogsetting.sidebar_article_count]
    dates = Article.objects.datetimes('created_time', 'month', order='DESC')
    links = Links.objects.all()
    commment_list = Comment.objects.filter(is_enable=True).order_by('-id')[:blogsetting.sidebar_comment_count]
    # show_adsense = settings.SHOW_GOOGLE_ADSENSE
    # 标签云 计算字体大小
    # 根据总数计算出平均值 大小为 (数目/平均值)*步长
    increment = 5
    tags = Tag.objects.all()
    sidebar_tags = None
    if tags and len(tags) > 0:
        s = list(map(lambda t: (t, t.get_article_count()), tags))
        count = sum(map(lambda t: t[1], s))
        dd = 1 if count == 0 else count / len(tags)
        sidebar_tags = list(map(lambda x: (x[0], x[1], (x[1] / dd) * increment + 10), s))

    return {
        'recent_articles': recent_articles,
        'sidebar_categorys': sidebar_categorys,
        'most_read_articles': most_read_articles,
        'article_dates': dates,
        'sidabar_links': links,
        'sidebar_comments': commment_list,
        'user': user,
        'show_google_adsense': blogsetting.show_google_adsense,
        'google_adsense_codes': blogsetting.google_adsense_codes,
        'open_site_comment': blogsetting.open_site_comment,
        'show_gongan_code': blogsetting.show_gongan_code,
        'sidebar_tags': sidebar_tags,
        'extra_sidebars': extra_sidebars
    }

@register.inclusion_tag('blog/tags/article_meta_info.html')
def load_article_metas(article, user):
    """
    获得文章meta信息
    :param article:
    :return:
    """
    return {
        'article': article,
        'user': user
    }

@register.inclusion_tag('blog/tags/article_pagination.html')
def load_pagination_info(page_obj, page_type, tag_name):
    previous_url = ''
    next_url = ''
    if page_type == '':
        if page_obj.has_next():
            next_number = page_obj.next_page_number()
            next_url = reverse('blog:index_page', kwargs={'page': next_number})
        if page_obj.has_previous():
            previous_number = page_obj.previous_page_number()
            previous_url = reverse('blog:index_page', kwargs={'page': previous_number})
    if page_type == '分类标签归档':
        tag = get_object_or_404(Tag, name=tag_name)
        if page_obj.has_next():
            next_number = page_obj.next_page_number()
            next_url = reverse('blog:tag_detail_page', kwargs={'page': next_number, 'tag_name': tag.slug})
        if page_obj.has_previous():
            previous_number = page_obj.previous_page_number()
            previous_url = reverse('blog:tag_detail_page', kwargs={'page': previous_number, 'tag_name': tag.slug})
    if page_type == '作者文章归档':
        if page_obj.has_next():
            next_number = page_obj.next_page_number()
            next_url = reverse('blog:author_detail_page', kwargs={'page': next_number, 'author_name': tag_name})
        if page_obj.has_previous():
            previous_number = page_obj.previous_page_number()
            previous_url = reverse('blog:author_detail_page', kwargs={'page': previous_number, 'author_name': tag_name})

    if page_type == '分类目录归档':
        category = get_object_or_404(Category, name=tag_name)
        if page_obj.has_next():
            next_number = page_obj.next_page_number()
            next_url = reverse('blog:category_detail_page',
                               kwargs={'page': next_number, 'category_name': category.slug})
        if page_obj.has_previous():
            previous_number = page_obj.previous_page_number()
            previous_url = reverse('blog:category_detail_page',
                                   kwargs={'page': previous_number, 'category_name': category.slug})

    return {
        'previous_url': previous_url,
        'next_url': next_url,
        'page_obj': page_obj
    }

"""
@register.inclusion_tag('nav.html')
def load_nav_info():
    category_list = Category.objects.all()
    return {
        'nav_category_list': category_list
    }
"""

@register.inclusion_tag('blog/tags/article_info.html')
def load_article_detail(article, isindex, user):
    """
    加载文章详情
    :param article:
    :param isindex:是否列表页，若是列表页只显示摘要
    :return:
    """
    from DjangoBlog.utils import get_blog_setting
    blogsetting = get_blog_setting()

    return {
        'article': article,
        'isindex': isindex,
        'user': user,
        'open_site_comment': blogsetting.open_site_comment,
    }

# return only the URL of the gravatar
# TEMPLATE USE:  {{ email|gravatar_url:150 }}
@register.filter
def gravatar_url(email, size=40):
    """获得gravatar头像"""
    cachekey = 'gravatat/' + email
    if cache.get(cachekey):
        return cache.get(cachekey)
    else:
        usermodels = OAuthUser.objects.filter(email=email)
        if usermodels:
            o = list(filter(lambda x: x.picture is not None, usermodels))
            if o:
                return o[0].picture
        email = email.encode('utf-8')

        default = "https://resource.lylinux.net/image/2017/03/26/120117.jpg".encode('utf-8')

        url = "https://www.gravatar.com/avatar/%s?%s" % (
            hashlib.md5(email.lower()).hexdigest(), urllib.parse.urlencode({'d': default, 's': str(size)}))
        cache.set(cachekey, url, 60 * 60 * 10)
        return url

# return an image tag with the gravatar
# TEMPLATE USE:  {{ email|gravatar:150 }}
@register.filter
def gravatar(email, size=40):
    """获得gravatar头像"""
    url = gravatar_url(email, size)
    return mark_safe('' % (url, size, size))

@register.simple_tag
def query(qs, **kwargs):
    """ template tag which allows queryset filtering. Usage:
          {% query books author=author as mybooks %}
          {% for book in mybooks %}
            ...
          {% endfor %}
    """
    return qs.filter(**kwargs)
<|endoftext|>


<commit_before>#
# Copyright 2013-2015 eNovance <licensing@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import abc

import cotyledon
from oslo_config import cfg
from oslo_log import log
import oslo_messaging
from oslo_utils import netutils
import six
from stevedore import extension

from aodh.i18n import _LE
from aodh import messaging

LOG = log.getLogger(__name__)

OPTS = [
    cfg.IntOpt('batch_size',
               default=1,
               help='Number of notification messages to wait before '
                    'dispatching them.'),
    cfg.IntOpt('batch_timeout',
               help='Number of seconds to wait before dispatching samples '
                    'when batch_size is not reached (None means indefinitely).'
               ),
]

@six.add_metaclass(abc.ABCMeta)
class AlarmNotifier(object):
    """Base class for alarm notifier plugins."""

    @staticmethod
    def __init__(conf):
        pass

    @abc.abstractmethod
    def notify(self, action, alarm_id, alarm_name, severity, previous,
               current, reason, reason_data):
        """Notify that an alarm has been triggered.

        :param action: The action that is being attended, as a parsed URL.
        :param alarm_id: The triggered alarm.
        :param alarm_name: The name of triggered alarm.
        :param severity: The level of triggered alarm
        :param previous: The previous state of the alarm.
        :param current: The current state of the alarm.
        :param reason: The reason the alarm changed its state.
        :param reason_data: A dict representation of the reason.
        """

class AlarmNotifierService(cotyledon.Service):
    NOTIFIER_EXTENSIONS_NAMESPACE = "aodh.notifier"

    def __init__(self, worker_id, conf):
        self.conf = conf
        transport = messaging.get_transport(self.conf)
        self.notifiers = extension.ExtensionManager(
            self.NOTIFIER_EXTENSIONS_NAMESPACE,
            invoke_on_load=True,
            invoke_args=(self.conf,))

        target = oslo_messaging.Target(topic=self.conf.notifier_topic)
        self.listener = messaging.get_batch_notification_listener(
            transport, [target], [AlarmEndpoint(self.notifiers)], False,
            self.conf.notifier.batch_size, self.conf.notifier.batch_timeout)
        self.listener.start()

    def terminate(self):
        self.listener.stop()
        self.listener.wait()

class AlarmEndpoint(object):

    def __init__(self, notifiers):
        self.notifiers = notifiers

    def sample(self, notifications):
        """Endpoint for alarm notifications"""
        LOG.debug('Received %s messages in batch.', len(notifications))
        for notification in notifications:
            self._process_alarm(self.notifiers, notification['payload'])

    @staticmethod
    def _handle_action(notifiers, action, alarm_id, alarm_name, severity,
                       previous, current, reason, reason_data):
        """Process action on alarm

        :param notifiers: list of possible notifiers.
        :param action: The action that is being attended, as a parsed URL.
        :param alarm_id: The triggered alarm.
        :param alarm_name: The name of triggered alarm.
        :param severity: The level of triggered alarm
        :param previous: The previous state of the alarm.
        :param current: The current state of the alarm.
        :param reason: The reason the alarm changed its state.
        :param reason_data: A dict representation of the reason.
        """

        try:
            action = netutils.urlsplit(action)
        except Exception:
            LOG.error(
                _LE("Unable to parse action %(action)s for alarm "
                    "%(alarm_id)s"), {'action': action, 'alarm_id': alarm_id})
            return

        try:
            notifier = notifiers[action.scheme].obj
        except KeyError:
            scheme = action.scheme
            LOG.error(
                _LE("Action %(scheme)s for alarm %(alarm_id)s is unknown, "
                    "cannot notify"),
                {'scheme': scheme, 'alarm_id': alarm_id})
            return

        try:
            LOG.debug("Notifying alarm %(id)s with action %(act)s",
                      {'id': alarm_id, 'act': action})
            notifier.notify(action, alarm_id, alarm_name, severity,
                            previous, current, reason, reason_data)
        except Exception:
            LOG.exception(_LE("Unable to notify alarm %s"), alarm_id)

    @staticmethod
    def _process_alarm(notifiers, data):
        """Notify that alarm has been triggered.

        :param notifiers: list of possible notifiers
        :param data: (dict): alarm data
        """

        actions = data.get('actions')
        if not actions:
            LOG.error(_LE("Unable to notify for an alarm with no action"))
            return

        for action in actions:
            AlarmEndpoint._handle_action(notifiers, action,
                                         data.get('alarm_id'),
                                         data.get('alarm_name'),
                                         data.get('severity'),
                                         data.get('previous'),
                                         data.get('current'),
                                         data.get('reason'),
                                         data.get('reason_data'))
<commit_msg>aodh-notifier: fix the launch bug<commit_after>#
# Copyright 2013-2015 eNovance <licensing@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import abc

import cotyledon
from oslo_config import cfg
from oslo_log import log
import oslo_messaging
from oslo_utils import netutils
import six
from stevedore import extension

from aodh.i18n import _LE
from aodh import messaging

LOG = log.getLogger(__name__)

OPTS = [
    cfg.IntOpt('batch_size',
               default=1,
               help='Number of notification messages to wait before '
                    'dispatching them.'),
    cfg.IntOpt('batch_timeout',
               help='Number of seconds to wait before dispatching samples '
                    'when batch_size is not reached (None means indefinitely).'
               ),
]

@six.add_metaclass(abc.ABCMeta)
class AlarmNotifier(object):
    """Base class for alarm notifier plugins."""

    @staticmethod
    def __init__(conf):
        pass

    @abc.abstractmethod
    def notify(self, action, alarm_id, alarm_name, severity, previous,
               current, reason, reason_data):
        """Notify that an alarm has been triggered.

        :param action: The action that is being attended, as a parsed URL.
        :param alarm_id: The triggered alarm.
        :param alarm_name: The name of triggered alarm.
        :param severity: The level of triggered alarm
        :param previous: The previous state of the alarm.
        :param current: The current state of the alarm.
        :param reason: The reason the alarm changed its state.
        :param reason_data: A dict representation of the reason.
        """

class AlarmNotifierService(cotyledon.Service):
    NOTIFIER_EXTENSIONS_NAMESPACE = "aodh.notifier"

    def __init__(self, worker_id, conf):
        super(AlarmNotifierService, self).__init__(worker_id)
        self.conf = conf
        transport = messaging.get_transport(self.conf)
        self.notifiers = extension.ExtensionManager(
            self.NOTIFIER_EXTENSIONS_NAMESPACE,
            invoke_on_load=True,
            invoke_args=(self.conf,))

        target = oslo_messaging.Target(topic=self.conf.notifier_topic)
        self.listener = messaging.get_batch_notification_listener(
            transport, [target], [AlarmEndpoint(self.notifiers)], False,
            self.conf.notifier.batch_size, self.conf.notifier.batch_timeout)
        self.listener.start()

    def terminate(self):
        self.listener.stop()
        self.listener.wait()

class AlarmEndpoint(object):

    def __init__(self, notifiers):
        self.notifiers = notifiers

    def sample(self, notifications):
        """Endpoint for alarm notifications"""
        LOG.debug('Received %s messages in batch.', len(notifications))
        for notification in notifications:
            self._process_alarm(self.notifiers, notification['payload'])

    @staticmethod
    def _handle_action(notifiers, action, alarm_id, alarm_name, severity,
                       previous, current, reason, reason_data):
        """Process action on alarm

        :param notifiers: list of possible notifiers.
        :param action: The action that is being attended, as a parsed URL.
        :param alarm_id: The triggered alarm.
        :param alarm_name: The name of triggered alarm.
        :param severity: The level of triggered alarm
        :param previous: The previous state of the alarm.
        :param current: The current state of the alarm.
        :param reason: The reason the alarm changed its state.
        :param reason_data: A dict representation of the reason.
        """

        try:
            action = netutils.urlsplit(action)
        except Exception:
            LOG.error(
                _LE("Unable to parse action %(action)s for alarm "
                    "%(alarm_id)s"), {'action': action, 'alarm_id': alarm_id})
            return

        try:
            notifier = notifiers[action.scheme].obj
        except KeyError:
            scheme = action.scheme
            LOG.error(
                _LE("Action %(scheme)s for alarm %(alarm_id)s is unknown, "
                    "cannot notify"),
                {'scheme': scheme, 'alarm_id': alarm_id})
            return

        try:
            LOG.debug("Notifying alarm %(id)s with action %(act)s",
                      {'id': alarm_id, 'act': action})
            notifier.notify(action, alarm_id, alarm_name, severity,
                            previous, current, reason, reason_data)
        except Exception:
            LOG.exception(_LE("Unable to notify alarm %s"), alarm_id)

    @staticmethod
    def _process_alarm(notifiers, data):
        """Notify that alarm has been triggered.

        :param notifiers: list of possible notifiers
        :param data: (dict): alarm data
        """

        actions = data.get('actions')
        if not actions:
            LOG.error(_LE("Unable to notify for an alarm with no action"))
            return

        for action in actions:
            AlarmEndpoint._handle_action(notifiers, action,
                                         data.get('alarm_id'),
                                         data.get('alarm_name'),
                                         data.get('severity'),
                                         data.get('previous'),
                                         data.get('current'),
                                         data.get('reason'),
                                         data.get('reason_data'))
<|endoftext|>


<commit_before>baseQuery = {
  "query": {
      "multi_match": {
          "query": "test",
          "fields": ["title", "overview"]
       }
   },
  "rescore": {
      "query": {
        "rescore_query": {
            "sltr": {
                "params": {
                    "keywords": ""
                },
                "model": "",
            }
         }
      }
   }
}

def ltrQuery(keywords, modelName):
    import json
    baseQuery['rescore']['query']['rescore_query']['sltr']['model'] = model
    baseQuery['query']['multi_match']['query'] = keywords
    baseQuery['rescore']['query']['rescore_query']['sltr']['params']['keywords'] = keywords
    print("%s" % json.dumps(baseQuery))
    return baseQuery

if __name__ == "__main__":
    import configparser
    from sys import argv
    from utils import Elasticsearch

    es = Elasticsearch(timeout=1000)
    model = "test_6"
    if len(argv) > 2:
        model = argv[2]
    results = es.search(index='tmdb', doc_type='movie', body=ltrQuery(argv[1], model))
    for result in results['hits']['hits']:
        print(result['_source']['title'])

<commit_msg>Use correct model in demo search<commit_after>baseQuery = {
  "query": {
      "multi_match": {
          "query": "test",
          "fields": ["title", "overview"]
       }
   },
  "rescore": {
      "query": {
        "rescore_query": {
            "sltr": {
                "params": {
                    "keywords": ""
                },
                "model": "",
            }
         }
      }
   }
}

def ltrQuery(keywords, modelName):
    import json
    baseQuery['rescore']['query']['rescore_query']['sltr']['model'] = modelName
    baseQuery['query']['multi_match']['query'] = keywords
    baseQuery['rescore']['query']['rescore_query']['sltr']['params']['keywords'] = keywords
    print("%s" % json.dumps(baseQuery, indent=2))
    return baseQuery

if __name__ == "__main__":
    import configparser
    from sys import argv
    from utils import Elasticsearch

    es = Elasticsearch(timeout=1000)
    model = "test_6"
    if len(argv) > 2:
        model = argv[2]
    results = es.search(index='tmdb', doc_type='movie', body=ltrQuery(argv[1], model))
    for result in results['hits']['hits']:
        print(result['_source']['title'])

<|endoftext|>