<commit_before>#!usr/bin/env python

#===============================================================================
# Import modules
#===============================================================================
# Standard Library
import os
import sys
import datetime
import time
import json

# Third party modules
import hashlib
import binascii
import evernote.edam.notestore.NoteStore as NoteStore
import evernote.edam.userstore.constants as UserStoreConstants
import evernote.edam.type.ttypes as Types
import evernote.edam.error.ttypes as Errors
from evernote.api.client import EvernoteClient

# Application modules
import settings as s
import log
import logging
import check_process
# import watchdog as wd

#---------------------------------------------------------------------------
# Get guid for tag
#---------------------------------------------------------------------------
def get_guid(items, search_string):
    '''
    Function returns the guid from a evernote list

    items           Struct: List item as defined by Evernote
    search_string   string of tag name to search

    '''

    return {item.guid: item.name for item in items if item.name.find(search_string) > -1}

#---------------------------------------------------------------------------
# Get all children from parent tag
#---------------------------------------------------------------------------
def get_tag_children(tag_list, parent_guid):
    '''
    Function returns all children from the tag guid provided

    tag_list        Struct: Tag as defined by Evernote
    search_string   string of tag name to search

    '''

    return {tag.guid: tag.name for tag in tag_list if tag.parentGuid == parent_guid}

#---------------------------------------------------------------------------
 # Check state tags file is correct and up to date
#---------------------------------------------------------------------------
def check_state_tags(tag_list, filename):
    '''
    Function checks states in state tag file is up to date

    tag_list        State tags from evernote
    filename        State tag JSON file

    '''

    #---------------------------------------------------------------------------
    # SET UP LOGGER
    #---------------------------------------------------------------------------
    logger = logging.getLogger('root')

    #---------------------------------------------------------------------------
    # GET DATA FROM FILE
    #---------------------------------------------------------------------------
    try:
        with open(filename, 'r') as f:
            state_config = json.load(f)

    except Exception, e:
        logger.warning('Warning ({error_v}).'.format(error_v=e), exc_info=True)
        state_config = {}

    #---------------------------------------------------------------------------
    # CHECK & SORT DATA
    #---------------------------------------------------------------------------
    tag_by_name_cfg_file = {values['name']: key for key, values in state_config.items()}

    for guid, name in tag_list.items():

        if guid in state_config.keys():
            if name not in state_config[guid]['name']:
                state_config[guid]['name'] = name
            continue

        if name in tag_by_name_cfg_file.keys():
            state_config[guid] = state_config[tag_by_name_cfg_file[name]]
            del state_config[tag_by_name_cfg_file[name]]
            continue

        state_config[guid] = {
            "color":    "",
            "name":     name,
            "symbol":   "",
            "event":    ""
        }

    #---------------------------------------------------------------------------
    # WRITE DATA TO FILE
    #---------------------------------------------------------------------------
    with open(filename, 'w') as f:
        json.dump(state_config, f)

    logger.debug('Writting state configuration to file: COMPLETE')

#---------------------------------------------------------------------------
# Get evernote data
#---------------------------------------------------------------------------
def get_evernote_data(key, gardening_notes, cfg):
    '''
    Function gets evernote data and updates gardening JSON file

    key
    gardening_notes Dictionary with results from file
    cfg             Configuration set up
                        "gardening_tag":    Evernote gardening tag parent
                        "notebook":         Evernote notebook
                        "plant_tag_id":     Evernote plant tag parent
                        "location_tag_id":  Evernote location tag parent
                        "state_tag_id":     Evernote plant state tag parent
                        "sand_box":         Use Evernote sandbox account,
                        "force_sync":       Force sync

    '''

    #---------------------------------------------------------------------------
    # SET UP LOGGER
    #---------------------------------------------------------------------------
    logger = logging.getLogger('root')

    #---------------------------------------------------------------------------
    # GET EVERNOTE DATA AND WRITE TO FILE
    #---------------------------------------------------------------------------
    try:

        client = EvernoteClient(token=key['AUTH_TOKEN'], sandbox=cfg['sand_box'])

        note_store  = client.get_note_store()
        user_store  = client.get_user_store()
        user        = user_store.getUser()
        user_public = user_store.getPublicUserInfo(user.username)

        #------------------------------------------------------------------------
        # Check evernote API
        #------------------------------------------------------------------------
        version_ok = user_store.checkVersion(
            "Evernote EDAMTest (Python)",
            UserStoreConstants.EDAM_VERSION_MAJOR,
            UserStoreConstants.EDAM_VERSION_MINOR)

        if not version_ok:
            logger.warning("Evernote API version is not up to date.")

        #------------------------------------------------------------------------
        # Check sync state
        #------------------------------------------------------------------------
        state = note_store.getSyncState()

        afterUSN = gardening_notes['lastUpdateCount']

        if cfg['force_sync']:
            afterUSN = 0

        if state.updateCount <= afterUSN:
            logger.info('Local file is in sync with Evernote. Exiting...')
            sys.exit()

        if afterUSN > 0:
            logger.info('Sync: INCREMENTAL')
        else:
            logger.info('Sync: FULL')

        #------------------------------------------------------------------------
        # Find notebook
        #------------------------------------------------------------------------
        notebooks       = note_store.listNotebooks()
        notebook_tag    = get_guid(notebooks, cfg['notebook']).keys()[0]

        #------------------------------------------------------------------------
        # Get all plant and location tags
        #------------------------------------------------------------------------
        tags            = note_store.listTags()
        gardening_tag   = get_guid(tags, cfg['gardening_tag']).keys()[0]

        gardening_notes['plant_tags']  = get_guid(tags, cfg['plant_tag_id'])

        gardening_loc_tag = get_guid(tags, cfg['location_tag_id']).keys()

        if len(gardening_loc_tag) > 1:
            logger.error("More than one Location Tag Parent found. Exiting...")
            sys.exit()

        gardening_notes['location_tags']  = get_tag_children(tags, gardening_loc_tag[0])

        gardening_state_tag = get_guid(tags, cfg['state_tag_id']).keys()

        if len(gardening_state_tag) > 1:
            logger.error("More than one State Tag Parent found. Exiting...")
            sys.exit()

        gardening_notes['state_tags']  = get_tag_children(tags, gardening_state_tag[0])

        #------------------------------------------------------------------------
        # Get all notes with specific tag from notebook
        #------------------------------------------------------------------------
        filter = NoteStore.NoteFilter()
        filter.tagGuids     = [gardening_tag]
        filter.notebookGuid = notebook_tag

        note_count = note_store.findNoteCounts(key['AUTH_TOKEN'], filter, False)
        note_count = note_count.notebookCounts[notebook_tag]

        if note_count <= 0:
            logger.error('No notes found. Exiting...')
            sys.exit()

        spec = NoteStore.SyncChunkFilter()
        spec.includeNotes = True
        spec.includeNoteResources = True
        spec.notebookGuids = [notebook_tag]

        logger.debug('Number of gardening notes found: {count}'.format(count= note_count))
        logger.debug('Downloading note metadata from evernote - START')

        note_list = []

        while True:
            logger.debug('USN {usn} / {count}'.format(usn= afterUSN, count= state.updateCount))
            download_data = note_store.getFilteredSyncChunk(key['AUTH_TOKEN'], afterUSN, 500, spec)

            afterUSN = download_data.chunkHighUSN

            if download_data.notes:
                note_list += download_data.notes
                logger.debug('Number of notes downloaded {count}'.format(count= len(download_data.notes)))

            if afterUSN >= state.updateCount or not afterUSN:
                break

        logger.debug('Downloading note metadata from evernote - COMPLETE')

        #------------------------------------------------------------------------
        # Add new update count
        #------------------------------------------------------------------------
        gardening_notes['lastUpdateCount'] = download_data.updateCount

        #------------------------------------------------------------------------
        # Organize data and write to file
        #------------------------------------------------------------------------
        for note in note_list:

            if note.deleted:
                continue

            if notebook_tag not in note.notebookGuid:
                continue

            if gardening_tag not in note.tagGuids:
                continue

            plant_tags = [tag for tag in note.tagGuids if tag in gardening_notes['plant_tags'].keys()]
            if not plant_tags:
                continue

            if note.updateSequenceNum < gardening_notes['notes'][note.guid]['USN']:
                continue

            res_guid = []
            if note.resources:
                for i in xrange(len(note.resources)):
                    if note.resources[i].mime == 'image/jpeg':
                        res_guid.append('{user}res/{r_guid}'.format(
                            user= user_public.webApiUrlPrefix,
                            r_guid= note.resources[i].guid))

            logger.info('Note updated: {guid} - {title}'.format(guid= note.guid, title=note.title))

            gardening_notes['notes'][note.guid] = {
                'created':  note.created,
                'title':    note.title,
                'tags':     note.tagGuids,
                'res':      res_guid,
                'USN':      note.updateSequenceNum,
                'link':     'https://{service}/shard/{shardId}/nl/{userId}/{noteGuid}/'.format(
                                    service= key['SERVICE'],
                                    shardId= user.shardId,
                                    userId= user.id,
                                    noteGuid= note.guid)
            }

        logger.debug('Sorting data: COMPLETE')
        logger.debug('Notes stored: {stored_no} / {total_no}'.format(
                                stored_no= len(gardening_notes['notes']),
                                total_no= note_count))

        logger.info('Sync Complete')

        return gardening_notes

    #---------------------------------------------------------------------------
    # Error Handling
    #---------------------------------------------------------------------------
    except Errors.EDAMSystemException, e:
        if e.errorCode == Errors.EDAMErrorCode.RATE_LIMIT_REACHED:
            logger.error("Rate limit reached")
            logger.error("Retry your request in %d seconds" % e.rateLimitDuration)
            sys.exit()

    except Exception, e:
        logger.error('Script error ({error_v}). Exiting...'.format(
            error_v=e), exc_info=True)
        # wd_err.set()
        sys.exit()

#===============================================================================
# MAIN
#===============================================================================
def main():

    '''
    Syncronizes data from evernote locally
    '''

    script_name = os.path.basename(sys.argv[0])
    folder_loc  = os.path.dirname(os.path.realpath(sys.argv[0]))
    folder_loc  = folder_loc.replace('scripts', '')

    #---------------------------------------------------------------------------
    # SET UP LOGGER
    #---------------------------------------------------------------------------
    logger = log.setup('root', '{folder}/logs/{script}.log'.format(
                                                    folder= folder_loc,
                                                    script= script_name[:-3]))
    logger.info('')
    logger.info('--- Script {script} Started ---'.format(script= script_name))

    #---------------------------------------------------------------------------
    # SET UP WATCHDOG
    #---------------------------------------------------------------------------
    err_file    = '{fl}/data/error.json'.format(fl= folder_loc)
    # wd_err      = wd.ErrorCode(err_file, '0007')

    #---------------------------------------------------------------------------
    # CHECK SCRIPT IS NOT ALREADY RUNNING
    #---------------------------------------------------------------------------
    if check_process.is_running(script_name):
        # wd_err.set()
        sys.exit()

    #---------------------------------------------------------------------------
    # CHECK USER REQUESTS
    #---------------------------------------------------------------------------
    key_file = 'key.json'
    sand_box = False
    force_sync = False

    if len(sys.argv) > 1:
        if '-s' in sys.argv:
            key_file = 'key_sand.json'
            sand_box = True
        if '-f' in sys.argv:
            force_sync = True

    #---------------------------------------------------------------------------
    # GET CONFIG & GARDENING DATA
    #---------------------------------------------------------------------------
    try:
        with open('{fl}/data/gardening.json'.format(fl= folder_loc), 'r') as f:
            gardening_notes = json.load(f)

    except Exception, e:
        logger.warning('Warning ({error_v}).'.format(error_v=e), exc_info=True)

        gardening_notes = {
            'lastUpdateCount': 0,
            'plant_tags': {},
            'state_tags': {},
            'location_tags': {},
            'notes': {}
        }

    try:
        with open('{fl}/data/config.json'.format(fl= folder_loc), 'r') as f:
            config = json.load(f)

        with open('{fl}/data/{fk}'.format(fl= folder_loc, fk=key_file), 'r') as f:
            key = json.load(f)

        cfg = {
            "gardening_tag":    config['evernote']['GARDENING_TAG'],
            "notebook":         config['evernote']['NOTEBOOK'],
            "plant_tag_id":     config['evernote']['PLANT_TAG_ID'],
            "location_tag_id":  config['evernote']['LOCATION_TAG_ID'],
            "state_tag_id":     config['evernote']['STATE_TAG_ID'],
            "sand_box":         False,
            "force_sync":       False
        }

        gardening_notes = get_evernote_data(key, gardening_notes, cfg)

        with open('{fl}/data/gardening.json'.format(fl= folder_loc), 'w') as f:
            json.dump(gardening_notes, f)

        logger.debug('Writting data to file: COMPLETE')

        check_state_tags(gardening_notes['state_tags'], '{fl}/data/state_tags.json'.format(fl= folder_loc))

    except Exception, e:
        logger.error('Error ({error_v}). Exiting...'.format(error_v=e), exc_info=True)
        # wd_err.set()
        sys.exit()

    finally:
        logger.info('--- Script Finished ---')

#===============================================================================
# Boiler plate
#===============================================================================
if __name__=='__main__':
    main()

<commit_msg>Add location filter<commit_after>#!usr/bin/env python

#===============================================================================
# Import modules
#===============================================================================
# Standard Library
import os
import sys
import datetime
import time
import json

# Third party modules
import hashlib
import binascii
import evernote.edam.notestore.NoteStore as NoteStore
import evernote.edam.userstore.constants as UserStoreConstants
import evernote.edam.type.ttypes as Types
import evernote.edam.error.ttypes as Errors
from evernote.api.client import EvernoteClient

# Application modules
import settings as s
import log
import logging
import check_process
# import watchdog as wd

#---------------------------------------------------------------------------
# Get guid for tag
#---------------------------------------------------------------------------
def get_guid(items, search_string):
    '''
    Function returns the guid from a evernote list

    items           Struct: List item as defined by Evernote
    search_string   string of tag name to search

    '''

    return {item.guid: item.name for item in items if item.name.find(search_string) > -1}

#---------------------------------------------------------------------------
# Get all children from parent tag
#---------------------------------------------------------------------------
def get_tag_children(tag_list, parent_guid):
    '''
    Function returns all children from the tag guid provided

    tag_list        Struct: Tag as defined by Evernote
    search_string   string of tag name to search

    '''

    return {tag.guid: tag.name for tag in tag_list if tag.parentGuid == parent_guid}

#---------------------------------------------------------------------------
 # Check state tags file is correct and up to date
#---------------------------------------------------------------------------
def check_state_tags(tag_list, filename):
    '''
    Function checks states in state tag file is up to date

    tag_list        State tags from evernote
    filename        State tag JSON file

    '''

    #---------------------------------------------------------------------------
    # SET UP LOGGER
    #---------------------------------------------------------------------------
    logger = logging.getLogger('root')

    #---------------------------------------------------------------------------
    # GET DATA FROM FILE
    #---------------------------------------------------------------------------
    try:
        with open(filename, 'r') as f:
            state_config = json.load(f)

    except Exception, e:
        logger.warning('Warning ({error_v}).'.format(error_v=e), exc_info=True)
        state_config = {}

    #---------------------------------------------------------------------------
    # CHECK & SORT DATA
    #---------------------------------------------------------------------------
    tag_by_name_cfg_file = {values['name']: key for key, values in state_config.items()}

    for guid, name in tag_list.items():

        if guid in state_config.keys():
            if name not in state_config[guid]['name']:
                state_config[guid]['name'] = name
            continue

        if name in tag_by_name_cfg_file.keys():
            state_config[guid] = state_config[tag_by_name_cfg_file[name]]
            del state_config[tag_by_name_cfg_file[name]]
            continue

        state_config[guid] = {
            "color":    "",
            "name":     name,
            "symbol":   "",
            "event":    ""
        }

    #---------------------------------------------------------------------------
    # WRITE DATA TO FILE
    #---------------------------------------------------------------------------
    with open(filename, 'w') as f:
        json.dump(state_config, f)

    logger.debug('Writting state configuration to file: COMPLETE')

#---------------------------------------------------------------------------
# Get evernote data
#---------------------------------------------------------------------------
def get_evernote_data(key, gardening_notes, cfg):
    '''
    Function gets evernote data and updates gardening JSON file

    key
    gardening_notes Dictionary with results from file
    cfg             Configuration set up
                        "gardening_tag":    Evernote gardening tag parent
                        "notebook":         Evernote notebook
                        "plant_tag_id":     Evernote plant tag parent
                        "location_tag_id":  Evernote location tag parent
                        "state_tag_id":     Evernote plant state tag parent
                        "sand_box":         Use Evernote sandbox account,
                        "force_sync":       Force sync

    '''

    #---------------------------------------------------------------------------
    # SET UP LOGGER
    #---------------------------------------------------------------------------
    logger = logging.getLogger('root')

    #---------------------------------------------------------------------------
    # GET EVERNOTE DATA AND WRITE TO FILE
    #---------------------------------------------------------------------------
    try:

        client = EvernoteClient(token=key['AUTH_TOKEN'], sandbox=cfg['sand_box'])

        note_store  = client.get_note_store()
        user_store  = client.get_user_store()
        user        = user_store.getUser()
        user_public = user_store.getPublicUserInfo(user.username)

        #------------------------------------------------------------------------
        # Check evernote API
        #------------------------------------------------------------------------
        version_ok = user_store.checkVersion(
            "Evernote EDAMTest (Python)",
            UserStoreConstants.EDAM_VERSION_MAJOR,
            UserStoreConstants.EDAM_VERSION_MINOR)

        if not version_ok:
            logger.warning("Evernote API version is not up to date.")

        #------------------------------------------------------------------------
        # Check sync state
        #------------------------------------------------------------------------
        state = note_store.getSyncState()

        afterUSN = gardening_notes['lastUpdateCount']

        if cfg['force_sync']:
            afterUSN = 0

        if state.updateCount <= afterUSN:
            logger.info('Local file is in sync with Evernote. Exiting...')
            sys.exit()

        if afterUSN > 0:
            logger.info('Sync: INCREMENTAL')
        else:
            logger.info('Sync: FULL')

        #------------------------------------------------------------------------
        # Find notebook
        #------------------------------------------------------------------------
        notebooks       = note_store.listNotebooks()
        notebook_tag    = get_guid(notebooks, cfg['notebook']).keys()[0]

        #------------------------------------------------------------------------
        # Get all plant and location tags
        #------------------------------------------------------------------------
        tags            = note_store.listTags()
        gardening_tag   = get_guid(tags, cfg['gardening_tag']).keys()[0]

        gardening_notes['plant_tags']  = get_guid(tags, cfg['plant_tag_id'])

        gardening_loc_tag = get_guid(tags, cfg['location_tag_id']).keys()

        if len(gardening_loc_tag) > 1:
            logger.error("More than one Location Tag Parent found. Exiting...")
            sys.exit()

        gardening_notes['location_tags']  = get_tag_children(tags, gardening_loc_tag[0])

        gardening_state_tag = get_guid(tags, cfg['state_tag_id']).keys()

        if len(gardening_state_tag) > 1:
            logger.error("More than one State Tag Parent found. Exiting...")
            sys.exit()

        gardening_notes['state_tags']  = get_tag_children(tags, gardening_state_tag[0])

        #------------------------------------------------------------------------
        # Get all notes with specific tag from notebook
        #------------------------------------------------------------------------
        filter = NoteStore.NoteFilter()
        filter.tagGuids     = [gardening_tag]
        filter.notebookGuid = notebook_tag

        note_count = note_store.findNoteCounts(key['AUTH_TOKEN'], filter, False)
        note_count = note_count.notebookCounts[notebook_tag]

        if note_count <= 0:
            logger.error('No notes found. Exiting...')
            sys.exit()

        spec = NoteStore.SyncChunkFilter()
        spec.includeNotes = True
        spec.includeNoteResources = True
        spec.notebookGuids = [notebook_tag]

        logger.debug('Number of gardening notes found: {count}'.format(count= note_count))
        logger.debug('Downloading note metadata from evernote - START')

        note_list = []

        while True:
            logger.debug('USN {usn} / {count}'.format(usn= afterUSN, count= state.updateCount))
            download_data = note_store.getFilteredSyncChunk(key['AUTH_TOKEN'], afterUSN, 500, spec)

            afterUSN = download_data.chunkHighUSN

            if download_data.notes:
                note_list += download_data.notes
                logger.debug('Number of notes downloaded {count}'.format(count= len(download_data.notes)))

            if afterUSN >= state.updateCount or not afterUSN:
                break

        logger.debug('Downloading note metadata from evernote - COMPLETE')

        #------------------------------------------------------------------------
        # Add new update count
        #------------------------------------------------------------------------
        gardening_notes['lastUpdateCount'] = download_data.updateCount

        #------------------------------------------------------------------------
        # Organize data and write to file
        #------------------------------------------------------------------------
        for note in note_list:

            if note.deleted:
                continue

            if notebook_tag not in note.notebookGuid:
                continue

            if gardening_tag not in note.tagGuids:
                continue

            all_plant_tags = gardening_notes['plant_tags'].keys()
            plant_tags = [tag for tag in note.tagGuids if tag in all_plant_tags]
            if not plant_tags:
                continue

            all_loc_tags = gardening_notes['location_tags'].keys()
            loc_tags = [tag for tag in note.tagGuids if loc in all_loc_tags]
            if not loc_tags:
                continue

            if note.updateSequenceNum < gardening_notes['notes'][note.guid]['USN']:
                continue

            res_guid = []
            if note.resources:
                for i in xrange(len(note.resources)):
                    if note.resources[i].mime == 'image/jpeg':
                        res_guid.append('{user}res/{r_guid}'.format(
                            user= user_public.webApiUrlPrefix,
                            r_guid= note.resources[i].guid))

            logger.info('Note updated: {guid} - {title}'.format(guid= note.guid, title=note.title))

            gardening_notes['notes'][note.guid] = {
                'created':  note.created,
                'title':    note.title,
                'tags':     note.tagGuids,
                'res':      res_guid,
                'USN':      note.updateSequenceNum,
                'link':     'https://{service}/shard/{shardId}/nl/{userId}/{noteGuid}/'.format(
                                    service= key['SERVICE'],
                                    shardId= user.shardId,
                                    userId= user.id,
                                    noteGuid= note.guid)
            }

        logger.debug('Sorting data: COMPLETE')
        logger.debug('Notes stored: {stored_no} / {total_no}'.format(
                                stored_no= len(gardening_notes['notes']),
                                total_no= note_count))

        logger.info('Sync Complete')

        return gardening_notes

    #---------------------------------------------------------------------------
    # Error Handling
    #---------------------------------------------------------------------------
    except Errors.EDAMSystemException, e:
        if e.errorCode == Errors.EDAMErrorCode.RATE_LIMIT_REACHED:
            logger.error("Rate limit reached")
            logger.error("Retry your request in %d seconds" % e.rateLimitDuration)
            sys.exit()

    except Exception, e:
        logger.error('Script error ({error_v}). Exiting...'.format(
            error_v=e), exc_info=True)
        # wd_err.set()
        sys.exit()

#===============================================================================
# MAIN
#===============================================================================
def main():

    '''
    Syncronizes data from evernote locally
    '''

    script_name = os.path.basename(sys.argv[0])
    folder_loc  = os.path.dirname(os.path.realpath(sys.argv[0]))
    folder_loc  = folder_loc.replace('scripts', '')

    #---------------------------------------------------------------------------
    # SET UP LOGGER
    #---------------------------------------------------------------------------
    logger = log.setup('root', '{folder}/logs/{script}.log'.format(
                                                    folder= folder_loc,
                                                    script= script_name[:-3]))
    logger.info('')
    logger.info('--- Script {script} Started ---'.format(script= script_name))

    #---------------------------------------------------------------------------
    # SET UP WATCHDOG
    #---------------------------------------------------------------------------
    err_file    = '{fl}/data/error.json'.format(fl= folder_loc)
    # wd_err      = wd.ErrorCode(err_file, '0007')

    #---------------------------------------------------------------------------
    # CHECK SCRIPT IS NOT ALREADY RUNNING
    #---------------------------------------------------------------------------
    if check_process.is_running(script_name):
        # wd_err.set()
        sys.exit()

    #---------------------------------------------------------------------------
    # CHECK USER REQUESTS
    #---------------------------------------------------------------------------
    key_file = 'key.json'
    sand_box = False
    force_sync = False

    if len(sys.argv) > 1:
        if '-s' in sys.argv:
            key_file = 'key_sand.json'
            sand_box = True
        if '-f' in sys.argv:
            force_sync = True

    #---------------------------------------------------------------------------
    # GET CONFIG & GARDENING DATA
    #---------------------------------------------------------------------------
    try:
        with open('{fl}/data/gardening.json'.format(fl= folder_loc), 'r') as f:
            gardening_notes = json.load(f)

    except Exception, e:
        logger.warning('Warning ({error_v}).'.format(error_v=e), exc_info=True)

        gardening_notes = {
            'lastUpdateCount': 0,
            'plant_tags': {},
            'state_tags': {},
            'location_tags': {},
            'notes': {}
        }

    try:
        with open('{fl}/data/config.json'.format(fl= folder_loc), 'r') as f:
            config = json.load(f)

        with open('{fl}/data/{fk}'.format(fl= folder_loc, fk=key_file), 'r') as f:
            key = json.load(f)

        cfg = {
            "gardening_tag":    config['evernote']['GARDENING_TAG'],
            "notebook":         config['evernote']['NOTEBOOK'],
            "plant_tag_id":     config['evernote']['PLANT_TAG_ID'],
            "location_tag_id":  config['evernote']['LOCATION_TAG_ID'],
            "state_tag_id":     config['evernote']['STATE_TAG_ID'],
            "sand_box":         False,
            "force_sync":       False
        }

        gardening_notes = get_evernote_data(key, gardening_notes, cfg)

        with open('{fl}/data/gardening.json'.format(fl= folder_loc), 'w') as f:
            json.dump(gardening_notes, f)

        logger.debug('Writting data to file: COMPLETE')

        check_state_tags(gardening_notes['state_tags'], '{fl}/data/state_tags.json'.format(fl= folder_loc))

    except Exception, e:
        logger.error('Error ({error_v}). Exiting...'.format(error_v=e), exc_info=True)
        # wd_err.set()
        sys.exit()

    finally:
        logger.info('--- Script Finished ---')

#===============================================================================
# Boiler plate
#===============================================================================
if __name__=='__main__':
    main()

<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright (C) 2012-2014 Zeex
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import cidl
import itertools
import os
import sys

idl_to_c_type_in = {
  'int'    : 'int',
  'bool'   : 'bool',
  'float'  : 'float',
  'string' : 'const char *',
}

idl_to_c_type_out = {
  'int'    : 'int *',
  'bool'   : 'bool *',
  'float'  : 'float *',
  'string' : 'char *',
}

class Parameter(cidl.Parameter):
  def __init__(self, type, name, default=None, attrlist=None):
    cidl.Parameter.__init__(self, type, name, default, attrlist)

  @property
  def c_type(self):
    try:
      if self.is_out:
        return idl_to_c_type_out[self.type]
      if self.is_in:
        return idl_to_c_type_in[self.type]
    except KeyError:
      if self.is_out:
        return '%s *' % self.type
      if self.is_in:
        return self.type
    return None

  @property
  def is_ref(self):
    return self.is_out or self.type == 'string'

  @property
  def is_value(self):
    return not self.is_ref

class Value(cidl.Value):
  def __init__(self, type, data):
    cidl.Value.__init__(self, type, data)

  def as_string(self):
    if self.is_bool():
      return ('false', 'true')[self.data]
    elif self.is_int():
      if self.is_hex():
        return '0x%s' % hex(self.data)[2:].upper()
      elif self.is_oct():
        return '%s' % oct(self.data)
      return '%s' % int(self.data)
    elif self.is_float():
      return '%s' % self.data
    elif self.is_string():
      return '"%s"' % self.data
    return None

  def __str__(self):
    string = self.as_string()
    if string is None:
      return '_codegen_invalid_literal_'
    return string

class ParamList:
  def __init__(self, params):
    self._params = params

  def __str__(self):
    return ', '.join(['%s %s' % (p.c_type, p.name) for p in
                      filter(lambda x: x.default is None, self._params)])

class ArgList:
  def __init__(self, params):
    self._params = params

  def __str__(self):
    return ', '.join(['%s' % p.name for p in self._params])

def previous_and_next(iterable):
    prevs, items, nexts = itertools.tee(iterable, 3)
    prevs = itertools.chain([None], prevs)
    nexts = itertools.chain(itertools.islice(nexts, 1, None), [None])
    try:
      return itertools.izip(prevs, items, nexts)
    except AttributeError:
      return zip(prevs, items, nexts)

def generate_api_file(module_name, idl, file):
  for f in filter(lambda x: x.has_attr('native'), idl.functions):
    file.write('sampgdk_%s\n' % f.name)

def generate_header_file(module_name, idl, file):
  file.write('#include <sampgdk/bool.h>\n')
  file.write('#include <sampgdk/export.h>\n')
  file.write('#include <sampgdk/types.h>\n')
  file.write('\n')

  natives = list(filter(lambda x: x.has_attr('native'),
                        idl.functions))
  file.write('#ifndef DOXYGEN\n')
  for func in natives:
    generate_native_alias(file, func)
  file.write('#endif\n\n')

  for func in natives:
    generate_native_decl(file, func)
  file.write('\n')

  for const in idl.constants:
    generate_constant(file, const)
  file.write('\n')

  callbacks = list(filter(lambda x: x.has_attr('callback'),
                          idl.functions))
  for func in callbacks:
    generate_callback_decl(file, func)

def generate_source_file(module_name, idl, file):
  file.write('#include <sampgdk/%s.h>\n' % module_name)
  file.write('\n')
  file.write('#include "internal/callback.h"\n')
  file.write('#include "internal/fakeamx.h"\n')
  file.write('#include "internal/init.h"\n')
  file.write('#include "internal/likely.h"\n')
  file.write('#include "internal/native.h"\n')
  file.write('#include "internal/param.h"\n')
  file.write('\n')

  natives = list(filter(lambda x: x.has_attr('native') and not
                                  x.has_attr('noimpl'),
                        idl.functions))
  for func in natives:
    generate_native_impl(file, func)
    file.write('\n')

  callbacks = list(filter(lambda x: x.has_attr('callback'), idl.functions))
  for func in callbacks:
    generate_callback_impl(file, func);
    file.write('\n')

  file.write('SAMPGDK_MODULE_INIT(%s) {\n' % module_name)
  sorted_callbacks = list(sorted(callbacks, key=lambda x: x.name, reverse=True))
  if sorted_callbacks:
    file.write('  int error;\n')
  for func in sorted_callbacks:
    file.write('  if ((error = sampgdk_callback_register("%s", _%s)) < 0) {\n' %
               (func.name, func.name))
    file.write('    return error;\n')
    file.write('  }\n')
  file.write('  return 0;\n')
  file.write('}\n\n')

  file.write('SAMPGDK_MODULE_CLEANUP(%s) {\n' % module_name)
  for func in sorted_callbacks:
    file.write('  sampgdk_callback_unregister("%s");\n' % func.name)
  file.write('}\n\n')

def generate_constant(file, const):
  file.write('#define %s (%s)\n' % (const.name, const.value))

def generate_native_decl(file, func):
  file.write('\n')
  file.write('/**\n')
  file.write(' * \\ingroup natives\n')
  file.write(' * \\see '
             '%s on SA-MP Wiki\n' % (func.name, func.name))
  file.write(' */\n')
  file.write('SAMPGDK_NATIVE(%s, %s(%s));\n'
             % (func.type, func.name, ParamList(func.params)))

def generate_native_alias(file, func):
  file.write('#undef  %s\n' % func.name)
  file.write('#define %s sampgdk_%s\n' % (func.name, func.name))

def generate_native_impl(file, func):
  file.write('SAMPGDK_NATIVE(%s, %s(%s)) {\n' %
             (func.type, func.name, ParamList(func.params)))
  file.write('  static AMX_NATIVE native;\n')
  file.write('  cell retval;\n')

  if func.params:
    file.write('  cell params[%d];\n' % (len(func.params) + 1))

    for p in filter(lambda p: p.is_ref, func.params):
      file.write('  cell %s_;\n' % p.name)

  file.write('  if (unlikely(native == NULL)) {\n')
  file.write('    native = sampgdk_native_find_warn_stub("%s");\n' % func.name)
  file.write('  }\n')

  if func.params:
    for pprev, p, pnext in previous_and_next(func.params):
      if p.is_ref:
        if p.default is not None:
          value = p.default
        else:
          value = p.name
        if p.type == 'string':
          if p.is_out:
            file.write('  sampgdk_fakeamx_push(%s, &%s_);\n' %
                       (pnext.name, p.name))
          else:
            file.write('  sampgdk_fakeamx_push_string(%s, NULL, &%s_);\n' %
                       (value, p.name))
        else:
          file.write('  sampgdk_fakeamx_push(1, &%s_);\n' % p.name)

    file.write('  params[0] = %d * sizeof(cell);\n' % len(func.params))
    for index, p in enumerate(func.params, 1):
      if p.is_value:
        if p.default is not None:
          value = p.default
        else:
          value = p.name
        file.write('  params[%d] = %s;\n' % (index,
          {
            'int'   : '(cell)%s' % value,
            'bool'  : '(cell)%s' % value,
            'float' : 'amx_ftoc(%s)' % value,
          }[p.type]
        ))
      else:
        file.write('  params[%d] = %s_;\n' % (index, p.name))

  file.write('  retval = native(sampgdk_fakeamx_amx(), %s);\n' %
             ('NULL', 'params')[bool(func.params)])

  if func.params:
    for pprev, p, pnext in previous_and_next(func.params):
      if p.is_out:
        if p.type == 'string':
          file.write('  sampgdk_fakeamx_get_string(%s_, %s, %s);\n' %
                     (p.name, p.name, pnext.name))
        else:
          file.write('  sampgdk_fakeamx_get_%s(%s_, %s);\n' % (
            {
              'int'    : 'cell',
              'bool'   : 'bool',
              'float'  : 'float',
              'string' : 'string',
            }[p.type]
          ,
          p.name, p.name))

    for p in reversed(list(filter(lambda p: p.is_ref, func.params))):
      file.write('  sampgdk_fakeamx_pop(%s_);\n' % p.name)

  file.write('  return %s(retval);\n' % ({
      'int'   : '(int)',
      'bool'  : '!!',
      'float' : 'amx_ctof',
    }[func.type]
  ))

  file.write('}\n')

def generate_callback_decl(file, func):
  file.write('\n')
  file.write('/**\n')
  file.write(' * \\ingroup callbacks\n')
  file.write(' * \\see '
             '%s on SA-MP Wiki\n' % (func.name, func.name))
  file.write(' */\n')
  file.write('SAMPGDK_CALLBACK(%s, %s(%s));\n' %
               (func.type, func.name, ParamList(func.params)))

def generate_callback_impl(file, func):
  file.write('typedef %s (SAMPGDK_CALLBACK_CALL *%s_func)(%s);\n' %
             (func.type, func.name, ParamList(func.params)))
  file.write('static bool _%s(AMX *amx, void *callback, cell *retval)'
             ' {\n' % func.name)

  badret = func.get_attr('badret')
  if badret.value is not None:
    file.write('  bool retval_;\n')

  for p in func.params:
    file.write('  %s %s;\n' % (p.c_type, p.name))

  for index, p in enumerate(func.params):
    file.write('  sampgdk_param_get_%s(amx, %d, (void *)&%s);\n' % ({
        'int'    : 'cell',
        'bool'   : 'bool',
        'float'  : 'float',
        'string' : 'string',
      }[p.type], index, p.name)
    )

  if badret.value is not None:
    file.write('  retval_ = ((%s_func)callback)(%s);\n' %
               (func.name, ArgList(func.params)))
    file.write('  if (retval != NULL) {\n')
    file.write('    *retval = (cell)retval_;\n')
    file.write('  }\n')
  else:
    file.write('  ((%s_func)callback)(%s);\n' %
                 (func.name, ArgList(func.params)))

  for p in filter(lambda p: p.type == 'string', func.params):
    file.write('  free((void *)%s);\n' % p.name)

  if badret.value is not None:
    file.write('  return (retval_ != %s);\n' % badret.value)
  else:
    file.write('  return true;\n')

  file.write('}\n')

def parse_args(argv):
  argparser = argparse.ArgumentParser()
  argparser.add_argument('--module', dest='module_name', required=True)
  argparser.add_argument('--idl', dest='idl_file', required=True)
  argparser.add_argument('--api', dest='api_file')
  argparser.add_argument('--header', dest='header_file')
  argparser.add_argument('--source', dest='source_file')
  return argparser.parse_args(argv)

def main(argv):
  args = parse_args(argv[1:])

  try:
    idlparser = cidl.Parser(value_class=Value, param_class=Parameter)
    idl = idlparser.parse(open(args.idl_file, 'r').read())

    def ensure_dir(filename):
      directory = os.path.dirname(filename)
      if not os.path.exists(directory):
        os.makedirs(directory)

    api = None
    if args.api_file is not None:
      ensure_dir(args.api_file)
      with open(args.api_file, 'w') as file:
        generate_api_file(args.module_name, idl, file)

    header = None
    if args.header_file is not None:
      ensure_dir(args.header_file)
      with open(args.header_file, 'w') as file:
        generate_header_file(args.module_name, idl, file)

    source = None
    if args.source_file is not None:
      ensure_dir(args.source_file)
      with open(args.source_file, 'w') as file:
        generate_source_file(args.module_name, idl, file)

  except cidl.Error as e:
    sys.stderr.write('%s\n' % e)

if __name__ == '__main__':
  main(sys.argv)
<commit_msg>Change _func to _callback fo consistency<commit_after>#!/usr/bin/env python
#
# Copyright (C) 2012-2014 Zeex
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import cidl
import itertools
import os
import sys

idl_to_c_type_in = {
  'int'    : 'int',
  'bool'   : 'bool',
  'float'  : 'float',
  'string' : 'const char *',
}

idl_to_c_type_out = {
  'int'    : 'int *',
  'bool'   : 'bool *',
  'float'  : 'float *',
  'string' : 'char *',
}

class Parameter(cidl.Parameter):
  def __init__(self, type, name, default=None, attrlist=None):
    cidl.Parameter.__init__(self, type, name, default, attrlist)

  @property
  def c_type(self):
    try:
      if self.is_out:
        return idl_to_c_type_out[self.type]
      if self.is_in:
        return idl_to_c_type_in[self.type]
    except KeyError:
      if self.is_out:
        return '%s *' % self.type
      if self.is_in:
        return self.type
    return None

  @property
  def is_ref(self):
    return self.is_out or self.type == 'string'

  @property
  def is_value(self):
    return not self.is_ref

class Value(cidl.Value):
  def __init__(self, type, data):
    cidl.Value.__init__(self, type, data)

  def as_string(self):
    if self.is_bool():
      return ('false', 'true')[self.data]
    elif self.is_int():
      if self.is_hex():
        return '0x%s' % hex(self.data)[2:].upper()
      elif self.is_oct():
        return '%s' % oct(self.data)
      return '%s' % int(self.data)
    elif self.is_float():
      return '%s' % self.data
    elif self.is_string():
      return '"%s"' % self.data
    return None

  def __str__(self):
    string = self.as_string()
    if string is None:
      return '_codegen_invalid_literal_'
    return string

class ParamList:
  def __init__(self, params):
    self._params = params

  def __str__(self):
    return ', '.join(['%s %s' % (p.c_type, p.name) for p in
                      filter(lambda x: x.default is None, self._params)])

class ArgList:
  def __init__(self, params):
    self._params = params

  def __str__(self):
    return ', '.join(['%s' % p.name for p in self._params])

def previous_and_next(iterable):
    prevs, items, nexts = itertools.tee(iterable, 3)
    prevs = itertools.chain([None], prevs)
    nexts = itertools.chain(itertools.islice(nexts, 1, None), [None])
    try:
      return itertools.izip(prevs, items, nexts)
    except AttributeError:
      return zip(prevs, items, nexts)

def generate_api_file(module_name, idl, file):
  for f in filter(lambda x: x.has_attr('native'), idl.functions):
    file.write('sampgdk_%s\n' % f.name)

def generate_header_file(module_name, idl, file):
  file.write('#include <sampgdk/bool.h>\n')
  file.write('#include <sampgdk/export.h>\n')
  file.write('#include <sampgdk/types.h>\n')
  file.write('\n')

  natives = list(filter(lambda x: x.has_attr('native'),
                        idl.functions))
  file.write('#ifndef DOXYGEN\n')
  for func in natives:
    generate_native_alias(file, func)
  file.write('#endif\n\n')

  for func in natives:
    generate_native_decl(file, func)
  file.write('\n')

  for const in idl.constants:
    generate_constant(file, const)
  file.write('\n')

  callbacks = list(filter(lambda x: x.has_attr('callback'),
                          idl.functions))
  for func in callbacks:
    generate_callback_decl(file, func)

def generate_source_file(module_name, idl, file):
  file.write('#include <sampgdk/%s.h>\n' % module_name)
  file.write('\n')
  file.write('#include "internal/callback.h"\n')
  file.write('#include "internal/fakeamx.h"\n')
  file.write('#include "internal/init.h"\n')
  file.write('#include "internal/likely.h"\n')
  file.write('#include "internal/native.h"\n')
  file.write('#include "internal/param.h"\n')
  file.write('\n')

  natives = list(filter(lambda x: x.has_attr('native') and not
                                  x.has_attr('noimpl'),
                        idl.functions))
  for func in natives:
    generate_native_impl(file, func)
    file.write('\n')

  callbacks = list(filter(lambda x: x.has_attr('callback'), idl.functions))
  for func in callbacks:
    generate_callback_impl(file, func);
    file.write('\n')

  file.write('SAMPGDK_MODULE_INIT(%s) {\n' % module_name)
  sorted_callbacks = list(sorted(callbacks, key=lambda x: x.name, reverse=True))
  if sorted_callbacks:
    file.write('  int error;\n')
  for func in sorted_callbacks:
    file.write('  if ((error = sampgdk_callback_register("%s", _%s)) < 0) {\n' %
               (func.name, func.name))
    file.write('    return error;\n')
    file.write('  }\n')
  file.write('  return 0;\n')
  file.write('}\n\n')

  file.write('SAMPGDK_MODULE_CLEANUP(%s) {\n' % module_name)
  for func in sorted_callbacks:
    file.write('  sampgdk_callback_unregister("%s");\n' % func.name)
  file.write('}\n\n')

def generate_constant(file, const):
  file.write('#define %s (%s)\n' % (const.name, const.value))

def generate_native_decl(file, func):
  file.write('\n')
  file.write('/**\n')
  file.write(' * \\ingroup natives\n')
  file.write(' * \\see '
             '%s on SA-MP Wiki\n' % (func.name, func.name))
  file.write(' */\n')
  file.write('SAMPGDK_NATIVE(%s, %s(%s));\n'
             % (func.type, func.name, ParamList(func.params)))

def generate_native_alias(file, func):
  file.write('#undef  %s\n' % func.name)
  file.write('#define %s sampgdk_%s\n' % (func.name, func.name))

def generate_native_impl(file, func):
  file.write('SAMPGDK_NATIVE(%s, %s(%s)) {\n' %
             (func.type, func.name, ParamList(func.params)))
  file.write('  static AMX_NATIVE native;\n')
  file.write('  cell retval;\n')

  if func.params:
    file.write('  cell params[%d];\n' % (len(func.params) + 1))

    for p in filter(lambda p: p.is_ref, func.params):
      file.write('  cell %s_;\n' % p.name)

  file.write('  if (unlikely(native == NULL)) {\n')
  file.write('    native = sampgdk_native_find_warn_stub("%s");\n' % func.name)
  file.write('  }\n')

  if func.params:
    for pprev, p, pnext in previous_and_next(func.params):
      if p.is_ref:
        if p.default is not None:
          value = p.default
        else:
          value = p.name
        if p.type == 'string':
          if p.is_out:
            file.write('  sampgdk_fakeamx_push(%s, &%s_);\n' %
                       (pnext.name, p.name))
          else:
            file.write('  sampgdk_fakeamx_push_string(%s, NULL, &%s_);\n' %
                       (value, p.name))
        else:
          file.write('  sampgdk_fakeamx_push(1, &%s_);\n' % p.name)

    file.write('  params[0] = %d * sizeof(cell);\n' % len(func.params))
    for index, p in enumerate(func.params, 1):
      if p.is_value:
        if p.default is not None:
          value = p.default
        else:
          value = p.name
        file.write('  params[%d] = %s;\n' % (index,
          {
            'int'   : '(cell)%s' % value,
            'bool'  : '(cell)%s' % value,
            'float' : 'amx_ftoc(%s)' % value,
          }[p.type]
        ))
      else:
        file.write('  params[%d] = %s_;\n' % (index, p.name))

  file.write('  retval = native(sampgdk_fakeamx_amx(), %s);\n' %
             ('NULL', 'params')[bool(func.params)])

  if func.params:
    for pprev, p, pnext in previous_and_next(func.params):
      if p.is_out:
        if p.type == 'string':
          file.write('  sampgdk_fakeamx_get_string(%s_, %s, %s);\n' %
                     (p.name, p.name, pnext.name))
        else:
          file.write('  sampgdk_fakeamx_get_%s(%s_, %s);\n' % (
            {
              'int'    : 'cell',
              'bool'   : 'bool',
              'float'  : 'float',
              'string' : 'string',
            }[p.type]
          ,
          p.name, p.name))

    for p in reversed(list(filter(lambda p: p.is_ref, func.params))):
      file.write('  sampgdk_fakeamx_pop(%s_);\n' % p.name)

  file.write('  return %s(retval);\n' % ({
      'int'   : '(int)',
      'bool'  : '!!',
      'float' : 'amx_ctof',
    }[func.type]
  ))

  file.write('}\n')

def generate_callback_decl(file, func):
  file.write('\n')
  file.write('/**\n')
  file.write(' * \\ingroup callbacks\n')
  file.write(' * \\see '
             '%s on SA-MP Wiki\n' % (func.name, func.name))
  file.write(' */\n')
  file.write('SAMPGDK_CALLBACK(%s, %s(%s));\n' %
               (func.type, func.name, ParamList(func.params)))

def generate_callback_impl(file, func):
  file.write('typedef %s (SAMPGDK_CALLBACK_CALL *%s_callback)(%s);\n' %
             (func.type, func.name, ParamList(func.params)))
  file.write('static bool _%s(AMX *amx, void *callback, cell *retval)'
             ' {\n' % func.name)

  badret = func.get_attr('badret')
  if badret.value is not None:
    file.write('  bool retval_;\n')

  for p in func.params:
    file.write('  %s %s;\n' % (p.c_type, p.name))

  for index, p in enumerate(func.params):
    file.write('  sampgdk_param_get_%s(amx, %d, (void *)&%s);\n' % ({
        'int'    : 'cell',
        'bool'   : 'bool',
        'float'  : 'float',
        'string' : 'string',
      }[p.type], index, p.name)
    )

  if badret.value is not None:
    file.write('  retval_ = ((%s_callback)callback)(%s);\n' %
               (func.name, ArgList(func.params)))
    file.write('  if (retval != NULL) {\n')
    file.write('    *retval = (cell)retval_;\n')
    file.write('  }\n')
  else:
    file.write('  ((%s_callback)callback)(%s);\n' %
                 (func.name, ArgList(func.params)))

  for p in filter(lambda p: p.type == 'string', func.params):
    file.write('  free((void *)%s);\n' % p.name)

  if badret.value is not None:
    file.write('  return (retval_ != %s);\n' % badret.value)
  else:
    file.write('  return true;\n')

  file.write('}\n')

def parse_args(argv):
  argparser = argparse.ArgumentParser()
  argparser.add_argument('--module', dest='module_name', required=True)
  argparser.add_argument('--idl', dest='idl_file', required=True)
  argparser.add_argument('--api', dest='api_file')
  argparser.add_argument('--header', dest='header_file')
  argparser.add_argument('--source', dest='source_file')
  return argparser.parse_args(argv)

def main(argv):
  args = parse_args(argv[1:])

  try:
    idlparser = cidl.Parser(value_class=Value, param_class=Parameter)
    idl = idlparser.parse(open(args.idl_file, 'r').read())

    def ensure_dir(filename):
      directory = os.path.dirname(filename)
      if not os.path.exists(directory):
        os.makedirs(directory)

    api = None
    if args.api_file is not None:
      ensure_dir(args.api_file)
      with open(args.api_file, 'w') as file:
        generate_api_file(args.module_name, idl, file)

    header = None
    if args.header_file is not None:
      ensure_dir(args.header_file)
      with open(args.header_file, 'w') as file:
        generate_header_file(args.module_name, idl, file)

    source = None
    if args.source_file is not None:
      ensure_dir(args.source_file)
      with open(args.source_file, 'w') as file:
        generate_source_file(args.module_name, idl, file)

  except cidl.Error as e:
    sys.stderr.write('%s\n' % e)

if __name__ == '__main__':
  main(sys.argv)
<|endoftext|>


<commit_before>#!/usr/bin/env python

from pcs.packets.localhost import *
from pcs.packets.ethernet import *
from pcs.packets.ipv4 import *
from pcs.packets.igmpv2 import *
from pcs.packets.payload import *
from pcs import *
from time import sleep

# Send an IGMPv2 general or group-specific query.

def main():

    from optparse import OptionParser

    parser = OptionParser()

    parser.add_option("-I", "--ether_iface",
                      dest="ether_iface", default=None,
                      help="The name of the source interface.")

    parser.add_option("-e", "--ether_source",
                      dest="ether_source", default=None,
                      help="The host Ethernet source address.")

    parser.add_option("-s", "--ip_source",
                      dest="ip_source", default=None,
                      help="The IP source address.")

    parser.add_option("-G", "--igmp_group",
                      dest="igmp_group", default=None,
                      help="The IPv4 group for a group-specific query. "
			   "If omitted, send a general query.")

    parser.add_option("-M", "--maxresp",
                      dest="igmp_maxresp", default=None,
                      help="The maximum time for end-stations to respond "
			   "(in seconds).")

    parser.add_option("-c", "--count",
                      dest="count", default=None,
                      help="Stop after receiving at least count responses.")

    (options, args) = parser.parse_args()

    if options.ether_iface is None or \
       options.ether_source is None or \
       options.ip_source is None or \
       options.count is None:
        print "Non-optional argument missing."
        return

    maxresp = 3 * 10
    if options.igmp_maxresp is not None:
        maxresp = int(options.igmp_maxresp) * 10    # in units of deciseconds

    # Set up the vanilla packet
    ether = ethernet()
    ether.type = 0x0800
    ether.src = ether_atob(options.ether_source)
    ether.dst = "\x01\x00\x5e\x00\x00\x01"

    ip = ipv4()
    ip.version = 4
    ip.hlen = 5
    ip.tos = 0
    ip.id = 0
    ip.flags = 0x02		# DF (yes, it's byte swapped here).
    ip.offset = 0
    ip.ttl = 1
    ip.protocol = IPPROTO_IGMP
    ip.src = inet_atol(options.ip_source)

    igmp = igmpv2()
    igmp.type = IGMP_HOST_MEMBERSHIP_QUERY
    igmp.code = maxresp

    if options.igmp_group is None:
	# General query.
    	ip.dst = inet_atol("224.0.0.1")		# XXX should be a constant
        igmp.group = 0
    else:
	# Group-specific query.
    	ip.dst = inet_atol(options.igmp_group)
        igmp.group = ip.dst

    igmp_packet = Chain([igmp])
    igmp.checksum = igmp_packet.calc_checksum()

    # Queries don't contain the Router Alert option as they are
    # destined for end stations, not routers.

    ip.length = len(ip.bytes) + len(igmp.bytes)
    ip.checksum = ip.cksum()

    packet = Chain([ether, ip, igmp])
    packet.encode()

    input = PcapConnector(options.ether_iface)
    input.setfilter("igmp")

    output = PcapConnector(options.ether_iface)
    out = output.write(packet.bytes, len(packet.bytes))

    #
    # Wait for up to 'count' responses to the query to arrive and print them.
    #
    count = int(options.count)
    while count > 0:
        packet = input.readpkt()
        chain = packet.chain()
	if chain.packets[2].type == IGMP_v2_HOST_MEMBERSHIP_REPORT:
	    #print chain.packets[2].println()
	    print "%s is in %s" % \
	        (inet_ntop(AF_INET, struct.pack('!L', chain.packets[1].src)), \
	         inet_ntop(AF_INET, struct.pack('!L', chain.packets[2].group)))
	    count -= 1

main()
<commit_msg>Use symbolic constants from ipv4.py.<commit_after>#!/usr/bin/env python

from pcs.packets.localhost import *
from pcs.packets.ethernet import *
from pcs.packets.ipv4 import *
from pcs.packets.igmpv2 import *
from pcs.packets.payload import *
from pcs import *
from time import sleep

# Send an IGMPv2 general or group-specific query.

def main():

    from optparse import OptionParser

    parser = OptionParser()

    parser.add_option("-I", "--ether_iface",
                      dest="ether_iface", default=None,
                      help="The name of the source interface.")

    parser.add_option("-e", "--ether_source",
                      dest="ether_source", default=None,
                      help="The host Ethernet source address.")

    parser.add_option("-s", "--ip_source",
                      dest="ip_source", default=None,
                      help="The IP source address.")

    parser.add_option("-G", "--igmp_group",
                      dest="igmp_group", default=None,
                      help="The IPv4 group for a group-specific query. "
			   "If omitted, send a general query.")

    parser.add_option("-M", "--maxresp",
                      dest="igmp_maxresp", default=None,
                      help="The maximum time for end-stations to respond "
			   "(in seconds).")

    parser.add_option("-c", "--count",
                      dest="count", default=None,
                      help="Stop after receiving at least count responses.")

    (options, args) = parser.parse_args()

    if options.ether_iface is None or \
       options.ether_source is None or \
       options.ip_source is None or \
       options.count is None:
        print "Non-optional argument missing."
        return

    maxresp = 3 * 10
    if options.igmp_maxresp is not None:
        maxresp = int(options.igmp_maxresp) * 10    # in units of deciseconds

    # Set up the vanilla packet
    ether = ethernet()
    ether.type = 0x0800
    ether.src = ether_atob(options.ether_source)
    ether.dst = "\x01\x00\x5e\x00\x00\x01"

    ip = ipv4()
    ip.version = 4
    ip.hlen = 5
    ip.tos = 0
    ip.id = 0
    ip.flags = 0x02		# DF (yes, it's byte swapped here).
    ip.offset = 0
    ip.ttl = 1
    ip.protocol = IPPROTO_IGMP
    ip.src = inet_atol(options.ip_source)

    igmp = igmpv2()
    igmp.type = IGMP_HOST_MEMBERSHIP_QUERY
    igmp.code = maxresp

    if options.igmp_group is None:
	# General query.
    	ip.dst = INADDR_ALLHOSTS_GROUP
        igmp.group = INADDR_ANY
    else:
	# Group-specific query.
    	ip.dst = inet_atol(options.igmp_group)
        igmp.group = ip.dst

    igmp_packet = Chain([igmp])
    igmp.checksum = igmp_packet.calc_checksum()

    # Queries don't contain the Router Alert option as they are
    # destined for end stations, not routers.

    ip.length = len(ip.bytes) + len(igmp.bytes)
    ip.checksum = ip.cksum()

    packet = Chain([ether, ip, igmp])
    packet.encode()

    input = PcapConnector(options.ether_iface)
    input.setfilter("igmp")

    output = PcapConnector(options.ether_iface)
    out = output.write(packet.bytes, len(packet.bytes))

    #
    # Wait for up to 'count' responses to the query to arrive and print them.
    #
    count = int(options.count)
    while count > 0:
        packet = input.readpkt()
        chain = packet.chain()
	if chain.packets[2].type == IGMP_v2_HOST_MEMBERSHIP_REPORT:
	    #print chain.packets[2].println()
	    print "%s is in %s" % \
	        (inet_ntop(AF_INET, struct.pack('!L', chain.packets[1].src)), \
	         inet_ntop(AF_INET, struct.pack('!L', chain.packets[2].group)))
	    count -= 1

main()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Script to handle interwiki links based on Wikibase.

This script connects pages to Wikibase items using language links on the page.
If multiple language links are present, and they are connected to different
items, the bot skips. After connecting the page to an item, language links
can be removed from the page.

These command line parameters can be used to specify which pages to work on:

&params;

Furthermore, the following command line parameters are supported:

-clean            Clean pages.

-create           Create items only.

-summary:         Use your own edit summary for cleaning the page.
"""

# (C) Pywikibot team, 2015-2017
#
# Distributed under the terms of the MIT license.
#
from __future__ import unicode_literals, absolute_import

__version__ = '$Id$'
#

import pywikibot

from pywikibot import pagegenerators, output, warning
from pywikibot.bot import ExistingPageBot, SingleSiteBot, suggest_help

# This is required for the text that is shown when you run this script
# with the parameter -help.
docuReplacements = {
    '&params;': pagegenerators.parameterHelp,
}

# Allowed namespaces. main, project, template, category
namespaces = [0, 4, 10, 14]

# TODO: Some templates on pages, like csd, inuse and afd templates,
# should cause the bot to skip the page, see T134497

class IWBot(ExistingPageBot, SingleSiteBot):

    """The bot for interwiki."""

    def __init__(self, **kwargs):
        """Construct the bot."""
        self.availableOptions.update({
            'clean': False,
            'create': False,
            'summary': None,
            'ignore_ns': False,  # used by interwikidata_tests only
        })
        super(IWBot, self).__init__(**kwargs)
        if not self.site.has_data_repository:
            raise ValueError('{site} does not have a data repository, '
                             'use interwiki.py instead.'.format(
                                 site=self.site))
        self.repo = self.site.data_repository()
        if not self.getOption('summary'):
            self.options['summary'] = pywikibot.i18n.twtranslate(
                self.site, 'interwikidata-clean-summary')

    def treat_page(self):
        """Check page."""
        if (self.current_page.namespace() not in namespaces and
                not self.getOption('ignore_ns')):
            output('{page} is not in allowed namespaces, skipping'
                   .format(page=self.current_page.title(
                       asLink=True)))
            return False
        self.iwlangs = pywikibot.textlib.getLanguageLinks(
            self.current_page.text, insite=self.current_page.site)
        if not self.iwlangs:
            output('No interlanguagelinks on {page}'.format(
                page=self.current_page.title(asLink=True)))
            return False
        try:
            item = pywikibot.ItemPage.fromPage(self.current_page)
        except pywikibot.NoPage:
            item = None

        if item is None:
            item = self.try_to_add()
            if self.getOption('create') and item is None:
                item = self.create_item()

        self.current_item = item
        if item and self.getOption('clean'):
            self.clean_page()

    def create_item(self):
        """Create item in repo for current_page."""
        data = {'sitelinks':
                {self.site.dbName():
                 {'site': self.site.dbName(),
                  'title': self.current_page.title()}
                 },
                'labels':
                {self.site.lang:
                 {'language': self.site.lang,
                  'value': self.current_page.title()}
                 }
                }
        summary = (u'Bot: New item with sitelink from %s'
                   % self.current_page.title(asLink=True, insite=self.repo))

        item = pywikibot.ItemPage(self.repo)
        item.editEntity(data, new='item', summary=summary)
        output('Created item {item}'.format(item=item.getID()))
        return item

    def handle_complicated(self):
        """
        Handle pages when they have interwiki conflict.

        When this method returns True it means conflict has resolved
        and it's okay to clean old interwiki links.
        This method should change self.current_item and fix conflicts.
        Change it in subclasses.
        """
        return False

    def clean_page(self):
        """Clean interwiki links from the page."""
        if not self.iwlangs:
            return
        dbnames = [iw_site.dbName() for iw_site in self.iwlangs]
        if set(dbnames) < set(self.current_item.sitelinks.keys()):
            if not self.handle_complicated():
                warning('Interwiki conflict in %s, skipping...' %
                        self.current_page.title(asLink=True))
                return False
        output('Cleaning up the page')
        new_text = pywikibot.textlib.removeLanguageLinks(
            self.current_page.text, site=self.current_page.site)
        self.put_current(new_text, summary=self.getOption('summary'))

    def try_to_add(self):
        """Add current page in repo."""
        wd_data = set()
        for iw_page in self.iwlangs.values():
            try:
                wd_data.add(pywikibot.ItemPage.fromPage(iw_page))
            except pywikibot.NoPage:
                warning('Interwiki %s does not exist, skipping...' %
                        iw_page.title(asLink=True))
                continue
            except pywikibot.InvalidTitle:
                warning('Invalid title %s, skipping...' %
                        iw_page.title(asLink=True))
                continue
        if len(wd_data) != 1:
            warning('Interwiki conflict in %s, skipping...' %
                    self.current_page.title(asLink=True))
            return False
        item = list(wd_data).pop()
        if self.current_page.site.dbName() in item.sitelinks:
            warning('Interwiki conflict in %s, skipping...' %
                    item.title(asLink=True))
            return False
        output('Adding link to %s' % item.title())
        item.setSitelink(self.current_page)
        return item

def main(*args):
    """
    Process command line arguments and invoke bot.

    If args is an empty list, sys.argv is used.

    @param args: command line arguments
    @type args: list of unicode
    """
    local_args = pywikibot.handle_args(args)
    genFactory = pagegenerators.GeneratorFactory()
    options = {}
    for arg in local_args:
        if genFactory.handleArg(arg):
            continue
        option, sep, value = arg.partition(':')
        option = option[1:] if option.startswith('-') else None
        if option == 'summary':
            options[option] = value
        else:
            options[option] = True

    site = pywikibot.Site()

    generator = genFactory.getCombinedGenerator(preload=True)
    if generator:
        bot = IWBot(generator=generator, site=site, **options)
        bot.run()
    else:
        suggest_help(missing_generator=True)
        return False

if __name__ == '__main__':
    main()
<commit_msg>[IMPR] Create items with interwiki using interwikidata.py<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Script to handle interwiki links based on Wikibase.

This script connects pages to Wikibase items using language links on the page.
If multiple language links are present, and they are connected to different
items, the bot skips. After connecting the page to an item, language links
can be removed from the page.

These command line parameters can be used to specify which pages to work on:

&params;

Furthermore, the following command line parameters are supported:

-clean            Clean pages.

-create           Create items only.

-summary:         Use your own edit summary for cleaning the page.
"""

# (C) Pywikibot team, 2015-2017
#
# Distributed under the terms of the MIT license.
#
from __future__ import unicode_literals, absolute_import

__version__ = '$Id$'
#

import pywikibot

from pywikibot import pagegenerators, output, warning
from pywikibot.bot import ExistingPageBot, SingleSiteBot, suggest_help

# This is required for the text that is shown when you run this script
# with the parameter -help.
docuReplacements = {
    '&params;': pagegenerators.parameterHelp,
}

# Allowed namespaces. main, project, template, category
namespaces = [0, 4, 10, 14]

# TODO: Some templates on pages, like csd, inuse and afd templates,
# should cause the bot to skip the page, see T134497

class IWBot(ExistingPageBot, SingleSiteBot):

    """The bot for interwiki."""

    def __init__(self, **kwargs):
        """Construct the bot."""
        self.availableOptions.update({
            'clean': False,
            'create': False,
            'summary': None,
            'ignore_ns': False,  # used by interwikidata_tests only
        })
        super(IWBot, self).__init__(**kwargs)
        if not self.site.has_data_repository:
            raise ValueError('{site} does not have a data repository, '
                             'use interwiki.py instead.'.format(
                                 site=self.site))
        self.repo = self.site.data_repository()
        if not self.getOption('summary'):
            self.options['summary'] = pywikibot.i18n.twtranslate(
                self.site, 'interwikidata-clean-summary')

    def treat_page(self):
        """Check page."""
        if (self.current_page.namespace() not in namespaces and
                not self.getOption('ignore_ns')):
            output('{page} is not in allowed namespaces, skipping'
                   .format(page=self.current_page.title(
                       asLink=True)))
            return False
        self.iwlangs = pywikibot.textlib.getLanguageLinks(
            self.current_page.text, insite=self.current_page.site)
        if not self.iwlangs:
            output('No interlanguagelinks on {page}'.format(
                page=self.current_page.title(asLink=True)))
            return False
        try:
            item = pywikibot.ItemPage.fromPage(self.current_page)
        except pywikibot.NoPage:
            item = None

        if item is None:
            item = self.try_to_add()
            if self.getOption('create') and item is None:
                item = self.create_item()

        self.current_item = item
        if item and self.getOption('clean'):
            self.clean_page()

    def create_item(self):
        """Create item in repo for current_page."""
        data = {'sitelinks':
                {self.site.dbName():
                 {'site': self.site.dbName(),
                  'title': self.current_page.title()}
                 },
                'labels':
                {self.site.lang:
                 {'language': self.site.lang,
                  'value': self.current_page.title()}
                 }
                }
        for site, page in self.iwlangs.values():
            if not page.exists():
                continue
            dbname = site.dbName()
            title = page.title()
            data['sitelinks'][dbname] = {'site': dbname, 'title': title}
            data['labels'][site.lang] = {'language': site.lang, 'value': title}
        summary = ('Bot: New item with sitelink(s) from %s'
                   % self.current_page.title(asLink=True, insite=self.repo))

        item = pywikibot.ItemPage(self.repo)
        item.editEntity(data, new='item', summary=summary)
        output('Created item {item}'.format(item=item.getID()))
        return item

    def handle_complicated(self):
        """
        Handle pages when they have interwiki conflict.

        When this method returns True it means conflict has resolved
        and it's okay to clean old interwiki links.
        This method should change self.current_item and fix conflicts.
        Change it in subclasses.
        """
        return False

    def clean_page(self):
        """Clean interwiki links from the page."""
        if not self.iwlangs:
            return
        dbnames = [iw_site.dbName() for iw_site in self.iwlangs]
        if set(dbnames) < set(self.current_item.sitelinks.keys()):
            if not self.handle_complicated():
                warning('Interwiki conflict in %s, skipping...' %
                        self.current_page.title(asLink=True))
                return False
        output('Cleaning up the page')
        new_text = pywikibot.textlib.removeLanguageLinks(
            self.current_page.text, site=self.current_page.site)
        self.put_current(new_text, summary=self.getOption('summary'))

    def try_to_add(self):
        """Add current page in repo."""
        wd_data = set()
        for iw_page in self.iwlangs.values():
            try:
                wd_data.add(pywikibot.ItemPage.fromPage(iw_page))
            except pywikibot.NoPage:
                warning('Interwiki %s does not exist, skipping...' %
                        iw_page.title(asLink=True))
                continue
        if not wd_data:
            # will create a new item with interwiki
            return None
        if len(wd_data) > 1:
            warning('Interwiki conflict in %s, skipping...' %
                    self.current_page.title(asLink=True))
            return False
        item = list(wd_data).pop()
        if self.current_page.site.dbName() in item.sitelinks:
            warning('Interwiki conflict in %s, skipping...' %
                    item.title(asLink=True))
            return False
        output('Adding link to %s' % item.title())
        item.setSitelink(self.current_page)
        return item

def main(*args):
    """
    Process command line arguments and invoke bot.

    If args is an empty list, sys.argv is used.

    @param args: command line arguments
    @type args: list of unicode
    """
    local_args = pywikibot.handle_args(args)
    genFactory = pagegenerators.GeneratorFactory()
    options = {}
    for arg in local_args:
        if genFactory.handleArg(arg):
            continue
        option, sep, value = arg.partition(':')
        option = option[1:] if option.startswith('-') else None
        if option == 'summary':
            options[option] = value
        else:
            options[option] = True

    site = pywikibot.Site()

    generator = genFactory.getCombinedGenerator(preload=True)
    if generator:
        bot = IWBot(generator=generator, site=site, **options)
        bot.run()
    else:
        suggest_help(missing_generator=True)
        return False

if __name__ == '__main__':
    main()
<|endoftext|>