<commit_before>#!/usr/bin/env python
#
# Author: Todd Giles (todd.giles@gmail.com)
#
# Feel free to use as you'd like, but I'd love to hear back from you on any
# improvements, changes, etc...

# Modifications by: Chris Usey (chris.usey@gmail.com)

"""Check SMS messages from a Google Voice account

When executed, this script will check all the SMS messages from a Google Voice account
checking for either the "help" command, which will cause a help message to be sent back
to the original sender, or a single number indicating which song they are voting for.

When a song is voted for, the playlist file will be updated with the sender's cell
phone number to indicate it has received a vote from that caller.  This also enforces
only a single vote per phone number per song (until that song is played).

Sample usage:

sudo check_sms.py --playlist=/home/pi/music/.playlist

Third party dependencies:

pygooglevoice: http://sphinxdoc.github.io/pygooglevoice/
Beautiful Soup: http://www.crummy.com/software/BeautifulSoup/

Note, I used the following version of pygooglevoice w/auth fix:
https://code.google.com/r/bwpayne-pygooglevoice-auth-fix/
"""

import argparse
import csv
import fcntl
import sys
import time
import subprocess
import os

from googlevoice import Voice
from bs4 import BeautifulSoup

import log as l
import ConfigParser
import ast

# get configurations
home_directory = os.getenv("SYNCHRONIZED_LIGHTS_HOME")
config = ConfigParser.RawConfigParser()
config.read(home_directory + '/config/synchronized_lights.cfg')
try:
  admins = config.get('sms_settings','admins_list').split(',')
except:
  admins = []

try:
  specialguestlist = config.get('sms_settings','special_guest_list').split(',')
except:
  specialguestlist = []

try:
  requestsongslistcmd = config.get('sms_settings','request_songs_list_cmd')
except:
  requestsongslistcmd  = "help"

try:
  adminvolumemanagementcmd = config.get('sms_settings','admin_volume_management_cmd')
except:
  adminvolumemanagementcmd  = "volume"

try:
  admininterruptpreshowtimerscmd = config.get('sms_settings','admin_interrupt_preshow_timers_cmd')
except:
  admininterruptpreshowtimerscmd  = "play"

try:
  playlistpath = config.get('light_show_settings','playlist_path')
except:
  playlistpath  = "/home/pi/music/.playlist"

# get state
state = ConfigParser.RawConfigParser()
state.read(home_directory + '/py/synchronized_lights_state.cfg')

# ADMIN SETTINGS
volscript=home_directory + '/bin/vol'    # location of the volume script

parser = argparse.ArgumentParser()
parser.add_argument('--playlist', default=playlistpath, help='filename with the song playlist, one song per line in the format: <song name><tab><path to song>')
parser.add_argument('-v', '--verbosity', type=int, choices=[0, 1, 2], default=1, help='change output logging verbosity')
args = parser.parse_args()
l.verbosity = args.verbosity

# Setup username / password in ~/.gvoice file as follows:
#
# [auth]
# email=<google voice email address>
# password=<google voice password>
#
voice = Voice()
voice.login()

def song_played(song) :
    """Send an sms message to each requesting user that their song is now playing"""
    global voice
    for phonenumber in song[2]:
        voice.send_sms(phonenumber, '"' + song[0] + '" is playing!')

# extractsms - taken from http://sphinxdoc.github.io/pygooglevoice/examples.html
# originally written by John Nagle (nagle@animats.com)
def extractsms(htmlsms) :
    """
    extractsms  --  extract SMS messages from BeautifulSoup tree of Google Voice SMS HTML.

    Output is a list of dictionaries, one per message.
    """
    msgitems = []
    tree = BeautifulSoup(htmlsms)           # parse HTML into tree
    conversations = tree("div",attrs={"id" : True, "class" : "gc-message-unread"},recursive=False)
    for conversation in conversations :
        #   For each conversation, extract each row, which is one SMS message.
        rows = conversation(attrs={"class" : "gc-message-sms-row"})
        for row in rows :                               # for all rows
            #   For each row, which is one message, extract all the fields.
            msgitem = {"id" : conversation["id"]}       # tag this message with conversation ID
            spans = row("span",attrs={"class" : True}, recursive=False)
            for span in spans :                         # for all spans in row
                cl = span['class'][0].replace('gc-message-sms-', '')
                msgitem[cl] = (" ".join(span.findAll(text=True))).strip()   # put text in dict
            msgitems.append(msgitem)                    # add msg dictionary to list
    return msgitems

# Load playlist from file, notifying users of any of their requests that have now played
l.log('Loading playlist ' + args.playlist.replace("$SYNCHRONIZED_LIGHTS_HOME",home_directory), 2)
with open(args.playlist.replace("$SYNCHRONIZED_LIGHTS_HOME",home_directory), 'rb') as f:
    fcntl.lockf(f, fcntl.LOCK_SH)
    playlist = csv.reader(f, delimiter='\t')
    songs = []
    for song in playlist:
        l.log(song, 2)
        if len(song) < 2 or len(song) > 4:
            l.log('Invalid playlist', 0)
            sys.exit()
        elif len(song) == 2:
            song.append(set())
        elif len(song) >= 3:
            song[2] = set(song[2].split(','))
            if len(song) == 4:
                song_played(song)
                del song[3]
                song[2] = set()
        songs.append(song)
    fcntl.lockf(f, fcntl.LOCK_UN)

# Parse and act on any new sms messages
messages = voice.sms().messages
for msg in extractsms(voice.sms.html):
    l.log(str(msg), 2)
    try:
        song_num = int(msg['text'])
    except ValueError:
        song_num = 0
    if msg['from'] != 'Me' and song_num > 0 and song_num <= len(songs):
        song = songs[song_num-1]
        song[2].add(msg['from'])
        l.log('Song requested: ' + str(song))
        voice.send_sms(msg['from'], 'Thank you for requesting "' + song[0] + '", we\'ll notify you when it starts!')
    elif requestsongslistcmd in msg['text'].lower():
        l.log('Help requested from ' + msg['from'])
        songlist = ['']
        division = 0
        index = 1
        for song in songs:
            songlist[division] += str(index) + ' - ' + song[0] + '\n'
            index += 1
            if (index - 1) % 5 == 0:
                division += 1
                songlist.append('')
        header = 'Vote by texting the song #:\n'
        for division in songlist:
            voice.send_sms(msg['from'], header + division)
            header = ''
            time.sleep(5)
    # ADMIN - Volume management
    elif ((adminvolumemanagementcmd in msg['text'].lower()[0:len(adminvolumemanagementcmd)]) and ((msg['from'] in admins))):
        volmessage = msg['text'][len(adminvolumemanagementcmd):len(adminvolumemanagementcmd)+1]
        if ('-' in volmessage):
            l.log('Volume Down Request: ' + msg['from'])
            output, error = subprocess.Popen(volscript + " -", shell=True,stdout = subprocess.PIPE, stderr= subprocess.PIPE).communicate()
            if error:
                l.log('Volume Down Request Failed: ' + str(error))
            else:
                l.log('Volume decreased to: ' + str(output))
                voice.send_sms(msg['from'],'Volume decreased to: ' + str(output))
        elif ('+' in volmessage):
            l.log('Volume Increase Request: ' + msg['from'])
            output, error = subprocess.Popen(volscript + " +", shell=True,stdout = subprocess.PIPE, stderr= subprocess.PIPE).communicate()
            if error:
                l.log('Volume Increase Request Failed: ' + str(error))
            else:
                l.log('Volume increased to: ' + str(output))
                voice.send_sms(msg['from'],'Volume increased to: ' + str(output))
        else:
            try:
                volume = int(msg['text'][len(adminvolumemanagementcmd):])
                l.log('Volume Change To "' + str(volume) + '" Request: ' + msg['from'])
                if (volume >= 0 and volume < 100):
                    l.log('Volume Down Request: ' + msg['from'])
                    output, error = subprocess.Popen(volscript + " " + str(volume), shell=True,stdout = subprocess.PIPE, stderr= subprocess.PIPE).communicate()
                    if error:
                        l.log('Volume Set Request Failed: ' + str(error))
                    else:
                        l.log('Volume set to: ' + str(output))
                        voice.send_sms(msg['from'],'Volume set to: ' + str(output))
                else:
                    l.log('Volume Change Value Invalid: "' + str(volume) + '"')
                    voice.send_sms(msg['from'],'Ivalid Volume: Volume must be a value from 0-99')
            except ValueError:
                l.log('Volume Change Not Understood: "' + volmessage + '"')
                voice.send_sms(msg['from'], adminvolumemanagementcmd + ' Help: \n - "'+ adminvolumemanagementcmd +'-" to decrease \n - "'+ adminvolumemanagementcmd +'+" to increase \n - "'+ adminvolumemanagementcmd +'##" to set volume to ##')
    # ADMIN - immediatly play the next song - interrupt any pre show lights on off time.
    elif ((admininterruptpreshowtimerscmd in msg['text'].lower()[0:len(admininterruptpreshowtimerscmd)]) and ((msg['from'] in admins) or (msg['from'] in specialguestlist))):
        interruptrequest = msg['text'][len(admininterruptpreshowtimerscmd):].strip()
        # Check if "play" was the only thing sent
        if len(interruptrequest) == 0:
            try:
                state.set('do_not_modify','skip_pause','-1')
                with open(home_directory + '/py/synchronized_lights_state.cfg', 'wb') as statefile:
                    state.write(statefile)
                l.log('Request to interrupt pre show timers: "' + msg['text'] + '" from ' + msg['from'])
                voice.send_sms(msg['from'], 'The show will start shortly !')
            except ValueError:
                l.log('Exception with request: "' + volmessage + '"' +  ' (' + ValueError + ')')
                voice.send_sms(msg['from'], 'ERROR: Could not interrupt preshow timers, check logs')
        # Check what else was sent and if its a valid song choice
        else:
            try:
                interruptrequest = int(interruptrequest)
                # Check if the song selection is valid
                if interruptrequest > len(songs) or interruptrequest <= 0:
                    l.log('Invalid song requsted: "' + msg['text'] + '"')
                    voice.send_sms(msg['from'], 'Invalid song requested: ' + msg['text'])
                else:
                    state.set('do_not_modify','skip_pause',interruptrequest)
                    with open(home_directory + '/py/synchronized_lights_state.cfg', 'wb') as statefile:
                        state.write(statefile)
                    l.log('Request to interrupt pre show timers with specified song received: "' + msg['text'] + '"')
                    voice.send_sms(msg['from'], 'The show will start shortly !')
            except ValueError:
                l.log('Exception with request: "' + msg['text'] + '"')
                voice.send_sms(msg['from'], 'Help: \n - "' + admininterruptpreshowtimerscmd + '" to begin playing next show \n - "' + admininterruptpreshowtimerscmd + '##" to play song number ##')
    else:
        l.log('Unknown request: "' + msg['text'] + '" from ' + msg['from'])
        voice.send_sms(msg['from'], 'Hrm, not sure what you want.  Try texting "help" for... well some help!')

# Update playlist with latest votes
with open(args.playlist.replace("$SYNCHRONIZED_LIGHTS_HOME",home_directory), 'wb') as f:
    fcntl.lockf(f, fcntl.LOCK_EX)
    writer = csv.writer(f, delimiter='\t')
    for song in songs:
        if len(song[2]) > 0:
            song[2] = ",".join(song[2])
        else:
            del song[2]
    writer.writerows(songs)
    fcntl.lockf(f, fcntl.LOCK_UN)

# Delete all mesages now that we've processed them
for msg in messages:
    msg.delete(1)
<commit_msg>update all other config file references to updated location<commit_after>#!/usr/bin/env python
#
# Author: Todd Giles (todd.giles@gmail.com)
#
# Feel free to use as you'd like, but I'd love to hear back from you on any
# improvements, changes, etc...

# Modifications by: Chris Usey (chris.usey@gmail.com)

"""Check SMS messages from a Google Voice account

When executed, this script will check all the SMS messages from a Google Voice account
checking for either the "help" command, which will cause a help message to be sent back
to the original sender, or a single number indicating which song they are voting for.

When a song is voted for, the playlist file will be updated with the sender's cell
phone number to indicate it has received a vote from that caller.  This also enforces
only a single vote per phone number per song (until that song is played).

Sample usage:

sudo check_sms.py --playlist=/home/pi/music/.playlist

Third party dependencies:

pygooglevoice: http://sphinxdoc.github.io/pygooglevoice/
Beautiful Soup: http://www.crummy.com/software/BeautifulSoup/

Note, I used the following version of pygooglevoice w/auth fix:
https://code.google.com/r/bwpayne-pygooglevoice-auth-fix/
"""

import argparse
import csv
import fcntl
import sys
import time
import subprocess
import os

from googlevoice import Voice
from bs4 import BeautifulSoup

import log as l
import ConfigParser
import ast

# get configurations
home_directory = os.getenv("SYNCHRONIZED_LIGHTS_HOME")
config = ConfigParser.RawConfigParser()
config.read(home_directory + '/config/synchronized_lights.cfg')
try:
  admins = config.get('sms_settings','admins_list').split(',')
except:
  admins = []

try:
  specialguestlist = config.get('sms_settings','special_guest_list').split(',')
except:
  specialguestlist = []

try:
  requestsongslistcmd = config.get('sms_settings','request_songs_list_cmd')
except:
  requestsongslistcmd  = "help"

try:
  adminvolumemanagementcmd = config.get('sms_settings','admin_volume_management_cmd')
except:
  adminvolumemanagementcmd  = "volume"

try:
  admininterruptpreshowtimerscmd = config.get('sms_settings','admin_interrupt_preshow_timers_cmd')
except:
  admininterruptpreshowtimerscmd  = "play"

try:
  playlistpath = config.get('light_show_settings','playlist_path')
except:
  playlistpath  = "/home/pi/music/.playlist"

# get state
state = ConfigParser.RawConfigParser()
state.read(home_directory + '/config/synchronized_lights_state.cfg')

# ADMIN SETTINGS
volscript=home_directory + '/bin/vol'    # location of the volume script

parser = argparse.ArgumentParser()
parser.add_argument('--playlist', default=playlistpath, help='filename with the song playlist, one song per line in the format: <song name><tab><path to song>')
parser.add_argument('-v', '--verbosity', type=int, choices=[0, 1, 2], default=1, help='change output logging verbosity')
args = parser.parse_args()
l.verbosity = args.verbosity

# Setup username / password in ~/.gvoice file as follows:
#
# [auth]
# email=<google voice email address>
# password=<google voice password>
#
voice = Voice()
voice.login()

def song_played(song) :
    """Send an sms message to each requesting user that their song is now playing"""
    global voice
    for phonenumber in song[2]:
        voice.send_sms(phonenumber, '"' + song[0] + '" is playing!')

# extractsms - taken from http://sphinxdoc.github.io/pygooglevoice/examples.html
# originally written by John Nagle (nagle@animats.com)
def extractsms(htmlsms) :
    """
    extractsms  --  extract SMS messages from BeautifulSoup tree of Google Voice SMS HTML.

    Output is a list of dictionaries, one per message.
    """
    msgitems = []
    tree = BeautifulSoup(htmlsms)           # parse HTML into tree
    conversations = tree("div",attrs={"id" : True, "class" : "gc-message-unread"},recursive=False)
    for conversation in conversations :
        #   For each conversation, extract each row, which is one SMS message.
        rows = conversation(attrs={"class" : "gc-message-sms-row"})
        for row in rows :                               # for all rows
            #   For each row, which is one message, extract all the fields.
            msgitem = {"id" : conversation["id"]}       # tag this message with conversation ID
            spans = row("span",attrs={"class" : True}, recursive=False)
            for span in spans :                         # for all spans in row
                cl = span['class'][0].replace('gc-message-sms-', '')
                msgitem[cl] = (" ".join(span.findAll(text=True))).strip()   # put text in dict
            msgitems.append(msgitem)                    # add msg dictionary to list
    return msgitems

# Load playlist from file, notifying users of any of their requests that have now played
l.log('Loading playlist ' + args.playlist.replace("$SYNCHRONIZED_LIGHTS_HOME",home_directory), 2)
with open(args.playlist.replace("$SYNCHRONIZED_LIGHTS_HOME",home_directory), 'rb') as f:
    fcntl.lockf(f, fcntl.LOCK_SH)
    playlist = csv.reader(f, delimiter='\t')
    songs = []
    for song in playlist:
        l.log(song, 2)
        if len(song) < 2 or len(song) > 4:
            l.log('Invalid playlist', 0)
            sys.exit()
        elif len(song) == 2:
            song.append(set())
        elif len(song) >= 3:
            song[2] = set(song[2].split(','))
            if len(song) == 4:
                song_played(song)
                del song[3]
                song[2] = set()
        songs.append(song)
    fcntl.lockf(f, fcntl.LOCK_UN)

# Parse and act on any new sms messages
messages = voice.sms().messages
for msg in extractsms(voice.sms.html):
    l.log(str(msg), 2)
    try:
        song_num = int(msg['text'])
    except ValueError:
        song_num = 0
    if msg['from'] != 'Me' and song_num > 0 and song_num <= len(songs):
        song = songs[song_num-1]
        song[2].add(msg['from'])
        l.log('Song requested: ' + str(song))
        voice.send_sms(msg['from'], 'Thank you for requesting "' + song[0] + '", we\'ll notify you when it starts!')
    elif requestsongslistcmd in msg['text'].lower():
        l.log('Help requested from ' + msg['from'])
        songlist = ['']
        division = 0
        index = 1
        for song in songs:
            songlist[division] += str(index) + ' - ' + song[0] + '\n'
            index += 1
            if (index - 1) % 5 == 0:
                division += 1
                songlist.append('')
        header = 'Vote by texting the song #:\n'
        for division in songlist:
            voice.send_sms(msg['from'], header + division)
            header = ''
            time.sleep(5)
    # ADMIN - Volume management
    elif ((adminvolumemanagementcmd in msg['text'].lower()[0:len(adminvolumemanagementcmd)]) and ((msg['from'] in admins))):
        volmessage = msg['text'][len(adminvolumemanagementcmd):len(adminvolumemanagementcmd)+1]
        if ('-' in volmessage):
            l.log('Volume Down Request: ' + msg['from'])
            output, error = subprocess.Popen(volscript + " -", shell=True,stdout = subprocess.PIPE, stderr= subprocess.PIPE).communicate()
            if error:
                l.log('Volume Down Request Failed: ' + str(error))
            else:
                l.log('Volume decreased to: ' + str(output))
                voice.send_sms(msg['from'],'Volume decreased to: ' + str(output))
        elif ('+' in volmessage):
            l.log('Volume Increase Request: ' + msg['from'])
            output, error = subprocess.Popen(volscript + " +", shell=True,stdout = subprocess.PIPE, stderr= subprocess.PIPE).communicate()
            if error:
                l.log('Volume Increase Request Failed: ' + str(error))
            else:
                l.log('Volume increased to: ' + str(output))
                voice.send_sms(msg['from'],'Volume increased to: ' + str(output))
        else:
            try:
                volume = int(msg['text'][len(adminvolumemanagementcmd):])
                l.log('Volume Change To "' + str(volume) + '" Request: ' + msg['from'])
                if (volume >= 0 and volume < 100):
                    l.log('Volume Down Request: ' + msg['from'])
                    output, error = subprocess.Popen(volscript + " " + str(volume), shell=True,stdout = subprocess.PIPE, stderr= subprocess.PIPE).communicate()
                    if error:
                        l.log('Volume Set Request Failed: ' + str(error))
                    else:
                        l.log('Volume set to: ' + str(output))
                        voice.send_sms(msg['from'],'Volume set to: ' + str(output))
                else:
                    l.log('Volume Change Value Invalid: "' + str(volume) + '"')
                    voice.send_sms(msg['from'],'Ivalid Volume: Volume must be a value from 0-99')
            except ValueError:
                l.log('Volume Change Not Understood: "' + volmessage + '"')
                voice.send_sms(msg['from'], adminvolumemanagementcmd + ' Help: \n - "'+ adminvolumemanagementcmd +'-" to decrease \n - "'+ adminvolumemanagementcmd +'+" to increase \n - "'+ adminvolumemanagementcmd +'##" to set volume to ##')
    # ADMIN - immediately play the next song - interrupt any pre show lights on off time.
    elif ((admininterruptpreshowtimerscmd in msg['text'].lower()[0:len(admininterruptpreshowtimerscmd)]) and ((msg['from'] in admins) or (msg['from'] in specialguestlist))):
        interruptrequest = msg['text'][len(admininterruptpreshowtimerscmd):].strip()
        # Check if "play" was the only thing sent
        if len(interruptrequest) == 0:
            try:
                state.set('do_not_modify','skip_pause','-1')
                with open(home_directory + '/config/synchronized_lights_state.cfg', 'wb') as statefile:
                    state.write(statefile)
                l.log('Request to interrupt pre show timers: "' + msg['text'] + '" from ' + msg['from'])
                voice.send_sms(msg['from'], 'The show will start shortly !')
            except ValueError:
                l.log('Exception with request: "' + msg['text'] + '"' +  ' (' + ValueError + ')')
                voice.send_sms(msg['from'], 'ERROR: Could not interrupt preshow timers, check logs')
        # Check what else was sent and if its a valid song choice
        else:
            try:
                interruptrequest = int(interruptrequest)
                # Check if the song selection is valid
                if interruptrequest > len(songs) or interruptrequest <= 0:
                    l.log('Invalid song requested: "' + msg['text'] + '"')
                    voice.send_sms(msg['from'], 'Invalid song requested: ' + msg['text'])
                else:
                    state.set('do_not_modify','skip_pause',interruptrequest)
                    with open(home_directory + '/config/synchronized_lights_state.cfg', 'wb') as statefile:
                        state.write(statefile)
                    l.log('Request to interrupt pre show timers with specified song received: "' + msg['text'] + '"')
                    voice.send_sms(msg['from'], 'The show will start shortly !')
            except ValueError:
                l.log('Exception with request: "' + msg['text'] + '"')
                voice.send_sms(msg['from'], 'Help: \n - "' + admininterruptpreshowtimerscmd + '" to begin playing next show \n - "' + admininterruptpreshowtimerscmd + '##" to play song number ##')
    else:
        l.log('Unknown request: "' + msg['text'] + '" from ' + msg['from'])
        voice.send_sms(msg['from'], 'Hrm, not sure what you want.  Try texting "help" for... well some help!')

# Update playlist with latest votes
with open(args.playlist.replace("$SYNCHRONIZED_LIGHTS_HOME", home_directory), 'wb') as f:
    fcntl.lockf(f, fcntl.LOCK_EX)
    writer = csv.writer(f, delimiter='\t')
    for song in songs:
        if len(song[2]) > 0:
            song[2] = ",".join(song[2])
        else:
            del song[2]
    writer.writerows(songs)
    fcntl.lockf(f, fcntl.LOCK_UN)

# Delete all mesages now that we've processed them
for msg in messages:
    msg.delete(1)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#import cv2
import glob
import os
import sys
import gdal
import csv
from gdalconst import *
import postgres as po
import zipfile
import re

''' function: delete all ' ' from the file and write the file in an temporary file
    the original file will not be overwritten
    Called in: getStations()
'''
def cleanFile(finfo, csvRead, folderPath, tempFolder):

    ''' replace alls ' ' with '' to clean up the file '''
    txt = csvRead.replace(' ', '')

    txt_unicode = txt.decode("iso-8859-1")

    txt_utf8 = txt_unicode.encode("utf-8")

    writeFile = open(folderPath + tempFolder + finfo,'w')

    writeFile.write(txt_utf8)
    #writeFile.write('hallooooo')
    writeFile.close()

''' function: read the first line of the file to get the row names
    calls the temporary file
    Called in: getStations()
'''
def getFirstLine(filePath, fileName):

    with open(filePath + fileName, 'rb') as f:
        firstLine = f.readline()

        firstLineList = []
        firstLine = firstLine.replace(' ', '')

        for index in firstLine.split(';'):
            if len(index) > 1:
                #print index, len(index)
                firstLineList.append(index)

    return firstLineList

def readInZIP(folderPath, zipFolderPath):

    zfile = zipfile.ZipFile(zipFolderPath)
    for finfo in zfile.namelist():
        data = zfile.open(finfo)
        #temp = data.readlines()

        csvRead = data.read()
        #print finfo[0]
        #print finfo
        if 'Stationsmetadaten' in finfo:
            cleanFile(finfo, csvRead, folderPath,'station_metadata/')
        if 'produkt_klima' in finfo:
            cleanFile(finfo, csvRead, folderPath,'temperatur_data/')

''' function: read row by row data and put into an dictionary
    Called in: ?
'''
def getStations(filePath, fileName):

    #cleanFile(filePath, fileName)

    firstLineList = getFirstLine(filePath, fileName)

    with open(filePath + fileName, "rb") as csvfile:
        reader = csv.DictReader(csvfile, delimiter = ';')

        rowList = []

        for row in reader:
            #print row
            temp = {}
            for i in firstLineList:
                #print i
                temp[i] = row[i]

            rowList.append(temp)

        #print rowList
        return rowList

''' function: creates a list of all text files in an folder
    except of the ones that do not start with 'Stationsmetadaten'
    Called in: ?
'''
def createFileNameList(folderPath):

    ''' create List of file names in folder '''
    fileNameList = []

    for fileData in glob.glob(os.path.join(folderPath, '*.txt')):
        ''' create temporary dictionary '''
        namesDictionary = {}

        ''' seperate filename '''
        startIndex = fileData.rfind('/') + 1
        fileName = fileData[startIndex:]

        ''' put fileName into temporary dictionary '''
        #if 'Stationsmetadaten' in fileName:
        namesDictionary["name"] = fileName

        ''' add filename to list '''
        fileNameList.append(namesDictionary)

    return fileNameList

''' function: creates a list of all folders in an folder
    Called in:?
'''
def createFolderNameList(folderPath):

    ''' create emtpy list for folder names '''
    folderNames = []

    ''' read all folder names in the folderPath and add it to list folderNames '''
    for name in os.listdir(folderPath):
        folderNames.append(name)

    return folderNames

def createTempData(folderPath):
    folder = createFolderNameList(folderPath)
    #print folder

    for z in range(len(folder)):
        if 'zip' in folder[z]:
            #print folder[z]
            readInZIP(folderPath, folderPath + folder[z])

''' class: this class is supposed to carry over or replace the functions above
    Called in:?
    Discription:
'''
class csvZIP(object):

    def __init__ (self, zipFolderPath, fileType):
        self.zipFolderPath = zipFolderPath
        #self.readZIP()
    #def test(self):
        #t = self.zipFolderPath + 'XYT'

        #return t

    ''' function: this function change all string in utf-8
    '''
    def makeUTF8(self, txt):
        txt = txt.decode("iso-8859-1")
        txt = txt.encode("utf-8")
        return txt

    def getFirstLine(self, firstLine):
        regex = re.compile('[a-z_\?\.]+', re.IGNORECASE)
        #firstLine = firstLine.lower()
        firstLineList = regex.findall(firstLine)

        return firstLineList

    def readZIP(self):
        zfile = zipfile.ZipFile(self.zipFolderPath)

        dataDictionary = {}

        for finfo in zfile.namelist():
            #temp = data.readlines()
            temp = {}
            if fileType in finfo:
                ''' get the first line of the file '''
                fileData = zfile.open(finfo, 'r')
                firstLine = fileData.readline()
                data = fileData.readlines()
                #reader = csv.DictReader(fileData, delimiter = ';')
                fileData.close()

                firstLine = self.makeUTF8(firstLine)

                ''' delete all ' ' between the comma seperated values;  fvl = first line values'''
                firstLineList = self.getFirstLine(firstLine)
                temp['firstLine'] = firstLineList

                with zfile.open(finfo, "r") as csvfile:

                    reader = csv.DictReader(csvfile, delimiter = ';')

                    rowList = []
                    for row in reader:
                        temptemp = {}
                        for i in row:
                            i = self.makeUTF8(i)
                            for z in firstLineList:

                                if z in i:
                                    if row[i]:
                                        xy = row[i]

                                        #xy = re.sub('^[ {1,}\d+]', '', xy)
                                        xy = re.sub(' {2,}', '', xy)
                                        xy = self.makeUTF8(xy)
                                        #xy = row[i]
                                    else:
                                        xy = 'none'
                                    temptemp[z] = xy
                        rowList.append(temptemp)

                temp['data'] = rowList

                ''' add the temporary dictionary to dataDictionary '''
                dataDictionary[finfo] = temp

        return dataDictionary
        #return test

fileType = 'txt'
m = csvZIP('data/recent/tageswerte_KL_13711_akt.zip', fileType)
#m = csvZIP('data/recent/tageswerte_KL_03513_akt.zip', fileType)

test = m.readZIP()
#print test
#print len(test)
#print test['Stationsmetadaten_klima_stationen_13711_20131005_20150216.txt']
for key in test:
    if 'meta' in key:
        print test[key]['firstLine']
        #l = len(test[key]['data'])
        #print len(test[key]['data'])
        #for index in range(len(test[key]['data'])):
            #print test[key]['data'][index][']
        print test[key]['data'][0]
        #print test[key]['data'][2]
        #print test[key]['data'][3]

        #for row in readerX:
            #print row

#print test['Stationsmetadaten_klima_stationen_13711_20131005_20150216.txt']['firstLine']

#test = m.getFirstLine()

#x = test[7]
#x = x.decode("iso-8859-1")
#x = x.encode("utf-8")
#print x

#print len(test)

#readInZIP('../data/recent/temp', 'data/recent/tageswerte_KL_13711_akt.zip')

#createTempData('data/recent/')

<commit_msg>fixed: spaces in 'data'; add: documentation comments<commit_after># -*- coding: utf-8 -*-
#import cv2
import glob
import os
import sys
import gdal
import csv
from gdalconst import *
import postgres as po
import zipfile
import re

''' function: delete all ' ' from the file and write the file in an temporary file
    the original file will not be overwritten
    Called in: getStations()
'''
def cleanFile(finfo, csvRead, folderPath, tempFolder):

    ''' replace alls ' ' with '' to clean up the file '''
    txt = csvRead.replace(' ', '')

    txt_unicode = txt.decode("iso-8859-1")

    txt_utf8 = txt_unicode.encode("utf-8")

    writeFile = open(folderPath + tempFolder + finfo,'w')

    writeFile.write(txt_utf8)
    #writeFile.write('hallooooo')
    writeFile.close()

''' function: read the first line of the file to get the row names
    calls the temporary file
    Called in: getStations()
'''
def getFirstLine(filePath, fileName):

    with open(filePath + fileName, 'rb') as f:
        firstLine = f.readline()

        firstLineList = []
        firstLine = firstLine.replace(' ', '')

        for index in firstLine.split(';'):
            if len(index) > 1:
                #print index, len(index)
                firstLineList.append(index)

    return firstLineList

def readInZIP(folderPath, zipFolderPath):

    zfile = zipfile.ZipFile(zipFolderPath)
    for finfo in zfile.namelist():
        data = zfile.open(finfo)
        #temp = data.readlines()

        csvRead = data.read()
        #print finfo[0]
        #print finfo
        if 'Stationsmetadaten' in finfo:
            cleanFile(finfo, csvRead, folderPath,'station_metadata/')
        if 'produkt_klima' in finfo:
            cleanFile(finfo, csvRead, folderPath,'temperatur_data/')

''' function: read row by row data and put into an dictionary
    Called in: ?
'''
def getStations(filePath, fileName):

    #cleanFile(filePath, fileName)

    firstLineList = getFirstLine(filePath, fileName)

    with open(filePath + fileName, "rb") as csvfile:
        reader = csv.DictReader(csvfile, delimiter = ';')

        rowList = []

        for row in reader:
            #print row
            temp = {}
            for i in firstLineList:
                #print i
                temp[i] = row[i]

            rowList.append(temp)

        #print rowList
        return rowList

''' function: creates a list of all text files in an folder
    except of the ones that do not start with 'Stationsmetadaten'
    Called in: ?
'''
def createFileNameList(folderPath):

    ''' create List of file names in folder '''
    fileNameList = []

    for fileData in glob.glob(os.path.join(folderPath, '*.txt')):
        ''' create temporary dictionary '''
        namesDictionary = {}

        ''' seperate filename '''
        startIndex = fileData.rfind('/') + 1
        fileName = fileData[startIndex:]

        ''' put fileName into temporary dictionary '''
        #if 'Stationsmetadaten' in fileName:
        namesDictionary["name"] = fileName

        ''' add filename to list '''
        fileNameList.append(namesDictionary)

    return fileNameList

''' function: creates a list of all folders in an folder
    Called in:?
'''
def createFolderNameList(folderPath):

    ''' create emtpy list for folder names '''
    folderNames = []

    ''' read all folder names in the folderPath and add it to list folderNames '''
    for name in os.listdir(folderPath):
        folderNames.append(name)

    return folderNames

def createTempData(folderPath):
    folder = createFolderNameList(folderPath)
    #print folder

    for z in range(len(folder)):
        if 'zip' in folder[z]:
            #print folder[z]
            readInZIP(folderPath, folderPath + folder[z])

''' class: this class is supposed to carry over or replace the functions above
    Called in:?
    Discription:
'''
class csvZIP(object):

    ''' init function
    '''
    def __init__ (self, zipFolderPath, fileType):
        self.zipFolderPath = zipFolderPath
        self.fileType = fileType

    ''' function: change characters in utf-8 and
        returns the characters
    '''
    def makeUTF8(self, txt):
        txt = txt.decode("iso-8859-1")
        txt = txt.encode("utf-8")
        return txt

    ''' function: get strings in the first line of the csv file and store it in a list and
        returns the List
    '''
    def getFirstLine(self, firstLine):
        regex = re.compile('[a-z_\?\.]+', re.IGNORECASE)
        #firstLine = firstLine.lower()
        firstLineList = regex.findall(firstLine)

        return firstLineList

    ''' function: open zip file and reads all 'fileType' files
        returns from each file the first line as a list and the data as a list of dictionaries.
        first line:     ['row_1', 'row_2']
        data:           [ {'row_1': 'data_1, 'row_2': 'data_1}
                         ,{'row_1': 'data_2, 'row_2': 'data_2}
                         ,{'row_1': 'data_3, 'row_2': 'data_3}
                        ]
    '''
    def readZIP(self):
        ''' open zip file '''
        zfile = zipfile.ZipFile(self.zipFolderPath)

        ''' dictionary which will be returned '''
        dataDictionary = {}

        ''' go through the files in the zip file '''
        for finfo in zfile.namelist():

            ''' temporary dictionary to store the first line list '''
            tempList = {}

            ''' if-clause to get only files who ends with 'fileType'''
            if self.fileType in finfo:

                ''' get the first line of current file '''
                fileData = zfile.open(finfo, 'r')
                firstLine = fileData.readline()
                fileData.close()
                ''' turn first line in utf-8 '''
                firstLine = self.makeUTF8(firstLine)

                ''' create list of strings in the first line of current file;
                    delete all ' ' between the comma seperated values;
                    add firstLineList to temporary dictionary;
                '''
                firstLineList = self.getFirstLine(firstLine)
                tempList['firstLine'] = firstLineList

                ''' open current file as comma seperated file (csv)'''
                with zfile.open(finfo, "r") as csvfile:

                    ''' read csv-file as dictionary '''
                    reader = csv.DictReader(csvfile, delimiter = ';')

                    ''' list of rows or rather columns (see description of this function above )'''
                    dataList = []
                    for row in reader:
                        ''' temporary dictionary to store {'row': 'data'} '''
                        tempDictionary = {}
                        for data in row:
                            ''' turn 'data' in utf-8 '''
                            data = self.makeUTF8(data)
                            for z in firstLineList:
                                if z in data:
                                    if row[data]:
                                        xy = row[data]
                                        xy = str(xy).strip()
                                        xy = self.makeUTF8(xy)
                                    else:
                                        xy = 'none'
                                    tempDictionary[z] = xy
                        dataList.append(tempDictionary)

                tempList['data'] = dataList

                ''' add the temporary dictionary to dataDictionary '''
                dataDictionary[finfo] = tempList

        return dataDictionary

fileType = 'txt'
#m = csvZIP('data/recent/tageswerte_KL_13711_akt.zip', fileType)
m = csvZIP('data/recent/tageswerte_KL_03513_akt.zip', fileType)

test = m.readZIP()

#print test.keys()
for key in test:
    if 'meta' in key:
        print test[key]['firstLine']

        print test[key]['data']

<|endoftext|>


<commit_before>from PyQt4 import QtGui
from PyQt4 import QtCore
from UI.SlitSizeCalculator import Ui_SlitSizeCalculator
from sas.sascalc.dataloader.loader import Loader
from sas.sascalc.calculator.slit_length_calculator import SlitlengthCalculator

import os
import sys

class SlitSizeCalculator(QtGui.QDialog, Ui_SlitSizeCalculator):
    """
    Provides the slit length calculator GUI.
    """
    def __init__(self, parent=None):
        super(SlitSizeCalculator, self).__init__()
        self.setupUi(self)

        self.setWindowTitle("Slit Size Calculator")
        self._parent = parent

        self.thickness = SlitlengthCalculator()

        # signals
        self.helpButton.clicked.connect(self.onHelp)
        self.browseButton.clicked.connect(self.onBrowse)
        self.closeButton.clicked.connect(self.onClose)

        # no reason to have this widget resizable
        self.setFixedSize(self.minimumSizeHint())

    def onHelp(self):
        """
        Bring up the Slit Size Calculator calculator Documentation whenever
        the HELP button is clicked.
        Calls DocumentationWindow with the path of the location within the
        documentation tree (after /doc/ ....".
        """
        try:
            location = self._parent.HELP_DIRECTORY_LOCATION + \
                "/user/sasgui/perspectives/calculator/slit_calculator_help.html"

            self._parent._helpView.load(QtCore.QUrl(location))
            self._parent._helpView.show()
        except AttributeError:
            # No manager defined - testing and standalone runs
            pass

    def onBrowse(self):
        """
        Browse the file and calculate slit lenght upon loading
        """
        path_str = self.chooseFile()
        if not path_str:
            return
        loader = Loader()
        data = loader.load(path_str)

        self.data_file.setText(os.path.basename(path_str))
        self.calculateSlitSize(data)

    def chooseFile(self):
        """
        Shows the Open file dialog and returns the chosen path(s)
        """

        # Location is automatically saved - no need to keep track of the last dir
        # But only with Qt built-in dialog (non-platform native)
        path = QtGui.QFileDialog.getOpenFileName(self, "Choose a file", "",
                "SAXSess 1D data (*.txt *.TXT *.dat *.DAT)", None,
                QtGui.QFileDialog.DontUseNativeDialog)

        if path is None:
            return

        if isinstance(path, QtCore.QString):
            path = str(path)

        return path

    def onClose(self):
        """
        close the window containing this panel
        """
        self.close()

    def calculateSlitSize(self, data=None):
        """
        Computes slit lenght from given 1D data
        """

        if data is None:
            msg = "ERROR: Data hasn't been loaded correctly"
            raise RuntimeError, msg

        if data.__class__.__name__ == 'Data2D':
            msg = "Slit Length cannot be computed for 2D Data"
            raise Exception, msg

        #compute the slit size
        try:
             x = data.x
             y = data.y
             if x == [] or x is None or y == [] or y is None:
                 msg = "The current data is empty please check x and y"
                 raise ValueError, msg
             slit_length_calculator = SlitlengthCalculator()
             slit_length_calculator.set_data(x=x, y=y)
             slit_length = slit_length_calculator.calculate_slit_length()
        except:
             msg = "Slit Size Calculator: %s" % (sys.exc_value)
             raise RuntimeError, msg

        slit_length_str = "{:.5f}".format(slit_length)
        self.slit_length_out.setText(slit_length_str)

        #Display unit, which most likely needs to be 1/Ang but needs to be confirmed
        self.unit_out.setText("[Unknown]")

<commit_msg>Pylint fix<commit_after>"""
Slit Size Calculator Panel
"""
import os
import sys

from PyQt4 import QtGui
from PyQt4 import QtCore
from UI.SlitSizeCalculator import Ui_SlitSizeCalculator
from sas.sascalc.dataloader.loader import Loader
from sas.sascalc.calculator.slit_length_calculator import SlitlengthCalculator

class SlitSizeCalculator(QtGui.QDialog, Ui_SlitSizeCalculator):
    """
    Provides the slit length calculator GUI.
    """
    def __init__(self, parent=None):
        super(SlitSizeCalculator, self).__init__()
        self.setupUi(self)

        self.setWindowTitle("Slit Size Calculator")
        self._parent = parent

        self.thickness = SlitlengthCalculator()

        # signals
        self.helpButton.clicked.connect(self.onHelp)
        self.browseButton.clicked.connect(self.onBrowse)
        self.closeButton.clicked.connect(self.onClose)

        # no reason to have this widget resizable
        self.setFixedSize(self.minimumSizeHint())

    def onHelp(self):
        """
        Bring up the Slit Size Calculator calculator Documentation whenever
        the HELP button is clicked.
        Calls DocumentationWindow with the path of the location within the
        documentation tree (after /doc/ ....".
        """
        try:
            location = self._parent.HELP_DIRECTORY_LOCATION + \
                "/user/sasgui/perspectives/calculator/slit_calculator_help.html"

            self._parent._helpView.load(QtCore.QUrl(location))
            self._parent._helpView.show()
        except AttributeError:
            # No manager defined - testing and standalone runs
            pass

    def onBrowse(self):
        """
        Browse the file and calculate slit lenght upon loading
        """
        path_str = self.chooseFile()
        if not path_str:
            return
        loader = Loader()
        data = loader.load(path_str)

        self.data_file.setText(os.path.basename(path_str))
        self.calculateSlitSize(data)

    def chooseFile(self):
        """
        Shows the Open file dialog and returns the chosen path(s)
        """

        # Location is automatically saved - no need to keep track of the last dir
        # But only with Qt built-in dialog (non-platform native)
        path = QtGui.QFileDialog.getOpenFileName(self, "Choose a file", "",
                                                 "SAXSess 1D data (*.txt *.TXT *.dat *.DAT)", None,
                                                 QtGui.QFileDialog.DontUseNativeDialog)

        if path is None:
            return

        if isinstance(path, QtCore.QString):
            path = str(path)

        return path

    def onClose(self):
        """
        close the window containing this panel
        """
        self.close()

    def calculateSlitSize(self, data=None):
        """
        Computes slit lenght from given 1D data
        """

        if data is None:
            msg = "ERROR: Data hasn't been loaded correctly"
            raise RuntimeError, msg

        if data.__class__.__name__ == 'Data2D':
            msg = "Slit Length cannot be computed for 2D Data"
            raise Exception, msg

        #compute the slit size
        try:
            xdata = data.x
            ydata = data.y
            if xdata == [] or xdata is None or ydata == [] or ydata is None:
                msg = "The current data is empty please check x and y"
                raise ValueError, msg
            slit_length_calculator = SlitlengthCalculator()
            slit_length_calculator.set_data(x=xdata, y=ydata)
            slit_length = slit_length_calculator.calculate_slit_length()
        except:
            msg = "Slit Size Calculator: %s" % (sys.exc_value)
            raise RuntimeError, msg

        slit_length_str = "{:.5f}".format(slit_length)
        self.slit_length_out.setText(slit_length_str)

        #Display unit, which most likely needs to be 1/Ang but needs to be confirmed
        self.unit_out.setText("[Unknown]")

<|endoftext|>


<commit_before># Utility functions for py2deb.
#
# Authors:
#  - Arjan Verwer
#  - Peter Odding <peter.odding@paylogic.com>
# Last Change: June 13, 2014
# URL: https://py2deb.readthedocs.org

"""
The :py:mod:`py2deb.utils` module contains miscellaneous code.
"""

# Standard library modules.
import logging
import re
import shutil
import sys
import tempfile

# External dependencies.
from cached_property import cached_property
from deb_pkg_tools.package import find_package_archives
from six import string_types

# Semi-standard module versioning.
__version__ = '0.16'

# Initialize a logger.
logger = logging.getLogger(__name__)

class PackageRepository(object):

    """
    Very simply abstraction for a directory containing ``*.deb`` archives.

    Used by :py:class:`py2deb.converter.PackageConverter` to recognize which
    Python packages have previously been converted (and so can be skipped).
    """

    def __init__(self, directory):
        """
        Initialize a :py:class:`PackageRepository` object.

        :param directory: The pathname of the directory containing ``*.deb``
                          archives (a string).
        """
        self.directory = directory

    @cached_property
    def archives(self):
        """
        Find archive(s) in package repository / directory.

        :returns: A sorted list of package archives, same as the return value
                  of :py:func:`deb_pkg_tools.package.find_package_archives()`.

        An example:

        >>> from py2deb import PackageRepository
        >>> repo = PackageRepository('/tmp')
        >>> repo.archives
        [PackageFile(name='py2deb', version='0.1', architecture='all',
                     filename='/tmp/py2deb_0.1_all.deb'),
         PackageFile(name='py2deb-cached-property', version='0.1.5', architecture='all',
                     filename='/tmp/py2deb-cached-property_0.1.5_all.deb'),
         PackageFile(name='py2deb-chardet', version='2.2.1', architecture='all',
                     filename='/tmp/py2deb-chardet_2.2.1_all.deb'),
         PackageFile(name='py2deb-coloredlogs', version='0.5', architecture='all',
                     filename='/tmp/py2deb-coloredlogs_0.5_all.deb'),
         PackageFile(name='py2deb-deb-pkg-tools', version='1.20.4', architecture='all',
                     filename='/tmp/py2deb-deb-pkg-tools_1.20.4_all.deb'),
         PackageFile(name='py2deb-docutils', version='0.11', architecture='all',
                     filename='/tmp/py2deb-docutils_0.11_all.deb'),
         PackageFile(name='py2deb-executor', version='1.2', architecture='all',
                     filename='/tmp/py2deb-executor_1.2_all.deb'),
         PackageFile(name='py2deb-html2text', version='2014.4.5', architecture='all',
                     filename='/tmp/py2deb-html2text_2014.4.5_all.deb'),
         PackageFile(name='py2deb-humanfriendly', version='1.8.2', architecture='all',
                     filename='/tmp/py2deb-humanfriendly_1.8.2_all.deb'),
         PackageFile(name='py2deb-pkginfo', version='1.1', architecture='all',
                     filename='/tmp/py2deb-pkginfo_1.1_all.deb'),
         PackageFile(name='py2deb-python-debian', version='0.1.21-nmu2', architecture='all',
                     filename='/tmp/py2deb-python-debian_0.1.21-nmu2_all.deb'),
         PackageFile(name='py2deb-six', version='1.6.1', architecture='all',
                     filename='/tmp/py2deb-six_1.6.1_all.deb')]

        """
        return find_package_archives(self.directory)

    def get_package(self, package, version, architecture):
        """
        Find a package in the repository.

        Here's an example:

        >>> from py2deb import PackageRepository
        >>> repo = PackageRepository('/tmp')
        >>> repo.get_package('py2deb', '0.1', 'all')
        PackageFile(name='py2deb', version='0.1', architecture='all', filename='/tmp/py2deb_0.1_all.deb')

        :param package: The name of the package (a string).
        :param version: The version of the package (a string).
        :param architecture: The architecture of the package (a string).
        :returns: A :py:class:`deb_pkg_tools.package.PackageFile` object
                  or ``None``.
        """
        for archive in self.archives:
            if (archive.name == package and archive.version == version and archive.architecture == architecture):
                return archive

class TemporaryDirectory(object):

    """
    Easy temporary directory creation & cleanup using the :keyword:`with` statement.

    Here's an example of how to use this:

    .. code-block:: python

       with TemporaryDirectory() as directory:
           # Do something useful here.
           assert os.path.isdir(directory)
    """

    def __init__(self, **options):
        """
        Initialize context manager that manages creation & cleanup of temporary directory.

        :param options: Any keyword arguments are passed on to
                        :py:func:`tempfile.mkdtemp()`.
        """
        self.options = options

    def __enter__(self):
        """
        Create the temporary directory.
        """
        self.temporary_directory = tempfile.mkdtemp(**self.options)
        logger.debug("Created temporary directory: %s", self.temporary_directory)
        return self.temporary_directory

    def __exit__(self, exc_type, exc_value, traceback):
        """
        Destroy the temporary directory.
        """
        logger.debug("Cleaning up temporary directory: %s", self.temporary_directory)
        shutil.rmtree(self.temporary_directory)
        del self.temporary_directory

def python_version():
    """
    Find the version of Python we're running.

    This specifically returns a name matching the format of the names of the
    Debian packages providing the various available Python versions.

    :returns: A string like ``python2.6`` or ``python2.7``.
    """
    python_version = 'python%d.%d' % (sys.version_info[0], sys.version_info[1])
    logger.debug("Detected Python version: %s", python_version)
    return python_version

def normalize_package_name(python_package_name):
    """
    Normalize Python package name to be used as Debian package name.

    :param python_package_name: The name of a Python package
                                as found on PyPI (a string).
    :returns: The normalized name (a string).

    >>> from py2deb import normalize_package_name
    >>> normalize_package_name('MySQL-python')
    'mysql-python'
    >>> normalize_package_name('simple_json')
    'simple-json'
    """
    return re.sub('[^a-z0-9]+', '-', python_package_name.lower()).strip('-')

def compact_repeating_words(words):
    """
    Remove adjacent repeating words.

    :param words: A list of words (strings), assumed to already be normalized
                  (lowercased).
    :returns: The list of words with adjacent repeating words replaced by a
              single word.

    This is used to avoid awkward word repetitions in the package name
    conversion algorithm. Here's an example of what I mean:

    >>> from py2deb import compact_repeating_words
    >>> name_prefix = 'python'
    >>> package_name = 'python-mcrypt'
    >>> combined_words = [name_prefix] + package_name.split('-')
    >>> print combined_words
    ['python', 'python', 'mcrypt']
    >>> compacted_words = compact_repeating_words(combined_words)
    >>> print compacted_words
    ['python', 'mcrypt']
    """
    last_word = None
    for word in words:
        if word != last_word:
            yield word
        last_word = word

def coerce_to_boolean(value):
    """
    Coerce any value to a boolean.

    :param value: Any Python value. If the value is a string:

                  - The strings '1', 'yes', 'true' and 'on' are coerced to ``True``.
                  - The strings '0', 'no', 'false' and 'off' are coerced to ``False``.
                  - Other strings raise an exception.

                  Other Python values are coerced using :py:func:`bool()`.
    :returns: A proper boolean value.
    :raises: :py:exc:`exceptions.ValueError` when the value is a string but
             cannot be coerced with certainty.
    """
    if isinstance(value, string_types):
        normalized = str(value).strip().lower()
        if normalized in ('1', 'yes', 'true', 'on'):
            return True
        elif normalized in ('0', 'no', 'false', 'off'):
            return False
        else:
            msg = "Failed to coerce string to boolean! (%r)"
            raise ValueError(msg % value)
    else:
        return bool(value)
<commit_msg>Update documentation of compact_repeating_words()<commit_after># Utility functions for py2deb.
#
# Authors:
#  - Arjan Verwer
#  - Peter Odding <peter.odding@paylogic.com>
# Last Change: June 13, 2014
# URL: https://py2deb.readthedocs.org

"""
The :py:mod:`py2deb.utils` module contains miscellaneous code.
"""

# Standard library modules.
import logging
import re
import shutil
import sys
import tempfile

# External dependencies.
from cached_property import cached_property
from deb_pkg_tools.package import find_package_archives
from six import string_types

# Semi-standard module versioning.
__version__ = '0.16'

# Initialize a logger.
logger = logging.getLogger(__name__)

class PackageRepository(object):

    """
    Very simply abstraction for a directory containing ``*.deb`` archives.

    Used by :py:class:`py2deb.converter.PackageConverter` to recognize which
    Python packages have previously been converted (and so can be skipped).
    """

    def __init__(self, directory):
        """
        Initialize a :py:class:`PackageRepository` object.

        :param directory: The pathname of the directory containing ``*.deb``
                          archives (a string).
        """
        self.directory = directory

    @cached_property
    def archives(self):
        """
        Find archive(s) in package repository / directory.

        :returns: A sorted list of package archives, same as the return value
                  of :py:func:`deb_pkg_tools.package.find_package_archives()`.

        An example:

        >>> from py2deb import PackageRepository
        >>> repo = PackageRepository('/tmp')
        >>> repo.archives
        [PackageFile(name='py2deb', version='0.1', architecture='all',
                     filename='/tmp/py2deb_0.1_all.deb'),
         PackageFile(name='py2deb-cached-property', version='0.1.5', architecture='all',
                     filename='/tmp/py2deb-cached-property_0.1.5_all.deb'),
         PackageFile(name='py2deb-chardet', version='2.2.1', architecture='all',
                     filename='/tmp/py2deb-chardet_2.2.1_all.deb'),
         PackageFile(name='py2deb-coloredlogs', version='0.5', architecture='all',
                     filename='/tmp/py2deb-coloredlogs_0.5_all.deb'),
         PackageFile(name='py2deb-deb-pkg-tools', version='1.20.4', architecture='all',
                     filename='/tmp/py2deb-deb-pkg-tools_1.20.4_all.deb'),
         PackageFile(name='py2deb-docutils', version='0.11', architecture='all',
                     filename='/tmp/py2deb-docutils_0.11_all.deb'),
         PackageFile(name='py2deb-executor', version='1.2', architecture='all',
                     filename='/tmp/py2deb-executor_1.2_all.deb'),
         PackageFile(name='py2deb-html2text', version='2014.4.5', architecture='all',
                     filename='/tmp/py2deb-html2text_2014.4.5_all.deb'),
         PackageFile(name='py2deb-humanfriendly', version='1.8.2', architecture='all',
                     filename='/tmp/py2deb-humanfriendly_1.8.2_all.deb'),
         PackageFile(name='py2deb-pkginfo', version='1.1', architecture='all',
                     filename='/tmp/py2deb-pkginfo_1.1_all.deb'),
         PackageFile(name='py2deb-python-debian', version='0.1.21-nmu2', architecture='all',
                     filename='/tmp/py2deb-python-debian_0.1.21-nmu2_all.deb'),
         PackageFile(name='py2deb-six', version='1.6.1', architecture='all',
                     filename='/tmp/py2deb-six_1.6.1_all.deb')]

        """
        return find_package_archives(self.directory)

    def get_package(self, package, version, architecture):
        """
        Find a package in the repository.

        Here's an example:

        >>> from py2deb import PackageRepository
        >>> repo = PackageRepository('/tmp')
        >>> repo.get_package('py2deb', '0.1', 'all')
        PackageFile(name='py2deb', version='0.1', architecture='all', filename='/tmp/py2deb_0.1_all.deb')

        :param package: The name of the package (a string).
        :param version: The version of the package (a string).
        :param architecture: The architecture of the package (a string).
        :returns: A :py:class:`deb_pkg_tools.package.PackageFile` object
                  or ``None``.
        """
        for archive in self.archives:
            if (archive.name == package and archive.version == version and archive.architecture == architecture):
                return archive

class TemporaryDirectory(object):

    """
    Easy temporary directory creation & cleanup using the :keyword:`with` statement.

    Here's an example of how to use this:

    .. code-block:: python

       with TemporaryDirectory() as directory:
           # Do something useful here.
           assert os.path.isdir(directory)
    """

    def __init__(self, **options):
        """
        Initialize context manager that manages creation & cleanup of temporary directory.

        :param options: Any keyword arguments are passed on to
                        :py:func:`tempfile.mkdtemp()`.
        """
        self.options = options

    def __enter__(self):
        """
        Create the temporary directory.
        """
        self.temporary_directory = tempfile.mkdtemp(**self.options)
        logger.debug("Created temporary directory: %s", self.temporary_directory)
        return self.temporary_directory

    def __exit__(self, exc_type, exc_value, traceback):
        """
        Destroy the temporary directory.
        """
        logger.debug("Cleaning up temporary directory: %s", self.temporary_directory)
        shutil.rmtree(self.temporary_directory)
        del self.temporary_directory

def python_version():
    """
    Find the version of Python we're running.

    This specifically returns a name matching the format of the names of the
    Debian packages providing the various available Python versions.

    :returns: A string like ``python2.6`` or ``python2.7``.
    """
    python_version = 'python%d.%d' % (sys.version_info[0], sys.version_info[1])
    logger.debug("Detected Python version: %s", python_version)
    return python_version

def normalize_package_name(python_package_name):
    """
    Normalize Python package name to be used as Debian package name.

    :param python_package_name: The name of a Python package
                                as found on PyPI (a string).
    :returns: The normalized name (a string).

    >>> from py2deb import normalize_package_name
    >>> normalize_package_name('MySQL-python')
    'mysql-python'
    >>> normalize_package_name('simple_json')
    'simple-json'
    """
    return re.sub('[^a-z0-9]+', '-', python_package_name.lower()).strip('-')

def compact_repeating_words(words):
    """
    Remove adjacent repeating words.

    :param words: An iterable of words (strings), assumed to already be
                  normalized (lowercased).
    :returns: An iterable of words with adjacent repeating words replaced by a
              single word.

    This is used to avoid awkward word repetitions in the package name
    conversion algorithm. Here's an example of what I mean:

    >>> from py2deb import compact_repeating_words
    >>> name_prefix = 'python'
    >>> package_name = 'python-mcrypt'
    >>> combined_words = [name_prefix] + package_name.split('-')
    >>> print(list(combined_words))
    ['python', 'python', 'mcrypt']
    >>> compacted_words = compact_repeating_words(combined_words)
    >>> print(list(compacted_words))
    ['python', 'mcrypt']
    """
    last_word = None
    for word in words:
        if word != last_word:
            yield word
        last_word = word

def coerce_to_boolean(value):
    """
    Coerce any value to a boolean.

    :param value: Any Python value. If the value is a string:

                  - The strings '1', 'yes', 'true' and 'on' are coerced to ``True``.
                  - The strings '0', 'no', 'false' and 'off' are coerced to ``False``.
                  - Other strings raise an exception.

                  Other Python values are coerced using :py:func:`bool()`.
    :returns: A proper boolean value.
    :raises: :py:exc:`exceptions.ValueError` when the value is a string but
             cannot be coerced with certainty.
    """
    if isinstance(value, string_types):
        normalized = str(value).strip().lower()
        if normalized in ('1', 'yes', 'true', 'on'):
            return True
        elif normalized in ('0', 'no', 'false', 'off'):
            return False
        else:
            msg = "Failed to coerce string to boolean! (%r)"
            raise ValueError(msg % value)
    else:
        return bool(value)
<|endoftext|>


<commit_before>import os
import platform
from retriever.lib.models import Engine, no_cleanup

class DummyCursor:
    pass

class DummyConnection:
    def cursor(self):
        pass
    def commit(self):
        pass

class engine(Engine):
    """Engine instance for writing data to a CSV file."""
    name = "CSV"
    abbreviation = "c"
    datatypes = {
                 "auto": "INTEGER",
                 "int": "INTEGER",
                 "bigint": "INTEGER",
                 "double": "REAL",
                 "decimal": "REAL",
                 "char": "TEXT",
                 "bool": "INTEGER",
                 }
    required_opts = [
                     ("table_name",
                      "Format of table name",
                      "{db}_{table}.csv"),
                     ]

    def create_db(self):
        return None

    def create_table(self):
        self.output_file = open(self.tablename(), "w")
        self.output_file.write(','.join(['"%s"' % c[0] for c in self.table.columns]))

    def execute(self, statement, commit=True):
        self.output_file.write('\n' + statement)

    def format_insert_value(self, value, datatype):
        v = Engine.format_insert_value(self, value, datatype)
        if v == 'null': return ""
        try:
            if len(v) > 1 and v[0] == v[-1] == "'":
                v = '"%s"' % v[1:-1]
        except:
            pass
        return v

    def insert_statement(self, values):
        return ','.join([str(value) for value in values])

    def table_exists(self, dbname, tablename):
        try:
            tablename = self.tablename(name=tablename, dbname=dbname)
            return os.path.exists(tablename)
        except:
            return False

    def get_connection(self):
        """Gets the db connection."""
        self.get_input()
        return DummyConnection()
<commit_msg>Adding rollback method to CSV DummyConnection.<commit_after>import os
import platform
from retriever.lib.models import Engine, no_cleanup

class DummyConnection:
    def cursor(self):
        pass
    def commit(self):
        pass
    def rollback(self):
        pass

class DummyCursor(DummyConnection):
    pass

class engine(Engine):
    """Engine instance for writing data to a CSV file."""
    name = "CSV"
    abbreviation = "c"
    datatypes = {
                 "auto": "INTEGER",
                 "int": "INTEGER",
                 "bigint": "INTEGER",
                 "double": "REAL",
                 "decimal": "REAL",
                 "char": "TEXT",
                 "bool": "INTEGER",
                 }
    required_opts = [
                     ("table_name",
                      "Format of table name",
                      "{db}_{table}.csv"),
                     ]

    def create_db(self):
        return None

    def create_table(self):
        self.output_file = open(self.tablename(), "w")
        self.output_file.write(','.join(['"%s"' % c[0] for c in self.table.columns]))

    def execute(self, statement, commit=True):
        self.output_file.write('\n' + statement)

    def format_insert_value(self, value, datatype):
        v = Engine.format_insert_value(self, value, datatype)
        if v == 'null': return ""
        try:
            if len(v) > 1 and v[0] == v[-1] == "'":
                v = '"%s"' % v[1:-1]
        except:
            pass
        return v

    def insert_statement(self, values):
        return ','.join([str(value) for value in values])

    def table_exists(self, dbname, tablename):
        try:
            tablename = self.tablename(name=tablename, dbname=dbname)
            return os.path.exists(tablename)
        except:
            return False

    def get_connection(self):
        """Gets the db connection."""
        self.get_input()
        return DummyConnection()
<|endoftext|>


<commit_before>import unicodedata
from ftfy import chardata
import pathlib
from pkg_resources import resource_filename

CATEGORIES = [unicodedata.category(chr(i)) for i in range(0x110000)]
DATA_PATH = pathlib.Path(resource_filename('wordfreq', 'data'))

def func_to_regex(accept_func):
    """
    Given a function that returns True or False for a numerical codepoint,
    return a regex character class accepting the characters resulting in True.
    Ranges separated only by unassigned characters are merged for efficiency.
    """
    parsing_range = False
    ranges = []

    for codepoint, category in enumerate(CATEGORIES):
        if accept_func(codepoint):
            if not parsing_range:
                ranges.append([codepoint, codepoint])
                parsing_range = True
            else:
                ranges[-1][1] = codepoint
        elif category != 'Cn':
            parsing_range = False

    return '[%s]' % ''.join('%c-%c' % tuple(r) for r in ranges)

def cache_regex_from_func(filename, func):
    """
    Generates a regex from a function that accepts a single unicode character,
    and caches it in the data path at filename.
    """
    with (DATA_PATH / filename).open(mode='w') as file:
        file.write(func_to_regex(func))

def _is_emoji_codepoint(i):
    """
    Report whether a numerical codepoint is (likely) an emoji: a Unicode 'So'
    character (as future-proofed by the ftfy chardata module) but excluding
    symbols like © and ™ below U+2600 and the replacement character U+FFFD.
    """
    return chardata.CHAR_CLASS_STRING[i] == '3' and i >= 0x2600 and i != 0xfffd

def _is_non_punct_codepoint(i):
    """
    Report whether a numerical codepoint is not one of the following classes:
    - P: punctuation
    - S: symbols
    - Z: separators
    - C: control characters
    This will classify symbols, including emoji, as punctuation; users that
    want to accept emoji should add them separately.
    """
    return CATEGORIES[i][0] not in 'PSZC'

def _is_combining_mark_codepoint(i):
    """
    Report whether a numerical codepoint is a combining mark (Unicode 'M').
    """
    return CATEGORIES[i][0] == 'M'

if __name__ == '__main__':
    cache_regex_from_func('emoji.txt', _is_emoji_codepoint)
    cache_regex_from_func('non_punct.txt', _is_non_punct_codepoint)
    cache_regex_from_func('combining_mark.txt', _is_combining_mark_codepoint)
<commit_msg>added comment about parsing_range<commit_after>import unicodedata
from ftfy import chardata
import pathlib
from pkg_resources import resource_filename

CATEGORIES = [unicodedata.category(chr(i)) for i in range(0x110000)]
DATA_PATH = pathlib.Path(resource_filename('wordfreq', 'data'))

def func_to_regex(accept_func):
    """
    Given a function that returns True or False for a numerical codepoint,
    return a regex character class accepting the characters resulting in True.
    Ranges separated only by unassigned characters are merged for efficiency.
    """
    # parsing_range is True if the current codepoint might be in a range that
    # the regex will accept
    parsing_range = False
    ranges = []

    for codepoint, category in enumerate(CATEGORIES):
        if accept_func(codepoint):
            if not parsing_range:
                ranges.append([codepoint, codepoint])
                parsing_range = True
            else:
                ranges[-1][1] = codepoint
        elif category != 'Cn':
            parsing_range = False

    return '[%s]' % ''.join('%c-%c' % tuple(r) for r in ranges)

def cache_regex_from_func(filename, func):
    """
    Generates a regex from a function that accepts a single unicode character,
    and caches it in the data path at filename.
    """
    with (DATA_PATH / filename).open(mode='w') as file:
        file.write(func_to_regex(func))

def _is_emoji_codepoint(i):
    """
    Report whether a numerical codepoint is (likely) an emoji: a Unicode 'So'
    character (as future-proofed by the ftfy chardata module) but excluding
    symbols like © and ™ below U+2600 and the replacement character U+FFFD.
    """
    return chardata.CHAR_CLASS_STRING[i] == '3' and i >= 0x2600 and i != 0xfffd

def _is_non_punct_codepoint(i):
    """
    Report whether a numerical codepoint is not one of the following classes:
    - P: punctuation
    - S: symbols
    - Z: separators
    - C: control characters
    This will classify symbols, including emoji, as punctuation; users that
    want to accept emoji should add them separately.
    """
    return CATEGORIES[i][0] not in 'PSZC'

def _is_combining_mark_codepoint(i):
    """
    Report whether a numerical codepoint is a combining mark (Unicode 'M').
    """
    return CATEGORIES[i][0] == 'M'

if __name__ == '__main__':
    cache_regex_from_func('emoji.txt', _is_emoji_codepoint)
    cache_regex_from_func('non_punct.txt', _is_non_punct_codepoint)
    cache_regex_from_func('combining_mark.txt', _is_combining_mark_codepoint)
<|endoftext|>