<commit_before>import json, urllib, urllib2

#Update schema
__url__ = "https://raw.githubusercontent.com/KittyHawkIrc/modules/production/" + __name__ + ".py"
__version__ = 1.0

fCountries = ['Belize', 'Guam', 'Puerto Rico', 'United States', 'US Virgin Islands']
iCountries = ['Liberia', 'Myanmar', 'United States']

def declare():
  return {"w": "privmsg", "setlocation": "privmsg"}

def callback(self):
    channel = self.channel
    command = self.command
    user = self.user.split('!')[0].lower()
    msg = self.msg
    message = self.message.split(command, 1)[1].strip()
    if command == 'w':
        try:
            if message:
                try:
                    query = self.locker.location[message.lower()]
                except:
                    query = message
            else:
                try:
                    query = self.locker.location[user]
                except:
                    return msg(channel, 'You have not set a location yet.')

            #sourced from example code for Yahoo Weather API
            baseurl = 'https://query.yahooapis.com/v1/public/yql?'
            yql_query = 'select * from weather.forecast where woeid in (select woeid from geo.places where text="%s")' % query
            yql_url = baseurl + urllib.urlencode({'q':yql_query}) + "&format=json"
            result = urllib2.urlopen(yql_url)
            data = json.loads(result.read())
            result.close()

            try:
                city = data['query']['results']['channel'][0]['location']['city']
                region = data['query']['results']['channel'][0]['location']['region']
                country = data['query']['results']['channel'][0]['location']['country']
                cond = data['query']['results']['channel'][0]['item']['condition']['text']
                humid = data['query']['results']['channel'][0]['atmosphere']['humidity']
                wSpeed = data['query']['results']['channel'][0]['wind']['speed']
                wDir = degToDirection(int(data['query']['results']['channel'][0]['wind']['direction']))

                if country.strip() in fCountries:
                    temp = data['query']['results']['channel'][0]['item']['condition']['temp'] + '%sF' % u'\xb0'
                    high = data['query']['results']['channel'][0]['item']['forecast'][0]['high'] + '%sF' % u'\xb0'
                    low = data['query']['results']['channel'][0]['item']['forecast'][0]['low'] + '%sF' % u'\xb0'
                else:
                    temp = FToC(data['query']['results']['channel'][0]['item']['condition']['temp']) + '%sC' % u'\xb0'
                    high = FToC(data['query']['results']['channel'][0]['item']['forecast'][0]['high']) + '%sC' % u'\xb0'
                    low = FToC(data['query']['results']['channel'][0]['item']['forecast'][0]['low']) + '%sC' % u'\xb0'

                if country.strip() in iCountries:
                    wSpeed = data['query']['results']['channel'][0]['wind']['speed'] + 'mph'
                else:
                    wSpeed = miToKm(data['query']['results']['channel'][0]['wind']['speed']) + 'km/h'

                weather = '%s, %s, %s / %s / %s / Humidity: %s%% / High: %s / Low: %s' % (city, region, country, cond, temp, humid, high, low)

                weather = ' '.join(weather.split())

                return msg(channel, unicode(weather))
            except Exception, e:
                return msg(channel, 'I cannot find the weather for %s' % str(e))
        except:
            return msg(channel, 'I cannot fetch the weather at this moment.')
    if command == 'setlocation':
        if len(message) > 0:
            try:
                self.locker.location[user] = message
            except:
                self.locker.location = {user: message}
            self.cache_save()
            return msg(channel, 'Location for user %s set to %s' % (self.user.split('!')[0], message))
        return msg(channel, 'You did not give me a location to set!')

def FToC(fahrenheit):
    return str(int(round((int(fahrenheit) - 32) / 1.8)))

def miToKm(miles):
    return str(int(round(int(miles) * 1.60934)))

def degToDirection(deg):
    directions = ['NNE','NE','ENE','E','ESE','SE','SSE','S','SSW','SW','WSW','W','WNW','NW','NNW','N']
    start = 11.25
    span = 22.5
    count = 0

    while(start + span < 360):
        if deg >= start and deg <= start + span:
            return directions[count]
        count += 1
        start += span

    return 'N'

class api:
	def msg(self, channel, text):
		return "[%s] %s" % (channel, text)
class empty:
	pass

'''
# interactive testing:
api = api()
def cache_save():
	print 'Cache saved'
setattr(api, 'cache_save', cache_save)
setattr(api, 'type', 'privmsg')
setattr(api, 'channel', "#test")
setattr(api, 'command', 'w')
setattr(api, 'locker', empty)
setattr(api, 'user', 'joe!username@hostmask')
while(True):
    _input = raw_input('Enter message here: ')
    if 'setlocation' in _input:
        setattr(api, 'command', 'setlocation')
    else:
        setattr(api, 'command', 'w')
    setattr(api, 'message', _input)
    print callback(api)
'''

if __name__ == "__main__":
    def cache_save():
        print 'Cache saved'
    api = api()
    setattr(api, 'cache_save', cache_save)
    setattr(api, 'type', 'privmsg')
    setattr(api, 'channel', "#test")
    setattr(api, 'command', 'w')
    setattr(api, 'locker', empty)
    setattr(api, 'user', 'joe!username@hostmask')

    setattr(api, 'command', 'w')
    setattr(api, 'message', '^w')
    print callback(api)
    if 'You have not' not in callback(api):
    	exit(1)

    setattr(api, 'message', '^w Los Angeles')
    print callback(api)
    if 'Los Angeles, CA' not in callback(api):
    	exit(2)

    setattr(api, 'command', 'setlocation')
    setattr(api, 'message', '^setlocation Los Angeles')
    print callback(api)
    if 'Location for' not in callback(api):
    	exit(3)

    setattr(api, 'command', 'w')
    setattr(api, 'message', '^w')
    print callback(api)
    if 'Los Angeles, CA' not in callback(api):
    	exit(4)

    setattr(api, 'user', 'jeb!username@hostmask')
    setattr(api, 'message', '^w joe')
    print callback(api)
    if 'Los Angeles, CA' not in callback(api):
    	exit(5)
<commit_msg>removed attempt to get degrees sign<commit_after>import json, urllib, urllib2

#Update schema
__url__ = "https://raw.githubusercontent.com/KittyHawkIrc/modules/production/" + __name__ + ".py"
__version__ = 1.0

fCountries = ['Belize', 'Guam', 'Puerto Rico', 'United States', 'US Virgin Islands']
iCountries = ['Liberia', 'Myanmar', 'United States']

def declare():
  return {"w": "privmsg", "setlocation": "privmsg"}

def callback(self):
    channel = self.channel
    command = self.command
    user = self.user.split('!')[0].lower()
    msg = self.msg
    message = self.message.split(command, 1)[1].strip()
    if command == 'w':
        try:
            if message:
                try:
                    query = self.locker.location[message.lower()]
                except:
                    query = message
            else:
                try:
                    query = self.locker.location[user]
                except:
                    return msg(channel, 'You have not set a location yet.')

            #sourced from example code for Yahoo Weather API
            baseurl = 'https://query.yahooapis.com/v1/public/yql?'
            yql_query = 'select * from weather.forecast where woeid in (select woeid from geo.places where text="%s")' % query
            yql_url = baseurl + urllib.urlencode({'q':yql_query}) + "&format=json"
            result = urllib2.urlopen(yql_url)
            data = json.loads(result.read())
            result.close()

            try:
                city = data['query']['results']['channel'][0]['location']['city']
                region = data['query']['results']['channel'][0]['location']['region']
                country = data['query']['results']['channel'][0]['location']['country']
                cond = data['query']['results']['channel'][0]['item']['condition']['text']
                humid = data['query']['results']['channel'][0]['atmosphere']['humidity']
                wSpeed = data['query']['results']['channel'][0]['wind']['speed']
                wDir = degToDirection(int(data['query']['results']['channel'][0]['wind']['direction']))

                if country.strip() in fCountries:
                    temp = data['query']['results']['channel'][0]['item']['condition']['temp'] + 'F'
                    high = data['query']['results']['channel'][0]['item']['forecast'][0]['high'] + 'F'
                    low = data['query']['results']['channel'][0]['item']['forecast'][0]['low'] + 'F''
                else:
                    temp = FToC(data['query']['results']['channel'][0]['item']['condition']['temp']) + 'C'
                    high = FToC(data['query']['results']['channel'][0]['item']['forecast'][0]['high']) + 'C'
                    low = FToC(data['query']['results']['channel'][0]['item']['forecast'][0]['low']) + 'C'

                if country.strip() in iCountries:
                    wSpeed = data['query']['results']['channel'][0]['wind']['speed'] + 'mph'
                else:
                    wSpeed = miToKm(data['query']['results']['channel'][0]['wind']['speed']) + 'km/h'

                weather = '%s, %s, %s / %s / %s / Humidity: %s%% / High: %s / Low: %s' % (city, region, country, cond, temp, humid, high, low)

                weather = ' '.join(weather.split())

                return msg(channel, weather)
            except:
                return msg(channel, 'I cannot find the weather for %s' % message)
        except:
            return msg(channel, 'I cannot fetch the weather at this moment.')
    if command == 'setlocation':
        if len(message) > 0:
            try:
                self.locker.location[user] = message
            except:
                self.locker.location = {user: message}
            self.cache_save()
            return msg(channel, 'Location for user %s set to %s' % (self.user.split('!')[0], message))
        return msg(channel, 'You did not give me a location to set!')

def FToC(fahrenheit):
    return str(int(round((int(fahrenheit) - 32) / 1.8)))

def miToKm(miles):
    return str(int(round(int(miles) * 1.60934)))

def degToDirection(deg):
    directions = ['NNE','NE','ENE','E','ESE','SE','SSE','S','SSW','SW','WSW','W','WNW','NW','NNW','N']
    start = 11.25
    span = 22.5
    count = 0

    while(start + span < 360):
        if deg >= start and deg <= start + span:
            return directions[count]
        count += 1
        start += span

    return 'N'

class api:
	def msg(self, channel, text):
		return "[%s] %s" % (channel, text)
class empty:
	pass

'''
# interactive testing:
api = api()
def cache_save():
	print 'Cache saved'
setattr(api, 'cache_save', cache_save)
setattr(api, 'type', 'privmsg')
setattr(api, 'channel', "#test")
setattr(api, 'command', 'w')
setattr(api, 'locker', empty)
setattr(api, 'user', 'joe!username@hostmask')
while(True):
    _input = raw_input('Enter message here: ')
    if 'setlocation' in _input:
        setattr(api, 'command', 'setlocation')
    else:
        setattr(api, 'command', 'w')
    setattr(api, 'message', _input)
    print callback(api)
'''

if __name__ == "__main__":
    def cache_save():
        print 'Cache saved'
    api = api()
    setattr(api, 'cache_save', cache_save)
    setattr(api, 'type', 'privmsg')
    setattr(api, 'channel', "#test")
    setattr(api, 'command', 'w')
    setattr(api, 'locker', empty)
    setattr(api, 'user', 'joe!username@hostmask')

    setattr(api, 'command', 'w')
    setattr(api, 'message', '^w')
    print callback(api)
    if 'You have not' not in callback(api):
    	exit(1)

    setattr(api, 'message', '^w Los Angeles')
    print callback(api)
    if 'Los Angeles, CA' not in callback(api):
    	exit(2)

    setattr(api, 'command', 'setlocation')
    setattr(api, 'message', '^setlocation Los Angeles')
    print callback(api)
    if 'Location for' not in callback(api):
    	exit(3)

    setattr(api, 'command', 'w')
    setattr(api, 'message', '^w')
    print callback(api)
    if 'Los Angeles, CA' not in callback(api):
    	exit(4)

    setattr(api, 'user', 'jeb!username@hostmask')
    setattr(api, 'message', '^w joe')
    print callback(api)
    if 'Los Angeles, CA' not in callback(api):
    	exit(5)
<|endoftext|>


<commit_before>
import csv
import logging
from math import ceil
import struct
import socket

import plyvel
import simplejson as json

logger = logging.getLogger(__name__)

#
# Utilities
#

IP_STRUCT = struct.Struct('>L')

def incr_ip(ip):
    n = IP_STRUCT.unpack(ip)[0] + 1
    try:
        return IP_STRUCT.pack(n)
    except struct.error:
        return None

def transform_record(rec):
    start_ip = IP_STRUCT.pack(int(rec['start_ip_int']))
    end_ip = IP_STRUCT.pack(int(rec['end_ip_int']))

    # TODO: carrier_id, tld_id, sld_id, reg_org_id,
    # phone_number_prefix, asn, cidr

    tz = rec['timezone']
    if tz != '999':
        timezone = None
    else:
        timezone = '{:+04d}'.format(int(ceil(100 * float(tz))))

    value = json.dumps(dict(
        begin=socket.inet_ntoa(start_ip),
        end=socket.inet_ntoa(end_ip),
        continent=(rec['continent'], 1.),
        country=(rec['country_iso2'],
                 float(rec['country_cf']) / 100),
        state=(rec['state'],
               float(rec['state_cf']) / 100),
        city=(rec['city'],
              float(rec['city_cf']) / 100),
        postal_code=rec['postal_code'],
        type=rec['connectiontype'],
        routing=rec['ip_routingtype'],
        coordinates=(float(rec['latitude']),
                     float(rec['longitude'])),
        timezone=timezone,
        line_speed=rec['linespeed'],
        asn=rec['asn'],
    ))

    # The start IP is the key; the end IP is prepended to the value
    key = start_ip
    value = end_ip + value

    return key, value

#
# Public API
#

class Database(object):
    def __init__(self, database_dir, create_if_missing=False):
        logger.debug("Opening database %s", database_dir)
        self.db = plyvel.DB(
            database_dir,
            create_if_missing=create_if_missing,
            lru_cache_size=128 * 1024 * 1024)
        self._make_iter()

    def _make_iter(self):
        """Make an iterator for the current database.

        Iterator construction is relatively costly, so reuse it for
        performance reasons. The iterator won't see any data written
        after its construction, but this is not a problem since the data
        set is static.
        """
        self.iter = self.db.iterator(reverse=True)

    def load(self, fp):
        """Load CSV data from an open file-like object"""
        dr = csv.DictReader(fp, delimiter='\t')
        put = self.db.put
        _transform = transform_record
        for n, rec in enumerate(dr, 1):
            key, value = _transform(rec)
            put(key, value)

            if n % 100000 == 0:
                logger.info('Indexed %d records', n)

        # Refresh iterator so that it sees the new data
        self._make_iter()

    def lookup(self, ip):
        """Lookup a single ip address in the database"""
        range_key = incr_ip(ip)
        self.iter.seek(range_key)
        try:
            key, value = next(self.iter)
        except StopIteration:
            # Start of range, no hit
            return None

        # Looked up ip must be within the range
        end = value[:4]
        if ip > end:
            return None

        return value[4:]
<commit_msg>Tiny cleanup in legacy code<commit_after>
import csv
import logging
import math
import struct
import socket

import plyvel
import simplejson as json

logger = logging.getLogger(__name__)

IP_STRUCT = struct.Struct('>L')

def incr_ip(ip):
    n = IP_STRUCT.unpack(ip)[0] + 1
    try:
        return IP_STRUCT.pack(n)
    except struct.error:
        return None

def transform_record(rec):
    start_ip = IP_STRUCT.pack(int(rec['start_ip_int']))
    end_ip = IP_STRUCT.pack(int(rec['end_ip_int']))

    # TODO: carrier_id, tld_id, sld_id, reg_org_id,
    # phone_number_prefix, asn, cidr

    tz = rec['timezone']
    if tz != '999':
        timezone = None
    else:
        timezone = '{:+04d}'.format(int(math.ceil(100 * float(tz))))

    value = json.dumps(dict(
        begin=socket.inet_ntoa(start_ip),
        end=socket.inet_ntoa(end_ip),
        continent=(rec['continent'], 1.),
        country=(rec['country_iso2'],
                 float(rec['country_cf']) / 100),
        state=(rec['state'],
               float(rec['state_cf']) / 100),
        city=(rec['city'],
              float(rec['city_cf']) / 100),
        postal_code=rec['postal_code'],
        type=rec['connectiontype'],
        routing=rec['ip_routingtype'],
        coordinates=(float(rec['latitude']),
                     float(rec['longitude'])),
        timezone=timezone,
        line_speed=rec['linespeed'],
        asn=rec['asn'],
    ))

    # The start IP is the key; the end IP is prepended to the value
    key = start_ip
    value = end_ip + value

    return key, value

class Database(object):
    def __init__(self, database_dir, create_if_missing=False):
        logger.debug("Opening database %s", database_dir)
        self.db = plyvel.DB(
            database_dir,
            create_if_missing=create_if_missing,
            lru_cache_size=128 * 1024 * 1024)
        self._make_iter()

    def _make_iter(self):
        """Make an iterator for the current database.

        Iterator construction is relatively costly, so reuse it for
        performance reasons. The iterator won't see any data written
        after its construction, but this is not a problem since the data
        set is static.
        """
        self.iter = self.db.iterator(reverse=True)

    def load(self, fp):
        """Load CSV data from an open file-like object"""
        dr = csv.DictReader(fp, delimiter='\t')
        put = self.db.put
        _transform = transform_record
        for n, rec in enumerate(dr, 1):
            key, value = _transform(rec)
            put(key, value)

            if n % 100000 == 0:
                logger.info('Indexed %d records', n)

        # Refresh iterator so that it sees the new data
        self._make_iter()

    def lookup(self, ip):
        """Lookup a single ip address in the database"""
        range_key = incr_ip(ip)
        self.iter.seek(range_key)
        try:
            key, value = next(self.iter)
        except StopIteration:
            # Start of range, no hit
            return None

        # Looked up ip must be within the range
        end = value[:4]
        if ip > end:
            return None

        return value[4:]
<|endoftext|>


<commit_before>from datetime import date
from optparse import OptionParser
import requests
from bs4 import BeautifulSoup

def get_command_line_options():
    parser = OptionParser()
    parser.add_option("-q", "--quiet",
                  action="store_const", const=0, dest="verbose")
    parser.add_option("-v", "--verbose",
                  action="store_const", const=1, dest="verbose")
    parser.add_option("--dry-run",
                  action="store_true", dest="dry_run")
    parser.add_option("-m", "--meal", dest="meal",
                      choices=['breakfast', 'lunch', 'dinner'],
                      help="This switch controls the meal this is posted to twitter.")
    (options, args) = parser.parse_args()
    return options

def request_todays_menu_html(date):
    payload = {'dt': date, 'venueName': 'Pulse On Dining Marketplace'}
    r = requests.get("http://www.dineoncampus.com/wpi/webtrition/webtrition.cfm", params=payload)
    assert(r.status_code is 200) #sanity check
    return r.text

MENU_TABLE_CLASS = 'menu-content'
BREAKFAST_TABLE_ID = 'menu-tbl-1'
AFTERNOON_SNACK_TABLE_ID = 'menu-tbl-2'
LUNCH_TABLE_ID = 'menu-tbl-3'
DINNER_TABLE_ID = 'menu-tbl-4'
LATE_NIGHT_TABLE_ID = 'menu-tbl-5'

# iterate through a list pushing everything before the conditional is found into
# the first partition. Then everything after the conditional is found into
# the second partition
def partition(iterable, func):
    first_partition = []
    second_partition = []
    partition_found = False
    for i in iterable:
        partition_found = partition_found or func(i)
        if partition_found:
            second_partition.append(i)
        else:
            first_partition.append(i)
    return (first_partition, second_partition)

def find_breakfast_items(soup):
    breakfast_table = soup.find(id=BREAKFAST_TABLE_ID)
    items = breakfast_table.find_all('span', 'item-name')
    items_text = [i.text.strip() for i in items]
    return items_text

def find_interesting_items(table_soup):
    items = table_soup.find_all('span', class_=['item-name', 'station-name'])
    (dull, interesting_items) = partition(items,
                                          lambda x: x.text.strip().lower() == 'kitchen')
    items_text = [span.text.strip() for span in interesting_items if 'item-name' in span.attrs['class'] ]
    return items_text

def find_lunch_items(soup):
    lunch_table = soup.find(id=LUNCH_TABLE_ID)
    return find_interesting_items(lunch_table)

def find_dinner_items(soup):
    dinner_table = soup.find(id=DINNER_TABLE_ID)
    return find_interesting_items(dinner_table)

def format_items_message(items):
    header = 'Now serving: '
    elipsis = '...'
    link = 'http://bit.ly/wpidaka'
    max_size = 140 - len(header) + len(elipsis)
    items_listed = ', '.join(items)
    if (len(items_listed) > max_size):
        truncated_items_listed = items_listed[0:max_size]
        truncated_items_listed = truncated_items_listed.rpartition(',')[0]
        truncated_items_listed += elipsis
    else:
        truncated_items_listed = items_listed
    return header + truncated_items_listed

def post_msg_to_twitter(msg):
    consumer_key = os.environ.get('DAKA_CONSUMER_KEY')
    consumer_secret = os.environ.get('DAKA_CONSUMER_SECRET')
    access_token = os.environ.get('DAKA_ACCESS_TOKEN')
    access_token_secret = os.environ.get('DAKA_ACCESS_TOKEN_SECRET')

    api = twitter.Api(consumer_key=consumer_key,
                      consumer_secret=consumer_secret,
                      access_token_key=access_token,
                      access_token_secret=access_token_secret)

    api.PostUpdate(msg)

def main():
    options = get_command_line_options()
    if (not options.meal):
        print 'please supply a meal. Usage python wpidaka.py -m (breakfast|lunch|dinner_'
        return
    todays_date = date.today().strftime('%m/%d/%y')
    menu_html = request_todays_menu_html(todays_date)
    soup = BeautifulSoup(menu_html)
    if options.meal == 'breakfast':
        items = find_breakfast_items(soup)
    elif options.meal == 'lunch':
        items = find_lunch_items(soup)
    elif options.meal == 'dinner':
        items = find_dinner_items(soup)

    msg = format_items_message(items)

    if options.verbose:
        print msg

    if options.dry_run:
        return

    assert(len(msg) > 40) # sanity check

    post_msg_to_twitter(msg)

if __name__=="__main__":
    main()
<commit_msg>add missing imports<commit_after>import os
from datetime import date
from optparse import OptionParser
import requests
import twitter
from bs4 import BeautifulSoup

def get_command_line_options():
    parser = OptionParser()
    parser.add_option("-q", "--quiet",
                  action="store_const", const=0, dest="verbose")
    parser.add_option("-v", "--verbose",
                  action="store_const", const=1, dest="verbose")
    parser.add_option("--dry-run",
                  action="store_true", dest="dry_run")
    parser.add_option("-m", "--meal", dest="meal",
                      choices=['breakfast', 'lunch', 'dinner'],
                      help="This switch controls the meal this is posted to twitter.")
    (options, args) = parser.parse_args()
    return options

def request_todays_menu_html(date):
    payload = {'dt': date, 'venueName': 'Pulse On Dining Marketplace'}
    r = requests.get("http://www.dineoncampus.com/wpi/webtrition/webtrition.cfm", params=payload)
    assert(r.status_code is 200) #sanity check
    return r.text

MENU_TABLE_CLASS = 'menu-content'
BREAKFAST_TABLE_ID = 'menu-tbl-1'
AFTERNOON_SNACK_TABLE_ID = 'menu-tbl-2'
LUNCH_TABLE_ID = 'menu-tbl-3'
DINNER_TABLE_ID = 'menu-tbl-4'
LATE_NIGHT_TABLE_ID = 'menu-tbl-5'

# iterate through a list pushing everything before the conditional is found into
# the first partition. Then everything after the conditional is found into
# the second partition
def partition(iterable, func):
    first_partition = []
    second_partition = []
    partition_found = False
    for i in iterable:
        partition_found = partition_found or func(i)
        if partition_found:
            second_partition.append(i)
        else:
            first_partition.append(i)
    return (first_partition, second_partition)

def find_breakfast_items(soup):
    breakfast_table = soup.find(id=BREAKFAST_TABLE_ID)
    items = breakfast_table.find_all('span', 'item-name')
    items_text = [i.text.strip() for i in items]
    return items_text

def find_interesting_items(table_soup):
    items = table_soup.find_all('span', class_=['item-name', 'station-name'])
    (dull, interesting_items) = partition(items,
                                          lambda x: x.text.strip().lower() == 'kitchen')
    items_text = [span.text.strip() for span in interesting_items if 'item-name' in span.attrs['class'] ]
    return items_text

def find_lunch_items(soup):
    lunch_table = soup.find(id=LUNCH_TABLE_ID)
    return find_interesting_items(lunch_table)

def find_dinner_items(soup):
    dinner_table = soup.find(id=DINNER_TABLE_ID)
    return find_interesting_items(dinner_table)

def format_items_message(items):
    header = 'Now serving: '
    elipsis = '...'
    link = 'http://bit.ly/wpidaka'
    max_size = 140 - len(header) + len(elipsis)
    items_listed = ', '.join(items)
    if (len(items_listed) > max_size):
        truncated_items_listed = items_listed[0:max_size]
        truncated_items_listed = truncated_items_listed.rpartition(',')[0]
        truncated_items_listed += elipsis
    else:
        truncated_items_listed = items_listed
    return header + truncated_items_listed

def post_msg_to_twitter(msg):
    consumer_key = os.environ.get('DAKA_CONSUMER_KEY')
    consumer_secret = os.environ.get('DAKA_CONSUMER_SECRET')
    access_token = os.environ.get('DAKA_ACCESS_TOKEN')
    access_token_secret = os.environ.get('DAKA_ACCESS_TOKEN_SECRET')

    api = twitter.Api(consumer_key=consumer_key,
                      consumer_secret=consumer_secret,
                      access_token_key=access_token,
                      access_token_secret=access_token_secret)

    api.PostUpdate(msg)

def main():
    options = get_command_line_options()
    if (not options.meal):
        print 'please supply a meal. Usage python wpidaka.py -m (breakfast|lunch|dinner_'
        return
    todays_date = date.today().strftime('%m/%d/%y')
    menu_html = request_todays_menu_html(todays_date)
    soup = BeautifulSoup(menu_html)
    if options.meal == 'breakfast':
        items = find_breakfast_items(soup)
    elif options.meal == 'lunch':
        items = find_lunch_items(soup)
    elif options.meal == 'dinner':
        items = find_dinner_items(soup)

    msg = format_items_message(items)

    if options.verbose:
        print msg

    if options.dry_run:
        return

    assert(len(msg) > 40) # sanity check

    post_msg_to_twitter(msg)

if __name__=="__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# © 2014 Ian Eure.
# Author: Ian Eure <ian.eure@gmail.com>
#

"""Yar CLI."""

import sys
import format
from optparse import OptionParser, OptionGroup

import serial

from control import Yar
from devices.matcher import match
import devices.unipak2b as unipak2b
import cksum

def summarize(f):
    """Return the function's doc summary."""
    return (f.__doc__ or "").split("\n", 1)[0].strip()

def get_commands():
    return sorted(
        (name[:-4], summarize(f), f)
        for (name, f) in globals().iteritems() if name.endswith("_cmd"))

def get_commands_map():
    return dict((name, f) for (name, _, f) in get_commands())

def get_parser():
    usage = "Usage: %prog [OPTIONS] COMMAND [ARG ARG ...]\n\nCommands:\n\n"

    for (name, doc, _) in get_commands():
        usage += "  %s - %s\n" % (name, doc)

    p = OptionParser(usage[:-1])

    cg = OptionGroup(p, "Connection options")

    cg.add_option("--port", "-p", default="/dev/cu.usbserial",
                  help="Serial port programmer is connected to")
    cg.add_option("--baud", "-b", default=9600,
                  help="Communication speed. Default: 9600.")
    cg.add_option("--parity", default="N81", help="Connection parity/data/stop. Default: N81")
    p.add_option_group(cg)

    p.add_option("--pak", help="Pak instaled in the programmer: uni, uni2, uni2b, logic, gang")

    p.add_option("--device", help="Device ID; looks up family/pinout")

    p.add_option("--family", help="Device family")
    p.add_option("--pinout", help="Device pinout")
    p.add_option("--detect", help="Autodetect device (on some Paks)")

    return p

def load_pak(name):
    """Return Pak  data."""
    if name == "uni2b":
        return unipak2b

def connect(yar):
    """Ensure that the programmer is connected."""
    yar.flush()
    c = yar.ping()
    if c:
        return

    sys.stdout.write("Put device in remote mode: SELECT F1 START START")
    while not yar.ping():
        pass
    print " connected."

def configure(yar, opts):
    """Configure the programmer for the given options"""
    connect()
    if opts.detect:
        yar.select(0xBD)
        yar.set_device(0xFF, 0xFF)

    if opts.family and opts.pinout:
        yar.set_device(int(opts.family, 16),
                       int(opts.pinout, 16))

def await_operator(yar, msg):
    sys.stdout.write(msg or "Press START...")
    r = yar.await_start()
    if not r:
        raise Exception("Timed out")
    print "ok."
    return r

def checksum_cmd(yar, *args):
    """Checksum files"""
    for file_ in args:
        with open(file_, 'rb') as fp:
            print "%s - %06x" % (
                file_, cksum.checksum(cksum.file_to_bytes(fp)))
    return 0

def dumpram_cmd(yar, output):
    """Dump programmer RAM to file."""
    yar.flush()
    if not yar.ping():
        print yar.last_error()
        return 1

    print "Dumping"
    with open(output, 'wb') as fp:
        yar.dump_to(fp)

    return 0

def loaddev_cmd(yar):
    """Load device contents into programmer RAM"""
    yar.clear_ram()
    await_operator("Insert device into indicated socket, then press START...")
    yar.load()
    return 0

def loadfile_cmd(yar, file_):
    """Load file contents into programmer RAM"""
    connect(yar)
    yar.clear_ram()
    with open(file_, "rb") as inp:
        yar.load_from(inp)

def lookup_cmd(yar, device):
    """Look up a device family/pinout"""
    pak = load_pak("uni2b")
    try:
        (family, pinout) = match(pak.DEVICES, device)
        print "%s family %03x pinout %03x" % (device, family, pinout)
        return 0
    except ValueError, e:
        print e
        return 1

def main():
    """Yar main entry point"""
    p = get_parser()
    (opts, cmd_args) = p.parse_args()
    (cmd, args) = (cmd_args[0], cmd_args[1:])
    c = Yar(opts.port)           # FIXME, use all settings
    cmds = get_commands_map()
    if cmd not in cmds:
        print "No such command: `%s'" % cmd
        p.print_help()
        sys.exit(-1)
    sys.exit(cmds[cmd](c, *args))
<commit_msg>Refactor CLI:<commit_after># -*- coding: utf-8 -*-
#
# © 2014 Ian Eure.
# Author: Ian Eure <ian.eure@gmail.com>
#

"""Yar CLI."""

import sys
import re
import format
from optparse import OptionParser, OptionGroup

import serial

from control import Yar
from devices.matcher import match
import yar.pak as pak
import yar.cksum as cksum

def summarize(f):
    """Return the function's doc summary."""
    return (f.__doc__ or "").split("\n", 1)[0].strip()

def get_commands():
    return sorted(
        (name[:-4], summarize(f), f)
        for (name, f) in globals().iteritems() if name.endswith("_cmd"))

def get_commands_map():
    return dict((name, f) for (name, _, f) in get_commands())

def expand_command(cmd, cmds):
    """Match a command, returning the name of the full, unambiguous cmd."""
    cmd_re = re.compile("^" + ".*".join(cmd) + ".*", re.I)
    cs = [name for name in cmds if cmd_re.match(name)]
    if len(cs) == 1:
        return cs[0]
    return None

def get_parser():
    usage = "Usage: %prog [OPTIONS] COMMAND [ARG ARG ...]\n\nCommands:\n\n"

    for (name, doc, _) in get_commands():
        usage += "  %s - %s\n" % (name, doc)

    p = OptionParser(usage[:-1])

    cg = OptionGroup(p, "Connection options")

    cg.add_option("--port", "-p", default="/dev/cu.usbserial",
                  help="Serial port programmer is connected to")
    cg.add_option("--baud", "-b", default=9600,
                  help="Communication speed. Default: 9600.")
    cg.add_option("--parity", default="N81", help="Connection parity/data/stop. Default: N81")
    p.add_option_group(cg)

    dg = OptionGroup(p, "Device options")
    dg.add_option("--pak", help="Pak instaled in the programmer: unipak, unipak2, unipak2b, logicpak, gangpak")

    dg.add_option("--device", help="Device ID; looks up family/pinout")

    dg.add_option("--family", help="Device family")
    dg.add_option("--pinout", help="Device pinout")
    dg.add_option("--detect", help="Autodetect device (on some Paks)",
                  action="store_true")
    p.add_option_group(dg)

    return p

def await_operator(yar, msg):
    sys.stdout.write(msg or "Press START...")
    sys.stdout.flush()
    r = yar.await_start()
    if not r:
        raise Exception("Timed out")
    print "ok."
    return r

 # Commands

def checksum_cmd(_, *args):
    """Checksum files"""
    for file_ in args:
        with open(file_, 'rb') as fp:
            print "%s - %06x" % (
                file_, cksum.checksum(cksum.file_to_bytes(fp)))
    return 0

def dumpram_cmd(s, output):
    """Dump programmer RAM to file."""
    yar = s.yar()
    s.yar().flush()
    if not yar.ping():
        print yar.last_error()
        return 1

    print "Dumping"
    with open(output, 'wb') as fp:
        yar.dump_to(fp)

    return 0

def loaddev_cmd(s):
    """Load device contents into programmer RAM"""
    yar = s.yar()
    yar.clear_ram()
    await_operator(
        yar, "Insert device into indicated socket, then press START...")
    sum = yar.load()
    print "Checksum: %06x" % sum
    return 0

def loadfile_cmd(yar, file_):
    """Load file contents into programmer RAM"""
    connect(yar)
    yar.clear_ram()
    with open(file_, "rb") as inp:
        yar.load_from(inp)

def lookup_cmd(s, device):
    """Look up a device family/pinout"""
    try:
        (family, pinout) = match(s.pak().DEVICES, device)
        print "%s family %03x pinout %03x" % (device, family, pinout)
        return 0
    except ValueError, e:
        print e
        return 1

 # Main code

class GlobalState():

    _yar = None
    opts = None
    args = None

    def __init__(self, opts, args):
        self.opts = opts
        self.args = args

    def yar(self):
        """Return a configured Yar instance."""
        if not self._yar:
            self._yar = self._construct()
            self._connect(self._yar)
            self._configure(self._yar)
        return self._yar

    def pak(self):
        """Return the configured Pak."""
        return pak.load(self.opts.pak)

    def _construct(self):
        return Yar(self.opts.port)           # FIXME, use all settings

    def _connect(self, yar):
        """Ensure that the programmer is connected."""
        yar.flush()
        c = yar.ping()
        if c:
            return

            sys.stdout.write("Put device in remote mode: SELECT F1 START START")
            while not yar.ping():
                pass
                print " connected."

    def _configure(self, yar):
        """Configure the programmer for the given options"""
        if self.opts.detect:
            yar.select(0xBD)
            yar.set_device(0xFF, 0xFF)

        if self.opts.device:
            (family, pinout) = match(self.pak().DEVICES, self.opts.device)
            yar.set_device(family, pinout)

        if self.opts.family and self.opts.pinout:
                yar.set_device(int(self.opts.family, 16),
                               int(self.opts.pinout, 16))

def main():
    """Yar main entry point"""
    p = get_parser()
    (opts, cmd_args) = p.parse_args()
    if not cmd_args:
        p.print_help()
        sys.exit(-1)
    (cmd, args) = (cmd_args[0], cmd_args[1:])
    cmds = get_commands_map()
    cmd_ = expand_command(cmd, cmds.keys())
    cmd = cmd_ or cmd
    if cmd not in cmds:
        print "No such command: `%s'" % cmd
        p.print_help()
        sys.exit(-1)

    s = GlobalState(opts, args)

    # try:
    sys.exit(cmds[cmd](s, *args))
    # except Exception, e:
    #     print "%s" % e
    #     sys.exit(-2)
<|endoftext|>


<commit_before>import cookielib
from rrtest import FatalError
from rrtest.check import ExpectedError
from rrtest.interaction import Interaction
from rrtest.interaction import Operation
from rrtest.interaction import InteractionNeeded
from rrtest.status import STATUSCODE

__author__ = 'rolandh'

class Conversation():
    def __init__(self, client, config, trace, interaction,
                 check_factory=None, msg_factory=None,
                 features=None, verbose=False):
        self.client = client
        self.client_config = config
        self.trace = trace
        self.test_output = []
        self.features = features
        self.verbose = verbose
        self.check_factory = check_factory
        self.msg_factory = msg_factory

        self.cjar = {"browser": cookielib.CookieJar(),
                     "rp": cookielib.CookieJar(),
                     "service": cookielib.CookieJar()}

        self.last_response = None
        self.last_content = None
        self.response = None
        self.interaction = Interaction(self.client, interaction)
        self.exception = None

    def check_severity(self, stat):
        if stat["status"] >= 4:
            self.trace.error("WHERE: %s" % stat["id"])
            self.trace.error("STATUS:%s" % STATUSCODE[stat["status"]])
            try:
                self.trace.error("HTTP STATUS: %s" % stat["http_status"])
            except KeyError:
                pass
            try:
                self.trace.error("INFO: %s" % stat["message"])
            except KeyError:
                pass

            raise FatalError

    def do_check(self, test, **kwargs):
        if isinstance(test, basestring):
            chk = self.check_factory(test)(**kwargs)
        else:
            chk = test(**kwargs)
        stat = chk(self, self.test_output)
        self.check_severity(stat)

    def err_check(self, test, err=None, bryt=True):
        if err:
            self.exception = err
        chk = self.check_factory(test)()
        chk(self, self.test_output)
        if bryt:
            raise FatalError(test)

    def test_sequence(self, sequence):
        for test in sequence:
            if isinstance(test, tuple):
                test, kwargs = test
            else:
                kwargs = {}
            self.do_check(test, **kwargs)
            if test == ExpectedError:
                return False
        return True

    def my_endpoints(self):
        pass

    def intermit(self):
        _response = self.last_response
        if _response.status_code >= 400:
            done = True
        else:
            done = False

        url = _response.url
        content = _response.text
        while not done:
            rdseq = []
            while _response.status_code in [302, 301, 303]:
                url = _response.headers["location"]
                if url in rdseq:
                    raise FatalError("Loop detected in redirects")
                else:
                    rdseq.append(url)
                    if len(rdseq) > 8:
                        raise FatalError(
                            "Too long sequence of redirects: %s" % rdseq)

                self.trace.reply("REDIRECT TO: %s" % url)
                # If back to me
                for_me = False
                for redirect_uri in self.my_endpoints():
                    if url.startswith(redirect_uri):
                        # Back at the RP
                        self.client.cookiejar = self.cjar["rp"]
                        for_me = True

                if for_me:
                    done = True
                    break
                else:
                    try:
                        _response = self.client.send(url, "GET")
                    except Exception, err:
                        raise FatalError("%s" % err)

                    content = _response.text
                    self.trace.reply("CONTENT: %s" % content)
                    self.position = url
                    self.last_content = content
                    self.response = _response

                    self.do_check("check-http-response")

            if done or url is None:
                break

            _base = url.split("?")[0]

            try:
                _spec = self.interaction.pick_interaction(_base, content)
            except InteractionNeeded:
                self.position = url
                self.trace.error("Page Content: %s" % content)
                raise
            except KeyError:
                self.position = url
                self.trace.error("Page Content: %s" % content)
                self.err_check("interaction-needed")

            if len(_spec) > 2:
                self.trace.info(">> %s <<" % _spec["page-type"])
                if _spec["page-type"] == "login":
                    self.login_page = content

            _op = Operation(_spec["control"])

            try:
                _response = _op(self.client, self, self.trace, url,
                                _response, content, self.features)
                if isinstance(_response, dict):
                    self.last_response = _response
                    self.last_content = _response
                    return _response
                content = _response.text
                self.position = url
                self.last_content = content
                self.response = _response

                self.do_check("check-http-response")
            except FatalError:
                raise
            except Exception, err:
                self.err_check("exception", err, False)

        self.last_response = _response
        try:
            self.last_content = _response.text
        except AttributeError:
            self.last_content = None

    def init(self, phase):
        pass

    def setup_request(self):
        pass

    def send(self):
        pass

    def handle_result(self):
        pass

    def do_query(self):
        self.setup_request()
        self.send()
        if not self.handle_result():
            self.intermit()
            self.handle_result()

    def do_sequence(self, oper):

        try:
            self.test_sequence(oper["tests"]["pre"])
        except KeyError:
            pass

        for phase in oper["sequence"]:
            self.init(phase)
            try:
                self.do_query()
            except InteractionNeeded:
                self.test_output.append({"status": 4,
                                         "message": self.last_content,
                                         "id": "exception",
                                         "name": "interaction needed"})
            except FatalError:
                raise
            except Exception, err:
                self.err_check("exception", err)

        try:
            self.test_sequence(oper["tests"]["post"])
        except KeyError:
            pass
<commit_msg>Use the status code defined for "interaction needed".<commit_after>import cookielib
from rrtest import FatalError
from rrtest.check import ExpectedError
from rrtest.check import INTERACTION
from rrtest.interaction import Interaction
from rrtest.interaction import Operation
from rrtest.interaction import InteractionNeeded
from rrtest.status import STATUSCODE

__author__ = 'rolandh'

class Conversation():
    def __init__(self, client, config, trace, interaction,
                 check_factory=None, msg_factory=None,
                 features=None, verbose=False):
        self.client = client
        self.client_config = config
        self.trace = trace
        self.test_output = []
        self.features = features
        self.verbose = verbose
        self.check_factory = check_factory
        self.msg_factory = msg_factory

        self.cjar = {"browser": cookielib.CookieJar(),
                     "rp": cookielib.CookieJar(),
                     "service": cookielib.CookieJar()}

        self.last_response = None
        self.last_content = None
        self.response = None
        self.interaction = Interaction(self.client, interaction)
        self.exception = None

    def check_severity(self, stat):
        if stat["status"] >= 4:
            self.trace.error("WHERE: %s" % stat["id"])
            self.trace.error("STATUS:%s" % STATUSCODE[stat["status"]])
            try:
                self.trace.error("HTTP STATUS: %s" % stat["http_status"])
            except KeyError:
                pass
            try:
                self.trace.error("INFO: %s" % stat["message"])
            except KeyError:
                pass

            raise FatalError

    def do_check(self, test, **kwargs):
        if isinstance(test, basestring):
            chk = self.check_factory(test)(**kwargs)
        else:
            chk = test(**kwargs)
        stat = chk(self, self.test_output)
        self.check_severity(stat)

    def err_check(self, test, err=None, bryt=True):
        if err:
            self.exception = err
        chk = self.check_factory(test)()
        chk(self, self.test_output)
        if bryt:
            raise FatalError(test)

    def test_sequence(self, sequence):
        for test in sequence:
            if isinstance(test, tuple):
                test, kwargs = test
            else:
                kwargs = {}
            self.do_check(test, **kwargs)
            if test == ExpectedError:
                return False
        return True

    def my_endpoints(self):
        pass

    def intermit(self):
        _response = self.last_response
        if _response.status_code >= 400:
            done = True
        else:
            done = False

        url = _response.url
        content = _response.text
        while not done:
            rdseq = []
            while _response.status_code in [302, 301, 303]:
                url = _response.headers["location"]
                if url in rdseq:
                    raise FatalError("Loop detected in redirects")
                else:
                    rdseq.append(url)
                    if len(rdseq) > 8:
                        raise FatalError(
                            "Too long sequence of redirects: %s" % rdseq)

                self.trace.reply("REDIRECT TO: %s" % url)
                # If back to me
                for_me = False
                for redirect_uri in self.my_endpoints():
                    if url.startswith(redirect_uri):
                        # Back at the RP
                        self.client.cookiejar = self.cjar["rp"]
                        for_me = True

                if for_me:
                    done = True
                    break
                else:
                    try:
                        _response = self.client.send(url, "GET")
                    except Exception, err:
                        raise FatalError("%s" % err)

                    content = _response.text
                    self.trace.reply("CONTENT: %s" % content)
                    self.position = url
                    self.last_content = content
                    self.response = _response

                    self.do_check("check-http-response")

            if done or url is None:
                break

            _base = url.split("?")[0]

            try:
                _spec = self.interaction.pick_interaction(_base, content)
            except InteractionNeeded:
                self.position = url
                self.trace.error("Page Content: %s" % content)
                raise
            except KeyError:
                self.position = url
                self.trace.error("Page Content: %s" % content)
                self.err_check("interaction-needed")

            if len(_spec) > 2:
                self.trace.info(">> %s <<" % _spec["page-type"])
                if _spec["page-type"] == "login":
                    self.login_page = content

            _op = Operation(_spec["control"])

            try:
                _response = _op(self.client, self, self.trace, url,
                                _response, content, self.features)
                if isinstance(_response, dict):
                    self.last_response = _response
                    self.last_content = _response
                    return _response
                content = _response.text
                self.position = url
                self.last_content = content
                self.response = _response

                self.do_check("check-http-response")
            except FatalError:
                raise
            except Exception, err:
                self.err_check("exception", err, False)

        self.last_response = _response
        try:
            self.last_content = _response.text
        except AttributeError:
            self.last_content = None

    def init(self, phase):
        pass

    def setup_request(self):
        pass

    def send(self):
        pass

    def handle_result(self):
        pass

    def do_query(self):
        self.setup_request()
        self.send()
        if not self.handle_result():
            self.intermit()
            self.handle_result()

    def do_sequence(self, oper):

        try:
            self.test_sequence(oper["tests"]["pre"])
        except KeyError:
            pass

        for phase in oper["sequence"]:
            self.init(phase)
            try:
                self.do_query()
            except InteractionNeeded:
                self.test_output.append({"status": INTERACTION,
                                         "message": self.last_content,
                                         "id": "exception",
                                         "name": "interaction needed"})
            except FatalError:
                raise
            except Exception, err:
                self.err_check("exception", err)

        try:
            self.test_sequence(oper["tests"]["post"])
        except KeyError:
            pass
<|endoftext|>


<commit_before>from model import *
import util
import datetime
from sqlalchemy import and_

#MNPY_RECEIVER = 'weiyi@papayamobile.com'
MNPY_RECEIVER = 'appflood_engineer@papayamobile.com'

MNPY_SNIPPET = """

%s

"""

def notify(body='\xe4\xbb\x80\xe4\xb9\x88\xe9\x83\xbd\xe6\xb2\xa1\xe6\x9c\x89......'):
    title = '[Python]\xe7\xa0\x81\xe5\x86\x9c\xe5\x91\xa8\xe5\x88\x8aPython\xe7\x9b\xb8\xe5\x85\xb3\xe4\xb8\xbb\xe9\xa2\x98\xe6\x9b\xb4\xe6\x96\xb0'
    util.send_email(title, body, MNPY_RECEIVER)

def task():
    today = datetime.datetime.now()
    yesterday = today - datetime.timedelta(1)
    mps = ManongPython.query.filter(and_(ManongPython.create_time>yesterday, ManongPython.create_time<today)).all()
    #mps = ManongPython.query.all()
    print '=============', len(mps)
    body = '<meta charset="UTF-8">'
    body += MNPY_SNIPPET % ('http://appflood.ml/mnpy.html', u'\u6240\u6709\u7ed3\u679c\u7684\u9875\u9762')
    if mps:
        for mp in mps:
            body += MNPY_SNIPPET % (mp.url, mp.title)
        notify(body)
        html()

def html():
    mps = ManongPython.query.all()
    body = ''
    for mp in mps:
        body += MNPY_SNIPPET % (mp.url, mp.title)
    #print body
    with open('/var/server/www/mnpy.html', 'w') as f:
        f.write(body.encode('utf8'))

<commit_msg>fix encoding html;<commit_after>from model import *
import util
import datetime
from sqlalchemy import and_

#MNPY_RECEIVER = 'weiyi@papayamobile.com'
MNPY_RECEIVER = 'appflood_engineer@papayamobile.com'

MNPY_SNIPPET = """

%s

"""

def notify(body='\xe4\xbb\x80\xe4\xb9\x88\xe9\x83\xbd\xe6\xb2\xa1\xe6\x9c\x89......'):
    title = '[Python]\xe7\xa0\x81\xe5\x86\x9c\xe5\x91\xa8\xe5\x88\x8aPython\xe7\x9b\xb8\xe5\x85\xb3\xe4\xb8\xbb\xe9\xa2\x98\xe6\x9b\xb4\xe6\x96\xb0'
    util.send_email(title, body, MNPY_RECEIVER)

def task():
    today = datetime.datetime.now()
    yesterday = today - datetime.timedelta(1)
    mps = ManongPython.query.filter(and_(ManongPython.create_time>yesterday, ManongPython.create_time<today)).all()
    #mps = ManongPython.query.all()
    print '=============', len(mps)
    body = MNPY_SNIPPET % ('http://appflood.ml/mnpy.html', u'\u6240\u6709\u7ed3\u679c\u7684\u9875\u9762')
    if mps:
        for mp in mps:
            body += MNPY_SNIPPET % (mp.url, mp.title)
        notify(body)
        html()

def html():
    mps = ManongPython.query.all()
    body = '<meta charset="UTF-8">'
    for mp in mps:
        body += MNPY_SNIPPET % (mp.url, mp.title)
    #print body
    with open('/var/server/www/mnpy.html', 'w') as f:
        f.write(body.encode('utf8'))

<|endoftext|>


<commit_before># get level-1 model for each dataset, for example the averaged model
# for data 5-57-21k, which includes predictions from xgboost,
# random forest ..., projected features from pca, ica, and selected features
# from some models, the model should be able to give prediction score
# for each biz_id, used for level-2 model

from __future__ import division
from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

from os import path
import os
import pickle
import argparse

import numpy as np
import xgboost as xgb
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.feature_selection import SelectFromModel
from sets import Set
from utils import *

np.random.seed(123213)

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--data',
                        choices=['21k', 'v3', 'colHist'],
                        default="21k")
    parser.add_argument('--prob',
                        choices=['75', '50'],
                        default='75')
    parser.add_argument('--reps',
                        choices=['5', '9'],
                        default='5')
    parser.add_argument('--yix', type=int, default=0)
    return parser.parse_args()

def get_data_train(args):
    data_dir = path.join("../feature/", args.reps + "_" + args.prob)
    if args.data == '21k':
        X_path = path.join(data_dir, "X_train.npy")
        y_path = path.join(data_dir, "y_train.npy")
        X = np.load(X_path)
        y = np.load(y_path)
        sel_range = range(0, 2048)
        return X[:, sel_range], y[:, args.yix]
    elif args.data == 'colHist':
        X_path = path.join(data_dir, "X_train.npy")
        y_path = path.join(data_dir, "y_train.npy")
        X = np.load(X_path)
        y = np.load(y_path)
        sel_range = range(2048, 2048 + 772)
        return X[:, sel_range], y[:, args.yix]
    elif args.data == 'v3':
        X_path = path.join(data_dir, "X_train.npy")
        y_path = path.join(data_dir, "y_train.npy")
        X = np.load(X_path)
        y = np.load(y_path)
        sel_range = range(2048 + 772, 6916)
        return X[:, sel_range], y[:, args.yix]

def get_data_test(args):
    data_dir = path.join("../feature/", args.reps + "_" + args.prob)
    if args.data == '21k':
        X_path = path.join(data_dir, "X_test.npy")
        biz_path = path.join(data_dir, "bizlist.npy")
        X = np.load(X_path)
        biz_list = np.load(biz_path)
        sel_range = range(0, 2048)
        return X[:, sel_range], biz_list
    elif args.data == 'colHist':
        X_path = path.join(data_dir, "X_test.npy")
        biz_path = path.join(data_dir, "bizlist.npy")
        X = np.load(X_path)
        biz_list = np.load(biz_path)
        sel_range = range(2048, 2048 + 772)
        return X[:, sel_range], biz_list
    elif args.data == 'v3':
        X_path = path.join(data_dir, "X_test.npy")
        biz_path = path.join(data_dir, "bizlist.npy")
        X = np.load(X_path)
        biz_list = np.load(biz_path)
        sel_range = range(2048 + 772, 6916)
        return X[:, sel_range], biz_list

class level1_model:
    def get_fitted(self):
        pass

    def get_feature_list(self, X_train=None, y_train=None):
        pass

    def predict(self, X_test):
        pass

class level1_xgboost(level1_model):
    def __init__(self, args):
        self.args = args
        model_dir = "_".join(("xgboost_f1", args.reps, args.prob, args.data))
        model_name = "model_" + str(args.yix) + ".pkl"
        outfold_name = "outFold_" + str(args.yix) + ".npy"

        model_path = path.join(path.dirname(__file__), model_dir, model_name)
        outfold_path = path.join(
            path.dirname(__file__), model_dir, outfold_name)

        with open(model_path, 'r') as f:
            self.clf = pickle.load(f)
        self.outfold_pred = np.load(outfold_path)

    def get_fitted(self):
        return np.array([self.outfold_pred]).T

    def get_feature_list(self, X_train=None, y_train=None):
        f_score = self.clf.get_fscore()
        k = np.array(f_score.keys())
        v = np.array(f_score.values())
        f_ix = k[v > np.mean(v)]
        f_list = map(lambda x: int(x[1:]), f_ix)
        return {args.data: f_list}

    def predict(self, X_test):
        dtest = xgb.DMatrix(X_test)
        return np.array([self.clf.predict(dtest, output_margin=True)]).T

class level1_rf(level1_model):
    def __init__(self, args):
        self.args = args
        model_dir = "_".join(("randomForest", args.reps, args.prob, args.data))
        model_name = "model_" + str(args.yix) + ".pkl"
        outfold_name = "outFold_" + str(args.yix) + ".npy"
        param_name = "param_" + str(args.yix) + ".pkl"

        model_path = path.join(path.dirname(__file__), model_dir, model_name)
        outfold_path = path.join(
            path.dirname(__file__), model_dir, outfold_name)
        param_path = path.join(path.dirname(__file__), model_dir, param_name)

        with open(model_path, 'r') as f:
            self.clf = pickle.load(f)
        with open(param_path, 'r') as f:
            self.params = pickle.load(f)
        self.outfold_pred = np.load(outfold_path)

    def get_fitted(self):
        return np.array([self.outfold_pred]).T

    def get_feature_list(self, X_train, y_train):
        weight = y_train.shape[0] / (2 * np.bincount(y_train))
        sample_weight = np.array([weight[i] for i in y_train])
        clf = RandomForestClassifier(**self.params)
        clf.fit(X_train, y_train, sample_weight)
        score = np.array(clf.feature_importances_)
        sel_ix = np.arange(score.shape[0])[score > np.mean(score)]
        return {args.data: list(sel_ix)}

    def predict(self, X_test):
        pred_list = []
        for cclf in self.clf:
            pred = cclf.predict_proba(X_test)[:, 1]
            pred_list.append(pred)
        return np.array([np.mean(np.array(pred_list), axis=0)]).T

class level1_extree(level1_model):
    def __init__(self, args):
        self.args = args
        model_dir = "_".join(("extraTree", args.reps, args.prob, args.data))
        model_name = "model_" + str(args.yix) + ".pkl"
        outfold_name = "outFold_" + str(args.yix) + ".npy"
        param_name = "param_" + str(args.yix) + ".pkl"

        model_path = path.join(path.dirname(__file__), model_dir, model_name)
        outfold_path = path.join(
            path.dirname(__file__), model_dir, outfold_name)
        param_path = path.join(path.dirname(__file__), model_dir, param_name)

        with open(model_path, 'r') as f:
            self.clf = pickle.load(f)
        with open(param_path, 'r') as f:
            self.params = pickle.load(f)
        self.outfold_pred = np.load(outfold_path)

    def get_fitted(self):
        return np.array([self.outfold_pred]).T

    def get_feature_list(self, X_train, y_train):
        weight = y_train.shape[0] / (2 * np.bincount(y_train))
        sample_weight = np.array([weight[i] for i in y_train])
        clf = ExtraTreesClassifier(**self.params)
        clf.fit(X_train, y_train, sample_weight)
        score = np.array(clf.feature_importances_)
        sel_ix = np.arange(score.shape[0])[score > np.mean(score)]
        return {args.data: list(sel_ix)}

    def predict(self, X_test):
        pred_list = []
        for cclf in self.clf:
            pred = cclf.predict_proba(X_test)[:, 1]
            pred_list.append(pred)
        return np.array([np.mean(np.array(pred_list), axis=0)]).T

class level1_pca(level1_model):
    def __init__(self, args):
        self.args = args
        model_dir = "_".join(("pca", args.reps, args.prob, args.data))
        model_name = "model.pkl"
        outfold_name = "outFold.npy"
        param_name = "param.pkl"

        model_path = path.join(path.dirname(__file__), model_dir, model_name)
        outfold_path = path.join(
            path.dirname(__file__), model_dir, outfold_name)
        param_path = path.join(path.dirname(__file__), model_dir, param_name)

        with open(model_path, 'r') as f:
            self.clf = pickle.load(f)
        with open(param_path, 'r') as f:
            self.params = pickle.load(f)
        self.outfold_pred = np.load(outfold_path)

    def get_fitted(self):
        return self.outfold_pred

    def get_feature_list(self, X_train=None, y_train=None):
        return {args.data: []}

    def predict(self, X_test):
        return self.clf.transform(X_test)

class level1_ica(level1_model):
    def __init__(self, args):
        self.args = args
        model_dir = "_".join(("ica", args.reps, args.prob, args.data))
        model_name = "model.pkl"
        outfold_name = "outFold.npy"
        param_name = "param.pkl"

        model_path = path.join(path.dirname(__file__), model_dir, model_name)
        outfold_path = path.join(
            path.dirname(__file__), model_dir, outfold_name)
        param_path = path.join(path.dirname(__file__), model_dir, param_name)

        with open(model_path, 'r') as f:
            self.clf = pickle.load(f)
        with open(param_path, 'r') as f:
            self.params = pickle.load(f)
        self.outfold_pred = np.load(outfold_path)

    def get_fitted(self):
        return self.outfold_pred

    def get_feature_list(self, X_train=None, y_train=None):
        return {args.data: []}

    def predict(self, X_test):
        return self.clf.transform(X_test)

class level1_glm(level1_model):
    def __init__(self, args):
        self.args = args
        model_dir = "_".join(("glm_l1", args.reps, args.prob, args.data))
        model_name = "model_" + str(args.yix) + ".pkl"
        outfold_name = "outFold_" + str(args.yix) + ".npy"
        param_name = "param_" + str(args.yix) + ".pkl"

        model_path = path.join(path.dirname(__file__), model_dir, model_name)
        outfold_path = path.join(
            path.dirname(__file__), model_dir, outfold_name)
        param_path = path.join(path.dirname(__file__), model_dir, param_name)

        with open(model_path, 'r') as f:
            self.clf = pickle.load(f)
        with open(param_path, 'r') as f:
            self.params = pickle.load(f)
        self.outfold_pred = np.load(outfold_path)

    def get_fitted(self):
        return np.array([self.outfold_pred]).T

    def get_feature_list(self, X_train=None, y_train=None):
        sel_ix = SelectFromModel(self.clf.steps[1][1],
                                 threshold="4 * mean",
                                 prefit=True).get_support(True)
        return {args.data: list(sel_ix)}

    def predict(self, X_test):
        return np.array([self.clf.predict_proba(X_test)[:, 1]]).T

def get_new_train(args, X):
    models = [level1_xgboost(args), level1_rf(args), level1_extree(args),
              level1_glm(args), level1_pca(args), level1_ica(args)]
    X_new = None
    for model in models:
        fitted = model.get_fitted()
        if X_new is None:
            X_new = fitted
        else:
            X_new = np.concatenate((X_new, fitted), axis=1)
    return X_new

def get_new_test(args, X):
    models = [level1_xgboost(args), level1_rf(args), level1_extree(args),
              level1_glm(args), level1_pca(args), level1_ica(args)]
    X_new = None
    for model in models:
        preds = model.predict(X)
        if X_new is None:
            X_new = preds
        else:
            X_new = np.concatenate((X_new, preds), axis=1)
    return X_new

def get_sel_features(args, X_train, y_train):
    models = [level1_rf(args), level1_extree(args), level1_glm(args)]
    feature_list = Set()
    for model in models:
        feature_list.update(
            model.get_feature_list(X_train, y_train).values()[0])
    return sorted(feature_list)

args = parse_args()
X_train, y_train = get_data_train(args)
X_test, biz_list = get_data_test(args)

X_train_new = get_new_train(args, X_train)
X_test_new = get_new_test(args, X_test)
feature_list = get_sel_features(args, X_train, y_train)
print(args.data)
print(feature_list)

X_train_new = np.hstack((X_train_new, X_train[:, feature_list]))
X_test_new = np.hstack((X_test_new, X_test[:, feature_list]))

save_dir = "_".join(("../level2-feature/" + args.reps, args.prob,
                    args.data, str(args.yix)))

if not path.exists(save_dir):
    os.mkdir(save_dir)
np.save(path.join(save_dir, "X_train.npy"), X_train_new)
np.save(path.join(save_dir, "y_train.npy"), y_train)
np.save(path.join(save_dir, "X_test.npy"), X_test_new)
np.save(path.join(save_dir, "feature_list.npy"), feature_list)
<commit_msg>change name for the name folder<commit_after># get level-1 model for each dataset, for example the averaged model
# for data 5-57-21k, which includes predictions from xgboost,
# random forest ..., projected features from pca, ica, and selected features
# from some models, the model should be able to give prediction score
# for each biz_id, used for level-2 model

from __future__ import division
from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

from os import path
import os
import pickle
import argparse

import numpy as np
import xgboost as xgb
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.feature_selection import SelectFromModel
from sets import Set
from utils import *

np.random.seed(123213)

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--data',
                        choices=['21k', 'v3', 'colHist'],
                        default="21k")
    parser.add_argument('--prob',
                        choices=['75', '50'],
                        default='75')
    parser.add_argument('--reps',
                        choices=['5', '9'],
                        default='5')
    parser.add_argument('--yix', type=int, default=0)
    return parser.parse_args()

def get_data_train(args):
    data_dir = path.join("../feature/", args.reps + "_" + args.prob)
    if args.data == '21k':
        X_path = path.join(data_dir, "X_train.npy")
        y_path = path.join(data_dir, "y_train.npy")
        X = np.load(X_path)
        y = np.load(y_path)
        sel_range = range(0, 2048)
        return X[:, sel_range], y[:, args.yix]
    elif args.data == 'colHist':
        X_path = path.join(data_dir, "X_train.npy")
        y_path = path.join(data_dir, "y_train.npy")
        X = np.load(X_path)
        y = np.load(y_path)
        sel_range = range(2048, 2048 + 772)
        return X[:, sel_range], y[:, args.yix]
    elif args.data == 'v3':
        X_path = path.join(data_dir, "X_train.npy")
        y_path = path.join(data_dir, "y_train.npy")
        X = np.load(X_path)
        y = np.load(y_path)
        sel_range = range(2048 + 772, 6916)
        return X[:, sel_range], y[:, args.yix]

def get_data_test(args):
    data_dir = path.join("../feature/", args.reps + "_" + args.prob)
    if args.data == '21k':
        X_path = path.join(data_dir, "X_test.npy")
        biz_path = path.join(data_dir, "bizlist.npy")
        X = np.load(X_path)
        biz_list = np.load(biz_path)
        sel_range = range(0, 2048)
        return X[:, sel_range], biz_list
    elif args.data == 'colHist':
        X_path = path.join(data_dir, "X_test.npy")
        biz_path = path.join(data_dir, "bizlist.npy")
        X = np.load(X_path)
        biz_list = np.load(biz_path)
        sel_range = range(2048, 2048 + 772)
        return X[:, sel_range], biz_list
    elif args.data == 'v3':
        X_path = path.join(data_dir, "X_test.npy")
        biz_path = path.join(data_dir, "bizlist.npy")
        X = np.load(X_path)
        biz_list = np.load(biz_path)
        sel_range = range(2048 + 772, 6916)
        return X[:, sel_range], biz_list

class level1_model:
    def get_fitted(self):
        pass

    def get_feature_list(self, X_train=None, y_train=None):
        pass

    def predict(self, X_test):
        pass

class level1_xgboost(level1_model):
    def __init__(self, args):
        self.args = args
        model_dir = "_".join(("xgboost_f1", args.reps, args.prob, args.data))
        model_name = "model_" + str(args.yix) + ".pkl"
        outfold_name = "outFold_" + str(args.yix) + ".npy"

        model_path = path.join(path.dirname(__file__), model_dir, model_name)
        outfold_path = path.join(
            path.dirname(__file__), model_dir, outfold_name)

        with open(model_path, 'r') as f:
            self.clf = pickle.load(f)
        self.outfold_pred = np.load(outfold_path)

    def get_fitted(self):
        return np.array([self.outfold_pred]).T

    def get_feature_list(self, X_train=None, y_train=None):
        f_score = self.clf.get_fscore()
        k = np.array(f_score.keys())
        v = np.array(f_score.values())
        f_ix = k[v > np.mean(v)]
        f_list = map(lambda x: int(x[1:]), f_ix)
        return {args.data: f_list}

    def predict(self, X_test):
        dtest = xgb.DMatrix(X_test)
        return np.array([self.clf.predict(dtest, output_margin=True)]).T

class level1_rf(level1_model):
    def __init__(self, args):
        self.args = args
        model_dir = "_".join(("randomForest", args.reps, args.prob, args.data))
        model_name = "model_" + str(args.yix) + ".pkl"
        outfold_name = "outFold_" + str(args.yix) + ".npy"
        param_name = "param_" + str(args.yix) + ".pkl"

        model_path = path.join(path.dirname(__file__), model_dir, model_name)
        outfold_path = path.join(
            path.dirname(__file__), model_dir, outfold_name)
        param_path = path.join(path.dirname(__file__), model_dir, param_name)

        with open(model_path, 'r') as f:
            self.clf = pickle.load(f)
        with open(param_path, 'r') as f:
            self.params = pickle.load(f)
        self.outfold_pred = np.load(outfold_path)

    def get_fitted(self):
        return np.array([self.outfold_pred]).T

    def get_feature_list(self, X_train, y_train):
        weight = y_train.shape[0] / (2 * np.bincount(y_train))
        sample_weight = np.array([weight[i] for i in y_train])
        clf = RandomForestClassifier(**self.params)
        clf.fit(X_train, y_train, sample_weight)
        score = np.array(clf.feature_importances_)
        sel_ix = np.arange(score.shape[0])[score > np.mean(score)]
        return {args.data: list(sel_ix)}

    def predict(self, X_test):
        pred_list = []
        for cclf in self.clf:
            pred = cclf.predict_proba(X_test)[:, 1]
            pred_list.append(pred)
        return np.array([np.mean(np.array(pred_list), axis=0)]).T

class level1_extree(level1_model):
    def __init__(self, args):
        self.args = args
        model_dir = "_".join(("extraTree", args.reps, args.prob, args.data))
        model_name = "model_" + str(args.yix) + ".pkl"
        outfold_name = "outFold_" + str(args.yix) + ".npy"
        param_name = "param_" + str(args.yix) + ".pkl"

        model_path = path.join(path.dirname(__file__), model_dir, model_name)
        outfold_path = path.join(
            path.dirname(__file__), model_dir, outfold_name)
        param_path = path.join(path.dirname(__file__), model_dir, param_name)

        with open(model_path, 'r') as f:
            self.clf = pickle.load(f)
        with open(param_path, 'r') as f:
            self.params = pickle.load(f)
        self.outfold_pred = np.load(outfold_path)

    def get_fitted(self):
        return np.array([self.outfold_pred]).T

    def get_feature_list(self, X_train, y_train):
        weight = y_train.shape[0] / (2 * np.bincount(y_train))
        sample_weight = np.array([weight[i] for i in y_train])
        clf = ExtraTreesClassifier(**self.params)
        clf.fit(X_train, y_train, sample_weight)
        score = np.array(clf.feature_importances_)
        sel_ix = np.arange(score.shape[0])[score > np.mean(score)]
        return {args.data: list(sel_ix)}

    def predict(self, X_test):
        pred_list = []
        for cclf in self.clf:
            pred = cclf.predict_proba(X_test)[:, 1]
            pred_list.append(pred)
        return np.array([np.mean(np.array(pred_list), axis=0)]).T

class level1_pca(level1_model):
    def __init__(self, args):
        self.args = args
        model_dir = "_".join(("pca", args.reps, args.prob, args.data))
        model_name = "model.pkl"
        outfold_name = "outFold.npy"
        param_name = "param.pkl"

        model_path = path.join(path.dirname(__file__), model_dir, model_name)
        outfold_path = path.join(
            path.dirname(__file__), model_dir, outfold_name)
        param_path = path.join(path.dirname(__file__), model_dir, param_name)

        with open(model_path, 'r') as f:
            self.clf = pickle.load(f)
        with open(param_path, 'r') as f:
            self.params = pickle.load(f)
        self.outfold_pred = np.load(outfold_path)

    def get_fitted(self):
        return self.outfold_pred

    def get_feature_list(self, X_train=None, y_train=None):
        return {args.data: []}

    def predict(self, X_test):
        return self.clf.transform(X_test)

class level1_ica(level1_model):
    def __init__(self, args):
        self.args = args
        model_dir = "_".join(("ica", args.reps, args.prob, args.data))
        model_name = "model.pkl"
        outfold_name = "outFold.npy"
        param_name = "param.pkl"

        model_path = path.join(path.dirname(__file__), model_dir, model_name)
        outfold_path = path.join(
            path.dirname(__file__), model_dir, outfold_name)
        param_path = path.join(path.dirname(__file__), model_dir, param_name)

        with open(model_path, 'r') as f:
            self.clf = pickle.load(f)
        with open(param_path, 'r') as f:
            self.params = pickle.load(f)
        self.outfold_pred = np.load(outfold_path)

    def get_fitted(self):
        return self.outfold_pred

    def get_feature_list(self, X_train=None, y_train=None):
        return {args.data: []}

    def predict(self, X_test):
        return self.clf.transform(X_test)

class level1_glm(level1_model):
    def __init__(self, args):
        self.args = args
        model_dir = "_".join(("glm_l1", args.reps, args.prob, args.data))
        model_name = "model_" + str(args.yix) + ".pkl"
        outfold_name = "outFold_" + str(args.yix) + ".npy"
        param_name = "param_" + str(args.yix) + ".pkl"

        model_path = path.join(path.dirname(__file__), model_dir, model_name)
        outfold_path = path.join(
            path.dirname(__file__), model_dir, outfold_name)
        param_path = path.join(path.dirname(__file__), model_dir, param_name)

        with open(model_path, 'r') as f:
            self.clf = pickle.load(f)
        with open(param_path, 'r') as f:
            self.params = pickle.load(f)
        self.outfold_pred = np.load(outfold_path)

    def get_fitted(self):
        return np.array([self.outfold_pred]).T

    def get_feature_list(self, X_train=None, y_train=None):
        sel_ix = SelectFromModel(self.clf.steps[1][1],
                                 threshold="4 * mean",
                                 prefit=True).get_support(True)
        return {args.data: list(sel_ix)}

    def predict(self, X_test):
        return np.array([self.clf.predict_proba(X_test)[:, 1]]).T

def get_new_train(args, X):
    models = [level1_xgboost(args), level1_rf(args), level1_extree(args),
              level1_glm(args), level1_pca(args), level1_ica(args)]
    X_new = None
    for model in models:
        fitted = model.get_fitted()
        if X_new is None:
            X_new = fitted
        else:
            X_new = np.concatenate((X_new, fitted), axis=1)
    return X_new

def get_new_test(args, X):
    models = [level1_xgboost(args), level1_rf(args), level1_extree(args),
              level1_glm(args), level1_pca(args), level1_ica(args)]
    X_new = None
    for model in models:
        preds = model.predict(X)
        if X_new is None:
            X_new = preds
        else:
            X_new = np.concatenate((X_new, preds), axis=1)
    return X_new

def get_sel_features(args, X_train, y_train):
    models = [level1_rf(args), level1_extree(args), level1_glm(args)]
    feature_list = Set()
    for model in models:
        feature_list.update(
            model.get_feature_list(X_train, y_train).values()[0])
    return sorted(feature_list)

args = parse_args()
X_train, y_train = get_data_train(args)
X_test, biz_list = get_data_test(args)

X_train_new = get_new_train(args, X_train)
X_test_new = get_new_test(args, X_test)
feature_list = get_sel_features(args, X_train, y_train)
print(args.data)
print(feature_list)

X_train_new = np.hstack((X_train_new, X_train[:, feature_list]))
X_test_new = np.hstack((X_test_new, X_test[:, feature_list]))

save_dir = "_".join(("../level2-feature/" + str(args.yix) + "/" +
                     args.reps, args.prob, args.data))

if not path.exists(save_dir):
    os.makedirs(save_dir)
np.save(path.join(save_dir, "X_train.npy"), X_train_new)
np.save(path.join(save_dir, "y_train.npy"), y_train)
np.save(path.join(save_dir, "X_test.npy"), X_test_new)
np.save(path.join(save_dir, "feature_list.npy"), feature_list)
<|endoftext|>


<commit_before>
class Mail:

	def __init__(self, db):
		self.db = db

	def create_mail(self, content, date_sent, sender, recipient, subject):
		cursor = self.db.cursor()
		cursor.execute("insert into mail (content, date_sent, sender, recipient, subject) values (?,?,?,?,?)",
				(content,
				date_sent,
				sender,
				recipient,
				subject) )
		self.db.commit()

	def get_mail(self, id):
		cursor = self.db.cursor()
		cursor.execute("select * from mail where id==?", (id,) )

	def get_mail_for_user(name):
		cursor.execute("select * from mail where recipient==?", (name,))
		mails = cursor.fetchall()
		return [
				{
					"content":omail[0],
					"date_sent":omail[1],
					"sender":omail[2],
					"recipient":omail[3],
					"subject":omail[4],
				}
			] for omail in mails

<commit_msg>fixed error with brackets in mail.py<commit_after>
class Mail:

	def __init__(self, db):
		self.db = db

	def create_mail(self, content, date_sent, sender, recipient, subject):
		cursor = self.db.cursor()
		cursor.execute("insert into mail (content, date_sent, sender, recipient, subject) values (?,?,?,?,?)",
				(content,
				date_sent,
				sender,
				recipient,
				subject) )
		self.db.commit()

	def get_mail(self, id):
		cursor = self.db.cursor()
		cursor.execute("select * from mail where id==?", (id,) )

	def get_mail_for_user(name):
		cursor.execute("select * from mail where recipient==?", (name,))
		mails = cursor.fetchall()
		return [
				{
					"content":omail[0],
					"date_sent":omail[1],
					"sender":omail[2],
					"recipient":omail[3],
					"subject":omail[4],
				}
			 for omail in mails ]

<|endoftext|>


<commit_before>class CommandModule:
	def __init__(self, circa):
		self.circa = circa

		self.listeners = {
			"message": [self.command]
		}

	def command(self, fr, to, text, msg):
		if text.startswith(self.circa.conf["prefix"]):
			cmd, *params = text.split()
			self.circa.emit("cmd." + cmd[1:], fr, to, params)

module = CommandModule
<commit_msg>remove msg param from command module<commit_after>class CommandModule:
	def __init__(self, circa):
		self.circa = circa

		self.listeners = {
			"message": [self.command]
		}

	def command(self, fr, to, text):
		if text.startswith(self.circa.conf["prefix"]):
			cmd, *params = text.split()
			self.circa.emit("cmd." + cmd[1:], fr, to, params)

module = CommandModule
<|endoftext|>


<commit_before># This file is part of Viper - https://github.com/botherder/viper
# See the file 'LICENSE' for copying permission.

import getopt
import hashlib
import datetime
import tempfile
import re

try:
    from elftools.elf.elffile import ELFFile
    HAVE_ELFTOOLS = True
except ImportError:
    HAVE_ELFTOOLS = False

try:
    import magic
    HAVE_MAGIC = True
except ImportError:
    HAVE_MAGIC = False

from viper.common.out import *
from viper.common.objects import File
from viper.common.abstracts import Module
from viper.core.database import Database
from viper.core.storage import get_sample_path
from viper.core.session import __sessions__

class PE(Module):
    cmd = 'elf'
    description = 'Extract information from ELF headers'
    authors = ['emdel']

    def __init__(self):
        self.elf = None

    def __check_session(self):
        if not __sessions__.is_set():
            print_error("No session opened")
            return False

        if not self.elf:
            try:
                fd = open(__sessions__.current.file.path, 'rb')
                self.elf = ELFFile(fd)
            except:
                print_error("Unable to parse ELF file")
                return False

        return True

    def __get_filetype(self, data):
        if not HAVE_MAGIC:
            return None

        try:
            ms = magic.open(magic.MAGIC_NONE)
            ms.load()
            file_type = ms.buffer(data)
        except:
            try:
                file_type = magic.from_buffer(data)
            except Exception:
                return None

        return file_type

    def __get_md5(self, data):
        md5 = hashlib.md5()
        md5.update(data)
        return md5.hexdigest()

    def segments(self):
        if not self.__check_session():
            return

        rows = []
        for segment in self.elf.iter_segments():
            rows.append([segment['p_type'],
                         segment['p_vaddr'],
                         segment['p_filesz'],
                         segment['p_memsz'],
                         segment['p_flags']
                         ])

        print_info("ELF Segments:")
        print(table(header=['Type', 'VirtAddr', 'FileSize', 'MemSize', 'Flags'], rows=rows))

    def sections(self):
        if not self.__check_session():
            return

        rows = []
        for section in self.elf.iter_sections():
            rows.append([
                section.name,
                hex(section['sh_addr']),
                hex(section['sh_size']),
                section['sh_type'],
                section['sh_flags'],
                0
                #section.get_entropy()
            ])

        print_info("ELF Sections:")
        print(table(header=['Name', 'Addr', 'Size', 'Type', 'Flags', 'Entropy'], rows=rows))

    def usage(self):
        print("usage: elf <command>")

    def help(self):
        self.usage()
        print("")
        print("Options:")
        print("\thelp\t\tShow this help message")
        print("\tsections\t\tList ELF sections")
        print("\tsegments\t\tList ELF segments")
        print("")

    def run(self):
        if not HAVE_ELFTOOLS:
            print_error("Missing dependency, install pyelftools")
            return

        if len(self.args) == 0:
            self.help()
            return

        if self.args[0] == 'help':
            self.help()
        elif self.args[0] == 'sections':
            self.sections()
        elif self.args[0] == 'segments':
            self.segments()
<commit_msg>Support for ELF symbols<commit_after># This file is part of Viper - https://github.com/botherder/viper
# See the file 'LICENSE' for copying permission.

import getopt
import hashlib
import datetime
import tempfile
import re

try:
    from elftools.elf.elffile import ELFFile
    from elftools.elf.sections import SymbolTableSection
    HAVE_ELFTOOLS = True
except ImportError:
    HAVE_ELFTOOLS = False

try:
    import magic
    HAVE_MAGIC = True
except ImportError:
    HAVE_MAGIC = False

from viper.common.out import *
from viper.common.objects import File
from viper.common.abstracts import Module
from viper.core.database import Database
from viper.core.storage import get_sample_path
from viper.core.session import __sessions__

# Have a look at scripts/readelf.py - pyelftools
class ELF(Module):
    cmd = 'elf'
    description = 'Extract information from ELF headers'
    authors = ['emdel']

    def __init__(self):
        self.elf = None

    def __check_session(self):
        if not __sessions__.is_set():
            print_error("No session opened")
            return False

        if not self.elf:
            try:
                fd = open(__sessions__.current.file.path, 'rb')
                self.elf = ELFFile(fd)
            except:
                print_error("Unable to parse ELF file")
                return False

        return True

    def __get_filetype(self, data):
        if not HAVE_MAGIC:
            return None

        try:
            ms = magic.open(magic.MAGIC_NONE)
            ms.load()
            file_type = ms.buffer(data)
        except:
            try:
                file_type = magic.from_buffer(data)
            except Exception:
                return None

        return file_type

    def __get_md5(self, data):
        md5 = hashlib.md5()
        md5.update(data)
        return md5.hexdigest()

    def segments(self):
        if not self.__check_session():
            return

        rows = []
        for segment in self.elf.iter_segments():
            rows.append([segment['p_type'],
                         segment['p_vaddr'],
                         segment['p_filesz'],
                         segment['p_memsz'],
                         segment['p_flags']
                         ])

        print_info("ELF Segments:")
        print(table(header=['Type', 'VirtAddr', 'FileSize', 'MemSize', 'Flags'], rows=rows))

    def sections(self):
        if not self.__check_session():
            return

        rows = []
        for section in self.elf.iter_sections():
            rows.append([
                section.name,
                hex(section['sh_addr']),
                hex(section['sh_size']),
                section['sh_type'],
                section['sh_flags'],
                0
                #section.get_entropy()
            ])

        print_info("ELF Sections:")
        print(table(header=['Name', 'Addr', 'Size', 'Type', 'Flags', 'Entropy'], rows=rows))

    def symbols(self):
        if not self.__check_session():
            return

        rows = []
        for section in self.elf.iter_sections():
            if not isinstance(section, SymbolTableSection): continue

            for cnt, symbol in enumerate(section.iter_symbols()):
                rows.append([
                    cnt,
                    hex(symbol['st_value']),
                    hex(symbol['st_size']),
                    symbol['st_info']['type'],
                    symbol.name
                    ])

        print_info("ELF Symbols:")
        print(table(header=['Num', 'Value', 'Size', 'Type', 'Name'], rows=rows))

    def usage(self):
        print("usage: elf <command>")

    def help(self):
        self.usage()
        print("")
        print("Options:")
        print("\thelp\t\tShow this help message")
        print("\tsections\t\tList ELF sections")
        print("\tsegments\t\tList ELF segments")
        print("\tsymbols\t\tList ELF symbols")
        print("")

    def run(self):
        if not HAVE_ELFTOOLS:
            print_error("Missing dependency, install pyelftools")
            return

        if len(self.args) == 0:
            self.help()
            return

        if self.args[0] == 'help':
            self.help()
        elif self.args[0] == 'sections':
            self.sections()
        elif self.args[0] == 'segments':
            self.segments()
        elif self.args[0] == 'symbols':
            self.symbols()
<|endoftext|>


<commit_before>import urllib.parse
from bs4 import BeautifulSoup
import re
import os
import syscmd

def fmi( self ):

	## For saving the city
	if len(self.msg) == 6:
		if "set" in self.msg:
			city = self.msg[5]
			setCity( self, city )
			self.send_notice( "Your city {0} has been saved!".format(city.title().strip()))
	else:
		try:
			if len(self.msg) == 4:	## when called only with !fmi, see if the city is saved
				city = getCity( self )
				if city == None:
					city = self.msg[4]
			else:
				city = self.msg[4].strip() #With !fmi City
		except IndexError:
			self.send_chan( "Usage: !fmi <city> | !fmi set <city>" )
			raise

		if "set" not in city and syscmd.checkCity( city ): # We don't want to look for a city named "set"

			city = city.title().strip()
			parameter = urllib.parse.quote(city)
			url = "http://ilmatieteenlaitos.fi/saa/" + parameter

			if city == "Oulu":
				url = "http://ilmatieteenlaitos.fi/saa/" + parameter + "?&station=101799"
			if city == "Helsinki":
				url = "http://ilmatieteenlaitos.fi/saa/" + parameter + "?&station=100971"

			html = syscmd.getHtml(self, url, True )

			try:
				soup = BeautifulSoup(html)
				text = ""
				## When the weather was updated
				time = soup.find_all("span", class_="time-stamp")
				time = time[0].string
				time = time[10:-12]

				str = soup.findAll("span", {"class" : "parameter-name-value"})

				## Loop the reusts into a string
				for a in str:
					text += "{0} - ".format(a)

				## Remove the Html tags
				trimmed = syscmd.delHtml(text)

				output = city.strip() + " klo " + time + ": " + trimmed
				self.send_chan( output )
			except Except as e:
				if self.config["debug"] == "true":
					print(e)
		else:
			self.send_chan( "City {0} doesn't exist!".format(city.title().strip()) )

## Return saved city for the nick
def getCity ( self ):

	nick = self.get_nick()
	file = "modules/data/fmi_nicks.txt"

	try:
		with open(file, "r", encoding="UTF-8") as f:
			for line in f:
				if re.search("\\b"+nick+":\\b", line, flags=re.IGNORECASE):
					city = line.split(":")
					return(city[1])
	except (OSError, IOError): ## Create an empty fmi_nicks.txt if it doesn't exist
		open(file, 'a').close()

## Save user city
def setCity ( self, city ):

	nick = self.get_nick()
	city = city.title().strip()
	file = "modules/data/fmi_nicks.txt"

	## If the nick is in the file, loop through it and replace the line containing the nick
	## with the new city. We write the whole new file to temp.txt and then move it back to fmi_nicks.txt
	try:
		#f = open(file)
		if re.search("\\b"+nick+":\\b", open(file).read(), flags=re.IGNORECASE):
			with open("modules/data/temp.txt", "w", encoding="UTF-8") as temp:
				for line in open(file):
					str = "{0}:{1}".format(nick,city)
					temp.write(re.sub("^{0}:.*$".format(nick), str, line))
				os.remove("modules/data/fmi_nicks.txt")
				os.rename("modules/data/temp.txt", file)
			return(True)
		## If the nick doesn't exist in the file, append it in there
		else:
			with open(file, "a", encoding="UTF-8") as file:
				str = "\r\n{0}:{1}".format(nick,city)
				file.write(str)
			return(True)
	except Exception as e:
		if self.config["debug"] == "true":
			print(e)
<commit_msg>unified string formatting<commit_after>import urllib.parse
from bs4 import BeautifulSoup
import re
import os
import syscmd

def fmi( self ):

	## For saving the city
	if len(self.msg) == 6:
		if "set" in self.msg:
			city = self.msg[5]
			setCity( self, city )
			self.send_notice( "Your city {0} has been saved!".format(city.title().strip()))
	else:
		try:
			if len(self.msg) == 4:	## when called only with !fmi, see if the city is saved
				city = getCity( self )
				if city == None:
					city = self.msg[4]
			else:
				city = self.msg[4].strip() #With !fmi City
		except IndexError:
			self.send_chan( "Usage: !fmi <city> | !fmi set <city>" )
			raise

		if "set" not in city and syscmd.checkCity( city ): # We don't want to look for a city named "set"

			city = city.title().strip()
			parameter = urllib.parse.quote(city)
			url = "http://ilmatieteenlaitos.fi/saa/" + parameter

			if city == "Oulu":
				url = "http://ilmatieteenlaitos.fi/saa/" + parameter + "?&station=101799"
			if city == "Helsinki":
				url = "http://ilmatieteenlaitos.fi/saa/" + parameter + "?&station=100971"

			html = syscmd.getHtml(self, url, True )

			try:
				soup = BeautifulSoup(html)
				text = ""
				## When the weather was updated
				time = soup.find_all("span", class_="time-stamp")
				time = time[0].string
				time = time[10:-12]

				str = soup.findAll("span", {"class" : "parameter-name-value"})

				## Loop the reusts into a string
				for a in str:
					text += "{0} - ".format(a)

				## Remove the Html tags
				trimmed = syscmd.delHtml(text)

				output = "{0} klo {1}: {2}".format(city.strip(), time, trimmed)
				self.send_chan( output )
			except Except as e:
				if self.config["debug"] == "true":
					print(e)
		else:
			self.send_chan( "City {0} doesn't exist!".format(city.title().strip()) )

## Return saved city for the nick
def getCity ( self ):

	nick = self.get_nick()
	file = "modules/data/fmi_nicks.txt"

	try:
		with open(file, "r", encoding="UTF-8") as f:
			for line in f:
				if re.search("\\b"+nick+":\\b", line, flags=re.IGNORECASE):
					city = line.split(":")
					return(city[1])
	except (OSError, IOError): ## Create an empty fmi_nicks.txt if it doesn't exist
		open(file, 'a').close()

## Save user city
def setCity ( self, city ):

	nick = self.get_nick()
	city = city.title().strip()
	file = "modules/data/fmi_nicks.txt"

	## If the nick is in the file, loop through it and replace the line containing the nick
	## with the new city. We write the whole new file to temp.txt and then move it back to fmi_nicks.txt
	try:
		#f = open(file)
		if re.search("\\b"+nick+":\\b", open(file).read(), flags=re.IGNORECASE):
			with open("modules/data/temp.txt", "w", encoding="UTF-8") as temp:
				for line in open(file):
					str = "{0}:{1}".format(nick,city)
					temp.write(re.sub("^{0}:.*$".format(nick), str, line))
				os.remove("modules/data/fmi_nicks.txt")
				os.rename("modules/data/temp.txt", file)
			return(True)
		## If the nick doesn't exist in the file, append it in there
		else:
			with open(file, "a", encoding="UTF-8") as file:
				str = "\r\n{0}:{1}".format(nick,city)
				file.write(str)
			return(True)
	except Exception as e:
		if self.config["debug"] == "true":
			print(e)
<|endoftext|>