<commit_before># encoding: utf-8

import json
import logging
from urllib2 import urlopen
from urllib import urlencode
from datetime import datetime

from google.appengine.api import urlfetch
from google.appengine.ext import ndb
import webapp2

TOKEN = open('bot.token').read()
BASE_URL = 'https://api.telegram.org/bot' + TOKEN + '/'

# ================================
# Data store classes

class Subscriber(ndb.Model):
    chat_id = ndb.IntegerProperty()

class Movie(ndb.Model):
    title = ndb.StringProperty()
    date = ndb.DateTimeProperty()
    url = ndb.StringProperty()
    imdbLink = ndb.StringProperty()
    imdbRating = ndb.StringProperty()

# ================================
# Helper functions

def add_subscriber(c_id):
    check = Subscriber.query(Subscriber.chat_id == c_id).fetch()
    if check:
        return 0
    else:
        subscriber = Subscriber(chat_id=c_id)
        subscriber.put()
        return 1

def remove_subscriber(c_id):
    check = Subscriber.query(Subscriber.chat_id == c_id).fetch()
    if check:
        check[0].key.delete()
        return 1
    return 0

def add_movie(title, date, url, imdblink, imdbrating):
    Movie(title=title,
          date=datetime.strptime(date, '%Y-%m-%dT%H:%M'),
          url=url,
          imdbLink=imdblink,
          imdbRating=imdbrating).put()

def get_formatted_movie_list():
    today = datetime.now()
    all_movies = Movie.query(Movie.date >= today).fetch()
    all_movies.sort(key=lambda movie_element: movie_element.date)
    movie_list = ''
    for movie in all_movies:
        movie_list += (movie.date.strftime('%d.%m.%Y') + ': ' + movie.title + '\n')
    return movie_list

def get_next_movie():
    today = datetime.now()
    query_movies_sorted = Movie.query(Movie.date >= today).fetch()
    query_movies_sorted.sort(key=lambda movie: movie.date)
    return query_movies_sorted[0]

def get_formatted_short_reminder(next_movie):
    return 'Nicht vergessen, heute im TU Film: ' \
           + next_movie.title + '\nBeginn um ' \
           + next_movie.date.strftime('%H:%M Uhr')

def get_formatted_movie(head, title, date=None, url=None, imdblink=None, imdbrating=None):
    text = '' + head + '\n' + title
    if date:
        text += '\n' + date.strftime('%d.%m.%Y um %H:%M Uhr')
        if url:
            text += '\nInfos: Link'
            if imdblink:
                text += '\nIMDb: Link'
                if imdbrating:
                    text += u'\nIMDb ★: ' + imdbrating
    return text

# Send a message from the Bot
# HTML encoded
def reply(chat_id, msg=None):
    if msg:
        resp = urlopen(BASE_URL + 'sendMessage', urlencode({
            'chat_id': str(chat_id),
            'text': msg.encode('utf-8'),
            'parse_mode': 'HTML',
            'disable_web_page_preview': 'true',
        })).read()
    else:
        logging.error('no message specified')
        resp = None

    logging.info('send response:')
    logging.info(resp)

# ================================
# Google App Request Handlers

# set the web hook for the Telegram API
class SetWebHookHandler(webapp2.RequestHandler):
    def get(self):
        urlfetch.set_default_fetch_deadline(60)
        url = self.request.get('url')
        if url:
            self.response.write(json.dumps(json.load(urlopen(BASE_URL + 'setWebhook', urlencode({'url': url})))))

# triggered over cron job
class ReminderHandler(webapp2.RequestHandler):
    def get(self):
        next_movie = get_next_movie()
        if datetime.date(datetime.now()) == next_movie.date.date():
            all_subscriber = Subscriber.query(projection=["chat_id"], distinct=True)
            # send reminder message to every subscriber
            # care if to many subscribers -> limitations from telegram api
            msg = get_formatted_short_reminder(next_movie)
            for sub in all_subscriber:
                urlopen(BASE_URL + 'sendMessage', urlencode({
                    'chat_id': str(sub.chat_id),
                    'text': msg.encode('utf-8'),
                    'disable_web_page_preview': 'true',
                })).read()

# Handler for adding a new Movie over a URL
class MovieHandler(webapp2.RequestHandler):
    def get(self):
        urlfetch.set_default_fetch_deadline(60)
        title = self.request.get('title')
        date = self.request.get('date')
        url = self.request.get('url')
        imdblink = self.request.get('imdblink')
        imdbrating = self.request.get('imdbrating')
        add_movie(title, date, url, imdblink, imdbrating)

# Starts the crawler
# Does nothing currently because the crawler script ist not yet imported
class GetMoviesHandler(webapp2.RedirectHandler):
    # links = get_all_title_links()
    # movie_list = get_movie_details(links)
    movie_list = []
    for movie in movie_list:
        add_movie(movie[0], movie[1], movie[2], movie[3], movie[4])

# Handles messages from Telegram
class WebHookHandler(webapp2.RequestHandler):
    def post(self):
        urlfetch.set_default_fetch_deadline(60)
        json_body = json.loads(self.request.body)

        # log request
        logging.info('request body:')
        logging.info(json_body)
        self.response.write(json.dumps(json_body))

        # get chat and message information
        message = json_body['message']
        text = message.get('text')
        chat = message['chat']
        chat_id = chat['id']

        if not text:
            logging.info('no text')
            return

        if text.startswith('/'):
            if text == '/subscribe':
                if add_subscriber(chat_id) == 1:
                    reply(chat_id, 'Subscribed to movie reminder!')
                else:
                    reply(chat_id, 'Already subscribed!')
            elif text == '/unsubscribe':
                if remove_subscriber(chat_id) == 1:
                    reply(chat_id, 'Unsubscribed from movie reminder!')
                else:
                    reply(chat_id, 'Not subscribed!')
            elif text == '/listall':
                reply(chat_id, get_formatted_movie_list())
            elif text == '/next':
                next_movie = get_next_movie()
                reply(chat_id, get_formatted_movie(u'Als nächstes im TU Film',
                                                   next_movie.title,
                                                   next_movie.date,
                                                   next_movie.url,
                                                   next_movie.imdbLink,
                                                   next_movie.imdbRating))
            else:
                reply(chat_id, "Command not known, use / to get an overview over possible commands.")

        else:
            # reply(chat_id, "Use /<command>. See possible commands with /? or /commands.")
            return

# The App
app = webapp2.WSGIApplication([
    ('/set_webhook', SetWebHookHandler),
    ('/webhook', WebHookHandler),
    ('/reminder', ReminderHandler),
    ('/add_movie', MovieHandler),
    ('/get_movies', GetMoviesHandler),
], debug=True)
<commit_msg>fixed commands for groups<commit_after># encoding: utf-8

import json
import logging
from urllib2 import urlopen
from urllib import urlencode
from datetime import datetime

from google.appengine.api import urlfetch
from google.appengine.ext import ndb
import webapp2

TOKEN = open('bot.token').read()
BASE_URL = 'https://api.telegram.org/bot' + TOKEN + '/'

# ================================
# Data store classes

class Subscriber(ndb.Model):
    chat_id = ndb.IntegerProperty()

class Movie(ndb.Model):
    title = ndb.StringProperty()
    date = ndb.DateTimeProperty()
    url = ndb.StringProperty()
    imdbLink = ndb.StringProperty()
    imdbRating = ndb.StringProperty()

# ================================
# Helper functions

def add_subscriber(c_id):
    check = Subscriber.query(Subscriber.chat_id == c_id).fetch()
    if check:
        return 0
    else:
        subscriber = Subscriber(chat_id=c_id)
        subscriber.put()
        return 1

def remove_subscriber(c_id):
    check = Subscriber.query(Subscriber.chat_id == c_id).fetch()
    if check:
        check[0].key.delete()
        return 1
    return 0

def add_movie(title, date, url, imdblink, imdbrating):
    Movie(title=title,
          date=datetime.strptime(date, '%Y-%m-%dT%H:%M'),
          url=url,
          imdbLink=imdblink,
          imdbRating=imdbrating).put()

def get_formatted_movie_list():
    today = datetime.now()
    all_movies = Movie.query(Movie.date >= today).fetch()
    all_movies.sort(key=lambda movie_element: movie_element.date)
    movie_list = ''
    for movie in all_movies:
        movie_list += (movie.date.strftime('%d.%m.%Y') + ': ' + movie.title + '\n')
    return movie_list

def get_next_movie():
    today = datetime.now()
    query_movies_sorted = Movie.query(Movie.date >= today).fetch()
    query_movies_sorted.sort(key=lambda movie: movie.date)
    return query_movies_sorted[0]

def get_formatted_short_reminder(next_movie):
    return 'Nicht vergessen, heute im TU Film: ' \
           + next_movie.title + '\nBeginn um ' \
           + next_movie.date.strftime('%H:%M Uhr')

def get_formatted_movie(head, title, date=None, url=None, imdblink=None, imdbrating=None):
    text = '' + head + '\n' + title
    if date:
        text += '\n' + date.strftime('%d.%m.%Y um %H:%M Uhr')
        if url:
            text += '\nInfos: Link'
            if imdblink:
                text += '\nIMDb: Link'
                if imdbrating:
                    text += u'\nIMDb ★: ' + imdbrating
    return text

# Send a message from the Bot
# HTML encoded
def reply(chat_id, msg=None):
    if msg:
        resp = urlopen(BASE_URL + 'sendMessage', urlencode({
            'chat_id': str(chat_id),
            'text': msg.encode('utf-8'),
            'parse_mode': 'HTML',
            'disable_web_page_preview': 'true',
        })).read()
    else:
        logging.error('no message specified')
        resp = None

    logging.info('send response:')
    logging.info(resp)

# ================================
# Google App Request Handlers

# set the web hook for the Telegram API
class SetWebHookHandler(webapp2.RequestHandler):
    def get(self):
        urlfetch.set_default_fetch_deadline(60)
        url = self.request.get('url')
        if url:
            self.response.write(json.dumps(json.load(urlopen(BASE_URL + 'setWebhook', urlencode({'url': url})))))

# triggered over cron job
class ReminderHandler(webapp2.RequestHandler):
    def get(self):
        next_movie = get_next_movie()
        if datetime.date(datetime.now()) == next_movie.date.date():
            all_subscriber = Subscriber.query(projection=["chat_id"], distinct=True)
            # send reminder message to every subscriber
            # care if to many subscribers -> limitations from telegram api
            msg = get_formatted_short_reminder(next_movie)
            for sub in all_subscriber:
                urlopen(BASE_URL + 'sendMessage', urlencode({
                    'chat_id': str(sub.chat_id),
                    'text': msg.encode('utf-8'),
                    'disable_web_page_preview': 'true',
                })).read()

# Handler for adding a new Movie over a URL
class MovieHandler(webapp2.RequestHandler):
    def get(self):
        urlfetch.set_default_fetch_deadline(60)
        title = self.request.get('title')
        date = self.request.get('date')
        url = self.request.get('url')
        imdblink = self.request.get('imdblink')
        imdbrating = self.request.get('imdbrating')
        add_movie(title, date, url, imdblink, imdbrating)

# Starts the crawler
# Does nothing currently because the crawler script ist not yet imported
class GetMoviesHandler(webapp2.RedirectHandler):
    # links = get_all_title_links()
    # movie_list = get_movie_details(links)
    movie_list = []
    for movie in movie_list:
        add_movie(movie[0], movie[1], movie[2], movie[3], movie[4])

# Handles messages from Telegram
class WebHookHandler(webapp2.RequestHandler):
    def post(self):
        urlfetch.set_default_fetch_deadline(60)
        json_body = json.loads(self.request.body)

        # log request
        logging.info('request body:')
        logging.info(json_body)
        self.response.write(json.dumps(json_body))

        # get chat and message information
        message = json_body['message']
        text = message.get('text')
        chat = message['chat']
        chat_id = chat['id']

        if not text:
            logging.info('no text')
            return

        if text.startswith('/'):
            if text.find('@') > 0:
                text = text[0:text.find('@')]
            if text == '/subscribe':
                if add_subscriber(chat_id) == 1:
                    reply(chat_id, 'Subscribed to movie reminder!')
                else:
                    reply(chat_id, 'Already subscribed!')
            elif text == '/unsubscribe':
                if remove_subscriber(chat_id) == 1:
                    reply(chat_id, 'Unsubscribed from movie reminder!')
                else:
                    reply(chat_id, 'Not subscribed!')
            elif text == '/listall':
                reply(chat_id, get_formatted_movie_list())
            elif text == '/next':
                next_movie = get_next_movie()
                reply(chat_id, get_formatted_movie(u'Als nächstes im TU Film',
                                                   next_movie.title,
                                                   next_movie.date,
                                                   next_movie.url,
                                                   next_movie.imdbLink,
                                                   next_movie.imdbRating))
            else:
                reply(chat_id, "Command not known, use / to get an overview over possible commands.")

        else:
            # reply(chat_id, "Use /<command>. See possible commands with /? or /commands.")
            return

# The App
app = webapp2.WSGIApplication([
    ('/set_webhook', SetWebHookHandler),
    ('/webhook', WebHookHandler),
    ('/reminder', ReminderHandler),
    ('/add_movie', MovieHandler),
    ('/get_movies', GetMoviesHandler),
], debug=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  untitled.py
#
#  Copyright 2013 tusharmakkar08 <tusharmakkar08@tusharmakkar08-Satellite-C660>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
#

""" Importing Modules """

import csv

""" Main Code Starts Here """
def wheref(filename,condition):
	"""
		Code for where in select
	"""

def joinf(tables,conditions):
	"""
		Code for joining tables
	"""
	if conditions!="-1":
		pass
	else:
		try:
			filename1=tables[0]+".csv"
			filename2=tables[1]+".csv"
			with open(filename1) and open(filename2):
				sreader1=csv.reader(open(filename1,"rb"))  #reader for the given file
				sreader2=csv.reader(open(filename2,"rb"))  #reader for the given file
				swriter=csv.writer(open("mix.csv","wb"))
				co=0
				for i in sreader2 :
					sreader1=csv.reader(open(filename1,"rb"))  #reader for the given file
					co1=0
					for j in sreader1:
						if co==0 and co1==0:	# For adding details
							k=[];k1=[]
							for t in i:
								k.append(tables[0]+"."+t)
							for t in j:
								k1.append(tables[1]+"."+t)
							swriter.writerow(k+k1)
						if co!=0 and co1!=0:
							swriter.writerow(i+j)
						co1+=1
					co+=1

		except IOError:
			print "File Not Found"

def parsing(inp):
	"""
		Code for parsing the functions
	"""
	newstr="" 		# newstr made to remove paranthesis

	for i in inp:
		if i!='(' and i!=')'and i!='-':
			newstr+=i
		if i=="-":
			newstr+=" "

	trialList=newstr.strip().split()
	tableName=trialList[1].strip().split(",")
	if trialList[0]=="Join"or"JOIN"or"join":
		if len(trialList)<=2:
			joinf(tableName,"-1")
		else:
			joinf(tablename,trialList[3:])

def selectf(listvar1,var2,varwhere):
	"""
		Code for selecting from table
	"""
	try:
		if var2[0]=="(":
			parsing(var2)
			selectf(listvar1,"mix","-1")
			return
		else:
			filename=var2+".csv"
		with open(filename):
			if varwhere!="-1":
					wheref(filename,varwhere)
			else:
				sreader=csv.reader(open(filename,"rb"))  #reader for the given file
				var1=listvar1.strip().split(',') #Splitting the columns in var1
				if var1[0]=="*": # For All columns
					co=0 		# Temporary Variable
					for row in sreader:
						if co!=0:
							print row
						co+=1
				else:
					co=0
					for row in sreader:
						if co==0:
							row_name=row
							break
					kinglist=[]  # List containing the index of columns in the table
					for j in var1:
						oldlength=len(kinglist)
						co=0
						for i in row_name:
							if i==j:
								kinglist.append(co)
								break
							co+=1
						if len(kinglist)==oldlength:
							print j,"Column Not Found"
					if len(kinglist)==0:
						print "Column Not Found"
					else:
						for row in sreader:
							for j in kinglist:
								print row[j],
							print
	except IOError:
		print "File Not Found"

def main():
	while 1:
		a=raw_input("Enter Regular Expression Statement and -1 for exit \n").strip().split()
		if a[0]=="-1":
			break
		L=len(a);i=0
		while i<L:
			if a[i]=="SELECT"or"select"or"Select" and a[i+2]=="FROM"or"from"or"From" :
				var1=a[i+1]
				var2=a[i+3]
				try:
					if a[i+5]=="where"or"WHERE"or"Where":
						var3=a[i+6:]
				except:
					var3="-1"
					pass
				selectf(var1,var2,var3)
				i+=3
			i+=1
	return 0

if __name__ == '__main__':
	main()

"""
	Format till now :
	Select "any_no_of_arguments" from
	"either_single_filename_or_
	(Join-filenames_with_comma_seperated where conditions)" where conditions
"""
<commit_msg>Parsing methodology for where clause done<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  main.py
#
#  Copyright 2013 tusharmakkar08 <tusharmakkar08@gmail.com>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
#

""" Importing Modules """

import csv
import os

""" Main Code Starts Here """

def wheref(filename,condition):
	"""
		Code for where in select
	"""
	print filename,condition

def joinf(tables,conditions):
	"""
		Code for joining tables
	"""
	if conditions!="-1":
		pass
	else:
		try:
			filename1=tables[0]+".csv"
			filename2=tables[1]+".csv"
			with open(filename1) and open(filename2):
				sreader1=csv.reader(open(filename1,"rb"))  #reader for the given file
				sreader2=csv.reader(open(filename2,"rb"))  #reader for the given file
				swriter=csv.writer(open("mix.csv","wb"))
				co=0
				for i in sreader2 :
					sreader1=csv.reader(open(filename1,"rb"))  #reader for the given file
					co1=0
					for j in sreader1:
						if co==0 and co1==0:	# For adding details
							k=[];k1=[]
							for t in i:
								k.append(tables[0]+"."+t)
							for t in j:
								k1.append(tables[1]+"."+t)
							swriter.writerow(k+k1)
						if co!=0 and co1!=0:
							swriter.writerow(i+j)
						co1+=1
					co+=1

		except IOError:
			print "File Not Found"

def parsing(filename,inp):
	"""
		Code for parsing the functions
	"""
	newstr="" 		# newstr made to remove paranthesis

	for i in inp:
		if i!='(' and i!=')'and i!='-' and i!="[" and i!="]" and i!="'":
			newstr+=i
		if i=="-":
			newstr+=" "

	trialList=newstr.strip().split()
	tableName=trialList[1].strip().split(",")
	if trialList[0]=="Join"or trialList[0]=="JOIN"or trialList[0]=="join":
		if len(trialList)<=2:
			joinf(tableName,"-1")
		else:
			joinf(tablename,trialList[3:])
	if trialList[0]=='cond'or trialList[0]=="COND" or trialList[0]=="Cond":
		wheref(filename,trialList[1])

def selectf(listvar1,var2,varwhere):
	"""
		Code for selecting from table
	"""
	try:
		if var2[0]=="(":
			parsing("-1",var2)
			selectf(listvar1,"mix",varwhere)
			#os.remove("mix.csv")	# To be added in final code
			return
		else:
			filename=var2+".csv"
		with open(filename):
			if varwhere!="-1":
				parsing(filename,str(varwhere))
			else:
				sreader=csv.reader(open(filename,"rb"))  #reader for the given file
				var1=listvar1.strip().split(',') #Splitting the columns in var1
				if var1[0]=="*": # For All columns
					co=0 		# Temporary Variable
					for row in sreader:
						if co!=0:
							print row
						co+=1
				else:
					co=0
					for row in sreader:
						if co==0:
							row_name=row
							break
					kinglist=[]  # List containing the index of columns in the table
					for j in var1:
						oldlength=len(kinglist)
						co=0
						for i in row_name:
							if i==j:
								kinglist.append(co)
								break
							co+=1
						if len(kinglist)==oldlength:
							print j,"Column Not Found"
					if len(kinglist)==0:
						print "Column Not Found"
					else:
						for row in sreader:
							for j in kinglist:
								print row[j],
							print
	except IOError:
		print "File Not Found"

def main():
	while 1:
		a=raw_input("Enter Regular Expression Statement and -1 for exit \n").strip().split()
		if a[0]=="-1":
			break
		L=len(a);i=0
		while i+2<L: 	# i+2 because in other cases it gives error
			if a[i]=="SELECT"or a[i]=="select"or a[i]=="Select" and a[i+2]=="FROM"or a[i+2]=="from"or a[i+2]=="From" :
				var1=a[i+1]
				var2=a[i+3]
				try:
					if a[i+5]=="where"or"WHERE"or"Where":
						var3=a[i+6:]
				except:
					var3="-1"
					pass
				selectf(var1,var2,var3)
				i+=3
			i+=1
	return 0

if __name__ == '__main__':
	main()

"""
	Format till now :
	Select "any_no_of_arguments" from
	"either_single_filename_or_
	(Join-filenames_with_comma_seperated where conditions)" where
	(cond-any|no|of|conditions|using|"|")
"""
<|endoftext|>


<commit_before>from __future__ import division
from __future__ import print_function

import os
import random
import logging

import torch
import torch.nn as nn
import torch.optim as optim

# IMPORT CONSTANTS
from treelstm import Constants
# NEURAL NETWORK MODULES/LAYERS
from treelstm import SimilarityTreeLSTM
# DATA HANDLING CLASSES
from treelstm import Vocab
# DATASET CLASS FOR SICK DATASET
from treelstm import SICKDataset
# METRICS CLASS FOR EVALUATION
from treelstm import Metrics
# UTILITY FUNCTIONS
from treelstm import utils
# TRAIN AND TEST HELPER FUNCTIONS
from treelstm import Trainer
# CONFIG PARSER
from config import parse_args

# MAIN BLOCK
def main():
    global args
    args = parse_args()
    # global logger
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)
    formatter = logging.Formatter("[%(asctime)s] %(levelname)s:%(name)s:%(message)s")
    # file logger
    fh = logging.FileHandler(os.path.join(args.save, args.expname)+'.log', mode='w')
    fh.setLevel(logging.INFO)
    fh.setFormatter(formatter)
    logger.addHandler(fh)
    # console logger
    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    ch.setFormatter(formatter)
    logger.addHandler(ch)
    # argument validation
    args.cuda = args.cuda and torch.cuda.is_available()
    device = torch.device("cuda:0" if args.cuda else "cpu")
    if args.sparse and args.wd != 0:
        logger.error('Sparsity and weight decay are incompatible, pick one!')
        exit()
    logger.debug(args)
    torch.manual_seed(args.seed)
    random.seed(args.seed)
    if args.cuda:
        torch.cuda.manual_seed(args.seed)
        torch.backends.cudnn.benchmark = True
    if not os.path.exists(args.save):
        os.makedirs(args.save)

    train_dir = os.path.join(args.data, 'train/')
    dev_dir = os.path.join(args.data, 'dev/')
    test_dir = os.path.join(args.data, 'test/')

    # write unique words from all token files
    sick_vocab_file = os.path.join(args.data, 'sick.vocab')
    if not os.path.isfile(sick_vocab_file):
        token_files_b = [os.path.join(split, 'b.toks') for split in [train_dir, dev_dir, test_dir]]
        token_files_a = [os.path.join(split, 'a.toks') for split in [train_dir, dev_dir, test_dir]]
        token_files = token_files_a + token_files_b
        sick_vocab_file = os.path.join(args.data, 'sick.vocab')
        utils.build_vocab(token_files, sick_vocab_file)

    # get vocab object from vocab file previously written
    vocab = Vocab(filename=sick_vocab_file,
                  data=[Constants.PAD_WORD, Constants.UNK_WORD,
                        Constants.BOS_WORD, Constants.EOS_WORD])
    logger.debug('==> SICK vocabulary size : %d ' % vocab.size())

    # load SICK dataset splits
    train_file = os.path.join(args.data, 'sick_train.pth')
    if os.path.isfile(train_file):
        train_dataset = torch.load(train_file)
    else:
        train_dataset = SICKDataset(train_dir, vocab, args.num_classes)
        torch.save(train_dataset, train_file)
    logger.debug('==> Size of train data   : %d ' % len(train_dataset))
    dev_file = os.path.join(args.data, 'sick_dev.pth')
    if os.path.isfile(dev_file):
        dev_dataset = torch.load(dev_file)
    else:
        dev_dataset = SICKDataset(dev_dir, vocab, args.num_classes)
        torch.save(dev_dataset, dev_file)
    logger.debug('==> Size of dev data     : %d ' % len(dev_dataset))
    test_file = os.path.join(args.data, 'sick_test.pth')
    if os.path.isfile(test_file):
        test_dataset = torch.load(test_file)
    else:
        test_dataset = SICKDataset(test_dir, vocab, args.num_classes)
        torch.save(test_dataset, test_file)
    logger.debug('==> Size of test data    : %d ' % len(test_dataset))

    # initialize model, criterion/loss_function, optimizer
    model = SimilarityTreeLSTM(
        vocab.size(),
        args.input_dim,
        args.mem_dim,
        args.hidden_dim,
        args.num_classes,
        args.sparse,
        args.freeze_embed)
    criterion = nn.KLDivLoss()

    # for words common to dataset vocab and GLOVE, use GLOVE vectors
    # for other words in dataset vocab, use random normal vectors
    emb_file = os.path.join(args.data, 'sick_embed.pth')
    if os.path.isfile(emb_file):
        emb = torch.load(emb_file)
    else:
        # load glove embeddings and vocab
        glove_vocab, glove_emb = utils.load_word_vectors(
            os.path.join(args.glove, 'glove.840B.300d'))
        logger.debug('==> GLOVE vocabulary size: %d ' % glove_vocab.size())
        emb = torch.zeros(vocab.size(), glove_emb.size(1), dtype=torch.float, device=device)
        emb.normal_(0, 0.05)
        # zero out the embeddings for padding and other special words if they are absent in vocab
        for idx, item in enumerate([Constants.PAD_WORD, Constants.UNK_WORD,
                                    Constants.BOS_WORD, Constants.EOS_WORD]):
            emb[idx].zero_()
        for word in vocab.labelToIdx.keys():
            if glove_vocab.getIndex(word):
                emb[vocab.getIndex(word)] = glove_emb[glove_vocab.getIndex(word)]
        torch.save(emb, emb_file)
    # plug these into embedding matrix inside model
    model.emb.weight.copy_(emb)

    model.to(device), criterion.to(device)
    if args.optim == 'adam':
        optimizer = optim.Adam(filter(lambda p: p.requires_grad,
                                      model.parameters()), lr=args.lr, weight_decay=args.wd)
    elif args.optim == 'adagrad':
        optimizer = optim.Adagrad(filter(lambda p: p.requires_grad,
                                         model.parameters()), lr=args.lr, weight_decay=args.wd)
    elif args.optim == 'sgd':
        optimizer = optim.SGD(filter(lambda p: p.requires_grad,
                                     model.parameters()), lr=args.lr, weight_decay=args.wd)
    metrics = Metrics(args.num_classes)

    # create trainer object for training and testing
    trainer = Trainer(args, model, criterion, optimizer, device)

    best = -float('inf')
    for epoch in range(args.epochs):
        train_loss = trainer.train(train_dataset)
        train_loss, train_pred = trainer.test(train_dataset)
        dev_loss, dev_pred = trainer.test(dev_dataset)
        test_loss, test_pred = trainer.test(test_dataset)

        train_pearson = metrics.pearson(train_pred, train_dataset.labels)
        train_mse = metrics.mse(train_pred, train_dataset.labels)
        logger.info('==> Epoch {}, Train \tLoss: {}\tPearson: {}\tMSE: {}'.format(
            epoch, train_loss, train_pearson, train_mse))
        dev_pearson = metrics.pearson(dev_pred, dev_dataset.labels)
        dev_mse = metrics.mse(dev_pred, dev_dataset.labels)
        logger.info('==> Epoch {}, Dev \tLoss: {}\tPearson: {}\tMSE: {}'.format(
            epoch, dev_loss, dev_pearson, dev_mse))
        test_pearson = metrics.pearson(test_pred, test_dataset.labels)
        test_mse = metrics.mse(test_pred, test_dataset.labels)
        logger.info('==> Epoch {}, Test \tLoss: {}\tPearson: {}\tMSE: {}'.format(
            epoch, test_loss, test_pearson, test_mse))

        if best < test_pearson:
            best = test_pearson
            checkpoint = {
                'model': trainer.model.state_dict(),
                'optim': trainer.optimizer,
                'pearson': test_pearson, 'mse': test_mse,
                'args': args, 'epoch': epoch
            }
            logger.debug('==> New optimum found, checkpointing everything now...')
            torch.save(checkpoint, '%s.pt' % os.path.join(args.save, args.expname))

if __name__ == "__main__":
    main()
<commit_msg>Fix RuntimeError: a leaf Variable that requires grad has been used in an in-place operation.<commit_after>from __future__ import division
from __future__ import print_function

import os
import random
import logging

import torch
import torch.nn as nn
import torch.optim as optim

# IMPORT CONSTANTS
from treelstm import Constants
# NEURAL NETWORK MODULES/LAYERS
from treelstm import SimilarityTreeLSTM
# DATA HANDLING CLASSES
from treelstm import Vocab
# DATASET CLASS FOR SICK DATASET
from treelstm import SICKDataset
# METRICS CLASS FOR EVALUATION
from treelstm import Metrics
# UTILITY FUNCTIONS
from treelstm import utils
# TRAIN AND TEST HELPER FUNCTIONS
from treelstm import Trainer
# CONFIG PARSER
from config import parse_args

# MAIN BLOCK
def main():
    global args
    args = parse_args()
    # global logger
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)
    formatter = logging.Formatter("[%(asctime)s] %(levelname)s:%(name)s:%(message)s")
    # file logger
    fh = logging.FileHandler(os.path.join(args.save, args.expname)+'.log', mode='w')
    fh.setLevel(logging.INFO)
    fh.setFormatter(formatter)
    logger.addHandler(fh)
    # console logger
    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    ch.setFormatter(formatter)
    logger.addHandler(ch)
    # argument validation
    args.cuda = args.cuda and torch.cuda.is_available()
    device = torch.device("cuda:0" if args.cuda else "cpu")
    if args.sparse and args.wd != 0:
        logger.error('Sparsity and weight decay are incompatible, pick one!')
        exit()
    logger.debug(args)
    torch.manual_seed(args.seed)
    random.seed(args.seed)
    if args.cuda:
        torch.cuda.manual_seed(args.seed)
        torch.backends.cudnn.benchmark = True
    if not os.path.exists(args.save):
        os.makedirs(args.save)

    train_dir = os.path.join(args.data, 'train/')
    dev_dir = os.path.join(args.data, 'dev/')
    test_dir = os.path.join(args.data, 'test/')

    # write unique words from all token files
    sick_vocab_file = os.path.join(args.data, 'sick.vocab')
    if not os.path.isfile(sick_vocab_file):
        token_files_b = [os.path.join(split, 'b.toks') for split in [train_dir, dev_dir, test_dir]]
        token_files_a = [os.path.join(split, 'a.toks') for split in [train_dir, dev_dir, test_dir]]
        token_files = token_files_a + token_files_b
        sick_vocab_file = os.path.join(args.data, 'sick.vocab')
        utils.build_vocab(token_files, sick_vocab_file)

    # get vocab object from vocab file previously written
    vocab = Vocab(filename=sick_vocab_file,
                  data=[Constants.PAD_WORD, Constants.UNK_WORD,
                        Constants.BOS_WORD, Constants.EOS_WORD])
    logger.debug('==> SICK vocabulary size : %d ' % vocab.size())

    # load SICK dataset splits
    train_file = os.path.join(args.data, 'sick_train.pth')
    if os.path.isfile(train_file):
        train_dataset = torch.load(train_file)
    else:
        train_dataset = SICKDataset(train_dir, vocab, args.num_classes)
        torch.save(train_dataset, train_file)
    logger.debug('==> Size of train data   : %d ' % len(train_dataset))
    dev_file = os.path.join(args.data, 'sick_dev.pth')
    if os.path.isfile(dev_file):
        dev_dataset = torch.load(dev_file)
    else:
        dev_dataset = SICKDataset(dev_dir, vocab, args.num_classes)
        torch.save(dev_dataset, dev_file)
    logger.debug('==> Size of dev data     : %d ' % len(dev_dataset))
    test_file = os.path.join(args.data, 'sick_test.pth')
    if os.path.isfile(test_file):
        test_dataset = torch.load(test_file)
    else:
        test_dataset = SICKDataset(test_dir, vocab, args.num_classes)
        torch.save(test_dataset, test_file)
    logger.debug('==> Size of test data    : %d ' % len(test_dataset))

    # initialize model, criterion/loss_function, optimizer
    model = SimilarityTreeLSTM(
        vocab.size(),
        args.input_dim,
        args.mem_dim,
        args.hidden_dim,
        args.num_classes,
        args.sparse,
        args.freeze_embed)
    criterion = nn.KLDivLoss()

    # for words common to dataset vocab and GLOVE, use GLOVE vectors
    # for other words in dataset vocab, use random normal vectors
    emb_file = os.path.join(args.data, 'sick_embed.pth')
    if os.path.isfile(emb_file):
        emb = torch.load(emb_file)
    else:
        # load glove embeddings and vocab
        glove_vocab, glove_emb = utils.load_word_vectors(
            os.path.join(args.glove, 'glove.840B.300d'))
        logger.debug('==> GLOVE vocabulary size: %d ' % glove_vocab.size())
        emb = torch.zeros(vocab.size(), glove_emb.size(1), dtype=torch.float, device=device)
        emb.normal_(0, 0.05)
        # zero out the embeddings for padding and other special words if they are absent in vocab
        for idx, item in enumerate([Constants.PAD_WORD, Constants.UNK_WORD,
                                    Constants.BOS_WORD, Constants.EOS_WORD]):
            emb[idx].zero_()
        for word in vocab.labelToIdx.keys():
            if glove_vocab.getIndex(word):
                emb[vocab.getIndex(word)] = glove_emb[glove_vocab.getIndex(word)]
        torch.save(emb, emb_file)
    # plug these into embedding matrix inside model
    model.emb.weight.data.copy_(emb)

    model.to(device), criterion.to(device)
    if args.optim == 'adam':
        optimizer = optim.Adam(filter(lambda p: p.requires_grad,
                                      model.parameters()), lr=args.lr, weight_decay=args.wd)
    elif args.optim == 'adagrad':
        optimizer = optim.Adagrad(filter(lambda p: p.requires_grad,
                                         model.parameters()), lr=args.lr, weight_decay=args.wd)
    elif args.optim == 'sgd':
        optimizer = optim.SGD(filter(lambda p: p.requires_grad,
                                     model.parameters()), lr=args.lr, weight_decay=args.wd)
    metrics = Metrics(args.num_classes)

    # create trainer object for training and testing
    trainer = Trainer(args, model, criterion, optimizer, device)

    best = -float('inf')
    for epoch in range(args.epochs):
        train_loss = trainer.train(train_dataset)
        train_loss, train_pred = trainer.test(train_dataset)
        dev_loss, dev_pred = trainer.test(dev_dataset)
        test_loss, test_pred = trainer.test(test_dataset)

        train_pearson = metrics.pearson(train_pred, train_dataset.labels)
        train_mse = metrics.mse(train_pred, train_dataset.labels)
        logger.info('==> Epoch {}, Train \tLoss: {}\tPearson: {}\tMSE: {}'.format(
            epoch, train_loss, train_pearson, train_mse))
        dev_pearson = metrics.pearson(dev_pred, dev_dataset.labels)
        dev_mse = metrics.mse(dev_pred, dev_dataset.labels)
        logger.info('==> Epoch {}, Dev \tLoss: {}\tPearson: {}\tMSE: {}'.format(
            epoch, dev_loss, dev_pearson, dev_mse))
        test_pearson = metrics.pearson(test_pred, test_dataset.labels)
        test_mse = metrics.mse(test_pred, test_dataset.labels)
        logger.info('==> Epoch {}, Test \tLoss: {}\tPearson: {}\tMSE: {}'.format(
            epoch, test_loss, test_pearson, test_mse))

        if best < test_pearson:
            best = test_pearson
            checkpoint = {
                'model': trainer.model.state_dict(),
                'optim': trainer.optimizer,
                'pearson': test_pearson, 'mse': test_mse,
                'args': args, 'epoch': epoch
            }
            logger.debug('==> New optimum found, checkpointing everything now...')
            torch.save(checkpoint, '%s.pt' % os.path.join(args.save, args.expname))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import matplotlib
matplotlib.use('Agg')
import argparse
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.backends.backend_pdf import PdfPages
import pysam
from multiprocessing import Process, Queue

def Parser():
    parser = argparse.ArgumentParser(description='miRNAs counts and coverages')
    parser.add_argument('-a', '--alignment', metavar='FILE', type=str,
                        dest='alignment_file', help='Alignment bam file')
    parser.add_argument('--gff', metavar='FILE', type=str, dest='gff_file',
                        help='GFF3 describing both pre-miRNAs\
                              and mature miRNAs')
    parser.add_argument('-q', '--quality_threshold', type=int,
                        dest='quality_threshold',
                        help='Quality threshold for coverage (default=10)',
                        default=10)
    parser.add_argument('-p', '--pre_mirs', type=str, dest='pre_mirs',
                        help='pre-miRNAs count file path', metavar='FILE')
    parser.add_argument('-m', '--mirs', type=str, dest='mirs',
                        help='mature miRNA count file path', metavar='FILE')
    parser.add_argument('--lattice', metavar='FILE', type=str, dest='lattice',
                        help='Output file for the lattice dataframe.')
    args = parser.parse_args()
    return args

def get_pre_mir_counts(bamfile):
    """
    Takes a AlignmentFile object and returns a dictionary of counts for reads
    aligning with pre_mirs (as keys)
    """
    count = dict()
    for ref_name in bamfile.references:
        count[ref_name] = bamfile.count(reference=ref_name)
    return count

def get_pre_mir_coverage(bamfile, quality=10):
    """
    Takes a AlignmentFile object and returns a dictionary of lists
    of coverage along the coordinates of pre_mirs (as keys)
    """
    coverage = dict()
    for ref_name, ref_len in zip(bamfile.references, bamfile.lengths):
        coverage[ref_name] = bamfile.count_coverage(reference=ref_name,
                                                    start=0, end=ref_len,
                                                    quality_threshold=quality)
        """ Add the 4 coverage values """
        coverage[ref_name] = np.array([sum(x) for x in
                                       zip(*coverage[ref_name])])
    return coverage

def get_mir_counts(bamfile, gff_file):
    """
    Takes a AlignmentFile and a gff file and computes for
    each 'miRNA' region of the gff the number of reads that hit it
    returns a dict[mir_name] = count
    """
    counts = dict()
    for line in open(gff_file, 'r'):
        if line[0] != '#':
            gff_fields = line[:-1].split("\t")
            if gff_fields[2] == 'miRNA':
                mir_name = gff_fields[0]
                premir_name = gff_fields[8].split('=')[-1]
                mir_start = int(gff_fields[3])
                mir_end = int(gff_fields[4])
                # GFF is 1-based, pysam is 0-based.
                counts[mir_name] = bamfile.count(reference=premir_name,
                                                 start=mir_start-1,
                                                 end=mir_end-1)
    return counts

def write_dataframe_coverage(countdict, outfile):
    """
    Takes a dict[pre_mir reference name] = [coverage list]
    and writes a dataframe with columns:
    <gene_type name>, offset, normoffset, counts and normcounts
    in the outfile
    """
    F = open(outfile, 'w')
    F.write('Mir_hairpin\tOffset\tNorm_offset\tCount\tNorm_count\n')
    for ref in sorted(countdict):
        """
        For each reference name in mirs,
        write the coverage of each of its positions
        """
        maximum = max(countdict[ref])
        reference_length = len(countdict[ref])
        for pos, c in enumerate(countdict[ref]):
            """ Compute and write value for each reference position"""
            F.write('%s\t%s\t%s\t%s\t%s\n' % (ref, str(pos + 1),
                    str(float(pos+1)/reference_length), str(float(c)),
                    str(float(c)/maximum) if maximum != 0 else '0'))
    F.close()

def plot_coverage_par(countdict, outfile):
    """
    Tkes a dict[pre_mir reference name] = [coverage list]
    and plots the coverage of each pre_mir reference name
    """
    pdf = PdfPages('multipage.pdf')
    xax = 0
    yax = 0
    process_list = list()
    nb_images = int(len(countdict) / 60)
    figure_queue = Queue(6)
    num_img = 1
    sref = sorted(countdict)
    nprocess = 0
    nb_pages = 0
    previous_page = 0
    for pages in range(1,nb_images+2):
        print(pages)
        current_page = pages*60
        if nprocess <4:
            nprocess += 1
            process_list.append(Process(target=plot_img,args=(countdict,
                                                              sref[previous_page:current_page],
                                                              figure_queue,)))
            process_list[len(process_list)-1].start()
        else:
            p = process_list.pop(0)
            while figure_queue.empty():
                p.join(1)
            if not figure_queue.empty():
                pdf.savefig(figure_queue.get())
            process_list.append(Process(target=plot_img,args=(countdict,
                                                              sref[previous_page:current_page],
                                                              figure_queue,)))
            process_list[len(process_list)-1].start()
        previous_page = current_page
    while nb_pages < len(process_list):
        if not figure_queue.empty():
            nb_pages += 1
            pdf.savefig(figure_queue.get())
    pdf.close()
    for p in process_list:
        p.join()

def plot_img_par(countdict, ref_list, figure_queue):
    fig, axarr = plt.subplots(15, 4, figsize=(10, 20), sharey='row')
    xax = 0
    yax = 0
    for ref in ref_list:
        m = max(countdict[ref])
        max_coord = len(countdict[ref])
        if m > 0:
            l = [x/m for x in countdict[ref]]
        else:
            l = countdict[ref]
        l_coord = [x/max_coord for x in range(0,max_coord)]
        axarr[xax,yax].plot(l_coord,l)
        axarr[xax,yax].set_title(ref)
        axarr[xax,yax].set_ylim(-0.01,1.1)
        yax += 1
        if yax == 4:
            yax = 0
            xax += 1
    plt.setp([a.get_yticklabels() for a in fig.axes[1:3:60]], visible=True)
    plt.setp([a.set_xticks([]) for a in fig.axes[:-4]])
    plt.setp([a.get_xticklabels() for a in fig.axes[-4:60:2]], visible=False)
    plt.tight_layout(h_pad=0,w_pad=-1.1)
    figure_queue.put(fig)

def plot_coverage(countdict, outfile):
    pdf = PdfPages('multipage.pdf')
    pages_number = int(len(countdict)/60)
    uncomplete_last = len(countdict) % 60
    ref_list = sorted(countdict)
    if len(ref_list) >= 60:
        fig, axarr = plt.subplots( 15, 4, figsize=(8,11))
    counter = 0
    boo = 0
    """ Format figure """
    for a in fig.axes:
        if counter % 4 != 0:
            plt.setp(a.get_yticklabels(), visible=False)
            plt.setp(a.set_yticks([]))
        elif boo != 0:
            plt.setp(a.get_yticklabels(), visible=False)
            boo = 0
        else:
            boo = 1
        if counter < 56:
            plt.setp(a.get_xticklabels(), visible=False)
            plt.setp(a.set_xticks([]))
        elif counter % 2 != 0:
            plt.setp(a.get_xticklabels(), visible=False)
        counter += 1
    """ Plot first page """
    xax = 0
    yax = 0
    line_list = list()
    for i in range(0,60):
        ref = ref_list.pop(0)
        m = max(countdict[ref])
        max_coord = len(countdict[ref])
        if m > 0:
            l = [x/m for x in countdict[ref]]
        else:
            l = countdict[ref]
        l_coord = [x/max_coord for x in range(0,max_coord)]
        line, = axarr[xax,yax].plot(l_coord,l)
        line_list.append(line)
        axarr[xax,yax].set_title(ref, fontsize=7)
        axarr[xax,yax].set_ylim(-0.01,1.1)
        yax += 1
        if yax == 4:
            yax = 0
            xax += 1
    fig.text(0.02, 0.5, 'Normalized counts', va='center', rotation='vertical', fontsize=12)
    fig.text(0.4, 0.02, 'Normalized coordinates', va='center', rotation='horizontal', fontsize=12)
    fig.suptitle('miRNA coverage maps',fontsize=14)
    fig.tight_layout(rect=[0.05, 0.03, 1, 0.95],h_pad=-0.5,w_pad=-1.1)
    pdf.savefig(fig)
    if pages_number > 1:
        for page in range(1,pages_number):
            plot_number = 0
            for xax in range(0,15):
                for yax in range(0,4):
                    ref = ref_list.pop(0)
                    m = max(countdict[ref])
                    max_coord = len(countdict[ref])
                    if m > 0:
                        l = [x/m for x in countdict[ref]]
                    else:
                        l = countdict[ref]
                    l_coord = [x/max_coord for x in range(0,max_coord)]
                    axarr[xax,yax].set_title(ref, fontsize=7)
                    line_list[plot_number].set_ydata(l)
                    line_list[plot_number].set_xdata(l_coord)
                    plot_number += 1
            pdf.savefig(fig)
    if uncomplete_last > 0:
        print(uncomplete_last)
        fig, axarr = plt.subplots(nrows=15, ncols=4, figsize=(8,11))
        counter = 0
        boo = 0
        """ Format figure """
        for a in fig.axes:
            if counter < uncomplete_last:
                if counter % 4 != 0:
                    plt.setp(a.get_yticklabels(), visible=False)
                    plt.setp(a.set_yticks([]))
                elif boo != 0:
                    plt.setp(a.get_yticklabels(), visible=False)
                    boo = 0
                else:
                    boo = 1
                if counter < 56:
                    plt.setp(a.get_xticklabels(), visible=False)
                    plt.setp(a.set_xticks([]))
                elif counter % 2 != 0:
                    plt.setp(a.get_xticklabels(), visible=False)
            else:
                fig.delaxes(a)
            counter += 1
        """ Plot last """
        xax = 0
        yax = 0
        nb = 0
        for ref in ref_list:
            if nb < uncomplete_last:
                m = max(countdict[ref])
                max_coord = len(countdict[ref])
                if m > 0:
                    l = [x/m for x in countdict[ref]]
                else:
                    l = countdict[ref]
                l_coord = [x/max_coord for x in range(0,max_coord)]
                axarr[xax,yax].plot(l_coord,l)
                axarr[xax,yax].set_title(ref, fontsize=7)
                axarr[xax,yax].set_ylim(-0.01,1.1)
                yax += 1
                if yax == 4:
                    yax = 0
                    xax += 1
            nb += 1
        fig.text(0.02, 0.5, 'Normalized counts', va='center', rotation='vertical', fontsize=12)
        fig.text(0.4, 0.02, 'Normalized coordinates', va='center', rotation='horizontal', fontsize=12)
        fig.suptitle('miRNA coverage maps',fontsize=14)
        fig.tight_layout(rect=[0.05, 0.03, 1, 0.95],h_pad=-0.5,w_pad=-1.1)
        pdf.savefig(fig)
    pdf.close()

def write_counts(countdict, outfile):
    """
    Takes a dict[<gene_type name>]=count and
    writes a count table
    """
    F = open(outfile, 'w')
    for gene in sorted(countdict):
        F.write('%s\t%s\n' % (gene, str(countdict[gene])))
    F.close()

def main():
    args = Parser()
    bamfile = pysam.AlignmentFile(args.alignment_file, 'rb', check_sq=False)
    if args.pre_mirs:
        pre_mirs = get_pre_mir_counts(bamfile)
        write_counts(pre_mirs, args.pre_mirs)
        if args.lattice:
            pre_mirs_coverage = get_pre_mir_coverage(bamfile,
                                                     args.quality_threshold)
            plot_coverage(pre_mirs_coverage, args.lattice)#write_dataframe_coverage(pre_mirs_coverage, args.lattice)
    if args.mirs:
        mirs = get_mir_counts(bamfile, args.gff_file)
        write_counts(mirs, args.mirs)

if __name__ == '__main__':
    main()
<commit_msg>Modified the file so as to be more readable and commented a 'format figure' section to see if its necessary<commit_after>#!/usr/bin/env python3

import matplotlib
matplotlib.use('Agg')
import argparse
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.backends.backend_pdf import PdfPages
import pysam
from multiprocessing import Process, Queue

def Parser():
    parser = argparse.ArgumentParser(description='miRNAs counts and coverages')
    parser.add_argument('-a', '--alignment', metavar='FILE', type=str,
                        dest='alignment_file', help='Alignment bam file')
    parser.add_argument('--gff', metavar='FILE', type=str, dest='gff_file',
                        help='GFF3 describing both pre-miRNAs\
                              and mature miRNAs')
    parser.add_argument('-q', '--quality_threshold', type=int,
                        dest='quality_threshold',
                        help='Quality threshold for coverage (default=10)',
                        default=10)
    parser.add_argument('-p', '--pre_mirs', type=str, dest='pre_mirs',
                        help='pre-miRNAs count file path', metavar='FILE')
    parser.add_argument('-m', '--mirs', type=str, dest='mirs',
                        help='mature miRNA count file path', metavar='FILE')
    parser.add_argument('--lattice', metavar='FILE', type=str, dest='lattice',
                        help='Output file for the lattice dataframe.')
    args = parser.parse_args()
    return args

def get_pre_mir_counts(bamfile):
    """
    Takes a AlignmentFile object and returns a dictionary of counts for reads
    aligning with pre_mirs (as keys)
    """
    count = dict()
    for ref_name in bamfile.references:
        count[ref_name] = bamfile.count(reference=ref_name)
    return count

def get_pre_mir_coverage(bamfile, quality=10):
    """
    Takes a AlignmentFile object and returns a dictionary of lists
    of coverage along the coordinates of pre_mirs (as keys)
    """
    coverage = dict()
    for ref_name, ref_len in zip(bamfile.references, bamfile.lengths):
        coverage[ref_name] = bamfile.count_coverage(reference=ref_name,
                                                    start=0, end=ref_len,
                                                    quality_threshold=quality)
        """ Add the 4 coverage values """
        coverage[ref_name] = np.array([sum(x) for x in
                                       zip(*coverage[ref_name])])
    return coverage

def get_mir_counts(bamfile, gff_file):
    """
    Takes a AlignmentFile and a gff file and computes for
    each 'miRNA' region of the gff the number of reads that hit it
    returns a dict[mir_name] = count
    """
    counts = dict()
    for line in open(gff_file, 'r'):
        if line[0] != '#':
            gff_fields = line[:-1].split("\t")
            if gff_fields[2] == 'miRNA':
                mir_name = gff_fields[0]
                premir_name = gff_fields[8].split('=')[-1]
                mir_start = int(gff_fields[3])
                mir_end = int(gff_fields[4])
                # GFF is 1-based, pysam is 0-based.
                counts[mir_name] = bamfile.count(reference=premir_name,
                                                 start=mir_start-1,
                                                 end=mir_end-1)
    return counts

def write_dataframe_coverage(countdict, outfile):
    """
    Takes a dict[pre_mir reference name] = [coverage list]
    and writes a dataframe with columns:
    <gene_type name>, offset, normoffset, counts and normcounts
    in the outfile
    """
    F = open(outfile, 'w')
    F.write('Mir_hairpin\tOffset\tNorm_offset\tCount\tNorm_count\n')
    for ref in sorted(countdict):
        """
        For each reference name in mirs,
        write the coverage of each of its positions
        """
        maximum = max(countdict[ref])
        reference_length = len(countdict[ref])
        for pos, c in enumerate(countdict[ref]):
            """ Compute and write value for each reference position"""
            F.write('%s\t%s\t%s\t%s\t%s\n' % (ref, str(pos + 1),
                    str(float(pos+1)/reference_length), str(float(c)),
                    str(float(c)/maximum) if maximum != 0 else '0'))
    F.close()

def plot_coverage_par(countdict, outfile):
    """
    Tkes a dict[pre_mir reference name] = [coverage list]
    and plots the coverage of each pre_mir reference name
    """
    pdf = PdfPages('multipage.pdf')
    xax = 0
    yax = 0
    process_list = list()
    nb_images = int(len(countdict) / 60)
    figure_queue = Queue(6)
    num_img = 1
    sref = sorted(countdict)
    nprocess = 0
    nb_pages = 0
    previous_page = 0
    for pages in range(1,nb_images+2):
        print(pages)
        current_page = pages*60
        if nprocess <4:
            nprocess += 1
            process_list.append(Process(target=plot_img,args=(countdict,
                                                              sref[previous_page:current_page],
                                                              figure_queue,)))
            process_list[len(process_list)-1].start()
        else:
            p = process_list.pop(0)
            while figure_queue.empty():
                p.join(1)
            if not figure_queue.empty():
                pdf.savefig(figure_queue.get())
            process_list.append(Process(target=plot_img,args=(countdict,
                                                              sref[previous_page:current_page],
                                                              figure_queue,)))
            process_list[len(process_list)-1].start()
        previous_page = current_page
    while nb_pages < len(process_list):
        if not figure_queue.empty():
            nb_pages += 1
            pdf.savefig(figure_queue.get())
    pdf.close()
    for p in process_list:
        p.join()

def plot_img_par(countdict, ref_list, figure_queue):
    fig, axarr = plt.subplots(15, 4, figsize=(10, 20), sharey='row')
    xax = 0
    yax = 0
    for ref in ref_list:
        m = max(countdict[ref])
        max_coord = len(countdict[ref])
        if m > 0:
            l = [x/m for x in countdict[ref]]
        else:
            l = countdict[ref]
        l_coord = [x/max_coord for x in range(0,max_coord)]
        axarr[xax,yax].plot(l_coord,l)
        axarr[xax,yax].set_title(ref)
        axarr[xax,yax].set_ylim(-0.01,1.1)
        yax += 1
        if yax == 4:
            yax = 0
            xax += 1
    plt.setp([a.get_yticklabels() for a in fig.axes[1:3:60]], visible=True)
    plt.setp([a.set_xticks([]) for a in fig.axes[:-4]])
    plt.setp([a.get_xticklabels() for a in fig.axes[-4:60:2]], visible=False)
    plt.tight_layout(h_pad=0,w_pad=-1.1)
    figure_queue.put(fig)

def plot_coverage(countdict, outfile):
    pdf = PdfPages('multipage.pdf')
    pages_number = int(len(countdict)/60)
    uncomplete_last = len(countdict) % 60
    ref_list = sorted(countdict)
    if len(ref_list) >= 60:
        fig, axarr = plt.subplots( 15, 4, figsize=(8,11))
    counter = 0
    boo = 0
    """ Format figure
    for a in fig.axes:
        if counter % 4 != 0:
            plt.setp(a.get_yticklabels(), visible=False)
            plt.setp(a.set_yticks([]))
        elif boo != 0:
            plt.setp(a.get_yticklabels(), visible=False)
            boo = 0
        else:
            boo = 1
        if counter < 56:
            plt.setp(a.get_xticklabels(), visible=False)
            plt.setp(a.set_xticks([]))
        elif counter % 2 != 0:
            plt.setp(a.get_xticklabels(), visible=False)
        counter += 1 """
    """ Plot first page """
    xax = 0
    yax = 0
    line_list = list()
    for i in range(0,60):
        ref = ref_list.pop(0)
        plot_coverage_plotting(ref, countdict, xax, yax, axarr,
                               first=True)
        yax += 1
        if yax == 4:
            yax = 0
            xax += 1
    fig.text(0.02, 0.5, 'Normalized counts', va='center', rotation='vertical', fontsize=12)
    fig.text(0.4, 0.02, 'Normalized coordinates', va='center', rotation='horizontal', fontsize=12)
    fig.suptitle('miRNA coverage maps',fontsize=14)
    fig.tight_layout(rect=[0.05, 0.03, 1, 0.95],h_pad=-0.5,w_pad=-1.1)
    pdf.savefig(fig)
    if pages_number > 1:
        for page in range(1,pages_number):
            plot_number = 0
            for xax in range(0,15):
                for yax in range(0,4):
                    ref = ref_list.pop(0)
                    plot_coverage_plotting(ref, countdict, xax, yax, axarr,
                                           line_list=line_list)
                    plot_number += 1
            pdf.savefig(fig)
    if uncomplete_last > 0:
        print(uncomplete_last)
        fig, axarr = plt.subplots(nrows=15, ncols=4, figsize=(8,11))
        counter = 0
        boo = 0
        """ Format figure """
        for a in fig.axes:
            if counter < uncomplete_last:
                if counter % 4 != 0:
                    plt.setp(a.get_yticklabels(), visible=False)
                    plt.setp(a.set_yticks([]))
                elif boo != 0:
                    plt.setp(a.get_yticklabels(), visible=False)
                    boo = 0
                else:
                    boo = 1
                if counter < 56:
                    plt.setp(a.get_xticklabels(), visible=False)
                    plt.setp(a.set_xticks([]))
                elif counter % 2 != 0:
                    plt.setp(a.get_xticklabels(), visible=False)
            else:
                fig.delaxes(a)
            counter += 1
        """ Plot last """
        xax = 0
        yax = 0
        nb = 0
        for ref in ref_list:
            if nb < uncomplete_last:
                plot_coverage_plotting(ref, countdict, xax, yax, axarr,
                                       last=False)
                yax += 1
                if yax == 4:
                    yax = 0
                    xax += 1
            nb += 1
        fig.text(0.02, 0.5, 'Normalized counts', va='center', rotation='vertical', fontsize=12)
        fig.text(0.4, 0.02, 'Normalized coordinates', va='center', rotation='horizontal', fontsize=12)
        fig.suptitle('miRNA coverage maps',fontsize=14)
        fig.tight_layout(rect=[0.05, 0.03, 1, 0.95],h_pad=-0.5,w_pad=-1.1)
        pdf.savefig(fig)
    pdf.close()

def plot_coverage_plotting(ref, countdict, xax, yax, axarr, first=False,
                           last=False, line_list=None):
    m = max(countdict[ref])
    max_coord = len(countdict[ref])
    if m > 0:
        l = [x/m for x in countdict[ref]]
    else:
        l = countdict[ref]
    l_coord = [x/max_coord for x in range(0,max_coord)]
    axarr[xax,yax].set_title(ref, fontsize=7)
    if first:
        line, = axarr[xax,yax].plot(l_coord,l)
        line_list.append(line)
        axarr[xax,yax].set_ylim(-0.01,1.1)
        return line
    elif last:
        axarr[xax,yax].plot(l_coord,l)
        axarr[xax,yax].set_ylim(-0.01,1.1)
    else:
        line_list[plot_number].set_ydata(l)
        line_list[plot_number].set_xdata(l_coord)

def write_counts(countdict, outfile):
    """
    Takes a dict[<gene_type name>]=count and
    writes a count table
    """
    F = open(outfile, 'w')
    for gene in sorted(countdict):
        F.write('%s\t%s\n' % (gene, str(countdict[gene])))
    F.close()

def main():
    args = Parser()
    bamfile = pysam.AlignmentFile(args.alignment_file, 'rb', check_sq=False)
    if args.pre_mirs:
        pre_mirs = get_pre_mir_counts(bamfile)
        write_counts(pre_mirs, args.pre_mirs)
        if args.lattice:
            pre_mirs_coverage = get_pre_mir_coverage(bamfile,
                                                     args.quality_threshold)
            plot_coverage(pre_mirs_coverage, args.lattice)#write_dataframe_coverage(pre_mirs_coverage, args.lattice)
    if args.mirs:
        mirs = get_mir_counts(bamfile, args.gff_file)
        write_counts(mirs, args.mirs)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from progressbar import ProgressBar
from pretty import *
import argparse
import lasagne
import theano
import theano.tensor as T
import numpy
import time

import os
import os.path

parser = argparse.ArgumentParser()
parser.add_argument('tagged', help='path to directory containing prepared files')
parser.add_argument('-e', '--epochs', type=int, help='number of epochs', default=100) # TODO: dictated by number of learning rates
parser.add_argument('-r', '--reserve', type=float, help='percentage of samples to reserve for validation and testing', default=5)
parser.add_argument('-l', '--learning', type=float, help='learning rate', default=0.01) # TODO: variable learning rate
parser.add_argument('-m', '--momentum', type=float, help='momentum', default=0.9)
parser.add_argument('-b', '--batchsize', type=int, help='size of each mini batch', default=256)
parser.add_argument('-s', '--stop', type=int, help='stop after this many batches each epoch', default=0)
parser.add_argument('-v', '--verbose', action='count')
args = parser.parse_args()

section("Setup")
task("Loading data")
y_train = numpy.memmap(os.path.join(args.tagged, "train.labels.db"), dtype=numpy.int32, mode='r')
X_train = numpy.memmap(os.path.join(args.tagged, "train.images.db"), dtype=numpy.float32, mode='r', shape=(len(y_train), 3, 128, 128))
cats = numpy.max(y_train)+1

# use X% of data for validation, and X% for testing
reserved = int(args.reserve * len(y_train) / 100.0)
X_train, X_val, X_test = X_train[:-2*reserved], X_train[-2*reserved:-reserved], X_train[-reserved:]
y_train, y_val, y_test = y_train[:-2*reserved], y_train[-2*reserved:-reserved], y_train[-reserved:]

task("Building model and compiling functions")
# create Theano variables for input and target minibatch
input_var = T.tensor4('X')
target_var = T.ivector('y')

# create a small convolutional neural network
from lasagne.nonlinearities import leaky_rectify, softmax
network = lasagne.layers.InputLayer((None, 3, 128, 128), input_var)
# 1st
network = lasagne.layers.Conv2DLayer(network, 64, (8, 8), stride=2,
                                     nonlinearity=leaky_rectify)
network = lasagne.layers.MaxPool2DLayer(network, (3, 3), stride=2)
# 2nd
network = lasagne.layers.Conv2DLayer(network, 96, (5, 5), stride=1,
                                     nonlinearity=leaky_rectify)
network = lasagne.layers.MaxPool2DLayer(network, (3, 3), stride=2)
# 3rd
network = lasagne.layers.Conv2DLayer(network, 128, (3, 3), stride=1,
                                     nonlinearity=leaky_rectify)
network = lasagne.layers.MaxPool2DLayer(network, (3, 3), stride=2)
# 4th
network = lasagne.layers.DenseLayer(lasagne.layers.dropout(network, 0.5),
                                    512, nonlinearity=leaky_rectify,
                                    W=lasagne.init.Orthogonal())
# 5th
network = lasagne.layers.DenseLayer(lasagne.layers.dropout(network, 0.5),
                                    cats, nonlinearity=softmax)

# create loss function
prediction = lasagne.layers.get_output(network)
loss = lasagne.objectives.categorical_crossentropy(prediction, target_var)
loss = loss.mean() + 1e-4 * lasagne.regularization.regularize_network_params(
        network, lasagne.regularization.l2)

# create parameter update expressions
params = lasagne.layers.get_all_params(network, trainable=True)
updates = lasagne.updates.nesterov_momentum(loss, params, learning_rate=args.learning, momentum=args.momentum)

# compile training function that updates parameters and returns training loss
train_fn = theano.function([input_var, target_var], loss, updates=updates)

# Create a loss expression for validation/testing. The crucial difference here
# is that we do a deterministic forward pass through the network, disabling
# dropout layers.
test_prediction = lasagne.layers.get_output(network, deterministic=True)
test_loss = lasagne.objectives.categorical_crossentropy(test_prediction,
                                                        target_var)
test_loss = test_loss.mean()
# As a bonus, also create an expression for the classification accuracy:
test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var),
                  dtype=theano.config.floatX)

# compile a second function computing the validation loss and accuracy:
val_fn = theano.function([input_var, target_var], [test_loss, test_acc])

def iterate_minibatches(inputs, targets, batchsize, shuffle=False):
    assert len(inputs) == len(targets)
    if shuffle:
        indices = numpy.arange(len(inputs))
        numpy.random.shuffle(indices)
    for start_idx in range(0, len(inputs) - batchsize + 1, batchsize):
        if shuffle:
            excerpt = indices[start_idx:start_idx + batchsize]
        else:
            excerpt = slice(start_idx, start_idx + batchsize)
        yield inputs[excerpt], targets[excerpt]

section("Training")
# Finally, launch the training loop.
for epoch in range(args.epochs):
    task("Starting training epoch {}".format(epoch))
    start_time = time.time()

    # In each epoch, we do a pass over minibatches of the training data:
    train_err = 0
    train_batches = len(range(0, len(X_train) - args.batchsize + 1, args.batchsize))
    p = ProgressBar(max_value = train_batches).start()
    i = 1
    for batch in iterate_minibatches(X_train, y_train, args.batchsize, shuffle=True):
        train_err += train_fn(batch[0], batch[1])
        p.update(i)
        i = i+1
        if args.stop != 0 and i > args.stop:
            break

    subtask("Doing forward pass on validation data (size: {})".format(len(X_val)))
    # Also do a validation data forward pass
    val_err = 0
    val_acc = 0
    val_batches = len(range(0, len(X_val) - args.batchsize + 1, args.batchsize))
    p = ProgressBar(max_value = val_batches).start()
    i = 1
    for batch in iterate_minibatches(X_val, y_val, args.batchsize, shuffle=False):
        err, acc = val_fn(batch[0], batch[1])
        val_err += err
        val_acc += acc
        p.update(i)
        i = i+1

    # Then we print the results for this epoch:
    subtask("Epoch results: {:.6f}/{:.6f}/{:.2f}% (tloss, vloss, vacc)".format(
        train_err / train_batches,
        val_err / val_batches,
        val_acc / val_batches * 100
    ))

section("Evaluation")
# After training, we compute and print the test error:
task("Evaluating performance on test data set")
test_err = 0
test_acc = 0
test_batches = len(range(0, len(X_test) - args.batchsize + 1, args.batchsize))
p = ProgressBar(max_value = test_batches).start()
i = 1
for batch in iterate_minibatches(X_test, y_test, args.batchsize, shuffle=False):
    err, acc = val_fn(batch[0], batch[1])
    test_err += err
    test_acc += acc
    p.update(i)
    i = i+1

print(colored(" ==> Final results: {:.6f} loss, {:.2f}% accuracy <== ".format(
    test_err / test_batches,
    test_acc / test_batches * 100
), "green", attrs=["bold"]))
<commit_msg>Slightly better org<commit_after>#!/usr/bin/env python3

from progressbar import ProgressBar
from pretty import *
import argparse
import lasagne
import theano
import theano.tensor as T
import numpy
import time

import os
import os.path

parser = argparse.ArgumentParser()
parser.add_argument('tagged', help='path to directory containing prepared files')
parser.add_argument('-e', '--epochs', type=int, help='number of epochs', default=100) # TODO: dictated by number of learning rates
parser.add_argument('-r', '--reserve', type=float, help='percentage of samples to reserve for validation and testing', default=5)
parser.add_argument('-l', '--learning', type=float, help='learning rate', default=0.01) # TODO: variable learning rate
parser.add_argument('-m', '--momentum', type=float, help='momentum', default=0.9)
parser.add_argument('-b', '--batchsize', type=int, help='size of each mini batch', default=256)
parser.add_argument('-s', '--stop', type=int, help='stop after this many batches each epoch', default=0)
parser.add_argument('-v', '--verbose', action='count')
args = parser.parse_args()

section("Setup")
task("Loading data")
y_train = numpy.memmap(os.path.join(args.tagged, "train.labels.db"), dtype=numpy.int32, mode='r')
X_train = numpy.memmap(os.path.join(args.tagged, "train.images.db"), dtype=numpy.float32, mode='r', shape=(len(y_train), 3, 128, 128))
cats = numpy.max(y_train)+1

# use X% of data for validation, and X% for testing
reserved = int(args.reserve * len(y_train) / 100.0)
X_train, X_val, X_test = X_train[:-2*reserved], X_train[-2*reserved:-reserved], X_train[-reserved:]
y_train, y_val, y_test = y_train[:-2*reserved], y_train[-2*reserved:-reserved], y_train[-reserved:]

task("Building model and compiling functions")
# create Theano variables for input and target minibatch
input_var = T.tensor4('X')
target_var = T.ivector('y')

# create a small convolutional neural network
from lasagne.nonlinearities import leaky_rectify, softmax
network = lasagne.layers.InputLayer((None, 3, 128, 128), input_var)
# 1st
network = lasagne.layers.Conv2DLayer(network, 64, (8, 8), stride=2,
                                     nonlinearity=leaky_rectify)
network = lasagne.layers.MaxPool2DLayer(network, (3, 3), stride=2)
# 2nd
network = lasagne.layers.Conv2DLayer(network, 96, (5, 5), stride=1, pad='same',
                                     nonlinearity=leaky_rectify)
network = lasagne.layers.MaxPool2DLayer(network, (3, 3), stride=2)
# 3rd
network = lasagne.layers.Conv2DLayer(network, 128, (3, 3), stride=1, pad='same',
                                     nonlinearity=leaky_rectify)
network = lasagne.layers.MaxPool2DLayer(network, (3, 3), stride=2)
# 4th
network = lasagne.layers.DenseLayer(network, 512, nonlinearity=leaky_rectify)
network = lasagne.layers.DropoutLayer(network)
# 5th
network = lasagne.layers.DenseLayer(network, cats, nonlinearity=softmax)

# create loss function
prediction = lasagne.layers.get_output(network)
loss = lasagne.objectives.categorical_crossentropy(prediction, target_var)
loss = loss.mean() + 1e-4 * lasagne.regularization.regularize_network_params(
        network, lasagne.regularization.l2)

# create parameter update expressions
params = lasagne.layers.get_all_params(network, trainable=True)
updates = lasagne.updates.nesterov_momentum(loss, params, learning_rate=args.learning, momentum=args.momentum)

# compile training function that updates parameters and returns training loss
train_fn = theano.function([input_var, target_var], loss, updates=updates)

# Create a loss expression for validation/testing. The crucial difference here
# is that we do a deterministic forward pass through the network, disabling
# dropout layers.
test_prediction = lasagne.layers.get_output(network, deterministic=True)
test_loss = lasagne.objectives.categorical_crossentropy(test_prediction,
                                                        target_var)
test_loss = test_loss.mean()
# As a bonus, also create an expression for the classification accuracy:
test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var),
                  dtype=theano.config.floatX)

# compile a second function computing the validation loss and accuracy:
val_fn = theano.function([input_var, target_var], [test_loss, test_acc])

def iterate_minibatches(inputs, targets, batchsize, shuffle=False):
    assert len(inputs) == len(targets)
    if shuffle:
        indices = numpy.arange(len(inputs))
        numpy.random.shuffle(indices)
    for start_idx in range(0, len(inputs) - batchsize + 1, batchsize):
        if shuffle:
            excerpt = indices[start_idx:start_idx + batchsize]
        else:
            excerpt = slice(start_idx, start_idx + batchsize)
        yield inputs[excerpt], targets[excerpt]

section("Training")
# Finally, launch the training loop.
for epoch in range(args.epochs):
    task("Starting training epoch {}".format(epoch))
    start_time = time.time()

    # In each epoch, we do a pass over minibatches of the training data:
    train_err = 0
    train_batches = len(range(0, len(X_train) - args.batchsize + 1, args.batchsize))
    p = ProgressBar(max_value = train_batches).start()
    i = 1
    for batch in iterate_minibatches(X_train, y_train, args.batchsize, shuffle=True):
        train_err += train_fn(batch[0], batch[1])
        p.update(i)
        i = i+1
        if args.stop != 0 and i > args.stop:
            break

    subtask("Doing forward pass on validation data (size: {})".format(len(X_val)))
    # Also do a validation data forward pass
    val_err = 0
    val_acc = 0
    val_batches = len(range(0, len(X_val) - args.batchsize + 1, args.batchsize))
    p = ProgressBar(max_value = val_batches).start()
    i = 1
    for batch in iterate_minibatches(X_val, y_val, args.batchsize, shuffle=False):
        err, acc = val_fn(batch[0], batch[1])
        val_err += err
        val_acc += acc
        p.update(i)
        i = i+1

    # Then we print the results for this epoch:
    subtask("Epoch results: {:.6f}/{:.6f}/{:.2f}% (tloss, vloss, vacc)".format(
        train_err / train_batches,
        val_err / val_batches,
        val_acc / val_batches * 100
    ))

section("Evaluation")
# After training, we compute and print the test error:
task("Evaluating performance on test data set")
test_err = 0
test_acc = 0
test_batches = len(range(0, len(X_test) - args.batchsize + 1, args.batchsize))
p = ProgressBar(max_value = test_batches).start()
i = 1
for batch in iterate_minibatches(X_test, y_test, args.batchsize, shuffle=False):
    err, acc = val_fn(batch[0], batch[1])
    test_err += err
    test_acc += acc
    p.update(i)
    i = i+1

print(colored(" ==> Final results: {:.6f} loss, {:.2f}% accuracy <== ".format(
    test_err / test_batches,
    test_acc / test_batches * 100
), "green", attrs=["bold"]))
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2008-2015, Ekevoo.com.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
# compliance with the License. You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
#
import ctypes
from logging import getLogger, StreamHandler, DEBUG
from os.path import dirname, join
from platform import system
from os import statvfs

from tendo.singleton import SingleInstance
from yaml import load

from pull import YoutubeChannelVideoFeed, BlogVideoFeed, append_to_queue
from dedup import Deduplicator
from youtube_dl import YoutubeDL

BASEDIR = dirname(__file__)
log = getLogger('tubeforme.main')

def main():
    sentinel = SingleInstance()
    try:
        log.info('Starting. %d MB free.', get_free_space_mb(BASEDIR))
        find_new_stuff()
        deduplicate()
        dowload_found_stuff()
    finally:
        del sentinel

def find_new_stuff():
    subscriptions = open('subscriptions.yaml', 'r')
    settings = load(subscriptions)
    for channel in settings['youtube_channels']:
        append_to_queue(YoutubeChannelVideoFeed(channel))
    for blog in settings['blogs']:
        append_to_queue(BlogVideoFeed(blog))

def deduplicate():
    ded = Deduplicator(BASEDIR, 'videos', 'queue.txt')
    ded.main()

def dowload_found_stuff():
    ydl_opts = {
        'writedescription': True,
        'noprogress': True,
        'writesubtitles': True,
    }
    with YoutubeDL(ydl_opts) as ydl:
        for link in open(join(BASEDIR, 'queue.txt')):
            space = get_free_space_mb(BASEDIR)
            link = link.strip()
            if link and space > 500:
                ydl.download([link])
            else:
                log.fatal("Disk space low, exiting! %d MB free.", space)
                exit(1)

def get_free_space_mb(pathname):
    """Return folder/drive free space (in megabytes)."""
    if system() == 'Windows':
        free_bytes = ctypes.c_ulonglong(0)
        ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(pathname), None, None, ctypes.pointer(free_bytes))
        return free_bytes.value / 1024 / 1024
    else:
        st = statvfs(pathname)
        return st.f_bavail * st.f_frsize / 1024 / 1024

if __name__ == '__main__':
    toplog = getLogger('tubeforme')
    toplog.addHandler(StreamHandler())
    toplog.level = DEBUG
    main()
<commit_msg>Added zipimporter. Not sure yet if it works.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2008-2015, Ekevoo.com.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
# compliance with the License. You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and limitations under the License.
#
import ctypes
from logging import getLogger, StreamHandler, DEBUG
from os.path import dirname, join
from platform import system
from os import statvfs

from tendo.singleton import SingleInstance
from yaml import load

from pull import YoutubeChannelVideoFeed, BlogVideoFeed, append_to_queue
from dedup import Deduplicator
from zipimport import zipimporter

BASEDIR = dirname(__file__)
log = getLogger('tubeforme.main')

def main():
    sentinel = SingleInstance()
    try:
        log.info('Starting. %d MB free.', get_free_space_mb(BASEDIR))
        find_new_stuff()
        deduplicate()
        dowload_found_stuff()
    finally:
        del sentinel

def find_new_stuff():
    subscriptions = open('subscriptions.yaml', 'r')
    settings = load(subscriptions)
    for channel in settings['youtube_channels']:
        append_to_queue(YoutubeChannelVideoFeed(channel))
    for blog in settings['blogs']:
        append_to_queue(BlogVideoFeed(blog))

def deduplicate():
    ded = Deduplicator(BASEDIR, 'videos', 'queue.txt')
    ded.main()

def dowload_found_stuff():
    ydl_opts = {
        'writedescription': True,
        'noprogress': True,
        'writesubtitles': True,
    }
    ydl = zipimporter('youtube-dl').load_module('youtube_dl')
    with ydl.YoutubeDL(ydl_opts) as ydl:
        for link in open(join(BASEDIR, 'queue.txt')):
            space = get_free_space_mb(BASEDIR)
            link = link.strip()
            if link and space > 500:
                ydl.download([link])
            else:
                log.fatal("Disk space low, exiting! %d MB free.", space)
                exit(1)

def get_free_space_mb(pathname):
    """Return folder/drive free space (in megabytes)."""
    if system() == 'Windows':
        free_bytes = ctypes.c_ulonglong(0)
        ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(pathname), None, None, ctypes.pointer(free_bytes))
        return free_bytes.value / 1024 / 1024
    else:
        st = statvfs(pathname)
        return st.f_bavail * st.f_frsize / 1024 / 1024

if __name__ == '__main__':
    toplog = getLogger('tubeforme')
    toplog.addHandler(StreamHandler())
    toplog.level = DEBUG
    main()
<|endoftext|>


<commit_before>import kaptan
from sh import tmux, cut
from pprint import pprint

class Session(object):

    __FORMATS__ = [
        'session_attached', 'session_created', 'session_created_string',
        'session_group', 'session_grouped', 'session_height', 'session_id',
        'session_name', 'session_width', 'session_windows']

    def __init__(self, session_name):
        self._session_name = session_name

    @property
    def session_name(self):
        return self._session_name

    @property
    def windows(self):
        """
            ``tmux list-windows`` outputs 1 session per line ``\n``.

            -F (FORMATS) allows returning custom sessings, we delimit with
            a tab ``\t``.

            we then use dict+zip to align the format variable with the
            output.

            the ``Window`` object accepts the returned properties  and
            ``Session`` (``self``) object.
        """

        if hasattr(self, '_session_name'):
            formats = WINDOW_FORMATS
            tmux_formats = ['#{%s}\t' % format for format in formats]

            windows = cut(
                tmux(
                    'list-windows',                 # ``tmux list-windows``
                    '-t%s' % self.session_name,     # target (session name)
                    '-F%s' % ''.join(tmux_formats)  # output
                ), '-f1', '-d:'
            )

            # combine format keys with values returned from ``tmux list-windows``
            windows = [dict(zip(formats, window.split('\t'))) for window in windows]

            # clear up empty dict
            windows = [
                dict((k, v) for k, v in window.iteritems() if v) for window in windows
            ]

            windows = [Window(session=self, **window) for window in windows]

            return windows
        else:
            return None  # session is not bound to a current session, the user
                         # is using Session imported config file to launch a
                         # new session

    def __repr__(self):
        # todo test without session_name
        return "%s(%s)" % (self.__class__, self.session_name)

class Window(object):
    """
        todo
            - @has_session property, or throw an Error
            - Pane
            - __cmd__
    """

    def __init__(self, **kwargs):

        if 'session' in kwargs:
            if isinstance(kwargs['session'], Session):
                self._session = kwargs['session']

        [setattr(self, k, v) for (k, v) in kwargs.items() if k is not 'session']

    def __repr__(self):
        # todo test without session_name
        return "%s(%s)" % (self.__class__, self.__dict__)

    @property
    def session(self):
        return self._session if self._session else None

    @property
    def panes(self):
        """
            ``tmux list-panes`` outputs 1 session per line ``\n``.

            -F (FORMATS) allows returning custom sessings, we delimit with
            a tab ``\t``.

            we then use dict+zip to align the format variable with the
            output.

            the ``Pane`` object accepts the returned properties, ``Session``
            object and ``Window`` (``self``) object.
        """

        if hasattr(self.session, '_session_name'):
            formats = PANE_FORMATS
            tmux_formats = ['#{%s}\t' % format for format in formats]

            panes = cut(
                tmux(
                    'list-panes',
                    '-s',                                # for sessions
                    '-t%s' % self.session.session_name,  # target (name of session)
                    '-F%s' % ''.join(tmux_formats)       # output
                ), '-f1', '-d:'
            )

            # `tmux list-panes` outputs a session per-line,
            # separate every line from `tmux list-panes` into a pane
            panes = str(panes).split('\n')

            # zip and map the results into the dict of formats used above
            panes = [dict(zip(formats, pane.split('\t'))) for pane in panes]

            # clear up empty dict
            panes = [
                dict((k, v) for k, v in pane.iteritems() if v) for pane in panes
            ]

            panes = [Pane(**pane) for pane in panes]

            return panes
        else:
            return None  # session is not bound to a current session, the user
                         # is using ``Session`` imported config file to launch a
                         # new session

class Pane(object):
    def __init__(self, **kwargs):
        for (k, v) in kwargs.items():
            setattr(self, k, v)

    def __repr__(self):
        # todo test without session_name
        return "%s(%s)" % (self.__class__, self.__dict__)

def get_sessions():
    for session in cut(tmux('list-sessions'), '-f1', '-d:'):
        yield Session(session.strip('\n'))

"""
    experiment, ALL_FORMATS parsing for session, window, pane object and
    remove blank values

    http://sourceforge.net/p/tmux/tmux-code/ci/master/tree/format.c
"""

SESSION_FORMATS = [
    'session_group',
    'session_name',
    'session_windows',
    'session_width',
    'session_height',
    'session_id',
    'session_grouped',
    'session_group',
    'session_created',
    'session_created_string',
    'session_attached',
]

CLIENT_FORMATS = [
    'client_cwd',
    'client_height',
    'client_width',
    'client_tty',
    'client_termname',
    'client_created',
    'client_created_string',
    'client_activity',
    'client_activity_string',
    'client_prefix',
    'client_utf8',
    'client_readonly',
    'client_session',
    'client_last_session',
]

WINDOW_FORMATS = [
    # format_window()
    'window_id',
    'window_name',
    'window_width',
    'window_height',
    'window_layout',
    'window_panes',
    # format_winlink()
    'window_index',
    'window_flags',
    'window_active',
    'window_bell_flag',
    'window_activity_flag',
    'window_silence_flag',
]

PANE_FORMATS = [
    'history_size',
    'history_limit',
    'history_bytes',
    'pane_index',
    'pane_width',
    'pane_height',
    'pane_title',
    'pane_id',
    'pane_active',
    'pane_dead',
    'pane_in_mode',
    'pane_synchronized',
    'pane_tty',
    'pane_pid',
    'pane_start_command',
    'pane_start_path',
    'pane_current_path',
    'pane_current_command',

    'cursor_x',
    'cursor_y',
    'scroll_region_upper',
    'scroll_region_lower',
    'saved_cursor_x',
    'saved_cursor_y',

    'alternate_on',
    'alternate_saved_x',
    'alternate_saved_y',

    'cursor_flag',
    'insert_flag',
    'keypad_cursor_flag',
    'keypad_flag',
    'wrap_flag',

    'mouse_standard_flag',
    'mouse_button_flag',
    'mouse_any_flag',
    'mouse_utf8_flag',
]

config = kaptan.Kaptan(handler="yaml")
config.import_config("""
    windows:
        - editor:
            layout: main-vertical
            panes:
                - vim
                - cowsay "hey"
        - server: htop
        - logs: tail -f logs/development.log
    """)

print config
pprint(config.__dict__)
""" expand inline config
    dict({'session_name': { dict })

    to

    dict({ name='session_name', **dict})
"""
windows = list()
for window in config.get('windows'):

    if len(window) == int(1):
        name = window.iterkeys().next()  # get window name

        """expand
            window[name] = 'command'

            to

            window[name] = {
                panes=['command']
            }
        """
        if isinstance(window[name], basestring):
            windowoptions = dict(
                panes=[window[name]]
            )
        else:
            windowoptions = window[name]

        window = dict(name=name, **windowoptions)
        if len(window['panes']) > int(1):
            pprint('omg multiple panes')

    windows.append(window)

for session in get_sessions():
    pprint(session)
    for window in session.windows:
        pprint(window)
        for pane in window.panes:
            #pprint(pane.__dict__)
            pass
<commit_msg>session information<commit_after>import kaptan
from sh import tmux, cut
from pprint import pprint

class Session(object):

    def __init__(self, **kwargs):
        for (k, v) in kwargs.items():
            if k is 'session_name':
                self._session_name = v
            else:
                setattr(self, k, v)

    @property
    def session_name(self):
        return self._session_name

    @property
    def windows(self):
        """
            ``tmux list-windows`` outputs 1 session per line ``\n``.

            -F (FORMATS) allows returning custom sessings, we delimit with
            a tab ``\t``.

            we then use dict+zip to align the format variable with the
            output.

            the ``Window`` object accepts the returned properties  and
            ``Session`` (``self``) object.
        """

        if hasattr(self, '_session_name'):
            formats = WINDOW_FORMATS
            tmux_formats = ['#{%s}\t' % format for format in formats]

            windows = cut(
                tmux(
                    'list-windows',                 # ``tmux list-windows``
                    '-t%s' % self.session_name,     # target (session name)
                    '-F%s' % ''.join(tmux_formats)  # output
                ), '-f1', '-d:'
            )

            # combine format keys with values returned from ``tmux list-windows``
            windows = [dict(zip(formats, window.split('\t'))) for window in windows]

            # clear up empty dict
            windows = [
                dict((k, v) for k, v in window.iteritems() if v) for window in windows
            ]

            windows = [Window(session=self, **window) for window in windows]

            return windows
        else:
            return None  # session is not bound to a current session, the user
                         # is using Session imported config file to launch a
                         # new session

    def __repr__(self):
        # todo test without session_name
        return "%s(%s)" % (self.__class__, self.session_name)

class Window(object):
    """
        todo
            - @has_session property, or throw an Error
            - Pane
            - __cmd__
    """

    def __init__(self, **kwargs):

        if 'session' in kwargs:
            if isinstance(kwargs['session'], Session):
                self._session = kwargs['session']

        [setattr(self, k, v) for (k, v) in kwargs.items() if k is not 'session']

    def __repr__(self):
        # todo test without session_name
        return "%s(%s)" % (self.__class__, self.__dict__)

    @property
    def session(self):
        return self._session if self._session else None

    @property
    def panes(self):
        """
            ``tmux list-panes`` outputs 1 session per line ``\n``.

            -F (FORMATS) allows returning custom sessings, we delimit with
            a tab ``\t``.

            we then use dict+zip to align the format variable with the
            output.

            the ``Pane`` object accepts the returned properties, ``Session``
            object and ``Window`` (``self``) object.
        """

        if hasattr(self.session, '_session_name'):
            formats = PANE_FORMATS
            tmux_formats = ['#{%s}\t' % format for format in formats]

            panes = cut(
                tmux(
                    'list-panes',
                    '-s',                                # for sessions
                    '-t%s' % self.session.session_name,  # target (name of session)
                    '-F%s' % ''.join(tmux_formats)       # output
                ), '-f1', '-d:'
            )

            # `tmux list-panes` outputs a session per-line,
            # separate every line from `tmux list-panes` into a pane
            panes = str(panes).split('\n')

            # zip and map the results into the dict of formats used above
            panes = [dict(zip(formats, pane.split('\t'))) for pane in panes]

            # clear up empty dict
            panes = [
                dict((k, v) for k, v in pane.iteritems() if v) for pane in panes
            ]

            panes = [Pane(**pane) for pane in panes]

            return panes
        else:
            return None  # session is not bound to a current session, the user
                         # is using ``Session`` imported config file to launch a
                         # new session

class Pane(object):
    def __init__(self, **kwargs):
        for (k, v) in kwargs.items():
            setattr(self, k, v)

    def __repr__(self):
        # todo test without session_name
        return "%s(%s)" % (self.__class__, self.__dict__)

def get_sessions():
    formats = SESSION_FORMATS
    tmux_formats = ['#{%s}\t' % format for format in formats]

    sessions = cut(
        tmux(
            'list-sessions',                 # ``tmux list-windows``
            '-F%s' % ''.join(tmux_formats)  # output
        ), '-f1', '-d:'
    )

    # combine format keys with values returned from ``tmux list-windows``
    sessions = [dict(zip(formats, session.split('\t'))) for session in sessions]

    # clear up empty dict
    sessions = [
        dict((k, v) for k, v in session.iteritems() if v) for session in sessions
    ]

    sessions = [Session(**session) for session in sessions]

    for session in sessions:
        yield session

"""
    experiment, ALL_FORMATS parsing for session, window, pane object and
    remove blank values

    http://sourceforge.net/p/tmux/tmux-code/ci/master/tree/format.c
"""

SESSION_FORMATS = [
    'session_group',
    'session_name',
    'session_windows',
    'session_width',
    'session_height',
    'session_id',
    'session_grouped',
    'session_group',
    'session_created',
    'session_created_string',
    'session_attached',
]

CLIENT_FORMATS = [
    'client_cwd',
    'client_height',
    'client_width',
    'client_tty',
    'client_termname',
    'client_created',
    'client_created_string',
    'client_activity',
    'client_activity_string',
    'client_prefix',
    'client_utf8',
    'client_readonly',
    'client_session',
    'client_last_session',
]

WINDOW_FORMATS = [
    # format_window()
    'window_id',
    'window_name',
    'window_width',
    'window_height',
    'window_layout',
    'window_panes',
    # format_winlink()
    'window_index',
    'window_flags',
    'window_active',
    'window_bell_flag',
    'window_activity_flag',
    'window_silence_flag',
]

PANE_FORMATS = [
    'history_size',
    'history_limit',
    'history_bytes',
    'pane_index',
    'pane_width',
    'pane_height',
    'pane_title',
    'pane_id',
    'pane_active',
    'pane_dead',
    'pane_in_mode',
    'pane_synchronized',
    'pane_tty',
    'pane_pid',
    'pane_start_command',
    'pane_start_path',
    'pane_current_path',
    'pane_current_command',

    'cursor_x',
    'cursor_y',
    'scroll_region_upper',
    'scroll_region_lower',
    'saved_cursor_x',
    'saved_cursor_y',

    'alternate_on',
    'alternate_saved_x',
    'alternate_saved_y',

    'cursor_flag',
    'insert_flag',
    'keypad_cursor_flag',
    'keypad_flag',
    'wrap_flag',

    'mouse_standard_flag',
    'mouse_button_flag',
    'mouse_any_flag',
    'mouse_utf8_flag',
]

config = kaptan.Kaptan(handler="yaml")
config.import_config("""
    windows:
        - editor:
            layout: main-vertical
            panes:
                - vim
                - cowsay "hey"
        - server: htop
        - logs: tail -f logs/development.log
    """)

print config
pprint(config.__dict__)
""" expand inline config
    dict({'session_name': { dict })

    to

    dict({ name='session_name', **dict})
"""
windows = list()
for window in config.get('windows'):

    if len(window) == int(1):
        name = window.iterkeys().next()  # get window name

        """expand
            window[name] = 'command'

            to

            window[name] = {
                panes=['command']
            }
        """
        if isinstance(window[name], basestring):
            windowoptions = dict(
                panes=[window[name]]
            )
        else:
            windowoptions = window[name]

        window = dict(name=name, **windowoptions)
        if len(window['panes']) > int(1):
            pprint('omg multiple panes')

    windows.append(window)

for session in get_sessions():

    pprint(session)
    for window in session.windows:
        pprint(window)
        for pane in window.panes:
            #pprint(pane.__dict__)
            pass
<|endoftext|>


<commit_before>import re

def remove_prefix(string, prefix):
    return re.sub("^{}".format(prefix), "", string)

def remove_suffix(string, suffix):
    return re.sub("{}$".format(suffix), "", string)
<commit_msg>Add missed convert_unicode_to_string function back to util.py<commit_after>import re

def remove_prefix(string, prefix):
    return re.sub("^{}".format(prefix), "", string)

def remove_suffix(string, suffix):
    return re.sub("{}$".format(suffix), "", string)

def convert_unicode_to_string(self, obj):
    """
    Convert all unicode parts of a dictionary or list to standard strings.

    This method may not handle special characters well!
    :param obj: the dictionary, list, or unicode string
    :return: the same data type as obj, but with unicode strings converted to python strings.
    """
    if isinstance(obj, dict):
        return {self.convert_unicode_to_string(key): self.convert_unicode_to_string(value)
                for key, value in obj.items()}
    elif isinstance(obj, list):
        return [self.convert_unicode_to_string(element) for element in obj]
    elif isinstance(obj, unicode):
        return obj.encode('utf-8')

    return obj
<|endoftext|>