<commit_before>import os
import pipes
import sys
import urllib
import StringIO
import tempfile
import time
import tvdb_api
from tvdb_api import tvdb_api
from mutagen.mp4 import MP4, MP4Cover

class Tvdb_mp4:
    def __init__(self, show, season, episode):
        attempts = 0
        while attempts < 3:
            try:
                self.tvdb_show = tvdb_api.Tvdb(interactive=False, cache=False, banners=True, actors=True, forceConnect=True)
                self.show = show
                self.season = season
                self.episode = episode
                self.HD = None

                #Gather information from theTVDB
                self.showdata = self.tvdb_show[self.show]
                self.episodedata = self.showdata[self.season][self.episode]
                #episodedata = self.tvdb_show[self.show][self.season][self.episode]

                self.show = self.showdata['seriesname']
                self.genre = self.showdata['genre']
                self.network = self.showdata['network']

                self.title = self.episodedata['episodename']
                self.description = self.episodedata['overview']
                self.airdate = self.episodedata['firstaired']
                self.director = self.episodedata['director']
                self.writer = self.episodedata['writer']

                #Generate XML tags for Actors/Writers/Directors
                self.xml = self.xmlTags()
                break
            except:
                print "Failed to connect to TVDB, trying again in 20 seconds"
                time.sleep(20)
                attempts += 1
        #end __init__

    def writeTags(self, mp4Path):
        print "Tagging file :" + mp4Path
        if not mp4Path.endswith('mp4') and not mp4Path.endswith('m4v'):
            print "Error: File is not the correct format"
            return
        ##Artwork will also require additional processing -- see artwork
        path = self.getArtwork()
        cover = open(path, 'rb').read()

        video=MP4(mp4Path)
        video["tvsh"] = self.show #TV show title
        video["\xa9nam"] = self.title #Video title
        video["tven"] = self.title #Episode title
        video["desc"] = self.description #Short description
        video["ldes"] = self.description #Long description (same a short for tv)
        video["tvnn"] = self.network #Network
        if self.airdate != "0000-00-00":
            video["\xa9day"] = self.airdate #Airdate
        video["tvsn"] = [self.season] #Season number
        video["tves"] = [self.episode] #Episode number
        video["stik"] = [10] #TV show iTunes category
        if self.HD is not None:
            video["hdvd"] = self.HD
        if self.genre != None:
            video["\xa9gen"] = self.genre.replace('|',',')[1:-1] #Genre(s)
        video["----:com.apple.iTunes:iTunMOVI"] = self.xml #XML - see xmlTags method
        if path.endswith('png'):
            video["covr"] = [MP4Cover(cover, MP4Cover.FORMAT_PNG)] #png poster
        else:
            video["covr"] = [MP4Cover(cover, MP4Cover.FORMAT_JPEG)] #jpeg poster

        video.pprint()
        attempts = 0
        while attempts < 3:
            try:
                print "Trying to write tags"
                MP4(mp4Path).delete(mp4Path)
                video.save()
                print "Tags written successfully"
                break
            except IOError:
                print "Failed"
                time.sleep(5)
                attempts += 1
    #end writeTags

    def setHD(self, width, height):
        if width >= 1280 or height >= 720:
            self.HD = True
        else:
            self.HD = False

    def xmlTags(self):
        #constants
        header = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\"><plist version=\"1.0\"><dict>\n"
        castheader = "<key>cast</key><array>\n"
        writerheader = "<key>screenwriters</key><array>\n"
        directorheader = "<key>directors</key><array>\n"
        subfooter = "</array>\n"
        footer = "</dict></plist>\n"

        output = StringIO.StringIO()
        output.write(header)
        output.write(castheader)

        count = 0
        #Write Actors
        while (count < len(self.showdata['_actors'])-1):
            name = self.showdata['_actors'][count]['name']
            if name != None:
                output.write("<dict><key>name</key><string>" + name.encode('ascii', errors='ignore') + "</string></dict>\n")
            count = count+1
        output.write(subfooter)
        #write Screenwriter
        if self.writer != None:
            output.write(writerheader)
            for name in self.writer.split("|"):
                output.write("<dict><key>name</key><string>" + name.encode('ascii', errors='ignore') + "</string></dict>\n")
            output.write(subfooter)
        #write Director
        if self.director != None:
            output.write(directorheader)
            for name in self.director.split("|"):
                output.write("<dict><key>name</key><string>" + name.encode('ascii', errors='ignore') + "</string></dict>\n")
            output.write(subfooter)
        output.write(footer)
        return output.getvalue()
        output.close()
    #end xmlTags

    def getArtwork(self):
        #Pulls down all the poster metadata for the correct season and sorts them into the Poster object
        posters = posterCollection()
        for bannerid in self.showdata['_banners']['season']['season'].keys():
            if str(self.showdata['_banners']['season']['season'][bannerid]['season']) == str(self.season):
                poster = Poster()
                poster.ratingcount = int(self.showdata['_banners']['season']['season'][bannerid]['ratingcount'])
                if poster.ratingcount > 0:
                    poster.rating = float(self.showdata['_banners']['season']['season'][bannerid]['rating'])
                poster.bannerpath = self.showdata['_banners']['season']['season'][bannerid]['_bannerpath']
                posters.addPoster(poster)
        return urllib.urlretrieve(posters.topPoster().bannerpath, tempfile.gettempdir() + "\poster.jpg")[0]
    #end artwork
#end tvdb_mp4

class Poster:
    #simple container for all the poster parameters needed
    def __init__(self, rating = 0, ratingcount = 0, bannerpath = ""):
        self.rating = rating
        self.bannerpath = bannerpath
        self.ratingcount = ratingcount
        #self.language
    #end poster

class posterCollection:
    def __init__(self):
        self.posters = []
    def topPoster(self):
        #Determines which poster has the highest rating, returns the Poster object
        top = Poster()
        for poster in self.posters:
            if poster.rating > top.rating:
                top = poster
            if poster.rating == top.rating and poster.ratingcount > top.ratingcount:
                top = poster
        return top
    def addPoster(self, inputPoster):
        self.posters.append(inputPoster)

def main():
    if len(sys.argv) > 3:
        tvdb_instance = tvdb_api.Tvdb(interactive=False, cache=False, banners=True, actors=True)
        mp4 = str(sys.argv[1]).replace("\\","\\\\").replace("\\\\\\\\","\\\\")
        tvdb_id = str(sys.argv[3])
        if tvdb_id.isdigit():
            tvdb_id = int(tvdb_id)
        tvdb_season = int(sys.argv[4])
        tvdb_episode = int(sys.argv[5])
        tvdb_mp4_instance = Tvdb_mp4(show=tvdb_id, season=tvdb_season, episode=tvdb_episode)
        if mp4.endswith(".mp4") or mp4.endswith(".m4v"):
            tvdb_mp4_instance.writeTags(mp4)
        else:
            print "Wrong file type (must be .mp4 or .m4v)"

if __name__ == '__main__':
    main()    <commit_msg>2.3.3<commit_after>import os
import pipes
import sys
import urllib
import StringIO
import tempfile
import time
import tvdb_api
from tvdb_api import tvdb_api
from mutagen.mp4 import MP4, MP4Cover

class Tvdb_mp4:
    def __init__(self, show, season, episode):
        attempts = 0
        while attempts < 3:
            try:
                self.tvdb_show = tvdb_api.Tvdb(interactive=False, cache=False, banners=True, actors=True, forceConnect=True)
                self.show = show
                self.season = season
                self.episode = episode
                self.HD = None

                #Gather information from theTVDB
                self.showdata = self.tvdb_show[self.show]
                self.seasondata = self.showdata[self.season]
                self.episodedata = self.seasondata[self.episode]
                #episodedata = self.tvdb_show[self.show][self.season][self.episode]

                self.show = self.showdata['seriesname']
                self.genre = self.showdata['genre']
                self.network = self.showdata['network']

                self.title = self.episodedata['episodename']
                self.description = self.episodedata['overview']
                self.airdate = self.episodedata['firstaired']
                self.director = self.episodedata['director']
                self.writer = self.episodedata['writer']

                #Generate XML tags for Actors/Writers/Directors
                self.xml = self.xmlTags()
                break
            except:
                print "Failed to connect to TVDB, trying again in 20 seconds"
                time.sleep(20)
                attempts += 1
        #end __init__

    def writeTags(self, mp4Path):
        print "Tagging file :" + mp4Path
        if not mp4Path.endswith('mp4') and not mp4Path.endswith('m4v'):
            print "Error: File is not the correct format"
            return
        ##Artwork will also require additional processing -- see artwork
        path = self.getArtwork()
        cover = open(path, 'rb').read()

        video=MP4(mp4Path)
        video["tvsh"] = self.show #TV show title
        video["\xa9nam"] = self.title #Video title
        video["tven"] = self.title #Episode title
        video["desc"] = self.description #Short description
        video["ldes"] = self.description #Long description (same a short for tv)
        video["tvnn"] = self.network #Network
        if self.airdate != "0000-00-00":
            video["\xa9day"] = self.airdate #Airdate
        video["tvsn"] = [self.season] #Season number
        video["\xa9alb"] = self.show + ", Season " + str(self.season)
        video["tves"] = [self.episode] #Episode number
        video["trkn"] = [(int(self.episode), len(self.seasondata))] #Episode number iTunes
        video["stik"] = [10] #TV show iTunes category
        if self.HD is not None:
            video["hdvd"] = self.HD
        if self.genre != None:
            video["\xa9gen"] = self.genre.replace('|',',')[1:-1] #Genre(s)
        video["----:com.apple.iTunes:iTunMOVI"] = self.xml #XML - see xmlTags method
        if path.endswith('png'):
            video["covr"] = [MP4Cover(cover, MP4Cover.FORMAT_PNG)] #png poster
        else:
            video["covr"] = [MP4Cover(cover, MP4Cover.FORMAT_JPEG)] #jpeg poster

        video.pprint()
        attempts = 0
        MP4(mp4Path).delete(mp4Path)
        video.save()
        while attempts < 3:
            try:
                print "Trying to write tags"
                MP4(mp4Path).delete(mp4Path)
                video.save()
                print "Tags written successfully"
                break
            except IOError:
                print IOError
                time.sleep(5)
                attempts += 1
    #end writeTags

    def setHD(self, width, height):
        if width >= 1280 or height >= 720:
            self.HD = True
        else:
            self.HD = False

    def xmlTags(self):
        #constants
        header = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\"><plist version=\"1.0\"><dict>\n"
        castheader = "<key>cast</key><array>\n"
        writerheader = "<key>screenwriters</key><array>\n"
        directorheader = "<key>directors</key><array>\n"
        subfooter = "</array>\n"
        footer = "</dict></plist>\n"

        output = StringIO.StringIO()
        output.write(header)
        output.write(castheader)

        count = 0
        #Write Actors
        while (count < len(self.showdata['_actors'])-1):
            name = self.showdata['_actors'][count]['name']
            if name != None:
                output.write("<dict><key>name</key><string>" + name.encode('ascii', errors='ignore') + "</string></dict>\n")
            count = count+1
        output.write(subfooter)
        #write Screenwriter
        if self.writer != None:
            output.write(writerheader)
            for name in self.writer.split("|"):
                if name != "":
                    output.write("<dict><key>name</key><string>" + name.encode('ascii', errors='ignore') + "</string></dict>\n")
            output.write(subfooter)
        #write Director
        if self.director != None:
            output.write(directorheader)
            for name in self.director.split("|"):
                if name != "":
                    output.write("<dict><key>name</key><string>" + name.encode('ascii', errors='ignore') + "</string></dict>\n")
            output.write(subfooter)
        output.write(footer)
        return output.getvalue()
        output.close()
    #end xmlTags

    def getArtwork(self):
        #Pulls down all the poster metadata for the correct season and sorts them into the Poster object
        posters = posterCollection()
        for bannerid in self.showdata['_banners']['season']['season'].keys():
            if str(self.showdata['_banners']['season']['season'][bannerid]['season']) == str(self.season):
                poster = Poster()
                poster.ratingcount = int(self.showdata['_banners']['season']['season'][bannerid]['ratingcount'])
                if poster.ratingcount > 0:
                    poster.rating = float(self.showdata['_banners']['season']['season'][bannerid]['rating'])
                poster.bannerpath = self.showdata['_banners']['season']['season'][bannerid]['_bannerpath']
                posters.addPoster(poster)
        return urllib.urlretrieve(posters.topPoster().bannerpath, tempfile.gettempdir() + "\poster.jpg")[0]
    #end artwork
#end tvdb_mp4

class Poster:
    #simple container for all the poster parameters needed
    def __init__(self, rating = 0, ratingcount = 0, bannerpath = ""):
        self.rating = rating
        self.bannerpath = bannerpath
        self.ratingcount = ratingcount
        #self.language
    #end poster

class posterCollection:
    def __init__(self):
        self.posters = []
    def topPoster(self):
        #Determines which poster has the highest rating, returns the Poster object
        top = Poster()
        for poster in self.posters:
            if poster.rating > top.rating:
                top = poster
            if poster.rating == top.rating and poster.ratingcount > top.ratingcount:
                top = poster
        return top
    def addPoster(self, inputPoster):
        self.posters.append(inputPoster)

def main():
    if len(sys.argv) > 3:
        tvdb_instance = tvdb_api.Tvdb(interactive=False, cache=False, banners=True, actors=True)
        mp4 = str(sys.argv[1]).replace("\\","\\\\").replace("\\\\\\\\","\\\\")
        tvdb_id = str(sys.argv[3])
        if tvdb_id.isdigit():
            tvdb_id = int(tvdb_id)
        tvdb_season = int(sys.argv[4])
        tvdb_episode = int(sys.argv[5])
        tvdb_mp4_instance = Tvdb_mp4(show=tvdb_id, season=tvdb_season, episode=tvdb_episode)
        if mp4.endswith(".mp4") or mp4.endswith(".m4v"):
            tvdb_mp4_instance.writeTags(mp4)
        else:
            print "Wrong file type (must be .mp4 or .m4v)"

if __name__ == '__main__':
    main()    <|endoftext|>


<commit_before>#!/usr/bin/python

import os
import sys
from optparse import OptionParser, SUPPRESS_HELP

from main import TvRenamr
from errors import *

parser = OptionParser()
# parser.add_option('--config', dest='config', help='')
parser.add_option('-c', '--canonical', dest='canonical', help='Set the show\'s canonical name to use when performing the online lookup.')
parser.add_option('--debug', action='store_true', dest='debug', help=SUPPRESS_HELP)
parser.add_option('--deluge', action='store_true', dest='deluge', help='Checks Deluge to make sure the file has been completed before renaming.')
parser.add_option('--deluge-ratio', dest='deluge_ratio', help='Checks Deluge for completed and that the file has at least reached X share ratio.')
parser.add_option('-d', '--dry-run', dest='dry', action='store_true', help='Dry run your renaming.')
parser.add_option('-e', '--episode', dest='episode', help='Set the episode number. Currently this will cause errors when working with more than one file.')
parser.add_option('--ignore-recursive', action='store_true', dest='ignore_recursive', help='Only use files from the root of a given directory, not entering any sub-directories.')
parser.add_option('-l', '--log_file', dest='log_file', help='Set the log file location.')
parser.add_option('--log_level', dest='log', help='Set the log level. Options: debug, info, warning, error and critical.')
parser.add_option('--library', dest='library', default='thetvdb', help='Set the library to use for retrieving episode titles. Options: thetvdb & tvrage.')
parser.add_option('-n', '--name', dest='name', help='Set the show\'s name. This will be used as the show\'s when the renaming is completed.')
# parser.add_option('--no-config', dest='no_config', action='store_true', help='Ignore the config')
parser.add_option('-o', '--output', dest='output_format', help='Set the output format for the episodes being renamed.')
parser.add_option('--organise', action='store_true', dest='organise', help='Organise renamed files into folders based on their show name and season number.')
parser.add_option('--no-organise', action='store_false', dest='organise', help='Explicitly tell Tv Renamr not to organise renamed files. Used to override the config.')
parser.add_option('-q', '--quiet', action='store_true', dest='quiet', help='Don\'t output logs to the command line')
parser.add_option('-r', "--rename-dir", dest='rename_dir', help='The directory to move renamed files to, if not specified the working directory is used.')
parser.add_option('--no-rename-dir', action='store_false', dest='rename_dir', help='Explicity tell Tv Renamr not to move renamed files. Used to override the config.')
parser.add_option('--regex', dest='regex', help='The regular expression to use when extracting information from files.')
parser.add_option('-s', '--season', dest='season', help='Set the season number.')
parser.add_option('-t', '--the', action='store_true', dest='the', help='Set the position of \'The\' in a show\'s name to the end of the file')
options, args = parser.parse_args()

def __determine_type(path, ignore_recursive=False, ignore_filelist=None):
    """
    Determines which files need to be processed for renaming.

    :param path: The input file or directory.
    :param ignore_recursive: To ignore a recursive search for files if 'path' is a directory.
    Default is False.
    :param ignore_filelist: Optional set of files to ignore from renaming. Often used by filtering
    methods such as Deluge.

    :returns: A list of files to be renamed.
    :rtype: A list of dictionaries, who's keys are 'directory' and 'filename'.
    """
    if os.path.isdir(path):
        filelist = []
        for root, dirs, files in os.walk(path):
            for fname in files:
                # If we have a file we should be ignoring and skipping it.
                if ignore_filelist is not None and (os.path.join(root, fname) in ignore_filelist):
                    continue
                filelist.append((root, fname))
            # Don't want a recusive walk?
            if ignore_recursive: break
        return filelist
    elif os.path.isfile(path):
        return [os.path.split(path)]
    else:
        raise Exception

def __start_dry_run():
    log.info('Dry Run beginning.')
    log.info('-'*70)
    log.info('')

def __stop_dry_run():
    log.info('')
    log.info('-'*70)
    log.info('Dry Run complete. No files were harmed in the process.')
    log.info('')

def rename(path):
    try:
        details = __determine_type(path)
    except Exception:
        print '%s is not a file or directory. Ruh Roe!' % path
    for full_path in details:
        working, filename = full_path
        tv = TvRenamr(working, options.log, options.log_file, options.debug, options.quiet, options.dry)
        if options.dry: __start_dry_run()
        try:
            credentials = tv.extract_details_from_file(filename, user_regex=options.regex)

            if options.season: credentials['season']=options.season
            if options.episode: credentials['episode']=options.episode

            credentials['title'] = tv.retrieve_episode_name(library=options.library,
                                                        canonical=options.canonical, **credentials)
            credentials['show'] = tv.format_show_name(show=credentials['show'], the=options.the,
                                                        override=options.name)

            path = tv.build_path(dry=options.dry, rename_dir=options.rename_dir, \
                                organise=options.organise, format=options.output_format, **credentials)

            tv.rename(filename,path)
        except Exception, e:
            print e
            pass
        if options.dry: __stop_dry_run()

if __name__=="__main__":
    if args[0] is None: parser.error('You must specify a file or directory')

    # Need to capture the Deluge arguments here, before we enter rename so we can instead pass it
    # as a callback to be called once we've fetched the required information from deluge.
    if options.deluge or options.deluge_ratio:
        if options.deluge and not options.deluge_ratio: options.deluge_ratio = 0
        from lib.filter_deluge import get_deluge_ignore_file_list
        get_deluge_ignore_file_list(rename, options.deluge_ratio, args[0])
    else: rename(args[0])
else: print 'This script is only designed to be run standalone'<commit_msg>changed print statement to parser error when target file/directory isnt valid<commit_after>#!/usr/bin/python

import os
import sys
from optparse import OptionParser, SUPPRESS_HELP

from main import TvRenamr
from errors import *

parser = OptionParser()
# parser.add_option('--config', dest='config', help='')
parser.add_option('-c', '--canonical', dest='canonical', help='Set the show\'s canonical name to use when performing the online lookup.')
parser.add_option('--debug', action='store_true', dest='debug', help=SUPPRESS_HELP)
parser.add_option('--deluge', action='store_true', dest='deluge', help='Checks Deluge to make sure the file has been completed before renaming.')
parser.add_option('--deluge-ratio', dest='deluge_ratio', help='Checks Deluge for completed and that the file has at least reached X share ratio.')
parser.add_option('-d', '--dry-run', dest='dry', action='store_true', help='Dry run your renaming.')
parser.add_option('-e', '--episode', dest='episode', help='Set the episode number. Currently this will cause errors when working with more than one file.')
parser.add_option('--ignore-recursive', action='store_true', dest='ignore_recursive', help='Only use files from the root of a given directory, not entering any sub-directories.')
parser.add_option('-l', '--log_file', dest='log_file', help='Set the log file location.')
parser.add_option('--log_level', dest='log', help='Set the log level. Options: debug, info, warning, error and critical.')
parser.add_option('--library', dest='library', default='thetvdb', help='Set the library to use for retrieving episode titles. Options: thetvdb & tvrage.')
parser.add_option('-n', '--name', dest='name', help='Set the show\'s name. This will be used as the show\'s when the renaming is completed.')
# parser.add_option('--no-config', dest='no_config', action='store_true', help='Ignore the config')
parser.add_option('-o', '--output', dest='output_format', help='Set the output format for the episodes being renamed.')
parser.add_option('--organise', action='store_true', dest='organise', help='Organise renamed files into folders based on their show name and season number.')
parser.add_option('--no-organise', action='store_false', dest='organise', help='Explicitly tell Tv Renamr not to organise renamed files. Used to override the config.')
parser.add_option('-q', '--quiet', action='store_true', dest='quiet', help='Don\'t output logs to the command line')
parser.add_option('-r', "--rename-dir", dest='rename_dir', help='The directory to move renamed files to, if not specified the working directory is used.')
parser.add_option('--no-rename-dir', action='store_false', dest='rename_dir', help='Explicity tell Tv Renamr not to move renamed files. Used to override the config.')
parser.add_option('--regex', dest='regex', help='The regular expression to use when extracting information from files.')
parser.add_option('-s', '--season', dest='season', help='Set the season number.')
parser.add_option('-t', '--the', action='store_true', dest='the', help='Set the position of \'The\' in a show\'s name to the end of the file')
options, args = parser.parse_args()

def __determine_type(path, ignore_recursive=False, ignore_filelist=None):
    """
    Determines which files need to be processed for renaming.

    :param path: The input file or directory.
    :param ignore_recursive: To ignore a recursive search for files if 'path' is a directory.
    Default is False.
    :param ignore_filelist: Optional set of files to ignore from renaming. Often used by filtering
    methods such as Deluge.

    :returns: A list of files to be renamed.
    :rtype: A list of dictionaries, who's keys are 'directory' and 'filename'.
    """
    if os.path.isdir(path):
        filelist = []
        for root, dirs, files in os.walk(path):
            for fname in files:
                # If we have a file we should be ignoring and skipping it.
                if ignore_filelist is not None and (os.path.join(root, fname) in ignore_filelist):
                    continue
                filelist.append((root, fname))
            # Don't want a recusive walk?
            if ignore_recursive: break
        return filelist
    elif os.path.isfile(path):
        return [os.path.split(path)]
    else:
        raise Exception

def __start_dry_run():
    log.info('Dry Run beginning.')
    log.info('-'*70)
    log.info('')

def __stop_dry_run():
    log.info('')
    log.info('-'*70)
    log.info('Dry Run complete. No files were harmed in the process.')
    log.info('')

def rename(path):
    try:
        details = __determine_type(path)
    except Exception: parser.error('\'%s\' is not a file or directory. Ruh Roe!' % path)
    for full_path in details:
        working, filename = full_path
        tv = TvRenamr(working, options.log, options.log_file, options.debug, options.quiet, options.dry)
        if options.dry: __start_dry_run()
        try:
            credentials = tv.extract_details_from_file(filename, user_regex=options.regex)

            if options.season: credentials['season']=options.season
            if options.episode: credentials['episode']=options.episode

            credentials['title'] = tv.retrieve_episode_name(library=options.library,
                                                        canonical=options.canonical, **credentials)
            credentials['show'] = tv.format_show_name(show=credentials['show'], the=options.the,
                                                        override=options.name)

            path = tv.build_path(dry=options.dry, rename_dir=options.rename_dir, \
                                organise=options.organise, format=options.output_format, **credentials)

            tv.rename(filename,path)
        except Exception, e:
            print e
            pass
        if options.dry: __stop_dry_run()

if __name__=="__main__":
    if args[0] is None: parser.error('You must specify a file or directory')

    # Need to capture the Deluge arguments here, before we enter rename so we can instead pass it
    # as a callback to be called once we've fetched the required information from deluge.
    if options.deluge or options.deluge_ratio:
        if options.deluge and not options.deluge_ratio: options.deluge_ratio = 0
        from lib.filter_deluge import get_deluge_ignore_file_list
        get_deluge_ignore_file_list(rename, options.deluge_ratio, args[0])
    else: rename(args[0])
else: print 'This script is only designed to be run standalone'<|endoftext|>


<commit_before>import re
import decimal
import datetime
import collections

import mt940

from . import _compat
from . import processors

class Model(object):
    pass

class FixedOffset(datetime.tzinfo):
    '''Fixed time offset based on the Python docs
    Source: https://docs.python.org/2/library/datetime.html#tzinfo-objects

    >>> offset = FixedOffset(60)
    >>> offset.utcoffset(None)
    datetime.timedelta(0, 3600)
    >>> offset.dst(None)
    datetime.timedelta(0)
    >>> offset.tzname(None)
    '60'
    '''

    def __init__(self, offset=0, name=None):
        self._name = name or str(offset)

        if not isinstance(offset, int):
            offset = int(offset)
        self._offset = datetime.timedelta(minutes=offset)

    def utcoffset(self, dt):
        return self._offset

    def dst(self, dt):
        return datetime.timedelta(0)

    def tzname(self, dt):
        return self._name

class DateTime(datetime.datetime, Model):
    '''Just a regular datetime object which supports dates given as strings

    >>> DateTime(year='2000', month='1', day='2', hour='3', minute='4',
    ...          second='5', microsecond='6')
    DateTime(2000, 1, 2, 3, 4, 5, 6)

    >>> DateTime(year='123', month='1', day='2', hour='3', minute='4',
    ...          second='5', microsecond='6')
    DateTime(2123, 1, 2, 3, 4, 5, 6)

    >>> DateTime(2000, 1, 2, 3, 4, 5, 6)
    DateTime(2000, 1, 2, 3, 4, 5, 6)

    >>> DateTime(year='123', month='1', day='2', hour='3', minute='4',
    ...          second='5', microsecond='6', tzinfo=FixedOffset('60'))
    DateTime(2123, 1, 2, 3, 4, 5, 6, tzinfo=<mt940.models.FixedOffset ...>)

    Args:
        year (str): Year (0-100), will automatically add 2000 when needed
        month (str): Month
        day (str): Day
        hour (str): Hour
        minute (str): Minute
        second (str): Second
        microsecond (str): Microsecond
        tzinfo (tzinfo): Timezone information. Overwrites `offset`
        offset (str): Timezone offset in minutes, generates a tzinfo object
                      with the given offset if no tzinfo is available.
    '''

    def __new__(cls, *args, **kwargs):
        if kwargs:
            values = dict(
                year=None,
                month=None,
                day=None,
                hour='0',
                minute='0',
                second='0',
                microsecond='0', )

            # The list makes sure this works in both Python 2 and 3

            for key, default in list(values.items()):
                # Fetch the value or the default
                value = kwargs.get(key, default)
                # Convert the value to integer and force base 10 to make sure
                # it doesn't get recognized as octal
                value = int(value, 10)
                # Save the values again
                values[key] = value

            if values['year'] < 1000:
                values['year'] += 2000

            values['tzinfo'] = None

            if kwargs.get('tzinfo'):
                values['tzinfo'] = kwargs['tzinfo']

            if kwargs.get('offset'):
                values['tzinfo'] = FixedOffset(kwargs['offset'])

            return datetime.datetime.__new__(cls, **values)
        else:
            return datetime.datetime.__new__(cls, *args, **kwargs)

class Date(datetime.date, Model):
    '''Just a regular date object which supports dates given as strings

    >>> Date(year='2000', month='1', day='2')
    Date(2000, 1, 2)

    >>> Date(year='123', month='1', day='2')
    Date(2123, 1, 2)

    Args:
        year (str): Year (0-100), will automatically add 2000 when needed
        month (str): Month
        day (str): Day
    '''

    def __new__(cls, *args, **kwargs):
        if kwargs:
            dt = DateTime(*args, **kwargs).date()

            return datetime.date.__new__(cls, dt.year, dt.month, dt.day)
        else:
            return datetime.date.__new__(cls, *args, **kwargs)

class Amount(Model):
    '''Amount object containing currency and amount

    Args:
        amount (str): Amount using either a , or a . as decimal separator
        status (str): Either C or D for credit or debit respectively
        currency (str): A 3 letter currency (e.g. EUR)

    >>> Amount('123.45', 'C', 'EUR')
    <123.45 EUR>
    >>> Amount('123.45', 'D', 'EUR')
    <-123.45 EUR>
    '''

    def __init__(self, amount, status, currency=None, **kwargs):
        self.amount = decimal.Decimal(amount.replace(',', '.'))
        self.currency = currency

        # C = credit, D = debit

        if status == 'D':
            self.amount = -self.amount

    def __repr__(self):
        return '<%s %s>' % (
            self.amount,
            self.currency, )

class SumAmount(Amount):
    def __init__(self, *args, **kwargs):
        number = kwargs.pop('number')
        super(SumAmount, self).__init__(*args, **kwargs)
        self.number = number

    def __repr__(self):
        return '<%s %s in %s stmts)>' % (self.amount, self.currency,
                                         self.number)

class Balance(Model):
    '''Parse balance statement

    Args:
        status (str): Either C or D for credit or debit respectively
        amount (Amount): Object containing the amount and currency
        date (date): The balance date

    >>> balance = Balance('C', '0.00', Date(2010, 7, 22))
    >>> balance.status
    'C'
    >>> balance.amount.amount
    Decimal('0.00')
    >>> isinstance(balance.date, Date)
    True
    >>> balance.date.year, balance.date.month, balance.date.day
    (2010, 7, 22)

    >>> Balance()
    <None @ None>
    '''

    def __init__(self, status=None, amount=None, date=None, **kwargs):
        if amount and not isinstance(amount, Amount):
            amount = Amount(amount, status, kwargs.get('currency'))
        self.status = status
        self.amount = amount
        self.date = date

    def __repr__(self):
        return '<%s>' % self

    def __str__(self):
        return '%s @ %s' % (
            self.amount,
            self.date, )

class Transactions(collections.Sequence):
    '''
    Collection of :py:class:`Transaction` objects with global properties such
    as begin and end balance

    '''

    #: Using the processors you can pre-process data before creating objects
    #: and modify them after creating the objects
    DEFAULT_PROCESSORS = dict(
        pre_account_identification=[],
        post_account_identification=[],
        pre_available_balance=[],
        post_available_balance=[],
        pre_closing_balance=[],
        post_closing_balance=[],
        pre_intermediate_closing_balance=[],
        post_intermediate_closing_balance=[],
        pre_final_closing_balance=[],
        post_final_closing_balance=[],
        pre_forward_available_balance=[],
        post_forward_available_balance=[],
        pre_opening_balance=[],
        post_opening_balance=[],
        pre_intermediate_opening_balance=[],
        post_intermediate_opening_balance=[],
        pre_final_opening_balance=[],
        post_final_opening_balance=[],
        pre_related_reference=[],
        post_related_reference=[],
        pre_statement=[processors.date_fixup_pre_processor],
        post_statement=[processors.date_cleanup_post_processor],
        pre_statement_number=[],
        post_statement_number=[],
        pre_non_swift=[],
        post_non_swift=[],
        pre_transaction_details=[],
        post_transaction_details=[
            processors.transaction_details_post_processor
        ],
        pre_transaction_reference_number=[],
        post_transaction_reference_number=[],
        pre_floor_limit_indicator=[],
        post_floor_limit_indicator=[],
        pre_date_time_indication=[],
        post_date_time_indication=[],
        pre_sum_credit_entries=[],
        post_sum_credit_entries=[],
        pre_sum_debit_entries=[],
        post_sum_debit_entries=[])

    def __init__(self, processors=None):
        self.processors = self.DEFAULT_PROCESSORS.copy()

        if processors:
            self.processors.update(processors)

        self.transactions = []
        self.data = {}

    @property
    def currency(self):
        balance = mt940.utils.coalesce(
            self.data.get('final_opening_balance'),
            self.data.get('opening_balance'),
            self.data.get('intermediate_opening_balance'),
            self.data.get('available_balance'),
            self.data.get('forward_available_balance'),
            self.data.get('final_closing_balance'),
            self.data.get('closing_balance'),
            self.data.get('intermediate_closing_balance'),
            self.data.get('c_floor_limit'),
            self.data.get('d_floor_limit'), )

        if balance:
            if isinstance(balance, Amount):
                return balance.currency

            return balance.amount.currency

    @classmethod
    def strip(cls, lines):
        for line in lines:
            # We don't like carriage returns in case of Windows files so let's
            # just replace them with nothing
            line = line.replace('\r', '')

            # Strip trailing whitespace from lines since they cause incorrect
            # files
            line = line.rstrip()

            # Skip separators

            if line.strip() == '-':
                continue

            # Return actual lines

            if line:
                yield line

    @classmethod
    def normalize_tag_id(cls, tag_id):
        # Since non-digit tags exist, make the conversion optional
        if tag_id.isdigit():
            tag_id = int(tag_id)

        return tag_id

    @classmethod
    def sanatize_tag_id_matches(cls, matches):
        i_next = 0
        for i, match in enumerate(matches):
            # match was rejected
            if i < i_next:
                continue

            # next match would be
            i_next = i + 1

            # normalize tag id
            tag_id = cls.normalize_tag_id(match.group('tag'))

            # tag should be known
            assert tag_id in mt940.tags.TAG_BY_ID, 'Unknown tag %r ' \
                'in line: %r' % (tag_id, match.group(0))

            # special treatment for long tag content with possible
            # bad line wrap which produces tag_id like line beginnings
            # seen with :86: tag
            if tag_id == mt940.tags.Tags.TRANSACTION_DETAILS.value.id:
                # search subsequent tags for unknown tag ids
                # these lines likely belong to the previous tag
                for j in range(i_next, len(matches)):
                    next_tag_id = cls.normalize_tag_id(matches[j].group('tag'))
                    if next_tag_id in mt940.tags.TAG_BY_ID:
                        # this one is the next valid match
                        i_next = j
                        break
                    # else reject match

            # a valid match
            yield match

    def parse(self, data):
        '''Parses mt940 data, expects a string with data

        Args:
            data (str): The MT940 data

        Returns: :py:class:`list` of :py:class:`Transaction`
        '''
        # Remove extraneous whitespace and such
        data = '\n'.join(self.strip(data.split('\n')))

        # The pattern is a bit annoying to match by regex, even with a greedy
        # match it's difficult to get both the beginning and the end so we're
        # working around it in a safer way to get everything.
        tag_re = re.compile(
            r'^:(?P<full_tag>(?P<tag>[0-9]{2}|NS)(?P<sub_tag>[A-Z])?):',
            re.MULTILINE)
        matches = list(tag_re.finditer(data))

        # identify valid matches
        valid_matches = list(self.sanatize_tag_id_matches(matches))

        for i, match in enumerate(valid_matches):
            tag_id = self.normalize_tag_id(match.group('tag'))

            # get tag instance corresponding to tag id
            tag = mt940.tags.TAG_BY_ID.get(match.group('full_tag')) \
                or mt940.tags.TAG_BY_ID[tag_id]

            # Nice trick to get all the text that is part of this tag, python
            # regex matches have a `end()` and `start()` to indicate the start
            # and end index of the match.

            if valid_matches[i + 1:]:
                tag_data =
                    data[match.end():valid_matches[i + 1].start()].strip()
            else:
                tag_data = data[match.end():].strip()

            tag_dict = tag.parse(self, tag_data)

            # Preprocess data before creating the object

            for processor in self.processors.get('pre_%s' % tag.slug):
                tag_dict = processor(self, tag, tag_dict)

            result = tag(self, tag_dict)

            # Postprocess the object

            for processor in self.processors.get('post_%s' % tag.slug):
                result = processor(self, tag, tag_dict, result)

            # Creating a new transaction for :20: and :61: tags allows the
            # tags from :20: to :61: to be captured as part of the transaction.

            if isinstance(tag, mt940.tags.Statement):
                # Transactions only get a Transaction Reference Code ID from a
                # :61: tag which is why a new transaction is created if the
                # 'id' has a value.

                if not self.transactions:
                    transaction = Transaction(self)
                    self.transactions.append(transaction)

                if transaction.data.get('id'):
                    transaction = Transaction(self, result)
                    self.transactions.append(transaction)
                else:
                    transaction.data.update(result)
            elif issubclass(tag.scope, Transaction) and self.transactions:
                # Combine multiple results together as one string, Rabobank has
                # multiple :86: tags for a single transaction

                for k, v in _compat.iteritems(result):
                    if k in transaction.data and hasattr(v, 'strip'):
                        transaction.data[k] += '\n%s' % v.strip()
                    else:
                        transaction.data[k] = v

            elif issubclass(tag.scope, Transactions):  # pragma: no branch
                self.data.update(result)

        return self.transactions

    def __getitem__(self, key):
        return self.transactions[key]

    def __len__(self):
        return len(self.transactions)

    def __repr__(self):
        return '<%s[%s]>' % (
            self.__class__.__name__,
            ']['.join('%s: %s' % (k.replace('_balance', ''), v)
                      for k, v in _compat.iteritems(self.data)
                      if k.endswith('balance')))

class Transaction(Model):
    def __init__(self, transactions, data=None):
        self.transactions = transactions
        self.data = {}
        self.update(data)

    def update(self, data):
        if data:
            self.data.update(data)

    def __repr__(self):
        return '<%s[%s] %s>' % (
            self.__class__.__name__,
            self.data.get('date'),
            self.data.get('amount'), )
<commit_msg>Fix build again<commit_after>import re
import decimal
import datetime
import collections

import mt940

from . import _compat
from . import processors

class Model(object):
    pass

class FixedOffset(datetime.tzinfo):
    '''Fixed time offset based on the Python docs
    Source: https://docs.python.org/2/library/datetime.html#tzinfo-objects

    >>> offset = FixedOffset(60)
    >>> offset.utcoffset(None)
    datetime.timedelta(0, 3600)
    >>> offset.dst(None)
    datetime.timedelta(0)
    >>> offset.tzname(None)
    '60'
    '''

    def __init__(self, offset=0, name=None):
        self._name = name or str(offset)

        if not isinstance(offset, int):
            offset = int(offset)
        self._offset = datetime.timedelta(minutes=offset)

    def utcoffset(self, dt):
        return self._offset

    def dst(self, dt):
        return datetime.timedelta(0)

    def tzname(self, dt):
        return self._name

class DateTime(datetime.datetime, Model):
    '''Just a regular datetime object which supports dates given as strings

    >>> DateTime(year='2000', month='1', day='2', hour='3', minute='4',
    ...          second='5', microsecond='6')
    DateTime(2000, 1, 2, 3, 4, 5, 6)

    >>> DateTime(year='123', month='1', day='2', hour='3', minute='4',
    ...          second='5', microsecond='6')
    DateTime(2123, 1, 2, 3, 4, 5, 6)

    >>> DateTime(2000, 1, 2, 3, 4, 5, 6)
    DateTime(2000, 1, 2, 3, 4, 5, 6)

    >>> DateTime(year='123', month='1', day='2', hour='3', minute='4',
    ...          second='5', microsecond='6', tzinfo=FixedOffset('60'))
    DateTime(2123, 1, 2, 3, 4, 5, 6, tzinfo=<mt940.models.FixedOffset ...>)

    Args:
        year (str): Year (0-100), will automatically add 2000 when needed
        month (str): Month
        day (str): Day
        hour (str): Hour
        minute (str): Minute
        second (str): Second
        microsecond (str): Microsecond
        tzinfo (tzinfo): Timezone information. Overwrites `offset`
        offset (str): Timezone offset in minutes, generates a tzinfo object
                      with the given offset if no tzinfo is available.
    '''

    def __new__(cls, *args, **kwargs):
        if kwargs:
            values = dict(
                year=None,
                month=None,
                day=None,
                hour='0',
                minute='0',
                second='0',
                microsecond='0', )

            # The list makes sure this works in both Python 2 and 3

            for key, default in list(values.items()):
                # Fetch the value or the default
                value = kwargs.get(key, default)
                # Convert the value to integer and force base 10 to make sure
                # it doesn't get recognized as octal
                value = int(value, 10)
                # Save the values again
                values[key] = value

            if values['year'] < 1000:
                values['year'] += 2000

            values['tzinfo'] = None

            if kwargs.get('tzinfo'):
                values['tzinfo'] = kwargs['tzinfo']

            if kwargs.get('offset'):
                values['tzinfo'] = FixedOffset(kwargs['offset'])

            return datetime.datetime.__new__(cls, **values)
        else:
            return datetime.datetime.__new__(cls, *args, **kwargs)

class Date(datetime.date, Model):
    '''Just a regular date object which supports dates given as strings

    >>> Date(year='2000', month='1', day='2')
    Date(2000, 1, 2)

    >>> Date(year='123', month='1', day='2')
    Date(2123, 1, 2)

    Args:
        year (str): Year (0-100), will automatically add 2000 when needed
        month (str): Month
        day (str): Day
    '''

    def __new__(cls, *args, **kwargs):
        if kwargs:
            dt = DateTime(*args, **kwargs).date()

            return datetime.date.__new__(cls, dt.year, dt.month, dt.day)
        else:
            return datetime.date.__new__(cls, *args, **kwargs)

class Amount(Model):
    '''Amount object containing currency and amount

    Args:
        amount (str): Amount using either a , or a . as decimal separator
        status (str): Either C or D for credit or debit respectively
        currency (str): A 3 letter currency (e.g. EUR)

    >>> Amount('123.45', 'C', 'EUR')
    <123.45 EUR>
    >>> Amount('123.45', 'D', 'EUR')
    <-123.45 EUR>
    '''

    def __init__(self, amount, status, currency=None, **kwargs):
        self.amount = decimal.Decimal(amount.replace(',', '.'))
        self.currency = currency

        # C = credit, D = debit

        if status == 'D':
            self.amount = -self.amount

    def __repr__(self):
        return '<%s %s>' % (
            self.amount,
            self.currency, )

class SumAmount(Amount):
    def __init__(self, *args, **kwargs):
        number = kwargs.pop('number')
        super(SumAmount, self).__init__(*args, **kwargs)
        self.number = number

    def __repr__(self):
        return '<%s %s in %s stmts)>' % (self.amount, self.currency,
                                         self.number)

class Balance(Model):
    '''Parse balance statement

    Args:
        status (str): Either C or D for credit or debit respectively
        amount (Amount): Object containing the amount and currency
        date (date): The balance date

    >>> balance = Balance('C', '0.00', Date(2010, 7, 22))
    >>> balance.status
    'C'
    >>> balance.amount.amount
    Decimal('0.00')
    >>> isinstance(balance.date, Date)
    True
    >>> balance.date.year, balance.date.month, balance.date.day
    (2010, 7, 22)

    >>> Balance()
    <None @ None>
    '''

    def __init__(self, status=None, amount=None, date=None, **kwargs):
        if amount and not isinstance(amount, Amount):
            amount = Amount(amount, status, kwargs.get('currency'))
        self.status = status
        self.amount = amount
        self.date = date

    def __repr__(self):
        return '<%s>' % self

    def __str__(self):
        return '%s @ %s' % (
            self.amount,
            self.date, )

class Transactions(collections.Sequence):
    '''
    Collection of :py:class:`Transaction` objects with global properties such
    as begin and end balance

    '''

    #: Using the processors you can pre-process data before creating objects
    #: and modify them after creating the objects
    DEFAULT_PROCESSORS = dict(
        pre_account_identification=[],
        post_account_identification=[],
        pre_available_balance=[],
        post_available_balance=[],
        pre_closing_balance=[],
        post_closing_balance=[],
        pre_intermediate_closing_balance=[],
        post_intermediate_closing_balance=[],
        pre_final_closing_balance=[],
        post_final_closing_balance=[],
        pre_forward_available_balance=[],
        post_forward_available_balance=[],
        pre_opening_balance=[],
        post_opening_balance=[],
        pre_intermediate_opening_balance=[],
        post_intermediate_opening_balance=[],
        pre_final_opening_balance=[],
        post_final_opening_balance=[],
        pre_related_reference=[],
        post_related_reference=[],
        pre_statement=[processors.date_fixup_pre_processor],
        post_statement=[processors.date_cleanup_post_processor],
        pre_statement_number=[],
        post_statement_number=[],
        pre_non_swift=[],
        post_non_swift=[],
        pre_transaction_details=[],
        post_transaction_details=[
            processors.transaction_details_post_processor
        ],
        pre_transaction_reference_number=[],
        post_transaction_reference_number=[],
        pre_floor_limit_indicator=[],
        post_floor_limit_indicator=[],
        pre_date_time_indication=[],
        post_date_time_indication=[],
        pre_sum_credit_entries=[],
        post_sum_credit_entries=[],
        pre_sum_debit_entries=[],
        post_sum_debit_entries=[])

    def __init__(self, processors=None):
        self.processors = self.DEFAULT_PROCESSORS.copy()

        if processors:
            self.processors.update(processors)

        self.transactions = []
        self.data = {}

    @property
    def currency(self):
        balance = mt940.utils.coalesce(
            self.data.get('final_opening_balance'),
            self.data.get('opening_balance'),
            self.data.get('intermediate_opening_balance'),
            self.data.get('available_balance'),
            self.data.get('forward_available_balance'),
            self.data.get('final_closing_balance'),
            self.data.get('closing_balance'),
            self.data.get('intermediate_closing_balance'),
            self.data.get('c_floor_limit'),
            self.data.get('d_floor_limit'), )

        if balance:
            if isinstance(balance, Amount):
                return balance.currency

            return balance.amount.currency

    @classmethod
    def strip(cls, lines):
        for line in lines:
            # We don't like carriage returns in case of Windows files so let's
            # just replace them with nothing
            line = line.replace('\r', '')

            # Strip trailing whitespace from lines since they cause incorrect
            # files
            line = line.rstrip()

            # Skip separators

            if line.strip() == '-':
                continue

            # Return actual lines

            if line:
                yield line

    @classmethod
    def normalize_tag_id(cls, tag_id):
        # Since non-digit tags exist, make the conversion optional
        if tag_id.isdigit():
            tag_id = int(tag_id)

        return tag_id

    @classmethod
    def sanatize_tag_id_matches(cls, matches):
        i_next = 0
        for i, match in enumerate(matches):
            # match was rejected
            if i < i_next:
                continue

            # next match would be
            i_next = i + 1

            # normalize tag id
            tag_id = cls.normalize_tag_id(match.group('tag'))

            # tag should be known
            assert tag_id in mt940.tags.TAG_BY_ID, 'Unknown tag %r ' \
                'in line: %r' % (tag_id, match.group(0))

            # special treatment for long tag content with possible
            # bad line wrap which produces tag_id like line beginnings
            # seen with :86: tag
            if tag_id == mt940.tags.Tags.TRANSACTION_DETAILS.value.id:
                # search subsequent tags for unknown tag ids
                # these lines likely belong to the previous tag
                for j in range(i_next, len(matches)):
                    next_tag_id = cls.normalize_tag_id(matches[j].group('tag'))
                    if next_tag_id in mt940.tags.TAG_BY_ID:
                        # this one is the next valid match
                        i_next = j
                        break
                    # else reject match

            # a valid match
            yield match

    def parse(self, data):
        '''Parses mt940 data, expects a string with data

        Args:
            data (str): The MT940 data

        Returns: :py:class:`list` of :py:class:`Transaction`
        '''
        # Remove extraneous whitespace and such
        data = '\n'.join(self.strip(data.split('\n')))

        # The pattern is a bit annoying to match by regex, even with a greedy
        # match it's difficult to get both the beginning and the end so we're
        # working around it in a safer way to get everything.
        tag_re = re.compile(
            r'^:(?P<full_tag>(?P<tag>[0-9]{2}|NS)(?P<sub_tag>[A-Z])?):',
            re.MULTILINE)
        matches = list(tag_re.finditer(data))

        # identify valid matches
        valid_matches = list(self.sanatize_tag_id_matches(matches))

        for i, match in enumerate(valid_matches):
            tag_id = self.normalize_tag_id(match.group('tag'))

            # get tag instance corresponding to tag id
            tag = mt940.tags.TAG_BY_ID.get(match.group('full_tag')) \
                or mt940.tags.TAG_BY_ID[tag_id]

            # Nice trick to get all the text that is part of this tag, python
            # regex matches have a `end()` and `start()` to indicate the start
            # and end index of the match.

            if valid_matches[i + 1:]:
                tag_data = \
                    data[match.end():valid_matches[i + 1].start()].strip()
            else:
                tag_data = data[match.end():].strip()

            tag_dict = tag.parse(self, tag_data)

            # Preprocess data before creating the object

            for processor in self.processors.get('pre_%s' % tag.slug):
                tag_dict = processor(self, tag, tag_dict)

            result = tag(self, tag_dict)

            # Postprocess the object

            for processor in self.processors.get('post_%s' % tag.slug):
                result = processor(self, tag, tag_dict, result)

            # Creating a new transaction for :20: and :61: tags allows the
            # tags from :20: to :61: to be captured as part of the transaction.

            if isinstance(tag, mt940.tags.Statement):
                # Transactions only get a Transaction Reference Code ID from a
                # :61: tag which is why a new transaction is created if the
                # 'id' has a value.

                if not self.transactions:
                    transaction = Transaction(self)
                    self.transactions.append(transaction)

                if transaction.data.get('id'):
                    transaction = Transaction(self, result)
                    self.transactions.append(transaction)
                else:
                    transaction.data.update(result)
            elif issubclass(tag.scope, Transaction) and self.transactions:
                # Combine multiple results together as one string, Rabobank has
                # multiple :86: tags for a single transaction

                for k, v in _compat.iteritems(result):
                    if k in transaction.data and hasattr(v, 'strip'):
                        transaction.data[k] += '\n%s' % v.strip()
                    else:
                        transaction.data[k] = v

            elif issubclass(tag.scope, Transactions):  # pragma: no branch
                self.data.update(result)

        return self.transactions

    def __getitem__(self, key):
        return self.transactions[key]

    def __len__(self):
        return len(self.transactions)

    def __repr__(self):
        return '<%s[%s]>' % (
            self.__class__.__name__,
            ']['.join('%s: %s' % (k.replace('_balance', ''), v)
                      for k, v in _compat.iteritems(self.data)
                      if k.endswith('balance')))

class Transaction(Model):
    def __init__(self, transactions, data=None):
        self.transactions = transactions
        self.data = {}
        self.update(data)

    def update(self, data):
        if data:
            self.data.update(data)

    def __repr__(self):
        return '<%s[%s] %s>' % (
            self.__class__.__name__,
            self.data.get('date'),
            self.data.get('amount'), )
<|endoftext|>


<commit_before># coding=UTF-8

# TODO 后台组: 解析指定XML文件
def parse_xml():
 try:
    import xml.etree.cElementTree as ET
 except ImportError:
    import xml.etree.ElementTree as ET
 import sys

 try:
    tree = ET.parse("TX2015.xml")  # 打开xml文档
    root = tree.getroot()  # 获得root节点
 except Exception, e:
    print "Error:cannot parse file:TX2015.xml."
    sys.exit(1)
 for node in root:
    print node.get('templateId'),node.get('name'),node.get('category'),node.get('emulation'),node.get('os'),node.get('image'),node.get('x'),node.get('y'),
    for config in node.findall('config'):
        print config.get('name'), config.get('type'), config.get('value'), config.get('unit'),config.get('index'),
        for config in config.findall('config'):
            print config.get('name'),config.get('type'),config.get('value'),config.get('unit'),config.get('index'),
            for config in config.findall('config'):
               print config.get('name'),config.get('type'),config.get('value'),config.get('unit'),
    for interface in node.findall('interface'):
        print interface.get('name'), interface.get('index'), interface.get('toNode'), interface.get('toIndex'),
        for config in interface.findall('config'):
           print config.get('name'), config.get('type'), config.get('value'), config.get('unit'),
    print

parse_xml()

<commit_msg>Update main.py<commit_after># coding=UTF-8

# TODO 后台组: 解析指定XML文件
try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET
import sys

try:
    tree = ET.parse("TX2015.xml")  # 打开xml文档
    root = tree.getroot()  # 获得root节点
except Exception, e:
    print "Error:cannot parse file:TX2015.xml."
    sys.exit(1)

for node in root:
    node_templateId = node.get('templateId')
    node_name=node.get('name')
    node_category=node.get('category')
    node_emulation=node.get('emulation')
    node_os=node.get('os')
    node_image=node.get('image')
    node_x=node.get('x')
    node_y=node.get('y')
    for config in node.findall('config'):
        node_config_name=config.get('name')
        node_config_type=config.get('type')
        node_config_value=config.get('value')
        node_config_unit=config.get('unit')
        node_config_index=config.get('index')
        for config in config.findall('config'):
            node_config_config_name = config.get('name')
            node_config_config_type = config.get('type')
            node_config_config_value = config.get('value')
            node_config_config_unit = config.get('unit')
            node_config_config_index = config.get('index')
            for config in config.findall('config'):
                node_config_config_config_name = config.get('name')
                node_config_config_config_type = config.get('type')
                node_config_config_config_value = config.get('value')
                node_config_config_config_unit = config.get('unit')
                node_config_config_config_index = config.get('index')
    for interface in node.findall('interface'):
        node_interface_name=interface.get('name')
        node_interface_name = interface.get('index')
        node_interface_name = interface.get('toNode')
        node_interface_name = interface.get('toIndex')
        for config in interface.findall('config'):
           node_interface_config_name = config.get('name')
           node_interface_config_type = config.get('type')
           node_interface_config_value = config.get('value')
           node_interface_config_unit = config.get('unit')
    print

<|endoftext|>


<commit_before>from functools import partial
from pathlib import Path
import time

from keras import backend as K
from keras.callbacks import CSVLogger
from keras.callbacks import ModelCheckpoint
from keras.preprocessing.image import img_to_array
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

from toolbox.data import load_image_pair
from toolbox.image import array_to_img
from toolbox.metrics import psnr
from toolbox.models import bicubic
from toolbox.paths import data_dir

class Experiment(object):
    def __init__(self, scale=3, load_set=None, build_model=None, save_dir='.'):
        self.scale = scale
        self.load_set = partial(load_set, scale=scale)
        self.build_model = partial(build_model, scale=scale)
        self.save_dir = Path(save_dir)
        self.save_dir.mkdir(parents=True, exist_ok=True)

        self.config_file = self.save_dir / 'config.yaml'
        self.model_file = self.save_dir / 'model.hdf5'

        self.train_dir = self.save_dir / 'train'
        self.train_dir.mkdir(exist_ok=True)
        self.history_file = self.train_dir / 'history.csv'
        self.weights_dir = self.train_dir / 'weights'
        self.weights_dir.mkdir(exist_ok=True)

        self.test_dir = self.save_dir / 'test'
        self.test_dir.mkdir(exist_ok=True)

    def weights_file(self, epoch=None):
        if epoch is None:
            return self.weights_dir / 'ep{epoch:04d}.hdf5'
        else:
            return self.weights_dir / f'ep{epoch:04d}.hdf5'

    @property
    def latest_epoch(self):
        try:
            return pd.read_csv(str(self.history_file))['epoch'].iloc[-1]
        except (FileNotFoundError, pd.io.common.EmptyDataError):
            pass
        return -1

    def _ensure_dimension(self, array, dim):
        while len(array.shape) < dim:
            array = array[np.newaxis, ...]
        return array

    def _ensure_channel(self, array, c):
        return array[..., c:c+1]

    def pre_process(self, array):
        array = self._ensure_dimension(array, 4)
        array = self._ensure_channel(array, 0)
        return array

    def post_process(self, array, auxiliary_array):
        array = np.concatenate([array, auxiliary_array[..., 1:]], axis=-1)
        array = np.clip(array, 0, 255)
        return array

    def inverse_post_process(self, array):
        array = self._ensure_dimension(array, 4)
        array = self._ensure_channel(array, 0)
        return array

    def compile(self, model, optimizer='adam', loss='mse', metrics=[psnr],
                **kwargs):
        """Compile model with default settings."""
        model.compile(optimizer=optimizer, loss=loss, metrics=metrics,
                      **kwargs)
        return model

    def train(self, train_set='91-image', val_set='Set5', epochs=1,
              resume=True):
        # Load and process data
        x_train, y_train = self.load_set(train_set)
        x_val, y_val = self.load_set(val_set)
        x_train, x_val = [self.pre_process(x)
                          for x in [x_train, x_val]]
        y_train, y_val = [self.inverse_post_process(y)
                          for y in [y_train, y_val]]

        # Compile model
        model = self.compile(self.build_model(x_train))
        model.summary()

        # Save model architecture
        # Currently in Keras 2 it's not possible to load a model with custom
        # layers. So we just save it without checking consistency.
        self.config_file.write_text(model.to_yaml())

        # Inherit weights
        if resume:
            latest_epoch = self.latest_epoch
            if latest_epoch > -1:
                weights_file = self.weights_file(epoch=latest_epoch)
                model.load_weights(str(weights_file))
            initial_epoch = latest_epoch + 1
        else:
            initial_epoch = 0

        # Set up callbacks
        callbacks = []
        callbacks += [ModelCheckpoint(str(self.model_file))]
        callbacks += [ModelCheckpoint(str(self.weights_file()),
                                      save_weights_only=True)]
        callbacks += [CSVLogger(str(self.history_file), append=resume)]

        # Train
        model.fit(x_train, y_train, epochs=epochs, callbacks=callbacks,
                  validation_data=(x_val, y_val), initial_epoch=initial_epoch)

        # Plot metrics history
        prefix = str(self.history_file).rsplit('.', maxsplit=1)[0]
        df = pd.read_csv(str(self.history_file))
        epoch = df['epoch']
        for metric in ['Loss', 'PSNR']:
            train = df[metric.lower()]
            val = df['val_' + metric.lower()]
            plt.figure()
            plt.plot(epoch, train, label='train')
            plt.plot(epoch, val, label='val')
            plt.legend(loc='best')
            plt.xlabel('Epoch')
            plt.ylabel(metric)
            plt.savefig('.'.join([prefix, metric.lower(), 'png']))
            plt.close()

    def test(self, test_set='Set5', metrics=[psnr]):
        print('Test on', test_set)
        image_dir = self.test_dir / test_set
        image_dir.mkdir(exist_ok=True)

        # Evaluate metrics on each image
        rows = []
        for image_path in (data_dir / test_set).glob('*'):
            rows += [self.test_on_image(str(image_path),
                                        str(image_dir / image_path.stem),
                                        metrics=metrics)]
        df = pd.DataFrame(rows)

        # Compute average metrics
        row = pd.Series()
        row['name'] = 'average'
        for col in df:
            if col != 'name':
                row[col] = df[col].mean()
        df = df.append(row, ignore_index=True)

        df.to_csv(str(self.test_dir / f'{test_set}/metrics.csv'))

    def test_on_image(self, path, prefix, suffix='png', metrics=[psnr]):
        # Load images
        lr_image, hr_image = load_image_pair(path, scale=self.scale)

        # Generate bicubic image
        x = img_to_array(lr_image)[np.newaxis, ...]
        bicubic_model = bicubic(x, scale=self.scale)
        y = bicubic_model.predict_on_batch(x)
        bicubic_array = np.clip(y[0], 0, 255)

        # Generate output image and measure run time
        start = time.perf_counter()
        x = self.pre_process(x)
        model = self.compile(self.build_model(x))
        if self.model_file.exists():
            model.load_weights(str(self.model_file))
        y_pred = model.predict_on_batch(x)
        output_array = self.post_process(y_pred[0], bicubic_array)
        output_image = array_to_img(output_array, mode='YCbCr')
        end = time.perf_counter()

        # Record metrics
        row = pd.Series()
        row['name'] = Path(path).stem
        row['time'] = end - start
        y_true = self.inverse_post_process(img_to_array(hr_image))
        for metric in metrics:
            row[metric.__name__] = K.eval(metric(y_true, y_pred))

        # Save images
        images_to_save = []
        images_to_save += [(hr_image, 'original')]
        images_to_save += [(output_image, 'output')]
        images_to_save += [(lr_image, 'input')]
        for img, label in images_to_save:
            img.convert(mode='RGB').save('.'.join([prefix, label, suffix]))

        return row
<commit_msg>Change the way run time is measured<commit_after>from functools import partial
from pathlib import Path
import time

from keras import backend as K
from keras.callbacks import CSVLogger
from keras.callbacks import ModelCheckpoint
from keras.preprocessing.image import img_to_array
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

from toolbox.data import load_image_pair
from toolbox.image import array_to_img
from toolbox.metrics import psnr
from toolbox.models import bicubic
from toolbox.paths import data_dir

class Experiment(object):
    def __init__(self, scale=3, load_set=None, build_model=None, save_dir='.'):
        self.scale = scale
        self.load_set = partial(load_set, scale=scale)
        self.build_model = partial(build_model, scale=scale)
        self.save_dir = Path(save_dir)
        self.save_dir.mkdir(parents=True, exist_ok=True)

        self.config_file = self.save_dir / 'config.yaml'
        self.model_file = self.save_dir / 'model.hdf5'

        self.train_dir = self.save_dir / 'train'
        self.train_dir.mkdir(exist_ok=True)
        self.history_file = self.train_dir / 'history.csv'
        self.weights_dir = self.train_dir / 'weights'
        self.weights_dir.mkdir(exist_ok=True)

        self.test_dir = self.save_dir / 'test'
        self.test_dir.mkdir(exist_ok=True)

    def weights_file(self, epoch=None):
        if epoch is None:
            return self.weights_dir / 'ep{epoch:04d}.hdf5'
        else:
            return self.weights_dir / f'ep{epoch:04d}.hdf5'

    @property
    def latest_epoch(self):
        try:
            return pd.read_csv(str(self.history_file))['epoch'].iloc[-1]
        except (FileNotFoundError, pd.io.common.EmptyDataError):
            pass
        return -1

    def _ensure_dimension(self, array, dim):
        while len(array.shape) < dim:
            array = array[np.newaxis, ...]
        return array

    def _ensure_channel(self, array, c):
        return array[..., c:c+1]

    def pre_process(self, array):
        array = self._ensure_dimension(array, 4)
        array = self._ensure_channel(array, 0)
        return array

    def post_process(self, array, auxiliary_array):
        array = np.concatenate([array, auxiliary_array[..., 1:]], axis=-1)
        array = np.clip(array, 0, 255)
        return array

    def inverse_post_process(self, array):
        array = self._ensure_dimension(array, 4)
        array = self._ensure_channel(array, 0)
        return array

    def compile(self, model, optimizer='adam', loss='mse', metrics=[psnr],
                **kwargs):
        """Compile model with default settings."""
        model.compile(optimizer=optimizer, loss=loss, metrics=metrics,
                      **kwargs)
        return model

    def train(self, train_set='91-image', val_set='Set5', epochs=1,
              resume=True):
        # Load and process data
        x_train, y_train = self.load_set(train_set)
        x_val, y_val = self.load_set(val_set)
        x_train, x_val = [self.pre_process(x)
                          for x in [x_train, x_val]]
        y_train, y_val = [self.inverse_post_process(y)
                          for y in [y_train, y_val]]

        # Compile model
        model = self.compile(self.build_model(x_train))
        model.summary()

        # Save model architecture
        # Currently in Keras 2 it's not possible to load a model with custom
        # layers. So we just save it without checking consistency.
        self.config_file.write_text(model.to_yaml())

        # Inherit weights
        if resume:
            latest_epoch = self.latest_epoch
            if latest_epoch > -1:
                weights_file = self.weights_file(epoch=latest_epoch)
                model.load_weights(str(weights_file))
            initial_epoch = latest_epoch + 1
        else:
            initial_epoch = 0

        # Set up callbacks
        callbacks = []
        callbacks += [ModelCheckpoint(str(self.model_file))]
        callbacks += [ModelCheckpoint(str(self.weights_file()),
                                      save_weights_only=True)]
        callbacks += [CSVLogger(str(self.history_file), append=resume)]

        # Train
        model.fit(x_train, y_train, epochs=epochs, callbacks=callbacks,
                  validation_data=(x_val, y_val), initial_epoch=initial_epoch)

        # Plot metrics history
        prefix = str(self.history_file).rsplit('.', maxsplit=1)[0]
        df = pd.read_csv(str(self.history_file))
        epoch = df['epoch']
        for metric in ['Loss', 'PSNR']:
            train = df[metric.lower()]
            val = df['val_' + metric.lower()]
            plt.figure()
            plt.plot(epoch, train, label='train')
            plt.plot(epoch, val, label='val')
            plt.legend(loc='best')
            plt.xlabel('Epoch')
            plt.ylabel(metric)
            plt.savefig('.'.join([prefix, metric.lower(), 'png']))
            plt.close()

    def test(self, test_set='Set5', metrics=[psnr]):
        print('Test on', test_set)
        image_dir = self.test_dir / test_set
        image_dir.mkdir(exist_ok=True)

        # Evaluate metrics on each image
        rows = []
        for image_path in (data_dir / test_set).glob('*'):
            rows += [self.test_on_image(str(image_path),
                                        str(image_dir / image_path.stem),
                                        metrics=metrics)]
        df = pd.DataFrame(rows)

        # Compute average metrics
        row = pd.Series()
        row['name'] = 'average'
        for col in df:
            if col != 'name':
                row[col] = df[col].mean()
        df = df.append(row, ignore_index=True)

        df.to_csv(str(self.test_dir / f'{test_set}/metrics.csv'))

    def test_on_image(self, path, prefix, suffix='png', metrics=[psnr]):
        # Load images
        lr_image, hr_image = load_image_pair(path, scale=self.scale)

        # Generate bicubic image
        x = img_to_array(lr_image)[np.newaxis, ...]
        bicubic_model = bicubic(x, scale=self.scale)
        y = bicubic_model.predict_on_batch(x)
        bicubic_array = np.clip(y[0], 0, 255)

        # Generate output image and measure run time
        x = self.pre_process(x)
        model = self.compile(self.build_model(x))
        if self.model_file.exists():
            model.load_weights(str(self.model_file))
        start = time.perf_counter()
        y_pred = model.predict_on_batch(x)
        end = time.perf_counter()
        output_array = self.post_process(y_pred[0], bicubic_array)
        output_image = array_to_img(output_array, mode='YCbCr')

        # Record metrics
        row = pd.Series()
        row['name'] = Path(path).stem
        row['time'] = end - start
        y_true = self.inverse_post_process(img_to_array(hr_image))
        for metric in metrics:
            row[metric.__name__] = K.eval(metric(y_true, y_pred))

        # Save images
        images_to_save = []
        images_to_save += [(hr_image, 'original')]
        images_to_save += [(output_image, 'output')]
        images_to_save += [(lr_image, 'input')]
        for img, label in images_to_save:
            img.convert(mode='RGB').save('.'.join([prefix, label, suffix]))

        return row
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2015, Foxugly. All rights reserved.
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
from __future__ import absolute_import, unicode_literals
from django.utils.translation import ugettext_lazy as _
from mta import database

"""
Django settings for MyTaxAccountant project.

For more information on this file, see
https://docs.djangoproject.com/en/1.7/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'changeme'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
ALLOWED_HOSTS = ['*']
DATABASES = database.db

# Application definition

INSTALLED_APPS = (
    'fileupload.apps.FileuploadConfig',
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django_extensions',
    'debug_toolbar',
    'djcelery',
    'django_celery_results',
    'hijack',
    'compat',
    'hijack_admin',
    'bootstrap3',
    'companies',
    'years',
    'trimesters',
    'categories',
    'documents',
    'utils',
    'users',
    'error',
    'webshell',
)

MIDDLEWARE = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    #'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'debug_toolbar.middleware.DebugToolbarMiddleware',
)

ROOT_URLCONF = 'mta.urls'
WSGI_APPLICATION = 'mta.wsgi.application'

UPLOAD_LOG = '/tmp/upload_log'

LANGUAGE_CODE = 'fr'
TIME_ZONE = 'Europe/Brussels'
USE_I18N = True
USE_L10N = True
USE_TZ = True
AUTH_PROFILE_MODULE = 'users.UserProfile'
LOGIN_REDIRECT_URL = 'login'
LOGIN_URL = '/user/login/'

LANGUAGES = (
  ('fr', _('Francais')),
  ('nl', _('Nederlands')),
  ('en', _('English')),
)

DEFAULT_LANGUAGE = 1

MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
MEDIA_URL = '/media/'
UPLOAD_DIR = 'upload'
STOCK_DIR = 'folders'
# ACTIVE TO PROD / COMMENT TO TEST
STATIC_ROOT = os.path.join(BASE_DIR, 'static')
STATIC_URL = '/static/'
TMP_ROOT = MEDIA_ROOT + '/tmp/'
TMP_URL = MEDIA_URL + 'tmp/'
ADMIN_MEDIA_PREFIX = '/static/admin/'
# COMMENT TO PROD / ACTIVE TO TEST
# STATICFILES_DIRS = [os.path.join(BASE_DIR, 'static'), ]

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            os.path.join(BASE_DIR, 'templates'),
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.template.context_processors.i18n',
                'django.template.context_processors.media',
                'django.template.context_processors.static',
                'django.template.context_processors.tz',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'simple': {
            'format': '%(levelname)s %(message)s'
        },
    },
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
        },
    },
    'core.handlers': {
        'level': 'DEBUG',
        'handlers': ['console']
    },
    'root': {
        'level': 'DEBUG',
        'handlers': ['console']
    },
    'loggers': {
        'django': {
            'level': 'INFO',
            'handlers': ['console']
        },
    },
}

HIJACK_LOGIN_REDIRECT_URL = '/'
HIJACK_LOGOUT_REDIRECT_URL = '/admin/auth/user/'
HIJACK_DISPLAY_WARNINGS = True
HIJACK_USE_BOOTSTRAP = True
HIJACK_ALLOW_GET_REQUESTS = True

if DEBUG:
    def show_toolbar(request):
        return True

    DEBUG_TOOLBAR_CONFIG = {
        'SHOW_TOOLBAR_CALLBACK': 'mta.settings.show_toolbar',
    }

BROKER_URL = 'redis://localhost:6379'
# CELERY_RESULT_BACKEND = 'redis://localhost:6379'
CELERY_RESULT_BACKEND = 'django-db'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = 'Europe/Brussels'
<commit_msg>logger<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2015, Foxugly. All rights reserved.
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
from __future__ import absolute_import, unicode_literals
from django.utils.translation import ugettext_lazy as _
from mta import database

"""
Django settings for MyTaxAccountant project.

For more information on this file, see
https://docs.djangoproject.com/en/1.7/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
import logging

BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'changeme'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
LOGGER = logging.DEBUG
ALLOWED_HOSTS = ['*']
DATABASES = database.db

# Application definition

INSTALLED_APPS = (
    'fileupload.apps.FileuploadConfig',
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django_extensions',
    'debug_toolbar',
    'djcelery',
    'django_celery_results',
    'hijack',
    'compat',
    'hijack_admin',
    'bootstrap3',
    'companies',
    'years',
    'trimesters',
    'categories',
    'documents',
    'utils',
    'users',
    'error',
    'webshell',
)

MIDDLEWARE = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    #'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'debug_toolbar.middleware.DebugToolbarMiddleware',
)

ROOT_URLCONF = 'mta.urls'
WSGI_APPLICATION = 'mta.wsgi.application'

UPLOAD_LOG = '/tmp/upload_log'

LANGUAGE_CODE = 'fr'
TIME_ZONE = 'Europe/Brussels'
USE_I18N = True
USE_L10N = True
USE_TZ = True
AUTH_PROFILE_MODULE = 'users.UserProfile'
LOGIN_REDIRECT_URL = 'login'
LOGIN_URL = '/user/login/'

LANGUAGES = (
  ('fr', _('Francais')),
  ('nl', _('Nederlands')),
  ('en', _('English')),
)

DEFAULT_LANGUAGE = 1

MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
MEDIA_URL = '/media/'
UPLOAD_DIR = 'upload'
STOCK_DIR = 'folders'
# ACTIVE TO PROD / COMMENT TO TEST
STATIC_ROOT = os.path.join(BASE_DIR, 'static')
STATIC_URL = '/static/'
TMP_ROOT = MEDIA_ROOT + '/tmp/'
TMP_URL = MEDIA_URL + 'tmp/'
ADMIN_MEDIA_PREFIX = '/static/admin/'
# COMMENT TO PROD / ACTIVE TO TEST
# STATICFILES_DIRS = [os.path.join(BASE_DIR, 'static'), ]

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            os.path.join(BASE_DIR, 'templates'),
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.template.context_processors.i18n',
                'django.template.context_processors.media',
                'django.template.context_processors.static',
                'django.template.context_processors.tz',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'simple': {
            'format': '%(levelname)s %(message)s'
        },
    },
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
        },
    },
    'core.handlers': {
        'level': 'DEBUG',
        'handlers': ['console']
    },
    'root': {
        'level': 'DEBUG',
        'handlers': ['console']
    },
    'loggers': {
        'django': {
            'level': 'INFO',
            'handlers': ['console']
        },
    },
}

HIJACK_LOGIN_REDIRECT_URL = '/'
HIJACK_LOGOUT_REDIRECT_URL = '/admin/auth/user/'
HIJACK_DISPLAY_WARNINGS = True
HIJACK_USE_BOOTSTRAP = True
HIJACK_ALLOW_GET_REQUESTS = True

if DEBUG:
    def show_toolbar(request):
        return True

    DEBUG_TOOLBAR_CONFIG = {
        'SHOW_TOOLBAR_CALLBACK': 'mta.settings.show_toolbar',
    }

BROKER_URL = 'redis://localhost:6379'
# CELERY_RESULT_BACKEND = 'redis://localhost:6379'
CELERY_RESULT_BACKEND = 'django-db'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = 'Europe/Brussels'
<|endoftext|>