<commit_before>from flask import Flask, render_template, Blueprint, redirect, url_for, flash, request
from datetime import timedelta
from PhoenixNow.decorators import login_notrequired, admin_required, check_verified, check_notverified
from PhoenixNow.model import db, User, Checkin
from PhoenixNow.user import get_weekly_checkins, admin_weekly_checkins
from flask_login import login_required, login_user, logout_user
from PhoenixNow.forms import UserForm, CalendarForm, ScheduleForm
import datetime
import requests
import json
import os

admin = Blueprint('admin', __name__, template_folder='templates', static_folder='static')

@admin.route('/grade/<int:grade>', methods=['GET', 'POST'])
@admin.route('/', methods=['GET', 'POST'])
@login_required
@admin_required
def home(grade=None):
    form = CalendarForm()
    if request.method == 'POST':
        try:
            stringdate = form.date.data
            searchdate = datetime.datetime.strptime(stringdate, '%Y-%m-%d').date()
        except:
            return "Improper Date Format" + form.date.data

        if request.form['submit'] == "Next Week":
            searchdate = searchdate + timedelta(days=7)
            form.date.data = searchdate

        if request.form['submit'] == "Previous Week":
            searchdate = searchdate + timedelta(days=-7)
            form.date.data = searchdate

    if request.method == 'GET':
        searchdate = datetime.date.today()

    if grade is None:
      users = User.query.all()
    else:
      users = User.query.filter_by(grade=grade).all()

    users.sort(key=lambda user: (user.grade, user.lastname)) # sort by grade and name
    weekly_checkins = admin_weekly_checkins(searchdate)
    for checkin in weekly_checkins:
      day = checkin.checkin_timestamp.strftime("%A")
      if day == 'Monday':
        checkin.user.monday = "present"
      elif day == 'Tuesday':
        checkin.user.tuesday = "present"
      elif day == 'Wednesday':
        checkin.user.wednesday = "present"
      elif day == 'Thursday':
        checkin.user.thursday = 'present'
      elif day == 'Friday':
        checkin.user.friday = 'present'
    return render_template('admin.html', searchdate=searchdate, grade=grade, form=form, users=users, checkins=weekly_checkins)

@admin.route('/user/<int:user_id>', methods=['GET', 'POST'])
@login_required
@admin_required
def user(user_id):

  form = UserForm()
  schedule_form = ScheduleForm()

  if request.method == 'POST':
    user = User.query.filter_by(id=user_id).first_or_404()
    if form.firstname.data:
      user.firstname = form.firstname.data
    if form.lastname.data:
      user.lastname = form.lastname.data
    if form.grade.data:
      user.grade = form.grade.data
    db.session.commit()
    return redirect(url_for('admin.user', user_id=user_id))

  elif request.method == 'GET':
    user = User.query.filter_by(id=user_id).first_or_404()
    return render_template('user.html', schedule_form=schedule_form, user=user, form=form)

@admin.route('/user/<int:user_id>/schedule', methods=['POST'])
@login_required
@admin_required
def schedule(user_id):
    form = ScheduleForm()

    user = User.query.filter_by(id=user_id).first_or_404()

    if form.validate_on_submit():
        user.schedule = ""
        user.schedule_monday = False
        user.schedule_tuesday = False
        user.schedule_wednesday = False
        user.schedule_thursday = False
        user.schedule_friday = False
        if form.monday.data:
            user.schedule = "M"
            user.schedule_monday = True
        if form.tuesday.data:
            user.schedule = "%s:T" % (user.schedule)
            user.schedule_tuesday = True
        if form.wednesday.data:
            user.schedule = "%s:W" % (user.schedule)
            user.schedule_wednesday = True
        if form.thursday.data:
            user.schedule = "%s:R" % (user.schedule)
            user.schedule_thursday = True
        if form.friday.data:
            user.schedule = "%s:F" % (user.schedule)
            user.schedule_friday = True
        if user.schedule == "M:T:W:R:F":
            user.schedule_verified = True
        else:
            user.schedule_verified = False
        db.session.commit()

    return redirect(url_for('admin.user', user_id=user_id))<commit_msg>Fix admin pages always show all checkins<commit_after>from flask import Flask, render_template, Blueprint, redirect, url_for, flash, request
from datetime import timedelta
from PhoenixNow.decorators import login_notrequired, admin_required, check_verified, check_notverified
from PhoenixNow.model import db, User, Checkin
from PhoenixNow.user import get_weekly_checkins, admin_weekly_checkins
from flask_login import login_required, login_user, logout_user
from PhoenixNow.forms import UserForm, CalendarForm, ScheduleForm
import datetime
import requests
import json
import os

admin = Blueprint('admin', __name__, template_folder='templates', static_folder='static')

@admin.route('/grade/<int:grade>', methods=['GET', 'POST'])
@admin.route('/', methods=['GET', 'POST'])
@login_required
@admin_required
def home(grade=None):
    form = CalendarForm()
    if request.method == 'POST':
        try:
            stringdate = form.date.data
            searchdate = datetime.datetime.strptime(stringdate, '%Y-%m-%d').date()
        except:
            return "Improper Date Format" + form.date.data

        if request.form['submit'] == "Next Week":
            searchdate = searchdate + timedelta(days=7)
            form.date.data = searchdate

        if request.form['submit'] == "Previous Week":
            searchdate = searchdate + timedelta(days=-7)
            form.date.data = searchdate

    if request.method == 'GET':
        searchdate = datetime.date.today()

    if grade is None:
      users = User.query.all()
    else:
      users = User.query.filter_by(grade=grade).all()

    users.sort(key=lambda user: (user.grade, user.lastname)) # sort by grade and name
    weekly_checkins = admin_weekly_checkins(searchdate, grade)
    for checkin in weekly_checkins:
      day = checkin.checkin_timestamp.strftime("%A")
      if day == 'Monday':
        checkin.user.monday = "present"
      elif day == 'Tuesday':
        checkin.user.tuesday = "present"
      elif day == 'Wednesday':
        checkin.user.wednesday = "present"
      elif day == 'Thursday':
        checkin.user.thursday = 'present'
      elif day == 'Friday':
        checkin.user.friday = 'present'
    return render_template('admin.html', searchdate=searchdate, grade=grade, form=form, users=users, checkins=weekly_checkins)

@admin.route('/user/<int:user_id>', methods=['GET', 'POST'])
@login_required
@admin_required
def user(user_id):

  form = UserForm()
  schedule_form = ScheduleForm()

  if request.method == 'POST':
    user = User.query.filter_by(id=user_id).first_or_404()
    if form.firstname.data:
      user.firstname = form.firstname.data
    if form.lastname.data:
      user.lastname = form.lastname.data
    if form.grade.data:
      user.grade = form.grade.data
    db.session.commit()
    return redirect(url_for('admin.user', user_id=user_id))

  elif request.method == 'GET':
    user = User.query.filter_by(id=user_id).first_or_404()
    return render_template('user.html', schedule_form=schedule_form, user=user, form=form)

@admin.route('/user/<int:user_id>/schedule', methods=['POST'])
@login_required
@admin_required
def schedule(user_id):
    form = ScheduleForm()

    user = User.query.filter_by(id=user_id).first_or_404()

    if form.validate_on_submit():
        user.schedule = ""
        user.schedule_monday = False
        user.schedule_tuesday = False
        user.schedule_wednesday = False
        user.schedule_thursday = False
        user.schedule_friday = False
        if form.monday.data:
            user.schedule = "M"
            user.schedule_monday = True
        if form.tuesday.data:
            user.schedule = "%s:T" % (user.schedule)
            user.schedule_tuesday = True
        if form.wednesday.data:
            user.schedule = "%s:W" % (user.schedule)
            user.schedule_wednesday = True
        if form.thursday.data:
            user.schedule = "%s:R" % (user.schedule)
            user.schedule_thursday = True
        if form.friday.data:
            user.schedule = "%s:F" % (user.schedule)
            user.schedule_friday = True
        if user.schedule == "M:T:W:R:F":
            user.schedule_verified = True
        else:
            user.schedule_verified = False
        db.session.commit()

    return redirect(url_for('admin.user', user_id=user_id))<|endoftext|>


<commit_before>import sys
import time
from cassandra.cluster import Cluster
from cassandra.query import SimpleStatement
from cassandra import ConsistencyLevel

# Get the arguments
cass_keyspace = sys.argv[1]  # Cassandra Keyspace containing the Table to tickle
cass_table = sys.argv[2]  # Cassandra Table to tickle
cass_ip = sys.argv[3]  # Cassandra Port
cass_port = sys.argv[4]  # CQL Port
cass_throttle = sys.argv[5]  # microseconds

# print cass_keyspace  # debug
# print cass_table  # debug
# print cass_ip  # debug
# print cass_port  # debug
# print cass_throttle  # debug

# Set the connections to the cluster
cluster = Cluster(
    [cass_ip],
    port=cass_port, compression=False)

# Connect to the KS
session = cluster.connect(cass_keyspace)

# Get the PK column
# TODO get the PK column

# read every key of the table
query = 'SELECT id FROM ' + cass_table
statement = SimpleStatement(query, fetch_size=1000)
print 'Starting to repair table ' + cass_table
row_count = 0
for user_row in session.execute(statement):
    # print user_row.id  # debug
    row_count += 1
    repair_query = 'SELECT COUNT(1) FROM ' + cass_table + ' WHERE id = ' + str(user_row.id)
    repair_statement = SimpleStatement(repair_query, consistency_level=ConsistencyLevel.ALL)
    session.execute(repair_statement)
    time.sleep(float(cass_throttle) / 1000000)  # delay in microseconds between reading each row
    if (row_count % 1000) == 0:
        print str(row_count) + ' rows processed'
print 'Repair of table ' + cass_table + ' complete'
print str(row_count) + ' rows read and repaired'
<commit_msg>Put back code that find the Primary key of a table.<commit_after>import sys
import time
from cassandra.cluster import Cluster
from cassandra.query import SimpleStatement
from cassandra import ConsistencyLevel

# Get the arguments
cass_keyspace = sys.argv[1]  # Cassandra Keyspace containing the Table to tickle
cass_table = sys.argv[2]  # Cassandra Table to tickle
cass_ip = sys.argv[3]  # Cassandra Port
cass_port = sys.argv[4]  # CQL Port
cass_throttle = sys.argv[5]  # microseconds

# print cass_keyspace  # debug
# print cass_table  # debug
# print cass_ip  # debug
# print cass_port  # debug
# print cass_throttle  # debug

# Set the connections to the cluster
cluster = Cluster(
    [cass_ip],
    port=cass_port, compression=False)

# Connect to the KS
session = cluster.connect(cass_keyspace)

# get the primary key of the table
# This is dirty. May fail if the table has a composite PK
for yo in cluster.metadata.keyspaces[cass_keyspace].tables[cass_table].primary_key:
    primary_key = yo.name

# print primary_key # debug

# read every key of the table
query = 'SELECT id FROM ' + cass_table
statement = SimpleStatement(query, fetch_size=1000)
print 'Starting to repair table ' + cass_table
row_count = 0
for user_row in session.execute(statement):
    # print user_row.id  # debug
    row_count += 1
    repair_query = 'SELECT COUNT(1) FROM ' + cass_table + ' WHERE ' + primary_key + ' = ' + str(user_row.id)
    repair_statement = SimpleStatement(repair_query, consistency_level=ConsistencyLevel.ALL)
    session.execute(repair_statement)
    time.sleep(float(cass_throttle) / 1000000)  # delay in microseconds between reading each row
    if (row_count % 1000) == 0:
        print str(row_count) + ' rows processed'
print 'Repair of table ' + cass_table + ' complete'
print str(row_count) + ' rows read and repaired'
<|endoftext|>


<commit_before>#!/usr/bin/env python2
'''
tmuxssh

Usage:
  tmuxssh [--template=<template>] <host>...
  tmuxssh --version
  tmuxssh -h | --help

'''

import subprocess
import sys
import os
import docopt

class TmuxSession(object):

    def __init__(self, session_name):
        self._session_name = session_name
        subprocess.call(['tmux', 'new-session', '-ds', self._session_name])

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.kill_session()

    def split_window(self, command):
        if isinstance(command, str):
            command = [command]
        subprocess.call(['tmux', 'split-window', '-v', '-t', self._session_name] + command)

    def select_layout(self, layout):
        subprocess.call(['tmux', 'select-layout', '-t', self._session_name, layout])

    def attach(self):
        subprocess.call(['tmux', 'attach', '-t', self._session_name])

    def set_window_option(self, option, value):
        subprocess.call(['tmux', 'set-window-option', '-t', self._session_name, option, value])

    def kill_pane(self, n):
        subprocess.call(['tmux', 'kill-pane', '-t', str(n)])

    def kill_session(self):
        subprocess.call(['tmux', 'kill-session', '-t', self._session_name])

def tmux_commands(commands):
    with TmuxSession('tmux-{}'.format(os.getpid())) as tmux:
        for command in commands:
            tmux.split_window(command)
            tmux.select_layout('tiled')
        tmux.kill_pane(0)
        tmux.select_layout('tiled')
        tmux.set_window_option('synchronize-panes', 'on')
        tmux.attach()

def ssh_command(host, template=None):
    if template is None:
        return 'ssh {}'.format(host)
    else:
        return template.format(host)

def main():
    args = docopt.docopt(__doc__, version='1.0.0')
    hosts = (h for host in args.get('<host>') for h in host.split())
    template = args.get('--template')
    commands = (ssh_command(host, template) for host in hosts)
    tmux_commands(commands)

if __name__ == '__main__':
    main()
<commit_msg>remove unused import<commit_after>#!/usr/bin/env python2
'''
tmuxssh

Usage:
  tmuxssh [--template=<template>] <host>...
  tmuxssh --version
  tmuxssh -h | --help
'''

import subprocess
import os
import docopt

class TmuxSession(object):

    def __init__(self, session_name):
        self._session_name = session_name
        subprocess.call(['tmux', 'new-session', '-ds', self._session_name])

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.kill_session()

    def split_window(self, command):
        if isinstance(command, str):
            command = [command]
        subprocess.call(['tmux', 'split-window', '-v', '-t', self._session_name] + command)

    def select_layout(self, layout):
        subprocess.call(['tmux', 'select-layout', '-t', self._session_name, layout])

    def attach(self):
        subprocess.call(['tmux', 'attach', '-t', self._session_name])

    def set_window_option(self, option, value):
        subprocess.call(['tmux', 'set-window-option', '-t', self._session_name, option, value])

    def kill_pane(self, n):
        subprocess.call(['tmux', 'kill-pane', '-t', str(n)])

    def kill_session(self):
        subprocess.call(['tmux', 'kill-session', '-t', self._session_name])

def tmux_commands(commands):
    with TmuxSession('tmux-{}'.format(os.getpid())) as tmux:
        for command in commands:
            tmux.split_window(command)
            tmux.select_layout('tiled')
        tmux.kill_pane(0)
        tmux.select_layout('tiled')
        tmux.set_window_option('synchronize-panes', 'on')
        tmux.attach()

def ssh_command(host, template=None):
    if template is None:
        return 'ssh {}'.format(host)
    else:
        return template.format(host)

def main():
    args = docopt.docopt(__doc__, version='1.0.0')
    hosts = (h for host in args.get('<host>') for h in host.split())
    template = args.get('--template')
    commands = (ssh_command(host, template) for host in hosts)
    tmux_commands(commands)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import sys
import time
import urllib
import hashlib
import bencode

class TrackerTalkError(Exception):
    def __init__(self, error):
        self.error = error

    def __str__(self):
        return self.error

class Torrent(object):
    '''
    Accepts a single .torrent file and parses it for metadata
    '''

    def __init__(self, filename):
        self.filename = filename
        self.parse_metainfo()

    def parse_metainfo(self):
        torrent_dict = bencode.bdecode(open(self.filename).read())

        self.announce_url = torrent_dict['announce']
        self.info = torrent_dict['info']
        self.info_hash = hashlib.sha1(bencode.bencode(self.info)).digest()
        self.piece_length = self.info['piece length']
        self.num_files = 1 if 'file' in self.info else len(self.info['files'])
        ###actually do
        self.length = 26356589

class Tracker(object):
    '''
    A torrent tracker object, connects over HTTP[s], responsible for routing
    requests
    '''

    def __init__(self, **kwargs):
        self.client_id = str(time.time()) + '901asdf0293fasljz23raasd'
        if len(self.client_id) > 20:
            self.client_id = self.client_id[0:20]
        self.port = 6881

    def _build_url(self, torrent, completed=False):
        event = 'completed' if completed else 'started'

        announce_query = {
            'info_hash': torrent.info_hash,
            'peer_id': self.client_id,
            'port': self.port,
            'uploaded': 0,
            'downloaded': 0,
            'left': torrent.length, #remove
            'event': event,
        }
        tracker_url = torrent.announce_url + '?' + urllib.urlencode(announce_query)
        return tracker_url

    def request(self, torrent):
        url =  self._build_url(torrent)
        response = urllib.urlopen(url).read()
        if not response:
            raise TrackerTalkError('No response from tracker for url %s' % url)
        return response

if __name__ == '__main__':
    track = Tracker()
    torrent = Torrent(sys.argv[1])
    track.request(torrent)
<commit_msg>num files bug, parsing length, decoding tracker<commit_after>import sys
import time
import urllib
import hashlib
import bencode

class TrackerTalkError(Exception):
    def __init__(self, error):
        self.error = error

    def __str__(self):
        return self.error

class Torrent(object):
    '''
    Accepts a single .torrent file and parses it for metadata
    '''

    def __init__(self, filename):
        self.filename = filename
        self.parse_metainfo()

    def parse_metainfo(self):
        torrent_dict = bencode.bdecode(open(self.filename).read())

        self.announce_url = torrent_dict['announce']
        self.info = torrent_dict['info']
        self.info_hash = hashlib.sha1(bencode.bencode(self.info)).digest()
        self.piece_length = self.info['piece length']
        self.num_files = len(self.info['files']) if 'files' in self.info else 1

        if self.num_files == 1:
            self.length = self.info['length']
        else:
            self.length = sum(int(f['length']) for f in self.info['files'])

class Tracker(object):
    '''
    A torrent tracker object, connects over HTTP[s], responsible for routing
    requests. 1 to 1 relationship between torrent and tracker.

    Requires an instance of the Torrent class object to be passed to it.
    '''

    def __init__(self, torrent, **kwargs):
        self.client_id = str(time.time()) + '901asdf0293fasljz23raasd'
        if len(self.client_id) > 20:
            self.client_id = self.client_id[0:20]
        self.port = 6881
        self.torrent = torrent
        self.decode_and_parse()

    def _build_url(self, completed=False):
        event = 'completed' if completed else 'started'

        announce_query = {
            'info_hash': self.torrent.info_hash,
            'peer_id': self.client_id,
            'port': self.port,
            'uploaded': 0,
            'downloaded': 0,
            'left': self.torrent.length,
            'event': event,
        }
        tracker_url = self.torrent.announce_url + '?' + urllib.urlencode(announce_query)
        return tracker_url

    def request(self, **kwargs):
        url = self._build_url(**kwargs)
        response = urllib.urlopen(url).read()
        if not response:
            raise TrackerTalkError('No response from tracker for url %s' % url)
        return response

    def decode_and_parse(self):
        response = self.request()
        tracker_dict = bencode.bdecode(response)

        if 'failure reason' in tracker_dict:
            raise TrackerTalkError('''failure reason key in tracker response\
                                    %s:''' % tracker_dict['failure reason'])

if __name__ == '__main__':
    torrent = Torrent(sys.argv[1])
    track = Tracker(torrent=torrent)
<|endoftext|>


<commit_before># Simple torrent class. Creates attributes from a dictionary of kwargs
# Preprocessing of the .torrent performed with bencode

class torrent():

    def __init__(self, *args, **kwargs):
        while kwargs:
            item = kwargs.popitem()
            key = str(item[0]).replace(' ', '_')
            setattr(self, key, item[1])
<commit_msg>Got the URL encoding for GET request working.<commit_after># Preprocessing of the .torrent performed with bdecode

import requests
import socket
import tparser
import hashlib
import urllib
# import pudb

class torrent():

    def __init__(self, tdict):
        self.tdict = tdict

    def get_request(self):
        assert self.tdict['info']
        hashed_info = hashlib.sha1(tparser.bencode(self.tdict['info']))
        # Get a hashy string
        url_string = urllib.urlencode({'myhashything': hashed_info.digest()})
        print url_string.strip('myhashything=')

def main():
    # pudb.set_trace()
    tor_dict = tparser.bdecode('torrent.torrent')
    mytorrent = torrent(tor_dict)
    mytorrent.get_request()

    # http://thomasballinger.com:6969/announce?
    # info_hash=%2b%15%ca%2b%fdH%cd%d7m9%ecU%a3%ab%1b%8aW%18%0a%09
    # peer_id=-TR2820-jlt0de3likus
    # port=55251
    # uploaded=0
    # downloaded=0
    # left=1277987
    # numwant=0
    # key=556b1696
    # compact=1&
    # upportcrypto=1
    # event=stopped

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""Attribute-value matrix."""

from matcher import Matcher
from StringIO import StringIO

class AVM(object):
    TYPE, REQUIRED, DEFAULT, VALUE = xrange(4)

    def __init__(self):
        self.__data = {}  # {key: [type, required, default_value, value]}

    def add_attribute(self, key, datatype, required=False, default_value=None):
        """Adds a new attribute to the "matrix"."""
        if not isinstance(required, bool):
            raise ValueError("required must be a bool, not " + type(required))
        if not isinstance(datatype, Matcher):
            raise ValueError("datatype must be a Matcher, not " +
                             type(datatype))
        self.__data[key] = [datatype, required, default_value, default_value]
        # TODO: do we need a default_value?

    def satisfied(self):
        """Returns @c True, if all required arguments are filled in."""
        for value in self.__data.values():
            if value[AVM.REQUIRED] and value[AVM.VALUE] is None:
                return False
        else:
            return True

    def get_attribute(self, key):
        """Returns the whole attribute data tuple for @p key."""
        return self.__data[key]

    def get_field(self, key, field):
        """
        Returns the specified field from the data tuple for @p key. The valid
        values for @p field are @c TYPE, @c REQUIRED, @c DEFAULT and @c VALUE.
        """
        return self.__data[key][field]

    def __getitem__(self, key):
        """Gets the current value of an attribute."""
        return self.__data[key][AVM.VALUE]

    def __setitem__(self, key, value):
        """Sets the current value of an attribute."""
        self.__data[key][AVM.VALUE] = value

    def __iter__(self):
        """Iterates through the attribute keys."""
        return self.__data.__iter__()

    def __unicode__(self):
        return u'{' + u', '.join(u'{0}: {1}'.format(key, self[key]) for key in self) + u'}'

<commit_msg>AVM class can now return a dict of attribute key - value pairs. 	modified:   ../pymachine/src/avm.py<commit_after>"""Attribute-value matrix."""

from matcher import Matcher

class AVM(object):
    TYPE, REQUIRED, DEFAULT, VALUE = xrange(4)

    def __init__(self):
        self.__data = {}  # {key: [type, required, default_value, value]}

    def add_attribute(self, key, datatype, required=False, default_value=None):
        """Adds a new attribute to the "matrix"."""
        if not isinstance(required, bool):
            raise ValueError("required must be a bool, not " + type(required))
        if not isinstance(datatype, Matcher):
            raise ValueError("datatype must be a Matcher, not " +
                             type(datatype))
        self.__data[key] = [datatype, required, default_value, default_value]
        # TODO: do we need a default_value?

    def satisfied(self):
        """Returns @c True, if all required arguments are filled in."""
        for value in self.__data.values():
            if value[AVM.REQUIRED] and value[AVM.VALUE] is None:
                return False
        else:
            return True

    def get_attribute(self, key):
        """Returns the whole attribute data tuple for @p key."""
        return self.__data[key]

    def get_field(self, key, field):
        """
        Returns the specified field from the data tuple for @p key. The valid
        values for @p field are @c TYPE, @c REQUIRED, @c DEFAULT and @c VALUE.
        """
        return self.__data[key][field]

    def get_dict(self):
        """Returns the attribute-value dictionary in a Python dict."""
        return dict((k, v[AVM.VALUE]) for k, v in self.__data.iteritems())

    def __getitem__(self, key):
        """Gets the current value of an attribute."""
        return self.__data[key][AVM.VALUE]

    def __setitem__(self, key, value):
        """Sets the current value of an attribute."""
        self.__data[key][AVM.VALUE] = value

    def __iter__(self):
        """Iterates through the attribute keys."""
        return self.__data.__iter__()

    def __unicode__(self):
        return u'{' + u', '.join(u'{0}: {1}'.format(key, self[key]) for key in self) + u'}'

<|endoftext|>


<commit_before># -*- coding: iso8859-1 -*-
#
# Copyright (C) 2003, 2004 Edgewall Software
# Copyright (C) 2003, 2004 Jonas Borgstrm <jonas@edgewall.com>
#
# Trac is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Trac is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# Author: Jonas Borgstrm <jonas@edgewall.com>

import os
import os.path
import sqlite

class Database(sqlite.Connection):
    def __init__(self, db_name):
        directory = os.path.dirname(db_name)
        if not os.access(db_name, os.R_OK + os.W_OK) or \
               not os.access(directory, os.R_OK + os.W_OK):
            tmp = db_name
            db_name = None
            raise EnvironmentError, \
                  'The web server user requires read _and_ write permission\n' \
                  'to the database %s and the directory this file is located in.' % tmp
        sqlite.Connection.__init__(self, db_name, timeout=10000)

    def load_config(self):
        """
        load configuration from the config table.

        The configuration is returned as a section-dictionary containing
        name-value dictionaries.
        """
        cursor = self.cursor()
        cursor.execute('SELECT section, name, value FROM config')
        config = {}
        rows = cursor.fetchall()
        for row in rows:
            if not config.has_key(row[0]):
                config[row[0]] = {}
            config[row[0]][row[1]] = row[2]
        return config
<commit_msg>Give a better more correct error message when the database is actually missing and not only a permission problem.<commit_after># -*- coding: iso8859-1 -*-
#
# Copyright (C) 2003, 2004 Edgewall Software
# Copyright (C) 2003, 2004 Jonas Borgstrm <jonas@edgewall.com>
#
# Trac is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Trac is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# Author: Jonas Borgstrm <jonas@edgewall.com>

import os
import os.path
import sqlite

class Database(sqlite.Connection):
    def __init__(self, db_name):
        if not os.access(db_name, os.F_OK):
            raise EnvironmentError, 'Database "%s" not found.' % db_name

        directory = os.path.dirname(db_name)
        if not os.access(db_name, os.R_OK + os.W_OK) or \
               not os.access(directory, os.R_OK + os.W_OK):
            tmp = db_name
            db_name = None
            raise EnvironmentError, \
                  'The web server user requires read _and_ write permission\n' \
                  'to the database %s and the directory this file is located in.' % tmp
        sqlite.Connection.__init__(self, db_name, timeout=10000)

    def load_config(self):
        """
        load configuration from the config table.

        The configuration is returned as a section-dictionary containing
        name-value dictionaries.
        """
        cursor = self.cursor()
        cursor.execute('SELECT section, name, value FROM config')
        config = {}
        rows = cursor.fetchall()
        for row in rows:
            if not config.has_key(row[0]):
                config[row[0]] = {}
            config[row[0]][row[1]] = row[2]
        return config
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Yang Gao <gaoyang.public@gmail.com>
# vim: ts=4 et ai

import sys
import requests
import json
import time
import datetime
import yaml
import sqlite3
from requests_oauthlib import OAuth1Session
from mako.template import Template

TRADEME_BASE_API = 'http://api.trademe.co.nz/v1/Search/%s.json'
CONSUMER_KEY     = ''
CONSUMER_SECRET  = ''
OAUTH_TOKEN_SECRET = ''
OAUTH_TOKEN      = ''

def json_encode(s):
    return json.dumps(s, separators=(',',':'))

def json_decode(s):
    return json.loads(s)

def md5(s):
    import hashlib
    m = hashlib.md5()
    m.update(s)
    return m.hexdigest()

class Trademe(object):
    trademe = requests

    def __init__(self):
        pass

    def _authenticate(self, consumer_key,
                           consumer_secret):
#                           oauth_token,
#                           oauth_secret):
        request_token_url = 'https://secure.trademe.co.nz/Oauth/RequestToken?scope=MyTradeMeRead,MyTradeMeWrite'
        authorization_base_url = 'https://secure.trademe.co.nz/Oauth/Authorize'
        access_token_url = 'https://secure.trademe.co.nz/Oauth/AccessToken'
        kw = dict()
        kw['client_secret']=consumer_secret
        #kw['resource_owner_key']=oauth_token
        #kw['resource_owner_secret']=oauth_secret
        self.trademe = OAuth1Session(consumer_key, **kw)
        self.trademe.fetch_request_token(request_token_url)
        authorization_url = self.trademe.authorization_url(authorization_base_url)
        print(authorization_url)
        redirect_response = raw_input('Paste the full redirect URL here:')
        self.trademe.parse_authorization_response(redirect_response)
        oauthxx = self.trademe.fetch_access_token(access_token_url)
        print oauthxx

    def authenticate(self, consumer_key,
                           consumer_secret,
                           oauth_token,
                           oauth_secret):
        kw = dict()
        kw['client_secret'] = consumer_secret
        kw['resource_owner_key'] = oauth_token
        kw['resource_owner_secret'] = oauth_secret
        self.trademe = OAuth1Session(consumer_key, **kw)

    def getListings(self, api_path="General",
                          feedback_func=None, **kw):

        api = TRADEME_BASE_API % api_path
        result = None

        params = dict(kw)
        try:
            resp = self.trademe.get(api, params=params, timeout=30)
            result = resp.json()
        except Exception as e:
            raise

        if not feedback_func is None:
            result = feedback_func(result)
        return result

    def getMyWatchList(self):
        api = 'https://api.trademe.co.nz/v1/MyTradeMe/Watchlist/All.json'
        try:
            resp = self.trademe.get(api)
            result = resp.json()
        except Exception as e:
            raise
        return result

def feedback_searching_result(data):
    def _collect(raw_listing):
        listing = filter(lambda x: x[0] in f_keys,
                              raw_listing.items())
        listing = dict(listing)
        listing['ListingUrl'] = listing_url % listing['ListingId']

        which_category = filter(lambda c:
                            c['Category'] == listing['Category'],
                            data['FoundCategories'])
        try:
            listing['CategoryName'] = which_category[0]['Name']
        except IndexError as e:
            listing['CategoryName'] = ''

        return listing

    listing_url = 'http://www.trademe.co.nz/Browse/Listing.aspx?id=%s'
    f_keys = ['ListingId', 'Title', 'Category', 'PictureHref',
              'Region', 'Suburb', 'PriceDisplay', 'StartPrice',
              'BuyNowPrice']
    ret = map(_collect, data['List'])
    return ret

def getConfig(cfile):
    def _merge(data):
        config = dict()
        for item, value in data.items():
            if item.startswith('.'): continue
            config[item] = value.copy()
            if value.has_key('.include'):
                included = dict(data[value['.include']])
                config[item] = dict(included, **config[item])
                del config[item]['.include']
        return config

    config = dict()
    try:
        with open(cfile, 'rb') as f:
            sys, search = yaml.load_all(f.read())
            config['system'] = sys
            config['search'] = _merge(search)
            f.close()
    except IOError as e:
        print str(e)
        sys.exit(2)
    except KeyError as e:
        print 'Include config(%s) does not exist.' % str(e)
        sys.exit(3)
    except Exception as e:
        raise
    return config

def sendEmail(smtp, user, passwd, me, send_to,
               subject, content, cc_to=None):
    '''
    @smtp SMTP server ip or domain.formart: "smtp.gmail.com:587"
    @user SMTP login user name.
    @passwd SMTP login password.
    @me display this string at "from" when he received mail.
    @to send to who.
    @subject email subject
    @content email content
    '''
    import email
    import mimetypes
    import smtplib
    from email.MIMEMultipart import MIMEMultipart
    from email.MIMEText import MIMEText
    from email.MIMEImage import MIMEImage
    from email.Header import Header
    from email.utils import COMMASPACE,formatdate

    assert(isinstance(send_to, list))
    smtp_host, smtp_port = smtp.split(":")

    toAll = list()
    msg = MIMEText(content.encode('utf-8'), 'html', 'utf-8')
    msg['Subject'] = Header(subject, 'utf-8')
    msg['From'] = me
    msg['To'] = COMMASPACE.join(send_to)
    msg['Date'] = formatdate(localtime=True)

    [toAll.append(i) for i in send_to]

    if cc_to:
        msg['Cc'] = COMMASPACE.join(cc_to)
        [toAll.append(i) for i in cc_to]

    mailServer = smtplib.SMTP(smtp_host, smtp_port)
    mailServer.ehlo()
    mailServer.starttls()
    mailServer.ehlo()
    mailServer.login(user, passwd)
    mailServer.sendmail(user, toAll, msg.as_string())
    mailServer.close()

def template(data):
    pass

class ListingModel(object):
    TABLE = 'TradeMe'

    SQL_CREATE_TABLE = '''
CREATE TABLE IF NOT EXISTS %(tb)s(id INTEGER PRIMARY KEY DESC, title, price, buynow, category, url, pic, region, suburb, md5);
CREATE UNIQUE INDEX IF NOT EXISTS TradeMe_md5 ON %(tb)s(md5);
'''

    SQL_SELECT_LISTING = 'SELECT id FROM %(tb)s WHERE md5=%(md5)s;'

    def __init__(self):
        self.db = self._conn('trademe.db')
        c = self.db.cursor()
        c.executescript(self.SQL_CREATE_TABLE % dict( tb = self.TABLE))

    def _conn(self, database):
        conn = sqlite3.connect(database)
        return conn

    def save(self, listing):
        assert(isinstance(listing, dict))
        fields = ['id', 'title', 'price', 'buynow',
                  'category', 'url', 'pic', 'region',
                  'suburb', 'md5']

        sql = "INSERT INTO %s(%s) VALUES(%s)" % (self.TABLE,
                ', '.join(map(lambda x: "%s" % x, fields)),
                ', '.join(map(lambda x: "?", fields)))

        data = (
                listing['ListingId'],
                listing['Title'],
                listing.get('PriceDisplay', 'None'),
                listing.get('BuyNowPrice', 'None'),
                listing['Category'],
                listing['ListingUrl'],
                listing['PictureHref'],
                listing['Region'],
                listing['Suburb'],
                md5('%s-%s' % ( listing['ListingId'],
                                listing['PriceDisplay'])),
                )

        c = self.db.cursor()
        c.execute(sql, data)
        self.db.commit()

def main():
    trademe = Trademe()
    #trademe._authenticate(CONSUMER_KEY, CONSUMER_SECRET)
    #                      OAUTH_TOKEN, OAUTH_SECRET)
    #watch_list = trademe.getMyWatchList()
    trademe.authenticate(CONSUMER_KEY, CONSUMER_SECRET,
                         OAUTH_TOKEN, OAUTH_TOKEN_SECRET)

    for key, value in config['search'].items():
        print 'Get %s listing...' % key
        params = value
        func = feedback_searching_result
        listings = trademe.getListings(feedback_func=func, **params)
        obj_listing = ListingModel()
        send_row = list()
        for row in listings:
            try:
                obj_listing.save(row)
            except sqlite3.IntegrityError as e:
                if not str(e) == 'column md5 is not unique':
                    raise
            else:
                send_row.append(row)

        ## template    = Template(filename='template/listing.htm')
        ## print template.render(listings=listings).encode('utf-8')
        ## sys.exit()

        if send_row:
            template    = Template(filename='template/listing.htm')
            SUBJECT = 'Elton Postman "%s"' % key
            CONTENT     = template.render(listings=send_row)
            sendEmail(SMTP, SMTP_USER, SMTP_PASS,
                       ME, SEND_TO, SUBJECT, CONTENT)
            print 'Sending email done.'

        print 'Sleep 2 seconds..'
        time.sleep(2)

if __name__ == '__main__':
    config = getConfig('prod.yaml')
    SMTP        = config['system'].get('SMTP_HOST')
    SMTP_USER   = config['system'].get('SMTP_USER')
    SMTP_PASS   = config['system'].get('SMTP_PASS')
    ME          = config['system'].get('DISPLAY_ME')
    SEND_TO     = config['system'].get('SEND_TO')

    CONSUMER_KEY        = config['system'].get('CONSUMER_KEY')
    CONSUMER_SECRET     = config['system'].get('CONSUMER_SECRET')
    OAUTH_TOKEN         = config['system'].get('OAUTH_TOKEN')
    OAUTH_TOKEN_SECRET  = config['system'].get('OAUTH_TOKEN_SECRET')

    try:
        main()
    except Exception as e:
        raise
        SUBJECT = 'EltonPostman crashed'
        CONTENT = 'Message: %s' % str(e)
        sendEmail(SMTP, SMTP_USER, SMTP_PASS,
                   ME, SEND_TO, SUBJECT, CONTENT)
<commit_msg>fix keyError.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Yang Gao <gaoyang.public@gmail.com>
# vim: ts=4 et ai

import sys
import requests
import json
import time
import datetime
import yaml
import sqlite3
from requests_oauthlib import OAuth1Session
from mako.template import Template

TRADEME_BASE_API = 'http://api.trademe.co.nz/v1/Search/%s.json'
CONSUMER_KEY     = ''
CONSUMER_SECRET  = ''
OAUTH_TOKEN_SECRET = ''
OAUTH_TOKEN      = ''

def json_encode(s):
    return json.dumps(s, separators=(',',':'))

def json_decode(s):
    return json.loads(s)

def md5(s):
    import hashlib
    m = hashlib.md5()
    m.update(s)
    return m.hexdigest()

class Trademe(object):
    trademe = requests

    def __init__(self):
        pass

    def _authenticate(self, consumer_key,
                           consumer_secret):
#                           oauth_token,
#                           oauth_secret):
        request_token_url = 'https://secure.trademe.co.nz/Oauth/RequestToken?scope=MyTradeMeRead,MyTradeMeWrite'
        authorization_base_url = 'https://secure.trademe.co.nz/Oauth/Authorize'
        access_token_url = 'https://secure.trademe.co.nz/Oauth/AccessToken'
        kw = dict()
        kw['client_secret']=consumer_secret
        #kw['resource_owner_key']=oauth_token
        #kw['resource_owner_secret']=oauth_secret
        self.trademe = OAuth1Session(consumer_key, **kw)
        self.trademe.fetch_request_token(request_token_url)
        authorization_url = self.trademe.authorization_url(authorization_base_url)
        print(authorization_url)
        redirect_response = raw_input('Paste the full redirect URL here:')
        self.trademe.parse_authorization_response(redirect_response)
        oauthxx = self.trademe.fetch_access_token(access_token_url)
        print oauthxx

    def authenticate(self, consumer_key,
                           consumer_secret,
                           oauth_token,
                           oauth_secret):
        kw = dict()
        kw['client_secret'] = consumer_secret
        kw['resource_owner_key'] = oauth_token
        kw['resource_owner_secret'] = oauth_secret
        self.trademe = OAuth1Session(consumer_key, **kw)

    def getListings(self, api_path="General",
                          feedback_func=None, **kw):

        api = TRADEME_BASE_API % api_path
        result = None

        params = dict(kw)
        try:
            resp = self.trademe.get(api, params=params, timeout=30)
            result = resp.json()
        except Exception as e:
            raise

        if not feedback_func is None:
            result = feedback_func(result)
        return result

    def getMyWatchList(self):
        api = 'https://api.trademe.co.nz/v1/MyTradeMe/Watchlist/All.json'
        try:
            resp = self.trademe.get(api)
            result = resp.json()
        except Exception as e:
            raise
        return result

def feedback_searching_result(data):
    def _collect(raw_listing):
        listing = filter(lambda x: x[0] in f_keys,
                              raw_listing.items())
        listing = dict(listing)
        listing['ListingUrl'] = listing_url % listing['ListingId']

        which_category = filter(lambda c:
                            c['Category'] == listing['Category'],
                            data['FoundCategories'])
        try:
            listing['CategoryName'] = which_category[0]['Name']
        except IndexError as e:
            listing['CategoryName'] = ''

        return listing

    listing_url = 'http://www.trademe.co.nz/Browse/Listing.aspx?id=%s'
    f_keys = ['ListingId', 'Title', 'Category', 'PictureHref',
              'Region', 'Suburb', 'PriceDisplay', 'StartPrice',
              'BuyNowPrice']
    ret = map(_collect, data['List'])
    return ret

def getConfig(cfile):
    def _merge(data):
        config = dict()
        for item, value in data.items():
            if item.startswith('.'): continue
            config[item] = value.copy()
            if value.has_key('.include'):
                included = dict(data[value['.include']])
                config[item] = dict(included, **config[item])
                del config[item]['.include']
        return config

    config = dict()
    try:
        with open(cfile, 'rb') as f:
            sys, search = yaml.load_all(f.read())
            config['system'] = sys
            config['search'] = _merge(search)
            f.close()
    except IOError as e:
        print str(e)
        sys.exit(2)
    except KeyError as e:
        print 'Include config(%s) does not exist.' % str(e)
        sys.exit(3)
    except Exception as e:
        raise
    return config

def sendEmail(smtp, user, passwd, me, send_to,
               subject, content, cc_to=None):
    '''
    @smtp SMTP server ip or domain.formart: "smtp.gmail.com:587"
    @user SMTP login user name.
    @passwd SMTP login password.
    @me display this string at "from" when he received mail.
    @to send to who.
    @subject email subject
    @content email content
    '''
    import email
    import mimetypes
    import smtplib
    from email.MIMEMultipart import MIMEMultipart
    from email.MIMEText import MIMEText
    from email.MIMEImage import MIMEImage
    from email.Header import Header
    from email.utils import COMMASPACE,formatdate

    assert(isinstance(send_to, list))
    smtp_host, smtp_port = smtp.split(":")

    toAll = list()
    msg = MIMEText(content.encode('utf-8'), 'html', 'utf-8')
    msg['Subject'] = Header(subject, 'utf-8')
    msg['From'] = me
    msg['To'] = COMMASPACE.join(send_to)
    msg['Date'] = formatdate(localtime=True)

    [toAll.append(i) for i in send_to]

    if cc_to:
        msg['Cc'] = COMMASPACE.join(cc_to)
        [toAll.append(i) for i in cc_to]

    mailServer = smtplib.SMTP(smtp_host, smtp_port)
    mailServer.ehlo()
    mailServer.starttls()
    mailServer.ehlo()
    mailServer.login(user, passwd)
    mailServer.sendmail(user, toAll, msg.as_string())
    mailServer.close()

def template(data):
    pass

class ListingModel(object):
    TABLE = 'TradeMe'

    SQL_CREATE_TABLE = '''
CREATE TABLE IF NOT EXISTS %(tb)s(id INTEGER PRIMARY KEY DESC, title, price, buynow, category, url, pic, region, suburb, md5);
CREATE UNIQUE INDEX IF NOT EXISTS TradeMe_md5 ON %(tb)s(md5);
'''

    SQL_SELECT_LISTING = 'SELECT id FROM %(tb)s WHERE md5=%(md5)s;'

    def __init__(self):
        self.db = self._conn('trademe.db')
        c = self.db.cursor()
        c.executescript(self.SQL_CREATE_TABLE % dict( tb = self.TABLE))

    def _conn(self, database):
        conn = sqlite3.connect(database)
        return conn

    def save(self, listing):
        assert(isinstance(listing, dict))
        fields = ['id', 'title', 'price', 'buynow',
                  'category', 'url', 'pic', 'region',
                  'suburb', 'md5']

        sql = "INSERT INTO %s(%s) VALUES(%s)" % (self.TABLE,
                ', '.join(map(lambda x: "%s" % x, fields)),
                ', '.join(map(lambda x: "?", fields)))

        try:
            data = (
                    listing['ListingId'],
                    listing['Title'],
                    listing.get('PriceDisplay', 'None'),
                    listing.get('BuyNowPrice', 'None'),
                    listing['Category'],
                    listing['ListingUrl'],
                    listing.get('PictureHref', 'http://www.trademe.co.nz/images/NewSearchCards/LVIcons/hasPhoto_160x120.png'),
                    listing['Region'],
                    listing['Suburb'],
                    md5('%s-%s' % ( listing['ListingId'],
                                    listing['PriceDisplay'])),
                    )
        except KeyError as e:
            print listing
            raise

        c = self.db.cursor()
        c.execute(sql, data)
        self.db.commit()

def main():
    trademe = Trademe()
    #trademe._authenticate(CONSUMER_KEY, CONSUMER_SECRET)
    #                      OAUTH_TOKEN, OAUTH_SECRET)
    #watch_list = trademe.getMyWatchList()
    trademe.authenticate(CONSUMER_KEY, CONSUMER_SECRET,
                         OAUTH_TOKEN, OAUTH_TOKEN_SECRET)

    for key, value in config['search'].items():
        print 'Get %s listing...' % key
        params = value
        func = feedback_searching_result
        listings = trademe.getListings(feedback_func=func, **params)
        obj_listing = ListingModel()
        send_row = list()
        for row in listings:
            try:
                obj_listing.save(row)
            except sqlite3.IntegrityError as e:
                if not str(e) == 'column md5 is not unique':
                    raise
            else:
                send_row.append(row)

        ## template    = Template(filename='template/listing.htm')
        ## print template.render(listings=listings).encode('utf-8')
        ## sys.exit()

        if send_row:
            template    = Template(filename='template/listing.htm')
            SUBJECT = 'Elton Postman "%s"' % key
            CONTENT     = template.render(listings=send_row)
            sendEmail(SMTP, SMTP_USER, SMTP_PASS,
                       ME, SEND_TO, SUBJECT, CONTENT)
            print 'Sending email done.'

        print 'Sleep 2 seconds..'
        time.sleep(2)

if __name__ == '__main__':
    config = getConfig('prod.yaml')
    SMTP        = config['system'].get('SMTP_HOST')
    SMTP_USER   = config['system'].get('SMTP_USER')
    SMTP_PASS   = config['system'].get('SMTP_PASS')
    ME          = config['system'].get('DISPLAY_ME')
    SEND_TO     = config['system'].get('SEND_TO')

    CONSUMER_KEY        = config['system'].get('CONSUMER_KEY')
    CONSUMER_SECRET     = config['system'].get('CONSUMER_SECRET')
    OAUTH_TOKEN         = config['system'].get('OAUTH_TOKEN')
    OAUTH_TOKEN_SECRET  = config['system'].get('OAUTH_TOKEN_SECRET')

    try:
        main()
    except Exception as e:
        raise
        SUBJECT = 'EltonPostman crashed'
        CONTENT = 'Message: %s' % str(e)
        sendEmail(SMTP, SMTP_USER, SMTP_PASS,
                   ME, SEND_TO, SUBJECT, CONTENT)
<|endoftext|>


<commit_before>import json
import os
import time

from django.conf import settings

from cnntools import caffefileproc
from cnntools.common_utils import ensuredir
from cnntools.models import CaffeCNNTrainingRun
from cnntools.snapshot_utils import upload_snapshot
from cnntools.utils import (add_caffe_to_path, add_to_path,
                            random_file_from_content)

def _load_training_run(caffe_cnn_trrun_id):
    entry = CaffeCNNTrainingRun.objects.get(id=caffe_cnn_trrun_id)
    outputs_str = json.loads(entry.outputs_json)
    # Convert to numbers
    outputs = []
    for op_str in outputs_str:
        op = {}
        for op_num_str, its_str in op_str.iteritems():
            its = {}
            for it_str, val_str in its_str.iteritems():
                its[int(it_str)] = float(val_str)
            op[int(op_num_str)] = its
        outputs.append(op)

    max_iter = entry.max_iteration

    return outputs, max_iter

def _refresh_training_run(caffe_cnn_trrun_id, outputs, output_names, itnum,
                          max_iter):
    outputs_json = json.dumps(outputs)
    output_names_json = json.dumps(output_names)
    entry = CaffeCNNTrainingRun.objects.get(id=caffe_cnn_trrun_id)
    entry.outputs_json = outputs_json
    entry.output_names_json = output_names_json
    entry.final_iteration = itnum
    entry.max_iteration = max_iter
    entry.save()

def _refresh_solverfile(caffe_cnn_trrun_id, solver_file_content):
    entry = CaffeCNNTrainingRun.objects.get(id=caffe_cnn_trrun_id)
    # Overwrite solverfile content with new data
    entry.solver_file_snapshot.delete(save=False)
    entry.solver_file_snapshot = random_file_from_content(solver_file_content)
    entry.save()

def extract_batchsize_testsetsize(model_file_content):
    from caffe.proto import caffe_pb2

    # Note that we handle only MULTI_IMAGE_DATA layer and some other layers (see below), we require include.phase be TEST
    model_params = caffefileproc.parse_model_definition_file_content(model_file_content)
    batch_size = None
    testset_size = None
    layer_types = [
        'ImageData',
        'MultiImageData',
        'Python',
    ]
    for layer in model_params.layer:
        if layer.type in layer_types:
            if not layer.include:
                continue
            if layer.include[0].phase == caffe_pb2.TEST:
                if layer.type == 'Python':
                    # Try to parse json
                    try:
                        params = json.loads(layer.python_param.param_str)
                        batch_size = params['batch_size']
                        source = params['source']
                    except Exception as e:
                        print 'Failed to parse param_str: ', e
                        print 'Skipping this layer.'
                        continue
                elif layer.type == 'ImageData':
                    batch_size = layer.image_data_param.batch_size
                    source = layer.image_data_param.source
                elif layer.type == 'MultiImageData':
                    batch_size = layer.multi_image_data_param.batch_size
                    source = layer.multi_image_data_param.source
                else:
                    continue

                # Note that the source should be relative to the caffe root path!
                testset_size = len(caffefileproc.freadlines(os.path.join(settings.CAFFE_ROOT, source)))
                break

    return batch_size, testset_size

def setup_solverfile(model_name, model_file_content, solver_file_content,
                     options, caffe_cnn_trrun_id, device_id):
    rand_name = '%s-%s' % (time.time(), caffe_cnn_trrun_id)
    root_path = os.path.join(
        settings.CAFFE_ROOT,
        'training_runs',
        '-'.join([rand_name, model_name])
    )

    ensuredir(root_path)
    trainfilename = 'train_val.prototxt'
    solverfilename = 'solver.prototxt'
    trainfile_path = os.path.join(root_path, trainfilename)
    solverfile_path = os.path.join(root_path, solverfilename)

    # Save the model_file_content to a file, so Caffe can read it
    with open(trainfile_path, 'w') as f:
        f.write(model_file_content)

    # copy the sample solver file and modify it
    solver_params = caffefileproc.parse_solver_file_content(solver_file_content)

    # modify solver params according to the command line parameters
    solver_params.net = trainfile_path
    if 'base_lr' in options and options['base_lr'] is not None:
        solver_params.base_lr = options['base_lr']

    # Switch on debug_info to facilitate debugging
    if 'debug_info' in options and options['debug_info'] is not None:
        solver_params.debug_info = options['debug_info']

    if 'weight_decay' in options and options['weight_decay'] is not None:
        solver_params.weight_decay = options['weight_decay']

    snapshot_path = os.path.join(root_path, 'snapshots')
    ensuredir(snapshot_path)
    solver_params.snapshot_prefix = os.path.join(
        snapshot_path,
        'train_{}-base_lr{}'.format(model_name, solver_params.base_lr)
    )

    # compute the proper test_iter
    batch_size, testset_size = extract_batchsize_testsetsize(model_file_content)

    if batch_size and testset_size:
        if options['verbose']:
            print 'Extracted batch_size ({0}) and testset_size ({1})'.format(
                batch_size, testset_size)
        # Note the solver file should have exactly one test_iter
        solver_params.test_iter[0] = int(testset_size/batch_size)
    else:
        if options['verbose']:
            print 'WARNING: Couldn\'t find the batch_size or the source file ' + \
                'containing the testset, please set the test_iter to ' + \
                'testset_size / batch_size!'

    # Setting random seed value for reproducible results
    solver_params.random_seed = settings.CAFFE_SEED

    add_caffe_to_path()
    import caffe
    from caffe.proto import caffe_pb2

    if settings.CAFFE_GPU and (options['cpu'] is None or not options['cpu']):
        if options['verbose']:
            print 'Using GPU'
        caffe.set_mode_gpu()
        caffe.set_device(device_id)
        solver_params.solver_mode = caffe_pb2.SolverParameter.GPU
    else:
        if options['verbose']:
            print 'Using CPU'
        caffe.set_mode_cpu()
        solver_params.solver_mode = caffe_pb2.SolverParameter.CPU

    caffefileproc.save_protobuf_file(solverfile_path, solver_params)

    return solver_params, solverfile_path

def get_solver_type(caffe, solver_params, verbose=True):
    if not hasattr(solver_params, 'type'):
        solver_type = caffe.SGDSolver
    else:
        valid_solver_names = [
            'SGD', 'AdaDelta', 'AdaGrad', 'Adam', 'Nesterov', 'RMSProp',
        ]
        if solver_params.type not in valid_solver_names:
            raise ValueError('Unexpected solver type: %s' % solver_params.type)

        solver_type = getattr(caffe, solver_params.type + 'Solver')

    if verbose:
        print 'Creating solver "%s"...' % solver_type.__name__
    return solver_type

def train_network(solver_params, solverfile_path, options,
                  caffe_cnn_trrun_id):
    add_caffe_to_path()
    import caffe

    for p in settings.TRAINING_EXTRA_PYTHON_PATH:
        add_to_path(p)

    restore = False

    solver = get_solver_type(caffe, solver_params)(solverfile_path)
    solver_nets = [solver.net] + list(solver.test_nets)
    if 'weights' in options and options['weights'] is not None:
        _, ext = os.path.splitext(options['weights'])
        if ext == '.solverstate':
            solver.restore(
                os.path.join(settings.CAFFE_ROOT, options['weights'])
            )
            restore = True
        else:
            for n in solver_nets:
                n.copy_from(
                    os.path.join(settings.CAFFE_ROOT, options['weights'])
                )

    print 'solver_net count:', len(solver_nets)
    for n in solver_nets:
        data_layer = n.layers[0]
        # If the data layer is python, we try to set the random seed for
        # reproducibility
        if data_layer.type == 'Python':
            # Note: The python implementation of the layer should have a
            # "set_random_seed" function. If we can't find a function with this name,
            # we won't set the random seed
            set_random_seed_func = getattr(data_layer, 'set_random_seed', None)
            if callable(set_random_seed_func):
                set_random_seed_func(settings.CAFFE_SEED)

            # Note: The python implementation of the layer should have a
            # "set_params" function.
            set_params_func = getattr(data_layer, 'set_params', None)
            if 'data_layer_params' in options and callable(set_params_func):
                set_params_func(options['data_layer_params'])

        n.reshape()

    # {key: output_num, value: output_name}
    output_names = [{}, {}]
    # for backward compatibility
    name_to_num = [{}, {}]
    # {key: output_num, value: {key: it_num value: output_value}}
    outputs = [{}, {}]

    for i, op in enumerate(solver.net.outputs):
        output_names[0][i] = op
        name_to_num[0][op] = i
        outputs[0][i] = {}

    op_num = 0
    for test_net in solver.test_nets:
        for op in test_net.outputs:
            output_names[1][op_num] = op
            name_to_num[1][op] = op_num
            outputs[1][op_num] = {}
            op_num += 1

    if restore:
        # Load back the saved outputs, so we can start from the figures where
        # we left off
        outputs, max_iter = _load_training_run(caffe_cnn_trrun_id)
        start_it = int(solver.iter)
        # Filter out data which happened after the snapshot
        for op in outputs:
            for op_num, its in op.iteritems():
                to_remove = []
                for it in its:
                    if it > start_it:
                        to_remove.append(it)
                for it in to_remove:
                    its.pop(it)
    else:
        max_iter = solver_params.max_iter
        start_it = 0

    final_snapshot_id = None
    for it in range(start_it, max_iter+1):
        #start = time.clock()
        solver.step(1)  # SGD by Caffe
        #elapsed = time.clock() - start
        #if options['verbose']:
            #print 'One iteration took {:.2f} seconds'.format(elapsed)

        display = solver_params.display and it % solver_params.display == 0
        if display:
            for op in solver.net.outputs:
                val = solver.net.blobs[op].data
                op_num = name_to_num[0][op]
                outputs[0][op_num][it] = float(val)

        test_display = solver_params.test_interval and \
            it % solver_params.test_interval == 0 and \
            (it != 0 or solver_params.test_initialization)
        if test_display:
            for i, test_net in enumerate(solver.test_nets):
                for op in test_net.outputs:
                    val = solver.test_mean_scores[i][op]
                    op_num = name_to_num[1][op]
                    outputs[1][op_num][it] = float(val)

        if display or test_display:
            _refresh_training_run(
                caffe_cnn_trrun_id,
                outputs,
                output_names,
                it,
                max_iter,
            )

        snapshot = it % solver_params.snapshot == 0 and it != 0
        if snapshot:
            snapshot_path = os.path.join(
                settings.CAFFE_ROOT,
                '{}_iter_{}.caffemodel'.format(
                    solver_params.snapshot_prefix,
                    it
                )
            )
            final_snapshot = upload_snapshot(
                caffe_cnn_trrun_id=caffe_cnn_trrun_id,
                snapshot_path=snapshot_path,
                it=it,
                verbose=options['verbose'],
            )
            final_snapshot_id = final_snapshot.id

    return final_snapshot_id

def start_training(model_name, model_file_content, solver_file_content,
                   options, caffe_cnn_trrun_id, device_id=0):
    if options['verbose']:
        print 'Running training for model {}...'.format(model_name)
        print 'with options: {}'.format(options)

    solver_params, solverfile_path = setup_solverfile(
        model_name=model_name,
        model_file_content=model_file_content,
        solver_file_content=solver_file_content,
        options=options,
        caffe_cnn_trrun_id=caffe_cnn_trrun_id,
        device_id=device_id,
    )

    # Save the final solver file's content to database
    _refresh_solverfile(
        caffe_cnn_trrun_id,
        caffefileproc.gen_protobuf_file_content(solver_params),
    )

    # Change working directory to Caffe
    os.chdir(settings.CAFFE_ROOT)
    # TODO: Figure out to detect divergence and that the loss doesn't decrease,
    # so we can stop training
    return train_network(
        solver_params, solverfile_path, options, caffe_cnn_trrun_id,
    )
<commit_msg>Fixed unicode bug in solver constructor, use only trrun ID<commit_after>import json
import os
import time

from django.conf import settings

from cnntools import caffefileproc
from cnntools.common_utils import ensuredir
from cnntools.models import CaffeCNNTrainingRun
from cnntools.snapshot_utils import upload_snapshot
from cnntools.utils import (add_caffe_to_path, add_to_path,
                            random_file_from_content)

def _load_training_run(caffe_cnn_trrun_id):
    entry = CaffeCNNTrainingRun.objects.get(id=caffe_cnn_trrun_id)
    outputs_str = json.loads(entry.outputs_json)
    # Convert to numbers
    outputs = []
    for op_str in outputs_str:
        op = {}
        for op_num_str, its_str in op_str.iteritems():
            its = {}
            for it_str, val_str in its_str.iteritems():
                its[int(it_str)] = float(val_str)
            op[int(op_num_str)] = its
        outputs.append(op)

    max_iter = entry.max_iteration

    return outputs, max_iter

def _refresh_training_run(caffe_cnn_trrun_id, outputs, output_names, itnum,
                          max_iter):
    outputs_json = json.dumps(outputs)
    output_names_json = json.dumps(output_names)
    entry = CaffeCNNTrainingRun.objects.get(id=caffe_cnn_trrun_id)
    entry.outputs_json = outputs_json
    entry.output_names_json = output_names_json
    entry.final_iteration = itnum
    entry.max_iteration = max_iter
    entry.save()

def _refresh_solverfile(caffe_cnn_trrun_id, solver_file_content):
    entry = CaffeCNNTrainingRun.objects.get(id=caffe_cnn_trrun_id)
    # Overwrite solverfile content with new data
    entry.solver_file_snapshot.delete(save=False)
    entry.solver_file_snapshot = random_file_from_content(solver_file_content)
    entry.save()

def extract_batchsize_testsetsize(model_file_content):
    from caffe.proto import caffe_pb2

    # Note that we handle only MULTI_IMAGE_DATA layer and some other layers (see below), we require include.phase be TEST
    model_params = caffefileproc.parse_model_definition_file_content(model_file_content)
    batch_size = None
    testset_size = None
    layer_types = [
        'ImageData',
        'MultiImageData',
        'Python',
    ]
    for layer in model_params.layer:
        if layer.type in layer_types:
            if not layer.include:
                continue
            if layer.include[0].phase == caffe_pb2.TEST:
                if layer.type == 'Python':
                    # Try to parse json
                    try:
                        params = json.loads(layer.python_param.param_str)
                        batch_size = params['batch_size']
                        source = params['source']
                    except Exception as e:
                        print 'Failed to parse param_str: ', e
                        print 'Skipping this layer.'
                        continue
                elif layer.type == 'ImageData':
                    batch_size = layer.image_data_param.batch_size
                    source = layer.image_data_param.source
                elif layer.type == 'MultiImageData':
                    batch_size = layer.multi_image_data_param.batch_size
                    source = layer.multi_image_data_param.source
                else:
                    continue

                # Note that the source should be relative to the caffe root path!
                testset_size = len(caffefileproc.freadlines(os.path.join(settings.CAFFE_ROOT, source)))
                break

    return batch_size, testset_size

def setup_solverfile(model_name, model_file_content, solver_file_content,
                     options, caffe_cnn_trrun_id, device_id):
    rand_name = '%d' % caffe_cnn_trrun_id
    root_path = os.path.join(
        settings.CAFFE_ROOT,
        'training_runs',
        '-'.join([rand_name, model_name])
    )

    ensuredir(root_path)
    trainfilename = 'train_val.prototxt'
    solverfilename = 'solver.prototxt'
    trainfile_path = os.path.join(root_path, trainfilename)
    solverfile_path = os.path.join(root_path, solverfilename)

    # Save the model_file_content to a file, so Caffe can read it
    with open(trainfile_path, 'w') as f:
        f.write(model_file_content)

    # copy the sample solver file and modify it
    solver_params = caffefileproc.parse_solver_file_content(solver_file_content)

    # modify solver params according to the command line parameters
    solver_params.net = trainfile_path
    if 'base_lr' in options and options['base_lr'] is not None:
        solver_params.base_lr = options['base_lr']

    # Switch on debug_info to facilitate debugging
    if 'debug_info' in options and options['debug_info'] is not None:
        solver_params.debug_info = options['debug_info']

    if 'weight_decay' in options and options['weight_decay'] is not None:
        solver_params.weight_decay = options['weight_decay']

    snapshot_path = os.path.join(root_path, 'snapshots')
    ensuredir(snapshot_path)
    solver_params.snapshot_prefix = os.path.join(
        snapshot_path,
        'train_{}-base_lr{}'.format(model_name, solver_params.base_lr)
    )

    # compute the proper test_iter
    batch_size, testset_size = extract_batchsize_testsetsize(model_file_content)

    if batch_size and testset_size:
        if options['verbose']:
            print 'Extracted batch_size ({0}) and testset_size ({1})'.format(
                batch_size, testset_size)
        # Note the solver file should have exactly one test_iter
        solver_params.test_iter[0] = int(testset_size/batch_size)
    else:
        if options['verbose']:
            print 'WARNING: Couldn\'t find the batch_size or the source file ' + \
                'containing the testset, please set the test_iter to ' + \
                'testset_size / batch_size!'

    # Setting random seed value for reproducible results
    solver_params.random_seed = settings.CAFFE_SEED

    add_caffe_to_path()
    import caffe
    from caffe.proto import caffe_pb2

    if settings.CAFFE_GPU and (options['cpu'] is None or not options['cpu']):
        if options['verbose']:
            print 'Using GPU'
        caffe.set_mode_gpu()
        caffe.set_device(device_id)
        solver_params.solver_mode = caffe_pb2.SolverParameter.GPU
    else:
        if options['verbose']:
            print 'Using CPU'
        caffe.set_mode_cpu()
        solver_params.solver_mode = caffe_pb2.SolverParameter.CPU

    caffefileproc.save_protobuf_file(solverfile_path, solver_params)

    return solver_params, solverfile_path

def get_solver_type(caffe, solver_params, verbose=True):
    if not hasattr(solver_params, 'type'):
        solver_type = caffe.SGDSolver
    else:
        valid_solver_names = [
            'SGD', 'AdaDelta', 'AdaGrad', 'Adam', 'Nesterov', 'RMSProp',
        ]
        if solver_params.type not in valid_solver_names:
            raise ValueError('Unexpected solver type: %s' % solver_params.type)

        solver_type = getattr(caffe, solver_params.type + 'Solver')

    if verbose:
        print 'Creating solver "%s"...' % solver_type.__name__
    return solver_type

def train_network(solver_params, solverfile_path, options,
                  caffe_cnn_trrun_id):
    add_caffe_to_path()
    import caffe

    for p in settings.TRAINING_EXTRA_PYTHON_PATH:
        add_to_path(p)

    restore = False

    solver = get_solver_type(caffe, solver_params)(str(solverfile_path))
    solver_nets = [solver.net] + list(solver.test_nets)
    if 'weights' in options and options['weights'] is not None:
        _, ext = os.path.splitext(options['weights'])
        if ext == '.solverstate':
            solver.restore(
                os.path.join(settings.CAFFE_ROOT, options['weights'])
            )
            restore = True
        else:
            for n in solver_nets:
                n.copy_from(
                    os.path.join(settings.CAFFE_ROOT, options['weights'])
                )

    print 'solver_net count:', len(solver_nets)
    for n in solver_nets:
        data_layer = n.layers[0]
        # If the data layer is python, we try to set the random seed for
        # reproducibility
        if data_layer.type == 'Python':
            # Note: The python implementation of the layer should have a
            # "set_random_seed" function. If we can't find a function with this name,
            # we won't set the random seed
            set_random_seed_func = getattr(data_layer, 'set_random_seed', None)
            if callable(set_random_seed_func):
                set_random_seed_func(settings.CAFFE_SEED)

            # Note: The python implementation of the layer should have a
            # "set_params" function.
            set_params_func = getattr(data_layer, 'set_params', None)
            if 'data_layer_params' in options and callable(set_params_func):
                set_params_func(options['data_layer_params'])

        n.reshape()

    # {key: output_num, value: output_name}
    output_names = [{}, {}]
    # for backward compatibility
    name_to_num = [{}, {}]
    # {key: output_num, value: {key: it_num value: output_value}}
    outputs = [{}, {}]

    for i, op in enumerate(solver.net.outputs):
        output_names[0][i] = op
        name_to_num[0][op] = i
        outputs[0][i] = {}

    op_num = 0
    for test_net in solver.test_nets:
        for op in test_net.outputs:
            output_names[1][op_num] = op
            name_to_num[1][op] = op_num
            outputs[1][op_num] = {}
            op_num += 1

    if restore:
        # Load back the saved outputs, so we can start from the figures where
        # we left off
        outputs, max_iter = _load_training_run(caffe_cnn_trrun_id)
        start_it = int(solver.iter)
        # Filter out data which happened after the snapshot
        for op in outputs:
            for op_num, its in op.iteritems():
                to_remove = []
                for it in its:
                    if it > start_it:
                        to_remove.append(it)
                for it in to_remove:
                    its.pop(it)
    else:
        max_iter = solver_params.max_iter
        start_it = 0

    final_snapshot_id = None
    for it in range(start_it, max_iter+1):
        #start = time.clock()
        solver.step(1)  # SGD by Caffe
        #elapsed = time.clock() - start
        #if options['verbose']:
            #print 'One iteration took {:.2f} seconds'.format(elapsed)

        display = solver_params.display and it % solver_params.display == 0
        if display:
            for op in solver.net.outputs:
                val = solver.net.blobs[op].data
                op_num = name_to_num[0][op]
                outputs[0][op_num][it] = float(val)

        test_display = solver_params.test_interval and \
            it % solver_params.test_interval == 0 and \
            (it != 0 or solver_params.test_initialization)
        if test_display:
            for i, test_net in enumerate(solver.test_nets):
                for op in test_net.outputs:
                    val = solver.test_mean_scores[i][op]
                    op_num = name_to_num[1][op]
                    outputs[1][op_num][it] = float(val)

        if display or test_display:
            _refresh_training_run(
                caffe_cnn_trrun_id,
                outputs,
                output_names,
                it,
                max_iter,
            )

        snapshot = it % solver_params.snapshot == 0 and it != 0
        if snapshot:
            snapshot_path = os.path.join(
                settings.CAFFE_ROOT,
                '{}_iter_{}.caffemodel'.format(
                    solver_params.snapshot_prefix,
                    it
                )
            )
            final_snapshot = upload_snapshot(
                caffe_cnn_trrun_id=caffe_cnn_trrun_id,
                snapshot_path=snapshot_path,
                it=it,
                verbose=options['verbose'],
            )
            final_snapshot_id = final_snapshot.id

    return final_snapshot_id

def start_training(model_name, model_file_content, solver_file_content,
                   options, caffe_cnn_trrun_id, device_id=0):
    if options['verbose']:
        print 'Running training for model {}...'.format(model_name)
        print 'with options: {}'.format(options)

    solver_params, solverfile_path = setup_solverfile(
        model_name=model_name,
        model_file_content=model_file_content,
        solver_file_content=solver_file_content,
        options=options,
        caffe_cnn_trrun_id=caffe_cnn_trrun_id,
        device_id=device_id,
    )

    # Save the final solver file's content to database
    _refresh_solverfile(
        caffe_cnn_trrun_id,
        caffefileproc.gen_protobuf_file_content(solver_params),
    )

    # Change working directory to Caffe
    os.chdir(settings.CAFFE_ROOT)
    # TODO: Figure out to detect divergence and that the loss doesn't decrease,
    # so we can stop training
    return train_network(
        solver_params, solverfile_path, options, caffe_cnn_trrun_id,
    )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.conf import settings
from django.db.models.fields import Field, CharField
from django.db.models.fields.related import (ForeignKey, OneToOneField,
                                             ManyToManyField)

from modeltranslation.utils import (get_language, build_localized_fieldname,
                                    build_localized_verbose_name)

class TranslationField(Field):
    """
    The translation field functions as a proxy to the original field which is
    wrapped.

    For every field defined in the model's ``TranslationOptions`` localized
    versions of that field are added to the model depending on the languages
    given in ``settings.LANGUAGES``.

    If for example there is a model ``News`` with a field ``title`` which is
    registered for translation and the ``settings.LANGUAGES`` contains the
    ``de`` and ``en`` languages, the fields ``title_de`` and ``title_en`` will
    be added to the model class. These fields are realized using this
    descriptor.

    The translation field needs to know which language it contains therefore
    that needs to be specified when the field is created.
    """
    def __init__(self, translated_field, language, *args, **kwargs):
        # Update the dict of this field with the content of the original one
        # This might be a bit radical?! Seems to work though...
        self.__dict__.update(translated_field.__dict__)

        # Common init
        self._post_init(translated_field, language)

    def _post_init(self, translated_field, language):
        """Common init for subclasses of TranslationField."""
        # Store the originally wrapped field for later
        self.translated_field = translated_field
        self.language = language

        # Translation are always optional (for now - maybe add some parameters
        # to the translation options for configuring this)
        self.null = True
        self.blank = True

        # Adjust the name of this field to reflect the language
        self.attname = build_localized_fieldname(self.translated_field.name,
                                                 self.language)
        self.name = self.attname

        # Copy the verbose name and append a language suffix
        # (will show up e.g. in the admin).
        self.verbose_name =\
        build_localized_verbose_name(translated_field.verbose_name,
                                     language)

    def pre_save(self, model_instance, add):
        val = super(TranslationField, self).pre_save(model_instance, add)
        if get_language() == self.language and not add:
            # Rule is: 3. Assigning a value to a translation field of the
            # default language also updates the original field
            model_instance.__dict__[self.translated_field.name] = val
        return val

    def get_db_prep_value(self, value, connection, prepared=False):
        if value == "":
            return None
        else:
            return value

    def get_internal_type(self):
        return self.translated_field.get_internal_type()

    def south_field_triple(self):
        """Returns a suitable description of this field for South."""
        # We'll just introspect the _actual_ field.
        from south.modelsinspector import introspector
        field_class = '%s.%s' % (self.translated_field.__class__.__module__,
                                 self.translated_field.__class__.__name__)
        args, kwargs = introspector(self.translated_field)
        # That's our definition!
        return (field_class, args, kwargs)

    def formfield(self, *args, **kwargs):
        """Preserves the widget of the translated field."""
        trans_formfield = self.translated_field.formfield(*args, **kwargs)
        defaults = {'widget': type(trans_formfield.widget)}
        defaults.update(kwargs)
        return super(TranslationField, self).formfield(*args, **defaults)

class RelatedTranslationField(object):
    """
    Mixin class which handles shared init of a translated relation field.
    """
    def _related_pre_init(self, translated_field, language, *args, **kwargs):
        self.translated_field = translated_field
        self.language = language

        self.field_name = self.translated_field.name
        self.translated_field_name = \
            build_localized_fieldname(self.translated_field.name,
                                      self.language)

        # Dynamically add a related_name to the original field
        translated_field.rel.related_name = \
            '%s%s' % (self.translated_field.model._meta.module_name,
                      self.field_name)

        TranslationField.__init__(self, self.translated_field, self.language,
                                  *args, **kwargs)

    def _related_post_init(self):
        # Dynamically add a related_name to the translation fields
        self.rel.related_name = \
            '%s%s' % (self.translated_field.model._meta.module_name,
                      self.translated_field_name)

        # ForeignKey's init overrides some essential values from
        # TranslationField, they have to be reassigned.
        TranslationField._post_init(self, self.translated_field, self.language)

class ForeignKeyTranslationField(ForeignKey, TranslationField,
                                 RelatedTranslationField):
    def __init__(self, translated_field, language, to, to_field=None, *args,
                 **kwargs):
        self._related_pre_init(translated_field, language, *args, **kwargs)
        ForeignKey.__init__(self, to, to_field, **kwargs)
        self._related_post_init()

class OneToOneTranslationField(OneToOneField, TranslationField,
                               RelatedTranslationField):
    def __init__(self, translated_field, language, to, to_field=None, *args,
                 **kwargs):
        self._related_pre_init(translated_field, language, *args, **kwargs)
        OneToOneField.__init__(self, to, to_field, **kwargs)
        self._related_post_init()

class ManyToManyTranslationField(ManyToManyField, TranslationField,
                                 RelatedTranslationField):
    def __init__(self, translated_field, language, to, *args, **kwargs):
        self._related_pre_init(translated_field, language, *args, **kwargs)
        ManyToManyField.__init__(self, to, **kwargs)
        self._related_post_init()
<commit_msg>Possible fix for regression in south_field_triple caused by r55.<commit_after># -*- coding: utf-8 -*-
from django.conf import settings
from django.db.models.fields import Field, CharField
from django.db.models.fields.related import (ForeignKey, OneToOneField,
                                             ManyToManyField)

from modeltranslation.utils import (get_language, build_localized_fieldname,
                                    build_localized_verbose_name)

class TranslationField(Field):
    """
    The translation field functions as a proxy to the original field which is
    wrapped.

    For every field defined in the model's ``TranslationOptions`` localized
    versions of that field are added to the model depending on the languages
    given in ``settings.LANGUAGES``.

    If for example there is a model ``News`` with a field ``title`` which is
    registered for translation and the ``settings.LANGUAGES`` contains the
    ``de`` and ``en`` languages, the fields ``title_de`` and ``title_en`` will
    be added to the model class. These fields are realized using this
    descriptor.

    The translation field needs to know which language it contains therefore
    that needs to be specified when the field is created.
    """
    def __init__(self, translated_field, language, *args, **kwargs):
        # Update the dict of this field with the content of the original one
        # This might be a bit radical?! Seems to work though...
        self.__dict__.update(translated_field.__dict__)

        # Common init
        self._post_init(translated_field, language)

    def _post_init(self, translated_field, language):
        """Common init for subclasses of TranslationField."""
        # Store the originally wrapped field for later
        self.translated_field = translated_field
        self.language = language

        # Translation are always optional (for now - maybe add some parameters
        # to the translation options for configuring this)
        self.null = True
        self.blank = True

        # Adjust the name of this field to reflect the language
        self.attname = build_localized_fieldname(self.translated_field.name,
                                                 self.language)
        self.name = self.attname

        # Copy the verbose name and append a language suffix
        # (will show up e.g. in the admin).
        self.verbose_name =\
        build_localized_verbose_name(translated_field.verbose_name,
                                     language)

    def pre_save(self, model_instance, add):
        val = super(TranslationField, self).pre_save(model_instance, add)
        if get_language() == self.language and not add:
            # Rule is: 3. Assigning a value to a translation field of the
            # default language also updates the original field
            model_instance.__dict__[self.translated_field.name] = val
        return val

    def get_db_prep_value(self, value, connection, prepared=False):
        if value == "":
            return None
        else:
            return value

    def get_internal_type(self):
        return self.translated_field.get_internal_type()

    def south_field_triple(self):
        """Returns a suitable description of this field for South."""
        # We'll just introspect the _actual_ field.
        from south.modelsinspector import introspector
        field_class = '%s.%s' % (self.translated_field.__class__.__module__,
                                 self.translated_field.__class__.__name__)
        args, kwargs = introspector(self)
        # That's our definition!
        return (field_class, args, kwargs)

    def formfield(self, *args, **kwargs):
        """Preserves the widget of the translated field."""
        trans_formfield = self.translated_field.formfield(*args, **kwargs)
        defaults = {'widget': type(trans_formfield.widget)}
        defaults.update(kwargs)
        return super(TranslationField, self).formfield(*args, **defaults)

class RelatedTranslationField(object):
    """
    Mixin class which handles shared init of a translated relation field.
    """
    def _related_pre_init(self, translated_field, language, *args, **kwargs):
        self.translated_field = translated_field
        self.language = language

        self.field_name = self.translated_field.name
        self.translated_field_name = \
            build_localized_fieldname(self.translated_field.name,
                                      self.language)

        # Dynamically add a related_name to the original field
        translated_field.rel.related_name = \
            '%s%s' % (self.translated_field.model._meta.module_name,
                      self.field_name)

        TranslationField.__init__(self, self.translated_field, self.language,
                                  *args, **kwargs)

    def _related_post_init(self):
        # Dynamically add a related_name to the translation fields
        self.rel.related_name = \
            '%s%s' % (self.translated_field.model._meta.module_name,
                      self.translated_field_name)

        # ForeignKey's init overrides some essential values from
        # TranslationField, they have to be reassigned.
        TranslationField._post_init(self, self.translated_field, self.language)

class ForeignKeyTranslationField(ForeignKey, TranslationField,
                                 RelatedTranslationField):
    def __init__(self, translated_field, language, to, to_field=None, *args,
                 **kwargs):
        self._related_pre_init(translated_field, language, *args, **kwargs)
        ForeignKey.__init__(self, to, to_field, **kwargs)
        self._related_post_init()

class OneToOneTranslationField(OneToOneField, TranslationField,
                               RelatedTranslationField):
    def __init__(self, translated_field, language, to, to_field=None, *args,
                 **kwargs):
        self._related_pre_init(translated_field, language, *args, **kwargs)
        OneToOneField.__init__(self, to, to_field, **kwargs)
        self._related_post_init()

class ManyToManyTranslationField(ManyToManyField, TranslationField,
                                 RelatedTranslationField):
    def __init__(self, translated_field, language, to, *args, **kwargs):
        self._related_pre_init(translated_field, language, *args, **kwargs)
        ManyToManyField.__init__(self, to, **kwargs)
        self._related_post_init()
<|endoftext|>