<commit_before># -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import DataMigration
from django.db import models

class Migration(DataMigration):

    def forwards(self, orm):
        """ Write your forwards methods here.
        Note: Don't use "from appname.models import ModelName".
        Use orm.ModelName to refer to models in this application,
        and orm['appname.ModelName'] for models in other applications.
        """

        for emis in orm.HeadTeacher.objects.values_list('emis__emis', flat=True).distinct():
            # Query below gets all the duplicate ids ordered by id descending and sliced from first index,
            # should give all the id's other than the max

            query = (orm.
                      HeadTeacher.
                      objects.
                      filter(emis__emis=emis).
                      values_list('id', flat=True).
                      order_by("-id"))

            duplicates = orm.HeadTeacher.objects.filter(pk__in=query[1:])

            for duplicate in duplicates:
                head_teacher_store = orm.HeadTeacherDuplicateStore(first_name=duplicate.first_name,
                                                      last_name=duplicate.last_name,
                                                      gender=duplicate.gender,
                                                      msisdn=duplicate.msisdn,
                                                      date_of_birth=duplicate.date_of_birth,
                                                      is_zonal_head=duplicate.is_zonal_head,
                                                      zonal_head_name=duplicate.zonal_head_name,
                                                      created_at=duplicate.created_at,
                                                      emis=duplicate.emis,
                                                      origin_id=duplicate.id
                                                      )
                head_teacher_store.save()

                if duplicate.schooldata_set.all():
                    school_data_store = orm.SchoolDataDuplicateStore(emis=duplicate.schooldata_set.all()[0].emis,
                                                          name=duplicate.schooldata_set.all()[0].name,
                                                          classrooms=duplicate.schooldata_set.all()[0].classrooms,
                                                          teachers=duplicate.schooldata_set.all()[0].teachers,
                                                          teachers_g1=duplicate.schooldata_set.all()[0].teachers_g1,
                                                          teachers_g2=duplicate.schooldata_set.all()[0].teachers_g2,
                                                          boys_g2=duplicate.schooldata_set.all()[0].boys_g2,
                                                          girls_g2=duplicate.schooldata_set.all()[0].girls_g2,
                                                          created_at=duplicate.schooldata_set.all()[0].created_at,
                                                          created_by=orm.HeadTeacherDuplicateStore.objects.get(origin_id=duplicate.id),
                                                          origin_id=duplicate.schooldata_set.all()[0].id
                                                          )
                    school_data_store.save()
            duplicates.delete()

    def backwards(self, orm):
        "Write your backwards methods here."

    models = {
        u'data.academicachievementcode': {
            'Meta': {'object_name': 'AcademicAchievementCode'},
            'achievement': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
        },
        u'data.headteacher': {
            'Meta': {'object_name': 'HeadTeacher'},
            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'date_of_birth': ('django.db.models.fields.DateField', [], {}),
            'emis': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.School']", 'null': 'True', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'gender': ('django.db.models.fields.CharField', [], {'max_length': '6'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_zonal_head': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'msisdn': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
            'zonal_head_name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        u'data.headteacherduplicatestore': {
            'Meta': {'object_name': 'HeadTeacherDuplicateStore'},
            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'date_of_birth': ('django.db.models.fields.DateField', [], {}),
            'emis': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.School']", 'null': 'True', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'gender': ('django.db.models.fields.CharField', [], {'max_length': '6'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_zonal_head': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'msisdn': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
            'origin_id': ('django.db.models.fields.IntegerField', [], {}),
            'zonal_head_name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        u'data.inboundsms': {
            'Meta': {'object_name': 'InboundSMS'},
            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'created_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['data.HeadTeacher']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'message': ('django.db.models.fields.CharField', [], {'max_length': '200'})
        },
        u'data.learnerperformancedata': {
            'Meta': {'object_name': 'LearnerPerformanceData'},
            'below_minimum_results': ('django.db.models.fields.IntegerField', [], {}),
            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'created_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['data.HeadTeacher']"}),
            'desirable_results': ('django.db.models.fields.IntegerField', [], {}),
            'emis': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.School']"}),
            'gender': ('django.db.models.fields.CharField', [], {'max_length': '6'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'minimum_results': ('django.db.models.fields.IntegerField', [], {}),
            'outstanding_results': ('django.db.models.fields.IntegerField', [], {}),
            'phonetic_awareness': ('django.db.models.fields.IntegerField', [], {}),
            'reading_comprehension': ('django.db.models.fields.IntegerField', [], {}),
            'total_number_pupils': ('django.db.models.fields.IntegerField', [], {}),
            'vocabulary': ('django.db.models.fields.IntegerField', [], {}),
            'writing_diction': ('django.db.models.fields.IntegerField', [], {})
        },
        u'data.schooldata': {
            'Meta': {'object_name': 'SchoolData'},
            'boys_g2': ('django.db.models.fields.IntegerField', [], {}),
            'classrooms': ('django.db.models.fields.IntegerField', [], {}),
            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'created_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['data.HeadTeacher']"}),
            'emis': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.School']"}),
            'girls_g2': ('django.db.models.fields.IntegerField', [], {}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'teachers': ('django.db.models.fields.IntegerField', [], {}),
            'teachers_g1': ('django.db.models.fields.IntegerField', [], {}),
            'teachers_g2': ('django.db.models.fields.IntegerField', [], {})
        },
        u'data.schooldataduplicatestore': {
            'Meta': {'object_name': 'SchoolDataDuplicateStore'},
            'boys_g2': ('django.db.models.fields.IntegerField', [], {}),
            'classrooms': ('django.db.models.fields.IntegerField', [], {}),
            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'created_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['data.HeadTeacherDuplicateStore']"}),
            'emis': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.School']"}),
            'girls_g2': ('django.db.models.fields.IntegerField', [], {}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'origin_id': ('django.db.models.fields.IntegerField', [], {}),
            'teachers': ('django.db.models.fields.IntegerField', [], {}),
            'teachers_g1': ('django.db.models.fields.IntegerField', [], {}),
            'teachers_g2': ('django.db.models.fields.IntegerField', [], {})
        },
        u'data.teacherperformancedata': {
            'Meta': {'object_name': 'TeacherPerformanceData'},
            'academic_level': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['data.AcademicAchievementCode']"}),
            'age': ('django.db.models.fields.IntegerField', [], {}),
            'attitudes_and_beliefs': ('django.db.models.fields.IntegerField', [], {}),
            'classroom_environment_score': ('django.db.models.fields.IntegerField', [], {}),
            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'created_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['data.HeadTeacher']"}),
            'emis': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.School']"}),
            'g2_pupils_present': ('django.db.models.fields.IntegerField', [], {}),
            'g2_pupils_registered': ('django.db.models.fields.IntegerField', [], {}),
            'gender': ('django.db.models.fields.CharField', [], {'max_length': '6'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'pupil_engagement_score': ('django.db.models.fields.IntegerField', [], {}),
            'pupils_books_number': ('django.db.models.fields.IntegerField', [], {}),
            'pupils_materials_score': ('django.db.models.fields.IntegerField', [], {}),
            'reading_assessment': ('django.db.models.fields.IntegerField', [], {'null': 'True'}),
            'reading_lesson': ('django.db.models.fields.IntegerField', [], {}),
            'reading_total': ('django.db.models.fields.IntegerField', [], {'null': 'True'}),
            't_l_materials': ('django.db.models.fields.IntegerField', [], {}),
            'training_subtotal': ('django.db.models.fields.IntegerField', [], {}),
            'ts_number': ('django.db.models.fields.IntegerField', [], {}),
            'years_experience': ('django.db.models.fields.CharField', [], {'max_length': '5'})
        },
        u'hierarchy.district': {
            'Meta': {'object_name': 'District'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'province': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.Province']"})
        },
        u'hierarchy.province': {
            'Meta': {'object_name': 'Province'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        u'hierarchy.school': {
            'Meta': {'object_name': 'School'},
            'emis': ('django.db.models.fields.IntegerField', [], {'unique': 'True', 'max_length': '8'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'zone': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.Zone']"})
        },
        u'hierarchy.zone': {
            'Meta': {'object_name': 'Zone'},
            'district': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.District']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        }
    }

    complete_apps = ['data']
    symmetrical = True
<commit_msg>using single query to get the required data<commit_after># -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import DataMigration
from django.db import models

class Migration(DataMigration):

    def forwards(self, orm):
        """ Write your forwards methods here.
        Note: Don't use "from appname.models import ModelName".
        Use orm.ModelName to refer to models in this application,
        and orm['appname.ModelName'] for models in other applications.
        """

        for emis in orm.HeadTeacher.objects.values_list('emis__emis', flat=True).distinct():
            # Query below gets all the duplicate ids ordered by id descending and sliced from first index,
            # should give all the id's other than the max

            duplicates = orm.HeadTeacher.objects.filter(emis__emis=emis).order_by("-id")[1:]

            for duplicate in duplicates:
                head_teacher_store = orm.HeadTeacherDuplicateStore(first_name=duplicate.first_name,
                                                      last_name=duplicate.last_name,
                                                      gender=duplicate.gender,
                                                      msisdn=duplicate.msisdn,
                                                      date_of_birth=duplicate.date_of_birth,
                                                      is_zonal_head=duplicate.is_zonal_head,
                                                      zonal_head_name=duplicate.zonal_head_name,
                                                      created_at=duplicate.created_at,
                                                      emis=duplicate.emis,
                                                      origin_id=duplicate.id
                                                      )
                head_teacher_store.save()

                if duplicate.schooldata_set.all():
                    school_data_store = orm.SchoolDataDuplicateStore(emis=duplicate.schooldata_set.all()[0].emis,
                                                          name=duplicate.schooldata_set.all()[0].name,
                                                          classrooms=duplicate.schooldata_set.all()[0].classrooms,
                                                          teachers=duplicate.schooldata_set.all()[0].teachers,
                                                          teachers_g1=duplicate.schooldata_set.all()[0].teachers_g1,
                                                          teachers_g2=duplicate.schooldata_set.all()[0].teachers_g2,
                                                          boys_g2=duplicate.schooldata_set.all()[0].boys_g2,
                                                          girls_g2=duplicate.schooldata_set.all()[0].girls_g2,
                                                          created_at=duplicate.schooldata_set.all()[0].created_at,
                                                          created_by=orm.HeadTeacherDuplicateStore.objects.get(origin_id=duplicate.id),
                                                          origin_id=duplicate.schooldata_set.all()[0].id
                                                          )
                    school_data_store.save()
                duplicate.delete()

    def backwards(self, orm):
        "Write your backwards methods here."

    models = {
        u'data.academicachievementcode': {
            'Meta': {'object_name': 'AcademicAchievementCode'},
            'achievement': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
        },
        u'data.headteacher': {
            'Meta': {'object_name': 'HeadTeacher'},
            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'date_of_birth': ('django.db.models.fields.DateField', [], {}),
            'emis': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.School']", 'null': 'True', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'gender': ('django.db.models.fields.CharField', [], {'max_length': '6'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_zonal_head': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'msisdn': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
            'zonal_head_name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        u'data.headteacherduplicatestore': {
            'Meta': {'object_name': 'HeadTeacherDuplicateStore'},
            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'date_of_birth': ('django.db.models.fields.DateField', [], {}),
            'emis': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.School']", 'null': 'True', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'gender': ('django.db.models.fields.CharField', [], {'max_length': '6'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_zonal_head': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'msisdn': ('django.db.models.fields.CharField', [], {'max_length': '20'}),
            'origin_id': ('django.db.models.fields.IntegerField', [], {}),
            'zonal_head_name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        u'data.inboundsms': {
            'Meta': {'object_name': 'InboundSMS'},
            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'created_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['data.HeadTeacher']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'message': ('django.db.models.fields.CharField', [], {'max_length': '200'})
        },
        u'data.learnerperformancedata': {
            'Meta': {'object_name': 'LearnerPerformanceData'},
            'below_minimum_results': ('django.db.models.fields.IntegerField', [], {}),
            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'created_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['data.HeadTeacher']"}),
            'desirable_results': ('django.db.models.fields.IntegerField', [], {}),
            'emis': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.School']"}),
            'gender': ('django.db.models.fields.CharField', [], {'max_length': '6'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'minimum_results': ('django.db.models.fields.IntegerField', [], {}),
            'outstanding_results': ('django.db.models.fields.IntegerField', [], {}),
            'phonetic_awareness': ('django.db.models.fields.IntegerField', [], {}),
            'reading_comprehension': ('django.db.models.fields.IntegerField', [], {}),
            'total_number_pupils': ('django.db.models.fields.IntegerField', [], {}),
            'vocabulary': ('django.db.models.fields.IntegerField', [], {}),
            'writing_diction': ('django.db.models.fields.IntegerField', [], {})
        },
        u'data.schooldata': {
            'Meta': {'object_name': 'SchoolData'},
            'boys_g2': ('django.db.models.fields.IntegerField', [], {}),
            'classrooms': ('django.db.models.fields.IntegerField', [], {}),
            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'created_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['data.HeadTeacher']"}),
            'emis': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.School']"}),
            'girls_g2': ('django.db.models.fields.IntegerField', [], {}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'teachers': ('django.db.models.fields.IntegerField', [], {}),
            'teachers_g1': ('django.db.models.fields.IntegerField', [], {}),
            'teachers_g2': ('django.db.models.fields.IntegerField', [], {})
        },
        u'data.schooldataduplicatestore': {
            'Meta': {'object_name': 'SchoolDataDuplicateStore'},
            'boys_g2': ('django.db.models.fields.IntegerField', [], {}),
            'classrooms': ('django.db.models.fields.IntegerField', [], {}),
            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'created_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['data.HeadTeacherDuplicateStore']"}),
            'emis': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.School']"}),
            'girls_g2': ('django.db.models.fields.IntegerField', [], {}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'origin_id': ('django.db.models.fields.IntegerField', [], {}),
            'teachers': ('django.db.models.fields.IntegerField', [], {}),
            'teachers_g1': ('django.db.models.fields.IntegerField', [], {}),
            'teachers_g2': ('django.db.models.fields.IntegerField', [], {})
        },
        u'data.teacherperformancedata': {
            'Meta': {'object_name': 'TeacherPerformanceData'},
            'academic_level': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['data.AcademicAchievementCode']"}),
            'age': ('django.db.models.fields.IntegerField', [], {}),
            'attitudes_and_beliefs': ('django.db.models.fields.IntegerField', [], {}),
            'classroom_environment_score': ('django.db.models.fields.IntegerField', [], {}),
            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'created_by': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['data.HeadTeacher']"}),
            'emis': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.School']"}),
            'g2_pupils_present': ('django.db.models.fields.IntegerField', [], {}),
            'g2_pupils_registered': ('django.db.models.fields.IntegerField', [], {}),
            'gender': ('django.db.models.fields.CharField', [], {'max_length': '6'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'pupil_engagement_score': ('django.db.models.fields.IntegerField', [], {}),
            'pupils_books_number': ('django.db.models.fields.IntegerField', [], {}),
            'pupils_materials_score': ('django.db.models.fields.IntegerField', [], {}),
            'reading_assessment': ('django.db.models.fields.IntegerField', [], {'null': 'True'}),
            'reading_lesson': ('django.db.models.fields.IntegerField', [], {}),
            'reading_total': ('django.db.models.fields.IntegerField', [], {'null': 'True'}),
            't_l_materials': ('django.db.models.fields.IntegerField', [], {}),
            'training_subtotal': ('django.db.models.fields.IntegerField', [], {}),
            'ts_number': ('django.db.models.fields.IntegerField', [], {}),
            'years_experience': ('django.db.models.fields.CharField', [], {'max_length': '5'})
        },
        u'hierarchy.district': {
            'Meta': {'object_name': 'District'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'province': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.Province']"})
        },
        u'hierarchy.province': {
            'Meta': {'object_name': 'Province'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        u'hierarchy.school': {
            'Meta': {'object_name': 'School'},
            'emis': ('django.db.models.fields.IntegerField', [], {'unique': 'True', 'max_length': '8'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
            'zone': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.Zone']"})
        },
        u'hierarchy.zone': {
            'Meta': {'object_name': 'Zone'},
            'district': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['hierarchy.District']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        }
    }

    complete_apps = ['data']
    symmetrical = True
<|endoftext|>


<commit_before>import traceback
from sqlalchemy.orm import joinedload
from dataactcore.models.baseInterface import BaseInterface
from dataactcore.models.jobModels import Job, JobDependency, JobStatus, JobType
from dataactcore.utils.statusCode import StatusCode
from dataactcore.utils.responseException import ResponseException
from dataactcore.utils.cloudLogger import CloudLogger
from dataactcore.utils.jobQueue import JobQueue
from dataactcore.config import CONFIG_DB

class JobTrackerInterface(BaseInterface):
    """Manages all interaction with the job tracker database."""
    dbConfig = CONFIG_DB
    dbName = dbConfig['job_db_name']
    Session = None
    engine = None
    session = None

    def __init__(self):
        self.dbName = self.dbConfig['job_db_name']
        self.jobQueue = JobQueue()
        super(JobTrackerInterface, self).__init__()

    @staticmethod
    def getDbName():
        """ Return database name"""
        return JobTrackerInterface.dbName

    @staticmethod
    def checkJobUnique(query):
        """ Checks if sqlalchemy queryResult has only one entry, error messages are specific to unique jobs

        Args:
        queryResult -- sqlalchemy query result

        Returns:
        True if single result, otherwise exception
        """
        return BaseInterface.runUniqueQuery(query, "Job ID not found in job table","Conflicting jobs found for this ID")

    def getSession(self):
        """ Return session object"""
        return self.session

    def getJobById(self,jobId):
        """ Return job model object based on ID """
        query = self.session.query(Job).filter(Job.job_id == jobId)
        return self.checkJobUnique(query)

    def getFileName(self,jobId):
        """ Get filename listed in database for this job """
        return self.getJobById(jobId).filename

    def getFileType(self,jobId):
        """ Get type of file associated with this job """
        query = self.session.query(Job).options(joinedload("file_type")).filter(Job.job_id == jobId)
        return self.checkJobUnique(query).file_type.name

    def getFileSize(self,jobId):
        """ Get size of the file associated with this job """
        return self.getJobById(jobId).file_size

    def getSubmissionId(self,jobId):
        """ Find submission that this job is part of """
        return self.getJobById(jobId).submission_id

    def getReportPath(self,jobId):
        """ Return the filename for the error report.  Does not include the folder to avoid conflicting with the S3 getSignedUrl method. """
        return  "submission_" + str(self.getSubmissionId(jobId)) + "_" + self.getFileType(jobId) + "_error_report.csv"

    def getCrossFileReportPath(self,submissionId):
        """ Returns the filename for the cross file error report. """
        return "".join(["submission_",str(submissionId),"_cross_file_error_report.csv"])

    def getJobsBySubmission(self,submissionId):
        """ Get list of jobs that are part of the specified submission

        Args:
            submissionId: submission to list jobs for

        Returns:
            List of job IDs
        """
        jobList = []
        queryResult = self.session.query(Job.job_id).filter(Job.submission_id == submissionId).all()
        for result in queryResult:
            jobList.append(result.job_id)
        return jobList

    def getJobStatusName(self, jobId):
        """

        Args:
            jobId: Job status to get

        Returns:
            status name of specified job
        """
        query = self.session.query(Job).options(joinedload("job_status")).filter(Job.job_id == jobId)
        return self.checkJobUnique(query).job_status.name

    def getJobType(self, jobId):
        """

        Args:
            jobId: Job to get description for

        Returns:
            description of specified job
        """

        query = self.session.query(Job).options(joinedload("job_type")).filter(Job.job_id == jobId)
        return self.checkJobUnique(query).job_type.name

    def getDependentJobs(self, jobId):
        """

        Args:
            jobId: job to get dependent jobs of
        Returns:
            list of jobs dependent on the specified job
        """

        dependents = []
        queryResult = self.session.query(JobDependency).filter(JobDependency.prerequisite_id == jobId).all()
        for result in queryResult:
            dependents.append(result.job_id)
        return dependents

    def markJobStatus(self,jobId,statusName):
        """ Mark job as having specified status.  Jobs being marked as finished will add dependent jobs to queue.

        Args:
            jobId: ID for job being marked
            statusName: Status to change job to
        """
        # Pull JobStatus for jobId
        prevStatus = self.getJobStatusName(jobId)

        query = self.session.query(Job).filter(Job.job_id == jobId)
        result = self.checkJobUnique(query)
        # Mark it finished
        result.job_status_id = self.getJobStatusId(statusName)
        # Push
        self.session.commit()

        # If status is changed to finished for the first time, check dependencies
        # and add to the job queue as necessary
        if prevStatus != 'finished' and statusName == 'finished':
            self.checkJobDependencies(jobId)

    def getJobStatus(self,jobId):
        """ Get status for specified job

        Args:
        jobId -- job to get status for

        Returns:
        status ID
        """
        status = None
        query = self.session.query(Job.job_status_id).filter(Job.job_id == jobId)
        result = self.checkJobUnique(query)
        status = result.job_status_id
        self.session.commit()
        return status

    def getJobStatusId(self,statusName):
        """ Return the status ID that corresponds to the given name """
        return self.getIdFromDict(
            JobStatus, "JOB_STATUS_DICT", "name", statusName, "job_status_id")

    def getJobTypeId(self,typeName):
        """ Return the type ID that corresponds to the given name """
        return self.getIdFromDict(JobType,"JOB_TYPE_DICT","name",typeName,"job_type_id")

    def getOriginalFilenameById(self,jobId):
        """ Get original filename for job matching ID """
        return self.getJobById(jobId).original_filename

    def getPrerequisiteJobs(self, jobId):
        """
        Get all the jobs of which the current job is a dependent

        Args:
            jobId: job to get dependent jobs of
        Returns:
            list of prerequisite jobs for the specified job
        """
        queryResult = self.session.query(JobDependency.prerequisite_id).filter(JobDependency.job_id == jobId).all()
        prerequisiteJobs = [result.prerequisite_id for result in queryResult]
        return prerequisiteJobs

    def checkJobDependencies(self,jobId):
        """ For specified job, check which of its dependencies are ready to be started, and add them to the queue """

        # raise exception if current job is not actually finished
        if self.getJobStatus(jobId) != self.getJobStatusId('finished'):
            raise ValueError('Current job not finished, unable to check dependencies')

        # check if dependent jobs are finished
        for depJobId in self.getDependentJobs(jobId):
            isReady = True
            if not (self.getJobStatus(depJobId) == self.getJobStatusId('waiting')):
                CloudLogger.logError("Job dependency is not in a 'waiting' state",
                                     ResponseException("Job dependency is not in a 'waiting' state",StatusCode.CLIENT_ERROR, ValueError),
                                     traceback.extract_stack())
                continue
            # if dependent jobs are finished, then check the jobs of which the current job is a dependent
            for preReqJobId in self.getPrerequisiteJobs(depJobId):
                if not (self.getJobStatus(preReqJobId) == self.getJobStatusId('finished')):
                    # Do nothing
                    isReady = False
                    break
            # The type check here is temporary and needs to be removed once the validator is able
            # to handle cross-file validation job
            if isReady and (self.getJobType(depJobId) == 'csv_record_validation' or self.getJobType(depJobId) == 'validation'):
                # mark job as ready
                self.markJobStatus(depJobId, 'ready')
                # add to the job queue
                jobQueueResult = self.jobQueue.enqueue.delay(depJobId)

    def getFileSizeById(self,jobId):
        """ Get file size for job matching ID """
        return self.getJobById(jobId).file_size

    def getNumberOfRowsById(self,jobId):
        """ Get number of rows in file for job matching ID """
        return self.getJobById(jobId).number_of_rows

    def setFileSizeById(self,jobId, fileSize):
        """ Set file size for job matching ID """
        job = self.getJobById(jobId)
        job.file_size = int(fileSize)
        self.session.commit()

    def setNumberOfRowsById(self,jobId, numRows):
        """ Set number of rows in file for job matching ID """
        job = self.getJobById(jobId)
        job.number_of_rows = int(numRows)
        self.session.commit()

    def getSubmissionStatus(self,submissionId):
        jobIds = self.getJobsBySubmission(submissionId)
        statuses = {"started": 0, "in_progress": 0, "has_errors": 0, "validated": 0}
        for jobId in jobIds:
            job = self.getJobById(jobId)
            job_status = job.job_status.name
            if job_status == "invalid" or job_status == "failed":
                statuses["has_errors"] += 1
            if job_status == "waiting":
                statuses["started"] += 1
            if job_status == "running":
                statuses["in_progress"] += 1
            if job_status == "finished":
                statuses["validated"] += 1
        if statuses["has_errors"] != 0:
            return "has errors"
        if statuses["started"] == len(jobIds):
            return "started"
        if statuses["validated"] == len(jobIds):
            return "validated"
        return "validation in progress"<commit_msg>Fixing submission status to map properly to the front-end<commit_after>import traceback
from sqlalchemy.orm import joinedload
from dataactcore.models.baseInterface import BaseInterface
from dataactcore.models.jobModels import Job, JobDependency, JobStatus, JobType
from dataactcore.utils.statusCode import StatusCode
from dataactcore.utils.responseException import ResponseException
from dataactcore.utils.cloudLogger import CloudLogger
from dataactcore.utils.jobQueue import JobQueue
from dataactcore.config import CONFIG_DB

class JobTrackerInterface(BaseInterface):
    """Manages all interaction with the job tracker database."""
    dbConfig = CONFIG_DB
    dbName = dbConfig['job_db_name']
    Session = None
    engine = None
    session = None

    def __init__(self):
        self.dbName = self.dbConfig['job_db_name']
        self.jobQueue = JobQueue()
        super(JobTrackerInterface, self).__init__()

    @staticmethod
    def getDbName():
        """ Return database name"""
        return JobTrackerInterface.dbName

    @staticmethod
    def checkJobUnique(query):
        """ Checks if sqlalchemy queryResult has only one entry, error messages are specific to unique jobs

        Args:
        queryResult -- sqlalchemy query result

        Returns:
        True if single result, otherwise exception
        """
        return BaseInterface.runUniqueQuery(query, "Job ID not found in job table","Conflicting jobs found for this ID")

    def getSession(self):
        """ Return session object"""
        return self.session

    def getJobById(self,jobId):
        """ Return job model object based on ID """
        query = self.session.query(Job).filter(Job.job_id == jobId)
        return self.checkJobUnique(query)

    def getFileName(self,jobId):
        """ Get filename listed in database for this job """
        return self.getJobById(jobId).filename

    def getFileType(self,jobId):
        """ Get type of file associated with this job """
        query = self.session.query(Job).options(joinedload("file_type")).filter(Job.job_id == jobId)
        return self.checkJobUnique(query).file_type.name

    def getFileSize(self,jobId):
        """ Get size of the file associated with this job """
        return self.getJobById(jobId).file_size

    def getSubmissionId(self,jobId):
        """ Find submission that this job is part of """
        return self.getJobById(jobId).submission_id

    def getReportPath(self,jobId):
        """ Return the filename for the error report.  Does not include the folder to avoid conflicting with the S3 getSignedUrl method. """
        return  "submission_" + str(self.getSubmissionId(jobId)) + "_" + self.getFileType(jobId) + "_error_report.csv"

    def getCrossFileReportPath(self,submissionId):
        """ Returns the filename for the cross file error report. """
        return "".join(["submission_",str(submissionId),"_cross_file_error_report.csv"])

    def getJobsBySubmission(self,submissionId):
        """ Get list of jobs that are part of the specified submission

        Args:
            submissionId: submission to list jobs for

        Returns:
            List of job IDs
        """
        jobList = []
        queryResult = self.session.query(Job.job_id).filter(Job.submission_id == submissionId).all()
        for result in queryResult:
            jobList.append(result.job_id)
        return jobList

    def getJobStatusName(self, jobId):
        """

        Args:
            jobId: Job status to get

        Returns:
            status name of specified job
        """
        query = self.session.query(Job).options(joinedload("job_status")).filter(Job.job_id == jobId)
        return self.checkJobUnique(query).job_status.name

    def getJobType(self, jobId):
        """

        Args:
            jobId: Job to get description for

        Returns:
            description of specified job
        """

        query = self.session.query(Job).options(joinedload("job_type")).filter(Job.job_id == jobId)
        return self.checkJobUnique(query).job_type.name

    def getDependentJobs(self, jobId):
        """

        Args:
            jobId: job to get dependent jobs of
        Returns:
            list of jobs dependent on the specified job
        """

        dependents = []
        queryResult = self.session.query(JobDependency).filter(JobDependency.prerequisite_id == jobId).all()
        for result in queryResult:
            dependents.append(result.job_id)
        return dependents

    def markJobStatus(self,jobId,statusName):
        """ Mark job as having specified status.  Jobs being marked as finished will add dependent jobs to queue.

        Args:
            jobId: ID for job being marked
            statusName: Status to change job to
        """
        # Pull JobStatus for jobId
        prevStatus = self.getJobStatusName(jobId)

        query = self.session.query(Job).filter(Job.job_id == jobId)
        result = self.checkJobUnique(query)
        # Mark it finished
        result.job_status_id = self.getJobStatusId(statusName)
        # Push
        self.session.commit()

        # If status is changed to finished for the first time, check dependencies
        # and add to the job queue as necessary
        if prevStatus != 'finished' and statusName == 'finished':
            self.checkJobDependencies(jobId)

    def getJobStatusNames(self):
        """ Get All Job Status names """
        job_status_names = []
        for job_status in self.session.query(JobStatus.name):
            job_status_names.append(str(job_status.name))
        return job_status_names

    def getJobStatus(self,jobId):
        """ Get status for specified job

        Args:
        jobId -- job to get status for

        Returns:
        status ID
        """
        status = None
        query = self.session.query(Job.job_status_id).filter(Job.job_id == jobId)
        result = self.checkJobUnique(query)
        status = result.job_status_id
        self.session.commit()
        return status

    def getJobStatusId(self,statusName):
        """ Return the status ID that corresponds to the given name """
        return self.getIdFromDict(
            JobStatus, "JOB_STATUS_DICT", "name", statusName, "job_status_id")

    def getJobTypeId(self,typeName):
        """ Return the type ID that corresponds to the given name """
        return self.getIdFromDict(JobType,"JOB_TYPE_DICT","name",typeName,"job_type_id")

    def getOriginalFilenameById(self,jobId):
        """ Get original filename for job matching ID """
        return self.getJobById(jobId).original_filename

    def getPrerequisiteJobs(self, jobId):
        """
        Get all the jobs of which the current job is a dependent

        Args:
            jobId: job to get dependent jobs of
        Returns:
            list of prerequisite jobs for the specified job
        """
        queryResult = self.session.query(JobDependency.prerequisite_id).filter(JobDependency.job_id == jobId).all()
        prerequisiteJobs = [result.prerequisite_id for result in queryResult]
        return prerequisiteJobs

    def checkJobDependencies(self,jobId):
        """ For specified job, check which of its dependencies are ready to be started, and add them to the queue """

        # raise exception if current job is not actually finished
        if self.getJobStatus(jobId) != self.getJobStatusId('finished'):
            raise ValueError('Current job not finished, unable to check dependencies')

        # check if dependent jobs are finished
        for depJobId in self.getDependentJobs(jobId):
            isReady = True
            if not (self.getJobStatus(depJobId) == self.getJobStatusId('waiting')):
                CloudLogger.logError("Job dependency is not in a 'waiting' state",
                                     ResponseException("Job dependency is not in a 'waiting' state",StatusCode.CLIENT_ERROR, ValueError),
                                     traceback.extract_stack())
                continue
            # if dependent jobs are finished, then check the jobs of which the current job is a dependent
            for preReqJobId in self.getPrerequisiteJobs(depJobId):
                if not (self.getJobStatus(preReqJobId) == self.getJobStatusId('finished')):
                    # Do nothing
                    isReady = False
                    break
            # The type check here is temporary and needs to be removed once the validator is able
            # to handle cross-file validation job
            if isReady and (self.getJobType(depJobId) == 'csv_record_validation' or self.getJobType(depJobId) == 'validation'):
                # mark job as ready
                self.markJobStatus(depJobId, 'ready')
                # add to the job queue
                jobQueueResult = self.jobQueue.enqueue.delay(depJobId)

    def getFileSizeById(self,jobId):
        """ Get file size for job matching ID """
        return self.getJobById(jobId).file_size

    def getNumberOfRowsById(self,jobId):
        """ Get number of rows in file for job matching ID """
        return self.getJobById(jobId).number_of_rows

    def setFileSizeById(self,jobId, fileSize):
        """ Set file size for job matching ID """
        job = self.getJobById(jobId)
        job.file_size = int(fileSize)
        self.session.commit()

    def setNumberOfRowsById(self,jobId, numRows):
        """ Set number of rows in file for job matching ID """
        job = self.getJobById(jobId)
        job.number_of_rows = int(numRows)
        self.session.commit()

    def getSubmissionStatus(self,submissionId):
        jobIds = self.getJobsBySubmission(submissionId)
        statuses = {}
        for status_name in self.getJobStatusNames():
            statuses[status_name] = 0

        print "DEBUG: " + str(statuses)

        for jobId in jobIds:
            job = self.getJobById(jobId)
            job_status = job.job_status.name
            statuses[job_status] += 1

        if statuses["failed"] != 0:
            return "failed"
        if statuses["invalid"] != 0:
            return "invalid"
        if statuses["running"] != 0:
            return "running"
        if statuses["waiting"] != 0:
            return "waiting"
        if statuses["ready"] != 0:
            return "ready"
        if statuses["finished"] == len(jobIds):
            return "finished"
        return "unknown"
<|endoftext|>


<commit_before>import csv
import os
from enum import Enum

import data_file_helper
from ..shared import GlobalConstants
from ..shared import GlobalVariables
from ..debug import DebugPersistor as debug

debug = debug()

class Persistor:
    NEWLINE = "\n"

    def __init__(self, p_state):
        if not data_file_helper.does_file_exist():
            data_file_helper.initialize_data_file()

        # TODO: Use states to fix this
        # True = 'active' = leave f open for all operations
        # False = 'disabled' = require f to be opened for any operation
        self.p_state = p_state
        self._prepare_data_file()

    def close(self):
        if self.f is None:
            return

        self.f.close()

    def protected_t_append(self, write_data):
        state = self._temporary_query(write_data)
        if (state is Identification.EXISTS) or (state is Identification.IS_NONE):
            return

        self._temporary_append(write_data)

    def append(self, write_data):
        if self.p_state:
            self._persisted_append(write_data)
        else:
            self._temporary_append(write_data)

    def query(self, write_data):
        # true: the item exists
        if self.p_state:
            return self._persisted_query(write_data)
        else:
            return self._temporary_query(write_data)

    def turn(self, new_internal_state):
        if not self.p_state and new_internal_state:
            self.f = open(GlobalConstants.PERSISTED_DATA_PATH, "r+")
        elif self.p_state and not new_internal_state:
            self.close()
        else:
            pass

    # private

    def _prepare_data_file(self):
        if self.p_state:
            self.f = open(GlobalConstants.PERSISTED_DATA_PATH, "r+")
        else:
            self.f = None

    def _persisted_append(self, write_data):
        self._write_with_debug(write_data)

    def _temporary_append(self, write_data):
        f = open(GlobalConstants.PERSISTED_DATA_PATH, "a") #append
        self._write_with_debug(write_data, f)
        f.close()

    def _persisted_query(self, new_data):
        return self._query_with_debug(new_data)

    def _temporary_query(self, new_data):
        f = open(GlobalConstants.PERSISTED_DATA_PATH, "r") #read only
        text_found = self._query_with_debug(new_data, f)
        f.close()

        return text_found

    def _write_with_debug(self, write_data, f=None):
        debug.attempted_written_data(write_data.as_csv_text(), self.p_state)

        ready_data = write_data.as_csv_text() + self.NEWLINE

        getattr(self._file(f), "write")(ready_data)

        debug.successful_written_data(write_data.as_csv_text(), self.p_state)

    def _query_with_debug(self, new_data, f=None):
        identifier = new_data.identifier()
        self._refresh_file()

        debug.attempted_query(new_data.as_csv_text(), self.p_state)

        openable_file = self._file(f)
        text = csv.reader(openable_file, delimiter=",")
        text_found = self._find_by_identifier(text, identifier)

        debug.successful_query(new_data.as_csv_text(), self.p_state, text_found)

        return text_found

    def _find_by_identifier(self, text, identifier):
        if (identifier is None) or (identifier is str(None)):
            # we don't want to write None
            return Identification.IS_NONE

        for row in text:
            if row[0] == identifier:
                return Identification.EXISTS

        return Identification.NEW_ENTRY

    def _file(self, f=None):
        if self.f:
            return self.f
        else:
            return f

    def _refresh_file(self):
        if not self.p_state:
            return
        self._refresh()

    def _refresh(self):
        self.f.close()
        # flush cache
        self.f = open(GlobalConstants.PERSISTED_DATA_PATH, "r+")

class Identification(Enum):
    NEW_ENTRY = 0
    EXISTS = 1
    IS_NONE = 2
<commit_msg>[Fix] making explicit that we only refresh if the last action was an append and persisted state is active<commit_after>import csv
import os
from enum import Enum

import data_file_helper
from ..shared import GlobalConstants
from ..shared import GlobalVariables
from ..debug import DebugPersistor as debug

debug = debug()

class Persistor:
    NEWLINE = "\n"

    def __init__(self, p_state):
        if not data_file_helper.does_file_exist():
            data_file_helper.initialize_data_file()

        # TODO: Use states to fix this
        # True = 'active' = leave f open for all operations
        # False = 'disabled' = require f to be opened for any operation
        self.p_state = p_state
        self._prepare_data_file()
        self.last_action = LastAction.INIT

    def close(self):
        if self.f is None:
            return

        self.f.close()

    def protected_t_append(self, write_data):
        state = self._temporary_query(write_data)
        if (state is Identification.EXISTS) or (state is Identification.IS_NONE):
            return

        self._temporary_append(write_data)

    def append(self, write_data):
        if self.p_state:
            self._persisted_append(write_data)
        else:
            self._temporary_append(write_data)

    def query(self, write_data):
        # true: the item exists
        if self.p_state:
            return self._persisted_query(write_data)
        else:
            return self._temporary_query(write_data)

    def turn(self, new_internal_state):
        if not self.p_state and new_internal_state:
            self.f = open(GlobalConstants.PERSISTED_DATA_PATH, "r+")
        elif self.p_state and not new_internal_state:
            self.close()
        else:
            pass

    # private

    def _prepare_data_file(self):
        if self.p_state:
            self.f = open(GlobalConstants.PERSISTED_DATA_PATH, "r+")
        else:
            self.f = None

    def _persisted_append(self, write_data):
        self._append_with_debug(write_data)

    def _temporary_append(self, write_data):
        f = open(GlobalConstants.PERSISTED_DATA_PATH, "a") #append
        self._append_with_debug(write_data, f)
        f.close()

    def _persisted_query(self, new_data):
        return self._query_with_debug(new_data)

    def _temporary_query(self, new_data):
        f = open(GlobalConstants.PERSISTED_DATA_PATH, "r") #read only
        text_found = self._query_with_debug(new_data, f)
        f.close()

        return text_found

    def _append_with_debug(self, write_data, f=None):
        debug.attempted_written_data(write_data.as_csv_text(), self.p_state)
        ready_data = write_data.as_csv_text() + self.NEWLINE
        getattr(self._file(f), "write")(ready_data)
        debug.successful_written_data(write_data.as_csv_text(), self.p_state)
        self.last_action = LastAction.APPEND

    def _query_with_debug(self, new_data, f=None):
        identifier = new_data.identifier()
        self._refresh_file()
        debug.attempted_query(new_data.as_csv_text(), self.p_state)
        openable_file = self._file(f)
        text = csv.reader(openable_file, delimiter=",")
        text_found = self._find_by_identifier(text, identifier)
        debug.successful_query(new_data.as_csv_text(), self.p_state, text_found)
        self.last_action = LastAction.QUERY

        return text_found

    def _find_by_identifier(self, text, identifier):
        if (identifier is None) or (identifier is str(None)):
            # we don't want to write None
            return Identification.IS_NONE

        for row in text:
            if row[0] == identifier:
                return Identification.EXISTS

        return Identification.NEW_ENTRY

    def _file(self, f=None):
        if self.f:
            return self.f
        else:
            return f

    def _refresh_file(self):
        if self.p_state and (self.last_action is LastAction.APPEND):
            self._refresh()
        return

    def _refresh(self):
        self.f.close()
        # flush cache
        self.f = open(GlobalConstants.PERSISTED_DATA_PATH, "r+")

class Identification(Enum):
    NEW_ENTRY = 0
    EXISTS = 1
    IS_NONE = 2

class LastAction(Enum):
    INIT = 0
    APPEND = 1
    QUERY = 2
<|endoftext|>


<commit_before>from django.db import models

class Access(models.Model):
    class Meta:
	permissions = ( ("search", "Can search photos through Flickr API"), )

<commit_msg>opendiamond.scopeserver.flickr: Typographic fix<commit_after>from django.db import models

class Access(models.Model):
    class Meta:
        permissions = ( ("search", "Can search photos through Flickr API"), )

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from osv import fields, osv
import tools

TRANSLATION_TYPE = [
    ('field', 'Field'),
    ('model', 'Object'),
    ('rml', 'RML  (deprecated - use Report)'), # Pending deprecation - to be replaced by report!
    ('report', 'Report/Template'),
    ('selection', 'Selection'),
    ('view', 'View'),
    ('wizard_button', 'Wizard Button'),
    ('wizard_field', 'Wizard Field'),
    ('wizard_view', 'Wizard View'),
    ('xsl', 'XSL'),
    ('help', 'Help'),
    ('code', 'Code'),
    ('constraint', 'Constraint'),
    ('sql_constraint', 'SQL Constraint')
]

class ir_translation(osv.osv):
    _name = "ir.translation"
    _log_access = False

    def _get_language(self, cr, uid, context):
        lang_model = self.pool.get('res.lang')
        lang_ids = lang_model.search(cr, uid, [('translatable', '=', True)], context=context)
        lang_data = lang_model.read(cr, uid, lang_ids, ['code', 'name'], context=context)
        return [(d['code'], d['name']) for d in lang_data]

    _columns = {
        'name': fields.char('Field Name', size=128, required=True),
        'res_id': fields.integer('Resource ID', select=True),
        'lang': fields.selection(_get_language, string='Language', size=16),
        'type': fields.selection(TRANSLATION_TYPE, string='Type', size=16, select=True),
        'src': fields.text('Source'),
        'value': fields.text('Translation Value'),
        # These two columns map to ir_model_data.module and ir_model_data.name.
        # They are used to resolve the res_id above after loading is done.
        'module': fields.char('Module', size=64, help='Maps to the ir_model_data for which this translation is provided.'),
        'xml_id': fields.char('External ID', size=128, help='Maps to the ir_model_data for which this translation is provided.'),
    }

    def _auto_init(self, cr, context={}):
        super(ir_translation, self)._auto_init(cr, context)

        # FIXME: there is a size limit on btree indexed values so we can't index src column with normal btree.
        cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = %s', ('ir_translation_ltns',))
        if cr.fetchone():
            #temporarily removed: cr.execute('CREATE INDEX ir_translation_ltns ON ir_translation (name, lang, type, src)')
            cr.execute('DROP INDEX ir_translation_ltns')
            cr.commit()
        cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = %s', ('ir_translation_lts',))
        if cr.fetchone():
            #temporarily removed: cr.execute('CREATE INDEX ir_translation_lts ON ir_translation (lang, type, src)')
            cr.execute('DROP INDEX ir_translation_lts')
            cr.commit()

        # add separate hash index on src (no size limit on values), as postgres 8.1+ is able to combine separate indexes
        cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = %s', ('ir_translation_src_hash_idx',))
        if not cr.fetchone():
            cr.execute('CREATE INDEX ir_translation_src_hash_idx ON ir_translation using hash (src)')

        cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = %s', ('ir_translation_ltn',))
        if not cr.fetchone():
            cr.execute('CREATE INDEX ir_translation_ltn ON ir_translation (name, lang, type)')
            cr.commit()

    @tools.ormcache_multi(skiparg=3, multi=6)
    def _get_ids(self, cr, uid, name, tt, lang, ids):
        translations = dict.fromkeys(ids, False)
        if ids:
            cr.execute('select res_id,value ' \
                    'from ir_translation ' \
                    'where lang=%s ' \
                        'and type=%s ' \
                        'and name=%s ' \
                        'and res_id IN %s',
                    (lang,tt,name,tuple(ids)))
            for res_id, value in cr.fetchall():
                translations[res_id] = value
        return translations

    def _set_ids(self, cr, uid, name, tt, lang, ids, value, src=None):
        # clear the caches
        tr = self._get_ids(cr, uid, name, tt, lang, ids)
        for res_id in tr:
            if tr[res_id]:
                self._get_source.clear_cache(self, uid, name, tt, lang, tr[res_id])
        self._get_source.clear_cache(self, uid, name, tt, lang)
        self._get_ids.clear_cache(self, uid, name, tt, lang, ids)

        cr.execute('delete from ir_translation ' \
                'where lang=%s ' \
                    'and type=%s ' \
                    'and name=%s ' \
                    'and res_id IN %s',
                (lang,tt,name,tuple(ids),))
        for id in ids:
            self.create(cr, uid, {
                'lang':lang,
                'type':tt,
                'name':name,
                'res_id':id,
                'value':value,
                'src':src,
                })
        return len(ids)

    @tools.ormcache(skiparg=3)
    def _get_source(self, cr, uid, name, types, lang, source=None):
        """
        Returns the translation for the given combination of name, type, language
        and source. All values passed to this method should be unicode (not byte strings),
        especially ``source``.

        :param name: identification of the term to translate, such as field name (optional if source is passed)
        :param types: single string defining type of term to translate (see ``type`` field on ir.translation), or sequence of allowed types (strings)
        :param lang: language code of the desired translation
        :param source: optional source term to translate (should be unicode)
        :rtype: unicode
        :return: the request translation, or an empty unicode string if no translation was
                 found and `source` was not passed
        """
        # FIXME: should assert that `source` is unicode and fix all callers to always pass unicode
        # so we can remove the string encoding/decoding.
        if not lang:
            return u''
        if isinstance(types, basestring):
            types = (types,)
        if source:
            query = """SELECT value
                       FROM ir_translation
                       WHERE lang=%s
                        AND type in %s
                        AND src=%s"""
            params = (lang or '', types, tools.ustr(source))
            if name:
                query += " AND name=%s"
                params += (tools.ustr(name),)
            cr.execute(query, params)
        else:
            cr.execute("""SELECT value
                          FROM ir_translation
                          WHERE lang=%s
                           AND type in %s
                           AND name=%s""",
                    (lang or '', types, tools.ustr(name)))
        res = cr.fetchone()
        trad = res and res[0] or u''
        if source and not trad:
            return tools.ustr(source)
        return trad

    def create(self, cr, uid, vals, context=None):
        if not context:
            context = {}
        ids = super(ir_translation, self).create(cr, uid, vals, context=context)
        self._get_source.clear_cache(self, uid, vals.get('name',0), vals.get('type',0),  vals.get('lang',0), vals.get('src',0))
        self._get_ids.clear_cache(self, uid, vals.get('name',0), vals.get('type',0), vals.get('lang',0), vals.get('res_id',0))
        return ids

    def write(self, cursor, user, ids, vals, context=None):
        if not context:
            context = {}
        if isinstance(ids, (int, long)):
            ids = [ids]
        result = super(ir_translation, self).write(cursor, user, ids, vals, context=context)
        for trans_obj in self.read(cursor, user, ids, ['name','type','res_id','src','lang'], context=context):
            self._get_source.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], trans_obj['src'])
            self._get_ids.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], [trans_obj['res_id']])
        return result

    def unlink(self, cursor, user, ids, context=None):
        if not context:
            context = {}
        if isinstance(ids, (int, long)):
            ids = [ids]
        for trans_obj in self.read(cursor, user, ids, ['name','type','res_id','src','lang'], context=context):
            self._get_source.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src'])
            self._get_ids.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], [trans_obj['res_id']])
        result = super(ir_translation, self).unlink(cursor, user, ids, context=context)
        return result

ir_translation()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<commit_msg>[FIX] ir_translation _get_ids clear_cache is not multi<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from osv import fields, osv
import tools

TRANSLATION_TYPE = [
    ('field', 'Field'),
    ('model', 'Object'),
    ('rml', 'RML  (deprecated - use Report)'), # Pending deprecation - to be replaced by report!
    ('report', 'Report/Template'),
    ('selection', 'Selection'),
    ('view', 'View'),
    ('wizard_button', 'Wizard Button'),
    ('wizard_field', 'Wizard Field'),
    ('wizard_view', 'Wizard View'),
    ('xsl', 'XSL'),
    ('help', 'Help'),
    ('code', 'Code'),
    ('constraint', 'Constraint'),
    ('sql_constraint', 'SQL Constraint')
]

class ir_translation(osv.osv):
    _name = "ir.translation"
    _log_access = False

    def _get_language(self, cr, uid, context):
        lang_model = self.pool.get('res.lang')
        lang_ids = lang_model.search(cr, uid, [('translatable', '=', True)], context=context)
        lang_data = lang_model.read(cr, uid, lang_ids, ['code', 'name'], context=context)
        return [(d['code'], d['name']) for d in lang_data]

    _columns = {
        'name': fields.char('Field Name', size=128, required=True),
        'res_id': fields.integer('Resource ID', select=True),
        'lang': fields.selection(_get_language, string='Language', size=16),
        'type': fields.selection(TRANSLATION_TYPE, string='Type', size=16, select=True),
        'src': fields.text('Source'),
        'value': fields.text('Translation Value'),
        # These two columns map to ir_model_data.module and ir_model_data.name.
        # They are used to resolve the res_id above after loading is done.
        'module': fields.char('Module', size=64, help='Maps to the ir_model_data for which this translation is provided.'),
        'xml_id': fields.char('External ID', size=128, help='Maps to the ir_model_data for which this translation is provided.'),
    }

    def _auto_init(self, cr, context={}):
        super(ir_translation, self)._auto_init(cr, context)

        # FIXME: there is a size limit on btree indexed values so we can't index src column with normal btree.
        cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = %s', ('ir_translation_ltns',))
        if cr.fetchone():
            #temporarily removed: cr.execute('CREATE INDEX ir_translation_ltns ON ir_translation (name, lang, type, src)')
            cr.execute('DROP INDEX ir_translation_ltns')
            cr.commit()
        cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = %s', ('ir_translation_lts',))
        if cr.fetchone():
            #temporarily removed: cr.execute('CREATE INDEX ir_translation_lts ON ir_translation (lang, type, src)')
            cr.execute('DROP INDEX ir_translation_lts')
            cr.commit()

        # add separate hash index on src (no size limit on values), as postgres 8.1+ is able to combine separate indexes
        cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = %s', ('ir_translation_src_hash_idx',))
        if not cr.fetchone():
            cr.execute('CREATE INDEX ir_translation_src_hash_idx ON ir_translation using hash (src)')

        cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = %s', ('ir_translation_ltn',))
        if not cr.fetchone():
            cr.execute('CREATE INDEX ir_translation_ltn ON ir_translation (name, lang, type)')
            cr.commit()

    @tools.ormcache_multi(skiparg=3, multi=6)
    def _get_ids(self, cr, uid, name, tt, lang, ids):
        translations = dict.fromkeys(ids, False)
        if ids:
            cr.execute('select res_id,value ' \
                    'from ir_translation ' \
                    'where lang=%s ' \
                        'and type=%s ' \
                        'and name=%s ' \
                        'and res_id IN %s',
                    (lang,tt,name,tuple(ids)))
            for res_id, value in cr.fetchall():
                translations[res_id] = value
        return translations

    def _set_ids(self, cr, uid, name, tt, lang, ids, value, src=None):
        # clear the caches
        tr = self._get_ids(cr, uid, name, tt, lang, ids)
        for res_id in tr:
            if tr[res_id]:
                self._get_source.clear_cache(self, uid, name, tt, lang, tr[res_id])
        self._get_source.clear_cache(self, uid, name, tt, lang)
        self._get_ids.clear_cache(self, uid, name, tt, lang, ids)

        cr.execute('delete from ir_translation ' \
                'where lang=%s ' \
                    'and type=%s ' \
                    'and name=%s ' \
                    'and res_id IN %s',
                (lang,tt,name,tuple(ids),))
        for id in ids:
            self.create(cr, uid, {
                'lang':lang,
                'type':tt,
                'name':name,
                'res_id':id,
                'value':value,
                'src':src,
                })
        return len(ids)

    @tools.ormcache(skiparg=3)
    def _get_source(self, cr, uid, name, types, lang, source=None):
        """
        Returns the translation for the given combination of name, type, language
        and source. All values passed to this method should be unicode (not byte strings),
        especially ``source``.

        :param name: identification of the term to translate, such as field name (optional if source is passed)
        :param types: single string defining type of term to translate (see ``type`` field on ir.translation), or sequence of allowed types (strings)
        :param lang: language code of the desired translation
        :param source: optional source term to translate (should be unicode)
        :rtype: unicode
        :return: the request translation, or an empty unicode string if no translation was
                 found and `source` was not passed
        """
        # FIXME: should assert that `source` is unicode and fix all callers to always pass unicode
        # so we can remove the string encoding/decoding.
        if not lang:
            return u''
        if isinstance(types, basestring):
            types = (types,)
        if source:
            query = """SELECT value
                       FROM ir_translation
                       WHERE lang=%s
                        AND type in %s
                        AND src=%s"""
            params = (lang or '', types, tools.ustr(source))
            if name:
                query += " AND name=%s"
                params += (tools.ustr(name),)
            cr.execute(query, params)
        else:
            cr.execute("""SELECT value
                          FROM ir_translation
                          WHERE lang=%s
                           AND type in %s
                           AND name=%s""",
                    (lang or '', types, tools.ustr(name)))
        res = cr.fetchone()
        trad = res and res[0] or u''
        if source and not trad:
            return tools.ustr(source)
        return trad

    def create(self, cr, uid, vals, context=None):
        if not context:
            context = {}
        ids = super(ir_translation, self).create(cr, uid, vals, context=context)
        self._get_source.clear_cache(self, uid, vals.get('name',0), vals.get('type',0),  vals.get('lang',0), vals.get('src',0))
        self._get_ids.clear_cache(self, uid, vals.get('name',0), vals.get('type',0), vals.get('lang',0), vals.get('res_id',0))
        return ids

    def write(self, cursor, user, ids, vals, context=None):
        if not context:
            context = {}
        if isinstance(ids, (int, long)):
            ids = [ids]
        result = super(ir_translation, self).write(cursor, user, ids, vals, context=context)
        for trans_obj in self.read(cursor, user, ids, ['name','type','res_id','src','lang'], context=context):
            self._get_source.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], trans_obj['src'])
            self._get_ids.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], trans_obj['res_id'])
        return result

    def unlink(self, cursor, user, ids, context=None):
        if not context:
            context = {}
        if isinstance(ids, (int, long)):
            ids = [ids]
        for trans_obj in self.read(cursor, user, ids, ['name','type','res_id','src','lang'], context=context):
            self._get_source.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], source=trans_obj['src'])
            self._get_ids.clear_cache(self, user, trans_obj['name'], trans_obj['type'], trans_obj['lang'], [trans_obj['res_id']])
        result = super(ir_translation, self).unlink(cursor, user, ids, context=context)
        return result

ir_translation()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<|endoftext|>


<commit_before>__author__ = 'Chris Krycho'
<commit_msg>Create about page. Fix height issues (use min-height rather than height) for content.<commit_after>__author__ = 'Chris Krycho'

from markdown import markdown
<|endoftext|>


<commit_before>import unittest2

import openerp.tests.common as common

class test_expression(common.TransactionCase):

    def test_search_order(self):

        registry, cr, uid = self.registry, self.cr, self.uid

        # Create 6 partners with a given name, and a given creation order to ensure the order of their ID. Some are set as unactive to verify they are by default excluded from the searchs and to provide a second order argument

        partners = registry('res.partner')
        c = partners.create(cr, uid, {'name': 'test_search_order_C'})
        d = partners.create(cr, uid, {'name': 'test_search_order_D', 'active': False})
        a = partners.create(cr, uid, {'name': 'test_search_order_A'})
        b = partners.create(cr, uid, {'name': 'test_search_order_B'})
        ab = partners.create(cr, uid, {'name': 'test_search_order_AB'})
        e = partners.create(cr, uid, {'name': 'test_search_order_E', 'active': False})

        # The tests.

        # The basic searchs should exclude records that have active = False. The order of ids returned
        # should be given by the 'order' parameter of search()

        name_asc = partners.search(cr, uid, [('name', 'like', 'test_search_order%')], order="name asc")
        self.assertEqual([a, ab, b, c], name_asc, "Search with 'NAME ASC' order failed.")
        name_desc = partners.search(cr, uid, [('name', 'like', 'test_search_order%')], order="name desc")
        self.assertEqual([c, b, ab, a], name_desc, "Search with 'NAME DESC' order failed.")
        id_asc = partners.search(cr, uid, [('name', 'like', 'test_search_order%')], order="id asc")
        self.assertEqual([c, a, b, ab], id_asc, "Search with 'ID ASC' order failed.")
        id_desc = partners.search(cr, uid, [('name', 'like', 'test_search_order%')], order="id desc")
        self.assertEqual([ab, b, a, c], id_desc, "Search with 'ID DESC' order failed.")

        # The inactive records shouldn't be ecxluded as soon as a condition on this field is present in the domain
        # criteria. The 'order' parameter of search() should support any valable coma-separated value

        active_asc_id_asc = partners.search(cr, uid, [('name', 'like', 'test_search_order%'), '|', ('active', '=', True), ('active', '=', False)], order="active asc, id asc")
        self.assertEqual([d, e, c, a, b, ab], active_asc_id_asc, "Search with 'ACTIVE ASC, ID ASC' order failed.")
        active_desc_id_asc = partners.search(cr, uid, [('name', 'like', 'test_search_order%'), '|', ('active', '=', True), ('active', '=', False)], order="active desc, id asc")
        self.assertEqual([c, a, b, ab, d, e], active_desc_id_asc, "Search with 'ACTIVE DESC, ID ASC' order failed.")
        active_asc_id_desc = partners.search(cr, uid, [('name', 'like', 'test_search_order%'), '|', ('active', '=', True), ('active', '=', False)], order="active asc, id desc")
        self.assertEqual([e, d, ab, b, a, c], active_asc_id_desc, "Search with 'ACTIVE ASC, ID DESC' order failed.")
        active_desc_id_desc = partners.search(cr, uid, [('name', 'like', 'test_search_order%'), '|', ('active', '=', True), ('active', '=', False)], order="active desc, id desc")
        self.assertEqual([ab, b, a, c, e, d], active_desc_id_desc, "Search with 'ACTIVE DESC, ID DESC' order failed.")
        id_asc_active_asc = partners.search(cr, uid, [('name', 'like', 'test_search_order%'), '|', ('active', '=', True), ('active', '=', False)], order="id asc, active asc")
        self.assertEqual([c, d, a, b, ab, e], id_asc_active_asc, "Search with 'ID ASC, ACTIVE ASC' order failed.")
        id_asc_active_desc = partners.search(cr, uid, [('name', 'like', 'test_search_order%'), '|', ('active', '=', True), ('active', '=', False)], order="id asc, active desc")
        self.assertEqual([c, d, a, b, ab, e], id_asc_active_desc, "Search with 'ID ASC, ACTIVE DESC' order failed.")
        id_desc_active_asc = partners.search(cr, uid, [('name', 'like', 'test_search_order%'), '|', ('active', '=', True), ('active', '=', False)], order="id desc, active asc")
        self.assertEqual([e, ab, b, a, d, c], id_desc_active_asc, "Search with 'ID DESC, ACTIVE ASC' order failed.")
        id_desc_active_desc = partners.search(cr, uid, [('name', 'like', 'test_search_order%'), '|', ('active', '=', True), ('active', '=', False)], order="id desc, active desc")
        self.assertEqual([e, ab, b, a, d, c], id_desc_active_desc, "Search with 'ID DESC, ACTIVE DESC' order failed.")

<commit_msg>[IMP] test_search: comment typos.<commit_after>import unittest2

import openerp.tests.common as common

class test_expression(common.TransactionCase):

    def test_search_order(self):

        registry, cr, uid = self.registry, self.cr, self.uid

	# Create 6 partners with a given name, and a given creation order to
	# ensure the order of their ID. Some are set as unactive to verify they
	# are by default excluded from the searches and to provide a second
	# `order` argument.

        partners = registry('res.partner')
        c = partners.create(cr, uid, {'name': 'test_search_order_C'})
        d = partners.create(cr, uid, {'name': 'test_search_order_D', 'active': False})
        a = partners.create(cr, uid, {'name': 'test_search_order_A'})
        b = partners.create(cr, uid, {'name': 'test_search_order_B'})
        ab = partners.create(cr, uid, {'name': 'test_search_order_AB'})
        e = partners.create(cr, uid, {'name': 'test_search_order_E', 'active': False})

        # The tests.

	# The basic searches should exclude records that have active = False.
	# The order of the returned ids should be given by the `order`
	# parameter of search().

        name_asc = partners.search(cr, uid, [('name', 'like', 'test_search_order%')], order="name asc")
        self.assertEqual([a, ab, b, c], name_asc, "Search with 'NAME ASC' order failed.")
        name_desc = partners.search(cr, uid, [('name', 'like', 'test_search_order%')], order="name desc")
        self.assertEqual([c, b, ab, a], name_desc, "Search with 'NAME DESC' order failed.")
        id_asc = partners.search(cr, uid, [('name', 'like', 'test_search_order%')], order="id asc")
        self.assertEqual([c, a, b, ab], id_asc, "Search with 'ID ASC' order failed.")
        id_desc = partners.search(cr, uid, [('name', 'like', 'test_search_order%')], order="id desc")
        self.assertEqual([ab, b, a, c], id_desc, "Search with 'ID DESC' order failed.")

	# The inactive records shouldn't be excluded as soon as a condition on
	# that field is present in the domain. The `order` parameter of
	# search() should support any legal coma-separated values.

        active_asc_id_asc = partners.search(cr, uid, [('name', 'like', 'test_search_order%'), '|', ('active', '=', True), ('active', '=', False)], order="active asc, id asc")
        self.assertEqual([d, e, c, a, b, ab], active_asc_id_asc, "Search with 'ACTIVE ASC, ID ASC' order failed.")
        active_desc_id_asc = partners.search(cr, uid, [('name', 'like', 'test_search_order%'), '|', ('active', '=', True), ('active', '=', False)], order="active desc, id asc")
        self.assertEqual([c, a, b, ab, d, e], active_desc_id_asc, "Search with 'ACTIVE DESC, ID ASC' order failed.")
        active_asc_id_desc = partners.search(cr, uid, [('name', 'like', 'test_search_order%'), '|', ('active', '=', True), ('active', '=', False)], order="active asc, id desc")
        self.assertEqual([e, d, ab, b, a, c], active_asc_id_desc, "Search with 'ACTIVE ASC, ID DESC' order failed.")
        active_desc_id_desc = partners.search(cr, uid, [('name', 'like', 'test_search_order%'), '|', ('active', '=', True), ('active', '=', False)], order="active desc, id desc")
        self.assertEqual([ab, b, a, c, e, d], active_desc_id_desc, "Search with 'ACTIVE DESC, ID DESC' order failed.")
        id_asc_active_asc = partners.search(cr, uid, [('name', 'like', 'test_search_order%'), '|', ('active', '=', True), ('active', '=', False)], order="id asc, active asc")
        self.assertEqual([c, d, a, b, ab, e], id_asc_active_asc, "Search with 'ID ASC, ACTIVE ASC' order failed.")
        id_asc_active_desc = partners.search(cr, uid, [('name', 'like', 'test_search_order%'), '|', ('active', '=', True), ('active', '=', False)], order="id asc, active desc")
        self.assertEqual([c, d, a, b, ab, e], id_asc_active_desc, "Search with 'ID ASC, ACTIVE DESC' order failed.")
        id_desc_active_asc = partners.search(cr, uid, [('name', 'like', 'test_search_order%'), '|', ('active', '=', True), ('active', '=', False)], order="id desc, active asc")
        self.assertEqual([e, ab, b, a, d, c], id_desc_active_asc, "Search with 'ID DESC, ACTIVE ASC' order failed.")
        id_desc_active_desc = partners.search(cr, uid, [('name', 'like', 'test_search_order%'), '|', ('active', '=', True), ('active', '=', False)], order="id desc, active desc")
        self.assertEqual([e, ab, b, a, d, c], id_desc_active_desc, "Search with 'ID DESC, ACTIVE DESC' order failed.")

<|endoftext|>


<commit_before># python municipal_finance/data_import/audit_reports.py audit_reports.csv
# cat audit_reports.csv|grep -v "Thumbs.db"|cut -d, -f-2  | sort |uniq -c|sort -n|grep -v ' 1 '|cut -d' ' -f 8- > multi
# grep -f multi audit_reports.csv
#
# Multiple audit reports should only be for a municipality with entity(ies). In the case of some metros with many entities there should be an audit report for each entity, one for the parent municipality and one for the consolidated whole.  In the latter case - link to the consolidated audit report.

from bs4 import BeautifulSoup
from requests.adapters import HTTPAdapter
import csv
import re
import requests
import sys
import traceback
import logging

root = 'http://mfma.treasury.gov.za'
fieldnames = ['demarcation_code', 'year', 'url']

class Main(object):
    def __init__(self, csv_file):
            self.writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
            self.writer.writeheader()
            self.session = requests.Session()
            self.session.mount(root, HTTPAdapter(max_retries=5))

    def run(self):
        year_dirs = list(self.expand_dir('/Documents/07.%20Audit%20Reports/', {}))
        year_dirs.reverse()
        for year_dir in year_dirs:
            for category_dir in self.expand_dir(year_dir, {}):
                for muni_dir in self.expand_dir(category_dir, {}):
                    report_paths = [p for p in list(self.expand_dir(muni_dir, {}))
                                    if 'Thumbs.db' not in p]

                    if len(report_paths) == 0:
                        pass
                    elif len(report_paths) == 1:
                        self.writerow(report_paths[0])
                    elif len(report_paths) > 1:
                        self.writerow(muni_dir)

    # /Documents/07.%20Audit%20Reports/2003-04/02.%20Local%20municipalities/EC125%20Buffalo%20City/EC125%20Buffalo%20City%20Audit%20Report%202003-04.pdf
    def writerow(self, report_path):
        try:
            regex = '/Documents/07.%20Audit%20Reports/\d{4}-(\d{2})/[\w.%]+/(\w{3,6})'
            match = re.search(regex, report_path)
            self.writer.writerow({
                'demarcation_code': match.group(2),
                'year': '20' + match.group(1),
                'url': root + report_path,
            })
        except:
            print("\nerror  %s\n" % report_path)
            traceback.print_exc()
            sys.stdout.flush()

    def expand_dir(self, dir, params):
        params['SortField'] = 'LinkFilename'
        params['SortDir'] = 'Asc'
        print("\nDir: %s %r" % (dir, params))
        try:
            r = requests.get(root + dir, params=params)
            r.raise_for_status()
            soup = BeautifulSoup(r.text.encode('utf8', 'replace'), "html.parser")
            for table in soup.find_all('table'):
                if u'url' in table.attrs:
                    child_path = table.attrs[u'url']
                    yield child_path
            next_arrow = soup.find(alt='Next')
            if next_arrow:
                # print
                print("  Current dir=%s" % dir)
                print("  Last child_path=%s" % child_path)
                last_name = child_path.replace(dir + '/', '')
                # print(last_name)
                # print
                for path in self.expand_dir(dir, {'p_FileLeafRef': last_name, 'Paged': 'TRUE'}):
                    yield path
        except:
            print("error", dir, params)
            traceback.print_exc()
            sys.stdout.flush()

if __name__ == "__main__":
    [outfile] = sys.argv[1:]
    with open(outfile, 'w') as csv_file:
        Main(csv_file).run()
<commit_msg>Update parser<commit_after># python municipal_finance/data_import/audit_reports.py audit_reports.csv
# cat audit_reports.csv|grep -v "Thumbs.db"|cut -d, -f-2  | sort |uniq -c|sort -n|grep -v ' 1 '|cut -d' ' -f 8- > multi
# grep -f multi audit_reports.csv
#
# Multiple audit reports should only be for a municipality with entity(ies). In the case of some metros with many entities there should be an audit report for each entity, one for the parent municipality and one for the consolidated whole.  In the latter case - link to the consolidated audit report.

from bs4 import BeautifulSoup
from requests.adapters import HTTPAdapter
import csv
import re
import requests
import sys
import traceback
import urllib.parse

root = 'http://mfma.treasury.gov.za'
fieldnames = ['demarcation_code', 'year', 'url']

class Main(object):
    def __init__(self, csv_file):
            self.writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
            self.writer.writeheader()
            self.session = requests.Session()
            self.session.mount(root, HTTPAdapter(max_retries=5))

    def run(self):
        year_dirs = list(self.expand_dir('/Documents/07.%20Audit%20Reports/', {}))
        year_dirs.reverse()
        for year_dir in year_dirs:
            for category_dir in self.expand_dir(year_dir, {}):
                for muni_dir in self.expand_dir(category_dir, {}):
                    report_paths = [p for p in list(self.expand_dir(muni_dir, {}))
                                    if 'Thumbs.db' not in p]

                    if len(report_paths) == 0:
                        pass
                    elif len(report_paths) == 1:
                        self.writerow(report_paths[0])
                    elif len(report_paths) > 1:
                        querystring = urllib.parse.urlsplit(muni_dir).query
                        directory = urllib.parse.parse_qs(querystring)["RootFolder"][0]
                        self.writerow(directory)

    # /Documents/07. Audit Reports/2003-04/02. Local municipalities/EC125 Buffalo City/EC125 Buffalo City Audit Report 2003-04.pdf
    def writerow(self, report_path):
        try:
            regex = '/Documents/07\. Audit Reports/\d{4}-(\d{2})/[^/]+/(\w{3,6})'
            match = re.search(regex, report_path)
            self.writer.writerow({
                'demarcation_code': match.group(2),
                'year': '20' + match.group(1),
                'url': root + report_path,
            })
        except:
            print("\nerror  %s\n" % report_path)
            traceback.print_exc()
            sys.stdout.flush()

    def expand_dir(self, dir, params):
        params['SortField'] = 'LinkFilename'
        params['SortDir'] = 'Asc'
        print("\nDir: %s %r" % (dir, params))
        try:
            r = requests.get(root + dir, params=params)
            r.raise_for_status()
            soup = BeautifulSoup(r.text.encode('utf8', 'replace'), "html.parser")
            for anchor in soup.select('table[Field="LinkFilename"] a'):
                child_path = anchor.attrs['href']
                yield child_path
            next_arrow = soup.find(alt='Next')
            if next_arrow:
                # print
                print("  Current dir=%s" % dir)
                print("  Last child_path=%s" % child_path)
                last_name = child_path.replace(dir + '/', '')
                # print(last_name)
                # print
                for path in self.expand_dir(dir, {'p_FileLeafRef': last_name, 'Paged': 'TRUE'}):
                    yield path
        except:
            print("error", dir, params)
            traceback.print_exc()
            sys.stdout.flush()

if __name__ == "__main__":
    [outfile] = sys.argv[1:]
    with open(outfile, 'w') as csv_file:
        Main(csv_file).run()
<|endoftext|>