<commit_before>#!/usr/bin/env python

from sys import stdout,stderr,exit, argv
from optparse import OptionParser
from os.path import basename, join
from itertools import combinations
from Bio import SeqIO
from Bio import pairwise2
from bisect import bisect
import logging
import csv

LOG = logging.getLogger(__name__)
LOG.setLevel(logging.DEBUG)
LOG_FILENAME = '%s.log' %basename(argv[0]).rsplit('.py', 1)[0]

MIN_LENGTH_DEFAULT = 0
QUORUM_DEFAULT = 1

def readFastaFiles(seqFiles, mauveFile):
    seqRecords = dict()
    chr_locations = dict()

    for f in seqFiles:
        # assume that fasta file contains a single entry
        ident= basename(f).rsplit('.', 1)[0]

        LOG.info('reading %s with identifier %s' %(f, ident))
        cur_len = 0
        for record in SeqIO.parse(open(f), 'fasta'):
            if ident not in seqRecords:
                seqRecords[ident] = [f, record]
                chr_locations[ident] = list()
            else:
                seqRecords[ident][1] += record
            chr_locations[ident].append((cur_len, record.id))
            cur_len += len(record)

    # put list into order indicated by mauve backbone filename
    seqList = list()
    chr_loc_list  = list()

    for ident in basename(mauveFile).rsplit('.',1)[0].split('_'):
        if ident not in seqRecords:
            LOG.error(('No corresponding fasta file found for identifier %s'
                + ' of backbone %s. Exiting.') %(ident, basename(mauveFile)))
            exit(1)

        chr_loc_list.append(tuple(chr_locations[ident]))
        seqList.append(tuple(seqRecords[ident]))

    return seqList, chr_loc_list

def parseMauveBackbone(seqData, chr_locations, mauveFile, minLength, quorum):

    genomes = [list() for _ in seqData]
    orthologies = list()

    isHeader = True
    outCount = 0

    for line in csv.reader(open(mauveFile), delimiter='\t'):
        if isHeader:
            isHeader = False
            continue

        cur_orth = list()
        seqRecords = [None] * (len(line)/2)
        for i in range(len(line)/2):
            if line[i*2] == '0':
                continue

            ident = basename(seqData[i][0]).rsplit('.', 1)[0]
            orient = int(line[i*2]) >= 0 and '+' or '-'
            start, end = abs(int(line[i*2]))-1, abs(int(line[i*2+1]))

            if end-start <= 0:
                LOG.warning(('sequence length of segment %s[%s:%s] is %s in' + \
                        ' line: \n\t\t%s') %(ident, line[i*2], line[i*2+1],
                            end-start-1, '\t'.join(line)))

            if end-start-1 < minLength:
                continue

            seqRecord = seqData[i][1][start:end]
            if orient == '-':
                seqRecord = seqRecord.reverse_complement()

            x = bisect(chr_locations[i], (start, chr(255)))
            chr_start, chr_name = chr_locations[i][x-1]
            gid = '%s_%s' %(ident, outCount)
            g1i = (gid, start+1-chr_start, end-chr_start, chr_name, orient)
            cur_orth.append((i, g1i))
            seqRecord.id = '%s|%s:%s|chromosome|%s|strand|%s' %g1i
            seqRecord.description = ''
            seqRecords[i] = seqRecord

        if len(cur_orth) >= quorum:
            for i, g1i in cur_orth:
                genomes[i].append((g1i, seqRecords[i]))
            orthologies.extend(combinations(map(lambda x: x[1], cur_orth), 2))
            outCount += 1

    for i in xrange(len(genomes)):
        genomes[i].sort(key=lambda x: (x[0][3], x[0][1], x[0][2]))
        genomes[i] = map(lambda x: x[1], genomes[i])

    LOG.info('Identified %s SBFs larger length %s' %(outCount, minLength))
    return genomes, orthologies

if __name__ == '__main__':

    usage = 'usage: %prog [options] <MAUVE BACKBONE> <FASTA FILE 1> ... <FASTA FILE N>'
    parser = OptionParser(usage=usage)

    parser.add_option('-q', '--quorum', dest='genomeQuorum', help='Minimum' + \
            ' number of genomes that must be covered by each marker ' + \
            '[default=%default]', type=int, default=QUORUM_DEFAULT,
            metavar='INT')

    parser.add_option('-l', '--minlength', dest='minLength',
            help='Miminimum length a segment must have to be included in ' \
                    + 'the output. [default=%default]',
                    type=int, default=MIN_LENGTH_DEFAULT, metavar='INT')

    parser.add_option('-o', '--output_dir', dest='outDir', type=str,
            default='.', help='Output directory [default=%default]')

    (options, args) = parser.parse_args()

    if len(args) < 3:
        parser.print_help()
        exit(1)

    mauveFile = args[0]
    seqFiles = args[1:]

    # setup logging
    ch = logging.StreamHandler(stderr)
    ch.setLevel(logging.ERROR)
    ch.setFormatter(logging.Formatter('!! %(message)s'))

    cf = logging.FileHandler('%s.log' %LOG_FILENAME, mode='w')
    cf.setLevel(logging.INFO)
    cf.setFormatter(logging.Formatter('%(levelname)s\t%(asctime)s\t++ %(message)s'))

    LOG.addHandler(cf)
    LOG.addHandler(ch)

    #
    # main
    #

    LOG.info(('start partitioning fasta sequences according to mauve ' \
            + 'backbone data using minimum segment length %s') %options.minLength)
    seqData, chr_locations = readFastaFiles(seqFiles, mauveFile)
    LOG.info('processing backbone file')
    genomes, _ = parseMauveBackbone(seqData, chr_locations, mauveFile,
            options.minLength, options.genomeQuorum)

    # initialize output handles
    for i in xrange(len(seqData)):
        out = open('%s.gos' %join(options.outDir, basename(seqData[i][0]).rsplit('.',
            1)[0]), 'w')
        SeqIO.write(genomes[i], out, 'fasta')

    LOG.info('finished')

<commit_msg>fixed logfile name<commit_after>#!/usr/bin/env python

from sys import stdout,stderr,exit, argv
from optparse import OptionParser
from os.path import basename, join
from itertools import combinations
from Bio import SeqIO
from Bio import pairwise2
from bisect import bisect
import logging
import csv

LOG = logging.getLogger(__name__)
LOG.setLevel(logging.DEBUG)
LOG_FILENAME = '%s.log' %basename(argv[0]).rsplit('.py', 1)[0]

MIN_LENGTH_DEFAULT = 0
QUORUM_DEFAULT = 1

def readFastaFiles(seqFiles, mauveFile):
    seqRecords = dict()
    chr_locations = dict()

    for f in seqFiles:
        # assume that fasta file contains a single entry
        ident= basename(f).rsplit('.', 1)[0]

        LOG.info('reading %s with identifier %s' %(f, ident))
        cur_len = 0
        for record in SeqIO.parse(open(f), 'fasta'):
            if ident not in seqRecords:
                seqRecords[ident] = [f, record]
                chr_locations[ident] = list()
            else:
                seqRecords[ident][1] += record
            chr_locations[ident].append((cur_len, record.id))
            cur_len += len(record)

    # put list into order indicated by mauve backbone filename
    seqList = list()
    chr_loc_list  = list()

    for ident in basename(mauveFile).rsplit('.',1)[0].split('_'):
        if ident not in seqRecords:
            LOG.error(('No corresponding fasta file found for identifier %s'
                + ' of backbone %s. Exiting.') %(ident, basename(mauveFile)))
            exit(1)

        chr_loc_list.append(tuple(chr_locations[ident]))
        seqList.append(tuple(seqRecords[ident]))

    return seqList, chr_loc_list

def parseMauveBackbone(seqData, chr_locations, mauveFile, minLength, quorum):

    genomes = [list() for _ in seqData]
    orthologies = list()

    isHeader = True
    outCount = 0

    for line in csv.reader(open(mauveFile), delimiter='\t'):
        if isHeader:
            isHeader = False
            continue

        cur_orth = list()
        seqRecords = [None] * (len(line)/2)
        for i in range(len(line)/2):
            if line[i*2] == '0':
                continue

            ident = basename(seqData[i][0]).rsplit('.', 1)[0]
            orient = int(line[i*2]) >= 0 and '+' or '-'
            start, end = abs(int(line[i*2]))-1, abs(int(line[i*2+1]))

            if end-start <= 0:
                LOG.warning(('sequence length of segment %s[%s:%s] is %s in' + \
                        ' line: \n\t\t%s') %(ident, line[i*2], line[i*2+1],
                            end-start-1, '\t'.join(line)))

            if end-start-1 < minLength:
                continue

            seqRecord = seqData[i][1][start:end]
            if orient == '-':
                seqRecord = seqRecord.reverse_complement()

            x = bisect(chr_locations[i], (start, chr(255)))
            chr_start, chr_name = chr_locations[i][x-1]
            gid = '%s_%s' %(ident, outCount)
            g1i = (gid, start+1-chr_start, end-chr_start, chr_name, orient)
            cur_orth.append((i, g1i))
            seqRecord.id = '%s|%s:%s|chromosome|%s|strand|%s' %g1i
            seqRecord.description = ''
            seqRecords[i] = seqRecord

        if len(cur_orth) >= quorum:
            for i, g1i in cur_orth:
                genomes[i].append((g1i, seqRecords[i]))
            orthologies.extend(combinations(map(lambda x: x[1], cur_orth), 2))
            outCount += 1

    for i in xrange(len(genomes)):
        genomes[i].sort(key=lambda x: (x[0][3], x[0][1], x[0][2]))
        genomes[i] = map(lambda x: x[1], genomes[i])

    LOG.info('Identified %s SBFs larger length %s' %(outCount, minLength))
    return genomes, orthologies

if __name__ == '__main__':

    usage = 'usage: %prog [options] <MAUVE BACKBONE> <FASTA FILE 1> ... <FASTA FILE N>'
    parser = OptionParser(usage=usage)

    parser.add_option('-q', '--quorum', dest='genomeQuorum', help='Minimum' + \
            ' number of genomes that must be covered by each marker ' + \
            '[default=%default]', type=int, default=QUORUM_DEFAULT,
            metavar='INT')

    parser.add_option('-l', '--minlength', dest='minLength',
            help='Miminimum length a segment must have to be included in ' \
                    + 'the output. [default=%default]',
                    type=int, default=MIN_LENGTH_DEFAULT, metavar='INT')

    parser.add_option('-o', '--output_dir', dest='outDir', type=str,
            default='.', help='Output directory [default=%default]')

    (options, args) = parser.parse_args()

    if len(args) < 3:
        parser.print_help()
        exit(1)

    mauveFile = args[0]
    seqFiles = args[1:]

    # setup logging
    ch = logging.StreamHandler(stderr)
    ch.setLevel(logging.ERROR)
    ch.setFormatter(logging.Formatter('!! %(message)s'))

    cf = logging.FileHandler(LOG_FILENAME, mode='w')
    cf.setLevel(logging.INFO)
    cf.setFormatter(logging.Formatter('%(levelname)s\t%(asctime)s\t++ %(message)s'))

    LOG.addHandler(cf)
    LOG.addHandler(ch)

    #
    # main
    #

    LOG.info(('start partitioning fasta sequences according to mauve ' \
            + 'backbone data using minimum segment length %s') %options.minLength)
    seqData, chr_locations = readFastaFiles(seqFiles, mauveFile)
    LOG.info('processing backbone file')
    genomes, _ = parseMauveBackbone(seqData, chr_locations, mauveFile,
            options.minLength, options.genomeQuorum)

    # initialize output handles
    for i in xrange(len(seqData)):
        out = open('%s.gos' %join(options.outDir, basename(seqData[i][0]).rsplit('.',
            1)[0]), 'w')
        SeqIO.write(genomes[i], out, 'fasta')

    LOG.info('finished')

<|endoftext|>


<commit_before>import re
import logging
from time import time
from threading import RLock
from itertools import count
from collections import defaultdict
# from unidecode import unidecode

from aleph.core import db
from aleph.util import normalize_strong
from aleph.model import Reference, Entity
from aleph.analyze.analyzer import Analyzer

log = logging.getLogger(__name__)
lock = RLock()
BATCH_SIZE = 10000

class EntityCache(object):

    def __init__(self):
        self.latest = None
        self.matches = {}
        self.regexes = []

    def generate(self):
        with lock:
            self._generate()

    def _generate(self):
        latest = Entity.latest()
        if self.latest is not None and self.latest >= latest:
            return

        self.latest = latest
        self.matches = defaultdict(set)

        for entity in Entity.all():
            for term in entity.terms:
                self.matches[normalize_strong(term)].add(entity.id)

        self.regexes = []
        terms = self.matches.keys()
        terms = [t for t in terms if len(t) > 2]
        for i in count(0):
            terms_slice = terms[i * BATCH_SIZE:(i + 1) * BATCH_SIZE]
            if not len(terms_slice):
                break
            body = '|'.join(terms_slice)
            rex = re.compile('( |^)(%s)( |$)' % body)
            # rex = re.compile('(%s)' % body)
            self.regexes.append(rex)

        log.info('Generating entity tagger: %r (%s terms)',
                 latest, len(terms))

class EntityAnalyzer(Analyzer):

    cache = EntityCache()

    def analyze(self, document, meta):
        begin_time = time()
        self.cache.generate()
        entities = defaultdict(int)
        for text, rec in document.text_parts():
            text = normalize_strong(text)
            if text is None or len(text) <= 2:
                continue
            for rex in self.cache.regexes:
                for match in rex.finditer(text):
                    match = match.group(2)
                    # match = match.group(1)
                    for entity_id in self.cache.matches.get(match, []):
                        entities[entity_id] += 1

        Reference.delete_document(document.id)
        for entity_id, weight in entities.items():
            ref = Reference()
            ref.document_id = document.id
            ref.entity_id = entity_id
            ref.weight = weight
            db.session.add(ref)
        self.save(document, meta)

        duration_time = int((time() - begin_time) * 1000)
        if len(entities):
            log.info("Tagged %r with %d entities (%sms)",
                     document, len(entities), duration_time)
        else:
            log.info("No entities on %r (%sms)",
                     document, len(entities), duration_time)
<commit_msg>outch. <commit_after>import re
import logging
from time import time
from threading import RLock
from itertools import count
from collections import defaultdict
# from unidecode import unidecode

from aleph.core import db
from aleph.util import normalize_strong
from aleph.model import Reference, Entity
from aleph.analyze.analyzer import Analyzer

log = logging.getLogger(__name__)
lock = RLock()
BATCH_SIZE = 10000

class EntityCache(object):

    def __init__(self):
        self.latest = None
        self.matches = {}
        self.regexes = []

    def generate(self):
        with lock:
            self._generate()

    def _generate(self):
        latest = Entity.latest()
        if self.latest is not None and self.latest >= latest:
            return

        self.latest = latest
        self.matches = defaultdict(set)

        for entity in Entity.all():
            for term in entity.terms:
                self.matches[normalize_strong(term)].add(entity.id)

        self.regexes = []
        terms = self.matches.keys()
        terms = [t for t in terms if len(t) > 2]
        for i in count(0):
            terms_slice = terms[i * BATCH_SIZE:(i + 1) * BATCH_SIZE]
            if not len(terms_slice):
                break
            body = '|'.join(terms_slice)
            rex = re.compile('( |^)(%s)( |$)' % body)
            # rex = re.compile('(%s)' % body)
            self.regexes.append(rex)

        log.info('Generating entity tagger: %r (%s terms)',
                 latest, len(terms))

class EntityAnalyzer(Analyzer):

    cache = EntityCache()

    def analyze(self, document, meta):
        begin_time = time()
        self.cache.generate()
        entities = defaultdict(int)
        for text, rec in document.text_parts():
            text = normalize_strong(text)
            if text is None or len(text) <= 2:
                continue
            for rex in self.cache.regexes:
                for match in rex.finditer(text):
                    match = match.group(2)
                    # match = match.group(1)
                    for entity_id in self.cache.matches.get(match, []):
                        entities[entity_id] += 1

        Reference.delete_document(document.id)
        for entity_id, weight in entities.items():
            ref = Reference()
            ref.document_id = document.id
            ref.entity_id = entity_id
            ref.weight = weight
            db.session.add(ref)
        self.save(document, meta)

        duration_time = int((time() - begin_time) * 1000)
        if len(entities):
            log.info("Tagged %r with %d entities (%sms)",
                     document, len(entities), duration_time)
        else:
            log.info("No entities on %r (%sms)",
                     document, duration_time)
<|endoftext|>


<commit_before>#!/usr/bin/env python
##
##  Copyright 2016 SRI International
##  See COPYING file distributed along with the package for the copyright and license terms.
##
"""
Report: Baseline and Year 1 Cases with Scans
============================================
Creates a csv files containing all cases that are included in the study, hanve
not missed their 1 year followup, and have a scanning session that was not
marked as missing

Usage:
python baseline_1yr_cases.py
"""
import os
import sys

import redcap

def get_project(args):
    # First REDCap connection for the Summary project (this is where we put data)
    summary_key_file = open(os.path.join(os.path.expanduser("~"),
                                         '.server_config/redcap-dataentry-token' ),
                                          'r')
    summary_api_key = summary_key_file.read().strip()
    rc_summary = redcap.Project('https://ncanda.sri.com/redcap/api/',
                                summary_api_key,
                                verify_ssl=False)

    # Get all the mri session reports for baseline and 1r
    if args.baseline:
        events = ['baseline_visit_arm_1']
    elif args.yearonefollowup:
        events = ['1y_visit_arm_1']
    else:
        events = ['baseline_visit_arm_1','1y_visit_arm_1']
    mri  = rc_summary.export_records(fields=['study_id', 'exclude',
                                             'visit_ignore___yes', 'mri_missing'],
                                     forms=['mr_session_report', 'visit_date',
                                            'demographics'],
                                     events=events,
                                     format='df')
    return mri

def mri_filter_dataframe(dataframe):
    # Create filters for cases that are included
    case_included = dataframe.exclude != 1 # baseline has 'exclude' in demographics
    visit_included = dataframe.visit_ignore___yes != 1 # Not consistent with 'exclude'
    mri_collected = dataframe.mri_missing != 1

    # Apply filters for results
    included = dataframe[case_included]
    results = included[visit_included & mri_collected]
    return results

def np_filter_dataframe(dataframe):
    # Create filters for cases that are included
    case_included = dataframe.exclude != 1 # subject excluded from NCANDA Study
    visit_included = dataframe.visit_ignore___yes != 1 # subject did not have a valid visit for this event

    # Apply filters for results
    included = dataframe[case_included]
    results = included[visit_included]
    return results

def main(args=None):
    if args.verbose:
        print("Connecting to REDCap...")
    project = get_project(args)
    if args.verbose:
        print("Filtering dataframe...")
    if args.subjectlist:
        with open(args.subjectlist, 'r') as f:
            subject_list = [line.strip() for line in f]
        project = project[project['mri_xnat_sid'].isin(subject_list)]
    if args.mri_cases:
        results = mri_filter_dataframe(project)
    elif args.np_cases:
        results = np_filter_dataframe(project)
    else:
        results = project
    if args.verbose:
        print("Writing results to {}...".format(args.outfile))
    # Write out results
    results.to_csv(os.path.join(args.csvdir, args.outfile),
                   columns=['mri_xnat_sid', 'mri_xnat_eids','mri_datetodvd'])

if __name__ == '__main__':
    import argparse

    formatter = argparse.RawDescriptionHelpFormatter
    default = 'default: %(default)s'
    parser = argparse.ArgumentParser(prog="baseline_1yr_cases.py",
                                     description=__doc__,
                                     formatter_class=formatter)
    parser.add_argument('-b', '--baseline', dest="baseline",
                        help="Select only baseline events", action='store_true')
    #parser.add_argument('-f', '--yearonefollowup', dest="yearonefollowup",
    #                    help="Select only 1y Follow-Up events", action='store_true')
    parser.add_argument( '-c','--csvdir',  action="store", default = '',
                        help="Directory where CSV will be stored.")
    parser.add_argument('-f', '--yearonefollowup', dest="yearonefollowup",
                        help="Select only 1y Follow-Up events", action='store_true')
    parser.add_argument('-o', '--outfile', dest="outfile",
                        help="File to write out. {}".format(default),
                        default='baseline_1yr_cases.csv')
    parser.add_argument('-m', '--mri_cases', dest="mri_cases",
                        help="Generate report for subjects with valid visit & MRI session", action='store_true')
    parser.add_argument('-n', '--np_cases', dest="np_cases",
                        help="Generate report for subjects with valid visit session", action='store_true')
    parser.add_argument('-s', '--subjectlist', dest="subjectlist",
                        help="Text file containing the SIDS for subjects of interest", action='store')
    parser.add_argument('-v', '--verbose', dest="verbose",
                        help="Turn on verbose", action='store_true')
    argv = parser.parse_args()
    sys.exit(main(args=argv))
<commit_msg>UPDATE: Error Reproting<commit_after>#!/usr/bin/env python
##
##  Copyright 2016 SRI International
##  See COPYING file distributed along with the package for the copyright and license terms.
##
"""
Report: Baseline and Year 1 Cases with Scans
============================================
Creates a csv files containing all cases that are included in the study, hanve
not missed their 1 year followup, and have a scanning session that was not
marked as missing

Usage:
python baseline_1yr_cases.py
"""
import os
import sys

import redcap

def get_project(args):
    # First REDCap connection for the Summary project (this is where we put data)
    summary_key_file = open(os.path.join(os.path.expanduser("~"),
                                         '.server_config/redcap-dataentry-token' ),
                                          'r')
    summary_api_key = summary_key_file.read().strip()
    rc_summary = redcap.Project('https://ncanda.sri.com/redcap/api/',
                                summary_api_key,
                                verify_ssl=False)

    # Get all the mri session reports for baseline and 1r
    if args.baseline:
        events = ['baseline_visit_arm_1']
    else:
        events = ['baseline_visit_arm_1','1y_visit_arm_1']
    mri  = rc_summary.export_records(fields=['study_id', 'exclude',
                                             'visit_ignore___yes', 'mri_missing'],
                                     forms=['mr_session_report', 'visit_date',
                                            'demographics'],
                                     events=events,
                                     format='df')
    return mri

def mri_filter_dataframe(dataframe):
    # Create filters for cases that are included
    case_included = dataframe.exclude != 1 # baseline has 'exclude' in demographics
    visit_included = dataframe.visit_ignore___yes != 1 # Not consistent with 'exclude'
    mri_collected = dataframe.mri_missing != 1

    # Apply filters for results
    included = dataframe[case_included]
    results = included[visit_included & mri_collected]
    return results

def np_filter_dataframe(dataframe):
    # Create filters for cases that are included
    case_included = dataframe.exclude != 1 # subject excluded from NCANDA Study
    visit_included = dataframe.visit_ignore___yes != 1 # subject did not have a valid visit for this event

    # Apply filters for results
    included = dataframe[case_included]
    results = included[visit_included]
    return results

def main(args=None):
    if args.verbose:
        print("Connecting to REDCap...")
    project = get_project(args)
    if args.verbose:
        print("Filtering dataframe...")
    if args.subjectlist:
        with open(args.subjectlist, 'r') as f:
            subject_list = [line.strip() for line in f]
        project = project[project['mri_xnat_sid'].isin(subject_list)]
    if args.mri_cases:
        results = mri_filter_dataframe(project)
    elif args.np_cases:
        results = np_filter_dataframe(project)
    else:
        results = project
    if args.verbose:
        print("Writing results to {}...".format(args.outfile))
    # Write out results
    results.to_csv(os.path.join(args.csvdir, args.outfile),
                   columns=['mri_xnat_sid', 'mri_xnat_eids','mri_datetodvd'])

if __name__ == '__main__':
    import argparse

    formatter = argparse.RawDescriptionHelpFormatter
    default = 'default: %(default)s'
    parser = argparse.ArgumentParser(prog="baseline_1yr_cases.py",
                                     description=__doc__,
                                     formatter_class=formatter)
    parser.add_argument('-b', '--baseline', dest="baseline",
                        help="Select only baseline events", action='store_true')
    parser.add_argument( '-c','--csvdir',  action="store", default = '',
                        help="Directory where CSV will be stored.")
    parser.add_argument('-f', '--yearonefollowup', dest="yearonefollowup",
                        help="Select only 1y Follow-Up events", action='store_true')
    parser.add_argument('-o', '--outfile', dest="outfile",
                        help="File to write out. {}".format(default),
                        default='baseline_1yr_cases.csv')
    parser.add_argument('-m', '--mri_cases', dest="mri_cases",
                        help="Generate report for subjects with valid visit & MRI session", action='store_true')
    parser.add_argument('-n', '--np_cases', dest="np_cases",
                        help="Generate report for subjects with valid visit session", action='store_true')
    parser.add_argument('-s', '--subjectlist', dest="subjectlist",
                        help="Text file containing the SIDS for subjects of interest", action='store')
    parser.add_argument('-v', '--verbose', dest="verbose",
                        help="Turn on verbose", action='store_true')
    argv = parser.parse_args()
    sys.exit(main(args=argv))
<|endoftext|>


<commit_before>import os
import sys
import time
import happybase
import numpy as np
from generic_indexer import GenericIndexer
from ..memex_tools.sha1_tools import get_SHA1_from_file, get_SHA1_from_data
from ..memex_tools.binary_file import read_binary_file

class HBaseIndexer(GenericIndexer):

    def read_conf(self):
        """ Reads configuration parameters.

        Will read parameters 'HBI_image_downloader', 'HBI_hasher',
        'HBI_feature_extractor', 'LI_master_update_filepath' and 'LI_base_update_path'
        from self.global_conf.
        """
        self.image_downloader_type = self.global_conf['HBI_image_downloader']
        self.hasher_type = self.global_conf['HBI_hasher']
        self.feature_extractor_type = self.global_conf['HBI_feature_extractor']
        self.hbase_host = self.global_conf['HBI_host']
        self.table_cdrinfos_name = self.global_conf['HBI_table_cdrinfos']
        self.table_sha1infos_name = self.global_conf['HBI_table_sha1infos']
        self.table_updateinfos_name = self.global_conf['HBI_table_updatesinfos']
        self.extractions_types = self.global_conf['HBI_extractions_types']
        self.extractions_columns = self.global_conf['HBI_extractions_columns']
        self.in_url_column = self.global_conf['HBI_in_url_column']
        self.sha1_column = self.global_conf['HBI_sha1_column']
        self.features_dim = self.global_conf["FE_features_dim"]
        self.bits_num = self.global_conf['HA_bits_num']
        if len(self.extractions_columns) != len(self.extractions_types):
            raise ValueError("[HBaseIngester.initialize_source: error] Dimensions mismatch {} vs. {} for extractions_columns vs. extractions_types".format(len(self.extractions_columns),len(self.extractions_types)))
        self.nb_threads = 2
        if 'HBI_pool_thread' in self.global_conf:
            self.nb_threads = self.global_conf['HBI_pool_thread']
        self.pool = happybase.ConnectionPool(size=self.nb_threads,host=self.hbase_host)
        # How to deal with additionals extractions?
        # Where should we store the update infos?
        # Start of batch being indexed should be end criterion for next batch.

    def initialize_indexer_backend(self):
        """ Initialize backend.
        """
        print "[HBaseIndexer: log] initialized with:\n\
                    \t- image_downloader: {}\n\
                    \t- feature_extractor_type: {}\n\
                    \t- hasher_type: {}".format(self.image_downloader_type,
                        self.feature_extractor_type,self.hasher_type)
        # Initialize image_downloader, feature_extractor and hasher
        self.initialize_image_downloader()
        self.initialize_feature_extractor()
        self.initialize_hasher()
        self.db = None

    def initialize_image_downloader(self):
        if self.image_downloader_type=="file_downloader":
            from ..image_downloader.file_downloader import FileDownloader
            self.image_downloader = FileDownloader(self.global_conf_filename)
        else:
            raise ValueError("[HBaseIndexer.initialize_indexer_backend error] Unsupported image_downloader_type: {}.".format(self.image_downloader_type))

    def initialize_feature_extractor(self):
        if self.feature_extractor_type=="sentibank_cmdline":
            from ..feature_extractor.sentibank_cmdline import SentiBankCmdLine
            self.feature_extractor = SentiBankCmdLine(self.global_conf_filename)
        else:
            raise ValueError("[HBaseIndexer.initialize_feature_extractor: error] Unknown feature_extractor_type: {}.".format(self.feature_extractor_type))

    def initialize_hasher(self):
        if self.hasher_type=="hasher_cmdline":
            from ..hasher.hasher_cmdline import HasherCmdLine
            self.hasher = HasherCmdLine(self.global_conf_filename)
        else:
            raise ValueError("[HBaseIndexer.initialize_hasher: error] Unknown hasher_type: {}.".format(self.hasher_type))

    def get_next_batch_start(self):
        """ Get start value for next update batch.

        :returns timestamp_cdrid: First row timestamp_cdrid in 'table_updateinfos_name'.
        """
        with self.pool.connection() as connection:
            table_updateinfos = connection.table(self.table_updateinfos_name)
            for row in table_updateinfos.scan():
                return row[0]

    def get_precomp_from_cdrids(self,list_cdrids,list_type):
        pass
        ## what to do with missing sha1s here?
        #sha1_list = self.get_sha1s_from_cdrids(list_cdrids)
        #res = self.get_precomp_from_sha1(self,sha1_list,list_type)

    def get_full_sha1_rows(self,list_sha1s):
        rows = None
        if list_sha1s:
            with self.pool.connection() as connection:
                table_sha1infos = connection.table(self.table_sha1infos_name)
                rows = table_sha1infos.rows(list_sha1s)
        return rows

    def get_precomp_from_sha1(self,list_sha1s,list_type):
        res = []
        ok_ids = []
        rows = self.get_full_sha1_rows(list_sha1s)
        # check if we have retrieved rows and extractions for each sha1
        retrieved_sha1s = [row[0] for row in rows]
        # building a list of ok_ids and res for each extraction type
        ok_ids = [[]]*len(list_type)
        res = [[]]*len(list_type)
        for i,sha1 in enumerate(retrieved_sha1s):
            for e,extr in enumerate(self.extractions_types):
                #print i,sha1,e,extr
                #print len(ok_ids)
                extr_column = self.extractions_columns[self.extractions_types.index(extr)]
                if extr_column in rows[i][1]:
                    ok_ids[e].append(list_sha1s.index(sha1))
                    res[e].append(rows[i][1][extr_column])
        return res,ok_ids

    def get_new_unique_images(self,sha1_images):
        new_files = []
        new_fulls = []
        old_to_be_merged = []
        # get unique images
        sha1_list = [img_item[-1] for img_item in sha1_images]
        unique_sha1 = sorted(set(sha1_list))
        print "[HBaseIndexer.get_new_unique_images: log] We have {} unique images.".format(len(unique_sha1))
        unique_idx = [sha1_list.index(sha1) for sha1 in unique_sha1]
        full_idx = [unique_sha1.index(sha1) for sha1 in sha1_list]
        # get what we already have indexed in 'table_sha1infos_name'
        res, ok_ids = self.get_precomp_from_sha1(list(unique_sha1),self.extractions_types)
        # check which images already exists and what extractions are present
        for i,sha1 in enumerate(unique_sha1):
            missing_extr = []
            for e,extr in enumerate(self.extractions_types):
                if i not in ok_ids[e]:
                    # we don't have this extraction for that image
                    missing_extr.append(extr)
            all_orig_ids = [k for k,j in enumerate(full_idx) if j==i]
            # we already have all extractions
            if not missing_extr:
                # we should just merge the cdr_ids and obj_parent etc
                old_to_be_merged.extend([sha1_images[k] for k in all_orig_ids])
            else:
                # we need to compute new extractions
                orig_pos = unique_idx[i]
                new_files.append((sha1_images[orig_pos][-2],missing_extr))
                new_fulls.append([sha1_images[k] for k in all_orig_ids])

        return new_files, new_fulls, old_to_be_merged

    def format_for_sha1_infos(self,list_images):
        sha1_infos = []
        unique_sha1 = set()
        # keys are "info:all_cdr_ids", "info:all_parent_ids", "info:all_htids", "info:featnorm_cu", "info:s3_url", "info:hash256_cu"
        for image in list_images:
            tmp = {}
            sha1 = image[-1]
            unique_sha1.add(sha1)
            tmp[sha1] = {}
            tmp[sha1]["info:all_cdr_ids"] = image[0]
            tmp[sha1]["info:all_parent_ids"] = image[4]["info:obj_parent"]
            tmp[sha1]["info:all_htids"] = image[4]["info:crawl_data.image_id"]
            tmp[sha1]["info:s3_url"] = image[1]
            sha1_infos.append(tmp)
        return sha1_infos,unique_sha1

    def group_by_sha1(self,list_images,extractions=None):
        out = dict{}
        for tmp in list_images:
            print tmp
            if "info:featnorm_cu" in tmp:
                print tmp["info:featnorm_cu"]
            if "info:hash256_cu" in tmp:
                print tmp["info:hash256_cu"]
        return out

    def index_batch(self,batch):
        """ Index a batch in the form of a list of (cdr_id,url,[extractions,ts_cdrid,other_data])
        """
        # Download images
        timestr = time.strftime("%b-%d-%Y-%H-%M-%S", time.localtime(time.time()))
        #startid = str(batch[0][0])
        #lastid = str(batch[-1][0])
        #update_id = timestr+'_'+startid+'_'+lastid
        update_id = timestr
        print "[HBaseIndexer.index_batch: log] Starting udpate {}".format(update_id)
        readable_images = self.image_downloader.download_images(batch,update_id)
        #print readable_images
        # Compute sha1
        sha1_images = [img+(get_SHA1_from_file(img[-1]),) for img in readable_images]
        # Now that we have sha1s, check if we actually don't already have all extractions
        new_files, new_fulls, old_to_be_merged = self.get_new_unique_images(sha1_images)
        print "[HBaseIndexer.index_batch: log] new_files: {}".format(new_files)
        print "[HBaseIndexer.index_batch: log] new_fulls: {}".format(new_fulls)
        print "[HBaseIndexer.index_batch: log] old_to_be_merged: {}".format(old_to_be_merged)
        # Compute missing extractions
        new_sb_files = []
        new_files_id = []
        for i,nf_extr in enumerate(new_files):
            nf,extr = nf_extr
            if "sentibank" in extr:
                new_sb_files.append(nf)
                new_files_id.append(new_fulls[i][0])
        if new_sb_files:
            print "[HBaseIndexer.index_batch: log] new_sb_files: {}".format(new_sb_files)
            # Compute features
            features_filename, ins_num = self.feature_extractor.compute_features(new_sb_files, update_id)
            # Compute hashcodes
            hashbits_filepath = self.hasher.compute_hashcodes(features_filename, ins_num, update_id)
            norm_features_filename = features_filename[:-4]+"_norm"
            # read features and hashcodes and pushback for insertion
            print "Initial features at {}, normalized features {} and hashcodes at {}.".format(features_filename,norm_features_filename,hashbits_filepath)
            feats,feats_ok_ids = read_binary_file(norm_features_filename,"feats",new_files_id,self.features_dim*4,np.float32)
            hashcodes,hash_ok_ids = read_binary_file(hashbits_filepath,"hashcodes",self.bits_num/8,np.uint8)
            print "Norm features {}\n Hashcodes {}".format(feats,hashcodes)
            if len(feats_ok_ids)!=len(new_files_id) or len(hash_ok_ids)!=len(new_files_id):
                print "[HBaseIndexer.index_batch: error] Dimensions mismatch. Are we missing features {} vs. {}, or hashcodes {} vs. {}.".format(len(feats_ok_ids),len(new_files_id),len(hash_ok_ids),len(new_files_id))
                return False
        # Need to update self.table_cdrinfos_name and self.table_sha1infos_name
        # in self.table_sha1infos_name,
        # merge "info:crawl_data.image_id", "info:doc_id", "info:obj_parent"
        # into "info:all_htids", "info:all_cdr_ids", "info:all_parent_ids"
        # merge old_to_be_merged, no new extractions just merge ids
        old_sha1_format, unique_sha1 = self.format_for_sha1_infos(old_to_be_merged)
        print "[HBaseIndexer.index_batch: log] old_sha1_format: {}".format(old_sha1_format)
        print "[HBaseIndexer.index_batch: log] unique_sha1: {}".format(unique_sha1)
        # get corresponding rows
        sha1_rows = self.get_full_sha1_rows(unique_sha1)
        print "[HBaseIndexer.index_batch: log] sha1_rows: {}".format(sha1_rows)
        # merge
        sha1_rows_merged = group_by_sha1(self,old_sha1_format.extend(sha1_rows)):
        # push merged old images infos

        # insert new images
        # First, insert sha1 in self.table_cdrinfos_name

        # Then insert sha1 row.

        # Finally udpate self.table_updateinfos_name

<commit_msg>update need to merge new features<commit_after>import os
import sys
import time
import base64
import happybase
import numpy as np
from generic_indexer import GenericIndexer
from ..memex_tools.sha1_tools import get_SHA1_from_file, get_SHA1_from_data
from ..memex_tools.binary_file import read_binary_file

class HBaseIndexer(GenericIndexer):

    def read_conf(self):
        """ Reads configuration parameters.

        Will read parameters 'HBI_image_downloader', 'HBI_hasher',
        'HBI_feature_extractor', 'LI_master_update_filepath' and 'LI_base_update_path'
        from self.global_conf.
        """
        self.image_downloader_type = self.global_conf['HBI_image_downloader']
        self.hasher_type = self.global_conf['HBI_hasher']
        self.feature_extractor_type = self.global_conf['HBI_feature_extractor']
        self.hbase_host = self.global_conf['HBI_host']
        self.table_cdrinfos_name = self.global_conf['HBI_table_cdrinfos']
        self.table_sha1infos_name = self.global_conf['HBI_table_sha1infos']
        self.table_updateinfos_name = self.global_conf['HBI_table_updatesinfos']
        self.extractions_types = self.global_conf['HBI_extractions_types']
        self.extractions_columns = self.global_conf['HBI_extractions_columns']
        self.in_url_column = self.global_conf['HBI_in_url_column']
        self.sha1_column = self.global_conf['HBI_sha1_column']
        self.features_dim = self.global_conf["FE_features_dim"]
        self.bits_num = self.global_conf['HA_bits_num']
        if len(self.extractions_columns) != len(self.extractions_types):
            raise ValueError("[HBaseIngester.initialize_source: error] Dimensions mismatch {} vs. {} for extractions_columns vs. extractions_types".format(len(self.extractions_columns),len(self.extractions_types)))
        self.nb_threads = 2
        if 'HBI_pool_thread' in self.global_conf:
            self.nb_threads = self.global_conf['HBI_pool_thread']
        self.pool = happybase.ConnectionPool(size=self.nb_threads,host=self.hbase_host)
        # How to deal with additionals extractions?
        # Where should we store the update infos?
        # Start of batch being indexed should be end criterion for next batch.

    def initialize_indexer_backend(self):
        """ Initialize backend.
        """
        print "[HBaseIndexer: log] initialized with:\n\
                    \t- image_downloader: {}\n\
                    \t- feature_extractor_type: {}\n\
                    \t- hasher_type: {}".format(self.image_downloader_type,
                        self.feature_extractor_type,self.hasher_type)
        # Initialize image_downloader, feature_extractor and hasher
        self.initialize_image_downloader()
        self.initialize_feature_extractor()
        self.initialize_hasher()
        self.db = None

    def initialize_image_downloader(self):
        if self.image_downloader_type=="file_downloader":
            from ..image_downloader.file_downloader import FileDownloader
            self.image_downloader = FileDownloader(self.global_conf_filename)
        else:
            raise ValueError("[HBaseIndexer.initialize_indexer_backend error] Unsupported image_downloader_type: {}.".format(self.image_downloader_type))

    def initialize_feature_extractor(self):
        if self.feature_extractor_type=="sentibank_cmdline":
            from ..feature_extractor.sentibank_cmdline import SentiBankCmdLine
            self.feature_extractor = SentiBankCmdLine(self.global_conf_filename)
        else:
            raise ValueError("[HBaseIndexer.initialize_feature_extractor: error] Unknown feature_extractor_type: {}.".format(self.feature_extractor_type))

    def initialize_hasher(self):
        if self.hasher_type=="hasher_cmdline":
            from ..hasher.hasher_cmdline import HasherCmdLine
            self.hasher = HasherCmdLine(self.global_conf_filename)
        else:
            raise ValueError("[HBaseIndexer.initialize_hasher: error] Unknown hasher_type: {}.".format(self.hasher_type))

    def get_next_batch_start(self):
        """ Get start value for next update batch.

        :returns timestamp_cdrid: First row timestamp_cdrid in 'table_updateinfos_name'.
        """
        with self.pool.connection() as connection:
            table_updateinfos = connection.table(self.table_updateinfos_name)
            for row in table_updateinfos.scan():
                return row[0]

    def get_precomp_from_cdrids(self,list_cdrids,list_type):
        pass
        ## what to do with missing sha1s here?
        #sha1_list = self.get_sha1s_from_cdrids(list_cdrids)
        #res = self.get_precomp_from_sha1(self,sha1_list,list_type)

    def get_full_sha1_rows(self,list_sha1s):
        rows = None
        if list_sha1s:
            with self.pool.connection() as connection:
                table_sha1infos = connection.table(self.table_sha1infos_name)
                rows = table_sha1infos.rows(list_sha1s)
        return rows

    def get_precomp_from_sha1(self,list_sha1s,list_type):
        res = []
        ok_ids = []
        rows = self.get_full_sha1_rows(list_sha1s)
        # check if we have retrieved rows and extractions for each sha1
        retrieved_sha1s = [row[0] for row in rows]
        # building a list of ok_ids and res for each extraction type
        ok_ids = [[]]*len(list_type)
        res = [[]]*len(list_type)
        for i,sha1 in enumerate(retrieved_sha1s):
            for e,extr in enumerate(self.extractions_types):
                #print i,sha1,e,extr
                #print len(ok_ids)
                extr_column = self.extractions_columns[self.extractions_types.index(extr)]
                if extr_column in rows[i][1]:
                    ok_ids[e].append(list_sha1s.index(sha1))
                    res[e].append(rows[i][1][extr_column])
        return res,ok_ids

    def get_new_unique_images(self,sha1_images):
        new_files = []
        new_fulls = []
        old_to_be_merged = []
        # get unique images
        sha1_list = [img_item[-1] for img_item in sha1_images]
        unique_sha1 = sorted(set(sha1_list))
        print "[HBaseIndexer.get_new_unique_images: log] We have {} unique images.".format(len(unique_sha1))
        unique_idx = [sha1_list.index(sha1) for sha1 in unique_sha1]
        full_idx = [unique_sha1.index(sha1) for sha1 in sha1_list]
        # get what we already have indexed in 'table_sha1infos_name'
        res, ok_ids = self.get_precomp_from_sha1(list(unique_sha1),self.extractions_types)
        # check which images already exists and what extractions are present
        for i,sha1 in enumerate(unique_sha1):
            missing_extr = []
            for e,extr in enumerate(self.extractions_types):
                if i not in ok_ids[e]:
                    # we don't have this extraction for that image
                    missing_extr.append(extr)
            all_orig_ids = [k for k,j in enumerate(full_idx) if j==i]
            # we already have all extractions
            if not missing_extr:
                # we should just merge the cdr_ids and obj_parent etc
                old_to_be_merged.extend([sha1_images[k] for k in all_orig_ids])
            else:
                # we need to compute new extractions
                orig_pos = unique_idx[i]
                new_files.append((sha1_images[orig_pos][-2],missing_extr))
                new_fulls.append([sha1_images[k] for k in all_orig_ids])

        return new_files, new_fulls, old_to_be_merged

    def format_for_sha1_infos(self,list_images):
        sha1_infos = []
        unique_sha1 = set()
        # keys are "info:all_cdr_ids", "info:all_parent_ids", "info:all_htids", "info:featnorm_cu", "info:s3_url", "info:hash256_cu"
        for image in list_images:
            tmp = dict()
            sha1 = image[-1]
            print "{}: {}".format(sha1,image)
            unique_sha1.add(sha1)
            tmp["info:all_cdr_ids"] = image[0]
            tmp["info:all_parent_ids"] = image[2][2]["info:obj_parent"]
            tmp["info:all_htids"] = image[2][2]["info:crawl_data.image_id"]
            tmp["info:s3_url"] = image[1]
            sha1_infos.append((sha1,tmp))
        return sha1_infos,unique_sha1

    def add_extractions_to_row(self,row,extractions):
        return row

    def merge_two_rows(self,row,tmp):
        if row[0]!=tmp[0]:
            raise ValueError("[HBaseIndexer.merge_two_rows: error] tried to merge two rows with different keys {} and {}.".format(row[0],tmp[0]))
        all_keys = set(row[1].keys())
        all_keys.union(set(tmp[1].keys()))
        out = dict()
        for key in all_keys:
            # some values are list and should be merged
            if key in row[1] and key in tmp[1] and key != 'info:s3_url' and key not in self.extractions_columns:
                out[key] = ','.join(list(set(row[1][key].split(',')) | set(tmp[1][key].split(','))))
            # for others a single value should be kept: info:s3_url, and the extractions.
            else:
                if key in row[1]:
                    out[key] = row[1][key]
                elif key in tmp[1]:
                    out[key] = tmp[1][key]
        out_row = (row[0],out)
        return out_row

    def group_by_sha1(self,list_images,extractions=None):
        out = []
        sha1_list = []
        print "[group_by_sha1: log] list_images {}.".format(list_images)
        for tmp in list_images:
            if tmp[0] not in sha1_list:
                sha1_list.append(tmp[0])
                if extractions and tmp[0] in extractions:
                    tmp = self.add_extractions_to_row(row,extractions[tmp[0]])
                out.append(tmp)
            else:
                pos = sha1_list.index(tmp[0])
                row = out[pos]
                # merge
                new_row = self.merge_two_rows(row,tmp)
                if extractions and tmp[0] in extractions:
                    new_row = self.add_extractions_to_row(row,extractions[tmp[0]])
                # update
                out[pos] = new_row
        return out

    def write_batch(self,batch,tab_out_name):
        with self.pool.connection() as connection:
            tab_out = connection.table(tab_out_name)
            batch_write = tab_out.batch()
            print "Pushing batch from {}.".format(batch[0][0])
            for row in batch:
                batch_write.put(row[0],row[1])
        batch_write.send()

    def index_batch(self,batch):
        """ Index a batch in the form of a list of (cdr_id,url,[extractions,ts_cdrid,other_data])
        """
        # Download images
        timestr = time.strftime("%b-%d-%Y-%H-%M-%S", time.localtime(time.time()))
        #startid = str(batch[0][0])
        #lastid = str(batch[-1][0])
        #update_id = timestr+'_'+startid+'_'+lastid
        update_id = timestr
        print "[HBaseIndexer.index_batch: log] Starting udpate {}".format(update_id)
        readable_images = self.image_downloader.download_images(batch,update_id)
        #print readable_images
        # Compute sha1
        sha1_images = [img+(get_SHA1_from_file(img[-1]),) for img in readable_images]
        # Now that we have sha1s, check if we actually don't already have all extractions
        new_files, new_fulls, old_to_be_merged = self.get_new_unique_images(sha1_images)
        print "[HBaseIndexer.index_batch: log] new_files: {}".format(new_files)
        print "[HBaseIndexer.index_batch: log] new_fulls: {}".format(new_fulls)
        print "[HBaseIndexer.index_batch: log] old_to_be_merged: {}".format(old_to_be_merged)
        # Compute missing extractions
        new_sb_files = []
        new_files_id = []
        for i,nf_extr in enumerate(new_files):
            nf,extr = nf_extr
            if "sentibank" in extr and new_fulls[i][0][-1] not in new_files_id:
                new_sb_files.append(nf)
                new_files_id.append(new_fulls[i][0][-1])
        if new_sb_files:
            print "[HBaseIndexer.index_batch: log] new_sb_files: {}".format(new_sb_files)
            print "[HBaseIndexer.index_batch: log] new_files_id: {}".format(new_files_id)
            # Compute features
            features_filename, ins_num = self.feature_extractor.compute_features(new_sb_files, update_id)
            # Compute hashcodes
            hashbits_filepath = self.hasher.compute_hashcodes(features_filename, ins_num, update_id)
            norm_features_filename = features_filename[:-4]+"_norm"
            # read features and hashcodes and pushback for insertion
            print "Initial features at {}, normalized features {} and hashcodes at {}.".format(features_filename,norm_features_filename,hashbits_filepath)
            feats,feats_ok_ids = read_binary_file(norm_features_filename,"feats",new_files_id,self.features_dim*4,np.float32)
            hashcodes,hash_ok_ids = read_binary_file(hashbits_filepath,"hashcodes",new_files_id,self.bits_num/8,np.uint8)
            print "Norm features {}\n Hashcodes {}".format(feats,hashcodes)
            if len(feats_ok_ids)!=len(new_files_id) or len(hash_ok_ids)!=len(new_files_id):
                print "[HBaseIndexer.index_batch: error] Dimensions mismatch. Are we missing features {} vs. {}, or hashcodes {} vs. {}.".format(len(feats_ok_ids),len(new_files_id),len(hash_ok_ids),len(new_files_id))
                return False
            extractions = dict()
            for i,sha1 in enumerate(new_files_id):
                extractions[sha1] = dict()
                sb_col_name = self.extractions_columns[self.extractions_types.index("sentibank")]
                hash_col_name = self.extractions_columns[self.extractions_types.index("hashcode")]
                extractions[sha1][sb_col_name] = base64.b64encode(feats[i])
                extractions[sha1][hash_col_name] = base64.b64encode(hashcodes[i])
        # Need to update self.table_cdrinfos_name and self.table_sha1infos_name
        # in self.table_sha1infos_name,
        # merge "info:crawl_data.image_id", "info:doc_id", "info:obj_parent"
        # into "info:all_htids", "info:all_cdr_ids", "info:all_parent_ids"
        # merge old_to_be_merged, no new extractions just merge ids
        old_sha1_format, unique_sha1 = self.format_for_sha1_infos(old_to_be_merged)
        print "[HBaseIndexer.index_batch: log] old_sha1_format: {}".format(old_sha1_format)
        print "[HBaseIndexer.index_batch: log] unique_sha1: {}".format(unique_sha1)
        # get corresponding rows
        sha1_rows = self.get_full_sha1_rows(unique_sha1)
        #print "[HBaseIndexer.index_batch: log] sha1_rows: {}".format(sha1_rows)
        # merge
        old_sha1_format.extend(sha1_rows)
        sha1_rows_merged = self.group_by_sha1(old_sha1_format)
        # push merged old images infos
        print "[HBaseIndexer.index_batch: log] sha1_rows_merged: {}".format(sha1_rows_merged)
        # insert new images
        print "[HBaseIndexer.index_batch: log] writing batch from {} to table {}.".format(sha1_rows_merged[0][0],self.table_sha1infos_name)
        self.write_batch(sha1_rows_merged,self.table_sha1infos_name)
        # First, insert sha1 in self.table_cdrinfos_name

        # Then insert sha1 row.

        # Finally udpate self.table_updateinfos_name

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.11.11 on 2018-04-17 11:35
from __future__ import unicode_literals

import copy

from django.db import migrations
from django.db.models import F
from fluent.syntax import FluentParser, FluentSerializer, ast
from pontoon.base import utils

parser = FluentParser()
serializer = FluentSerializer()

def adjust_stats(self, total_strings_diff, approved_strings_diff,
                 fuzzy_strings_diff, translated_strings_diff):
    self.total_strings = F('total_strings') + total_strings_diff
    self.approved_strings = F('approved_strings') + approved_strings_diff
    self.fuzzy_strings = F('fuzzy_strings') + fuzzy_strings_diff
    self.translated_strings = F('translated_strings') + translated_strings_diff

    self.save(update_fields=[
        'total_strings', 'approved_strings',
        'fuzzy_strings', 'translated_strings'
    ])

def calculate_stats(self, Translation, Entity, ProjectLocale):
    """Update stats, including denormalized ones."""
    resource = self.resource
    locale = self.locale
    project = resource.project

    entity_ids = Translation.objects.filter(locale=locale).values('entity')
    translated_entities = Entity.objects.filter(
        pk__in=entity_ids, resource=resource, obsolete=False)

    # Singular
    translations = Translation.objects.filter(
        entity__in=translated_entities.filter(string_plural=''), locale=locale)
    approved = translations.filter(approved=True).count()
    fuzzy = translations.filter(fuzzy=True).count()

    # Plural
    nplurals = len(map(int, locale.cldr_plurals.split(','))) or 1
    missing = 0
    for e in translated_entities.exclude(string_plural=''):
        translations = Translation.objects.filter(entity=e, locale=locale)
        plural_approved_count = translations.filter(approved=True).count()
        plural_fuzzy_count = translations.filter(fuzzy=True).count()
        plural_translated_count = translations.values('plural_form').distinct().count()

        if plural_approved_count == nplurals:
            approved += 1
        elif plural_fuzzy_count == nplurals:
            fuzzy += 1
        elif plural_translated_count < nplurals:
            missing += 1

    translated = max(translated_entities.count() - approved - fuzzy - missing, 0)

    # Calculate diffs to reduce DB queries
    total_strings_diff = resource.total_strings - self.total_strings
    approved_strings_diff = approved - self.approved_strings
    fuzzy_strings_diff = fuzzy - self.fuzzy_strings
    translated_strings_diff = translated - self.translated_strings

    # Translated Resource
    adjust_stats(
        self,
        total_strings_diff, approved_strings_diff,
        fuzzy_strings_diff, translated_strings_diff
    )

    # Project
    adjust_stats(
        project,
        total_strings_diff, approved_strings_diff,
        fuzzy_strings_diff, translated_strings_diff
    )

    # Locale
    adjust_stats(
        locale,
        total_strings_diff, approved_strings_diff,
        fuzzy_strings_diff, translated_strings_diff
    )

    # ProjectLocale
    project_locale = utils.get_object_or_none(
        ProjectLocale,
        project=project,
        locale=locale
    )
    if project_locale:
        adjust_stats(
            project_locale,
            total_strings_diff, approved_strings_diff,
            fuzzy_strings_diff, translated_strings_diff
        )

def update_plurals(apps, schema_editor):
    Locale = apps.get_model('base', 'Locale')
    Translation = apps.get_model('base', 'Translation')
    TranslatedResource = apps.get_model('base', 'TranslatedResource')
    ChangedEntityLocale = apps.get_model('base', 'ChangedEntityLocale')
    Entity = apps.get_model('base', 'Entity')
    ProjectLocale = apps.get_model('base', 'ProjectLocale')

    for code, data in LOCALES.items():
        locale = Locale.objects.get(code=code)

        # print ""
        # print locale.code
        # print "---------"

        # Cache for later use
        cldr_plurals_old = locale.cldr_plurals

        """
        1. Update Locale data
        """
        locale.cldr_plurals = data['cldr_plurals']
        locale.plural_rule = data['plural_rule']
        locale.save()

        translations = Translation.objects.filter(locale=locale, approved=True)

        """
        2. Update Gettext Translations
        """
        gettext_translations = translations.filter(
            entity__resource__format='po',
            plural_form__isnull=False,
        )

        # Update Translation.plural_form. It represents an index of
        # Locale.cldr_plurals_list(), which we're changing. Prefetch QuerySets
        # to prevent cycled updates.
        cldr_ids_old = map(int, cldr_plurals_old.split(','))
        cldr_ids = map(int, locale.cldr_plurals.split(','))

        translation_indexes = {}

        for index, plural_form in enumerate(cldr_ids_old):
            index_new = cldr_ids.index(plural_form)
            translation_indexes[index_new] = (
                list(gettext_translations.filter(plural_form=index).values_list('pk', flat=True))
            )

        for index_new, translation_pks in translation_indexes.items():
            Translation.objects.filter(
                pk__in=translation_pks
            ).update(
                plural_form=index_new
            )

        # Add suggestions to newly created plural forms, copied from the
        # approved plural forms with the highest index (incl. metadata).
        forms_all = range(0, len(cldr_ids))
        used_indexes = translation_indexes.keys()
        forms_new = [item for item in forms_all if item not in used_indexes]

        gettext_templates = gettext_translations.filter(
            plural_form=index_new  # Highest index
        )
        changed_entity_ids = gettext_templates.values_list(
            'entity__id',
            flat=True,
        ).distinct()

        gettext_translations_new = [
            Translation(
                entity=gettext_template.entity,
                locale=gettext_template.locale,
                plural_form=form_new,
                string=gettext_template.string,
                user=gettext_template.user,
                date=gettext_template.date,
            ) for form_new in forms_new
            for gettext_template in gettext_templates
        ]

        Translation.objects.bulk_create(gettext_translations_new)

        # Update stats
        translated_resources = TranslatedResource.objects.filter(
            locale__code=code,
            resource__in=gettext_templates.values('entity__resource'),
        )
        for tr in translated_resources:
            calculate_stats(tr, Translation, Entity, ProjectLocale)

        # Mark entities as changed, for later sync
        changed_entities = [
            ChangedEntityLocale(
                entity_id=changed_entity_id,
                locale=locale,
            ) for changed_entity_id in changed_entity_ids
        ]

        # print "Changed Gettext entities: " + str(list(changed_entity_ids))

        ChangedEntityLocale.objects.bulk_create(changed_entities)

        """
        3. Update Fluent Translations
        """
        CLDR_PLURALS = ['zero', 'one', 'two', 'few', 'many', 'other']
        locale_cldr_plurals = [CLDR_PLURALS[i] for i in cldr_ids]

        fluent_translation_candidates = translations.filter(
            entity__resource__format='ftl',
            # We need python to find pluralized FTL strings,
            # but we can narrow down the list of candidates
            # to only include selectors
            string__contains='->',
        )

        changed_fluent_entities = []

        for t in fluent_translation_candidates:
            translation_ast = parser.parse_entry(t.string)
            elements = []

            if translation_ast.value:
                elements += translation_ast.value.elements

            for attribute in translation_ast.attributes:
                if attribute.value:
                    elements += attribute.value.elements

            for element in elements:
                # Skip non-selector elements
                if not (
                    isinstance(element, ast.Placeable) and
                    hasattr(element.expression, 'variants')
                ):
                    continue

                # Keys of all variants of plural elements are either
                # CLDR plurals or numbers
                if all(
                    isinstance(variant.key, ast.NumberLiteral) or
                    (
                        isinstance(variant.key, ast.VariantName) and
                        variant.key.name in CLDR_PLURALS
                    )
                    for variant in element.expression.variants
                ):
                    # Populate missing plural forms
                    existing_plural_variants = [
                        variant
                        for variant in element.expression.variants
                        if isinstance(variant.key, ast.VariantName)
                    ]

                    for cldr_plural in locale_cldr_plurals:
                        numeric_variants = [x.key.value for x in element.expression.variants if isinstance(x.key, ast.NumberLiteral)]
                        # Skip 'zero' if 0 exists
                        if cldr_plural == 'zero' and '0' in numeric_variants:
                            continue

                        # Skip 'one' if 1 exists
                        if cldr_plural == 'one' and '1' in numeric_variants:
                            continue

                        # Skip 'two' if 2 exists
                        if cldr_plural == 'two' and '2' in numeric_variants:
                            continue

                        if existing_plural_variants and cldr_plural not in [variant.key.name for variant in existing_plural_variants]:
                            new_variant = copy.deepcopy(existing_plural_variants[-1])
                            new_variant.key.name = cldr_plural
                            new_variant.default = False
                            element.expression.variants.append(new_variant)

                    # Sort variants
                    element.expression.variants = sorted(
                        element.expression.variants,
                        key=lambda x: (['0'] + ['1'] + CLDR_PLURALS).index(x.key.name if hasattr(x.key, 'name') else x.key.value)
                    )

                    new_string = serializer.serialize_entry(translation_ast)

                    if t.string != new_string:
                        t.string = new_string
                        t.approved = False

                        # Does not run the custom Translation.save() method
                        t.save()

                        # Update stats
                        tr, _ = TranslatedResource.objects.get_or_create(
                            resource_id=t.entity.resource.id, locale__code=code
                        )
                        calculate_stats(tr, Translation, Entity, ProjectLocale)

                        # Mark entity as changed
                        ChangedEntityLocale.objects.get_or_create(entity_id=t.entity.id, locale_id=locale.id)

                        changed_fluent_entities.append(t.entity.pk)

        # print "Changed Fluent entities: " + str(changed_fluent_entities)

class Migration(migrations.Migration):

    dependencies = [
        ('base', '0119_plural_rule_max_length_1639'),
    ]

    operations = [
        migrations.RunPython(
            update_plurals,
            migrations.RunPython.noop,
        ),
    ]

LOCALES = {
    'ka': {
        'cldr_plurals': '1,5',
        'plural_rule': '(n != 1)',
    },
    'fa': {
        'cldr_plurals': '1,5',
        'plural_rule': '(n > 1)',
    },
    'tr': {
        'cldr_plurals': '1,5',
        'plural_rule': '(n != 1)',
    },
    'cy': {
        'cldr_plurals': '0,1,2,3,4,5',
        'plural_rule': '(n == 0) ? 0 : ((n == 1) ? 1 : ((n == 2) ? 2 : ((n == 3) ? 3 : ((n == 6) ? 4 : 5))))',
    },
    'uz': {
        'cldr_plurals': '1,5',
        'plural_rule': '(n != 1)',
    },
    'br': {
        'cldr_plurals': '1,2,3,4,5',
        'plural_rule': '(n % 10 == 1 && n % 100 != 11 && n % 100 != 71 && n % 100 != 91) ? 0 : ((n % 10 == 2 && n % 100 != 12 && n % 100 != 72 && n % 100 != 92) ? 1 : ((((n % 10 == 3 || n % 10 == 4) || n % 10 == 9) && (n % 100 < 10 || n % 100 > 19) && (n % 100 < 70 || n % 100 > 79) && (n % 100 < 90 || n % 100 > 99)) ? 2 : ((n != 0 && n % 1000000 == 0) ? 3 : 4)))',
    },
}
<commit_msg>Remove migration 0120 to avoid Fluent compatibility issues. (#1119)<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.11.11 on 2018-04-17 11:35

from django.db import migrations

# The previous version of this migration has been removed to avoid failures in the future.
# More info: https://github.com/mozilla/pontoon/pull/1119

class Migration(migrations.Migration):

    dependencies = [
        ('base', '0119_plural_rule_max_length_1639'),
    ]

    operations = []
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function, unicode_literals

from datetime import date
from dateutil.parser import parse
import re
import sys
import uuid

from amaascore.assets.children import Link
from amaascore.core.amaas_model import AMaaSModel
from amaascore.core.comment import Comment
from amaascore.core.reference import Reference

# This extremely ugly hack is due to the whole Python 2 vs 3 debacle.
type_check = str if sys.version_info >= (3, 0, 0) else (str, unicode)

class Asset(AMaaSModel):

    @staticmethod
    def children():
        return {'comments': Comment, 'links': Link, 'references': Reference}

    def __init__(self, asset_manager_id, fungible, asset_issuer_id=None, asset_id=None, asset_status='Active',
                 country_id=None, venue_id=None, currency=None, issue_date=date.min,
                 roll_price=True, display_name='', description='',
                 comments=None, links=None, references=None, client_additional=None,
                 *args, **kwargs):
        self.asset_manager_id = asset_manager_id
        self.asset_id = asset_id or uuid.uuid4().hex
        if not hasattr(self, 'asset_class'):  # A more specific child class may have already set this
            self.asset_class = 'Asset'
        # This can be overridden if necessary but I think it probably shouldn't
        self.asset_type_display = ' '.join(re.findall('[A-Z][^A-Z]*', self.__class__.__name__))
        self.asset_type = self.__class__.__name__
        self.fungible = fungible
        self.asset_issuer_id = asset_issuer_id
        self.asset_status = asset_status
        self.country_id = country_id
        self.venue_id = venue_id
        self.currency = currency
        self.issue_date = issue_date
        self.roll_price = roll_price
        self.display_name = display_name
        self.description = description
        self.client_additional = client_additional  # A field to allow people to build their own assets
        # Defaults are here not in constructor for mutability reasons.
        self.comments = comments.copy() if comments else {}
        self.links = links.copy() if links else {}
        self.references = references.copy() if references else {}
        self.references['AMaaS'] = Reference(reference_value=self.asset_id)  # Upserts the AMaaS Reference
        super(Asset, self).__init__(*args, **kwargs)

    def reference_types(self):
        """
        TODO - are these helper functions useful?
        :return:
        """
        return self.references.keys()

    @property
    def issue_date(self):
        if hasattr(self, '_issue_date'):
            return self._issue_date

    @issue_date.setter
    def issue_date(self, value):
        """
        The date on which the asset is issued
        :param value:
        :return:
        """
        if value:
            self._issue_date = parse(value).date() if isinstance(value, type_check) else value

    @property
    def maturity_date(self):
        if hasattr(self, '_maturity_date'):
            return self._maturity_date

    @maturity_date.setter
    def maturity_date(self, value):
        """
        The date on which the asset matures and no longer holds value
        :param value:
        :return:
        """
        if value:
            self._maturity_date = parse(value).date() if isinstance(value, type_check) else value

    def __str__(self):
        return "Asset object - ID: %s" % self.asset_id

    def get_country_codes(self):
        return [self.country_id]

    def get_country_and_exchange_codes(self):
        """
        This function returns dictionary in the format of {"country_codes": [XYZ, ABC], "venue_id": [ABC]}
        Where country_codes = self.country_id and venue_id = self.venue_id
        except for FX where we also return the country_ids associated with the legs of the FX transaction
        (e.g. USDJPY returns USA + JPN).
        Assuming country_codes and venue_id attributes' values are of list data type
        """
        codes = {'country_codes': self.get_country_codes(), 'venue_id': self.venue_id}
        return codes
<commit_msg>rename get_country_and_exchange_codes to “country_and_venue_codes”<commit_after>from __future__ import absolute_import, division, print_function, unicode_literals

from datetime import date
from dateutil.parser import parse
import re
import sys
import uuid

from amaascore.assets.children import Link
from amaascore.core.amaas_model import AMaaSModel
from amaascore.core.comment import Comment
from amaascore.core.reference import Reference

# This extremely ugly hack is due to the whole Python 2 vs 3 debacle.
type_check = str if sys.version_info >= (3, 0, 0) else (str, unicode)

class Asset(AMaaSModel):

    @staticmethod
    def children():
        return {'comments': Comment, 'links': Link, 'references': Reference}

    def __init__(self, asset_manager_id, fungible, asset_issuer_id=None, asset_id=None, asset_status='Active',
                 country_id=None, venue_id=None, currency=None, issue_date=date.min,
                 roll_price=True, display_name='', description='',
                 comments=None, links=None, references=None, client_additional=None,
                 *args, **kwargs):
        self.asset_manager_id = asset_manager_id
        self.asset_id = asset_id or uuid.uuid4().hex
        if not hasattr(self, 'asset_class'):  # A more specific child class may have already set this
            self.asset_class = 'Asset'
        # This can be overridden if necessary but I think it probably shouldn't
        self.asset_type_display = ' '.join(re.findall('[A-Z][^A-Z]*', self.__class__.__name__))
        self.asset_type = self.__class__.__name__
        self.fungible = fungible
        self.asset_issuer_id = asset_issuer_id
        self.asset_status = asset_status
        self.country_id = country_id
        self.venue_id = venue_id
        self.currency = currency
        self.issue_date = issue_date
        self.roll_price = roll_price
        self.display_name = display_name
        self.description = description
        self.client_additional = client_additional  # A field to allow people to build their own assets
        # Defaults are here not in constructor for mutability reasons.
        self.comments = comments.copy() if comments else {}
        self.links = links.copy() if links else {}
        self.references = references.copy() if references else {}
        self.references['AMaaS'] = Reference(reference_value=self.asset_id)  # Upserts the AMaaS Reference
        super(Asset, self).__init__(*args, **kwargs)

    def reference_types(self):
        """
        TODO - are these helper functions useful?
        :return:
        """
        return self.references.keys()

    @property
    def issue_date(self):
        if hasattr(self, '_issue_date'):
            return self._issue_date

    @issue_date.setter
    def issue_date(self, value):
        """
        The date on which the asset is issued
        :param value:
        :return:
        """
        if value:
            self._issue_date = parse(value).date() if isinstance(value, type_check) else value

    @property
    def maturity_date(self):
        if hasattr(self, '_maturity_date'):
            return self._maturity_date

    @maturity_date.setter
    def maturity_date(self, value):
        """
        The date on which the asset matures and no longer holds value
        :param value:
        :return:
        """
        if value:
            self._maturity_date = parse(value).date() if isinstance(value, type_check) else value

    def __str__(self):
        return "Asset object - ID: %s" % self.asset_id

    def get_country_codes(self):
        return [self.country_id]

    def country_and_venue_codes(self):
        """
        This function returns dictionary in the format of {"country_codes": [XYZ, ABC], "venue_id": [ABC]}
        Where country_codes = self.country_id and venue_id = self.venue_id
        except for FX where we also return the country_ids associated with the legs of the FX transaction
        (e.g. USDJPY returns USA + JPN).
        Assuming country_codes and venue_id attributes' values are of list data type
        """
        codes = {'country_codes': self.get_country_codes(), 'venue_id': self.venue_id}
        return codes
<|endoftext|>


<commit_before># To change this license header, choose License Headers in Project Properties.
# To change this template file, choose Tools | Templates
# and open the template in the editor.

__author__="John Kirkham <kirkhamj@janelia.hhmi.org>"
__date__ ="$Apr 14, 2014 5:42:07 PM$"

import logging
import functools

# Nothing fancy. Just the basic logging unless otherwise specified, in which case this does nothing.
logging.basicConfig(level=logging.DEBUG)

def log_call(logger, print_args = False):
    """
        Takes a given logger and uses it to log entering and leaving the decorated callable.

        Intended to be used as a decorator that takes a few arguments.

        Args:
            logger      (Logger):    Used for logging entry, exit and possibly arguments.

        Keyword Args:
            print_args  (bool):      Whether to output the arguments and keyword arguments passed to the function.
                                     This should not automatically be true as some arguments may not be printable or
                                     may be expensive to print. Thus, it should be up to the developer to use their
                                     own discretion. Further, we don't want to break their existing code.

        Returns:
            log_call_decorator (for wrapping)
    """

    def log_call_decorator(callable):
        """
            The actual decorator, which is what is returned to decorate the callable in question.

            Args:
                callable:    Anything that can be called function, method, functor, lambda, etc.

            Returns:
		log_call_callable_wrapped, which is wrapped around the function in question.
        """

        @functools.wraps(callable)
        def log_call_callable_wrapped(*args, **kwargs):
            """
                This is what will replace the orignal callable. It will behave the same except it will now log its
                entrance and exit. If set, it will log its parameters too.

                Args:
                    callable:    Anything that can be called function, method, functor, lambda, etc.

                Returns:
                    log_call_callable_wrapped, which is wrapped around the function in question.
            """

            # Log that we have entered the callable in question.
            logger.debug("Entering callable: \"" + callable.__name__ + "\".")

            # Output arguments and keyword arguments if acceptable. Note that this allows keyword arguments to be turned on or off at runtime.
            #
            # Note: We have used log_call_callable_wrapped.print_args. However, we cannot define this until after as wrapping will lose this variable.
            if (log_call_callable_wrapped.print_args):
               logger.debug("Called with the arguments: \"" + str(args) + "\" and with the keyword arguments: \"" + str(kwargs) + "\".")

            # We don't return immediately. Why? We want to know if this succeeded or failed.
            # So, we want the log message below to print after the function runs.
            result = callable(*args, **kwargs)

            # Log that we have exited the callable in question.
            logger.debug("Exiting " + callable.__name__ + ".")

            # Return the result even None.
            return(result)

        # Store the underlying callable. Automatic in Python 3.
        log_call_callable_wrapped.__wrapped__ = callable

        # Copy over the define value of print_args for later use.
        # Must be defined afterwards as functools.wraps will not copy it over to the wrapped instance.
        log_call_callable_wrapped.print_args = print_args

        # The callable wrapped.
        return(log_call_callable_wrapped)

    # The arguments passed to the decorator for easy access.
    return(log_call_decorator)<commit_msg>advanced_debugging: Fixed missing quotes on the callable name when printing the exiting line.<commit_after># To change this license header, choose License Headers in Project Properties.
# To change this template file, choose Tools | Templates
# and open the template in the editor.

__author__="John Kirkham <kirkhamj@janelia.hhmi.org>"
__date__ ="$Apr 14, 2014 5:42:07 PM$"

import logging
import functools

# Nothing fancy. Just the basic logging unless otherwise specified, in which case this does nothing.
logging.basicConfig(level=logging.DEBUG)

def log_call(logger, print_args = False):
    """
        Takes a given logger and uses it to log entering and leaving the decorated callable.

        Intended to be used as a decorator that takes a few arguments.

        Args:
            logger      (Logger):    Used for logging entry, exit and possibly arguments.

        Keyword Args:
            print_args  (bool):      Whether to output the arguments and keyword arguments passed to the function.
                                     This should not automatically be true as some arguments may not be printable or
                                     may be expensive to print. Thus, it should be up to the developer to use their
                                     own discretion. Further, we don't want to break their existing code.

        Returns:
            log_call_decorator (for wrapping)
    """

    def log_call_decorator(callable):
        """
            The actual decorator, which is what is returned to decorate the callable in question.

            Args:
                callable:    Anything that can be called function, method, functor, lambda, etc.

            Returns:
		log_call_callable_wrapped, which is wrapped around the function in question.
        """

        @functools.wraps(callable)
        def log_call_callable_wrapped(*args, **kwargs):
            """
                This is what will replace the orignal callable. It will behave the same except it will now log its
                entrance and exit. If set, it will log its parameters too.

                Args:
                    callable:    Anything that can be called function, method, functor, lambda, etc.

                Returns:
                    log_call_callable_wrapped, which is wrapped around the function in question.
            """

            # Log that we have entered the callable in question.
            logger.debug("Entering callable: \"" + callable.__name__ + "\".")

            # Output arguments and keyword arguments if acceptable. Note that this allows keyword arguments to be turned on or off at runtime.
            #
            # Note: We have used log_call_callable_wrapped.print_args. However, we cannot define this until after as wrapping will lose this variable.
            if (log_call_callable_wrapped.print_args):
               logger.debug("Called with the arguments: \"" + str(args) + "\" and with the keyword arguments: \"" + str(kwargs) + "\".")

            # We don't return immediately. Why? We want to know if this succeeded or failed.
            # So, we want the log message below to print after the function runs.
            result = callable(*args, **kwargs)

            # Log that we have exited the callable in question.
            logger.debug("Exiting \"" + callable.__name__ + "\".")

            # Return the result even None.
            return(result)

        # Store the underlying callable. Automatic in Python 3.
        log_call_callable_wrapped.__wrapped__ = callable

        # Copy over the define value of print_args for later use.
        # Must be defined afterwards as functools.wraps will not copy it over to the wrapped instance.
        log_call_callable_wrapped.print_args = print_args

        # The callable wrapped.
        return(log_call_callable_wrapped)

    # The arguments passed to the decorator for easy access.
    return(log_call_decorator)<|endoftext|>


<commit_before>from custom.icds_reports.models.views import BiharDemographicsView, BiharVaccineView, BiharAPIMotherView
from custom.icds_reports.const import CAS_API_PAGE_SIZE
from custom.icds_reports.cache import icds_quickcache
from dateutil.relativedelta import relativedelta
from dimagi.utils.dates import force_to_date

# cache for 2 hours because it wont change atleast for 24 hours.
# This API will be hit in a loop of something and they should be able to scrape all
# the records in 2 hours.
@icds_quickcache(['model_classname', 'month', 'state_id'], timeout=60 * 60 * 2)
def get_total_records_count(model_classname, month, state_id, month_end_11yr=None,
                            month_start_14yr=None):
    classes = {
        BiharDemographicsView.__name__: BiharDemographicsView,
        BiharAPIMotherView.__name__: BiharAPIMotherView,
        BiharVaccineView.__name__: BiharVaccineView,
    }
    if month_start_14yr is None:
        return classes[model_classname].objects.filter(
            month=month,
            state_id=state_id
        ).count()
    else:
        return classes[model_classname].objects.filter(
            month=month,
            state_id=state_id,
            dob__lt=month_end_11yr,
            dob__gte=month_start_14yr,
            gender='F',
        ).count()

def get_api_demographics_data(month, state_id, last_person_case_id):
    demographics_data_query = BiharDemographicsView.objects.filter(
        month=month,
        state_id=state_id,
        person_id__gt=last_person_case_id
    ).order_by('person_id').values(
        'state_name',
        'state_site_code',
        'district_name',
        'district_site_code',
        'block_name',
        'block_site_code',
        'supervisor_name',
        'supervisor_site_code',
        'awc_name',
        'awc_site_code',
        'ward_number',
        'household_id',
        'household_name',
        'hh_reg_date',
        'hh_num',
        'hh_gps_location',
        'hh_caste',
        'hh_bpl_apl',
        'hh_minority',
        'hh_religion',
        'hh_member_number',
        'person_id',
        'person_name',
        'has_adhaar',
        'bank_account_number',
        'ifsc_code',
        'age_at_reg',
        'dob',
        'gender',
        'blood_group',
        'disabled',
        'disability_type',
        'referral_status',
        'migration_status',
        'resident',
        'registered_status',
        'rch_id',
        'mcts_id',
        'phone_number',
        'date_death',
        'site_death',
        'closed_on',
        'reason_closure'
    )

    # To apply pagination on database query with data size length
    limited_demographics_data = list(demographics_data_query[:CAS_API_PAGE_SIZE])
    return limited_demographics_data, get_total_records_count(BiharDemographicsView.__name__, month, state_id)

def get_mother_details(month, state_id, last_ccs_case_id):
    bihar_mother_details = BiharAPIMotherView.objects.filter(
        month=month,
        state_id=state_id,
        ccs_case_id__gt=last_ccs_case_id
    ).order_by('ccs_case_id').values(
        'household_id',
        'ccs_case_id',
        'person_id',
        'married',
        'husband_name',
        'husband_id',
        'last_preg_year',
        'last_preg_tt',
        'is_pregnant',
        'preg_reg_date',
        'tt_1',
        'tt_2',
        'tt_booster',
        'add',
        'hb'
    )
    limited_mother_details_data = list(bihar_mother_details[:CAS_API_PAGE_SIZE])
    return limited_mother_details_data, get_total_records_count(BiharAPIMotherView.__name__, month, state_id)

def get_api_vaccine_data(month, state_id, last_person_case_id):
    vaccine_data_query = BiharVaccineView.objects.filter(
        month=month,
        state_id=state_id,
        person_id__gt=last_person_case_id
    ).order_by('person_id').values(
        'month',
        'person_id',
        'time_birth',
        'child_alive',
        'father_name',
        'mother_name',
        'father_id',
        'mother_id',
        'dob',
        'private_admit',
        'primary_admit',
        'date_last_private_admit',
        'date_return_private',
        'due_list_date_1g_dpt_1',
        'due_list_date_2g_dpt_2',
        'due_list_date_3g_dpt_3',
        'due_list_date_5g_dpt_booster',
        'due_list_date_7gdpt_booster_2',
        'due_list_date_0g_hep_b_0',
        'due_list_date_1g_hep_b_1',
        'due_list_date_2g_hep_b_2',
        'due_list_date_3g_hep_b_3',
        'due_list_date_3g_ipv',
        'due_list_date_4g_je_1',
        'due_list_date_5g_je_2',
        'due_list_date_5g_measles_booster',
        'due_list_date_4g_measles',
        'due_list_date_0g_opv_0',
        'due_list_date_1g_opv_1',
        'due_list_date_2g_opv_2',
        'due_list_date_3g_opv_3',
        'due_list_date_5g_opv_booster',
        'due_list_date_1g_penta_1',
        'due_list_date_2g_penta_2',
        'due_list_date_3g_penta_3',
        'due_list_date_1g_rv_1',
        'due_list_date_2g_rv_2',
        'due_list_date_3g_rv_3',
        'due_list_date_4g_vit_a_1',
        'due_list_date_5g_vit_a_2',
        'due_list_date_6g_vit_a_3',
        'due_list_date_6g_vit_a_4',
        'due_list_date_6g_vit_a_5',
        'due_list_date_6g_vit_a_6',
        'due_list_date_6g_vit_a_7',
        'due_list_date_6g_vit_a_8',
        'due_list_date_7g_vit_a_9',
        'due_list_date_1g_bcg'
    )

    # To apply pagination on database query with data size length
    limited_vaccine_data = list(vaccine_data_query[:CAS_API_PAGE_SIZE])
    return limited_vaccine_data, get_total_records_count(BiharVaccineView.__name__, month, state_id)

def get_api_ag_school_data(month, state_id, last_person_case_id):
    month_start = force_to_date(month).replace(day=1)
    month_end = month_start + relativedelta(months=1, seconds=-1)
    month_end_11yr = month_end - relativedelta(years=11)
    month_start_14yr = month_start - relativedelta(years=14)

    school_data_query = BiharDemographicsView.objects.filter(
        month=month,
        state_id=state_id,
        dob__lt=month_end_11yr,
        dob__gte=month_start_14yr,
        gender='F',
        person_id__gt=last_person_case_id
    ).order_by('person_id').values(
        'person_id',
        'person_name',
        'out_of_school_status',
        'last_class_attended_ever'
    )

    # To apply pagination on database query with data size length
    limited_school_data = list(school_data_query[:CAS_API_PAGE_SIZE])
    return limited_school_data, get_total_records_count(BiharVaccineView.__name__, month, state_id,
                                                        month_end_11yr, month_start_14yr)
<commit_msg>test case fix<commit_after>from custom.icds_reports.models.views import BiharDemographicsView, BiharVaccineView, BiharAPIMotherView
from custom.icds_reports.const import CAS_API_PAGE_SIZE
from custom.icds_reports.cache import icds_quickcache
from dateutil.relativedelta import relativedelta
from dimagi.utils.dates import force_to_date

# cache for 2 hours because it wont change atleast for 24 hours.
# This API will be hit in a loop of something and they should be able to scrape all
# the records in 2 hours.
@icds_quickcache(['model_classname', 'month', 'state_id'], timeout=60 * 60 * 2)
def get_total_records_count(model_classname, month, state_id, month_end_11yr=None,
                            month_start_14yr=None):
    classes = {
        BiharDemographicsView.__name__: BiharDemographicsView,
        BiharAPIMotherView.__name__: BiharAPIMotherView,
        BiharVaccineView.__name__: BiharVaccineView,
    }
    if month_start_14yr is None:
        return classes[model_classname].objects.filter(
            month=month,
            state_id=state_id
        ).count()
    else:
        return classes[model_classname].objects.filter(
            month=month,
            state_id=state_id,
            dob__lt=month_end_11yr,
            dob__gte=month_start_14yr,
            gender='F',
        ).count()

def get_api_demographics_data(month, state_id, last_person_case_id):
    demographics_data_query = BiharDemographicsView.objects.filter(
        month=month,
        state_id=state_id,
        person_id__gt=last_person_case_id
    ).order_by('person_id').values(
        'state_name',
        'state_site_code',
        'district_name',
        'district_site_code',
        'block_name',
        'block_site_code',
        'supervisor_name',
        'supervisor_site_code',
        'awc_name',
        'awc_site_code',
        'ward_number',
        'household_id',
        'household_name',
        'hh_reg_date',
        'hh_num',
        'hh_gps_location',
        'hh_caste',
        'hh_bpl_apl',
        'hh_minority',
        'hh_religion',
        'hh_member_number',
        'person_id',
        'person_name',
        'has_adhaar',
        'bank_account_number',
        'ifsc_code',
        'age_at_reg',
        'dob',
        'gender',
        'blood_group',
        'disabled',
        'disability_type',
        'referral_status',
        'migration_status',
        'resident',
        'registered_status',
        'rch_id',
        'mcts_id',
        'phone_number',
        'date_death',
        'site_death',
        'closed_on',
        'reason_closure'
    )

    # To apply pagination on database query with data size length
    limited_demographics_data = list(demographics_data_query[:CAS_API_PAGE_SIZE])
    return limited_demographics_data, get_total_records_count(BiharDemographicsView.__name__, month, state_id)

def get_mother_details(month, state_id, last_ccs_case_id):
    bihar_mother_details = BiharAPIMotherView.objects.filter(
        month=month,
        state_id=state_id,
        ccs_case_id__gt=last_ccs_case_id
    ).order_by('ccs_case_id').values(
        'household_id',
        'ccs_case_id',
        'person_id',
        'married',
        'husband_name',
        'husband_id',
        'last_preg_year',
        'last_preg_tt',
        'is_pregnant',
        'preg_reg_date',
        'tt_1',
        'tt_2',
        'tt_booster',
        'add',
        'hb'
    )
    limited_mother_details_data = list(bihar_mother_details[:CAS_API_PAGE_SIZE])
    return limited_mother_details_data, get_total_records_count(BiharAPIMotherView.__name__, month, state_id)

def get_api_vaccine_data(month, state_id, last_person_case_id):
    vaccine_data_query = BiharVaccineView.objects.filter(
        month=month,
        state_id=state_id,
        person_id__gt=last_person_case_id
    ).order_by('person_id').values(
        'month',
        'person_id',
        'time_birth',
        'child_alive',
        'father_name',
        'mother_name',
        'father_id',
        'mother_id',
        'dob',
        'private_admit',
        'primary_admit',
        'date_last_private_admit',
        'date_return_private',
        'due_list_date_1g_dpt_1',
        'due_list_date_2g_dpt_2',
        'due_list_date_3g_dpt_3',
        'due_list_date_5g_dpt_booster',
        'due_list_date_7gdpt_booster_2',
        'due_list_date_0g_hep_b_0',
        'due_list_date_1g_hep_b_1',
        'due_list_date_2g_hep_b_2',
        'due_list_date_3g_hep_b_3',
        'due_list_date_3g_ipv',
        'due_list_date_4g_je_1',
        'due_list_date_5g_je_2',
        'due_list_date_5g_measles_booster',
        'due_list_date_4g_measles',
        'due_list_date_0g_opv_0',
        'due_list_date_1g_opv_1',
        'due_list_date_2g_opv_2',
        'due_list_date_3g_opv_3',
        'due_list_date_5g_opv_booster',
        'due_list_date_1g_penta_1',
        'due_list_date_2g_penta_2',
        'due_list_date_3g_penta_3',
        'due_list_date_1g_rv_1',
        'due_list_date_2g_rv_2',
        'due_list_date_3g_rv_3',
        'due_list_date_4g_vit_a_1',
        'due_list_date_5g_vit_a_2',
        'due_list_date_6g_vit_a_3',
        'due_list_date_6g_vit_a_4',
        'due_list_date_6g_vit_a_5',
        'due_list_date_6g_vit_a_6',
        'due_list_date_6g_vit_a_7',
        'due_list_date_6g_vit_a_8',
        'due_list_date_7g_vit_a_9',
        'due_list_date_1g_bcg'
    )

    # To apply pagination on database query with data size length
    limited_vaccine_data = list(vaccine_data_query[:CAS_API_PAGE_SIZE])
    return limited_vaccine_data, get_total_records_count(BiharVaccineView.__name__, month, state_id)

def get_api_ag_school_data(month, state_id, last_person_case_id):
    month_start = force_to_date(month).replace(day=1)
    month_end = month_start + relativedelta(months=1, seconds=-1)
    month_end_11yr = month_end - relativedelta(years=11)
    month_start_14yr = month_start - relativedelta(years=14)

    school_data_query = BiharDemographicsView.objects.filter(
        month=month,
        state_id=state_id,
        dob__lt=month_end_11yr,
        dob__gte=month_start_14yr,
        gender='F',
        person_id__gt=last_person_case_id
    ).order_by('person_id').values(
        'person_id',
        'person_name',
        'out_of_school_status',
        'last_class_attended_ever'
    )

    # To apply pagination on database query with data size length
    limited_school_data = list(school_data_query[:CAS_API_PAGE_SIZE])
    return limited_school_data, get_total_records_count(BiharDemographicsView.__name__, month, state_id,
                                                        month_end_11yr, month_start_14yr)
<|endoftext|>