<commit_before>#!/usr/bin/python2.4
import sys
from bx.bitset import *
from bx.bitset_builders import *

"""
Readers for bed/gtf/gff formats

GeneReader: yields exons
CDSReader: yields cds_exons
FeatureReader: yields cds_exons,introns, exons

For gff/gtf, the start_codon stop_codon line types are merged with CDSs.

"""

def GeneReader( fh, format='gff' ):
""" yield chrom, strand, gene_exons, name """

    known_formats = ( 'gff', 'gtf', 'bed')
    if format not in known_formats:
        print >>sys.stderr,  '%s format not in %s' % (format, ",".join( known_formats ))
        raise '?'

    if format == 'bed':
        for line in fh:
            f = line.strip().split()
            chrom = f[0]
            chrom_start = int(f[1])
            name = f[4]
            strand = f[5]
            cdsStart = int(f[6])
            cdsEnd = int(f[7])
            blockCount = int(f[9])
            blockSizes = [ int(i) for i in f[10].strip(',').split(',') ]
            blockStarts = [ chrom_start + int(i) for i in f[11].strip(',').split(',') ]

            # grab cdsStart - cdsEnd
            gene_exons = []
            for base,offset in zip( blockStarts, blockSizes ):
                exon_start = base
                exon_end = base+offset
                gene_exons.append( (exon_start, exon_end) )
            yield chrom, strand, gene_exons, name
    genelist = {}
    grouplist = []
    if format == 'gff' or format == 'gtf':
        for line in fh:
            if line.startswith('#'): continue
            fields = line.strip().split('\t')
            if len( fields ) < 9: continue

            # fields

            chrom = fields[0]
            ex_st = int( fields[3] ) - 1 # make zero-centered
            ex_end = int( fields[4] ) #+ 1 # make exclusive
            strand = fields[6]

            if format == 'gtf':
                group = fields[8].split(';')[0]
            else:
                group = fields[8]

            if group not in grouplist: grouplist.append( group )
            if group not in genelist:
                genelist[group] = (chrom, strand, [])
            exons_i = 2
            genelist[group][exons_i].append( ( ex_st, ex_end ) )

        sp = lambda a,b: cmp( a[0], b[0] )

        #for gene in genelist.values():
        for gene in grouplist:
            chrom, strand, gene_exons = genelist[ gene ]
            gene_exons = bitset_union( gene_exons )
            yield chrom, strand, gene_exons, gene

def CDSReader( fh, format='gff' ):
""" yield chrom, strand, cds_exons, name """

    known_formats = ( 'gff', 'gtf', 'bed')
    if format not in known_formats:
        print >>sys.stderr,  '%s format not in %s' % (format, ",".join( known_formats ))
        raise '?'

    if format == 'bed':
        for line in fh:
            f = line.strip().split()
            chrom = f[0]
            chrom_start = int(f[1])
            name = f[4]
            strand = f[5]
            cdsStart = int(f[6])
            cdsEnd = int(f[7])
            blockCount = int(f[9])
            blockSizes = [ int(i) for i in f[10].strip(',').split(',') ]
            blockStarts = [ chrom_start + int(i) for i in f[11].strip(',').split(',') ]

            # grab cdsStart - cdsEnd
            cds_exons = []
            cds_seq = ''
            genome_seq_index = []
            for base,offset in zip( blockStarts, blockSizes ):
                if (base + offset) < cdsStart: continue
                if base > cdsEnd: continue
                exon_start = max( base, cdsStart )
                exon_end = min( base+offset, cdsEnd )
                cds_exons.append( (exon_start, exon_end) )
            yield chrom, strand, cds_exons, name

    genelist = {}
    grouplist = []
    if format == 'gff' or format == 'gtf':
        for line in fh:
            if line.startswith('#'): continue
            fields = line.strip().split('\t')
            if len( fields ) < 9: continue
            if fields[2] not in ('CDS', 'stop_codon', 'start_codon'): continue

            # fields

            chrom = fields[0]
            ex_st = int( fields[3] ) - 1 # make zero-centered
            ex_end = int( fields[4] ) #+ 1 # make exclusive
            strand = fields[6]

            if format == 'gtf':
                group = fields[8].split(';')[0]
            else:
                group = fields[8]

            if group not in grouplist: grouplist.append( group )
            if group not in genelist:
                genelist[group] = (chrom, strand, [])

            genelist[group][2].append( ( ex_st, ex_end ) )

        sp = lambda a,b: cmp( a[0], b[0] )

        #for gene in genelist.values():
        for gene in grouplist:
            chrom, strand, cds_exons = genelist[ gene ]
            seqlen = sum([ a[1]-a[0] for a in cds_exons ])
            overhang = seqlen % 3
            if overhang > 0:
                #print >>sys.stderr, "adjusting ", gene
                if strand == '+':
                    cds_exons[-1] = ( cds_exons[-1][0], cds_exons[-1][1] - overhang )
                else:
                    cds_exons[0] = ( cds_exons[0][0] + overhang, cds_exons[0][1] )
            cds_exons = bitset_union( cds_exons )
            yield chrom, strand, cds_exons, gene

def FeatureReader( fh, format='gff', alt_introns_subtract="exons", gtf_parse=None):
""" yield chrom, strand, cds_exons, introns, exons, name

gtf_parse Example:
# parse gene_id from transcript_id "AC073130.2-001"; gene_id "TES";
gene_name = lambda s: s.split(';')[1].split()[1].strip('"')

for chrom, strand, cds_exons, introns, exons, name in FeatureReader( sys.stdin, format='gtf', gtf_parse=gene_name )
"""

    known_formats = ( 'gff', 'gtf', 'bed')
    if format not in known_formats:
        print >>sys.stderr,  '%s format not in %s' % (format, ",".join( known_formats ))
        raise '?'

    if format == 'bed':
        for line in fh:
            f = line.strip().split()
            chrom = f[0]
            chrom_start = int(f[1])
            name = f[4]
            strand = f[5]
            cdsStart = int(f[6])
            cdsEnd = int(f[7])
            blockCount = int(f[9])
            blockSizes = [ int(i) for i in f[10].strip(',').split(',') ]
            blockStarts = [ chrom_start + int(i) for i in f[11].strip(',').split(',') ]

            # grab cdsStart - cdsEnd
            cds_exons = []
            exons = []

            cds_seq = ''
            genome_seq_index = []
            for base,offset in zip( blockStarts, blockSizes ):
                if (base + offset) < cdsStart: continue
                if base > cdsEnd: continue
                # exons
                exon_start = base
                exon_end = base+offset
                exons.append( (exon_start, exon_end) )
                # cds exons
                exon_start = max( base, cdsStart )
                exon_end = min( base+offset, cdsEnd )
                cds_exons.append( (exon_start, exon_end) )
            cds_exons = bitset_union( cds_exons )
            exons = bitset_union( exons )
            introns = bitset_complement( exons )
            yield chrom, strand, cds_exons, introns, exons, name

    genelist = {}
    grouplist = []
    if format == 'gff' or format == 'gtf':
        for line in fh:
            if line.startswith('#'): continue
            fields = line.strip().split('\t')
            if len( fields ) < 9: continue

            # fields

            chrom = fields[0]
            ex_st = int( fields[3] ) - 1 # make zero-centered
            ex_end = int( fields[4] ) #+ 1 # make exclusive
            strand = fields[6]

            if format == 'gtf':
                if not gtf_parse:
                    group = fields[8].split(';')[0]
                else:
                    group = gtf_parse( fields[8] )
            else:
                group = fields[8]

            if group not in grouplist: grouplist.append( group )
            if group not in genelist:
                # chrom, strand, cds_exons, introns, exons
                genelist[group] = (chrom, strand, [], [], [])

            if fields[2] == 'exon':
                genelist[group][4].append( ( ex_st, ex_end ) )
            elif fields[2] in ('CDS', 'stop_codon', 'start_codon'):
                genelist[group][2].append( ( ex_st, ex_end ) )
            elif fields[2] == 'intron':
                genelist[group][3].append( ( ex_st, ex_end ) )

        sp = lambda a,b: cmp( a[0], b[0] )

        #for gene in genelist.values():
        for gene in grouplist:
            chrom, strand, cds_exons, introns, exons = genelist[ gene ]
            seqlen = sum([ a[1]-a[0] for a in cds_exons ])
            overhang = seqlen % 3
            if overhang > 0:
                #print >>sys.stderr, "adjusting ", gene
                if strand == '+':
                    cds_exons[-1] = ( cds_exons[-1][0], cds_exons[-1][1] - overhang )
                else:
                    cds_exons[0] = ( cds_exons[0][0] + overhang, cds_exons[0][1] )

            cds_exons = bitset_union( cds_exons )
            exons = bitset_union( exons )

            if alt_introns_subtract:
                if alt_introns_subtract == 'exons':
                    introns = bitset_subtract( introns, exons )
                if alt_introns_subtract == 'cds_exons':
                    introns = bitset_subtract( introns, cds_exons )
            else:
                introns = bitset_union( introns )

            yield chrom, strand, cds_exons, introns, exons, gene

def bitset_subtract( ex1, ex2 ):
    bits1 = BinnedBitSet(MAX)
    for l in ex1:
        start, end = l[0], l[1]
        bits1.set_range( start, end - start )
    bits2 = BinnedBitSet(MAX)
    for l in ex1:
        start, end = l[0], l[1]
        bits2.set_range( start, end - start )

    bits2.invert()
    bits1.iand( bits2 )
    return bits2list( bits1 )

def bits2list( bits ):
    ex = []
    end = 0
    while 1:
        start = bits.next_set( end )
        if start == bits.size: break
        end = bits.next_clear( start )
        ex.append( (start, end) )

    return ex

def bitset_complement( exons ):
    bits = BinnedBitSet(MAX)
    introns = []
    for l in exons:
        start, end = l[0], l[1]
        bits.set_range( start, end - start )
    bits.invert()

    # only complement within the range of the list
    ex_start = min( [a[0] for a in exons] )
    ex_end = max( [a[1] for a in exons] )
    end = ex_start
    len = ex_end
    while 1:
            start = bits.next_set( end )
            if start == bits.size: break
            end = bits.next_clear( start )
            if end > len: end = len
            if start != end:
                introns.append( (start,end ) )
            if end == len: break
    return introns

def bitset_union( exons ):
    bits = BinnedBitSet(MAX)
    Uexons = []
    for l in exons:
        start, end = l[0], l[1]
        bits.set_range( start, end - start )
    end = 0
    while 1:
        start = bits.next_set( end )
        if start == bits.size: break
        end = bits.next_clear( start )
        Uexons.append( (start, end) )

    return Uexons
<commit_msg>fixed indent errors-- sometimes I think I'm in C<commit_after>#!/usr/bin/python2.4
import sys
from bx.bitset import *
from bx.bitset_builders import *

"""
Readers for bed/gtf/gff formats

GeneReader: yields exons
CDSReader: yields cds_exons
FeatureReader: yields cds_exons,introns, exons

For gff/gtf, the start_codon stop_codon line types are merged with CDSs.

"""

def GeneReader( fh, format='gff' ):
    """ yield chrom, strand, gene_exons, name """

    known_formats = ( 'gff', 'gtf', 'bed')
    if format not in known_formats:
        print >>sys.stderr,  '%s format not in %s' % (format, ",".join( known_formats ))
        raise '?'

    if format == 'bed':
        for line in fh:
            f = line.strip().split()
            chrom = f[0]
            chrom_start = int(f[1])
            name = f[4]
            strand = f[5]
            cdsStart = int(f[6])
            cdsEnd = int(f[7])
            blockCount = int(f[9])
            blockSizes = [ int(i) for i in f[10].strip(',').split(',') ]
            blockStarts = [ chrom_start + int(i) for i in f[11].strip(',').split(',') ]

            # grab cdsStart - cdsEnd
            gene_exons = []
            for base,offset in zip( blockStarts, blockSizes ):
                exon_start = base
                exon_end = base+offset
                gene_exons.append( (exon_start, exon_end) )
            yield chrom, strand, gene_exons, name
    genelist = {}
    grouplist = []
    if format == 'gff' or format == 'gtf':
        for line in fh:
            if line.startswith('#'): continue
            fields = line.strip().split('\t')
            if len( fields ) < 9: continue

            # fields

            chrom = fields[0]
            ex_st = int( fields[3] ) - 1 # make zero-centered
            ex_end = int( fields[4] ) #+ 1 # make exclusive
            strand = fields[6]

            if format == 'gtf':
                group = fields[8].split(';')[0]
            else:
                group = fields[8]

            if group not in grouplist: grouplist.append( group )
            if group not in genelist:
                genelist[group] = (chrom, strand, [])
            exons_i = 2
            genelist[group][exons_i].append( ( ex_st, ex_end ) )

        sp = lambda a,b: cmp( a[0], b[0] )

        #for gene in genelist.values():
        for gene in grouplist:
            chrom, strand, gene_exons = genelist[ gene ]
            gene_exons = bitset_union( gene_exons )
            yield chrom, strand, gene_exons, gene

def CDSReader( fh, format='gff' ):
    """ yield chrom, strand, cds_exons, name """

    known_formats = ( 'gff', 'gtf', 'bed')
    if format not in known_formats:
        print >>sys.stderr,  '%s format not in %s' % (format, ",".join( known_formats ))
        raise '?'

    if format == 'bed':
        for line in fh:
            f = line.strip().split()
            chrom = f[0]
            chrom_start = int(f[1])
            name = f[4]
            strand = f[5]
            cdsStart = int(f[6])
            cdsEnd = int(f[7])
            blockCount = int(f[9])
            blockSizes = [ int(i) for i in f[10].strip(',').split(',') ]
            blockStarts = [ chrom_start + int(i) for i in f[11].strip(',').split(',') ]

            # grab cdsStart - cdsEnd
            cds_exons = []
            cds_seq = ''
            genome_seq_index = []
            for base,offset in zip( blockStarts, blockSizes ):
                if (base + offset) < cdsStart: continue
                if base > cdsEnd: continue
                exon_start = max( base, cdsStart )
                exon_end = min( base+offset, cdsEnd )
                cds_exons.append( (exon_start, exon_end) )
            yield chrom, strand, cds_exons, name

    genelist = {}
    grouplist = []
    if format == 'gff' or format == 'gtf':
        for line in fh:
            if line.startswith('#'): continue
            fields = line.strip().split('\t')
            if len( fields ) < 9: continue
            if fields[2] not in ('CDS', 'stop_codon', 'start_codon'): continue

            # fields

            chrom = fields[0]
            ex_st = int( fields[3] ) - 1 # make zero-centered
            ex_end = int( fields[4] ) #+ 1 # make exclusive
            strand = fields[6]

            if format == 'gtf':
                group = fields[8].split(';')[0]
            else:
                group = fields[8]

            if group not in grouplist: grouplist.append( group )
            if group not in genelist:
                genelist[group] = (chrom, strand, [])

            genelist[group][2].append( ( ex_st, ex_end ) )

        sp = lambda a,b: cmp( a[0], b[0] )

        #for gene in genelist.values():
        for gene in grouplist:
            chrom, strand, cds_exons = genelist[ gene ]
            seqlen = sum([ a[1]-a[0] for a in cds_exons ])
            overhang = seqlen % 3
            if overhang > 0:
                #print >>sys.stderr, "adjusting ", gene
                if strand == '+':
                    cds_exons[-1] = ( cds_exons[-1][0], cds_exons[-1][1] - overhang )
                else:
                    cds_exons[0] = ( cds_exons[0][0] + overhang, cds_exons[0][1] )
            cds_exons = bitset_union( cds_exons )
            yield chrom, strand, cds_exons, gene

def FeatureReader( fh, format='gff', alt_introns_subtract="exons", gtf_parse=None):
    """
    yield chrom, strand, cds_exons, introns, exons, name

    gtf_parse Example:
    # parse gene_id from transcript_id "AC073130.2-001"; gene_id "TES";
    gene_name = lambda s: s.split(';')[1].split()[1].strip('"')

    for chrom, strand, cds_exons, introns, exons, name in FeatureReader( sys.stdin, format='gtf', gtf_parse=gene_name )
    """

    known_formats = ( 'gff', 'gtf', 'bed')
    if format not in known_formats:
        print >>sys.stderr,  '%s format not in %s' % (format, ",".join( known_formats ))
        raise '?'

    if format == 'bed':
        for line in fh:
            f = line.strip().split()
            chrom = f[0]
            chrom_start = int(f[1])
            name = f[4]
            strand = f[5]
            cdsStart = int(f[6])
            cdsEnd = int(f[7])
            blockCount = int(f[9])
            blockSizes = [ int(i) for i in f[10].strip(',').split(',') ]
            blockStarts = [ chrom_start + int(i) for i in f[11].strip(',').split(',') ]

            # grab cdsStart - cdsEnd
            cds_exons = []
            exons = []

            cds_seq = ''
            genome_seq_index = []
            for base,offset in zip( blockStarts, blockSizes ):
                if (base + offset) < cdsStart: continue
                if base > cdsEnd: continue
                # exons
                exon_start = base
                exon_end = base+offset
                exons.append( (exon_start, exon_end) )
                # cds exons
                exon_start = max( base, cdsStart )
                exon_end = min( base+offset, cdsEnd )
                cds_exons.append( (exon_start, exon_end) )
            cds_exons = bitset_union( cds_exons )
            exons = bitset_union( exons )
            introns = bitset_complement( exons )
            yield chrom, strand, cds_exons, introns, exons, name

    genelist = {}
    grouplist = []
    if format == 'gff' or format == 'gtf':
        for line in fh:
            if line.startswith('#'): continue
            fields = line.strip().split('\t')
            if len( fields ) < 9: continue

            # fields

            chrom = fields[0]
            ex_st = int( fields[3] ) - 1 # make zero-centered
            ex_end = int( fields[4] ) #+ 1 # make exclusive
            strand = fields[6]

            if format == 'gtf':
                if not gtf_parse:
                    group = fields[8].split(';')[0]
                else:
                    group = gtf_parse( fields[8] )
            else:
                group = fields[8]

            if group not in grouplist: grouplist.append( group )
            if group not in genelist:
                # chrom, strand, cds_exons, introns, exons
                genelist[group] = (chrom, strand, [], [], [])

            if fields[2] == 'exon':
                genelist[group][4].append( ( ex_st, ex_end ) )
            elif fields[2] in ('CDS', 'stop_codon', 'start_codon'):
                genelist[group][2].append( ( ex_st, ex_end ) )
            elif fields[2] == 'intron':
                genelist[group][3].append( ( ex_st, ex_end ) )

        sp = lambda a,b: cmp( a[0], b[0] )

        #for gene in genelist.values():
        for gene in grouplist:
            chrom, strand, cds_exons, introns, exons = genelist[ gene ]
            seqlen = sum([ a[1]-a[0] for a in cds_exons ])
            overhang = seqlen % 3
            if overhang > 0:
                #print >>sys.stderr, "adjusting ", gene
                if strand == '+':
                    cds_exons[-1] = ( cds_exons[-1][0], cds_exons[-1][1] - overhang )
                else:
                    cds_exons[0] = ( cds_exons[0][0] + overhang, cds_exons[0][1] )

            cds_exons = bitset_union( cds_exons )
            exons = bitset_union( exons )

            if alt_introns_subtract:
                if alt_introns_subtract == 'exons':
                    introns = bitset_subtract( introns, exons )
                if alt_introns_subtract == 'cds_exons':
                    introns = bitset_subtract( introns, cds_exons )
            else:
                introns = bitset_union( introns )

            yield chrom, strand, cds_exons, introns, exons, gene

def bitset_subtract( ex1, ex2 ):
    bits1 = BinnedBitSet(MAX)
    for l in ex1:
        start, end = l[0], l[1]
        bits1.set_range( start, end - start )
    bits2 = BinnedBitSet(MAX)
    for l in ex1:
        start, end = l[0], l[1]
        bits2.set_range( start, end - start )

    bits2.invert()
    bits1.iand( bits2 )
    return bits2list( bits1 )

def bits2list( bits ):
    ex = []
    end = 0
    while 1:
        start = bits.next_set( end )
        if start == bits.size: break
        end = bits.next_clear( start )
        ex.append( (start, end) )

    return ex

def bitset_complement( exons ):
    bits = BinnedBitSet(MAX)
    introns = []
    for l in exons:
        start, end = l[0], l[1]
        bits.set_range( start, end - start )
    bits.invert()

    # only complement within the range of the list
    ex_start = min( [a[0] for a in exons] )
    ex_end = max( [a[1] for a in exons] )
    end = ex_start
    len = ex_end
    while 1:
            start = bits.next_set( end )
            if start == bits.size: break
            end = bits.next_clear( start )
            if end > len: end = len
            if start != end:
                introns.append( (start,end ) )
            if end == len: break
    return introns

def bitset_union( exons ):
    bits = BinnedBitSet(MAX)
    Uexons = []
    for l in exons:
        start, end = l[0], l[1]
        bits.set_range( start, end - start )
    end = 0
    while 1:
        start = bits.next_set( end )
        if start == bits.size: break
        end = bits.next_clear( start )
        Uexons.append( (start, end) )

    return Uexons
<|endoftext|>


<commit_before>"""
Keyword Recommenders.
"""

import math
import random
from utils import timeit
from collections import defaultdict

class RandomRecommender(object):
    def __init__(self, limit, dbm, *ignored):
        """
        @param dbm a DatabaseManager
        @param limit the (maximal) number of recommended products at a time
        """
        self.limit = limit
        self.dbm = dbm
        self.all_products = []

    def __str__(self):
        return 'Random Recommender[N=%d]' % self.limit

    def use_keywords(self):
        return False

    @timeit
    def preprocess(self, query_train_table):
        # retrieve all products at once as there aren't many (< 4000)
        query = '''SELECT DISTINCT pageinfo FROM visit
            WHERE pagetype = 'product' AND pageinfo != '' AND pageinfo != 'null' AND userid IN (
                SELECT user_id FROM %s
            )''' % query_train_table
        self.all_products = [(row['pageinfo'], 1.0) for row in self.dbm.get_rows(query)]

    def round_statistics(self):
        pass

    def recommend(self, query):
        return random.sample(self.all_products, self.limit)

class HottestRecommender(object):
    def __init__(self, limit, dbm, *ignored):
        """
        @param dbm a DatabaseManager
        @param limit the (maximal) number of recommended products at a time
        """
        self.limit = limit
        self.dbm = dbm
        self.recommend_list = []

    def __str__(self):
        return 'Hottest Recommender[N=%d]' % self.limit

    def use_keywords(self):
        return False

    @timeit
    def preprocess(self, query_train_table):
        for row in self.dbm.get_rows('''SELECT pageinfo, COUNT(id) count FROM visit
            WHERE pagetype = 'product' AND pageinfo != '' AND userid IN (
                SELECT user_id FROM %s
            ) GROUP BY pageinfo ORDER BY count DESC LIMIT %d''' % (query_train_table, self.limit)):
            self.recommend_list.append((row['pageinfo'], row['count']))
        #print self.recommend_list

    def round_statistics(self):
        pass

    def recommend(self, query):
        return self.recommend_list

class KeywordRecommender(object):
    def __init__(self, limit, dbm, ws, rm):
        """
        Make sure to source rec_tables.sql before using this class.
        @param dbm a DatabaseManager
        @param ws a WordSegmenter
        @param rm a RelevanceMeasure
        """
        self.limit = limit
        self.dbm = dbm
        self.ws = ws
        self.rm = rm
        self._related_product_cache = {}

    def __str__(self):
        return 'Keyword Recommender with %s[N=%d]' % (self.rm, self.limit)

    def use_keywords(self):
        return True

    def preprocess(self, query_train_table):
        self.query_train_table = query_train_table
        # empty cache so that cache from last round does not interfere with next round
        self._related_product_cache = {}
        self._not_enough_recs = 0

        self.dbm.begin()
        self.dbm.query('TRUNCATE TABLE keyword');
        self.dbm.query('TRUNCATE TABLE keyword_query');
        self.dbm.query('TRUNCATE TABLE keyword_product_weight');

        # these methods can be overridden by sub-classes
        self._build_keyword_product_mapping()
        self._build_product_keyword_mapping()
        self._measure_relevance()

        self.dbm.commit()

    @timeit
    def _build_keyword_product_mapping(self):
        self.keyword_count = defaultdict(int)
        self.keyword_product_count = defaultdict(lambda: defaultdict(int))
        for qrow in self.dbm.get_rows('SELECT id, query FROM %s' % self.query_train_table):
            # GROUP_CONCAT returns a comma-separeted string
            products = [(qprow['product_name'], qprow['sequences']) for qprow in self.dbm.get_rows('SELECT product_name, GROUP_CONCAT(sequence) AS sequences FROM query_product WHERE query_id = %s GROUP BY product_name', (qrow['id'],))]

            # remove duplicate
            keywords = set(self.ws.segment(qrow['query']))
            for kw in keywords:
                self.keyword_count[kw] += 1
                # store keyword-query relations in db
                self.dbm.insert('INSERT INTO keyword_query (keyword, query_id) VALUES (%s, %s)', (kw, qrow['id']))

            for p, sequences in products:
                # get product sequence in this session
                count = self.get_browse_count(sequences)
                # update keyword_product_count
                for kw in keywords:
                    self.keyword_product_count[kw][p] += count

    def get_browse_count(self, sequences):
        """Overrideable by sub-class.
        Multiple browses in a session always count 1."""
        return 1

    @timeit
    def _build_product_keyword_mapping(self):
        # construct product_keyword_count
        # it's actually equivalent to keyword_product_count, but can let compute
        # related_product_count faster
        self.product_keyword_count = defaultdict(dict)
        for kw, dt in self.keyword_product_count.iteritems():
            for p, c in dt.iteritems():
                self.product_keyword_count[p][kw] = c

    @timeit
    def _measure_relevance(self):
        # calculate keyword-product relevance
        all_product_number = self.dbm.get_value('SELECT COUNT(DISTINCT product_name) FROM query_product')
        for keyword, count in self.keyword_count.iteritems():
            # will be used for statistics
            self.dbm.insert('INSERT INTO keyword (keyword, count) VALUES (%s, %s)', (keyword, count))
            related_product_number = len(self.keyword_product_count[keyword].keys())
            related_product_count = sum(self.keyword_product_count[keyword].values())

            for product, count in self.keyword_product_count[keyword].iteritems():
                related_keyword_number = len(self.product_keyword_count[product].keys())
                related_keyword_count = sum(self.product_keyword_count[product].values())

                # delegate to sub-classes
                relevance = self.rm.get_relevance(count, (related_product_number, related_product_count), (related_keyword_number, related_keyword_count), all_product_number)
                self.dbm.insert('INSERT INTO keyword_product_weight (keyword, product, weight) VALUES (%s, %s, %s)', (keyword, product, relevance))

    def round_statistics(self):
        """Get number of query, keywords, products, keyword-product relations of current round."""
        n_query = self.dbm.get_value("SELECT COUNT(*) FROM %s" % self.query_train_table)
        n_keyword = self.dbm.get_value("SELECT COUNT(*) FROM keyword")
        n_product = self.dbm.get_value("SELECT COUNT(DISTINCT product) FROM keyword_product_weight")
        n_relation = self.dbm.get_value("SELECT COUNT(*) FROM keyword_product_weight")

        print 'Round statistics: query: %d (not enough %d), keyword: %d, product: %d, relation: %d, A/M: %.2f%%' % (n_query, self._not_enough_recs, n_keyword, n_product, n_relation, 100.0*n_relation / (n_keyword*n_product))

    def recommend(self, query):
        keywords = self.ws.segment(query)
        product_weight = defaultdict(float)
        # gather product weights
        for kw in keywords:
            for product, weight in self.__fetch_related_products(kw):
                product_weight[product] += weight

        # convert dict to list for sorting
        product_weight_list = [item for item in product_weight.iteritems()]
        product_weight_list.sort(key=lambda t: t[1], reverse=True)

        if len(product_weight_list) < self.limit:
            self._not_enough_recs += 1

        return product_weight_list[:self.limit]

    def __fetch_related_products(self, keyword):
        if not self._related_product_cache.has_key(keyword):
            self._related_product_cache[keyword] = [(row['product'], row['weight']) for row in self.dbm.get_rows('SELECT product, weight FROM keyword_product_weight WHERE keyword = %s', (keyword,))]
        return self._related_product_cache[keyword]

class KeywordRecommenderHottestFallback(KeywordRecommender):
    """A recommender which uses KeywordRecommender's recommendations first,
    but turns to HottestRecommender if its recommendations are not enough."""

    def __init__(self, *args):
        """Identical to that of KeywordRecommender"""
        super(KeywordRecommenderHottestFallback, self).__init__(*args)
        self.hottest_recommender = HottestRecommender(*args)

    def __str__(self):
        return 'Keyword Recommender with Hottest Recommender fallback with %s[N=%d]' % (self.rm, self.limit)

    def preprocess(self, query_train_table):
        super(KeywordRecommenderHottestFallback, self).preprocess(query_train_table)
        self.hottest_recommender.preprocess(query_train_table)

    def recommend(self, query):
        recommendations = super(KeywordRecommenderHottestFallback, self).recommend(query)
        num_rec = len(recommendations)
        if num_rec == self.limit:
            return recommendations

        # ask HottestRecommender for more
        # note that create list in order not to break HottestRecommender.recommend_list
        hot_recommendations = self.hottest_recommender.recommend(query)[:self.limit-num_rec]

        # ensure hot_recommendations's weight is no greater than any from keyword recommendations
        max_hot_rec_weight = hot_recommendations[0][1]
        min_key_rec_weight = recommendations[-1][1] if num_rec > 0 else max_hot_rec_weight
        recommendations.extend((t[0], 1.0*min_key_rec_weight*t[1]/max_hot_rec_weight) for t in hot_recommendations)

        return recommendations

class WeightedSequenceRelevanceMixin(object):
    @timeit
    def _measure_relevance(self):
        # calculate keyword-product relevance
        all_product_number = self.dbm.get_value('SELECT COUNT(DISTINCT product_name) FROM query_product')
        for keyword, count in self.keyword_count.iteritems():
            self.dbm.insert('INSERT INTO keyword (keyword, count) VALUES (%s, %s)', (keyword, count))
            related_product_number = len(self.keyword_product_count[keyword].keys())
            related_product_count = sum(self.keyword_product_count[keyword].values())

            for product, count in self.keyword_product_count[keyword].iteritems():
                related_keyword_number = len(self.product_keyword_count[product].keys())
                related_keyword_count = sum(self.product_keyword_count[product].values())
                # get average sequence from database
                # TODO: very inefficient, get a group all average sequences for a keyword at once
                #avg_sequence = self.dbm.get_value('select avg(sequence) from query_product where product_name = %s AND query_id in (select query_id from keyword_query where keyword = %s)', (product, keyword))
                avg_sequence = 1

                relevance = self.rm.get_relevance(count, (related_product_number, related_product_count), (related_keyword_number, related_keyword_count), all_product_number, avg_sequence)

                # sub-class can override sequence_weight
                relevance *= self.sequence_weight(avg_sequence)
                self.dbm.insert('INSERT INTO keyword_product_weight (keyword, product, weight) VALUES (%s, %s, %s)', (keyword, product, relevance))

    def sequence_weight(self, avg_sequence):
        return 1

# ensure WSRM._measure_relevance will be called with putting it before KeywordRecommender
# ref: http://python-history.blogspot.com/2010/06/method-resolution-order.html
class SequenceKeywordRecommender(WeightedSequenceRelevanceMixin, KeywordRecommender):
    """This recommender weights browse count by distribution of sequence."""

    @timeit
    def preprocess(self, query_train_table):
        # first, get sequence distribution
        max_occurrence = self.dbm.get_value('SELECT MAX(c) FROM (SELECT sequence, COUNT(sequence) c FROM query_product WHERE query_id IN (SELECT id FROM %s) GROUP BY sequence) T' % query_train_table)
        self.sequence_dist = {row['sequence']: float(row['ratio']) for row in self.dbm.get_rows('SELECT sequence, COUNT(sequence)/%d ratio FROM query_product WHERE query_id IN (SELECT id FROM %s) GROUP BY sequence' % (max_occurrence,query_train_table))}

        # then, call KeywordRecommender's preprocess
        super(SequenceKeywordRecommender, self).preprocess(query_train_table)

    def get_browse_count(self, sequences):
        """Multiple browses in a session always count 1."""
        return sum((self.sequence_dist[int(seq)] for seq in sequences.split(',')))

    def __str__(self):
        return 'Sequenced Keyword Recommender with %s[N=%d]' % (self.rm, self.limit)

class RelevanceMeasure(object):
    """Defines the RelevanceMeasure interface."""

    def get_relevance(self, count, related_product_info, related_keyword_info, all_product_number, *args):
        """
        @param count number of times the keyword visit the product
        @param related_product_info the tuple (related_product_number, related_product_count)
        @param related_keyword_info the tuple (related_keyword_number, related_keyword_count)
        @param all_product_number number of all products
        """
        raise NotImplemented

class BCMeasure(RelevanceMeasure):
    def get_relevance(self, count, *ignored):
        return count

    def __str__(self):
        return 'BC'

class BCIPFMeasure(RelevanceMeasure):
    def get_relevance(self, count, related_product_info, related_keyword_info, all_product_number, *args):
        ipf = math.log(1.0 * all_product_number / related_product_info[0])
        return count * ipf

    def __str__(self):
        return 'BC-IPF'

class BFIPFMeasure(RelevanceMeasure):
    def get_relevance(self, count, related_product_info, related_keyword_info, all_product_number, *args):
        bf = 1.0 * count / related_keyword_info[1]
        ipf = math.log(1.0 * all_product_number / related_product_info[0])
        return bf * ipf

    def __str__(self):
        return 'BF-IPF'

if __name__ == '__main__':
    import config
    from database import DatabaseManager
    from word_segment import SpaceWordSegmenter
    dbm = DatabaseManager(config.DB_HOST, config.DB_USER, config.DB_PASSWORD, config.DB_NAME)
    try:
        word_segmenter = SpaceWordSegmenter()
        rmeasure = BCIPFMeasure()
        recommender = KeywordRecommender(10, dbm, word_segmenter, rmeasure)
        recommender.preprocess('query_train')
    finally:
        dbm.close()
<commit_msg>reset the counter when initialization this avoids errors when directly used to make recommendations<commit_after>"""
Keyword Recommenders.
"""

import math
import random
from utils import timeit
from collections import defaultdict

class RandomRecommender(object):
    def __init__(self, limit, dbm, *ignored):
        """
        @param dbm a DatabaseManager
        @param limit the (maximal) number of recommended products at a time
        """
        self.limit = limit
        self.dbm = dbm
        self.all_products = []

    def __str__(self):
        return 'Random Recommender[N=%d]' % self.limit

    def use_keywords(self):
        return False

    @timeit
    def preprocess(self, query_train_table):
        # retrieve all products at once as there aren't many (< 4000)
        query = '''SELECT DISTINCT pageinfo FROM visit
            WHERE pagetype = 'product' AND pageinfo != '' AND pageinfo != 'null' AND userid IN (
                SELECT user_id FROM %s
            )''' % query_train_table
        self.all_products = [(row['pageinfo'], 1.0) for row in self.dbm.get_rows(query)]

    def round_statistics(self):
        pass

    def recommend(self, query):
        return random.sample(self.all_products, self.limit)

class HottestRecommender(object):
    def __init__(self, limit, dbm, *ignored):
        """
        @param dbm a DatabaseManager
        @param limit the (maximal) number of recommended products at a time
        """
        self.limit = limit
        self.dbm = dbm
        self.recommend_list = []

    def __str__(self):
        return 'Hottest Recommender[N=%d]' % self.limit

    def use_keywords(self):
        return False

    @timeit
    def preprocess(self, query_train_table):
        for row in self.dbm.get_rows('''SELECT pageinfo, COUNT(id) count FROM visit
            WHERE pagetype = 'product' AND pageinfo != '' AND userid IN (
                SELECT user_id FROM %s
            ) GROUP BY pageinfo ORDER BY count DESC LIMIT %d''' % (query_train_table, self.limit)):
            self.recommend_list.append((row['pageinfo'], row['count']))
        #print self.recommend_list

    def round_statistics(self):
        pass

    def recommend(self, query):
        return self.recommend_list

class KeywordRecommender(object):
    def __init__(self, limit, dbm, ws, rm):
        """
        Make sure to source rec_tables.sql before using this class.
        @param dbm a DatabaseManager
        @param ws a WordSegmenter
        @param rm a RelevanceMeasure
        """
        self.limit = limit
        self.dbm = dbm
        self.ws = ws
        self.rm = rm
        self._related_product_cache = {}
        self._not_enough_recs = 0

    def __str__(self):
        return 'Keyword Recommender with %s[N=%d]' % (self.rm, self.limit)

    def use_keywords(self):
        return True

    def preprocess(self, query_train_table):
        self.query_train_table = query_train_table
        # empty cache so that cache from last round does not interfere with next round
        self._related_product_cache = {}
        self._not_enough_recs = 0

        self.dbm.begin()
        self.dbm.query('TRUNCATE TABLE keyword');
        self.dbm.query('TRUNCATE TABLE keyword_query');
        self.dbm.query('TRUNCATE TABLE keyword_product_weight');

        # these methods can be overridden by sub-classes
        self._build_keyword_product_mapping()
        self._build_product_keyword_mapping()
        self._measure_relevance()

        self.dbm.commit()

    @timeit
    def _build_keyword_product_mapping(self):
        self.keyword_count = defaultdict(int)
        self.keyword_product_count = defaultdict(lambda: defaultdict(int))
        for qrow in self.dbm.get_rows('SELECT id, query FROM %s' % self.query_train_table):
            # GROUP_CONCAT returns a comma-separeted string
            products = [(qprow['product_name'], qprow['sequences']) for qprow in self.dbm.get_rows('SELECT product_name, GROUP_CONCAT(sequence) AS sequences FROM query_product WHERE query_id = %s GROUP BY product_name', (qrow['id'],))]

            # remove duplicate
            keywords = set(self.ws.segment(qrow['query']))
            for kw in keywords:
                self.keyword_count[kw] += 1
                # store keyword-query relations in db
                self.dbm.insert('INSERT INTO keyword_query (keyword, query_id) VALUES (%s, %s)', (kw, qrow['id']))

            for p, sequences in products:
                # get product sequence in this session
                count = self.get_browse_count(sequences)
                # update keyword_product_count
                for kw in keywords:
                    self.keyword_product_count[kw][p] += count

    def get_browse_count(self, sequences):
        """Overrideable by sub-class.
        Multiple browses in a session always count 1."""
        return 1

    @timeit
    def _build_product_keyword_mapping(self):
        # construct product_keyword_count
        # it's actually equivalent to keyword_product_count, but can let compute
        # related_product_count faster
        self.product_keyword_count = defaultdict(dict)
        for kw, dt in self.keyword_product_count.iteritems():
            for p, c in dt.iteritems():
                self.product_keyword_count[p][kw] = c

    @timeit
    def _measure_relevance(self):
        # calculate keyword-product relevance
        all_product_number = self.dbm.get_value('SELECT COUNT(DISTINCT product_name) FROM query_product')
        for keyword, count in self.keyword_count.iteritems():
            # will be used for statistics
            self.dbm.insert('INSERT INTO keyword (keyword, count) VALUES (%s, %s)', (keyword, count))
            related_product_number = len(self.keyword_product_count[keyword].keys())
            related_product_count = sum(self.keyword_product_count[keyword].values())

            for product, count in self.keyword_product_count[keyword].iteritems():
                related_keyword_number = len(self.product_keyword_count[product].keys())
                related_keyword_count = sum(self.product_keyword_count[product].values())

                # delegate to sub-classes
                relevance = self.rm.get_relevance(count, (related_product_number, related_product_count), (related_keyword_number, related_keyword_count), all_product_number)
                self.dbm.insert('INSERT INTO keyword_product_weight (keyword, product, weight) VALUES (%s, %s, %s)', (keyword, product, relevance))

    def round_statistics(self):
        """Get number of query, keywords, products, keyword-product relations of current round."""
        n_query = self.dbm.get_value("SELECT COUNT(*) FROM %s" % self.query_train_table)
        n_keyword = self.dbm.get_value("SELECT COUNT(*) FROM keyword")
        n_product = self.dbm.get_value("SELECT COUNT(DISTINCT product) FROM keyword_product_weight")
        n_relation = self.dbm.get_value("SELECT COUNT(*) FROM keyword_product_weight")

        print 'Round statistics: query: %d (not enough %d), keyword: %d, product: %d, relation: %d, A/M: %.2f%%' % (n_query, self._not_enough_recs, n_keyword, n_product, n_relation, 100.0*n_relation / (n_keyword*n_product))

    def recommend(self, query):
        keywords = self.ws.segment(query)
        product_weight = defaultdict(float)
        # gather product weights
        for kw in keywords:
            for product, weight in self.__fetch_related_products(kw):
                product_weight[product] += weight

        # convert dict to list for sorting
        product_weight_list = [item for item in product_weight.iteritems()]
        product_weight_list.sort(key=lambda t: t[1], reverse=True)

        if len(product_weight_list) < self.limit:
            self._not_enough_recs += 1

        return product_weight_list[:self.limit]

    def __fetch_related_products(self, keyword):
        if not self._related_product_cache.has_key(keyword):
            self._related_product_cache[keyword] = [(row['product'], row['weight']) for row in self.dbm.get_rows('SELECT product, weight FROM keyword_product_weight WHERE keyword = %s', (keyword,))]
        return self._related_product_cache[keyword]

class KeywordRecommenderHottestFallback(KeywordRecommender):
    """A recommender which uses KeywordRecommender's recommendations first,
    but turns to HottestRecommender if its recommendations are not enough."""

    def __init__(self, *args):
        """Identical to that of KeywordRecommender"""
        super(KeywordRecommenderHottestFallback, self).__init__(*args)
        self.hottest_recommender = HottestRecommender(*args)

    def __str__(self):
        return 'Keyword Recommender with Hottest Recommender fallback with %s[N=%d]' % (self.rm, self.limit)

    def preprocess(self, query_train_table):
        super(KeywordRecommenderHottestFallback, self).preprocess(query_train_table)
        self.hottest_recommender.preprocess(query_train_table)

    def recommend(self, query):
        recommendations = super(KeywordRecommenderHottestFallback, self).recommend(query)
        num_rec = len(recommendations)
        if num_rec == self.limit:
            return recommendations

        # ask HottestRecommender for more
        # note that create list in order not to break HottestRecommender.recommend_list
        hot_recommendations = self.hottest_recommender.recommend(query)[:self.limit-num_rec]

        # ensure hot_recommendations's weight is no greater than any from keyword recommendations
        max_hot_rec_weight = hot_recommendations[0][1]
        min_key_rec_weight = recommendations[-1][1] if num_rec > 0 else max_hot_rec_weight
        recommendations.extend((t[0], 1.0*min_key_rec_weight*t[1]/max_hot_rec_weight) for t in hot_recommendations)

        return recommendations

class WeightedSequenceRelevanceMixin(object):
    @timeit
    def _measure_relevance(self):
        # calculate keyword-product relevance
        all_product_number = self.dbm.get_value('SELECT COUNT(DISTINCT product_name) FROM query_product')
        for keyword, count in self.keyword_count.iteritems():
            self.dbm.insert('INSERT INTO keyword (keyword, count) VALUES (%s, %s)', (keyword, count))
            related_product_number = len(self.keyword_product_count[keyword].keys())
            related_product_count = sum(self.keyword_product_count[keyword].values())

            for product, count in self.keyword_product_count[keyword].iteritems():
                related_keyword_number = len(self.product_keyword_count[product].keys())
                related_keyword_count = sum(self.product_keyword_count[product].values())
                # get average sequence from database
                # TODO: very inefficient, get a group all average sequences for a keyword at once
                #avg_sequence = self.dbm.get_value('select avg(sequence) from query_product where product_name = %s AND query_id in (select query_id from keyword_query where keyword = %s)', (product, keyword))
                avg_sequence = 1

                relevance = self.rm.get_relevance(count, (related_product_number, related_product_count), (related_keyword_number, related_keyword_count), all_product_number, avg_sequence)

                # sub-class can override sequence_weight
                relevance *= self.sequence_weight(avg_sequence)
                self.dbm.insert('INSERT INTO keyword_product_weight (keyword, product, weight) VALUES (%s, %s, %s)', (keyword, product, relevance))

    def sequence_weight(self, avg_sequence):
        return 1

# ensure WSRM._measure_relevance will be called with putting it before KeywordRecommender
# ref: http://python-history.blogspot.com/2010/06/method-resolution-order.html
class SequenceKeywordRecommender(WeightedSequenceRelevanceMixin, KeywordRecommender):
    """This recommender weights browse count by distribution of sequence."""

    @timeit
    def preprocess(self, query_train_table):
        # first, get sequence distribution
        max_occurrence = self.dbm.get_value('SELECT MAX(c) FROM (SELECT sequence, COUNT(sequence) c FROM query_product WHERE query_id IN (SELECT id FROM %s) GROUP BY sequence) T' % query_train_table)
        self.sequence_dist = {row['sequence']: float(row['ratio']) for row in self.dbm.get_rows('SELECT sequence, COUNT(sequence)/%d ratio FROM query_product WHERE query_id IN (SELECT id FROM %s) GROUP BY sequence' % (max_occurrence,query_train_table))}

        # then, call KeywordRecommender's preprocess
        super(SequenceKeywordRecommender, self).preprocess(query_train_table)

    def get_browse_count(self, sequences):
        """Multiple browses in a session always count 1."""
        return sum((self.sequence_dist[int(seq)] for seq in sequences.split(',')))

    def __str__(self):
        return 'Sequenced Keyword Recommender with %s[N=%d]' % (self.rm, self.limit)

class RelevanceMeasure(object):
    """Defines the RelevanceMeasure interface."""

    def get_relevance(self, count, related_product_info, related_keyword_info, all_product_number, *args):
        """
        @param count number of times the keyword visit the product
        @param related_product_info the tuple (related_product_number, related_product_count)
        @param related_keyword_info the tuple (related_keyword_number, related_keyword_count)
        @param all_product_number number of all products
        """
        raise NotImplemented

class BCMeasure(RelevanceMeasure):
    def get_relevance(self, count, *ignored):
        return count

    def __str__(self):
        return 'BC'

class BCIPFMeasure(RelevanceMeasure):
    def get_relevance(self, count, related_product_info, related_keyword_info, all_product_number, *args):
        ipf = math.log(1.0 * all_product_number / related_product_info[0])
        return count * ipf

    def __str__(self):
        return 'BC-IPF'

class BFIPFMeasure(RelevanceMeasure):
    def get_relevance(self, count, related_product_info, related_keyword_info, all_product_number, *args):
        bf = 1.0 * count / related_keyword_info[1]
        ipf = math.log(1.0 * all_product_number / related_product_info[0])
        return bf * ipf

    def __str__(self):
        return 'BF-IPF'

if __name__ == '__main__':
    import config
    from database import DatabaseManager
    from word_segment import SpaceWordSegmenter
    dbm = DatabaseManager(config.DB_HOST, config.DB_USER, config.DB_PASSWORD, config.DB_NAME)
    try:
        word_segmenter = SpaceWordSegmenter()
        rmeasure = BCIPFMeasure()
        recommender = KeywordRecommender(10, dbm, word_segmenter, rmeasure)
        recommender.preprocess('query_train')
    finally:
        dbm.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
Sisrec.py
Movies recomendation engine
'''

from typing import List  # noqa: F401
from scipy.spatial.distance import cosine
from concurrent.futures import ProcessPoolExecutor
import sys
import math
import json

# Importance of each column
DEFAULT_RATIO = {'director': 1,
                 'genres': 2,
                 'imdb_keywords': 3,
                 'lsa': 10}

def union(list_a: list, list_b: list) -> list:
    '''Return the union of two lists'''
    if list_a is None:
        list_a = [None]
    if list_b is None:
        list_b = [None]
    return list(set(list_a) | set(list_b))

def ordenar(comparacion: dict, length: int = 10) -> dict:
    '''Sort the results'''

    # Save 10 films ordered by similarity
    orden_similitud = sorted(comparacion['pels'],
                             key=lambda x: x['similitud'],
                             reverse=True)[:length]

    comparacion['pels'] = orden_similitud
    #print([x['pel']['title'] for x in comparacion['pels']])
    return comparacion

def compare(mov: dict, pel: dict, ratio: dict = DEFAULT_RATIO) -> float:
    '''Compute similarity between two movies'''

    similitudtotal = 0.0

    # Compare each element of each column which has a ratio assigned. For every
    # match the similarity increases. Basically Jaccard index of every column.
    try:
        # LSA compared separately!
        for campo_a, campo_b in [
                (x, y) for x, y
                in zip(mov.items(), pel.items()) if x[0]
                in ratio and x[0] is not 'lsa'
        ]:
            total = len(union(campo_a[1], campo_b[1]))
            if total > 0:
                coincidente = len(
                    [(x, y) for x in campo_a[1] for y in campo_b[1] if x == y]
                )
                similitudtotal += (coincidente/total) * ratio[campo_a[0]]

        # Only compare LSA if configured to do so
        if 'lsa' in ratio:
            # LSA is comapred using cosine distance
            syn_similarity = (
                1 - cosine(
                    pel['lsa'],
                    mov['lsa']
                )
            ) * ratio['lsa']
            # Sometimes LSA is not valid and syn_simality = NaN
            if not math.isnan(float(syn_similarity)):
                similitudtotal += syn_similarity

    except Exception:
        return 0.0

    return similitudtotal

def rec(mov: dict, pel: dict, ratio: dict = None) -> dict:
    similitudtotal: float = 0.0

    if ratio is not None:
        similitudtotal = compare(mov, pel, ratio)
    else:
        similitudtotal = compare(mov, pel)

    return {
        "similitud": similitudtotal,
        "pel": pel
    }

class Recommender:
    def __init__(self) -> None:
        print("loading")
        sys.stdout.flush()
        self.peliculas: List[dict] = self.load()

    def get_movies(self):
        r = []
        for pel in self.peliculas:
            r.append(
                {
                    'title': pel['title'],
                    'poster': pel['poster'],
                    'director': pel['director'],
                    'genres': ", ".join(pel['genres']),
                    'year': pel['year'],
                    'rating': pel['rating'],
                    'url': pel['url']
                }
            )
        return r

    def film_quantity(self):
        return len(self.peliculas)

    def recommend(self, movie, ratio=None, film_quantity = 10):
        print(movie)
        mov = None
        for m in self.peliculas:
            if m['title'].replace(u'\u00a0', '') == movie:
                mov = m

        if mov is None:
            return None

        comparacion: list = []
        # Compare mov with all films except itself
        with ProcessPoolExecutor() as executor:
            for pel in [
                    x for x in self.peliculas if x['title'] != mov['title']
            ]:
                if ratio is not None:
                    comparacion.append(executor.submit(rec, mov,
                                                       pel, ratio).result())
                else:
                    comparacion.append(executor.submit(rec, mov,
                                                       pel).result())

        comp = {'mov': mov, 'pels': comparacion}
        r = ordenar(comp, film_quantity)

        return ([x['pel'] for x in r['pels']])

    def ask_movie(self, peliculas: list) -> dict:
        '''Select a movie'''

        print("Select a movie")
        print("Press 0 for the complete list")
        print("Press a letter for the movies that start with that letter")

        while True:
            letter: str = input()

            # If the user inputs 0 the complete list
            # of movies and their id's is displayed
            if letter == "0":
                for pel in peliculas:
                    print(pel['title'])
            else:
                # Return first id coincidence
                # None if id is not found
                for pel in peliculas:
                    if pel['title'].replace(u'\u00a0', '') == letter:
                        return pel

    def clean(self, item: list) -> list:
        '''Each row is a string. Lists must be separated and stripped'''
        item = [x.replace("'", "")
                .replace('"', '')
                .replace('[', '')
                .replace(']', '')
                .split(',') for x in item]

        return item

    def load(self) -> list:
        '''Load every synopsis in a list'''
        peliculas: list = []

        with open('./db.json', 'r') as in_file:
            for film in json.load(in_file):
                peliculas.append(film)

        # Sort films by ID
        # Useful when asking the user to select a movie
        peliculas = sorted(peliculas, key=lambda x: x['title'])
        return peliculas

    def main_loop(self, peliculas: list) -> dict:
        '''Main loop'''

        mov: dict = self.ask_movie(peliculas)
        print(mov['title'])

        comparacion: list = []
        # Compare mov with all films except itself
        for pel in [x for x in peliculas if x['title'] != mov['title']]:
            similitudtotal = compare(mov, pel)
            comparacion.append({"similitud": similitudtotal, "pel": pel})

        comp = {'mov': mov, 'pels': comparacion}
        return comp

    def get_best_keyword(self, key: str, originals: dict) -> str:
        word = ''
        best = 0

        # TODO: Best keywords can be pre-computed, we shouldn't do this every
        # iteration

        # originals.json contains keys of lemmatized words and values of
        # original words. These words are associated with their number
        # of occurences in the downloaded texts.
        # Get the original word which is more common
        for keyword in originals[key].items():
            if keyword[1] > best:
                word = keyword[0]

        return word

    def main(self) -> None:
        '''Main'''

        while True:
            comparacion: dict = self.main_loop(self.peliculas)
            ordenar(comparacion)

if __name__ == "__main__":
    recommender: Recommender = Recommender()
    recommender.main()
<commit_msg>Use != instead of is not in recommender.py<commit_after>#!/usr/bin/env python

'''
Sisrec.py
Movies recomendation engine
'''

from typing import List  # noqa: F401
from scipy.spatial.distance import cosine
from concurrent.futures import ProcessPoolExecutor
import sys
import math
import json

# Importance of each column
DEFAULT_RATIO = {'director': 1,
                 'genres': 2,
                 'imdb_keywords': 3,
                 'lsa': 10}

def union(list_a: list, list_b: list) -> list:
    '''Return the union of two lists'''
    if list_a is None:
        list_a = [None]
    if list_b is None:
        list_b = [None]
    return list(set(list_a) | set(list_b))

def ordenar(comparacion: dict, length: int = 10) -> dict:
    '''Sort the results'''

    # Save 10 films ordered by similarity
    orden_similitud = sorted(comparacion['pels'],
                             key=lambda x: x['similitud'],
                             reverse=True)[:length]

    comparacion['pels'] = orden_similitud
    #print([x['pel']['title'] for x in comparacion['pels']])
    return comparacion

def compare(mov: dict, pel: dict, ratio: dict = DEFAULT_RATIO) -> float:
    '''Compute similarity between two movies'''

    similitudtotal = 0.0

    # Compare each element of each column which has a ratio assigned. For every
    # match the similarity increases. Basically Jaccard index of every column.
    try:
        # LSA compared separately!
        for campo_a, campo_b in [
                (x, y) for x, y
                in zip(mov.items(), pel.items()) if x[0]
                in ratio and x[0] != 'lsa'
        ]:
            total = len(union(campo_a[1], campo_b[1]))
            if total > 0:
                coincidente = len(
                    [(x, y) for x in campo_a[1] for y in campo_b[1] if x == y]
                )
                similitudtotal += (coincidente/total) * ratio[campo_a[0]]

        # Only compare LSA if configured to do so
        if 'lsa' in ratio:
            # LSA is comapred using cosine distance
            syn_similarity = (
                1 - cosine(
                    pel['lsa'],
                    mov['lsa']
                )
            ) * ratio['lsa']
            # Sometimes LSA is not valid and syn_simality = NaN
            if not math.isnan(float(syn_similarity)):
                similitudtotal += syn_similarity

    except Exception:
        return 0.0

    return similitudtotal

def rec(mov: dict, pel: dict, ratio: dict = None) -> dict:
    similitudtotal: float = 0.0

    if ratio is not None:
        similitudtotal = compare(mov, pel, ratio)
    else:
        similitudtotal = compare(mov, pel)

    return {
        "similitud": similitudtotal,
        "pel": pel
    }

class Recommender:
    def __init__(self) -> None:
        print("loading")
        sys.stdout.flush()
        self.peliculas: List[dict] = self.load()

    def get_movies(self):
        r = []
        for pel in self.peliculas:
            r.append(
                {
                    'title': pel['title'],
                    'poster': pel['poster'],
                    'director': pel['director'],
                    'genres': ", ".join(pel['genres']),
                    'year': pel['year'],
                    'rating': pel['rating'],
                    'url': pel['url']
                }
            )
        return r

    def film_quantity(self):
        return len(self.peliculas)

    def recommend(self, movie, ratio=None, film_quantity = 10):
        print(movie)
        mov = None
        for m in self.peliculas:
            if m['title'].replace(u'\u00a0', '') == movie:
                mov = m

        if mov is None:
            return None

        comparacion: list = []
        # Compare mov with all films except itself
        with ProcessPoolExecutor() as executor:
            for pel in [
                    x for x in self.peliculas if x['title'] != mov['title']
            ]:
                if ratio is not None:
                    comparacion.append(executor.submit(rec, mov,
                                                       pel, ratio).result())
                else:
                    comparacion.append(executor.submit(rec, mov,
                                                       pel).result())

        comp = {'mov': mov, 'pels': comparacion}
        r = ordenar(comp, film_quantity)

        return ([x['pel'] for x in r['pels']])

    def ask_movie(self, peliculas: list) -> dict:
        '''Select a movie'''

        print("Select a movie")
        print("Press 0 for the complete list")
        print("Press a letter for the movies that start with that letter")

        while True:
            letter: str = input()

            # If the user inputs 0 the complete list
            # of movies and their id's is displayed
            if letter == "0":
                for pel in peliculas:
                    print(pel['title'])
            else:
                # Return first id coincidence
                # None if id is not found
                for pel in peliculas:
                    if pel['title'].replace(u'\u00a0', '') == letter:
                        return pel

    def clean(self, item: list) -> list:
        '''Each row is a string. Lists must be separated and stripped'''
        item = [x.replace("'", "")
                .replace('"', '')
                .replace('[', '')
                .replace(']', '')
                .split(',') for x in item]

        return item

    def load(self) -> list:
        '''Load every synopsis in a list'''
        peliculas: list = []

        with open('./db.json', 'r') as in_file:
            for film in json.load(in_file):
                peliculas.append(film)

        # Sort films by ID
        # Useful when asking the user to select a movie
        peliculas = sorted(peliculas, key=lambda x: x['title'])
        return peliculas

    def main_loop(self, peliculas: list) -> dict:
        '''Main loop'''

        mov: dict = self.ask_movie(peliculas)
        print(mov['title'])

        comparacion: list = []
        # Compare mov with all films except itself
        for pel in [x for x in peliculas if x['title'] != mov['title']]:
            similitudtotal = compare(mov, pel)
            comparacion.append({"similitud": similitudtotal, "pel": pel})

        comp = {'mov': mov, 'pels': comparacion}
        return comp

    def get_best_keyword(self, key: str, originals: dict) -> str:
        word = ''
        best = 0

        # TODO: Best keywords can be pre-computed, we shouldn't do this every
        # iteration

        # originals.json contains keys of lemmatized words and values of
        # original words. These words are associated with their number
        # of occurences in the downloaded texts.
        # Get the original word which is more common
        for keyword in originals[key].items():
            if keyword[1] > best:
                word = keyword[0]

        return word

    def main(self) -> None:
        '''Main'''

        while True:
            comparacion: dict = self.main_loop(self.peliculas)
            ordenar(comparacion)

if __name__ == "__main__":
    recommender: Recommender = Recommender()
    recommender.main()
<|endoftext|>


<commit_before>import imp
from carbon.relayrules import loadRelayRules
from carbon.hashing import ConsistentHashRing

class DatapointRouter:
  "Interface for datapoint routing logic implementations"

  def addDestination(self, destination):
    "destination is a (host, port, instance) triple"

  def removeDestination(self, destination):
    "destination is a (host, port, instance) triple"

  def getDestinations(self, key):
    """Generate the destinations where the given routing key should map to. Only
    destinations which are configured (addDestination has been called for it)
    may be generated by this method."""

class RelayRulesRouter(DatapointRouter):
  def __init__(self, rules_path):
    self.rules_path = rules_path
    self.rules = loadRelayRules(rules_path)
    self.destinations = set()

  def addDestination(self, destination):
    self.destinations.add(destination)

  def removeDestination(self, destination):
    self.destinations.discard(destination)

  def getDestinations(self, key):
    for rule in self.rules:
      if rule.matches(key):
        for destination in rule.destinations:
          if destination in self.destinations:
            yield destination
        if not rule.continue_matching:
          return

class ConsistentHashingRouter(DatapointRouter):
  def __init__(self, replication_factor=1):
    self.replication_factor = int(replication_factor)
    self.instance_ports = {} # { (server, instance) : port }
    self.ring = ConsistentHashRing([])

  def addDestination(self, destination):
    (server, port, instance) = destination
    if (server, instance) in self.instance_ports:
      raise Exception("destination instance (%s, %s) already configured" % (server, instance))
    self.instance_ports[ (server, instance) ] = port
    self.ring.add_node( (server, instance) )

  def removeDestination(self, destination):
    (server, port, instance) = destination
    if (server, instance) not in self.instance_ports:
      raise Exception("destination instance (%s, %s) not configured" % (server, instance))
    del self.instance_ports[ (server, instance) ]
    self.ring.remove_node( (server, instance) )

  def getDestinations(self, metric):
    key = self.getKey(metric)

    used_servers = set()
    for (server, instance) in self.ring.get_nodes(key):
      if server in used_servers:
        continue
      else:
        used_servers.add(server)
        port = self.instance_ports[ (server, instance) ]
        yield (server, port, instance)

      if len(used_servers) >= self.replication_factor:
        return

  def getKey(self, metric):
    return metric

  def setKeyFunction(self, func):
    self.getKey = func

  def setKeyFunctionFromModule(self, keyfunc_spec):
    module_path, func_name = keyfunc_spec.rsplit(':', 1)
    module_file = open(module_path, 'U')
    description = ('.py', 'U', imp.PY_SOURCE)
    module = imp.load_module('keyfunc_module', module_file, module_path, description)
    keyfunc = getattr(module, func_name)
    self.setKeyFunction(keyfunc)

class AggregatedConsistentHashingRouter(DatapointRouter):
  def __init__(self, agg_rules_manager, replication_factor=1):
    self.hash_router = ConsistentHashingRouter(replication_factor)
    self.agg_rules_manager = agg_rules_manager

  def addDestination(self, destination):
    self.hash_router.addDestination(destination)

  def removeDestination(self, destination):
    self.hash_router.removeDestination(destination)

  def getDestinations(self, key):
    # resolve metric to aggregate forms
    aggregate_metrics = []
    for rule in self.agg_rules_manager.rules:
      aggregate_metric = rule.get_aggregate_metric(key)
      if aggregate_metric is None:
        continue
      else:
        aggregate_metrics.append(aggregate_metric)

    # get consistent hashing destinations based on aggregate forms
    destinations = set()
    for aggregate_metric in aggregate_metrics:
      for destination in self.hash_router.getDestinations(aggregate_metric):
        destinations.add(destination)

    for destination in destinations:
      yield destination
<commit_msg>aggregated-consistent-hashing unmatched passthru<commit_after>import imp
from carbon.relayrules import loadRelayRules
from carbon.hashing import ConsistentHashRing

class DatapointRouter:
  "Interface for datapoint routing logic implementations"

  def addDestination(self, destination):
    "destination is a (host, port, instance) triple"

  def removeDestination(self, destination):
    "destination is a (host, port, instance) triple"

  def getDestinations(self, key):
    """Generate the destinations where the given routing key should map to. Only
    destinations which are configured (addDestination has been called for it)
    may be generated by this method."""

class RelayRulesRouter(DatapointRouter):
  def __init__(self, rules_path):
    self.rules_path = rules_path
    self.rules = loadRelayRules(rules_path)
    self.destinations = set()

  def addDestination(self, destination):
    self.destinations.add(destination)

  def removeDestination(self, destination):
    self.destinations.discard(destination)

  def getDestinations(self, key):
    for rule in self.rules:
      if rule.matches(key):
        for destination in rule.destinations:
          if destination in self.destinations:
            yield destination
        if not rule.continue_matching:
          return

class ConsistentHashingRouter(DatapointRouter):
  def __init__(self, replication_factor=1):
    self.replication_factor = int(replication_factor)
    self.instance_ports = {} # { (server, instance) : port }
    self.ring = ConsistentHashRing([])

  def addDestination(self, destination):
    (server, port, instance) = destination
    if (server, instance) in self.instance_ports:
      raise Exception("destination instance (%s, %s) already configured" % (server, instance))
    self.instance_ports[ (server, instance) ] = port
    self.ring.add_node( (server, instance) )

  def removeDestination(self, destination):
    (server, port, instance) = destination
    if (server, instance) not in self.instance_ports:
      raise Exception("destination instance (%s, %s) not configured" % (server, instance))
    del self.instance_ports[ (server, instance) ]
    self.ring.remove_node( (server, instance) )

  def getDestinations(self, metric):
    key = self.getKey(metric)

    used_servers = set()
    for (server, instance) in self.ring.get_nodes(key):
      if server in used_servers:
        continue
      else:
        used_servers.add(server)
        port = self.instance_ports[ (server, instance) ]
        yield (server, port, instance)

      if len(used_servers) >= self.replication_factor:
        return

  def getKey(self, metric):
    return metric

  def setKeyFunction(self, func):
    self.getKey = func

  def setKeyFunctionFromModule(self, keyfunc_spec):
    module_path, func_name = keyfunc_spec.rsplit(':', 1)
    module_file = open(module_path, 'U')
    description = ('.py', 'U', imp.PY_SOURCE)
    module = imp.load_module('keyfunc_module', module_file, module_path, description)
    keyfunc = getattr(module, func_name)
    self.setKeyFunction(keyfunc)

class AggregatedConsistentHashingRouter(DatapointRouter):
  def __init__(self, agg_rules_manager, replication_factor=1):
    self.hash_router = ConsistentHashingRouter(replication_factor)
    self.agg_rules_manager = agg_rules_manager

  def addDestination(self, destination):
    self.hash_router.addDestination(destination)

  def removeDestination(self, destination):
    self.hash_router.removeDestination(destination)

  def getDestinations(self, key):
    # resolve metric to aggregate forms
    resolved_metrics = []
    for rule in self.agg_rules_manager.rules:
      aggregate_metric = rule.get_aggregate_metric(key)
      if aggregate_metric is None:
        continue
      else:
        resolved_metrics.append(aggregate_metric)

    # if the metric will not be aggregated, send it raw
    # (will pass through aggregation)
    if len(resolved_metrics) == 0:
      resolved_metrics.append(key)

    # get consistent hashing destinations based on aggregate forms
    destinations = set()
    for resolved_metric in resolved_metrics:
      for destination in self.hash_router.getDestinations(resolved_metric):
        destinations.add(destination)

    for destination in destinations:
      yield destination
<|endoftext|>


<commit_before>from zope.interface import implements
from twisted.web.client import getPage
from twisted.python import log

import re
import json

try:
    import urllib.parse as parser
except ImportError:
    import urllib as parser

from core.interface import IPlugin, IMsgHandler
from core.pluginmanager import plugin_manager

class Urban(object):
    implements(IMsgHandler)

    name = 'urbandict'
    version = '0.1'
    author = 'fbs'
    command = 'urban'
    BASEURL = 'http://api.urbandictionary.com/v0/define?term=%s'

    def accepts_command(self, cmd):
        if cmd == self.command:
            return True
        return False

    def _repairjson(self, original):
        original = original.replace(r'\r','').replace(r'\n','')
        original = original.replace('\\x', '\\u00')
        regex = re.compile(r'\\(?![/u"])')
        fixed = regex.sub(r"\\\\", original)
        return fixed

    def handle(self, proto, user, channel, args):
        user = user[0]
        url = self.BASEURL % (''.join(args))
        url = parser.quote_plus(url, ':=()/?')
        d = getPage(url)

        d.addCallback(self.parse_json, proto, user, channel)
        d.addErrback(self.fetch_failed, proto, user, channel)
        d.addCallback(self.send_definition, proto, user, channel)
        d.addErrback(self.send_definition_failed, proto, user, channel)

    def parse_json(self, page, proto, user, channel):
        jsondata = self._repairjson(page.decode('utf-8'))
        jsondata = json.loads(jsondata)
        return jsondata

    def send_definition(self, page, proto, user, channel):
        results = page.get('result_type')
        definitions = page.get('list')

        if results == 'exact':
            output = definitions[0]['definition'].encode('ascii').strip()
        else:
            output = "No results found"

        proto.msg(channel, "%s: %s" % (user, output))

    def parse_json_failed(self, failure, proto, user, channel):
        proto.msg(channel, '%s: failed to parse JSON data' % user)
        log.msg('Failed to parse json' % failure)

    def send_definition_failed(self, failure, proto, user, channel):
        proto.msg(channel, '%s: failed to find definitions' % user)
        log.msg('Failed to find definitions ', failure)

    def fetch_failed(self, failure, proto, user, channel):
        proto.msg(channel, '%s: Couldn\'t fetch json' % user)
        log.msg("Failed to fetch urban json: ", failure)

urban = Urban()
plugin_manager.register(urban)
<commit_msg>Plugin urbandict improved<commit_after>from zope.interface import implements
from twisted.web.client import getPage
from twisted.python import log
from twisted.web import error as web_error
from twisted.internet import error as internet_error

import re
import json

try:
    import urllib.parse as parser
except ImportError:
    import urllib as parser

from core.interface import IPlugin, IMsgHandler
from core.pluginmanager import plugin_manager

def _make_apiurl(query):
    APIURL = 'http://api.urbandictionary.com/v0/define?term=%s'
    return parser.quote_plus(APIURL % query, ':=()/?')

def _make_weburl(query):
    WEBURL = 'http://www.urbandictionary.com/define.php?term=%s'
    return parser.quote_plus(WEBURL % query, ':=()/?')

# Stolen from somewhere
def _repairjson(original):
    original = original.replace(r'\r','').replace(r'\n','')
    original = original.replace('\\x', '\\u00')
    regex = re.compile(r'\\(?![/u"])')
    fixed = regex.sub(r"\\\\", original)
    return fixed

class Urban(object):
    implements(IMsgHandler)

    name = 'urbandict'
    version = '0.1'
    author = 'fbs'
    command = 'urban'

    def accepts_command(self, cmd):
        if cmd == self.command:
            return True
        return False

    def handle(self, proto, user, channel, args):
        user = user[0]
        self.query  = ' '.join(args)
        self.weburl = _make_weburl(self.query)
        self.apiurl = _make_apiurl(self.query)

        d = getPage(self.apiurl)
        d.addCallback(self.handle_page, proto, user, channel)
        d.addCallback(self.search_page, proto, user, channel)
        d.addErrback(self.errback, proto, user, channel)

    def handle_page(self, page, proto, user, channel):
        jsondata = _repairjson(page.decode('utf-8'))
        jsondata = json.loads(jsondata)
        return jsondata

    def search_page(self, page, proto, user, channel):
        results = page.get('result_type')
        definitions = page.get('list')

        if results == 'exact':
            output = definitions[0]['definition'].encode('ascii').strip()
            log.msg(self.weburl)
            proto.msg(channel, '%s: %s, see %s for more.' %
                      (user, output, self.weburl))
        else:
            proto.msg(channel, '%s: Sorry, no results found' % user)

    def errback(self, failure, proto, user, channel):
        log.err('Plugin: %s: type: %s | message: %s' %
                (self.name, failure.type, failure.getErrorMessage()))

        r = failure.check(web_error.Error,
                         internet_error.ConnectError,
                         ValueError)
        if r == ValueError:
            proto.msg(channel, '%s: Urban: Couldn\'t decode the JSON, sorry :(' % user)
        elif r == web_error.Error:
            proto.msg(channel,  '%s: Urban: Couldn\'t fetch the page, sorry :(' % user)
        elif r == internet_error.ConnectError:
            proto.msg(channel, '%s: Urban: Couldn\'t connect to the server, sorry :(' % user)

urban = Urban()
plugin_manager.register(urban)
<|endoftext|>