<commit_before>#!/usr/bin/env python
#---------------------------------------------------------------------------------------------------
# Collects all the necessary data to generate rankings for all datasets in the AnalysisOps space.
#---------------------------------------------------------------------------------------------------
import sys, os, math, datetime, sqlite3, operator, random, ConfigParser
import phedexData, popDbData, dbApi

class weeklyRockerBoard():
    def __init__(self):
        config = ConfigParser.RawConfigParser()
        config.read('/usr/local/IntelROCCS/DataDealer/intelroccs.cfg')
        self.rankingsCachePath = config.get('DataDealer', 'cache')
        self.budget = config.getint('DataDealer', 'budget')
        self.lowerBudget = config.getint('DataDealer', 'lower_budget')
        self.lowerThreshold = config.getfloat('DataDealer', 'lower_threshold')
        self.upperThreshold = config.getfloat('DataDealer', 'upper_threshold')
        self.limit = config.getfloat('DataDealer', 'limit')
        self.upperLimit = config.getfloat('DataDealer', 'upper_limit')
        self.phedexData = phedexData.phedexData()
        self.popDbData = popDbData.popDbData()
        self.dbApi = dbApi.dbApi()

#===================================================================================================
#  H E L P E R S
#===================================================================================================
    def weightedChoice(self, choices):
        total = sum(w for c, w in choices.items())
        r = random.uniform(0, total)
        upto = 0
        for c, w in choices.items():
            if upto + w > r:
                return c
            upto += w

    def getPopularity(self, datasetName):
        popularity = 0
        today = datetime.date.today()
        date = today - datetime.timedelta(days=1)
        cpuh1 = self.popDbData.getDatasetCpus(datasetName, date.strftime('%Y-%m-%d'))
        for i in range(2, 15):
            date = today - datetime.timedelta(days=i)
            cpuh2 = self.popDbData.getDatasetCpus(datasetName, date.strftime('%Y-%m-%d'))
            popularity += cpuh1 - cpuh2
            cpuh1 = cpuh2
        return popularity

    def rankingsCache(self, datasetRankings, siteRankings):
        if not os.path.exists(self.rankingsCachePath):
            os.makedirs(self.rankingsCachePath)
        cacheFile = "%s/%s.db" % (self.rankingsCachePath, "rankingsCache")
        if os.path.isfile(cacheFile):
            os.remove(cacheFile)
        rankingsCache = sqlite3.connect(cacheFile)
        with rankingsCache:
            cur = rankingsCache.cursor()
            cur.execute('CREATE TABLE IF NOT EXISTS Datasets (DatasetName TEXT UNIQUE, Rank REAL)')
            cur.execute('CREATE TABLE IF NOT EXISTS Sites (SiteName TEXT UNIQUE, Rank REAL)')
            for datasetName, rank in datasetRankings.items():
                cur.execute('INSERT INTO Datasets(DatasetName, Rank) VALUES(?, ?)', (datasetName, rank))
            for siteName, rank in siteRankings.items():
                cur.execute('INSERT INTO Sites(SiteName, Rank) VALUES(?, ?)', (siteName, rank))

    def getDatasetRankings(self, datasets):
        alphaValues = dict()
        for datasetName in datasets:
            nReplicas = self.phedexData.getNumberReplicas(datasetName)
            sizeGb = self.phedexData.getDatasetSize(datasetName)
            popularity = self.getPopularity(datasetName)
            alpha = popularity/float(nReplicas*sizeGb)
            alphaValues[datasetName] = alpha
        mean = (1/len(alphaValues))*sum(v for v in alphaValues.values())
        datasetRankings = dict()
        for k, v in alphaValues.items():
            dev = v - mean
            datasetRankings[k] = dev
        return datasetRankings

    def getSiteRankings(self, sites, datasetRankings):
        siteRankings = dict()
        for siteName in sites:
            datasets = self.phedexData.getAnalysisOpsDatasetsAtSite(siteName)
            rank = sum(datasetRankings[d] for d in datasets)
            siteRankings[siteName] = rank
        return siteRankings

    def getNewReplicas(self, datasetRankings, siteRankings, totalQuota, totalUsed):
        subscriptions = dict()
        sizeSubscribedGb = 0
        maxRank = max(siteRankings.iteritems(), key=operator.itemgetter(1))[1]
        for siteName, rank in siteRankings.items():
            siteRankings[siteName] = maxRank - rank
        dataset = max(datasetRankings.iteritems(), key=operator.itemgetter(1))
        while (datasetRankings):
            datasetName = dataset[0]
            datasetSizeGb = self.phedexData.getDatasetSize(datasetName)
            if sizeSubscribedGb + datasetSizeGb > self.budget:
                break
            del datasetRankings[datasetName]
            siteRank = siteRankings
            invalidSites = self.phedexData.getSitesWithDataset(datasetName)
            for siteName in invalidSites:
                if siteName in siteRank:
                    del siteRank[siteName]
            siteName = self.weightedChoice(siteRank)
            sizeSubscribedGb += datasetSizeGb
            if siteName in subscriptions:
                subscriptions[siteName].append(datasetName)
            else:
                subscriptions[siteName] = [datasetName]
            dataset = max(datasetRankings.iteritems(), key=operator.itemgetter(1))
        return subscriptions

#===================================================================================================
#  M A I N
#===================================================================================================
    def weeklyRba(self, datasets, sites):
        subscriptions = []
        datasetRankings = self.getDatasetRankings(datasets)
        for i in range(10):
            dataset = max(datasetRankings.iteritems(), key=operator.itemgetter(1))
            print str(dataset[1]) + " " + str(dataset[0])
            del datasetRankings[dataset[0]]
        return subscriptions
        siteRankings = self.getSiteRankings(sites, datasetRankings)
        self.rankingsCache(datasetRankings, siteRankings)
        totalQuota = 0
        totalUsed = 0
        for siteName in sites:
            query = "SELECT Quotas.SizeTb FROM Quotas INNER JOIN Sites ON Quotas.SiteId=Sites.SiteId INNER JOIN Groups ON Groups.GroupId=Quotas.GroupId WHERE Sites.SiteName=%s AND Groups.GroupName=%s"
            values = [siteName, "AnalysisOps"]
            data = self.dbApi.dbQuery(query, values=values)
            totalQuota += data[0][0]*10**3
            used = self.phedexData.getSiteStorage(siteName)
            totalUsed += used
        subscriptions = self.getNewReplicas(datasetRankings, siteRankings, totalQuota, totalUsed)
        return subscriptions
<commit_msg>Testing site ranking<commit_after>#!/usr/bin/env python
#---------------------------------------------------------------------------------------------------
# Collects all the necessary data to generate rankings for all datasets in the AnalysisOps space.
#---------------------------------------------------------------------------------------------------
import sys, os, math, datetime, sqlite3, operator, random, ConfigParser
import phedexData, popDbData, dbApi

class weeklyRockerBoard():
    def __init__(self):
        config = ConfigParser.RawConfigParser()
        config.read('/usr/local/IntelROCCS/DataDealer/intelroccs.cfg')
        self.rankingsCachePath = config.get('DataDealer', 'cache')
        self.budget = config.getint('DataDealer', 'budget')
        self.lowerBudget = config.getint('DataDealer', 'lower_budget')
        self.lowerThreshold = config.getfloat('DataDealer', 'lower_threshold')
        self.upperThreshold = config.getfloat('DataDealer', 'upper_threshold')
        self.limit = config.getfloat('DataDealer', 'limit')
        self.upperLimit = config.getfloat('DataDealer', 'upper_limit')
        self.phedexData = phedexData.phedexData()
        self.popDbData = popDbData.popDbData()
        self.dbApi = dbApi.dbApi()

#===================================================================================================
#  H E L P E R S
#===================================================================================================
    def weightedChoice(self, choices):
        total = sum(w for c, w in choices.items())
        r = random.uniform(0, total)
        upto = 0
        for c, w in choices.items():
            if upto + w > r:
                return c
            upto += w

    def getPopularity(self, datasetName):
        popularity = 0
        today = datetime.date.today()
        date = today - datetime.timedelta(days=1)
        cpuh1 = self.popDbData.getDatasetCpus(datasetName, date.strftime('%Y-%m-%d'))
        for i in range(2, 15):
            date = today - datetime.timedelta(days=i)
            cpuh2 = self.popDbData.getDatasetCpus(datasetName, date.strftime('%Y-%m-%d'))
            popularity += cpuh1 - cpuh2
            cpuh1 = cpuh2
        return popularity

    def rankingsCache(self, datasetRankings, siteRankings):
        if not os.path.exists(self.rankingsCachePath):
            os.makedirs(self.rankingsCachePath)
        cacheFile = "%s/%s.db" % (self.rankingsCachePath, "rankingsCache")
        if os.path.isfile(cacheFile):
            os.remove(cacheFile)
        rankingsCache = sqlite3.connect(cacheFile)
        with rankingsCache:
            cur = rankingsCache.cursor()
            cur.execute('CREATE TABLE IF NOT EXISTS Datasets (DatasetName TEXT UNIQUE, Rank REAL)')
            cur.execute('CREATE TABLE IF NOT EXISTS Sites (SiteName TEXT UNIQUE, Rank REAL)')
            for datasetName, rank in datasetRankings.items():
                cur.execute('INSERT INTO Datasets(DatasetName, Rank) VALUES(?, ?)', (datasetName, rank))
            for siteName, rank in siteRankings.items():
                cur.execute('INSERT INTO Sites(SiteName, Rank) VALUES(?, ?)', (siteName, rank))

    def getDatasetRankings(self, datasets):
        alphaValues = dict()
        for datasetName in datasets:
            nReplicas = self.phedexData.getNumberReplicas(datasetName)
            sizeGb = self.phedexData.getDatasetSize(datasetName)
            popularity = self.getPopularity(datasetName)
            alpha = popularity/float(nReplicas*sizeGb)
            alphaValues[datasetName] = alpha
        mean = (1/len(alphaValues))*sum(v for v in alphaValues.values())
        datasetRankings = dict()
        for k, v in alphaValues.items():
            dev = v - mean
            datasetRankings[k] = dev
        return datasetRankings

    def getSiteRankings(self, sites, datasetRankings):
        siteRankings = dict()
        for siteName in sites:
            datasets = self.phedexData.getAnalysisOpsDatasetsAtSite(siteName)
            rank = sum(datasetRankings[d] for d in datasets)
            siteRankings[siteName] = rank
        return siteRankings

    def getNewReplicas(self, datasetRankings, siteRankings, totalQuota, totalUsed):
        subscriptions = dict()
        sizeSubscribedGb = 0
        maxRank = max(siteRankings.iteritems(), key=operator.itemgetter(1))[1]
        for siteName, rank in siteRankings.items():
            siteRankings[siteName] = maxRank - rank
        dataset = max(datasetRankings.iteritems(), key=operator.itemgetter(1))
        while (datasetRankings):
            datasetName = dataset[0]
            datasetSizeGb = self.phedexData.getDatasetSize(datasetName)
            if sizeSubscribedGb + datasetSizeGb > self.budget:
                break
            del datasetRankings[datasetName]
            siteRank = siteRankings
            invalidSites = self.phedexData.getSitesWithDataset(datasetName)
            for siteName in invalidSites:
                if siteName in siteRank:
                    del siteRank[siteName]
            siteName = self.weightedChoice(siteRank)
            sizeSubscribedGb += datasetSizeGb
            if siteName in subscriptions:
                subscriptions[siteName].append(datasetName)
            else:
                subscriptions[siteName] = [datasetName]
            dataset = max(datasetRankings.iteritems(), key=operator.itemgetter(1))
        return subscriptions

#===================================================================================================
#  M A I N
#===================================================================================================
    def weeklyRba(self, datasets, sites):
        subscriptions = []
        datasetRankings = self.getDatasetRankings(datasets)
        siteRankings = self.getSiteRankings(sites, datasetRankings)
        for siteName, rank in siteRankings.items():
            print str(rank) + " " + str(siteName)
        return subscriptions
        self.rankingsCache(datasetRankings, siteRankings)
        totalQuota = 0
        totalUsed = 0
        for siteName in sites:
            query = "SELECT Quotas.SizeTb FROM Quotas INNER JOIN Sites ON Quotas.SiteId=Sites.SiteId INNER JOIN Groups ON Groups.GroupId=Quotas.GroupId WHERE Sites.SiteName=%s AND Groups.GroupName=%s"
            values = [siteName, "AnalysisOps"]
            data = self.dbApi.dbQuery(query, values=values)
            totalQuota += data[0][0]*10**3
            used = self.phedexData.getSiteStorage(siteName)
            totalUsed += used
        subscriptions = self.getNewReplicas(datasetRankings, siteRankings, totalQuota, totalUsed)
        return subscriptions
<|endoftext|>


<commit_before>import pkg_resources
from pathlib import Path
import os
import subprocess
from Datasnakes.Tools.logit import LogIt
from Datasnakes.Manager.BioSQL.biosql_repo import sql
from Datasnakes.Manager.BioSQL.biosql_repo import scripts as sql_scripts

class BaseBioSQL(object):
    # TODO-ROB:  Organize the BioSQL files by driver/RDBMS
 # driver "mysql", "Pg", "Oracle", "SQLite"
    def __init__(self, database_name, database_type, driver):
        self.database_name = database_name
        self.database_type = database_type
        self.driver = driver
        self.biosqllog = LogIt().default(logname="BioSQL", logfile=None)

        self.ncbi_taxon_script = pkg_resources.resource_filename(sql_scripts.__name__, "load_ncbi_taxonomy.pl")
        pass

    def load_biosql_schema(self, cmd, schema_file):
        schema_cmd = "%s < %s" % (cmd, schema_file)
        schema_load = subprocess.Popen([schema_cmd], stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True,
                                       encoding='utf-8')
        error = schema_load.stderr.readlines()
        out = schema_load.stdout.readlines()

        self.biosqllog.info("Schema-Error: " + str(error))
        self.biosqllog.info("Schema-Out: " + str(out))
        return error, out

    def load_ncbi_taxonomy(self, cmd):
        # ./load_ncbi_taxonomy.pl --dbname bioseqdb --driver mysql --dbuser root --download true
        taxon_cmd = cmd
        taxon_load = subprocess.Popen([taxon_cmd], stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True,
                                      encoding='utf-8')

        error = taxon_load.stderr.readlines()
        out = taxon_load.stdout.readlines()

        self.biosqllog.info("Taxon-Error: " + str(error))
        self.biosqllog.info("Taxon-Out: " + str(out))
        return error, out
        pass

class SQLiteBioSQL(BaseBioSQL):
    def __init__(self, database_name, database_type):
        super().__init__(database_name=database_name, database_type=database_type, driver="SQLite")
        self.schema_cmd = "sqlite3 %s -echo" % database_name
        self.schema_file = "biosqldb-sqlite.sql"

        self.taxon_cmd = "%s --dbname %s --driver %s --download true" % \
                         (self.ncbi_taxon_script, database_name, self.driver)

    def sqlite_schema(self):
        schema_file = pkg_resources.resource_filename(sql.__name__, self.schema_file)
        error, out = self.load_biosql_schema(self.schema_cmd, schema_file)
        # TODO-ROB: Parse output and error

    def sqlite_taxonomy(self):
        error, out = self.load_ncbi_taxonomy(self.taxon_cmd)
        # TODO-ROB: Parse output and error<commit_msg>Added a function for changing the permissions to the perl files.<commit_after>import pkg_resources
from pathlib import Path
import os
import subprocess
from Datasnakes.Tools.logit import LogIt
from Datasnakes.Manager.BioSQL.biosql_repo import sql
from Datasnakes.Manager.BioSQL.biosql_repo import scripts as sql_scripts

class BaseBioSQL(object):
    # TODO-ROB:  Organize the BioSQL files by driver/RDBMS
 # driver "mysql", "Pg", "Oracle", "SQLite"
    def __init__(self, database_name, database_type, driver):
        self.database_name = database_name
        self.database_type = database_type
        self.driver = driver
        self.biosqllog = LogIt().default(logname="BioSQL", logfile=None)

        self.scripts = pkg_resources.resource_filename(sql_scripts.__name__, "")
        self.ncbi_taxon_script = pkg_resources.resource_filename(sql_scripts.__name__, "load_ncbi_taxonomy.pl")
        pass

    def load_biosql_schema(self, cmd, schema_file):
        schema_cmd = "%s < %s" % (cmd, schema_file)
        schema_load = subprocess.Popen([schema_cmd], stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True,
                                       encoding='utf-8')
        error = schema_load.stderr.readlines()
        out = schema_load.stdout.readlines()

        self.biosqllog.info("Schema-Error: " + str(error))
        self.biosqllog.info("Schema-Out: " + str(out))
        return error, out

    def load_ncbi_taxonomy(self, cmd):
        # ./load_ncbi_taxonomy.pl --dbname bioseqdb --driver mysql --dbuser root --download true
        taxon_cmd = cmd
        taxon_load = subprocess.Popen([taxon_cmd], stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True,
                                      encoding='utf-8')

        error = taxon_load.stderr.readlines()
        out = taxon_load.stdout.readlines()

        self.biosqllog.info("Taxon-Error: " + str(error))
        self.biosqllog.info("Taxon-Out: " + str(out))
        return error, out
        pass

    def create_executable_scripts(self):
        # Set up the permissions for the BioSQL Perl scripts
        biosql_scripts = self.scripts
        for file in os.listdir(biosql_scripts):
            if '.pl' in file:
                script_path = os.path.join(biosql_scripts, file)
                os.chmod(script_path, mode=755)

class SQLiteBioSQL(BaseBioSQL):
    def __init__(self, database_name, database_type):
        super().__init__(database_name=database_name, database_type=database_type, driver="SQLite")
        self.schema_cmd = "sqlite3 %s -echo" % database_name
        self.schema_file = "biosqldb-sqlite.sql"

        self.taxon_cmd = "%s --dbname %s --driver %s --download true" % \
                         (self.ncbi_taxon_script, database_name, self.driver)

    def sqlite_schema(self):
        schema_file = pkg_resources.resource_filename(sql.__name__, self.schema_file)
        error, out = self.load_biosql_schema(self.schema_cmd, schema_file)
        # TODO-ROB: Parse output and error

    def sqlite_taxonomy(self):
        error, out = self.load_ncbi_taxonomy(self.taxon_cmd)
        # TODO-ROB: Parse output and error<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import pickle
import time
import urlparse
import re
from urllib2 import quote

from bs4 import BeautifulSoup
from pysimplelog import Logger
from requests import Session

from FoodScan.ShopSync.Shops.shop import Shop
from FoodScan.items import ShopItem

class Kaufland(Shop):

    search_url_prefix = 'https://shop.kaufland.de/search?pageSize=48&sort=relevance&text='

    def __init__(self, email, password, captcha_service, cookie_file="kl_cookies"):
        self.logger = Logger('Kaufland')
        self.captcha_service = captcha_service
        self.base_url = 'https://shop.kaufland.de'
        self.login_url = "https://shop.kaufland.de/login"
        self.account_url = "https://shop.kaufland.de/my-account"
        self.take_url = 'https://shop.kaufland.de/cart/modify'
        self.basket_url = 'https://shop.kaufland.de/cart'

        Shop.__init__(self, email, password, cookie_file)

    @staticmethod
    def search_url(name):
        return Kaufland.search_url_prefix + quote(name.encode('utf-8'))

    def login(self):
        self.logger.info("Logging in...")
        self.session = Session()
        html = self.session.get(self.account_url).text
        blob = BeautifulSoup(html, "html.parser")
        token = blob.find('input', {'name': 'CSRFToken'}).get('value')
        site_key = blob.find('input', {'id': 'siteKey'}).get('value')

        job = self.captcha_service.create(self.login_url, site_key)
        time.sleep(5)

        data = [
            ('j_username', self.email),
            ('j_password', self.password),
            ('g-recaptcha-response', self.captcha_service.result(job)),
            ('siteKeyCaptcha', site_key),
            ('submit-form', ''),
            ('CSRFToken', token),
        ]

        self.session.post(self.base_url + '/j_spring_security_check', data=data)
        self.save_session()

    def is_logged_in(self, html=None):
        if not html:
            html = self.session.get(self.account_url).text
        return html.find('Abmelden') > 0

    def save_session(self):
        with open(self.cookie_file, 'w') as f:
            pickle.dump(self.session.cookies, f)

    def load_session(self):
        try:
            with open(self.cookie_file) as f:
                cookies = pickle.load(f)
                self.session = Session()
                self.session.cookies = cookies
                return self.is_logged_in()
        except IOError:
            return False

    def get(self, url):
        html = self.session.get(url).text
        if self.is_logged_in(html):
            self.save_session()
            return html

        self.login()
        html = self.session.get(url).text
        if self.is_logged_in(html):
            self.save_session()
            return html

        self.logger.error("Can not log in")
        exit(1)

    def cart(self):
        blob = BeautifulSoup(self.get(self.basket_url), "html.parser")
        r = blob.select('section.product-list')
        if len(r) == 0:
            return []

        r = r[0]
        ids = []
        for i in r.findAll('article'):
            a = i.find('a')
            link = urlparse.urljoin(self.base_url, a['href'])
            title = i.find('p', {'class': 'product-list__title'}).text.strip()
            amount = i.find('div', {'class': 'product-list__amount'})
            article_id = amount['data-dynamicblock']
            amount = int(amount.find('input', {'name': 'quantity'}).get('value'))
            price = i.find('div', {'data-dynamiccontent': 'prices'})
            red = price.find('span', {'class': 'product-list__reduced-price'})
            if red:
                price = red
            price = price.text.replace(u'€', u'').strip()
            price = int(float(price) * 100)

            item = ShopItem(article_id, amount, title, price, link)
            ids.append(item)

        return ids

    def search(self, term, sub_term=None):
        html = self.get(Kaufland.search_url(term))
        ids = self.parse_search(html)

        split_terms = [x for x in re.split('-| |\n', term) if len(x) > 1]

        if 0 < len(ids) < 48:
            return self.order_by_matches(split_terms, ids)

        if sub_term and len(ids) == 0:
            return self.search(term + " " + sub_term)

        if len(split_terms) > 1:
            ids = []
            for criteria in split_terms:
                if len(criteria) > 1:
                    ids += self.search(criteria)

        return self.order_by_matches(split_terms, ids, max=20, perfect=0.6, cut_off=0.25)

    def parse_search(self, html):
        blob = BeautifulSoup(html, "html.parser")
        ids = []
        r = blob.select('div.productmatrix')
        if len(r) > 0:
            r = r[0]
            for i in r.findAll('article'):
                a = i.find('a')
                article_id = i['data-dynamicblock'].split('_')[0]
                link = urlparse.urljoin(self.base_url, a['href'])
                title = a.find('p', {'class': 'product-tile__infos--title'}).text.strip()
                price = a.find('div', {'class': 'product-tile__price--regular'})
                if not price:
                    price = a.find('div', {'class': 'product-tile__price--reduced'})
                price = price.text.replace(u'€', u'').strip()
                price = int(float(price) * 100)

                item = ShopItem(article_id, 1, title, price, link)
                ids.append(item)
        return ids

    def order_by_matches(self, terms, ids, max=None, perfect=None, cut_off=None):
        if len(ids) == 0:
            return []
        normal_fit = {}
        perfect_fit = {}
        normal_ids = []
        perfect_ids = []
        for item in ids:
            if item in normal_ids or item in perfect_ids:
                continue
            match = len([x for x in terms if x.lower() in item.name.lower()])
            if not cut_off or match > len(terms) * cut_off:
                normal_ids.append(item)
                normal_fit[item] = match
            if perfect and match > len(terms) * perfect:
                perfect_ids.append(item)
                perfect_fit[item] = match

        if len(perfect_fit) > 0:
            normal_ids = perfect_ids
            normal_fit = perfect_fit

        ordered = sorted(normal_ids, key=normal_fit.__getitem__, reverse=True)
        if max:
            ordered = ordered[:max]
        return ordered

    def take(self, item):
        html = self.get(Kaufland.search_url(item.name))
        blob = BeautifulSoup(html, "html.parser")
        token = blob.find('input', {'name': 'CSRFToken'}).get('value')

        self.session.post(self.take_url, data=[
            ('qty', item.amount),
            ('productCodePost', item.article_id),
            ('pageTemplate', 'producttile'),
            ('CSRFToken', token),
        ])
        self.save_session()

    def shelf_life(self, item_link):
        pass

<commit_msg>better comparability with normalized unicode names<commit_after># -*- coding: utf-8 -*-

import unicodedata
import pickle
import time
import urlparse
import re
from urllib2 import quote

from bs4 import BeautifulSoup
from pysimplelog import Logger
from requests import Session

from FoodScan.ShopSync.Shops.shop import Shop
from FoodScan.items import ShopItem

class Kaufland(Shop):

    search_url_prefix = 'https://shop.kaufland.de/search?pageSize=48&sort=relevance&text='

    def __init__(self, email, password, captcha_service, cookie_file="kl_cookies"):
        self.logger = Logger('Kaufland')
        self.captcha_service = captcha_service
        self.base_url = 'https://shop.kaufland.de'
        self.login_url = "https://shop.kaufland.de/login"
        self.account_url = "https://shop.kaufland.de/my-account"
        self.take_url = 'https://shop.kaufland.de/cart/modify'
        self.basket_url = 'https://shop.kaufland.de/cart'

        Shop.__init__(self, email, password, cookie_file)

    @staticmethod
    def search_url(name):
        return Kaufland.search_url_prefix + quote(name.encode('utf-8'))

    def login(self):
        self.logger.info("Logging in...")
        self.session = Session()
        html = self.session.get(self.account_url).text
        blob = BeautifulSoup(html, "html.parser")
        token = blob.find('input', {'name': 'CSRFToken'}).get('value')
        site_key = blob.find('input', {'id': 'siteKey'}).get('value')

        job = self.captcha_service.create(self.login_url, site_key)
        time.sleep(5)

        data = [
            ('j_username', self.email),
            ('j_password', self.password),
            ('g-recaptcha-response', self.captcha_service.result(job)),
            ('siteKeyCaptcha', site_key),
            ('submit-form', ''),
            ('CSRFToken', token),
        ]

        self.session.post(self.base_url + '/j_spring_security_check', data=data)
        self.save_session()

    def is_logged_in(self, html=None):
        if not html:
            html = self.session.get(self.account_url).text
        return html.find('Abmelden') > 0

    def save_session(self):
        with open(self.cookie_file, 'w') as f:
            pickle.dump(self.session.cookies, f)

    def load_session(self):
        try:
            with open(self.cookie_file) as f:
                cookies = pickle.load(f)
                self.session = Session()
                self.session.cookies = cookies
                return self.is_logged_in()
        except IOError:
            return False

    def get(self, url):
        html = self.session.get(url).text
        if self.is_logged_in(html):
            self.save_session()
            return html

        self.login()
        html = self.session.get(url).text
        if self.is_logged_in(html):
            self.save_session()
            return html

        self.logger.error("Can not log in")
        exit(1)

    def cart(self):
        blob = BeautifulSoup(self.get(self.basket_url), "html.parser")
        r = blob.select('section.product-list')
        if len(r) == 0:
            return []

        r = r[0]
        ids = []
        for i in r.findAll('article'):
            a = i.find('a')
            link = urlparse.urljoin(self.base_url, a['href'])
            title = i.find('p', {'class': 'product-list__title'}).text.strip()
            amount = i.find('div', {'class': 'product-list__amount'})
            article_id = amount['data-dynamicblock']
            amount = int(amount.find('input', {'name': 'quantity'}).get('value'))
            price = i.find('div', {'data-dynamiccontent': 'prices'})
            red = price.find('span', {'class': 'product-list__reduced-price'})
            if red:
                price = red
            price = price.text.replace(u'€', u'').strip()
            price = int(float(price) * 100)

            title = unicodedata.normalize('NFKC', title)

            item = ShopItem(article_id, amount, title, price, link)
            ids.append(item)

        return ids

    def search(self, term, sub_term=None):
        html = self.get(Kaufland.search_url(term))
        ids = self.parse_search(html)

        split_terms = [x for x in re.split('-| |\n', term) if len(x) > 1]

        if 0 < len(ids) < 48:
            return self.order_by_matches(split_terms, ids)

        if sub_term and len(ids) == 0:
            return self.search(term + " " + sub_term)

        if len(split_terms) > 1:
            ids = []
            for criteria in split_terms:
                if len(criteria) > 1:
                    ids += self.search(criteria)

        return self.order_by_matches(split_terms, ids, max=20, perfect=0.6, cut_off=0.25)

    def parse_search(self, html):
        blob = BeautifulSoup(html, "html.parser")
        ids = []
        r = blob.select('div.productmatrix')
        if len(r) > 0:
            r = r[0]
            for i in r.findAll('article'):
                a = i.find('a')
                article_id = i['data-dynamicblock'].split('_')[0]
                link = urlparse.urljoin(self.base_url, a['href'])
                title = a.find('p', {'class': 'product-tile__infos--title'}).text.strip()
                price = a.find('div', {'class': 'product-tile__price--regular'})
                if not price:
                    price = a.find('div', {'class': 'product-tile__price--reduced'})
                price = price.text.replace(u'€', u'').strip()
                price = int(float(price) * 100)

                title = unicodedata.normalize('NFKC', title)

                item = ShopItem(article_id, 1, title, price, link)
                ids.append(item)
        return ids

    def order_by_matches(self, terms, ids, max=None, perfect=None, cut_off=None):
        if len(ids) == 0:
            return []
        normal_fit = {}
        perfect_fit = {}
        normal_ids = []
        perfect_ids = []
        for item in ids:
            if item in normal_ids or item in perfect_ids:
                continue
            match = len([x for x in terms if x.lower() in item.name.lower()])
            if not cut_off or match > len(terms) * cut_off:
                normal_ids.append(item)
                normal_fit[item] = match
            if perfect and match > len(terms) * perfect:
                perfect_ids.append(item)
                perfect_fit[item] = match

        if len(perfect_fit) > 0:
            normal_ids = perfect_ids
            normal_fit = perfect_fit

        ordered = sorted(normal_ids, key=normal_fit.__getitem__, reverse=True)
        if max:
            ordered = ordered[:max]
        return ordered

    def take(self, item):
        html = self.get(Kaufland.search_url(item.name))
        blob = BeautifulSoup(html, "html.parser")
        token = blob.find('input', {'name': 'CSRFToken'}).get('value')

        self.session.post(self.take_url, data=[
            ('qty', item.amount),
            ('productCodePost', item.article_id),
            ('pageTemplate', 'producttile'),
            ('CSRFToken', token),
        ])
        self.save_session()

    def shelf_life(self, item_link):
        pass

<|endoftext|>


<commit_before>#       Licensed to the Apache Software Foundation (ASF) under one
#       or more contributor license agreements.  See the NOTICE file
#       distributed with this work for additional information
#       regarding copyright ownership.  The ASF licenses this file
#       to you under the Apache License, Version 2.0 (the
#       "License"); you may not use this file except in compliance
#       with the License.  You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#       Unless required by applicable law or agreed to in writing,
#       software distributed under the License is distributed on an
#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#       KIND, either express or implied.  See the License for the
#       specific language governing permissions and limitations
#       under the License.

import os
import sys
import shutil
import string
import logging
import random
import itertools
from collections import namedtuple
from datetime import datetime
from glob import glob
import gzip

import tg
import git
import gitdb
from pylons import app_globals as g
from pylons import tmpl_context as c
from pymongo.errors import DuplicateKeyError
from paste.deploy.converters import asbool

from ming.base import Object
from ming.orm import Mapper, session, mapper
from ming.utils import LazyProperty

from allura.lib import helpers as h
from allura.lib import utils
from allura.model.repository import topological_sort
from allura import model as M

log = logging.getLogger(__name__)

gitdb.util.mman = gitdb.util.mman.__class__(
    max_open_handles=128)

class Repository(M.Repository):
    tool_name='Git'
    repo_id='git'
    type_s='Git Repository'
    class __mongometa__:
        name='git-repository'

    @LazyProperty
    def _impl(self):
        return GitImplementation(self)

    def suggested_clone_dest_path(self):
        return super(Repository, self).suggested_clone_dest_path()[:-4]

    def clone_url(self, category, username=''):
        return super(Repository, self).clone_url(category, username)[:-4]

    def merge_command(self, merge_request):
        '''Return the command to merge a given commit to a given target branch'''
        return 'git checkout %s\ngit fetch %s\ngit merge %s' % (
            merge_request.target_branch,
            merge_request.downstream_repo_url,
            merge_request.downstream.commit_id,
        )

class GitImplementation(M.RepositoryImplementation):
    post_receive_template = string.Template(
        '#!/bin/bash\n'
        '# The following is required for site integration, do not remove/modify.\n'
        '# Place user hook code in post-receive-user and it will be called from here.\n'
        'curl -s $url\n'
        '\n'
        'DIR="$$(dirname "$${BASH_SOURCE[0]}")"\n'
        'if [ -x $$DIR/post-receive-user ]; then\n'
        '  exec $$DIR/post-receive-user\n'
        'fi')

    def __init__(self, repo):
        self._repo = repo

    @LazyProperty
    def _git(self):
        try:
            return git.Repo(self._repo.full_fs_path, odbt=git.GitCmdObjectDB)
        except (git.exc.NoSuchPathError, git.exc.InvalidGitRepositoryError), err:
            log.error('Problem looking up repo: %r', err)
            return None

    def init(self):
        fullname = self._setup_paths()
        log.info('git init %s', fullname)
        if os.path.exists(fullname):
            shutil.rmtree(fullname)
        repo = git.Repo.init(
            path=fullname,
            mkdir=True,
            quiet=True,
            bare=True,
            shared='all')
        self.__dict__['_git'] = repo
        self._setup_special_files()
        self._repo.status = 'ready'

    def can_hotcopy(self, source_url):
        enabled = asbool(tg.config.get('scm.git.hotcopy', True))
        is_local = os.path.exists(source_url)
        requested = self._repo.app.config.options.get('hotcopy', False)
        return enabled and is_local and requested

    def clone_from(self, source_url):
        '''Initialize a repo as a clone of another'''
        self._repo.status = 'cloning'
        session(self._repo).flush(self._repo)
        log.info('Initialize %r as a clone of %s',
                 self._repo, source_url)
        try:
            fullname = self._setup_paths(create_repo_dir=False)
            if os.path.exists(fullname):
                shutil.rmtree(fullname)
            if self.can_hotcopy(source_url):
                shutil.copytree(source_url, fullname)
                post_receive = os.path.join(self._repo.full_fs_path, 'hooks', 'post-receive')
                if os.path.exists(post_receive):
                    os.rename(post_receive, post_receive + '-user')
                repo = git.Repo(fullname)
            else:
                repo = git.Repo.clone_from(
                    source_url,
                    to_path=fullname,
                    bare=True)
            self.__dict__['_git'] = repo
            self._setup_special_files(source_url)
        except:
            self._repo.status = 'ready'
            session(self._repo).flush(self._repo)
            raise

    def commit(self, rev):
        '''Return a Commit object.  rev can be _id or a branch/tag name'''
        # See if the rev is a named ref that we have cached, and use the sha1
        # from the cache. This ensures that we don't return a sha1 that we
        # don't have indexed into mongo yet.
        for ref in self._repo.heads + self._repo.branches + self._repo.repo_tags:
            if ref.name == rev:
                rev = ref.object_id
                break
        cache = getattr(c, 'model_cache', '') or M.repo.ModelCache()
        result = cache.get(M.repo.Commit, dict(_id=rev))
        if result is None:
            # find the id by branch/tag name
            try:
                impl = self._git.rev_parse(str(rev) + '^0')
                result = cache.get(M.repo.Commit, dict(_id=impl.hexsha))
            except Exception:
                url = ''
                try:
                    from tg import request
                    url = ' at ' + request.url
                except:
                    pass
                log.exception('Error with rev_parse(%s)%s' % (str(rev) + '^0', url))
        if result is None: return None
        result.set_context(self._repo)
        return result

    def all_commit_ids(self):
        """Yield commit ids, starting with the head(s) of the commit tree and
        ending with the root (first commit).
        """
        if not self._git.head.is_valid():
            return  # empty repo
        seen = set()
        for ci in self._git.iter_commits(all=True, topo_order=True):
            if ci.binsha in seen: continue
            seen.add(ci.binsha)
            yield ci.hexsha

    def new_commits(self, all_commits=False):
        graph = {}

        to_visit = [ self._git.commit(rev=hd.object_id) for hd in self._repo.heads ]
        while to_visit:
            obj = to_visit.pop()
            if obj.hexsha in graph: continue
            if not all_commits:
                # Look up the object
                if M.repo.Commit.query.find(dict(_id=obj.hexsha)).count():
                    graph[obj.hexsha] = set() # mark as parentless
                    continue
            graph[obj.hexsha] = set(p.hexsha for p in obj.parents)
            to_visit += obj.parents
        return list(topological_sort(graph))

    def refresh_heads(self):
        self._repo.heads = [
            Object(name=head.name, object_id=head.commit.hexsha)
            for head in self._git.heads
            if head.is_valid() ]
        self._repo.branches = [
            Object(name=head.name, object_id=head.commit.hexsha)
            for head in self._git.branches
            if head.is_valid() ]
        self._repo.repo_tags = [
            Object(name=tag.name, object_id=tag.commit.hexsha)
            for tag in self._git.tags
            if tag.is_valid() ]
        session(self._repo).flush()

    def refresh_commit_info(self, oid, seen, lazy=True):
        from allura.model.repo import CommitDoc
        ci_doc = CommitDoc.m.get(_id=oid)
        if ci_doc and lazy: return False
        ci = self._git.rev_parse(oid)
        args = dict(
            tree_id=ci.tree.hexsha,
            committed = Object(
                name=h.really_unicode(ci.committer.name),
                email=h.really_unicode(ci.committer.email),
                date=datetime.utcfromtimestamp(ci.committed_date)),
            authored = Object(
                name=h.really_unicode(ci.author.name),
                email=h.really_unicode(ci.author.email),
                date=datetime.utcfromtimestamp(ci.authored_date)),
            message=h.really_unicode(ci.message or ''),
            child_ids=[],
            parent_ids = [ p.hexsha for p in ci.parents ])
        if ci_doc:
            ci_doc.update(**args)
            ci_doc.m.save()
        else:
            ci_doc = CommitDoc(dict(args, _id=ci.hexsha))
            try:
                ci_doc.m.insert(safe=True)
            except DuplicateKeyError:
                if lazy: return False
        self.refresh_tree_info(ci.tree, seen, lazy)
        return True

    def refresh_tree_info(self, tree, seen, lazy=True):
        from allura.model.repo import TreeDoc
        if lazy and tree.binsha in seen: return
        seen.add(tree.binsha)
        doc = TreeDoc(dict(
                _id=tree.hexsha,
                tree_ids=[],
                blob_ids=[],
                other_ids=[]))
        for o in tree:
            if o.type == 'submodule':
                continue
            obj = Object(
                name=h.really_unicode(o.name),
                id=o.hexsha)
            if o.type == 'tree':
                self.refresh_tree_info(o, seen, lazy)
                doc.tree_ids.append(obj)
            elif o.type == 'blob':
                doc.blob_ids.append(obj)
            else:
                obj.type = o.type
                doc.other_ids.append(obj)
        doc.m.save(safe=False)
        return doc

    def commits(self, path=None, rev=None, skip=None, limit=None):
        params = dict(paths=path)
        if rev is not None:
            params['rev'] = rev
        if skip is not None:
            params['skip'] = skip
        if limit is not None:
            params['max_count'] = limit
        return (c.hexsha for c in self._git.iter_commits(**params))

    def commits_count(self, path=None, rev=None):
        commit = self._git.commit(rev)
        return commit.count(path)

    def log(self, object_id, skip, count):
        obj = self._git.commit(object_id)
        candidates = [ obj ]
        result = []
        seen = set()
        while count and candidates:
            candidates.sort(key=lambda c:c.committed_date)
            obj = candidates.pop(-1)
            if obj.hexsha in seen: continue
            seen.add(obj.hexsha)
            if skip == 0:
                result.append(obj.hexsha)
                count -= 1
            else:
                skip -= 1
            candidates += obj.parents
        return result, [ p.hexsha for p in candidates ]

    def open_blob(self, blob):
        return _OpenedGitBlob(
            self._object(blob._id).data_stream)

    def blob_size(self, blob):
        return self._object(blob._id).data_stream.size

    def _setup_hooks(self, source_path=None):
        'Set up the git post-commit hook'
        text = self.post_receive_template.substitute(
            url=tg.config.get('base_url', 'http://localhost:8080')
            + '/auth/refresh_repo' + self._repo.url())
        fn = os.path.join(self._repo.fs_path, self._repo.name, 'hooks', 'post-receive')
        with open(fn, 'w') as fp:
            fp.write(text)
        os.chmod(fn, 0755)

    def _object(self, oid):
        evens = oid[::2]
        odds = oid[1::2]
        binsha = ''
        for e,o in zip(evens, odds):
            binsha += chr(int(e+o, 16))
        return git.Object.new_from_sha(self._git, binsha)

    def symbolics_for_commit(self, commit):
        branch_heads, tags = super(self.__class__, self).symbolics_for_commit(commit)
        containing_branches = self._git.git.branch(contains=commit._id)
        containing_branches = [br.strip(' *') for br in containing_branches.split('\n')]
        return containing_branches, tags

    def compute_tree_new(self, commit, tree_path='/'):
        ci = self._git.rev_parse(commit._id)
        tree = self.refresh_tree_info(ci.tree, set())
        return tree._id

    def tarball(self, commit):
        if not os.path.exists(self._repo.tarball_path):
            os.makedirs(self._repo.tarball_path)
        archive_name = self._repo.tarball_filename(commit)
        filename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.zip'))
        tmpfilename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.tmp'))
        try:
            with open(tmpfilename, 'wb') as archive_file:
                self._git.archive(archive_file, format='zip', treeish=commit, prefix=archive_name + '/')
            os.rename(tmpfilename, filename)
        finally:
            if os.path.exists(tmpfilename):
                os.remove(tmpfilename)

    def is_empty(self):
        return not self._git or len(self._git.heads) == 0

class _OpenedGitBlob(object):
    CHUNK_SIZE=4096

    def __init__(self, stream):
        self._stream = stream

    def read(self):
        return self._stream.read()

    def __iter__(self):
        '''
        Yields one line at a time, reading from the stream
        '''
        buffer = ''
        while True:
            # Replenish buffer until we have a line break
            while '\n' not in buffer:
                chars = self._stream.read(self.CHUNK_SIZE)
                if not chars: break
                buffer += chars
            if not buffer: break
            eol = buffer.find('\n')
            if eol == -1:
                # end without \n
                yield buffer
                break
            yield buffer[:eol+1]
            buffer = buffer[eol+1:]

    def close(self):
        pass

Mapper.compile_all()
<commit_msg>[#6222] ticket:349 Handle git errors gracefully in symbolics_for_commit<commit_after>#       Licensed to the Apache Software Foundation (ASF) under one
#       or more contributor license agreements.  See the NOTICE file
#       distributed with this work for additional information
#       regarding copyright ownership.  The ASF licenses this file
#       to you under the Apache License, Version 2.0 (the
#       "License"); you may not use this file except in compliance
#       with the License.  You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#       Unless required by applicable law or agreed to in writing,
#       software distributed under the License is distributed on an
#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#       KIND, either express or implied.  See the License for the
#       specific language governing permissions and limitations
#       under the License.

import os
import sys
import shutil
import string
import logging
import random
import itertools
from collections import namedtuple
from datetime import datetime
from glob import glob
import gzip

import tg
import git
import gitdb
from pylons import app_globals as g
from pylons import tmpl_context as c
from pymongo.errors import DuplicateKeyError
from paste.deploy.converters import asbool

from ming.base import Object
from ming.orm import Mapper, session, mapper
from ming.utils import LazyProperty

from allura.lib import helpers as h
from allura.lib import utils
from allura.model.repository import topological_sort
from allura import model as M

log = logging.getLogger(__name__)

gitdb.util.mman = gitdb.util.mman.__class__(
    max_open_handles=128)

class Repository(M.Repository):
    tool_name='Git'
    repo_id='git'
    type_s='Git Repository'
    class __mongometa__:
        name='git-repository'

    @LazyProperty
    def _impl(self):
        return GitImplementation(self)

    def suggested_clone_dest_path(self):
        return super(Repository, self).suggested_clone_dest_path()[:-4]

    def clone_url(self, category, username=''):
        return super(Repository, self).clone_url(category, username)[:-4]

    def merge_command(self, merge_request):
        '''Return the command to merge a given commit to a given target branch'''
        return 'git checkout %s\ngit fetch %s\ngit merge %s' % (
            merge_request.target_branch,
            merge_request.downstream_repo_url,
            merge_request.downstream.commit_id,
        )

class GitImplementation(M.RepositoryImplementation):
    post_receive_template = string.Template(
        '#!/bin/bash\n'
        '# The following is required for site integration, do not remove/modify.\n'
        '# Place user hook code in post-receive-user and it will be called from here.\n'
        'curl -s $url\n'
        '\n'
        'DIR="$$(dirname "$${BASH_SOURCE[0]}")"\n'
        'if [ -x $$DIR/post-receive-user ]; then\n'
        '  exec $$DIR/post-receive-user\n'
        'fi')

    def __init__(self, repo):
        self._repo = repo

    @LazyProperty
    def _git(self):
        try:
            return git.Repo(self._repo.full_fs_path, odbt=git.GitCmdObjectDB)
        except (git.exc.NoSuchPathError, git.exc.InvalidGitRepositoryError), err:
            log.error('Problem looking up repo: %r', err)
            return None

    def init(self):
        fullname = self._setup_paths()
        log.info('git init %s', fullname)
        if os.path.exists(fullname):
            shutil.rmtree(fullname)
        repo = git.Repo.init(
            path=fullname,
            mkdir=True,
            quiet=True,
            bare=True,
            shared='all')
        self.__dict__['_git'] = repo
        self._setup_special_files()
        self._repo.status = 'ready'

    def can_hotcopy(self, source_url):
        enabled = asbool(tg.config.get('scm.git.hotcopy', True))
        is_local = os.path.exists(source_url)
        requested = self._repo.app.config.options.get('hotcopy', False)
        return enabled and is_local and requested

    def clone_from(self, source_url):
        '''Initialize a repo as a clone of another'''
        self._repo.status = 'cloning'
        session(self._repo).flush(self._repo)
        log.info('Initialize %r as a clone of %s',
                 self._repo, source_url)
        try:
            fullname = self._setup_paths(create_repo_dir=False)
            if os.path.exists(fullname):
                shutil.rmtree(fullname)
            if self.can_hotcopy(source_url):
                shutil.copytree(source_url, fullname)
                post_receive = os.path.join(self._repo.full_fs_path, 'hooks', 'post-receive')
                if os.path.exists(post_receive):
                    os.rename(post_receive, post_receive + '-user')
                repo = git.Repo(fullname)
            else:
                repo = git.Repo.clone_from(
                    source_url,
                    to_path=fullname,
                    bare=True)
            self.__dict__['_git'] = repo
            self._setup_special_files(source_url)
        except:
            self._repo.status = 'ready'
            session(self._repo).flush(self._repo)
            raise

    def commit(self, rev):
        '''Return a Commit object.  rev can be _id or a branch/tag name'''
        # See if the rev is a named ref that we have cached, and use the sha1
        # from the cache. This ensures that we don't return a sha1 that we
        # don't have indexed into mongo yet.
        for ref in self._repo.heads + self._repo.branches + self._repo.repo_tags:
            if ref.name == rev:
                rev = ref.object_id
                break
        cache = getattr(c, 'model_cache', '') or M.repo.ModelCache()
        result = cache.get(M.repo.Commit, dict(_id=rev))
        if result is None:
            # find the id by branch/tag name
            try:
                impl = self._git.rev_parse(str(rev) + '^0')
                result = cache.get(M.repo.Commit, dict(_id=impl.hexsha))
            except Exception:
                url = ''
                try:
                    from tg import request
                    url = ' at ' + request.url
                except:
                    pass
                log.exception('Error with rev_parse(%s)%s' % (str(rev) + '^0', url))
        if result is None: return None
        result.set_context(self._repo)
        return result

    def all_commit_ids(self):
        """Yield commit ids, starting with the head(s) of the commit tree and
        ending with the root (first commit).
        """
        if not self._git.head.is_valid():
            return  # empty repo
        seen = set()
        for ci in self._git.iter_commits(all=True, topo_order=True):
            if ci.binsha in seen: continue
            seen.add(ci.binsha)
            yield ci.hexsha

    def new_commits(self, all_commits=False):
        graph = {}

        to_visit = [ self._git.commit(rev=hd.object_id) for hd in self._repo.heads ]
        while to_visit:
            obj = to_visit.pop()
            if obj.hexsha in graph: continue
            if not all_commits:
                # Look up the object
                if M.repo.Commit.query.find(dict(_id=obj.hexsha)).count():
                    graph[obj.hexsha] = set() # mark as parentless
                    continue
            graph[obj.hexsha] = set(p.hexsha for p in obj.parents)
            to_visit += obj.parents
        return list(topological_sort(graph))

    def refresh_heads(self):
        self._repo.heads = [
            Object(name=head.name, object_id=head.commit.hexsha)
            for head in self._git.heads
            if head.is_valid() ]
        self._repo.branches = [
            Object(name=head.name, object_id=head.commit.hexsha)
            for head in self._git.branches
            if head.is_valid() ]
        self._repo.repo_tags = [
            Object(name=tag.name, object_id=tag.commit.hexsha)
            for tag in self._git.tags
            if tag.is_valid() ]
        session(self._repo).flush()

    def refresh_commit_info(self, oid, seen, lazy=True):
        from allura.model.repo import CommitDoc
        ci_doc = CommitDoc.m.get(_id=oid)
        if ci_doc and lazy: return False
        ci = self._git.rev_parse(oid)
        args = dict(
            tree_id=ci.tree.hexsha,
            committed = Object(
                name=h.really_unicode(ci.committer.name),
                email=h.really_unicode(ci.committer.email),
                date=datetime.utcfromtimestamp(ci.committed_date)),
            authored = Object(
                name=h.really_unicode(ci.author.name),
                email=h.really_unicode(ci.author.email),
                date=datetime.utcfromtimestamp(ci.authored_date)),
            message=h.really_unicode(ci.message or ''),
            child_ids=[],
            parent_ids = [ p.hexsha for p in ci.parents ])
        if ci_doc:
            ci_doc.update(**args)
            ci_doc.m.save()
        else:
            ci_doc = CommitDoc(dict(args, _id=ci.hexsha))
            try:
                ci_doc.m.insert(safe=True)
            except DuplicateKeyError:
                if lazy: return False
        self.refresh_tree_info(ci.tree, seen, lazy)
        return True

    def refresh_tree_info(self, tree, seen, lazy=True):
        from allura.model.repo import TreeDoc
        if lazy and tree.binsha in seen: return
        seen.add(tree.binsha)
        doc = TreeDoc(dict(
                _id=tree.hexsha,
                tree_ids=[],
                blob_ids=[],
                other_ids=[]))
        for o in tree:
            if o.type == 'submodule':
                continue
            obj = Object(
                name=h.really_unicode(o.name),
                id=o.hexsha)
            if o.type == 'tree':
                self.refresh_tree_info(o, seen, lazy)
                doc.tree_ids.append(obj)
            elif o.type == 'blob':
                doc.blob_ids.append(obj)
            else:
                obj.type = o.type
                doc.other_ids.append(obj)
        doc.m.save(safe=False)
        return doc

    def commits(self, path=None, rev=None, skip=None, limit=None):
        params = dict(paths=path)
        if rev is not None:
            params['rev'] = rev
        if skip is not None:
            params['skip'] = skip
        if limit is not None:
            params['max_count'] = limit
        return (c.hexsha for c in self._git.iter_commits(**params))

    def commits_count(self, path=None, rev=None):
        commit = self._git.commit(rev)
        return commit.count(path)

    def log(self, object_id, skip, count):
        obj = self._git.commit(object_id)
        candidates = [ obj ]
        result = []
        seen = set()
        while count and candidates:
            candidates.sort(key=lambda c:c.committed_date)
            obj = candidates.pop(-1)
            if obj.hexsha in seen: continue
            seen.add(obj.hexsha)
            if skip == 0:
                result.append(obj.hexsha)
                count -= 1
            else:
                skip -= 1
            candidates += obj.parents
        return result, [ p.hexsha for p in candidates ]

    def open_blob(self, blob):
        return _OpenedGitBlob(
            self._object(blob._id).data_stream)

    def blob_size(self, blob):
        return self._object(blob._id).data_stream.size

    def _setup_hooks(self, source_path=None):
        'Set up the git post-commit hook'
        text = self.post_receive_template.substitute(
            url=tg.config.get('base_url', 'http://localhost:8080')
            + '/auth/refresh_repo' + self._repo.url())
        fn = os.path.join(self._repo.fs_path, self._repo.name, 'hooks', 'post-receive')
        with open(fn, 'w') as fp:
            fp.write(text)
        os.chmod(fn, 0755)

    def _object(self, oid):
        evens = oid[::2]
        odds = oid[1::2]
        binsha = ''
        for e,o in zip(evens, odds):
            binsha += chr(int(e+o, 16))
        return git.Object.new_from_sha(self._git, binsha)

    def symbolics_for_commit(self, commit):
        branch_heads, tags = super(self.__class__, self).symbolics_for_commit(commit)
        try:
            containing_branches = self._git.git.branch(contains=commit._id)
        except git.GitCommandError:
            return []
        containing_branches = [br.strip(' *') for br in containing_branches.split('\n')]
        return containing_branches, tags

    def compute_tree_new(self, commit, tree_path='/'):
        ci = self._git.rev_parse(commit._id)
        tree = self.refresh_tree_info(ci.tree, set())
        return tree._id

    def tarball(self, commit):
        if not os.path.exists(self._repo.tarball_path):
            os.makedirs(self._repo.tarball_path)
        archive_name = self._repo.tarball_filename(commit)
        filename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.zip'))
        tmpfilename = os.path.join(self._repo.tarball_path, '%s%s' % (archive_name, '.tmp'))
        try:
            with open(tmpfilename, 'wb') as archive_file:
                self._git.archive(archive_file, format='zip', treeish=commit, prefix=archive_name + '/')
            os.rename(tmpfilename, filename)
        finally:
            if os.path.exists(tmpfilename):
                os.remove(tmpfilename)

    def is_empty(self):
        return not self._git or len(self._git.heads) == 0

class _OpenedGitBlob(object):
    CHUNK_SIZE=4096

    def __init__(self, stream):
        self._stream = stream

    def read(self):
        return self._stream.read()

    def __iter__(self):
        '''
        Yields one line at a time, reading from the stream
        '''
        buffer = ''
        while True:
            # Replenish buffer until we have a line break
            while '\n' not in buffer:
                chars = self._stream.read(self.CHUNK_SIZE)
                if not chars: break
                buffer += chars
            if not buffer: break
            eol = buffer.find('\n')
            if eol == -1:
                # end without \n
                yield buffer
                break
            yield buffer[:eol+1]
            buffer = buffer[eol+1:]

    def close(self):
        pass

Mapper.compile_all()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright (c) 2015 Orange
# guyrodrigue.koffi@orange.com
# morgan.richomme@orange.com
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
#
import re, json, os, urllib2, argparse, logging, yaml

with open('../functest.yaml') as f:
    functest_yaml = yaml.safe_load(f)
f.close()

""" get the date """
cmd = os.popen("date '+%d%m%Y_%H%M'")
test_date = cmd.read().rstrip()

HOME = os.environ['HOME']+"/"
SCENARIOS_DIR = HOME + functest_yaml.get("general").get("directories").get("dir_rally_scn")
RESULTS_DIR = HOME + functest_yaml.get("general").get("directories").get("dir_rally_res") + test_date + "/"

""" tests configuration """
tests = ['authenticate', 'glance', 'cinder', 'heat', 'keystone', 'neutron', 'nova', 'quotas', 'requests', 'tempest', 'vm', 'all', 'smoke']
parser = argparse.ArgumentParser()
parser.add_argument("test_name", help="The name of the test you want to perform with rally. "
                                      "Possible values are : "
                                      "[ {d[0]} | {d[1]} | {d[2]} | {d[3]} | {d[4]} | {d[5]} | {d[6]} "
                                      "| {d[7]} | {d[8]} | {d[9]} | {d[10]} | {d[11]} | {d[12]}]. The 'all' value performs all the  possible tests scenarios"
                                      "except 'tempest'".format(d=tests))

parser.add_argument("-d", "--debug", help="Debug mode",  action="store_true")
parser.add_argument("test_mode", help="Tempest test mode", nargs='?', default="smoke")
args = parser.parse_args()
test_mode=args.test_mode

if not args.test_name == "tempest":
    if not args.test_mode == "smoke":
        parser.error("test_mode is only used with tempest")

""" logging configuration """
logger = logging.getLogger('run_rally')
logger.setLevel(logging.DEBUG)

ch = logging.StreamHandler()
if args.debug:
    ch.setLevel(logging.DEBUG)
else:
    ch.setLevel(logging.INFO)

formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

def get_tempest_id(cmd_raw):
    """
    get task id from command rally result
    :param cmd_raw:
    :return: task_id as string
    """
    taskid_re = re.compile('^Verification UUID: (.*)$')
    for line in cmd_raw.splitlines(True):
        line = line.strip()
    match = taskid_re.match(line)

    if match:
        return match.group(1)
    return None

def get_task_id(cmd_raw):
    """
    get task id from command rally result
    :param cmd_raw:
    :return: task_id as string
    """
    taskid_re = re.compile('^Task +(.*): started$')
    for line in cmd_raw.splitlines(True):
        line = line.strip()
        match = taskid_re.match(line)
        if match:
            return match.group(1)
    return None

def task_succeed(json_raw):
    """
    Parse JSON from rally JSON results
    :param json_raw:
    :return: Bool
    """
    rally_report = json.loads(json_raw)
    rally_report = rally_report[0]
    if rally_report is None:
        return False
    if rally_report.get('result') is None:
        return False

    for result in rally_report.get('result'):
        if len(result.get('error')) > 0:
            return False

    return True

def run_tempest():
    """
    the function dedicated to Tempest (functional tests for OpenStack)
    :param test_mode: Tempest mode smoke (default), full, ..
    :return: void
    """
    logger.info('starting {} Tempest ...'.format(test_mode))

    cmd_line = "rally verify start {}".format(test_mode)
    logger.debug('running command line : {}'.format(cmd_line))
    cmd = os.popen(cmd_line)
    task_id = get_tempest_id(cmd.read())
    logger.debug('task_id : {}'.format(task_id))

    if task_id is None:
        logger.error("failed to retrieve task_id")
    exit(-1)

    """ check for result directory and create it otherwise """
    if not os.path.exists(RESULTS_DIR):
        logger.debug('does not exists, we create it'.format(RESULTS_DIR))
        os.makedirs(RESULTS_DIR)

    """ write log report file """
    report_file_name = '{}opnfv-tempest.log'.format(RESULTS_DIR)
    cmd_line = "rally verify detailed {} > {} ".format(task_id, report_file_name)
    logger.debug('running command line : {}'.format(cmd_line))
    os.popen(cmd_line)

def run_task(test_name):
    """
    the "main" function of the script who lunch rally for a task
    :param test_name: name for the rally test
    :return: void
    """
    logger.info('starting {} test ...'.format(test_name))

    """ check directory for scenarios test files or retrieve from git otherwise"""
    proceed_test = True
    test_file_name = '{}opnfv-{}.json'.format(SCENARIOS_DIR, test_name)
    if not os.path.exists(test_file_name):
        logger.debug('{} does not exists'.format(test_file_name))
        proceed_test = retrieve_test_cases_file(test_name, SCENARIOS_DIR)

    """ we do the test only if we have a scenario test file """
    if proceed_test:
        logger.debug('Scenario fetched from : {}'.format(test_file_name))
        cmd_line = "rally task start --abort-on-sla-failure %s" % test_file_name
        logger.debug('running command line : {}'.format(cmd_line))
        cmd = os.popen(cmd_line)
        task_id = get_task_id(cmd.read())
        logger.debug('task_id : {}'.format(task_id))

        if task_id is None:
            logger.error("failed to retrieve task_id")
            exit(-1)

        """ check for result directory and create it otherwise """
        if not os.path.exists(RESULTS_DIR):
            logger.debug('does not exists, we create it'.format(RESULTS_DIR))
            os.makedirs(RESULTS_DIR)

        """ write html report file """
        report_file_name = '{}opnfv-{}.html'.format(RESULTS_DIR, test_name)
        cmd_line = "rally task report %s --out %s" % (task_id, report_file_name)
        logger.debug('running command line : {}'.format(cmd_line))
        os.popen(cmd_line)

        """ get and save rally operation JSON result """
        cmd_line = "rally task results %s" % task_id
        logger.debug('running command line : {}'.format(cmd_line))
        cmd = os.popen(cmd_line)
        json_results = cmd.read()
        with open('{}opnfv-{}.json'.format(RESULTS_DIR, test_name), 'w') as f:
            logger.debug('saving json file')
            f.write(json_results)
            logger.debug('saving json file2')

        """ parse JSON operation result """
        if task_succeed(json_results):
            print 'Test OK'
        else:
            print 'Test KO'
    else:
        logger.error('{} test failed, unable to fetch a scenario test file'.format(test_name))

def retrieve_test_cases_file(test_name, tests_path):
    """
    Retrieve from github the sample test files
    :return: Boolean that indicates the retrieval status
    """

    """ do not add the "/" at the end """
    url_base = "https://git.opnfv.org/cgit/functest/plain/testcases/VIM/OpenStack/CI/suites"

    test_file_name = 'opnfv-{}.json'.format(test_name)
    logger.info('fetching {}/{} ...'.format(url_base, test_file_name))

    try:
        response = urllib2.urlopen('{}/{}'.format(url_base, test_file_name))
    except (urllib2.HTTPError, urllib2.URLError):
        return False
    file_raw = response.read()

    """ check if the test path exist otherwise we create it """
    if not os.path.exists(tests_path):
        os.makedirs(tests_path)

    with open('{}/{}'.format(tests_path, test_file_name), 'w') as f:
        f.write(file_raw)
    return True

def main():
    """ configure script """
    if not (args.test_name in tests):
        logger.error('argument not valid')
        exit(-1)

    if args.test_name == "all":
        for test_name in tests:
            if not (test_name == 'all' or test_name == 'tempest' or test_name == 'heat' or test_name == 'smoke' ):
                print(test_name)
                run_task(test_name)
    else:
        print(args.test_name)
        if args.test_name == 'tempest':
            run_tempest()
        else:
            run_task(args.test_name)

if __name__ == '__main__':
    main()
<commit_msg>run_rally.py: added consistent path for functest.yaml JIRA:FUNCTEST-1<commit_after>#!/usr/bin/env python
#
# Copyright (c) 2015 Orange
# guyrodrigue.koffi@orange.com
# morgan.richomme@orange.com
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
#
import re, json, os, urllib2, argparse, logging, yaml

HOME = os.environ['HOME']+"/"
with open(HOME+'.functest/functest.yaml') as f:
    functest_yaml = yaml.safe_load(f)
f.close()

""" get the date """
cmd = os.popen("date '+%d%m%Y_%H%M'")
test_date = cmd.read().rstrip()

HOME = os.environ['HOME']+"/"
SCENARIOS_DIR = HOME + functest_yaml.get("general").get("directories").get("dir_rally_scn")
RESULTS_DIR = HOME + functest_yaml.get("general").get("directories").get("dir_rally_res") + test_date + "/"

""" tests configuration """
tests = ['authenticate', 'glance', 'cinder', 'heat', 'keystone', 'neutron', 'nova', 'quotas', 'requests', 'tempest', 'vm', 'all', 'smoke']
parser = argparse.ArgumentParser()
parser.add_argument("test_name", help="The name of the test you want to perform with rally. "
                                      "Possible values are : "
                                      "[ {d[0]} | {d[1]} | {d[2]} | {d[3]} | {d[4]} | {d[5]} | {d[6]} "
                                      "| {d[7]} | {d[8]} | {d[9]} | {d[10]} | {d[11]} | {d[12]}]. The 'all' value performs all the  possible tests scenarios"
                                      "except 'tempest'".format(d=tests))

parser.add_argument("-d", "--debug", help="Debug mode",  action="store_true")
parser.add_argument("test_mode", help="Tempest test mode", nargs='?', default="smoke")
args = parser.parse_args()
test_mode=args.test_mode

if not args.test_name == "tempest":
    if not args.test_mode == "smoke":
        parser.error("test_mode is only used with tempest")

""" logging configuration """
logger = logging.getLogger('run_rally')
logger.setLevel(logging.DEBUG)

ch = logging.StreamHandler()
if args.debug:
    ch.setLevel(logging.DEBUG)
else:
    ch.setLevel(logging.INFO)

formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

def get_tempest_id(cmd_raw):
    """
    get task id from command rally result
    :param cmd_raw:
    :return: task_id as string
    """
    taskid_re = re.compile('^Verification UUID: (.*)$')
    for line in cmd_raw.splitlines(True):
        line = line.strip()
    match = taskid_re.match(line)

    if match:
        return match.group(1)
    return None

def get_task_id(cmd_raw):
    """
    get task id from command rally result
    :param cmd_raw:
    :return: task_id as string
    """
    taskid_re = re.compile('^Task +(.*): started$')
    for line in cmd_raw.splitlines(True):
        line = line.strip()
        match = taskid_re.match(line)
        if match:
            return match.group(1)
    return None

def task_succeed(json_raw):
    """
    Parse JSON from rally JSON results
    :param json_raw:
    :return: Bool
    """
    rally_report = json.loads(json_raw)
    rally_report = rally_report[0]
    if rally_report is None:
        return False
    if rally_report.get('result') is None:
        return False

    for result in rally_report.get('result'):
        if len(result.get('error')) > 0:
            return False

    return True

def run_tempest():
    """
    the function dedicated to Tempest (functional tests for OpenStack)
    :param test_mode: Tempest mode smoke (default), full, ..
    :return: void
    """
    logger.info('starting {} Tempest ...'.format(test_mode))

    cmd_line = "rally verify start {}".format(test_mode)
    logger.debug('running command line : {}'.format(cmd_line))
    cmd = os.popen(cmd_line)
    task_id = get_tempest_id(cmd.read())
    logger.debug('task_id : {}'.format(task_id))

    if task_id is None:
        logger.error("failed to retrieve task_id")
    exit(-1)

    """ check for result directory and create it otherwise """
    if not os.path.exists(RESULTS_DIR):
        logger.debug('does not exists, we create it'.format(RESULTS_DIR))
        os.makedirs(RESULTS_DIR)

    """ write log report file """
    report_file_name = '{}opnfv-tempest.log'.format(RESULTS_DIR)
    cmd_line = "rally verify detailed {} > {} ".format(task_id, report_file_name)
    logger.debug('running command line : {}'.format(cmd_line))
    os.popen(cmd_line)

def run_task(test_name):
    """
    the "main" function of the script who lunch rally for a task
    :param test_name: name for the rally test
    :return: void
    """
    logger.info('starting {} test ...'.format(test_name))

    """ check directory for scenarios test files or retrieve from git otherwise"""
    proceed_test = True
    test_file_name = '{}opnfv-{}.json'.format(SCENARIOS_DIR, test_name)
    if not os.path.exists(test_file_name):
        logger.debug('{} does not exists'.format(test_file_name))
        proceed_test = retrieve_test_cases_file(test_name, SCENARIOS_DIR)

    """ we do the test only if we have a scenario test file """
    if proceed_test:
        logger.debug('Scenario fetched from : {}'.format(test_file_name))
        cmd_line = "rally task start --abort-on-sla-failure %s" % test_file_name
        logger.debug('running command line : {}'.format(cmd_line))
        cmd = os.popen(cmd_line)
        task_id = get_task_id(cmd.read())
        logger.debug('task_id : {}'.format(task_id))

        if task_id is None:
            logger.error("failed to retrieve task_id")
            exit(-1)

        """ check for result directory and create it otherwise """
        if not os.path.exists(RESULTS_DIR):
            logger.debug('does not exists, we create it'.format(RESULTS_DIR))
            os.makedirs(RESULTS_DIR)

        """ write html report file """
        report_file_name = '{}opnfv-{}.html'.format(RESULTS_DIR, test_name)
        cmd_line = "rally task report %s --out %s" % (task_id, report_file_name)
        logger.debug('running command line : {}'.format(cmd_line))
        os.popen(cmd_line)

        """ get and save rally operation JSON result """
        cmd_line = "rally task results %s" % task_id
        logger.debug('running command line : {}'.format(cmd_line))
        cmd = os.popen(cmd_line)
        json_results = cmd.read()
        with open('{}opnfv-{}.json'.format(RESULTS_DIR, test_name), 'w') as f:
            logger.debug('saving json file')
            f.write(json_results)
            logger.debug('saving json file2')

        """ parse JSON operation result """
        if task_succeed(json_results):
            print 'Test OK'
        else:
            print 'Test KO'
    else:
        logger.error('{} test failed, unable to fetch a scenario test file'.format(test_name))

def retrieve_test_cases_file(test_name, tests_path):
    """
    Retrieve from github the sample test files
    :return: Boolean that indicates the retrieval status
    """

    """ do not add the "/" at the end """
    url_base = "https://git.opnfv.org/cgit/functest/plain/testcases/VIM/OpenStack/CI/suites"

    test_file_name = 'opnfv-{}.json'.format(test_name)
    logger.info('fetching {}/{} ...'.format(url_base, test_file_name))

    try:
        response = urllib2.urlopen('{}/{}'.format(url_base, test_file_name))
    except (urllib2.HTTPError, urllib2.URLError):
        return False
    file_raw = response.read()

    """ check if the test path exist otherwise we create it """
    if not os.path.exists(tests_path):
        os.makedirs(tests_path)

    with open('{}/{}'.format(tests_path, test_file_name), 'w') as f:
        f.write(file_raw)
    return True

def main():
    """ configure script """
    if not (args.test_name in tests):
        logger.error('argument not valid')
        exit(-1)

    if args.test_name == "all":
        for test_name in tests:
            if not (test_name == 'all' or test_name == 'tempest' or test_name == 'heat' or test_name == 'smoke' ):
                print(test_name)
                run_task(test_name)
    else:
        print(args.test_name)
        if args.test_name == 'tempest':
            run_tempest()
        else:
            run_task(args.test_name)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before><commit_msg>also vestigial code<commit_after><|endoftext|>


<commit_before><commit_msg>add main block for plotting grouped cluster output<commit_after><|endoftext|>


<commit_before>from sqlagg.base import AliasColumn
from sqlagg.columns import SimpleColumn
from sqlagg.filters import EQ, RawFilter, ORFilter, LTE
from sqlagg.sorting import OrderBy

from corehq.apps.reports.sqlreport import DatabaseColumn
from custom.icds_reports.sqldata.base import IcdsSqlData, ICDSDatabaseColumn
from custom.icds_reports.utils.mixins import ExportableMixin
from custom.icds_reports.utils import get_status, calculate_date_for_age, \
    current_month_stunting_column, \
    current_month_wasting_column, format_decimal, DATA_NOT_ENTERED, phone_number_function

class BeneficiaryExport(ExportableMixin, IcdsSqlData):
    title = 'Child Beneficiary'
    table_name = 'child_health_monthly_view'

    def __init__(self, config=None, loc_level=1, show_test=False, beta=False):
        config.update({
            '5_years': 60,
        })
        super(BeneficiaryExport, self).__init__(config, loc_level, show_test, beta,
                                                use_excluded_states=False)

    @property
    def group_by(self):
        group_by_columns = self.get_columns_by_loc_level
        group_by = []
        for column in group_by_columns:
            if column.slug != 'current_age':
                group_by.append(column.slug)
        return group_by

    def _map_filter_name_to_sql_filter(self, filter_name):
        return {
            'unweighed': RawFilter('recorded_weight IS NULL'),
            'umeasured': RawFilter('recorded_height IS NULL'),
            'severely_underweight': RawFilter("current_month_nutrition_status = 'severely_underweight'"),
            'moderately_underweight': RawFilter("current_month_nutrition_status = 'moderately_underweight'"),
            'normal_wfa': RawFilter("current_month_nutrition_status = 'normal'"),
            'severely_stunted': RawFilter("{} = 'severe'".format(current_month_stunting_column(self.beta))),
            'moderately_stunted': RawFilter("{} = 'moderate'".format(current_month_stunting_column(self.beta))),
            'normal_hfa': RawFilter("{} = 'normal'".format(current_month_stunting_column(self.beta))),
            'severely_wasted': RawFilter("{} = 'severe'".format(current_month_wasting_column(self.beta))),
            'moderately_wasted': RawFilter("{} = 'moderate'".format(current_month_wasting_column(self.beta))),
            'normal_wfh': RawFilter("{} = 'normal'".format(current_month_wasting_column(self.beta))),
        }[filter_name]

    def _build_additional_filters(self, filters):
        if len(filters) == 1:
            return self._map_filter_name_to_sql_filter(filters[0])
        return ORFilter([
            self._map_filter_name_to_sql_filter(filter_name)
            for filter_name in filters
        ])

    @property
    def filters(self):
        filters = [LTE('age_in_months', '5_years')]
        for key, value in self.config.items():
            if key == 'domain' or key == '5_years':
                continue
            elif key == 'filters':
                filters.append(self._build_additional_filters(value))
                continue
            filters.append(EQ(key, key))
        return filters

    @property
    def order_by(self):
        return [OrderBy('person_name')]

    @property
    def get_columns_by_loc_level(self):
        selected_month = self.config['month']

        def test_fucntion(x):
            return format_decimal(x) if x else DATA_NOT_ENTERED

        columns = [
            DatabaseColumn(
                'AWC Name',
                SimpleColumn('awc_name'),
                slug='awc_name'
            ),
            DatabaseColumn(
                'AWC Site Code',
                SimpleColumn('awc_site_code'),
                slug='awc_site_code'
            ),
            DatabaseColumn(
                'Supervisor Name',
                SimpleColumn('supervisor_name'),
                slug='supervisor_name'
            ),
            DatabaseColumn(
                'Block Name',
                SimpleColumn('block_name'),
                slug='block_name'
            ),
            DatabaseColumn(
                'AWW Phone Number',
                SimpleColumn('aww_phone_number'),
                format_fn=phone_number_function,
                slug='aww_phone_number'
            ),
            DatabaseColumn(
                'Mother name',
                SimpleColumn('mother_name'),
                slug='mother_name'
            ),
            DatabaseColumn(
                'Mother Phone Number',
                SimpleColumn('mother_phone_number'),
                format_fn=phone_number_function,
                slug='mother_phone_number'
            ),
            DatabaseColumn(
                'Child Name',
                SimpleColumn('person_name'),
                slug='person_name'
            ),
            DatabaseColumn(
                'Date of Birth',
                SimpleColumn('dob'),
                slug='dob'
            ),
            DatabaseColumn(
                'Current Age (as of {})'.format(selected_month.isoformat()),
                AliasColumn('dob'),
                format_fn=lambda x: calculate_date_for_age(x, self.config['month']),
                slug='current_age'
            ),
            DatabaseColumn(
                'Sex ',
                SimpleColumn('sex'),
                slug='sex'
            ),
            ICDSDatabaseColumn(
                '1 Year Immunizations Complete',
                SimpleColumn('fully_immunized'),
                format_fn=lambda x: 'Yes' if x else 'No'
            ),
            DatabaseColumn(
                'Month for data shown',
                SimpleColumn('month'),
                slug='month'
            ),
            DatabaseColumn(
                'Weight Recorded (in Month)',
                SimpleColumn('recorded_weight'),
                format_fn=test_fucntion,
                slug='recorded_weight'
            ),
            DatabaseColumn(
                'Height Recorded (in Month)',
                SimpleColumn('recorded_height'),
                format_fn=test_fucntion,
                slug='recorded_height'
            ),
            DatabaseColumn(
                'Weight-for-Age Status (in Month)',
                SimpleColumn('current_month_nutrition_status'),
                format_fn=lambda x: get_status(
                    x,
                    'underweight',
                    'Normal weight for age',
                    True
                ),
                slug='current_month_nutrition_status'
            ),
            DatabaseColumn(
                'Weight-for-Height Status (in Month)',
                SimpleColumn(current_month_wasting_column(self.beta)),
                format_fn=lambda x: get_status(
                    x,
                    'wasted',
                    'Normal weight for height',
                    True
                ),
                slug="current_month_wasting_v2"
            ),
            DatabaseColumn(
                'Height-for-Age status (in Month)',
                SimpleColumn(current_month_stunting_column(self.beta)),
                format_fn=lambda x: get_status(
                    x,
                    'stunted',
                    'Normal height for age',
                    True
                ),
                slug="current_month_stunting_v2"
            ),
            DatabaseColumn(
                'Days attended PSE (as of {})'.format(selected_month.isoformat()),
                SimpleColumn('pse_days_attended'),
                slug="pse_days_attended"
            ),
        ]
        return columns

    @property
    def columns(self):
        return self.get_columns_by_loc_level
<commit_msg>simpler super<commit_after>from sqlagg.base import AliasColumn
from sqlagg.columns import SimpleColumn
from sqlagg.filters import EQ, RawFilter, ORFilter, LTE
from sqlagg.sorting import OrderBy

from corehq.apps.reports.sqlreport import DatabaseColumn
from custom.icds_reports.sqldata.base import IcdsSqlData, ICDSDatabaseColumn
from custom.icds_reports.utils.mixins import ExportableMixin
from custom.icds_reports.utils import get_status, calculate_date_for_age, \
    current_month_stunting_column, \
    current_month_wasting_column, format_decimal, DATA_NOT_ENTERED, phone_number_function

class BeneficiaryExport(ExportableMixin, IcdsSqlData):
    title = 'Child Beneficiary'
    table_name = 'child_health_monthly_view'

    def __init__(self, config=None, loc_level=1, show_test=False, beta=False):
        config.update({
            '5_years': 60,
        })
        super().__init__(config, loc_level, show_test, beta, use_excluded_states=False)

    @property
    def group_by(self):
        group_by_columns = self.get_columns_by_loc_level
        group_by = []
        for column in group_by_columns:
            if column.slug != 'current_age':
                group_by.append(column.slug)
        return group_by

    def _map_filter_name_to_sql_filter(self, filter_name):
        return {
            'unweighed': RawFilter('recorded_weight IS NULL'),
            'umeasured': RawFilter('recorded_height IS NULL'),
            'severely_underweight': RawFilter("current_month_nutrition_status = 'severely_underweight'"),
            'moderately_underweight': RawFilter("current_month_nutrition_status = 'moderately_underweight'"),
            'normal_wfa': RawFilter("current_month_nutrition_status = 'normal'"),
            'severely_stunted': RawFilter("{} = 'severe'".format(current_month_stunting_column(self.beta))),
            'moderately_stunted': RawFilter("{} = 'moderate'".format(current_month_stunting_column(self.beta))),
            'normal_hfa': RawFilter("{} = 'normal'".format(current_month_stunting_column(self.beta))),
            'severely_wasted': RawFilter("{} = 'severe'".format(current_month_wasting_column(self.beta))),
            'moderately_wasted': RawFilter("{} = 'moderate'".format(current_month_wasting_column(self.beta))),
            'normal_wfh': RawFilter("{} = 'normal'".format(current_month_wasting_column(self.beta))),
        }[filter_name]

    def _build_additional_filters(self, filters):
        if len(filters) == 1:
            return self._map_filter_name_to_sql_filter(filters[0])
        return ORFilter([
            self._map_filter_name_to_sql_filter(filter_name)
            for filter_name in filters
        ])

    @property
    def filters(self):
        filters = [LTE('age_in_months', '5_years')]
        for key, value in self.config.items():
            if key == 'domain' or key == '5_years':
                continue
            elif key == 'filters':
                filters.append(self._build_additional_filters(value))
                continue
            filters.append(EQ(key, key))
        return filters

    @property
    def order_by(self):
        return [OrderBy('person_name')]

    @property
    def get_columns_by_loc_level(self):
        selected_month = self.config['month']

        def test_fucntion(x):
            return format_decimal(x) if x else DATA_NOT_ENTERED

        columns = [
            DatabaseColumn(
                'AWC Name',
                SimpleColumn('awc_name'),
                slug='awc_name'
            ),
            DatabaseColumn(
                'AWC Site Code',
                SimpleColumn('awc_site_code'),
                slug='awc_site_code'
            ),
            DatabaseColumn(
                'Supervisor Name',
                SimpleColumn('supervisor_name'),
                slug='supervisor_name'
            ),
            DatabaseColumn(
                'Block Name',
                SimpleColumn('block_name'),
                slug='block_name'
            ),
            DatabaseColumn(
                'AWW Phone Number',
                SimpleColumn('aww_phone_number'),
                format_fn=phone_number_function,
                slug='aww_phone_number'
            ),
            DatabaseColumn(
                'Mother name',
                SimpleColumn('mother_name'),
                slug='mother_name'
            ),
            DatabaseColumn(
                'Mother Phone Number',
                SimpleColumn('mother_phone_number'),
                format_fn=phone_number_function,
                slug='mother_phone_number'
            ),
            DatabaseColumn(
                'Child Name',
                SimpleColumn('person_name'),
                slug='person_name'
            ),
            DatabaseColumn(
                'Date of Birth',
                SimpleColumn('dob'),
                slug='dob'
            ),
            DatabaseColumn(
                'Current Age (as of {})'.format(selected_month.isoformat()),
                AliasColumn('dob'),
                format_fn=lambda x: calculate_date_for_age(x, self.config['month']),
                slug='current_age'
            ),
            DatabaseColumn(
                'Sex ',
                SimpleColumn('sex'),
                slug='sex'
            ),
            ICDSDatabaseColumn(
                '1 Year Immunizations Complete',
                SimpleColumn('fully_immunized'),
                format_fn=lambda x: 'Yes' if x else 'No'
            ),
            DatabaseColumn(
                'Month for data shown',
                SimpleColumn('month'),
                slug='month'
            ),
            DatabaseColumn(
                'Weight Recorded (in Month)',
                SimpleColumn('recorded_weight'),
                format_fn=test_fucntion,
                slug='recorded_weight'
            ),
            DatabaseColumn(
                'Height Recorded (in Month)',
                SimpleColumn('recorded_height'),
                format_fn=test_fucntion,
                slug='recorded_height'
            ),
            DatabaseColumn(
                'Weight-for-Age Status (in Month)',
                SimpleColumn('current_month_nutrition_status'),
                format_fn=lambda x: get_status(
                    x,
                    'underweight',
                    'Normal weight for age',
                    True
                ),
                slug='current_month_nutrition_status'
            ),
            DatabaseColumn(
                'Weight-for-Height Status (in Month)',
                SimpleColumn(current_month_wasting_column(self.beta)),
                format_fn=lambda x: get_status(
                    x,
                    'wasted',
                    'Normal weight for height',
                    True
                ),
                slug="current_month_wasting_v2"
            ),
            DatabaseColumn(
                'Height-for-Age status (in Month)',
                SimpleColumn(current_month_stunting_column(self.beta)),
                format_fn=lambda x: get_status(
                    x,
                    'stunted',
                    'Normal height for age',
                    True
                ),
                slug="current_month_stunting_v2"
            ),
            DatabaseColumn(
                'Days attended PSE (as of {})'.format(selected_month.isoformat()),
                SimpleColumn('pse_days_attended'),
                slug="pse_days_attended"
            ),
        ]
        return columns

    @property
    def columns(self):
        return self.get_columns_by_loc_level
<|endoftext|>