<commit_before>""" RHEAS module for retrieving MODIS evapotranspiration data (MOD16 product).

.. module:: mod16
   :synopsis: Retrieve MODIS evapotranspiration data

.. moduleauthor:: Kostas Andreadis <kandread@jpl.nasa.gov>

"""

import modis
import tempfile
from ftplib import FTP
import subprocess
import glob
import shutil
import dbio
import datasets
from datetime import timedelta
import logging

table = "evap.mod16"

def dates(dbname):
    dts = datasets.dates(dbname, table)
    return dts

def download(dbname, dts, bbox):
    """Downloads the MODIS evapotranspiration data product MOD16 for
    a set of dates *dt* and imports them into the PostGIS database *dbname*."""
    log = logging.getLogger(__name__)
    res = 0.01
    url = "ftp.ntsg.umt.edu"
    tiles = modis.findTiles(bbox)
    if tiles is not None:
        ftp = FTP(url)
        ftp.login()
        for dt in [dts[0] + timedelta(dti) for dti in range((dts[-1] - dts[0]).days + 1)]:
            try:
                ftp.cwd(
                    "pub/MODIS/NTSG_Products/MOD16/MOD16A2.105_MERRAGMAO/Y{0}".format(dt.year))
                days = ftp.nlst()
                datadir = "D{0}".format(dt.strftime("%j"))
                if datadir in days:
                    ftp.cwd(datadir)
                    files = [f for f in ftp.nlst() if any(
                        f.find("h{0:02d}v{1:02d}".format(t[1], t[0])) > 0 for t in tiles)]
                    outpath = tempfile.mkdtemp()
                    for fname in files:
                        with open("{0}/{1}".format(outpath, fname), 'wb') as f:
                            ftp.retrbinary("RETR {0}".format(fname), f.write)
                        proc = subprocess.Popen(["gdal_translate", "HDF4_EOS:EOS_GRID:{0}/{1}:MOD_Grid_MOD16A2:ET_1km".format(
                            outpath, fname), "{0}/{1}".format(outpath, fname).replace("hdf", "tif")], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                        out, err = proc.communicate()
                        log.debug(out)
                    tifs = glob.glob("{0}/*.tif".format(outpath))
                    proc = subprocess.Popen(
                        ["gdal_merge.py", "-o", "{0}/et.tif".format(outpath)] + tifs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                    out, err = proc.communicate()
                    log.debug(out)
                    proc = subprocess.Popen(["gdal_calc.py", "-A", "{0}/et.tif".format(outpath), "--outfile={0}/et1.tif".format(
                        outpath), "--NoDataValue=-9999", "--calc=\"(A<32701)*(0.1*A+9999)-9999\""], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                    out, err = proc.communicate()
                    log.debug(out)
                    proc = subprocess.Popen(["gdalwarp", "-t_srs", "'+proj=latlong +ellps=sphere'", "-tr", str(
                        res), str(-res), "{0}/et1.tif".format(outpath), "{0}/et2.tif".format(outpath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                    out, err = proc.communicate()
                    log.debug(out)
                    if bbox is None:
                        pstr = []
                    else:
                        pstr = ["-projwin", str(bbox[0]), str(bbox[3]), str(bbox[2]), str(bbox[1])]
                    proc = subprocess.Popen(["gdal_translate"] + pstr + ["-a_srs", "epsg:4326", "{0}/et2.tif".format(outpath), "{0}/et3.tif".format(outpath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                    out, err = proc.communicate()
                    log.debug(out)
                    dbio.ingest(
                        dbname, "{0}/et3.tif".format(outpath), dt, table, False)
                    shutil.rmtree(outpath)
            except:
                log.warning("MOD16 data not available for {0}. Skipping download!".format(
                    dt.strftime("%Y-%m-%d")))

class Mod16:

    def __init__(self):
        """Initialize MOD16 evapotranspiration object."""
        self.statevar = ["soil_moist"]
        self.obsvar = "evap"
        self.res = 0.01
        self.stddev = 1.0
        self.tablename = "evap.mod16"
<commit_msg>Fixes #64. Changed FTP to HTTP for MOD16 dataset.<commit_after>""" RHEAS module for retrieving MODIS evapotranspiration data (MOD16 product).

.. module:: mod16
   :synopsis: Retrieve MODIS evapotranspiration data

.. moduleauthor:: Kostas Andreadis <kandread@jpl.nasa.gov>

"""

import modis
import tempfile
import subprocess
import glob
import shutil
import dbio
import datasets
from datetime import timedelta
import requests
from bs4 import BeautifulSoup, SoupStrainer
from bs4.element import Tag
import logging

table = "evap.mod16"

def dates(dbname):
    dts = datasets.dates(dbname, table)
    return dts

def download(dbname, dts, bbox):
    """Downloads the MODIS evapotranspiration data product MOD16 for
    a set of dates *dt* and imports them into the PostGIS database *dbname*."""
    log = logging.getLogger(__name__)
    res = 0.01
    urlbase = "http://files.ntsg.umt.edu"
    tiles = modis.findTiles(bbox)
    if tiles is not None:
        for dt in [dts[0] + timedelta(dti) for dti in range((dts[-1] - dts[0]).days + 1)]:
            url = "{0}/data/NTSG_Products/MOD16/MOD16A2.105_MERRAGMAO/Y{1}".format(urlbase, dt.year)
            resp_year = requests.get(url)
            try:
                assert resp_year.status_code == 200
                days = [link for link in BeautifulSoup(resp_year.text, parse_only=SoupStrainer('a')) if isinstance(link, Tag) and link.text.find(dt.strftime("%j")) >= 0]
                assert len(days) > 0
                resp_day = requests.get("{0}{1}".format(urlbase, days[0].get('href')))
                assert resp_day.status_code == 200
                files = [link.get('href') for link in BeautifulSoup(resp_day.text, parse_only=SoupStrainer('a')) if isinstance(link, Tag) and link.text.find("hdf") > 0]
                files = [f for f in files if any(f.find("h{0:02d}v{1:02d}".format(t[1], t[0])) > 0 for t in tiles)]
                outpath = tempfile.mkdtemp()
                for fname in files:
                    resp_file = requests.get("{0}{1}".format(urlbase, fname))
                    filename = fname.split("/")[-1]
                    with open("{0}/{1}".format(outpath, filename), 'wb') as fout:
                        for chunk in resp_file:
                            fout.write(chunk)
                    proc = subprocess.Popen(["gdal_translate", "HDF4_EOS:EOS_GRID:{0}/{1}:MOD_Grid_MOD16A2:ET_1km".format(
                        outpath, filename), "{0}/{1}".format(outpath, filename).replace("hdf", "tif")], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                    out, err = proc.communicate()
                    log.debug(out)
                tifs = glob.glob("{0}/*.tif".format(outpath))
                proc = subprocess.Popen(
                    ["gdal_merge.py", "-o", "{0}/et.tif".format(outpath)] + tifs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                out, err = proc.communicate()
                log.debug(out)
                proc = subprocess.Popen(["gdal_calc.py", "-A", "{0}/et.tif".format(outpath), "--outfile={0}/et1.tif".format(
                    outpath), "--NoDataValue=-9999", "--calc=(A<32701)*(0.1*A+9999)-9999"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                out, err = proc.communicate()
                log.debug(out)
                proc = subprocess.Popen(["gdalwarp", "-t_srs", "+proj=latlong +ellps=sphere", "-tr", str(
                    res), str(-res), "{0}/et1.tif".format(outpath), "{0}/et2.tif".format(outpath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                out, err = proc.communicate()
                log.debug(out)
                if bbox is None:
                    pstr = []
                else:
                    pstr = ["-projwin", str(bbox[0]), str(bbox[3]), str(bbox[2]), str(bbox[1])]
                proc = subprocess.Popen(["gdal_translate"] + pstr + ["-a_srs", "epsg:4326", "{0}/et2.tif".format(outpath), "{0}/et3.tif".format(outpath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                out, err = proc.communicate()
                log.debug(out)
                dbio.ingest(
                    dbname, "{0}/et3.tif".format(outpath), dt, table, False)
                shutil.rmtree(outpath)
            except:
                log.warning("MOD16 data not available for {0}. Skipping download!".format(
                    dt.strftime("%Y-%m-%d")))

class Mod16:

    def __init__(self):
        """Initialize MOD16 evapotranspiration object."""
        self.statevar = ["soil_moist"]
        self.obsvar = "evap"
        self.res = 0.01
        self.stddev = 1.0
        self.tablename = "evap.mod16"
<|endoftext|>


<commit_before>import re, requests, ast, time
import parsedatetime as pdt
import datetime
from datetime import date
import bs4
import utilities
from gluon import *
class Profile(object):
    """
        Class containing methods for retrieving
        submissions of user
    """

    # -------------------------------------------------------------------------
    def __init__(self, site="", handle=""):

        self.site = None
        self.handle = None

        if site in utilities.SITES:
            self.handle = handle
            self.site = site

    @staticmethod
    # -------------------------------------------------------------------------
    def parsetime(time_str):
        try:
            dt = datetime.datetime.strptime(time_str, "%I:%M %p %d/%m/%y")
            return dt
        except ValueError:
            cal = pdt.Calendar()
            dt, flags = cal.parseDT(time_str)
            assert flags
            return dt

    # -------------------------------------------------------------------------
    def codechef(self, last_retrieved):

        if self.handle:
            handle = self.handle
        else:
            return {}

        user_url = "http://www.codechef.com/recent/user?user_handle=" + handle
        user_agent = "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5"

        while 1:
            tmp = requests.get(user_url,
                               headers={"User-Agent": user_agent},
                               proxies=utilities.PROXY)
            if tmp.status_code == 200:
                break

        d = ast.literal_eval(tmp.text)
        max_page = d["max_page"]
        submissions = {handle: {}}
        it = 1

        for page in xrange(0, max_page):
            user_url = "http://www.codechef.com/recent/user?user_handle=" + \
                       handle + \
                       "&page=" + \
                       str(page)

            while 1:
                tmp = requests.get(user_url,
                                   headers={"User-Agent": user_agent},
                                   proxies=utilities.PROXY)
                if tmp.status_code == 200:
                    break

            d = ast.literal_eval(tmp.text)["content"]

            submissions[handle][page] = {}
            x = bs4.BeautifulSoup(d)
            for i in x.find_all("tr"):
                try:
                    if i['class'][0] == "kol":

                        submissions[handle][page][it] = []
                        submission = submissions[handle][page][it]
                        append = submission.append

                        # tos = time_of_submission
                        tos = i.contents[0].contents[0]
                        tos = str(ast.literal_eval(repr(tos).replace("\\", "")))

                        # Do not retrieve any further because this leads to ambiguity
                        # If 2 hours ago => 2 hours 20 mins or 2 hours 14 mins ...
                        # Let the user come back later when the datetime is exact
                        # This prevents from redundant addition into database
                        # @ToDo: For now we are allowing redundant submissions
                        #        for codechef :/ . Find a way to change it.
                        #if tos.__contains__("hours"):
                        #   continue

                        tos = Profile.parsetime(tos)
                        curr = time.strptime(str(tos), "%Y-%m-%d %H:%M:%S")

                        if curr <= last_retrieved:
                            return submissions
                        append(str(tos))

                        # Problem name/url
                        prob = i.contents[1].contents[0]
                        prob["href"] = "http://www.codechef.com" + prob["href"]
                        append(eval(repr(prob["href"]).replace("\\", "")))
                        try:
                            append(prob.contents[0])
                        except IndexError:
                            append("")

                        # Submission status
                        stat = i.contents[2].contents[0]
                        stat = stat.find("img")["src"]
                        stat = repr(stat).replace("\\", "")
                        stat = stat[7:-5]
                        st = "AC"
                        if stat == "tick-icon":
                            st = "AC"
                        elif stat == "cross-icon":
                            st = "WA"
                        elif stat == "alert-icon":
                            st = "CE"
                        elif stat == "runtime-error":
                            st = "RE"
                        elif stat == "clock_error":
                            st = "TLE"
                        else:
                            st = "OTH"
                        append(st)

                        # Question points
                        pts = i.contents[2].contents[0].contents
                        try:
                            if  len(pts) >= 5:
                                points = pts[2] + " " + pts[4]
                            else:
                                points = pts[2]
                        except IndexError:
                            if st == "AC":
                                points = "100"
                            else:
                                points = "0"
                        append(points)

                        # Language
                        append(i.contents[3].contents[0].strip())
                        it += 1
                except KeyError:
                    pass
        return submissions

    # -------------------------------------------------------------------------
    def codeforces(self, last_retrieved):

        if self.handle:
            handle = self.handle
        else:
            return {}

        tmp = requests.get("http://codeforces.com/api/user.status?handle=" + \
                           handle + \
                           "&from=1&count=5000")

        submissions = {handle: {1: {}}}
        all_submissions = tmp.json()
        if all_submissions["status"] != "OK":
            return submissions
        all_submissions = all_submissions["result"]
        it = 0
        for row in all_submissions:

            submissions[handle][1][it] = []
            submission = submissions[handle][1][it]
            append = submission.append

            curr = time.gmtime(row["creationTimeSeconds"] + 330 * 60)
            if curr <= last_retrieved:
                return submissions
            append(str(time.strftime("%Y-%m-%d %H:%M:%S", curr)))
            time_stamp = time.strftime("%Y-%m-%d %H:%M:%S", curr)

            arg = "problem/"
            if len(str(row["contestId"])) != 3:
                arg = "gymProblem/"

            problem_link = "http://codeforces.com/problemset/" + arg + \
                           str(row["contestId"]) + "/" + \
                           row["problem"]["index"]
            append(problem_link)
            problem_name = row["problem"]["name"]
            append(problem_name)

            status = row["verdict"]
            st = "AC"
            if status == "OK":
                st = "AC"
            elif status == "WRONG_ANSWER":
                st = "WA"
            elif status == "COMPILATION_ERROR":
                st = "CE"
            elif status == "SKIPPED":
                st = "SK"
            elif status == "RUNTIME_ERROR":
                st = "RE"
            elif status == "TIME_LIMIT_EXCEEDED":
                st = "TLE"
            elif status == "CHALLENGED":
                st = "HCK"
            elif status == "MEMORY_LIMIT_EXCEEDED":
                st = "MLE"
            elif status == "TESTING":
                continue
            else:
                st = "OTH"
            append(st)

            if st == "AC":
                points = "100"
            elif st == "HCK":
                points = "-50"
            else:
                points = "0"
            append(points)

            append(row["programmingLanguage"])

            it += 1
        return submissions

    # -------------------------------------------------------------------------
    def spoj(self, last_retrieved):

        if self.handle:
            handle = self.handle
        else:
            return {}

        submissions = {handle: {}}
        start = 0
        it = 1

        previd = -1
        currid = 0
        page = 0
        url = "https://www.spoj.com/users/" + handle
        tmpreq = requests.get(url, proxies=utilities.PROXY)

        # Bad but correct way of checking if the handle exists
        if tmpreq.text.find("History of submissions") == -1:
            return submissions

        while 1:
            flag = 0
            url = "https://www.spoj.com/status/" + \
                  handle + \
                  "/all/start=" + \
                  str(start)

            start += 20
            t = requests.get(url, proxies=utilities.PROXY)
            soup = bs4.BeautifulSoup(t.text)
            table_body = soup.find("tbody")

            # Check if the page retrieved has no submissions
            if len(table_body) == 1:
                return submissions

            row = 0
            submissions[handle][page] = {}
            for i in table_body:
                submissions[handle][page][it] = []
                submission = submissions[handle][page][it]
                append = submission.append

                if isinstance(i, bs4.element.Tag):
                    if row == 0:
                        currid = i.contents[1].contents[0]
                        if currid == previd:
                            flag = 1
                            break
                    row += 1
                    previd = currid

                    # Time of submission
                    tos = i.contents[3].contents[1].contents[0]
                    curr = time.strptime(str(tos), "%Y-%m-%d %H:%M:%S")
                    if curr <= last_retrieved:
                        return submissions
                    append(str(tos))

                    # Problem URL
                    uri = i.contents[5].contents[0]
                    uri["href"] = "http://www.spoj.com" + uri["href"]
                    append(eval(repr(uri["href"]).replace("\\", "")))
                    append(uri.contents[0].strip())

                    # Problem Status
                    status = str(i.contents[6])
                    if status.__contains__("accepted"):
                        st = "AC"
                    elif status.__contains__("wrong"):
                        st = "WA"
                    elif status.__contains__("compilation"):
                        st = "CE"
                    elif status.__contains__("runtime"):
                        st = "RE"
                    elif status.__contains__("time limit"):
                        st = "TLE"
                    else:
                        st = "OTH"

                    append(st)

                    # Question Points
                    if st == "AC":
                        points = "100"
                    else:
                        points = "0"
                    append(points)

                    # Language
                    append(i.contents[12].contents[1].contents[0])

                    it += 1
            page += 1
            if flag == 1:
                break
        return submissions

    # -------------------------------------------------------------------------
    def hackerearth(self, last_retrieved):

        if self.handle:
            handle = self.handle
        else:
            return {}

        url = "https://www.hackerearth.com/submissions/" + handle
        t = requests.get(url, proxies=utilities.PROXY)
        tmp_string = t.headers['set-cookie']
        csrf_token = re.findall("csrftoken=\w*", tmp_string)[0][10:]
        url = "https://www.hackerearth.com/AJAX/feed/newsfeed/submission/user/" + handle + "/"

        response = {}
        response["host"] = "www.hackerearth.com"
        response["user-agent"] = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:40.0) Gecko/20100101 Firefox/40.0"
        response["accept"] = "application/json, text/javascript, */*; q=0.01"
        response["accept-language"] = "en-US,en;q=0.5"
        response["accept-encoding"] = "gzip, deflate"
        response["content-type"] = "application/x-www-form-urlencoded"
        response["X-CSRFToken"] = csrf_token
        response["X-Requested-With"] = "XMLHttpRequest"
        response["Referer"] = "https://www.hackerearth.com/submissions/raj454raj/"
        response["Connection"] = "keep-alive"
        response["Pragma"] = "no-cache"
        response["Cache-Control"] = "no-cache"
        response["Cookie"] = tmp_string

        it = 1
        submissions = {handle: {}}
        for i in xrange(0, 10000, 20):
            submissions[handle][i / 20] = {}
            data = {"index": str(i)}

            tmp = requests.post(url,
                                data=data,
                                proxies=utilities.PROXY,
                                headers=response)

            try:
                final_json = tmp.json()
            except:
                break

            if final_json["count"] == 0:
                break

            for feed_number in xrange(20):

                submissions[handle][i / 20][it] = []
                submission = submissions[handle][i / 20][it]
                append = submission.append

                try:
                    final = bs4.BeautifulSoup((tmp.json()["feed" + str(feed_number)]))
                except KeyError:
                    break

                all_as = final.find_all("a")
                all_tds = final.find_all("td")
                tos = all_tds[-1].contents[1]["title"]
                time_stamp = time.strptime(str(tos), "%Y-%m-%d %H:%M:%S")

                if time_stamp <= last_retrieved:
                    return submissions
                append(str(tos))

                problem_link = "https://hackerearth.com" + all_as[1]["href"]
                append(problem_link)
                problem_name = all_as[1].contents[0]
                append(problem_name)

                try:
                    status = all_tds[2].contents[1]["title"]
                except IndexError:
                    status = "Others"

                if status.__contains__("Accepted"):
                    status = "AC"
                elif status.__contains__("Wrong"):
                    status = "WA"
                elif status.__contains__("Compilation"):
                    status = "CE"
                elif status.__contains__("Runtime"):
                    status = "RE"
                elif status.__contains__("Memory"):
                    status = "MLE"
                elif status.__contains__("Time"):
                    status = "TLE"
                else:
                    status = "OTH"
                append(status)

                if status == "AC":
                    points = "100"
                else:
                    points = "0"
                append(points)

                language = all_tds[5].contents[0]
                append(language)
                it += 1
        return submissions

    # -------------------------------------------------------------------------
    def hackerrank(self, last_retrieved):

        if self.handle:
            handle = self.handle
        else:
            return {}

        tmp = requests.get("https://www.hackerrank.com/rest/hackers/" + \
                           handle + \
                           "/recent_challenges?offset=0&limit=50000")

        all_submissions = tmp.json()["models"]

        submissions = {handle: {1 : {}}}
        it = 0
        for row in all_submissions:

            submission = submissions[handle][1]
            submission[it] = []
            submission = submission[it]
            append = submission.append

            time_stamp = row["created_at"][:-5].split("T")
            curr = time.strptime(time_stamp[0] + " " + time_stamp[1],
                                 "%Y-%m-%d %H:%M:%S")

            if curr <= last_retrieved:
                return submissions
            append(" ".join(time_stamp))

            append("https://hackerrank.com/challenges/" + row["slug"])
            append(row["name"])
            append("AC")
            append("100")
            append("-")
            it += 1

        return submissions
<commit_msg>Booyeah 14:21 -> 4:11 :D<commit_after>import re, requests, ast, time
import parsedatetime as pdt
import datetime
from datetime import date
import bs4
import utilities
from gluon import *
import gevent
from gevent import monkey

monkey.patch_all(thread=False)
user_agent = "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5"

class Profile(object):
    """
        Class containing methods for retrieving
        submissions of user
    """

    # -------------------------------------------------------------------------
    def __init__(self, site="", handle=""):

        self.site = None
        self.handle = None
        self.submissions = {handle: {}}

        if site in utilities.SITES:
            self.handle = handle
            self.site = site

    @staticmethod
    # -------------------------------------------------------------------------
    def parsetime(time_str):
        try:
            dt = datetime.datetime.strptime(time_str, "%I:%M %p %d/%m/%y")
            return dt
        except ValueError:
            cal = pdt.Calendar()
            dt, flags = cal.parseDT(time_str)
            assert flags
            return dt

    # -------------------------------------------------------------------------
    def parallelize_codechef(self, handle, page):

        url = "https://www.codechef.com/recent/user?user_handle=" + \
               handle + \
               "&page=" + \
               str(page)

        while True:
            tmp = requests.get(url,
                               headers={"User-Agent": user_agent},
                               proxies=utilities.PROXY)
            if tmp.status_code == 200:
                break

        d = ast.literal_eval(tmp.text)["content"]

        it = 1
        self.submissions[handle][page] = {}
        x = bs4.BeautifulSoup(d)
        for i in x.find_all("tr"):
            try:
                if i['class'][0] == "kol":

                    self.submissions[handle][page][it] = []
                    submission = self.submissions[handle][page][it]
                    append = submission.append

                    # tos = time_of_submission
                    tos = i.contents[0].contents[0]
                    tos = str(ast.literal_eval(repr(tos).replace("\\", "")))
                    tos = Profile.parsetime(tos)
                    append(str(tos))

                    # Problem name/url
                    prob = i.contents[1].contents[0]
                    prob["href"] = "http://www.codechef.com" + prob["href"]
                    append(eval(repr(prob["href"]).replace("\\", "")))
                    try:
                        append(prob.contents[0])
                    except IndexError:
                        append("")

                    # Submission status
                    stat = i.contents[2].contents[0]
                    stat = stat.find("img")["src"]
                    stat = repr(stat).replace("\\", "")
                    stat = stat[7:-5]
                    st = "AC"
                    if stat == "tick-icon":
                        st = "AC"
                    elif stat == "cross-icon":
                        st = "WA"
                    elif stat == "alert-icon":
                        st = "CE"
                    elif stat == "runtime-error":
                        st = "RE"
                    elif stat == "clock_error":
                        st = "TLE"
                    else:
                        st = "OTH"
                    append(st)

                    # Question points
                    pts = i.contents[2].contents[0].contents
                    try:
                        if  len(pts) >= 5:
                            points = pts[2] + " " + pts[4]
                        else:
                            points = pts[2]
                    except IndexError:
                        if st == "AC":
                            points = "100"
                        else:
                            points = "0"
                    append(points)

                    # Language
                    append(i.contents[3].contents[0].strip())
                    it += 1
            except KeyError:
                pass

    # -------------------------------------------------------------------------
    def codechef(self, last_retrieved):

        if self.handle:
            handle = self.handle
        else:
            return {}

        user_url = "http://www.codechef.com/recent/user?user_handle=" + handle

        while 1:
            tmp = requests.get(user_url,
                               headers={"User-Agent": user_agent},
                               proxies=utilities.PROXY)
            if tmp.status_code == 200:
                break

        d = ast.literal_eval(tmp.text)
        max_page = d["max_page"]
        submissions = {handle: {}}
        it = 1
        if time.strptime("2013-01-01 00:00:00", "%Y-%m-%d %H:%M:%S") == last_retrieved:
            threads = []
            for i in xrange(max_page):
                threads.append(gevent.spawn(self.parallelize_codechef, handle, i))
            gevent.joinall(threads)
            return self.submissions

        else:
            for page in xrange(0, max_page):
                user_url = "http://www.codechef.com/recent/user?user_handle=" + \
                           handle + \
                           "&page=" + \
                           str(page)

                while 1:
                    tmp = requests.get(user_url,
                                       headers={"User-Agent": user_agent},
                                       proxies=utilities.PROXY)
                    if tmp.status_code == 200:
                        break

                d = ast.literal_eval(tmp.text)["content"]

                submissions[handle][page] = {}
                x = bs4.BeautifulSoup(d)
                for i in x.find_all("tr"):
                    try:
                        if i['class'][0] == "kol":

                            submissions[handle][page][it] = []
                            submission = submissions[handle][page][it]
                            append = submission.append

                            # tos = time_of_submission
                            tos = i.contents[0].contents[0]
                            tos = str(ast.literal_eval(repr(tos).replace("\\", "")))

                            # Do not retrieve any further because this leads to ambiguity
                            # If 2 hours ago => 2 hours 20 mins or 2 hours 14 mins ...
                            # Let the user come back later when the datetime is exact
                            # This prevents from redundant addition into database
                            # @ToDo: For now we are allowing redundant submissions
                            #        for codechef :/ . Find a way to change it.
                            #if tos.__contains__("hours"):
                            #   continue

                            tos = Profile.parsetime(tos)
                            curr = time.strptime(str(tos), "%Y-%m-%d %H:%M:%S")

                            if curr <= last_retrieved:
                                return submissions
                            append(str(tos))

                            # Problem name/url
                            prob = i.contents[1].contents[0]
                            prob["href"] = "http://www.codechef.com" + prob["href"]
                            append(eval(repr(prob["href"]).replace("\\", "")))
                            try:
                                append(prob.contents[0])
                            except IndexError:
                                append("")

                            # Submission status
                            stat = i.contents[2].contents[0]
                            stat = stat.find("img")["src"]
                            stat = repr(stat).replace("\\", "")
                            stat = stat[7:-5]
                            st = "AC"
                            if stat == "tick-icon":
                                st = "AC"
                            elif stat == "cross-icon":
                                st = "WA"
                            elif stat == "alert-icon":
                                st = "CE"
                            elif stat == "runtime-error":
                                st = "RE"
                            elif stat == "clock_error":
                                st = "TLE"
                            else:
                                st = "OTH"
                            append(st)

                            # Question points
                            pts = i.contents[2].contents[0].contents
                            try:
                                if  len(pts) >= 5:
                                    points = pts[2] + " " + pts[4]
                                else:
                                    points = pts[2]
                            except IndexError:
                                if st == "AC":
                                    points = "100"
                                else:
                                    points = "0"
                            append(points)

                            # Language
                            append(i.contents[3].contents[0].strip())
                            it += 1
                    except KeyError:
                        pass
            return submissions

    # -------------------------------------------------------------------------
    def codeforces(self, last_retrieved):

        if self.handle:
            handle = self.handle
        else:
            return {}

        tmp = requests.get("http://codeforces.com/api/user.status?handle=" + \
                           handle + \
                           "&from=1&count=5000")

        submissions = {handle: {1: {}}}
        all_submissions = tmp.json()
        if all_submissions["status"] != "OK":
            return submissions
        all_submissions = all_submissions["result"]
        it = 0
        for row in all_submissions:

            submissions[handle][1][it] = []
            submission = submissions[handle][1][it]
            append = submission.append

            curr = time.gmtime(row["creationTimeSeconds"] + 330 * 60)
            if curr <= last_retrieved:
                return submissions
            append(str(time.strftime("%Y-%m-%d %H:%M:%S", curr)))
            time_stamp = time.strftime("%Y-%m-%d %H:%M:%S", curr)

            arg = "problem/"
            if len(str(row["contestId"])) != 3:
                arg = "gymProblem/"

            problem_link = "http://codeforces.com/problemset/" + arg + \
                           str(row["contestId"]) + "/" + \
                           row["problem"]["index"]
            append(problem_link)
            problem_name = row["problem"]["name"]
            append(problem_name)

            status = row["verdict"]
            st = "AC"
            if status == "OK":
                st = "AC"
            elif status == "WRONG_ANSWER":
                st = "WA"
            elif status == "COMPILATION_ERROR":
                st = "CE"
            elif status == "SKIPPED":
                st = "SK"
            elif status == "RUNTIME_ERROR":
                st = "RE"
            elif status == "TIME_LIMIT_EXCEEDED":
                st = "TLE"
            elif status == "CHALLENGED":
                st = "HCK"
            elif status == "MEMORY_LIMIT_EXCEEDED":
                st = "MLE"
            elif status == "TESTING":
                continue
            else:
                st = "OTH"
            append(st)

            if st == "AC":
                points = "100"
            elif st == "HCK":
                points = "-50"
            else:
                points = "0"
            append(points)

            append(row["programmingLanguage"])

            it += 1
        return submissions

    # -------------------------------------------------------------------------
    def spoj(self, last_retrieved):

        if self.handle:
            handle = self.handle
        else:
            return {}

        submissions = {handle: {}}
        start = 0
        it = 1

        previd = -1
        currid = 0
        page = 0
        url = "https://www.spoj.com/users/" + handle
        tmpreq = requests.get(url, proxies=utilities.PROXY)

        # Bad but correct way of checking if the handle exists
        if tmpreq.text.find("History of submissions") == -1:
            return submissions

        while 1:
            flag = 0
            url = "https://www.spoj.com/status/" + \
                  handle + \
                  "/all/start=" + \
                  str(start)

            start += 20
            t = requests.get(url, proxies=utilities.PROXY)
            soup = bs4.BeautifulSoup(t.text)
            table_body = soup.find("tbody")

            # Check if the page retrieved has no submissions
            if len(table_body) == 1:
                return submissions

            row = 0
            submissions[handle][page] = {}
            for i in table_body:
                submissions[handle][page][it] = []
                submission = submissions[handle][page][it]
                append = submission.append

                if isinstance(i, bs4.element.Tag):
                    if row == 0:
                        currid = i.contents[1].contents[0]
                        if currid == previd:
                            flag = 1
                            break
                    row += 1
                    previd = currid

                    # Time of submission
                    tos = i.contents[3].contents[1].contents[0]
                    curr = time.strptime(str(tos), "%Y-%m-%d %H:%M:%S")
                    if curr <= last_retrieved:
                        return submissions
                    append(str(tos))

                    # Problem URL
                    uri = i.contents[5].contents[0]
                    uri["href"] = "http://www.spoj.com" + uri["href"]
                    append(eval(repr(uri["href"]).replace("\\", "")))
                    append(uri.contents[0].strip())

                    # Problem Status
                    status = str(i.contents[6])
                    if status.__contains__("accepted"):
                        st = "AC"
                    elif status.__contains__("wrong"):
                        st = "WA"
                    elif status.__contains__("compilation"):
                        st = "CE"
                    elif status.__contains__("runtime"):
                        st = "RE"
                    elif status.__contains__("time limit"):
                        st = "TLE"
                    else:
                        st = "OTH"

                    append(st)

                    # Question Points
                    if st == "AC":
                        points = "100"
                    else:
                        points = "0"
                    append(points)

                    # Language
                    append(i.contents[12].contents[1].contents[0])

                    it += 1
            page += 1
            if flag == 1:
                break
        return submissions

    # -------------------------------------------------------------------------
    def hackerearth(self, last_retrieved):

        if self.handle:
            handle = self.handle
        else:
            return {}

        url = "https://www.hackerearth.com/submissions/" + handle
        t = requests.get(url, proxies=utilities.PROXY)
        tmp_string = t.headers['set-cookie']
        csrf_token = re.findall("csrftoken=\w*", tmp_string)[0][10:]
        url = "https://www.hackerearth.com/AJAX/feed/newsfeed/submission/user/" + handle + "/"

        response = {}
        response["host"] = "www.hackerearth.com"
        response["user-agent"] = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:40.0) Gecko/20100101 Firefox/40.0"
        response["accept"] = "application/json, text/javascript, */*; q=0.01"
        response["accept-language"] = "en-US,en;q=0.5"
        response["accept-encoding"] = "gzip, deflate"
        response["content-type"] = "application/x-www-form-urlencoded"
        response["X-CSRFToken"] = csrf_token
        response["X-Requested-With"] = "XMLHttpRequest"
        response["Referer"] = "https://www.hackerearth.com/submissions/raj454raj/"
        response["Connection"] = "keep-alive"
        response["Pragma"] = "no-cache"
        response["Cache-Control"] = "no-cache"
        response["Cookie"] = tmp_string

        it = 1
        submissions = {handle: {}}
        for i in xrange(0, 10000, 20):
            submissions[handle][i / 20] = {}
            data = {"index": str(i)}

            tmp = requests.post(url,
                                data=data,
                                proxies=utilities.PROXY,
                                headers=response)

            try:
                final_json = tmp.json()
            except:
                break

            if final_json["count"] == 0:
                break

            for feed_number in xrange(20):

                submissions[handle][i / 20][it] = []
                submission = submissions[handle][i / 20][it]
                append = submission.append

                try:
                    final = bs4.BeautifulSoup((tmp.json()["feed" + str(feed_number)]))
                except KeyError:
                    break

                all_as = final.find_all("a")
                all_tds = final.find_all("td")
                tos = all_tds[-1].contents[1]["title"]
                time_stamp = time.strptime(str(tos), "%Y-%m-%d %H:%M:%S")

                if time_stamp <= last_retrieved:
                    return submissions
                append(str(tos))

                problem_link = "https://hackerearth.com" + all_as[1]["href"]
                append(problem_link)
                problem_name = all_as[1].contents[0]
                append(problem_name)

                try:
                    status = all_tds[2].contents[1]["title"]
                except IndexError:
                    status = "Others"

                if status.__contains__("Accepted"):
                    status = "AC"
                elif status.__contains__("Wrong"):
                    status = "WA"
                elif status.__contains__("Compilation"):
                    status = "CE"
                elif status.__contains__("Runtime"):
                    status = "RE"
                elif status.__contains__("Memory"):
                    status = "MLE"
                elif status.__contains__("Time"):
                    status = "TLE"
                else:
                    status = "OTH"
                append(status)

                if status == "AC":
                    points = "100"
                else:
                    points = "0"
                append(points)

                language = all_tds[5].contents[0]
                append(language)
                it += 1
        return submissions

    # -------------------------------------------------------------------------
    def hackerrank(self, last_retrieved):

        if self.handle:
            handle = self.handle
        else:
            return {}

        tmp = requests.get("https://www.hackerrank.com/rest/hackers/" + \
                           handle + \
                           "/recent_challenges?offset=0&limit=50000")

        all_submissions = tmp.json()["models"]

        submissions = {handle: {1 : {}}}
        it = 0
        for row in all_submissions:

            submission = submissions[handle][1]
            submission[it] = []
            submission = submission[it]
            append = submission.append

            time_stamp = row["created_at"][:-5].split("T")
            curr = time.strptime(time_stamp[0] + " " + time_stamp[1],
                                 "%Y-%m-%d %H:%M:%S")

            if curr <= last_retrieved:
                return submissions
            append(" ".join(time_stamp))

            append("https://hackerrank.com/challenges/" + row["slug"])
            append(row["name"])
            append("AC")
            append("100")
            append("-")
            it += 1

        return submissions
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_bambenek
# Purpose:      Checks if an IP, hostname or domain is malicious.
#
# Author:       steve@binarypool.com
#
# Created:     07/09/2018
# Copyright:   (c) Steve Micallef, 2018
# Licence:     GPL
# -------------------------------------------------------------------------------

import re

from netaddr import IPAddress, IPNetwork

from spiderfoot import SpiderFootEvent, SpiderFootPlugin

malchecks = {
    'Bambenek C&C IP List': {
        'id': 'bambip',
        'checks': ['ip', 'netblock'],
        'url': 'http://osint.bambenekconsulting.com/feeds/c2-ipmasterlist.txt',
        'regex': '^{0},.*'
    },
    'Bambenek C&C Domain List': {
        'id': 'bambdom',
        'checks': ['domain'],
        'url': 'http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt',
        'regex': '^{0},.*'
    }
}

class sfp_bambenek(SpiderFootPlugin):

    meta = {
        'name': "Bambenek C&C List",
        'summary': "Check if a host/domain or IP appears on Bambenek Consulting's C&C tracker lists.",
        'flags': [],
        'useCases': ["Investigate", "Passive"],
        'categories': ["Reputation Systems"],
        'dataSource': {
            'website': "http://www.bambenekconsulting.com/",
            'model': "FREE_NOAUTH_UNLIMITED",
            'references': [
                "http://www.bambenekconsulting.com/free-osint-tools/",
                "https://osint.bambenekconsulting.com/feeds/",
                "https://osint.bambenekconsulting.com/feeds/license.txt"
            ],
            'favIcon': "http://www.bambenekconsulting.com/wp-content/uploads/2013/04/mini-logo1.ico",
            'logo': "http://www.bambenekconsulting.com/wp-content/uploads/2013/04/logo_transparent21-300x84.png",
            'description': "Bambenek Consulting is an cybersecurity investigations and intelligence consulting firm "
            "focusing on tackling major criminal threats. "
            "Every day, there is another story about another company having their banking accounts drained, "
            "someone having their identity stolen, or critical infrastructure being taken offline by hostile entities. "
            "Led by IT security expert, John Bambenek, we have the resources to bring to your business so "
            "you can be sure your organization and your customers’ data is safe.",
        }
    }

    # Default options
    opts = {
        'bambip': True,
        'bambdom': True,
        'checkaffiliates': True,
        'checkcohosts': True,
        'cacheperiod': 18,
        'checknetblocks': True,
        'checksubnets': True
    }

    # Option descriptions
    optdescs = {
        'bambip': "Enable Bambenek IP check?",
        'bambdom': "Enable Bambenek Domains check?",
        'checkaffiliates': "Apply checks to affiliates?",
        'checkcohosts': "Apply checks to sites found to be co-hosted on the target's IP?",
        'cacheperiod': "Hours to cache list data before re-fetching.",
        'checknetblocks': "Report if any malicious IPs are found within owned netblocks?",
        'checksubnets': "Check if any malicious IPs are found within the same subnet of the target?"
    }

    # Be sure to completely clear any class variables in setup()
    # or you run the risk of data persisting between scan runs.

    results = None

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = self.tempStorage()

        # Clear / reset any other class member variables here
        # or you risk them persisting between threads.

        for opt in list(userOpts.keys()):
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    # * = be notified about all events.
    def watchedEvents(self):
        return ["INTERNET_NAME", "IP_ADDRESS",
                "NETBLOCK_MEMBER", "AFFILIATE_INTERNET_NAME", "AFFILIATE_IPADDR",
                "CO_HOSTED_SITE", "NETBLOCK_OWNER"]

    # What events this module produces
    # This is to support the end user in selecting modules based on events
    # produced.
    def producedEvents(self):
        return ["MALICIOUS_IPADDR", "MALICIOUS_INTERNET_NAME",
                "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_AFFILIATE_INTERNET_NAME",
                "MALICIOUS_SUBNET", "MALICIOUS_COHOST", "MALICIOUS_NETBLOCK"]

    # Look up 'list' type resources
    def resourceList(self, replaceme_id, target, targetType):
        targetDom = ''
        # Get the base domain if we're supplied a domain
        if targetType == "domain":
            targetDom = self.sf.hostDomain(target, self.opts['_internettlds'])
            if not targetDom:
                return None

        for check in list(malchecks.keys()):
            cid = malchecks[check]['id']
            if replaceme_id == cid:
                data = dict()
                url = malchecks[check]['url']
                data['content'] = self.sf.cacheGet("sfmal_" + cid, self.opts.get('cacheperiod', 0))
                if data['content'] is None:
                    data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])
                    if data['content'] is None:
                        self.error("Unable to fetch " + url)
                        return None
                    else:
                        self.sf.cachePut("sfmal_" + cid, data['content'])

                # If we're looking at netblocks
                if targetType == "netblock":
                    iplist = list()
                    # Get the regex, replace {0} with an IP address matcher to
                    # build a list of IP.
                    # Cycle through each IP and check if it's in the netblock.
                    if 'regex' in malchecks[check]:
                        rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
                        pat = re.compile(rx, re.IGNORECASE)
                        self.debug("New regex for " + check + ": " + rx)
                        for line in data['content'].split('\n'):
                            grp = re.findall(pat, line)
                            if len(grp) > 0:
                                # self.debug("Adding " + grp[0] + " to list.")
                                iplist.append(grp[0])
                    else:
                        iplist = data['content'].split('\n')

                    for ip in iplist:
                        if len(ip) < 8 or ip.startswith("#"):
                            continue
                        ip = ip.strip()

                        try:
                            if IPAddress(ip) in IPNetwork(target):
                                self.debug(f"{ip} found within netblock/subnet {target} in {check}")
                                return url
                        except Exception as e:
                            self.debug(f"Error encountered parsing: {e}")
                            continue

                    return None

                # If we're looking at hostnames/domains/IPs
                if 'regex' not in malchecks[check]:
                    for line in data['content'].split('\n'):
                        if line == target or (targetType == "domain" and line == targetDom):
                            self.debug(target + "/" + targetDom + " found in " + check + " list.")
                            return url
                else:
                    try:
                        # Check for the domain and the hostname
                        rxDom = str(malchecks[check]['regex']).format(targetDom)
                        rxTgt = str(malchecks[check]['regex']).format(target)
                        for line in data['content'].split('\n'):
                            if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
                                    re.match(rxTgt, line, re.IGNORECASE):
                                self.debug(target + "/" + targetDom + " found in " + check + " list.")
                                return url
                    except Exception as e:
                        self.debug("Error encountered parsing 2: " + str(e))
                        continue

        return None

    def lookupItem(self, resourceId, itemType, target):
        for check in list(malchecks.keys()):
            cid = malchecks[check]['id']
            if cid == resourceId and itemType in malchecks[check]['checks']:
                self.debug("Checking maliciousness of " + target + " (" + itemType + ") with: " + cid)
                return self.resourceList(cid, target, itemType)

        return None

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        self.debug(f"Received event, {eventName}, from {srcModuleName}")

        if eventData in self.results:
            self.debug(f"Skipping {eventData}, already checked.")
            return

        self.results[eventData] = True

        if eventName == 'CO_HOSTED_SITE' and not self.opts.get('checkcohosts', False):
            return
        if eventName == 'AFFILIATE_IPADDR' \
                and not self.opts.get('checkaffiliates', False):
            return
        if eventName == 'NETBLOCK_OWNER' and not self.opts.get('checknetblocks', False):
            return
        if eventName == 'NETBLOCK_MEMBER' and not self.opts.get('checksubnets', False):
            return

        for check in list(malchecks.keys()):
            cid = malchecks[check]['id']

            if eventName in ['IP_ADDRESS', 'AFFILIATE_IPADDR']:
                typeId = 'ip'
                if eventName == 'IP_ADDRESS':
                    evtType = 'MALICIOUS_IPADDR'
                else:
                    evtType = 'MALICIOUS_AFFILIATE_IPADDR'

            if eventName in ['INTERNET_NAME', 'CO_HOSTED_SITE',
                             'AFFILIATE_INTERNET_NAME']:
                typeId = 'domain'
                if eventName == "INTERNET_NAME":
                    evtType = "MALICIOUS_INTERNET_NAME"
                if eventName == 'AFFILIATE_INTERNET_NAME':
                    evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME'
                if eventName == 'CO_HOSTED_SITE':
                    evtType = 'MALICIOUS_COHOST'

            if eventName == 'NETBLOCK_OWNER':
                typeId = 'netblock'
                evtType = 'MALICIOUS_NETBLOCK'
            if eventName == 'NETBLOCK_MEMBER':
                typeId = 'netblock'
                evtType = 'MALICIOUS_SUBNET'

            url = self.lookupItem(cid, typeId, eventData)

            if self.checkForStop():
                return

            # Notify other modules of what you've found
            if url is not None:
                text = f"{check} [{eventData}]\n<SFURL>{url}</SFURL>"
                evt = SpiderFootEvent(evtType, text, self.__name__, event)
                self.notifyListeners(evt)

# End of sfp_bambenek class
<commit_msg>sfp_bambenek: Remove sfp_bambenek module (#1485)<commit_after><|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import logging
from collections import namedtuple

import requests
logger = logging.getLogger(__name__)

class ApiError(Exception):
    pass

class ConnectionError(ApiError):
    pass

CompanyInfo = namedtuple('CompanyInfo', [
    'id',
    'nazwa',
    'nazwa_skrocona',
    'nip',
    'adres',
    'liczba_wspolnikow',
    'score',
    'url']
)

class ApiClient:
    API_URL = 'https://api-v3.mojepanstwo.pl/'

    def __init__(self, url=API_URL):
        self.url = url
        self.session = requests.Session()

    def send_request(self, method, param, value):
        params = {
            param: value
            }
        resp = self.session.get(url=self.API_URL + method, params=params)

        print self.API_URL + method, params, resp.json()
        if resp.status_code != 200:
            raise ConnectionError({'status_code': resp.status_code})

        json = resp.json()

        if json['Dataobject'] is None:
            raise ApiError()

        return json

class Krs:
    def __init__(self, client=None):
        self.client = client or ApiClient()

    def json_to_company(self, json):
        company = dict()
        company['nazwa'] = Krs.unescape(json['krs_podmioty.nazwa'])
        company['nazwa_skrocona'] = \
            Krs.unescape(json['krs_podmioty.nazwa_skrocona'])
        company['nip'] = json['krs_podmioty.nip']
        company['adres'] = self.simplifyAddress(json)
        company['id'] = json['krs_podmioty.id']
        company['liczba_wspolnikow'] = \
            json['krs_podmioty.liczba_wspolnikow']
        company['score'] = ""
        company['url'] = ""

        return CompanyInfo(**company)

    def simplifyAddress(self, data):
        lokal = u" lok. {}".format(data['krs_podmioty.adres_lokal']) if \
            data['krs_podmioty.adres_lokal'] else u""
        adres = Krs.unescape(u"ul. {} {} {}\n{} {}\n{}".format(
            data['krs_podmioty.adres_ulica'],
            data['krs_podmioty.adres_numer'],
            lokal,
            data['krs_podmioty.adres_kod_pocztowy'],
            data['krs_podmioty.adres_miejscowosc'],
            data['krs_podmioty.adres_kraj'])
        )
        return adres

    def get_companies_by_name(self, name):
        return self.get_companies_by_krs_name(name) or \
            self.get_companies_by_search(name)

    def get_companies_by_search(self, name):
        normalized_name = Krs._normalize_name(name)
        response = self.client.send_request('dane/krs_podmioty', 'conditions[q]', normalized_name)
        return self.parse_companies_response(response)

    def get_companies_by_krs_name(self, name):
        normalized_name = Krs._normalize_name(name)
        response = self.client.send_request('dane/krs_podmioty', 'conditions[krs_podmioty.nazwa]', normalized_name)
        return self.parse_companies_response(response)

    def get_companies_by_krs_no(self, name):
        normalized_name = Krs._normalize_name(name)
        response = self.client.send_request('dane/krs_podmioty', 'conditions[krs_podmioty.krs]', normalized_name)
        return self.parse_companies_response(response)

    def get_companies_by_nip(self, nip):
        json = self.client.send_request('dane/krs_podmioty', 'conditions[krs_podmioty.nip]', nip)
        return self.parse_companies_response(json)

    def parse_companies_response(self, json):
        return map(lambda o: self.json_to_company(o['data']), json['Dataobject'])

    #remove unnecessary mojepanstwo escape
    @staticmethod
    def unescape(s):
        return s.replace('&', '&')

    COMMON_COMPANY_NAME_ENDINGS = \
        { u' S.A.':u' SPÓŁKA AKCYJNA',
          u' Sp. z o.o.' : u' SPÓŁKA Z OGRANICZONĄ ODPOWIEDZIALNOŚCIĄ',
          u' Spółka z o.o.' : u' SPÓŁKA Z OGRANICZONĄ ODPOWIEDZIALNOŚCIĄ',
          u'Sp. Jawna' : u'SPÓŁKA JAWNA',
          u'spółka z ograniczoną odpowiedzialnością sp.k.' :
            u'SPÓŁKA Z OGRANICZONĄ ODPOWIEDZIALNOŚCIĄ SPÓŁKA KOMANDYTOWA',
          u'sp. z o. o. sp.k.' :
            u'SPÓŁKA Z OGRANICZONĄ ODPOWIEDZIALNOŚCIĄ SPÓŁKA KOMANDYTOWA',
          u'Sp. z o.o. sp.k.' :
            u'SPÓŁKA Z OGRANICZONĄ ODPOWIEDZIALNOŚCIĄ SPÓŁKA KOMANDYTOWA',
          u'sp. z o.o. sp. k.' :
            u'SPÓŁKA Z OGRANICZONĄ ODPOWIEDZIALNOŚCIĄ SPÓŁKA KOMANDYTOWA'
          }

    @staticmethod
    def _normalize_name(name):
        name = name.upper()
        for key, value in Krs.COMMON_COMPANY_NAME_ENDINGS.items():
            if name.endswith(key.upper()):
                return name[:len(name)-len(key)] + value
        return name

    def query_shareholders(self, id):
        return self.client.send_request('dane/krs_podmioty/' + id, 'layers', 'wspolnicy')
<commit_msg>Reorder method, marks as private<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import logging
from collections import namedtuple

import requests
logger = logging.getLogger(__name__)

class ApiError(Exception):
    pass

class ConnectionError(ApiError):
    pass

CompanyInfo = namedtuple('CompanyInfo', [
    'id',
    'nazwa',
    'nazwa_skrocona',
    'nip',
    'adres',
    'liczba_wspolnikow',
    'score',
    'url']
)

class ApiClient:
    API_URL = 'https://api-v3.mojepanstwo.pl/'

    def __init__(self, url=API_URL):
        self.url = url
        self.session = requests.Session()

    def send_request(self, method, param, value):
        params = {
            param: value
            }
        resp = self.session.get(url=self.API_URL + method, params=params)

        print self.API_URL + method, params, resp.json()
        if resp.status_code != 200:
            raise ConnectionError({'status_code': resp.status_code})

        json = resp.json()

        if json['Dataobject'] is None:
            raise ApiError()

        return json

class Krs:
    def __init__(self, client=None):
        self.client = client or ApiClient()

    def get_companies_by_name(self, name):
        return self.get_companies_by_krs_name(name) or \
            self.get_companies_by_search(name)

    def get_companies_by_search(self, name):
        normalized_name = Krs._normalize_name(name)
        response = self.client.send_request('dane/krs_podmioty', 'conditions[q]', normalized_name)
        return self._parse_companies_response(response)

    def get_companies_by_krs_name(self, name):
        normalized_name = Krs._normalize_name(name)
        response = self.client.send_request('dane/krs_podmioty', 'conditions[krs_podmioty.nazwa]', normalized_name)
        return self._parse_companies_response(response)

    def get_companies_by_krs_no(self, name):
        normalized_name = Krs._normalize_name(name)
        response = self.client.send_request('dane/krs_podmioty', 'conditions[krs_podmioty.krs]', normalized_name)
        return self._parse_companies_response(response)

    def get_companies_by_nip(self, nip):
        json = self.client.send_request('dane/krs_podmioty', 'conditions[krs_podmioty.nip]', nip)
        return self._parse_companies_response(json)

    def query_shareholders(self, id):
        return self.client.send_request('dane/krs_podmioty/' + id, 'layers', 'wspolnicy')

    def _parse_companies_response(self, json):
        return map(lambda o: self._json_to_company(o['data']), json['Dataobject'])

    def _json_to_company(self, json):
        company = dict()
        company['nazwa'] = Krs._unescape(json['krs_podmioty.nazwa'])
        company['nazwa_skrocona'] = \
            Krs._unescape(json['krs_podmioty.nazwa_skrocona'])
        company['nip'] = json['krs_podmioty.nip']
        company['adres'] = self._simplify_address(json)
        company['id'] = json['krs_podmioty.id']
        company['liczba_wspolnikow'] = \
            json['krs_podmioty.liczba_wspolnikow']
        company['score'] = ""
        company['url'] = ""

        return CompanyInfo(**company)

    def _simplify_address(self, data):
        lokal = u" lok. {}".format(data['krs_podmioty.adres_lokal']) if \
            data['krs_podmioty.adres_lokal'] else u""
        adres = Krs._unescape(u"ul. {} {} {}\n{} {}\n{}".format(
            data['krs_podmioty.adres_ulica'],
            data['krs_podmioty.adres_numer'],
            lokal,
            data['krs_podmioty.adres_kod_pocztowy'],
            data['krs_podmioty.adres_miejscowosc'],
            data['krs_podmioty.adres_kraj'])
        )
        return adres

    #remove unnecessary mojepanstwo escape
    @staticmethod
    def _unescape(s):
        return s.replace('&', '&')

    COMMON_COMPANY_NAME_ENDINGS = \
        { u' S.A.':u' SPÓŁKA AKCYJNA',
          u' Sp. z o.o.' : u' SPÓŁKA Z OGRANICZONĄ ODPOWIEDZIALNOŚCIĄ',
          u' Spółka z o.o.' : u' SPÓŁKA Z OGRANICZONĄ ODPOWIEDZIALNOŚCIĄ',
          u'Sp. Jawna' : u'SPÓŁKA JAWNA',
          u'spółka z ograniczoną odpowiedzialnością sp.k.' :
            u'SPÓŁKA Z OGRANICZONĄ ODPOWIEDZIALNOŚCIĄ SPÓŁKA KOMANDYTOWA',
          u'sp. z o. o. sp.k.' :
            u'SPÓŁKA Z OGRANICZONĄ ODPOWIEDZIALNOŚCIĄ SPÓŁKA KOMANDYTOWA',
          u'Sp. z o.o. sp.k.' :
            u'SPÓŁKA Z OGRANICZONĄ ODPOWIEDZIALNOŚCIĄ SPÓŁKA KOMANDYTOWA',
          u'sp. z o.o. sp. k.' :
            u'SPÓŁKA Z OGRANICZONĄ ODPOWIEDZIALNOŚCIĄ SPÓŁKA KOMANDYTOWA'
          }

    @staticmethod
    def _normalize_name(name):
        name = name.upper()
        for key, value in Krs.COMMON_COMPANY_NAME_ENDINGS.items():
            if name.endswith(key.upper()):
                return name[:len(name)-len(key)] + value
        return name

<|endoftext|>


<commit_before>import itertools
import datetime
from logging import getLogger

from lxml import etree

from django.contrib.gis.geos import Point
from django.core.paginator import Page
from django.db import models

logger = getLogger(__name__)

class DateUnicode(unicode): pass
class DateTimeUnicode(unicode): pass

_XML_DATATYPES = (
    (DateUnicode, 'date'),
    (DateTimeUnicode, 'datetime'),
    (str, 'string'),
    (unicode, 'string'),
    (int, 'integer'),
    (float, 'float'),
)

def simplify_value(value):
    if hasattr(value, 'simplify_for_render'):
        return value.simplify_for_render(simplify_value, simplify_model)
    elif isinstance(value, dict):
        out = {}
        for key in value:
            new_key = key if isinstance(key, (basestring, int)) else str(key)
            try:
                out[new_key] = simplify_value(value[key])
            except NotImplementedError:
                logger.warn('Could not simplify a value', exc_info=True)
                pass
        return out
    elif isinstance(value, (list, tuple, set, frozenset)):
        out = []
        for subvalue in value:
            try:
                out.append(simplify_value(subvalue))
            except NotImplementedError:
                pass
        if isinstance(value, tuple):
            return tuple(out)
        else:
            return out
    elif isinstance(value, (basestring, int, float)):
        return value
    elif isinstance(value, datetime.datetime):
        return DateTimeUnicode(value.isoformat(' '))
    elif isinstance(value, datetime.date):
        return DateUnicode(value.isoformat())
    elif hasattr(type(value), '__mro__') and models.Model in type(value).__mro__:
        return simplify_model(value)
    elif isinstance(value, Page):
        return {
            'has_next': value.has_next(),
            'has_previous': value.has_next(),
            'next_page_number': value.has_next(),
            'previous_page_number': value.has_next(),
            'number': value.number,
            'objects': simplify_value(value.object_list)
        }
    elif value is None:
        return None
    elif isinstance(value, Point):
        return simplify_value(list(value))
    elif hasattr(value, '__iter__'):
        # Iterators may be unbounded; silently ignore elements once we've already had 1000.
        return [simplify_value(item) for item in itertools.islice(value, 1000)]
    else:
        raise NotImplementedError

def simplify_model(obj, terse=False):
    if obj is None:
        return None
    # It's a Model instance
    if hasattr(obj._meta, 'expose_fields'):
        expose_fields = obj._meta.expose_fields
    else:
        expose_fields = [f.name for f in obj._meta.fields]
    out = {
        '_type': '%s.%s' % (obj.__module__[:-7], obj._meta.object_name),
        '_pk': obj.pk,
    }
    if hasattr(obj, 'get_absolute_url'):
        out['_url'] = obj.get_absolute_url()
    if terse:
        out['_terse'] = True
    else:
        for field_name in expose_fields:
            if field_name in ('password',):
                continue
            try:
                value = getattr(obj, field_name)
                if isinstance(value, models.Model):
                    value = simplify_model(value, terse=True)
                out[field_name] = simplify_value(value)
            except NotImplementedError:
                pass

        # Add any non-field attributes
        for field in list(dir(obj)):
            if field[0] != '_' and field != 'objects' \
             and not isinstance(getattr(obj, field), models.Field):
                try:
                    out[field] = simplify_value(getattr(obj, field))
                except NotImplementedError:
                    pass
    return out

def serialize_to_xml(value):
    if value is None:
        node = etree.Element('null')
    elif isinstance(value, bool):
        node = etree.Element('literal')
        node.text = 'true' if value else 'false'
        node.attrib['type'] = 'boolean'
    elif isinstance(value, (basestring, int, float)):
        node = etree.Element('literal')
        node.text = unicode(value)
        node.attrib['type'] = [d[1] for d in _XML_DATATYPES if isinstance(value, d[0])][0]
    elif isinstance(value, dict):
        if '_type' in value:
            node = etree.Element('object', {'type': value['_type'], 'pk': unicode(value.get('_pk', ''))})
            del value['_type']
            del value['_pk']
            if '_url' in value:
                node.attrib['url'] = value['_url']
                del value['_url']
            if value.get('_terse'):
                node.attrib['terse'] = 'true'
                del value['_terse']
        else:
            node = etree.Element('collection', {'type': 'mapping'})
        for key in value:
            v = serialize_to_xml(value[key])
            subnode = etree.Element('item', {'key':key})
            subnode.append(v)
            node.append(subnode)
    elif isinstance(value, (list, tuple, set, frozenset)):
        for x,y in ((list, 'list'), (tuple, 'tuple')):
            if isinstance(value, x):
                node = etree.Element('collection', {'type': y})
                break
        else:
            node = etree.Element('collection', {'type':'set'})
        for item in value:
            v = serialize_to_xml(item)
            subnode = etree.Element('item')
            subnode.append(v)
            node.append(subnode)
    else:
        node = etree.Element('unknown')

    return node
<commit_msg>Downgrade simplification failures to info<commit_after>import itertools
import datetime
from logging import getLogger

from lxml import etree

from django.contrib.gis.geos import Point
from django.core.paginator import Page
from django.db import models

logger = getLogger(__name__)

class DateUnicode(unicode): pass
class DateTimeUnicode(unicode): pass

_XML_DATATYPES = (
    (DateUnicode, 'date'),
    (DateTimeUnicode, 'datetime'),
    (str, 'string'),
    (unicode, 'string'),
    (int, 'integer'),
    (float, 'float'),
)

def simplify_value(value):
    if hasattr(value, 'simplify_for_render'):
        return value.simplify_for_render(simplify_value, simplify_model)
    elif isinstance(value, dict):
        out = {}
        for key in value:
            new_key = key if isinstance(key, (basestring, int)) else str(key)
            try:
                out[new_key] = simplify_value(value[key])
            except NotImplementedError:
                logger.info('Could not simplify a value', exc_info=True)
                pass
        return out
    elif isinstance(value, (list, tuple, set, frozenset)):
        out = []
        for subvalue in value:
            try:
                out.append(simplify_value(subvalue))
            except NotImplementedError:
                pass
        if isinstance(value, tuple):
            return tuple(out)
        else:
            return out
    elif isinstance(value, (basestring, int, float)):
        return value
    elif isinstance(value, datetime.datetime):
        return DateTimeUnicode(value.isoformat(' '))
    elif isinstance(value, datetime.date):
        return DateUnicode(value.isoformat())
    elif hasattr(type(value), '__mro__') and models.Model in type(value).__mro__:
        return simplify_model(value)
    elif isinstance(value, Page):
        return {
            'has_next': value.has_next(),
            'has_previous': value.has_next(),
            'next_page_number': value.has_next(),
            'previous_page_number': value.has_next(),
            'number': value.number,
            'objects': simplify_value(value.object_list)
        }
    elif value is None:
        return None
    elif isinstance(value, Point):
        return simplify_value(list(value))
    elif hasattr(value, '__iter__'):
        # Iterators may be unbounded; silently ignore elements once we've already had 1000.
        return [simplify_value(item) for item in itertools.islice(value, 1000)]
    else:
        raise NotImplementedError

def simplify_model(obj, terse=False):
    if obj is None:
        return None
    # It's a Model instance
    if hasattr(obj._meta, 'expose_fields'):
        expose_fields = obj._meta.expose_fields
    else:
        expose_fields = [f.name for f in obj._meta.fields]
    out = {
        '_type': '%s.%s' % (obj.__module__[:-7], obj._meta.object_name),
        '_pk': obj.pk,
    }
    if hasattr(obj, 'get_absolute_url'):
        out['_url'] = obj.get_absolute_url()
    if terse:
        out['_terse'] = True
    else:
        for field_name in expose_fields:
            if field_name in ('password',):
                continue
            try:
                value = getattr(obj, field_name)
                if isinstance(value, models.Model):
                    value = simplify_model(value, terse=True)
                out[field_name] = simplify_value(value)
            except NotImplementedError:
                pass

        # Add any non-field attributes
        for field in list(dir(obj)):
            if field[0] != '_' and field != 'objects' \
             and not isinstance(getattr(obj, field), models.Field):
                try:
                    out[field] = simplify_value(getattr(obj, field))
                except NotImplementedError:
                    pass
    return out

def serialize_to_xml(value):
    if value is None:
        node = etree.Element('null')
    elif isinstance(value, bool):
        node = etree.Element('literal')
        node.text = 'true' if value else 'false'
        node.attrib['type'] = 'boolean'
    elif isinstance(value, (basestring, int, float)):
        node = etree.Element('literal')
        node.text = unicode(value)
        node.attrib['type'] = [d[1] for d in _XML_DATATYPES if isinstance(value, d[0])][0]
    elif isinstance(value, dict):
        if '_type' in value:
            node = etree.Element('object', {'type': value['_type'], 'pk': unicode(value.get('_pk', ''))})
            del value['_type']
            del value['_pk']
            if '_url' in value:
                node.attrib['url'] = value['_url']
                del value['_url']
            if value.get('_terse'):
                node.attrib['terse'] = 'true'
                del value['_terse']
        else:
            node = etree.Element('collection', {'type': 'mapping'})
        for key in value:
            v = serialize_to_xml(value[key])
            subnode = etree.Element('item', {'key':key})
            subnode.append(v)
            node.append(subnode)
    elif isinstance(value, (list, tuple, set, frozenset)):
        for x,y in ((list, 'list'), (tuple, 'tuple')):
            if isinstance(value, x):
                node = etree.Element('collection', {'type': y})
                break
        else:
            node = etree.Element('collection', {'type':'set'})
        for item in value:
            v = serialize_to_xml(item)
            subnode = etree.Element('item')
            subnode.append(v)
            node.append(subnode)
    else:
        node = etree.Element('unknown')

    return node
<|endoftext|>


<commit_before>import document
from document import *
import fields
from fields import *
import connection
from connection import *
import queryset
from queryset import *
import signals
from signals import *

__all__ = (document.__all__ + fields.__all__ + connection.__all__ +
           queryset.__all__ + signals.__all__)

VERSION = (0, 6, 1)

def get_version():
    version = '%s.%s' % (VERSION[0], VERSION[1])
    if VERSION[2]:
        version = '%s.%s' % (version, VERSION[2])
    return version

__version__ = get_version()
<commit_msg>Version Bump<commit_after>import document
from document import *
import fields
from fields import *
import connection
from connection import *
import queryset
from queryset import *
import signals
from signals import *

__all__ = (document.__all__ + fields.__all__ + connection.__all__ +
           queryset.__all__ + signals.__all__)

VERSION = (0, 6, 2)

def get_version():
    version = '%s.%s' % (VERSION[0], VERSION[1])
    if VERSION[2]:
        version = '%s.%s' % (version, VERSION[2])
    return version

__version__ = get_version()
<|endoftext|>


<commit_before>from PIL import Image
from base64 import b64encode
import datetime
import logging
import mimetypes
from pkg_resources import resource_stream
import xlrd
# http://www.lexicon.net/sjmachin/xlrd.html

logger = logging.getLogger('encoded')

def convert(type_, value):
    if type_ is None:
        return value
    if type_ is datetime.date:
        return datetime.date(*value[:3]).isoformat()
    elif type_ is datetime.datetime:
        return datetime.datetime(*value).isoformat()
    else:
        return type_(value)

def cell_value(sheet, row, col, hint=None):
    if hint is None:
        hint = {}
    cell = sheet.cell(row, col)
    ctype = cell.ctype
    value = cell.value
    type_ = hint.get('type')

    if ctype == xlrd.XL_CELL_ERROR:
        raise ValueError((row, col, repr(cell)))

    elif ctype == xlrd.XL_CELL_BOOLEAN:
        if type_ is None:
            type_ = bool

    elif ctype == xlrd.XL_CELL_NUMBER:
        if type_ is None:
            type_ = int if value.is_integer() else float

    elif ctype == xlrd.XL_CELL_DATE:
        value = xlrd.xldate_as_tuple(value, sheet.book.datemode)
        if type_ is None:
            if value[3:] == (0, 0, 0):
                type_ = datetime.date
            else:
                type_ = datetime.datetime

    elif ctype == xlrd.XL_CELL_TEXT:
        value = value.strip()
        if value == 'null':
            value = None
        elif value == '':
            value = None

    # Empty cell
    elif ctype in (xlrd.XL_CELL_EMPTY, xlrd.XL_CELL_BLANK):
        value = None

    else:
        raise ValueError((row, col, repr(cell), 'Unknown cell type.'))

    return convert(type_, value)

def iter_rows(sheet, headers=None, hints=None, start=1):
    if hints is None:
        hints = {}
    if headers is None:
        headers = [sheet.cell_value(0, col).lower().strip() for col in range(sheet.ncols)]
    for row in xrange(start, sheet.nrows):
        yield dict((name, cell_value(sheet, row, col, hints.get(name)))
            for col, name in enumerate(headers) if name)

def extract(filename, sheets):
    alldata = {}
    book = xlrd.open_workbook(filename)
    for name in sheets:
        data = alldata[name] = {}
        sheet = book.sheet_by_name(name)
        for row in iter_rows(sheet):
            uuid = row['%s_uuid' % name]
            if not uuid:
                continue
            row['_uuid'] = uuid
            data[uuid] = row
    return alldata

TYPE_URL = [
    ('organism', '/organisms/'),
    ('source', '/sources/'),
    ('target', '/targets/'),
    ('antibody_lot', '/antibody-lots/'),
    ('validation', '/validations/'),
    ('antibody_approval', '/antibodies/'),
    ]

def value_index(data, attribute):
    index = {}
    for uuid, value in data.iteritems():
        index_value = value[attribute]
        assert index_value not in index, index_value
        index[index_value] = uuid
    return index

def multi_index(data, attribute):
    index = {}
    for uuid, value in data.iteritems():
        index_value = value[attribute]
        index.setdefault(index_value, []).append(uuid)
    return index

def tuple_index(data, *attrs):
    index = {}
    for uuid, value in data.iteritems():
        index_value = tuple(value[attr] for attr in attrs)
        assert index_value not in index, index_value
        index[index_value] = uuid
    return index

def multi_tuple_index(data, *attrs):
    index = {}
    for uuid, value in data.iteritems():
        index_value = tuple(value[attr] for attr in attrs)
        index.setdefault(index_value, []).append(uuid)
    return index

def image_data(stream, filename=None):
    data = {}
    if filename is not None:
        data['download'] = filename
    im = Image.open(stream)
    im.verify()
    data['width'], data['height'] = im.size
    mime_type, _ = mimetypes.guess_type('name.%s' % im.format)
    data['type'] = mime_type
    stream.seek(0, 0)
    encoded_data = b64encode(stream.read())
    data['href'] = 'data:%s;base64,%s' % (mime_type, encoded_data)
    return data

def load_all(testapp, filename):
    sheets = [content_type for content_type, url in TYPE_URL]
    alldata = extract(filename, sheets)

    organism_index = value_index(alldata['organism'], 'organism_name')
    source_index = value_index(alldata['source'], 'source_name')
    target_index = tuple_index(alldata['target'], 'target_label', 'organism_name')
    # validation_index = multi_tuple_index(alldata['validation'], 'antibody_lot_uuid', 'target_label', 'organism_name')
    validation_index = multi_index(alldata['validation'], 'document_filename')

    for uuid, value in alldata['target'].iteritems():
        value['organism_uuid'] = organism_index[value.pop('organism_name')]
        aliases = value.pop('target_aliases') or ''
        alias_source = value['target_alias_source']
        value['dbxref'] = [
            {'db': alias_source, 'id': alias.strip()}
            for alias in aliases.split(';')]

    for uuid, value in list(alldata['antibody_lot'].iteritems()):
        source = value.pop('source')
        try:
            value['source_uuid'] = source_index[source]
        except KeyError:
            logger.warn('Unable to find source: %s (%s)' % (source, uuid))
            del alldata['antibody_lot'][uuid]

    antibody_lot_index = tuple_index(alldata['antibody_lot'], 'product_id', 'lot_id')

    for uuid, value in alldata['validation'].iteritems():
        value['target_uuid'] = target_index[(value.pop('target_label'), value.pop('organism_name'))]
        filename = value.pop('document_filename')
        value['document'] = image_data(
            resource_stream('encoded', 'tests/data/validation-docs/' + filename),
            filename,
            )

    for uuid, value in list(alldata['antibody_approval'].iteritems()):
        try:
            value['antibody_lot_uuid'] = antibody_lot_index[(value.pop('antibody_product_id'), value.pop('antibody_lot_id'))]
            #value['validation_uuids'] = validation_index.get((value['antibody_lot_uuid'], value['target_label'], value['organism_name']), [])
            value['validation_uuids'] = []
            filenames = (value.pop('validation_filenames') or '').split(';')
            for filename in filenames:
                validation_uuids = validation_index.get(filename, [])
                value['validation_uuids'].extend(validation_uuids)
            value['target_uuid'] = target_index[(value.pop('target_label'), value.pop('organism_name'))]
        except KeyError:
            logger.debug('Missing/skipped reference for antibody_approval: %s' % uuid)
            del alldata['antibody_approval'][uuid]

    for content_type, url in TYPE_URL:
        collection = alldata[content_type]
        for item in collection.itervalues():
            testapp.post_json(url, item, status=201)
<commit_msg>Better import logging.<commit_after>from PIL import Image
from base64 import b64encode
import datetime
import logging
import mimetypes
from pkg_resources import resource_stream
import xlrd
# http://www.lexicon.net/sjmachin/xlrd.html

logger = logging.getLogger('encoded')

def convert(type_, value):
    if type_ is None:
        return value
    if type_ is datetime.date:
        return datetime.date(*value[:3]).isoformat()
    elif type_ is datetime.datetime:
        return datetime.datetime(*value).isoformat()
    else:
        return type_(value)

def cell_value(sheet, row, col, hint=None):
    if hint is None:
        hint = {}
    cell = sheet.cell(row, col)
    ctype = cell.ctype
    value = cell.value
    type_ = hint.get('type')

    if ctype == xlrd.XL_CELL_ERROR:
        raise ValueError((row, col, repr(cell)))

    elif ctype == xlrd.XL_CELL_BOOLEAN:
        if type_ is None:
            type_ = bool

    elif ctype == xlrd.XL_CELL_NUMBER:
        if type_ is None:
            type_ = int if value.is_integer() else float

    elif ctype == xlrd.XL_CELL_DATE:
        value = xlrd.xldate_as_tuple(value, sheet.book.datemode)
        if type_ is None:
            if value[3:] == (0, 0, 0):
                type_ = datetime.date
            else:
                type_ = datetime.datetime

    elif ctype == xlrd.XL_CELL_TEXT:
        value = value.strip()
        if value == 'null':
            value = None
        elif value == '':
            value = None

    # Empty cell
    elif ctype in (xlrd.XL_CELL_EMPTY, xlrd.XL_CELL_BLANK):
        value = None

    else:
        raise ValueError((row, col, repr(cell), 'Unknown cell type.'))

    return convert(type_, value)

def iter_rows(sheet, headers=None, hints=None, start=1):
    if hints is None:
        hints = {}
    if headers is None:
        headers = [sheet.cell_value(0, col).lower().strip() for col in range(sheet.ncols)]
    for row in xrange(start, sheet.nrows):
        yield dict((name, cell_value(sheet, row, col, hints.get(name)))
            for col, name in enumerate(headers) if name)

def extract(filename, sheets):
    alldata = {}
    book = xlrd.open_workbook(filename)
    for name in sheets:
        data = alldata[name] = {}
        sheet = book.sheet_by_name(name)
        for row in iter_rows(sheet):
            uuid = row['%s_uuid' % name]
            if not uuid:
                continue
            row['_uuid'] = uuid
            data[uuid] = row
    return alldata

TYPE_URL = [
    ('organism', '/organisms/'),
    ('source', '/sources/'),
    ('target', '/targets/'),
    ('antibody_lot', '/antibody-lots/'),
    ('validation', '/validations/'),
    ('antibody_approval', '/antibodies/'),
    ]

def value_index(data, attribute):
    index = {}
    for uuid, value in data.iteritems():
        index_value = value[attribute]
        assert index_value not in index, index_value
        index[index_value] = uuid
    return index

def multi_index(data, attribute):
    index = {}
    for uuid, value in data.iteritems():
        index_value = value[attribute]
        index.setdefault(index_value, []).append(uuid)
    return index

def tuple_index(data, *attrs):
    index = {}
    for uuid, value in data.iteritems():
        index_value = tuple(value[attr] for attr in attrs)
        assert index_value not in index, index_value
        index[index_value] = uuid
    return index

def multi_tuple_index(data, *attrs):
    index = {}
    for uuid, value in data.iteritems():
        index_value = tuple(value[attr] for attr in attrs)
        index.setdefault(index_value, []).append(uuid)
    return index

def image_data(stream, filename=None):
    data = {}
    if filename is not None:
        data['download'] = filename
    im = Image.open(stream)
    im.verify()
    data['width'], data['height'] = im.size
    mime_type, _ = mimetypes.guess_type('name.%s' % im.format)
    data['type'] = mime_type
    stream.seek(0, 0)
    encoded_data = b64encode(stream.read())
    data['href'] = 'data:%s;base64,%s' % (mime_type, encoded_data)
    return data

def load_all(testapp, filename):
    sheets = [content_type for content_type, url in TYPE_URL]
    alldata = extract(filename, sheets)

    organism_index = value_index(alldata['organism'], 'organism_name')
    source_index = value_index(alldata['source'], 'source_name')
    target_index = tuple_index(alldata['target'], 'target_label', 'organism_name')
    # validation_index = multi_tuple_index(alldata['validation'], 'antibody_lot_uuid', 'target_label', 'organism_name')
    validation_index = multi_index(alldata['validation'], 'document_filename')

    for uuid, value in alldata['target'].iteritems():
        value['organism_uuid'] = organism_index[value.pop('organism_name')]
        aliases = value.pop('target_aliases') or ''
        alias_source = value['target_alias_source']
        value['dbxref'] = [
            {'db': alias_source, 'id': alias.strip()}
            for alias in aliases.split(';')]

    for uuid, value in list(alldata['antibody_lot'].iteritems()):
        source = value.pop('source')
        try:
            value['source_uuid'] = source_index[source]
        except KeyError:
            logger.warn('Unable to find source: %s (%s)' % (source, uuid))
            del alldata['antibody_lot'][uuid]

    antibody_lot_index = tuple_index(alldata['antibody_lot'], 'product_id', 'lot_id')

    for uuid, value in list(alldata['validation'].iteritems()):
        key = (value.pop('target_label'), value.pop('organism_name'))
        try:
            value['target_uuid'] = target_index[key]
        except KeyError:
            logger.warn('Unable to find target: %s for validation: %s' % (key, uuid))
            del alldata['validation'][uuid]
        else:
            filename = value.pop('document_filename')
            try:
                value['document'] = image_data(
                    resource_stream('encoded', 'tests/data/validation-docs/' + filename),
                    filename,
                    )
            except IOError:
                logger.debug('Referenced filename missing for %s: %s' % (uuid, filename))

    for uuid, value in list(alldata['antibody_approval'].iteritems()):
        try:
            value['antibody_lot_uuid'] = antibody_lot_index[(value.pop('antibody_product_id'), value.pop('antibody_lot_id'))]
        except KeyError:
            logger.debug('Missing/skipped antibody_lot reference for antibody_approval: %s' % uuid)
            del alldata['antibody_approval'][uuid]
            continue

        #value['validation_uuids'] = validation_index.get((value['antibody_lot_uuid'], value['target_label'], value['organism_name']), [])
        value['validation_uuids'] = []
        filenames = (value.pop('validation_filenames') or '').split(';')
        for filename in filenames:
            validation_uuids = validation_index.get(filename, [])
            value['validation_uuids'].extend(validation_uuids)

        try:
            value['target_uuid'] = target_index[(value.pop('target_label'), value.pop('organism_name'))]
        except KeyError:
            logger.debug('Missing/skipped target reference for antibody_approval: %s' % uuid)
            del alldata['antibody_approval'][uuid]
            continue

    for content_type, url in TYPE_URL:
        collection = alldata[content_type]
        for item in collection.itervalues():
            testapp.post_json(url, item, status=201)
<|endoftext|>


<commit_before>from pkg_resources import resource_string

def html_wrapper_tween_factory(handler, registry):

    index_html = resource_string('encoded', 'index.html')

    def html_wrapper_tween(request):
        '''Wrap the returned json with the single page html
        '''

        response = handler(request)
        if response.content_type == 'application/json':
            ## Later include the json in the page
            # json_string = response.body
            response.body = index_html
            response.content_type = 'text/html; charset=utf-8'

        return response

    return html_wrapper_tween
<commit_msg>Never used tween.<commit_after><|endoftext|>