<commit_before>#! /usr/bin/env python

from __future__ import print_function
import scipy.stats
import vcf
import argparse
import csv
import sys
import collections
import numpy

# Parse command line
parser=argparse.ArgumentParser(description='Annotates sites for differential read counts between carrier and non-carrier samples.')
parser.add_argument('-v', '--vcf', metavar='variants.vcf', required=True, dest='vcfFile', help='input vcf file (required)')
parser.add_argument('-out', '--out', metavar='out.vcf', required=True, dest='outFile', help='output vcf file (required)')
parser.add_argument('-t', '--type', metavar='DEL', required=True, dest='svType', help='SV type [DEL, DUP, INV] (required)')
parser.add_argument('-p', '--pvalue', metavar='1.0', required=False, dest='pValueCut', help='p-value threshold for read-depth')
parser.add_argument('-w', '--hwe', metavar='1.0', required=False, dest='hweCut', help='p-value threshold for HWEpval')
parser.add_argument('-s', '--size', metavar='0', required=False, dest='sizeCut', help='min. required size of SVs')
parser.add_argument('-f', '--filter', dest='siteFilter', action='store_true', help='Filter sites for PASS')
args = parser.parse_args()

# Set parameters
sizeCut=0
if (args.sizeCut):
    sizeCut=int(args.sizeCut)
hweCut=1.0
if (args.hweCut):
    hweCut=float(args.hweCut)
pValueCut=1.0
if (args.pValueCut):
    pValueCut=float(args.pValueCut)

# Parse vcf File
if args.vcfFile:
    vcf_reader=vcf.Reader(open(args.vcfFile), 'r')
    vcf_reader.infos['RDpval'] = vcf.parser._Info('RDpval', 1, 'Float', 'RD p-value estimated from carrier vs. non-carrier samples.')
    vcf_writer = vcf.Writer(open(args.outFile, 'w'), vcf_reader, lineterminator='\n')
    for record in vcf_reader:
        rcRef=[]
        rcAlt=[]
        for call in record.samples:
            if ((call.called) and (call['FT']=="PASS") and (call.gt_type==0)):
                rcRef.append(call['RC'])
            if ((call.called) and (call.gt_type!=0)):
                rcAlt.append(call['RC'])
        wilcoxonPval=1.0
        if ((len(rcRef)>0) and (len(rcAlt)>0)):
            wilcoxonStat=scipy.stats.ranksums(rcRef, rcAlt)
            if ((args.svType=="DEL") and (wilcoxonStat[0]>0)):
                wilcoxonPval=wilcoxonStat[1]
            elif ((args.svType=="DUP") and (wilcoxonStat[0]<0)):
                wilcoxonPval=wilcoxonStat[1]
            else:
                wilcoxonPval=wilcoxonStat[1]
        record.INFO['RDpval']=float(round(wilcoxonPval,6))
        if ((record.INFO['RDpval'] <= pValueCut) and (record.INFO['SVLEN']>=sizeCut) and (record.INFO['HWEpval'] <= hweCut)):
            if ((not args.siteFilter) or (len(record.FILTER)==0)):
                vcf_writer.write_record(record)
<commit_msg>Changed read-depth script.<commit_after>#! /usr/bin/env python

from __future__ import print_function
import scipy.stats
import vcf
import argparse
import csv
import sys
import collections
import numpy

# Parse command line
parser=argparse.ArgumentParser(description='Annotates sites for differential read counts between carrier and non-carrier samples.')
parser.add_argument('-v', '--vcf', metavar='variants.vcf', required=True, dest='vcfFile', help='input vcf file (required)')
parser.add_argument('-o', '--out', metavar='out.vcf', required=True, dest='outFile', help='output vcf file (required)')
parser.add_argument('-t', '--type', metavar='DEL', required=True, dest='svType', help='SV type [DEL, DUP, INV] (required)')
parser.add_argument('-p', '--pvalue', metavar='1.0', required=False, dest='pValueCut', help='p-value threshold for read-depth')
parser.add_argument('-w', '--hwe', metavar='1.0', required=False, dest='hweCut', help='p-value threshold for HWEpval')
parser.add_argument('-s', '--size', metavar='0', required=False, dest='sizeCut', help='min. required size of SVs')
parser.add_argument('-f', '--filter', dest='siteFilter', action='store_true', help='Filter sites for PASS')
args = parser.parse_args()

# Set parameters
sizeCut=0
if (args.sizeCut):
    sizeCut=int(args.sizeCut)
hweCut=1.0
if (args.hweCut):
    hweCut=float(args.hweCut)
pValueCut=1.0
if (args.pValueCut):
    pValueCut=float(args.pValueCut)

# Parse vcf File
if args.vcfFile:
    vcf_reader=vcf.Reader(open(args.vcfFile), 'r')
    vcf_reader.infos['RDpval'] = vcf.parser._Info('RDpval', 1, 'Float', 'RD p-value estimated from carrier vs. non-carrier samples.')
    vcf_writer = vcf.Writer(open(args.outFile, 'w'), vcf_reader, lineterminator='\n')
    for record in vcf_reader:
        rcRef=[]
        rcAlt=[]
        for call in record.samples:
            if ((call.called) and (call['FT']=="PASS") and (call.gt_type==0)):
                rcRef.append(call['RC'])
            if ((call.called) and (call.gt_type!=0)):
                rcAlt.append(call['RC'])
        wilcoxonPval=1.0
        if ((len(rcRef)>0) and (len(rcAlt)>0)):
            wilcoxonStat=scipy.stats.ranksums(rcRef, rcAlt)
            if ((args.svType=="DEL") and (wilcoxonStat[0]>0)):
                wilcoxonPval=wilcoxonStat[1]
            elif ((args.svType=="DUP") and (wilcoxonStat[0]<0)):
                wilcoxonPval=wilcoxonStat[1]
            else:
                wilcoxonPval=wilcoxonStat[1]
        record.INFO['RDpval']=float(round(wilcoxonPval,6))
        if ((record.INFO['RDpval'] <= pValueCut) and (record.INFO['SVLEN']>=sizeCut) and (record.INFO['HWEpval'] <= hweCut)):
            if ((not args.siteFilter) or (len(record.FILTER)==0)):
                vcf_writer.write_record(record)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf8
#from gluon import *  #temporary, until we're ready to integrate
from urllib2 import urlopen
from lxml import etree

## This module is used to load owl format ontologies.  There should probably be a class for returned ontology information, but a dict might suffice

OBO_PURL_PREFIX = "http://purl.obolibrary.org/obo/"
OWL_SUFFIX = ".owl"

RDF_RESOURCE = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource'
RDF_ABOUT = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}about'

class ontology(object):
    name_id = dict()
    name = ''
    def __init__(self,etree):
      name = "test"

class ClassTarget(object):
    def __init__(self):
        self.text = []
        self.class_list = []
        self.containerclass = None
    def start(self, tag, attrib):
        self.is_class = False
        self.is_label = False
        self.has_parent = False
        self.is_class_comment = False
        if tag.endswith('Class'):
            self.is_class = True
            if attrib:
                if attrib[RDF_ABOUT]:
                    self.containerclass = {'about': attrib[RDF_ABOUT]}
        elif self.containerclass:
            if tag.endswith('label'):
                self.is_label = True
            elif tag.endswith('subClassOf'):
                self.containerclass['parent'] = attrib[RDF_RESOURCE]
                self.has_parent = True
            elif tag.endswith('IAO_0000115'):
                self.is_class_comment = True
    def end(self, tag):
        if tag.endswith('Class'):
            self.class_list.append(self.containerclass)
            self.containerclass = None
        self.is_class = False
        self.is_label = False
        self.has_parent = False
        self.is_class_comment = False
        pass
    def data(self, data):
        if self.is_label and self.containerclass:
            self.containerclass['label'] = data
        elif self.is_class_comment and self.containerclass:
            self.containerclass['class_comment'] = data
    def close(self):
        return self.class_list

def load_from_obo(ontology_name):
    import pprint
    pp = pprint.PrettyPrinter(indent=2)
    ontology_source = urlopen(OBO_PURL_PREFIX+ontology_name+OWL_SUFFIX)
    parser = etree.XMLParser(target = ClassTarget())
    results = etree.parse(ontology_source, parser)
    for result in results:
        pp.pprint(result)

def process_tree(ont_tree):
    for child in ont_tree.getroot():
        print child.tag
    return ont_tree
<commit_msg>parsing and taxon scoping from NCBI taxonomy seems to work<commit_after>#!/usr/bin/env python
# coding: utf8
#from gluon import *  #temporary, until we're ready to integrate
from urllib2 import urlopen
from lxml import etree

## This module is used to load owl format ontologies.  There should probably be a class for returned ontology information, but a dict might suffice

RDF_PREFIX = "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}"
RDFS_PREFIX = "{http://www.w3.org/2000/01/rdf-schema#}"
OWL_PREFIX = "{http://www.w3.org/2002/07/owl#}"
OBO_PURL_PREFIX = "http://purl.obolibrary.org/obo/"

OWL_SUFFIX = ".owl"

RDF_RESOURCE = RDF_PREFIX + 'resource'
RDF_ABOUT = RDF_PREFIX + 'about'

RDFS_LABEL = RDFS_PREFIX + 'label'

OWL_CLASS = OWL_PREFIX + 'Class'
OWL_EQUIVALENTCLASS = OWL_PREFIX + 'equivalentClass'

class ClassTarget(object):
    def __init__(self):
        self.text = []
        self.class_list = []
        self.containerclass = None
    def start(self, tag, attrib):
        self.is_class = False
        self.is_label = False
        self.has_parent = False
        self.is_equivalent_class = False
        self.is_class_comment = False
        if tag == OWL_CLASS:
            self.is_class = True
            if attrib:
                if attrib[RDF_ABOUT]:
                    self.containerclass = {'about': attrib[RDF_ABOUT]}
        elif tag == OWL_EQUIVALENTCLASS:
            self.is_equivalent_class = True
        elif self.containerclass:
            if tag.endswith('label'):
                self.is_label = True
            elif tag.endswith('subClassOf'):
                self.containerclass['parent'] = attrib[RDF_RESOURCE]
                self.has_parent = True
            elif tag.endswith('IAO_0000115'):
                self.is_class_comment = True
    def end(self, tag):
        if tag == OWL_CLASS:
            if not self.is_equivalent_class:
                self.class_list.append(self.containerclass)
                self.containerclass = None
                self.is_class = False
        elif tag == OWL_EQUIVALENTCLASS:
            self.is_equivalent_class = False
        self.is_label = False
        self.has_parent = False
        self.is_class_comment = False
        pass
    def data(self, data):
        if self.is_label and self.containerclass:
            self.containerclass['label'] = data
        elif self.is_class_comment and self.containerclass:
            com = ''
            if 'class_comment' in self.containerclass:
                com = self.containerclass['class_comment']
            self.containerclass['class_comment'] = com + data
    def close(self):
        return self.class_list

def pplist(terms):
    import pprint
    pp = pprint.PrettyPrinter(indent=2)
    for term in terms:
        pp.pprint(term)

def process_tree(ont_tree):
    for child in ont_tree.getroot():
        print child.tag
    return ont_tree

ARACHNID_NODE = u'http://purl.obolibrary.org/obo/NCBITaxon_6854'

def build_ontology_tree(terms):
    import pprint
    pp = pprint.PrettyPrinter(indent=2)
    roots = []
    tree_dict = dict()
    parent_dict = dict()
    for term in terms:
        if term:
            if 'about' in term:
                tree_dict[term['about']] = term
            if 'parent' in term:
                tp = term['parent']
                if tp in parent_dict and parent_dict[tp] != None:
                   t1 = parent_dict[tp]
                   baz = t1.append(term)
                   parent_dict[tp] = t1
                else:
                   parent_dict[tp] = [term]
            else:
                roots.append(term)
    final_dict = dict()
    children = [tree_dict[ARACHNID_NODE]]
    while (children):
        child = children.pop(0)
        if 'about' in child:
            final_dict[child['about']] = child
        if child['about'] in parent_dict:
            newchildren = parent_dict[child['about']]
            children.extend(newchildren)
    print "added %d children of arachnid node" % len(final_dict)

def load_from_obo(ontology_name, processor):
    ontology_source = urlopen(OBO_PURL_PREFIX+ontology_name+OWL_SUFFIX)
    parser = etree.XMLParser(target = ClassTarget())
    results = etree.parse(ontology_source, parser)
    processor(results)

def demo():
    load_from_obo('ncbitaxon',build_ontology_tree)
<|endoftext|>


<commit_before># Copyright 2011 Element 34
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
========
HomePage
========
"""
import sys
from saunter.po import string_timeout
from saunter.po.page import Page
from pages.LoginPage import LoginPage
from SeleniumWrapper import SeleniumWrapper as wrapper

locators = {
    "login": "css=div.account_mast a:first",
    "signup": "css=div.account_mast a:last"
}

class HomePage(Page):
    """
    PO for the Home/Landing page
    """
    def __init__(self):
        self.se = wrapper().connection

    def open_default_url(self):
        """
        Goes to the default url for this PO
        """
        self.se.open("/")

    def go_to_login_page(self):
        """
        Goes to the login page

        :returns: :class:`pages.LoginPage.LoginPage`
        """
        self.se.click(locators['login'])
        self.se.wait_for_page_to_load(string_timeout)
        return LoginPage()
        <commit_msg>removing an unnecessary import<commit_after># Copyright 2011 Element 34
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
========
HomePage
========
"""
from saunter.po import string_timeout
from saunter.po.page import Page
from pages.LoginPage import LoginPage
from SeleniumWrapper import SeleniumWrapper as wrapper

locators = {
    "login": "css=div.account_mast a:first",
    "signup": "css=div.account_mast a:last"
}

class HomePage(Page):
    """
    PO for the Home/Landing page
    """
    def __init__(self):
        self.se = wrapper().connection

    def open_default_url(self):
        """
        Goes to the default url for this PO
        """
        self.se.open("/")

    def go_to_login_page(self):
        """
        Goes to the login page

        :returns: :class:`pages.LoginPage.LoginPage`
        """
        self.se.click(locators['login'])
        self.se.wait_for_page_to_load(string_timeout)
        return LoginPage()
        <|endoftext|>


<commit_before>"""
    Copyright (c) 2015-2019 Raj Patel(raj454raj@gmail.com), StopStalk

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to deal
    in the Software without restriction, including without limitation the rights
    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    THE SOFTWARE.
"""

from urllib import urlencode
from .init import *

PER_PAGE_LIMIT = 20
CODECHEF_API_URL = "https://api.codechef.com"
CODECHEF_SITE_URL = "https://www.codechef.com"
TIME_CONVERSION_STRING = "%Y-%m-%d %H:%M:%S"
SUBMISSION_REQUEST_PARAMS = {"year": None,
                             "username": "",
                             "limit": PER_PAGE_LIMIT,
                             "offset": 0,
                             "result": "",
                             "language": "",
                             "problemCode": "",
                             "contestCode": "",
                             "fields": ""}

class Profile(object):
    """
        Class containing methods for retrieving
        submissions of user
    """

    # --------------------------------------------------------------------------
    def __init__(self, handle=""):
        """
            @param handle (String): Codechef Handle
        """

        self.site = "CodeChef"
        self.handle = handle
        self.submissions = []
        self.access_token = None
        self.is_daily_retrieval = False
        self.last_retrieved_reached = False

    # --------------------------------------------------------------------------
    @staticmethod
    def get_tags(problem_link):
        """
            Get tags given a problem link

            @param problem_link (String): Problem URL
            @return (List): List of tags for the Problem
        """

        url = problem_link.split("/")
        url = url[2:]
        url.insert(1, "api/contests")
        if len(url) == 4:
            url.insert(2, "PRACTICE")
        url = "https://" + "/".join(url)

        response = get_request(url, headers={"User-Agent": user_agent})
        if response in REQUEST_FAILURES:
            # @ToDo: Need to blacklist 404 urls also
            return ["-"]

        t = response.json()
        all_tags = []
        try:
            tags = t["tags"]
            all_as = BeautifulSoup(str(tags), "lxml").find_all("a")
            for i in all_as:
                all_tags.append(i.contents[0].strip())
            return all_tags
        except KeyError:
            return all_tags

    # --------------------------------------------------------------------------
    @staticmethod
    def get_editorial_link(problem_link):
        """
            Get editorial link given a problem link

            @param problem_link (String): Problem URL
            @return (String/None): Editorial URL
        """
        editorial_link = None
        response = get_request(problem_link, headers={"User-Agent": user_agent})
        if response in REQUEST_FAILURES:
            return None

        soup = BeautifulSoup(response.text, "lxml")
        all_as = soup.find_all("a")

        for link in all_as:
            try:
                url = link.contents[0]
            except IndexError:
                continue
            if url.__contains__("discuss.codechef.com"):
                editorial_link = url
                break

        return editorial_link

    # -------------------------------------------------------------------------
    @staticmethod
    def is_invalid_handle(handle):
        # CodeChef is very flaky
        return True
        response = get_request("https://www.codechef.com/users/" + handle)
        if (response in REQUEST_FAILURES) or response.url.__contains__("teams/view"):
            return True
        return False

    # --------------------------------------------------------------------------
    def __validate_handle(self):
        """
            Make an API request to Codechef to see if a user exists
        """

        response = get_request("%s/users/%s" % (CODECHEF_API_URL, self.handle),
                               headers={"User-Agent": user_agent,
                                        "Authorization": "Bearer %s" % self.access_token},
                               timeout=10,
                               is_daily_retrieval=self.is_daily_retrieval)
        if response in REQUEST_FAILURES:
            return response

        # User was not found in Codechef database
        json_data = response.json()
        if json_data["result"]["data"]["code"] == 9003:
            return NOT_FOUND
        else:
            return "VALID_HANDLE"

    # --------------------------------------------------------------------------
    def __get_access_token(self):
        """
            Get CodeChef API access_token from the database
        """

        # Codechef has a TTL of 1 hour for every generated access token
        TTL_TIME = 60 * 60
        # We are assuming there might be cases when 1 submission retrieval takes
        # more than 10 minutes
        SAFE_TIME = 10 * 60

        redis_key = "codechef_access_token"
        redis_ttl = current.REDIS_CLIENT.ttl(redis_key)
        if redis_ttl is not None and redis_ttl > SAFE_TIME:
            return current.REDIS_CLIENT.get(redis_key)

        response = requests.post("%s/oauth/token" % CODECHEF_API_URL,
                                 data={"grant_type": "client_credentials",
                                       "scope": "public",
                                       "client_id": current.codechef_client_id,
                                       "client_secret": current.codechef_client_secret,
                                       "redirect_uri": ""})
        json_data = response.json()
        if response.status_code == 200 and json_data["status"] == "OK":
            access_token = json_data["result"]["data"]["access_token"]
            current.REDIS_CLIENT.set(redis_key,
                                     access_token,
                                     ex=TTL_TIME)
            return access_token
        else:
            print "Error requesting CodeChef API for access token"
            return

    # --------------------------------------------------------------------------
    def __get_problem_link(self, contest_code, problem_code):
        """
            Get the problem link given the contest_code and problem_code
            @params contest_code (String): Contest code of the problem
            @params problem_code (String): Problem code of the problem

            @return (String): URL of the problem
        """
        return "%s/%s/problems/%s" % (CODECHEF_SITE_URL,
                                      contest_code,
                                      problem_code)

    # --------------------------------------------------------------------------
    def __process_year_submissions(self, year, last_retrieved):
        SUBMISSION_REQUEST_PARAMS["year"] = year
        SUBMISSION_REQUEST_PARAMS["offset"] = 0
        submissions = []
        for _ in xrange(1000):
            response = get_request("%s/submissions" % CODECHEF_API_URL,
                                   headers={"Authorization": "Bearer %s" % self.access_token},
                                   params=SUBMISSION_REQUEST_PARAMS,
                                   timeout=10,
                                   is_daily_retrieval=self.is_daily_retrieval)
            if response in REQUEST_FAILURES:
                return response

            json_response = response.json()
            if json_response["result"]["data"]["code"] == 9003:
                # No submisions for the year
                return submissions

            for submission in json_response["result"]["data"]["content"]:
                curr = time.strptime(submission["date"],
                                     TIME_CONVERSION_STRING)
                if curr <= last_retrieved:
                    self.last_retrieved_reached = True
                    return submissions

                problem_link = self.__get_problem_link(submission["contestCode"],
                                                       submission["problemCode"])
                problem_name = submission["problemCode"]
                status = submission["result"]
                points = submission["score"]
                if status == "AC":
                    if float(points) > 0 and float(points) < 100:
                        status = "PS"
                elif status in ["WA", "TLE"]:
                    pass
                elif status == "CTE":
                    status = "CE"
                elif status == "RTE":
                    status = "RE"
                else:
                    print "*****************", status
                    status = "OTH"
                language = submission["language"]
                view_link = "%s/viewsolution/%d" % (CODECHEF_SITE_URL,
                                                    submission["id"])
                submissions.append((submission["date"],
                                    problem_link,
                                    problem_name,
                                    status,
                                    points,
                                    language,
                                    view_link))
            if len(json_response["result"]["data"]["content"]) < PER_PAGE_LIMIT:
                break

            SUBMISSION_REQUEST_PARAMS["offset"] += PER_PAGE_LIMIT

        return submissions

    # --------------------------------------------------------------------------
    def get_submissions(self, last_retrieved, is_daily_retrieval):
        """
            Retrieve CodeChef submissions after last retrieved timestamp

            @param last_retrieved (DateTime): Last retrieved timestamp for the user
            @param is_daily_retrieval (Boolean): If this call is from daily retrieval cron
            @return (Dict): Dictionary of submissions containing all the
                            information about the submissions
        """

        if current.environment == "development":
            # Locally client credentials for CodeChef API wouldn't be valid and
            # there is no other fallback method which can be used for retrieval
            return SERVER_FAILURE

        self.is_daily_retrieval = is_daily_retrieval
        try:
            # If the handle is a URL return NOT_FOUND
            re.match("https?://.*\.com", self.handle).group()
            return NOT_FOUND
        except AttributeError:
            pass

        start_year = last_retrieved.tm_year
        current_year = datetime.datetime.now().year
        str_init_time = time.strptime(str(current.INITIAL_DATE),
                                      "%Y-%m-%d %H:%M:%S")

        self.access_token = self.__get_access_token()
        if self.access_token is None:
            print "Access token found none"
            return SERVER_FAILURE

        # Test for invalid handles
        if  last_retrieved == str_init_time:
            response = self.__validate_handle()
            if response in REQUEST_FAILURES:
                return response

        SUBMISSION_REQUEST_PARAMS["username"] = self.handle
        self.submissions = []

        print start_year, current_year
        for year in xrange(current_year, start_year - 1, -1):
            print year
            # Years processed in the reverse order to break out when
            # last_retrieved time_stamp is matched

            result = self.__process_year_submissions(year, last_retrieved)
            if result in REQUEST_FAILURES:
                return result
            else:
                self.submissions.extend(result)
                if self.last_retrieved_reached:
                    return self.submissions

        return self.submissions

# =============================================================================
<commit_msg>remove log lines<commit_after>"""
    Copyright (c) 2015-2019 Raj Patel(raj454raj@gmail.com), StopStalk

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to deal
    in the Software without restriction, including without limitation the rights
    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    THE SOFTWARE.
"""

from urllib import urlencode
from .init import *

PER_PAGE_LIMIT = 20
CODECHEF_API_URL = "https://api.codechef.com"
CODECHEF_SITE_URL = "https://www.codechef.com"
TIME_CONVERSION_STRING = "%Y-%m-%d %H:%M:%S"
SUBMISSION_REQUEST_PARAMS = {"year": None,
                             "username": "",
                             "limit": PER_PAGE_LIMIT,
                             "offset": 0,
                             "result": "",
                             "language": "",
                             "problemCode": "",
                             "contestCode": "",
                             "fields": ""}

class Profile(object):
    """
        Class containing methods for retrieving
        submissions of user
    """

    # --------------------------------------------------------------------------
    def __init__(self, handle=""):
        """
            @param handle (String): Codechef Handle
        """

        self.site = "CodeChef"
        self.handle = handle
        self.submissions = []
        self.access_token = None
        self.is_daily_retrieval = False
        self.last_retrieved_reached = False

    # --------------------------------------------------------------------------
    @staticmethod
    def get_tags(problem_link):
        """
            Get tags given a problem link

            @param problem_link (String): Problem URL
            @return (List): List of tags for the Problem
        """

        url = problem_link.split("/")
        url = url[2:]
        url.insert(1, "api/contests")
        if len(url) == 4:
            url.insert(2, "PRACTICE")
        url = "https://" + "/".join(url)

        response = get_request(url, headers={"User-Agent": user_agent})
        if response in REQUEST_FAILURES:
            # @ToDo: Need to blacklist 404 urls also
            return ["-"]

        t = response.json()
        all_tags = []
        try:
            tags = t["tags"]
            all_as = BeautifulSoup(str(tags), "lxml").find_all("a")
            for i in all_as:
                all_tags.append(i.contents[0].strip())
            return all_tags
        except KeyError:
            return all_tags

    # --------------------------------------------------------------------------
    @staticmethod
    def get_editorial_link(problem_link):
        """
            Get editorial link given a problem link

            @param problem_link (String): Problem URL
            @return (String/None): Editorial URL
        """
        editorial_link = None
        response = get_request(problem_link, headers={"User-Agent": user_agent})
        if response in REQUEST_FAILURES:
            return None

        soup = BeautifulSoup(response.text, "lxml")
        all_as = soup.find_all("a")

        for link in all_as:
            try:
                url = link.contents[0]
            except IndexError:
                continue
            if url.__contains__("discuss.codechef.com"):
                editorial_link = url
                break

        return editorial_link

    # -------------------------------------------------------------------------
    @staticmethod
    def is_invalid_handle(handle):
        # CodeChef is very flaky
        return True
        response = get_request("https://www.codechef.com/users/" + handle)
        if (response in REQUEST_FAILURES) or response.url.__contains__("teams/view"):
            return True
        return False

    # --------------------------------------------------------------------------
    def __validate_handle(self):
        """
            Make an API request to Codechef to see if a user exists
        """

        response = get_request("%s/users/%s" % (CODECHEF_API_URL, self.handle),
                               headers={"User-Agent": user_agent,
                                        "Authorization": "Bearer %s" % self.access_token},
                               timeout=10,
                               is_daily_retrieval=self.is_daily_retrieval)
        if response in REQUEST_FAILURES:
            return response

        # User was not found in Codechef database
        json_data = response.json()
        if json_data["result"]["data"]["code"] == 9003:
            return NOT_FOUND
        else:
            return "VALID_HANDLE"

    # --------------------------------------------------------------------------
    def __get_access_token(self):
        """
            Get CodeChef API access_token from the database
        """

        # Codechef has a TTL of 1 hour for every generated access token
        TTL_TIME = 60 * 60
        # We are assuming there might be cases when 1 submission retrieval takes
        # more than 10 minutes
        SAFE_TIME = 10 * 60

        redis_key = "codechef_access_token"
        redis_ttl = current.REDIS_CLIENT.ttl(redis_key)
        if redis_ttl is not None and redis_ttl > SAFE_TIME:
            return current.REDIS_CLIENT.get(redis_key)

        response = requests.post("%s/oauth/token" % CODECHEF_API_URL,
                                 data={"grant_type": "client_credentials",
                                       "scope": "public",
                                       "client_id": current.codechef_client_id,
                                       "client_secret": current.codechef_client_secret,
                                       "redirect_uri": ""})
        json_data = response.json()
        if response.status_code == 200 and json_data["status"] == "OK":
            access_token = json_data["result"]["data"]["access_token"]
            current.REDIS_CLIENT.set(redis_key,
                                     access_token,
                                     ex=TTL_TIME)
            return access_token
        else:
            print "Error requesting CodeChef API for access token"
            return

    # --------------------------------------------------------------------------
    def __get_problem_link(self, contest_code, problem_code):
        """
            Get the problem link given the contest_code and problem_code
            @params contest_code (String): Contest code of the problem
            @params problem_code (String): Problem code of the problem

            @return (String): URL of the problem
        """
        return "%s/%s/problems/%s" % (CODECHEF_SITE_URL,
                                      contest_code,
                                      problem_code)

    # --------------------------------------------------------------------------
    def __process_year_submissions(self, year, last_retrieved):
        SUBMISSION_REQUEST_PARAMS["year"] = year
        SUBMISSION_REQUEST_PARAMS["offset"] = 0
        submissions = []
        for _ in xrange(1000):
            response = get_request("%s/submissions" % CODECHEF_API_URL,
                                   headers={"Authorization": "Bearer %s" % self.access_token},
                                   params=SUBMISSION_REQUEST_PARAMS,
                                   timeout=10,
                                   is_daily_retrieval=self.is_daily_retrieval)
            if response in REQUEST_FAILURES:
                return response

            json_response = response.json()
            if json_response["result"]["data"]["code"] == 9003:
                # No submisions for the year
                return submissions

            for submission in json_response["result"]["data"]["content"]:
                curr = time.strptime(submission["date"],
                                     TIME_CONVERSION_STRING)
                if curr <= last_retrieved:
                    self.last_retrieved_reached = True
                    return submissions

                problem_link = self.__get_problem_link(submission["contestCode"],
                                                       submission["problemCode"])
                problem_name = submission["problemCode"]
                status = submission["result"]
                points = submission["score"]
                if status == "AC":
                    if float(points) > 0 and float(points) < 100:
                        status = "PS"
                elif status in ["WA", "TLE"]:
                    pass
                elif status == "CTE":
                    status = "CE"
                elif status == "RTE":
                    status = "RE"
                else:
                    print "*****************", status
                    status = "OTH"
                language = submission["language"]
                view_link = "%s/viewsolution/%d" % (CODECHEF_SITE_URL,
                                                    submission["id"])
                submissions.append((submission["date"],
                                    problem_link,
                                    problem_name,
                                    status,
                                    points,
                                    language,
                                    view_link))
            if len(json_response["result"]["data"]["content"]) < PER_PAGE_LIMIT:
                break

            SUBMISSION_REQUEST_PARAMS["offset"] += PER_PAGE_LIMIT

        return submissions

    # --------------------------------------------------------------------------
    def get_submissions(self, last_retrieved, is_daily_retrieval):
        """
            Retrieve CodeChef submissions after last retrieved timestamp

            @param last_retrieved (DateTime): Last retrieved timestamp for the user
            @param is_daily_retrieval (Boolean): If this call is from daily retrieval cron
            @return (Dict): Dictionary of submissions containing all the
                            information about the submissions
        """

        if current.environment == "development":
            # Locally client credentials for CodeChef API wouldn't be valid and
            # there is no other fallback method which can be used for retrieval
            return SERVER_FAILURE

        self.is_daily_retrieval = is_daily_retrieval
        try:
            # If the handle is a URL return NOT_FOUND
            re.match("https?://.*\.com", self.handle).group()
            return NOT_FOUND
        except AttributeError:
            pass

        start_year = last_retrieved.tm_year
        current_year = datetime.datetime.now().year
        str_init_time = time.strptime(str(current.INITIAL_DATE),
                                      "%Y-%m-%d %H:%M:%S")

        self.access_token = self.__get_access_token()
        if self.access_token is None:
            print "Access token found none"
            return SERVER_FAILURE

        # Test for invalid handles
        if  last_retrieved == str_init_time:
            response = self.__validate_handle()
            if response in REQUEST_FAILURES:
                return response

        SUBMISSION_REQUEST_PARAMS["username"] = self.handle
        self.submissions = []

        for year in xrange(current_year, start_year - 1, -1):
            # Years processed in the reverse order to break out when
            # last_retrieved time_stamp is matched

            result = self.__process_year_submissions(year, last_retrieved)
            if result in REQUEST_FAILURES:
                return result
            else:
                self.submissions.extend(result)
                if self.last_retrieved_reached:
                    return self.submissions

        return self.submissions

# =============================================================================
<|endoftext|>


<commit_before>"""
    Copyright (c) 2015-2019 Raj Patel(raj454raj@gmail.com), StopStalk

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to deal
    in the Software without restriction, including without limitation the rights
    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    THE SOFTWARE.
"""

from urllib import urlencode
from .init import *

PER_PAGE_LIMIT = 20
CODECHEF_API_URL = "https://api.codechef.com"
CODECHEF_SITE_URL = "https://www.codechef.com"
TIME_CONVERSION_STRING = "%Y-%m-%d %H:%M:%S"
SUBMISSION_REQUEST_PARAMS = {"year": None,
                             "username": "",
                             "limit": PER_PAGE_LIMIT,
                             "offset": 0,
                             "result": "",
                             "language": "",
                             "problemCode": "",
                             "contestCode": "",
                             "fields": ""}

class Profile(object):
    """
        Class containing methods for retrieving
        submissions of user
    """
    site_name = "CodeChef"

    # --------------------------------------------------------------------------
    def __init__(self, handle=""):
        """
            @param handle (String): Codechef Handle
        """

        self.site = Profile.site_name
        self.handle = handle
        self.submissions = []
        self.access_token = None
        self.is_daily_retrieval = False
        self.last_retrieved_reached = False

    # -------------------------------------------------------------------------
    @staticmethod
    def is_website_down():
        return (Profile.site_name in current.REDIS_CLIENT.smembers("disabled_retrieval"))

    # --------------------------------------------------------------------------
    @staticmethod
    def get_tags(problem_link):
        """
            Get tags given a problem link

            @param problem_link (String): Problem URL
            @return (List): List of tags for the Problem
        """

        url = problem_link.split("/")
        url = url[2:]
        url.insert(1, "api/contests")
        if len(url) == 4:
            url.insert(2, "PRACTICE")
        url = "https://" + "/".join(url)

        response = get_request(url, headers={"User-Agent": user_agent})
        if response in REQUEST_FAILURES:
            # @ToDo: Need to blacklist 404 urls also
            return ["-"]

        t = response.json()
        all_tags = []
        try:
            tags = t["tags"]
            all_as = BeautifulSoup(str(tags), "lxml").find_all("a")
            for i in all_as:
                all_tags.append(i.contents[0].strip())
            return all_tags
        except KeyError:
            return all_tags

    # --------------------------------------------------------------------------
    @staticmethod
    def get_editorial_link(problem_link):
        """
            Get editorial link given a problem link

            @param problem_link (String): Problem URL
            @return (String/None): Editorial URL
        """
        editorial_link = None
        api_link = problem_link.replace("https://www.codechef.com/", "https://www.codechef.com/api/contests/")
        response = get_request(api_link + "?v=1554915627060",
                               headers={"User-Agent": user_agent})
        if response in REQUEST_FAILURES:
            return None
        response = response.json()
        try:
            editorial_link = response["editorial_url"]
        except KeyError:
            editorial_link = None

        try:
            tags = BeautifulSoup(response["tags"], "lxml").text
            tags = tags.split(" ")
            db = current.db
            ptable = db.problem
            row = db(ptable.link == problem_link).select().first()
            today = datetime.datetime.now().strftime("%Y-%m-%d")
            if row:
                row.update_record(tags=str(tags),
                                  tags_added_on=today)
        except Exception as e:
            pass

        return editorial_link

    # -------------------------------------------------------------------------
    @staticmethod
    def is_invalid_handle(handle):
        # CodeChef is very flaky
        return True
        response = get_request("https://www.codechef.com/users/" + handle)
        if (response in REQUEST_FAILURES) or response.url.__contains__("teams/view"):
            return True
        return False

    # -------------------------------------------------------------------------
    @staticmethod
    def download_submission(view_link):
        if Profile.is_website_down():
            return -1

        problem_id = view_link.strip("/").split("/")[-1]
        download_url = "https://www.codechef.com/viewplaintext/" + \
                       str(problem_id)
        response = get_request(download_url,
                               headers={"User-Agent": user_agent})
        if response in REQUEST_FAILURES:
            return -1
        try:
            return BeautifulSoup(response.text, "lxml").find("pre").text
        except:
            return -1

    # -------------------------------------------------------------------------
    @staticmethod
    def rating_graph_data(handle):
        url = "https://www.codechef.com/users/" + handle
        response = get_request(url, headers={"User-Agent": user_agent})

        if response in REQUEST_FAILURES:
            return response

        ratings = eval(re.search("var all_rating = .*?;", response.text).group()[17:-1].replace("null", "None"))

        long_contest_data = {}
        cookoff_contest_data = {}
        ltime_contest_data = {}

        for contest in ratings:
            this_obj = None
            if contest["code"].__contains__("COOK"):
                # Cook off contest
                this_obj = cookoff_contest_data
            elif contest["code"].__contains__("LTIME"):
                # Lunchtime contest
                this_obj = ltime_contest_data
            else:
                # Long contest
                this_obj = long_contest_data
            # getdate, getmonth and getyear give end_date only
            time_stamp = str(datetime.datetime.strptime(contest["end_date"], "%Y-%m-%d %H:%M:%S"))
            this_obj[time_stamp] = {"name": contest["name"],
                                    "url": "https://www.codechef.com/" + contest["code"],
                                    "rating": str(contest["rating"]),
                                    "rank": contest["rank"]}

        return [{"title": "CodeChef Long",
                 "data": long_contest_data},
                {"title": "CodeChef Cook-off",
                 "data": cookoff_contest_data},
                {"title": "CodeChef Lunchtime",
                 "data": ltime_contest_data}]

    # --------------------------------------------------------------------------
    def __validate_handle(self):
        """
            Make an API request to Codechef to see if a user exists
        """

        response = get_request("%s/users/%s" % (CODECHEF_API_URL, self.handle),
                               headers={"User-Agent": user_agent,
                                        "Authorization": "Bearer %s" % self.access_token},
                               timeout=10,
                               is_daily_retrieval=self.is_daily_retrieval)
        if response in REQUEST_FAILURES:
            return response

        # User was not found in Codechef database
        json_data = response.json()
        if json_data["result"]["data"]["code"] == 9003:
            return NOT_FOUND
        else:
            return "VALID_HANDLE"

    # --------------------------------------------------------------------------
    def __get_access_token(self):
        """
            Get CodeChef API access_token from the database
        """

        # Codechef has a TTL of 1 hour for every generated access token
        TTL_TIME = 60 * 60
        # We are assuming there might be cases when 1 submission retrieval takes
        # more than 10 minutes
        SAFE_TIME = 10 * 60

        redis_key = "codechef_access_token"
        redis_ttl = current.REDIS_CLIENT.ttl(redis_key)
        if redis_ttl is not None and redis_ttl > SAFE_TIME:
            return current.REDIS_CLIENT.get(redis_key)

        response = requests.post("%s/oauth/token" % CODECHEF_API_URL,
                                 data={"grant_type": "client_credentials",
                                       "scope": "public",
                                       "client_id": current.codechef_client_id,
                                       "client_secret": current.codechef_client_secret,
                                       "redirect_uri": ""})
        json_data = response.json()
        if response.status_code == 200 and json_data["status"] == "OK":
            access_token = json_data["result"]["data"]["access_token"]
            current.REDIS_CLIENT.set(redis_key,
                                     access_token,
                                     ex=TTL_TIME)
            return access_token
        else:
            print "Error requesting CodeChef API for access token"
            return

    # --------------------------------------------------------------------------
    def __get_problem_link(self, contest_code, problem_code):
        """
            Get the problem link given the contest_code and problem_code
            @params contest_code (String): Contest code of the problem
            @params problem_code (String): Problem code of the problem

            @return (String): URL of the problem
        """
        return "%s/%s/problems/%s" % (CODECHEF_SITE_URL,
                                      contest_code,
                                      problem_code)

    # --------------------------------------------------------------------------
    def __process_year_submissions(self, year, last_retrieved):
        SUBMISSION_REQUEST_PARAMS["year"] = year
        SUBMISSION_REQUEST_PARAMS["offset"] = 0
        submissions = []
        for _ in xrange(1000):
            response = get_request("%s/submissions" % CODECHEF_API_URL,
                                   headers={"Authorization": "Bearer %s" % self.access_token},
                                   params=SUBMISSION_REQUEST_PARAMS,
                                   timeout=10,
                                   is_daily_retrieval=self.is_daily_retrieval)
            if response in REQUEST_FAILURES:
                return response

            json_response = response.json()
            if json_response["result"]["data"]["code"] == 9003:
                # No submisions for the year
                return submissions

            for submission in json_response["result"]["data"]["content"]:
                curr = time.strptime(submission["date"],
                                     TIME_CONVERSION_STRING)
                if curr <= last_retrieved:
                    self.last_retrieved_reached = True
                    return submissions

                problem_link = self.__get_problem_link(submission["contestCode"],
                                                       submission["problemCode"])
                problem_name = submission["problemCode"]
                status = submission["result"]
                points = submission["score"]
                if status == "AC":
                    if float(points) > 0 and float(points) < 100:
                        status = "PS"
                elif status in ["WA", "TLE"]:
                    pass
                elif status == "CTE":
                    status = "CE"
                elif status == "RTE":
                    status = "RE"
                else:
                    print "*****************", status
                    status = "OTH"
                language = submission["language"]
                view_link = "%s/viewsolution/%d" % (CODECHEF_SITE_URL,
                                                    submission["id"])
                submissions.append((submission["date"],
                                    problem_link,
                                    problem_name,
                                    status,
                                    points,
                                    language,
                                    view_link))
            if len(json_response["result"]["data"]["content"]) < PER_PAGE_LIMIT:
                break

            SUBMISSION_REQUEST_PARAMS["offset"] += PER_PAGE_LIMIT

        return submissions

    # --------------------------------------------------------------------------
    def get_submissions(self, last_retrieved, is_daily_retrieval):
        """
            Retrieve CodeChef submissions after last retrieved timestamp

            @param last_retrieved (DateTime): Last retrieved timestamp for the user
            @param is_daily_retrieval (Boolean): If this call is from daily retrieval cron
            @return (Dict): Dictionary of submissions containing all the
                            information about the submissions
        """

        if current.environment == "development":
            # Locally client credentials for CodeChef API wouldn't be valid and
            # there is no other fallback method which can be used for retrieval
            return SERVER_FAILURE

        self.is_daily_retrieval = is_daily_retrieval
        try:
            # If the handle is a URL return NOT_FOUND
            re.match("https?://.*\.com", self.handle).group()
            return NOT_FOUND
        except AttributeError:
            pass

        start_year = last_retrieved.tm_year
        current_year = datetime.datetime.now().year
        str_init_time = time.strptime(str(current.INITIAL_DATE),
                                      "%Y-%m-%d %H:%M:%S")

        self.access_token = self.__get_access_token()
        if self.access_token is None:
            print "Access token found none"
            return SERVER_FAILURE

        # Test for invalid handles
        if  last_retrieved == str_init_time:
            response = self.__validate_handle()
            if response in REQUEST_FAILURES:
                return response

        SUBMISSION_REQUEST_PARAMS["username"] = self.handle
        self.submissions = []

        for year in xrange(current_year, start_year - 1, -1):
            # Years processed in the reverse order to break out when
            # last_retrieved time_stamp is matched

            result = self.__process_year_submissions(year, last_retrieved)
            if result in REQUEST_FAILURES:
                return result
            else:
                self.submissions.extend(result)
                if self.last_retrieved_reached:
                    return self.submissions

        return self.submissions

# =============================================================================
<commit_msg>Plug the commas in codechef tags<commit_after>"""
    Copyright (c) 2015-2019 Raj Patel(raj454raj@gmail.com), StopStalk

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to deal
    in the Software without restriction, including without limitation the rights
    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    THE SOFTWARE.
"""

from urllib import urlencode
from .init import *

PER_PAGE_LIMIT = 20
CODECHEF_API_URL = "https://api.codechef.com"
CODECHEF_SITE_URL = "https://www.codechef.com"
TIME_CONVERSION_STRING = "%Y-%m-%d %H:%M:%S"
SUBMISSION_REQUEST_PARAMS = {"year": None,
                             "username": "",
                             "limit": PER_PAGE_LIMIT,
                             "offset": 0,
                             "result": "",
                             "language": "",
                             "problemCode": "",
                             "contestCode": "",
                             "fields": ""}

class Profile(object):
    """
        Class containing methods for retrieving
        submissions of user
    """
    site_name = "CodeChef"

    # --------------------------------------------------------------------------
    def __init__(self, handle=""):
        """
            @param handle (String): Codechef Handle
        """

        self.site = Profile.site_name
        self.handle = handle
        self.submissions = []
        self.access_token = None
        self.is_daily_retrieval = False
        self.last_retrieved_reached = False

    # -------------------------------------------------------------------------
    @staticmethod
    def is_website_down():
        return (Profile.site_name in current.REDIS_CLIENT.smembers("disabled_retrieval"))

    # --------------------------------------------------------------------------
    @staticmethod
    def get_tags(problem_link):
        """
            Get tags given a problem link

            @param problem_link (String): Problem URL
            @return (List): List of tags for the Problem
        """

        url = problem_link.split("/")
        url = url[2:]
        url.insert(1, "api/contests")
        if len(url) == 4:
            url.insert(2, "PRACTICE")
        url = "https://" + "/".join(url)

        response = get_request(url, headers={"User-Agent": user_agent})
        if response in REQUEST_FAILURES:
            # @ToDo: Need to blacklist 404 urls also
            return ["-"]

        t = response.json()
        all_tags = []
        try:
            tags = t["tags"]
            all_as = BeautifulSoup(str(tags), "lxml").find_all("a")
            for i in all_as:
                all_tags.append(i.contents[0].strip())
            return all_tags
        except KeyError:
            return all_tags

    # --------------------------------------------------------------------------
    @staticmethod
    def get_editorial_link(problem_link):
        """
            Get editorial link given a problem link

            @param problem_link (String): Problem URL
            @return (String/None): Editorial URL
        """
        editorial_link = None
        api_link = problem_link.replace("https://www.codechef.com/", "https://www.codechef.com/api/contests/")
        response = get_request(api_link + "?v=1554915627060",
                               headers={"User-Agent": user_agent})
        if response in REQUEST_FAILURES:
            return None
        response = response.json()
        try:
            editorial_link = response["editorial_url"]
        except KeyError:
            editorial_link = None

        try:
            tags = BeautifulSoup(response["tags"], "lxml").text
            tags = [x.strip(",") for x in tags.split(" ")]
            db = current.db
            ptable = db.problem
            row = db(ptable.link == problem_link).select().first()
            today = datetime.datetime.now().strftime("%Y-%m-%d")
            if row:
                row.update_record(tags=str(tags),
                                  tags_added_on=today)
        except Exception as e:
            pass

        return editorial_link

    # -------------------------------------------------------------------------
    @staticmethod
    def is_invalid_handle(handle):
        # CodeChef is very flaky
        return True
        response = get_request("https://www.codechef.com/users/" + handle)
        if (response in REQUEST_FAILURES) or response.url.__contains__("teams/view"):
            return True
        return False

    # -------------------------------------------------------------------------
    @staticmethod
    def download_submission(view_link):
        if Profile.is_website_down():
            return -1

        problem_id = view_link.strip("/").split("/")[-1]
        download_url = "https://www.codechef.com/viewplaintext/" + \
                       str(problem_id)
        response = get_request(download_url,
                               headers={"User-Agent": user_agent})
        if response in REQUEST_FAILURES:
            return -1
        try:
            return BeautifulSoup(response.text, "lxml").find("pre").text
        except:
            return -1

    # -------------------------------------------------------------------------
    @staticmethod
    def rating_graph_data(handle):
        url = "https://www.codechef.com/users/" + handle
        response = get_request(url, headers={"User-Agent": user_agent})

        if response in REQUEST_FAILURES:
            return response

        ratings = eval(re.search("var all_rating = .*?;", response.text).group()[17:-1].replace("null", "None"))

        long_contest_data = {}
        cookoff_contest_data = {}
        ltime_contest_data = {}

        for contest in ratings:
            this_obj = None
            if contest["code"].__contains__("COOK"):
                # Cook off contest
                this_obj = cookoff_contest_data
            elif contest["code"].__contains__("LTIME"):
                # Lunchtime contest
                this_obj = ltime_contest_data
            else:
                # Long contest
                this_obj = long_contest_data
            # getdate, getmonth and getyear give end_date only
            time_stamp = str(datetime.datetime.strptime(contest["end_date"], "%Y-%m-%d %H:%M:%S"))
            this_obj[time_stamp] = {"name": contest["name"],
                                    "url": "https://www.codechef.com/" + contest["code"],
                                    "rating": str(contest["rating"]),
                                    "rank": contest["rank"]}

        return [{"title": "CodeChef Long",
                 "data": long_contest_data},
                {"title": "CodeChef Cook-off",
                 "data": cookoff_contest_data},
                {"title": "CodeChef Lunchtime",
                 "data": ltime_contest_data}]

    # --------------------------------------------------------------------------
    def __validate_handle(self):
        """
            Make an API request to Codechef to see if a user exists
        """

        response = get_request("%s/users/%s" % (CODECHEF_API_URL, self.handle),
                               headers={"User-Agent": user_agent,
                                        "Authorization": "Bearer %s" % self.access_token},
                               timeout=10,
                               is_daily_retrieval=self.is_daily_retrieval)
        if response in REQUEST_FAILURES:
            return response

        # User was not found in Codechef database
        json_data = response.json()
        if json_data["result"]["data"]["code"] == 9003:
            return NOT_FOUND
        else:
            return "VALID_HANDLE"

    # --------------------------------------------------------------------------
    def __get_access_token(self):
        """
            Get CodeChef API access_token from the database
        """

        # Codechef has a TTL of 1 hour for every generated access token
        TTL_TIME = 60 * 60
        # We are assuming there might be cases when 1 submission retrieval takes
        # more than 10 minutes
        SAFE_TIME = 10 * 60

        redis_key = "codechef_access_token"
        redis_ttl = current.REDIS_CLIENT.ttl(redis_key)
        if redis_ttl is not None and redis_ttl > SAFE_TIME:
            return current.REDIS_CLIENT.get(redis_key)

        response = requests.post("%s/oauth/token" % CODECHEF_API_URL,
                                 data={"grant_type": "client_credentials",
                                       "scope": "public",
                                       "client_id": current.codechef_client_id,
                                       "client_secret": current.codechef_client_secret,
                                       "redirect_uri": ""})
        json_data = response.json()
        if response.status_code == 200 and json_data["status"] == "OK":
            access_token = json_data["result"]["data"]["access_token"]
            current.REDIS_CLIENT.set(redis_key,
                                     access_token,
                                     ex=TTL_TIME)
            return access_token
        else:
            print "Error requesting CodeChef API for access token"
            return

    # --------------------------------------------------------------------------
    def __get_problem_link(self, contest_code, problem_code):
        """
            Get the problem link given the contest_code and problem_code
            @params contest_code (String): Contest code of the problem
            @params problem_code (String): Problem code of the problem

            @return (String): URL of the problem
        """
        return "%s/%s/problems/%s" % (CODECHEF_SITE_URL,
                                      contest_code,
                                      problem_code)

    # --------------------------------------------------------------------------
    def __process_year_submissions(self, year, last_retrieved):
        SUBMISSION_REQUEST_PARAMS["year"] = year
        SUBMISSION_REQUEST_PARAMS["offset"] = 0
        submissions = []
        for _ in xrange(1000):
            response = get_request("%s/submissions" % CODECHEF_API_URL,
                                   headers={"Authorization": "Bearer %s" % self.access_token},
                                   params=SUBMISSION_REQUEST_PARAMS,
                                   timeout=10,
                                   is_daily_retrieval=self.is_daily_retrieval)
            if response in REQUEST_FAILURES:
                return response

            json_response = response.json()
            if json_response["result"]["data"]["code"] == 9003:
                # No submisions for the year
                return submissions

            for submission in json_response["result"]["data"]["content"]:
                curr = time.strptime(submission["date"],
                                     TIME_CONVERSION_STRING)
                if curr <= last_retrieved:
                    self.last_retrieved_reached = True
                    return submissions

                problem_link = self.__get_problem_link(submission["contestCode"],
                                                       submission["problemCode"])
                problem_name = submission["problemCode"]
                status = submission["result"]
                points = submission["score"]
                if status == "AC":
                    if float(points) > 0 and float(points) < 100:
                        status = "PS"
                elif status in ["WA", "TLE"]:
                    pass
                elif status == "CTE":
                    status = "CE"
                elif status == "RTE":
                    status = "RE"
                else:
                    print "*****************", status
                    status = "OTH"
                language = submission["language"]
                view_link = "%s/viewsolution/%d" % (CODECHEF_SITE_URL,
                                                    submission["id"])
                submissions.append((submission["date"],
                                    problem_link,
                                    problem_name,
                                    status,
                                    points,
                                    language,
                                    view_link))
            if len(json_response["result"]["data"]["content"]) < PER_PAGE_LIMIT:
                break

            SUBMISSION_REQUEST_PARAMS["offset"] += PER_PAGE_LIMIT

        return submissions

    # --------------------------------------------------------------------------
    def get_submissions(self, last_retrieved, is_daily_retrieval):
        """
            Retrieve CodeChef submissions after last retrieved timestamp

            @param last_retrieved (DateTime): Last retrieved timestamp for the user
            @param is_daily_retrieval (Boolean): If this call is from daily retrieval cron
            @return (Dict): Dictionary of submissions containing all the
                            information about the submissions
        """

        if current.environment == "development":
            # Locally client credentials for CodeChef API wouldn't be valid and
            # there is no other fallback method which can be used for retrieval
            return SERVER_FAILURE

        self.is_daily_retrieval = is_daily_retrieval
        try:
            # If the handle is a URL return NOT_FOUND
            re.match("https?://.*\.com", self.handle).group()
            return NOT_FOUND
        except AttributeError:
            pass

        start_year = last_retrieved.tm_year
        current_year = datetime.datetime.now().year
        str_init_time = time.strptime(str(current.INITIAL_DATE),
                                      "%Y-%m-%d %H:%M:%S")

        self.access_token = self.__get_access_token()
        if self.access_token is None:
            print "Access token found none"
            return SERVER_FAILURE

        # Test for invalid handles
        if  last_retrieved == str_init_time:
            response = self.__validate_handle()
            if response in REQUEST_FAILURES:
                return response

        SUBMISSION_REQUEST_PARAMS["username"] = self.handle
        self.submissions = []

        for year in xrange(current_year, start_year - 1, -1):
            # Years processed in the reverse order to break out when
            # last_retrieved time_stamp is matched

            result = self.__process_year_submissions(year, last_retrieved)
            if result in REQUEST_FAILURES:
                return result
            else:
                self.submissions.extend(result)
                if self.last_retrieved_reached:
                    return self.submissions

        return self.submissions

# =============================================================================
<|endoftext|>


<commit_before># tasks.py
from iati_synchroniser.models import IatiXmlSource
from django_rq import job
import django_rq
from django.http import HttpResponse
import datetime

###############################
######## WORKER TASKS  ########
###############################

@job
def start_worker(queue_name, amount_of_workers):
    from rq import Queue, Connection, Worker
    import Redis

    redis_conn = Redis()
    queue = Queue('low', connection=redis_conn)

    amount_of_workers += 1

    with Connection():
        workername = "oipa-" + queue_name + "-" + str(amount_of_workers)
        w = Worker(queue, workername)
        w.work()

###############################
######## PARSING TASKS ########
###############################

@job
def parse_all_existing_sources():
    for e in IatiXmlSource.objects.all():
        queue = django_rq.get_queue("parser")
        queue.enqueue(parse_source_by_url, e.source_url)

@job
def get_new_sources_from_iati_api():
    from iati_synchroniser.dataset_syncer import DatasetSyncer
    ds = DatasetSyncer()
    ds.synchronize_with_iati_api(1)

@job
def parse_source_by_url(url):
    if IatiXmlSource.objects.filter(source_url=url).exists():
        xml_source = IatiXmlSource.objects.get(source_url=url)
        xml_source.process()

@job
def parse_all_not_parsed_in_x_days(days):
    for source in IatiXmlSource.objects.all():
        curdate = float(datetime.datetime.now().strftime('%s'))
        last_updated = float(source.date_updated.strftime('%s'))

        update_interval_time = 24 * 60 * 60 * int(days)

        if ((curdate - update_interval_time) > last_updated):
            source.save()

@job
def parse_all_over_parse_interval():
    for source in IatiXmlSource.objects.all():
        curdate = float(datetime.datetime.now().strftime('%s'))
        last_updated = float(source.date_updated.strftime('%s'))
        update_interval = source.update_interval

        if update_interval == "day":
            update_interval_time = 24 * 60 * 60
        if update_interval == "week":
            update_interval_time = 24 * 60 * 60 * 7
        if update_interval == "month":
            update_interval_time = 24 * 60 * 60 * 7 * 4.34
        if update_interval == "year":
            update_interval_time = 24 * 60 * 60 * 365

        if ((curdate - update_interval_time) > last_updated):
            source.save()

###############################
#### IATI MANAGEMENT TASKS ####
###############################

@job
def sync_codelist():
    from iati_synchroniser.codelist_importer import CodeListImporter
    syncer = CodeListImporter()

    syncer.synchronise_with_codelists()

###############################
######## GEODATA TASKS ########
###############################

@job
def update_all_geo_data():
    raise Exception("Not implemented yet")

###############################
######## INDICATOR TASKS ########
###############################

@job
def update_all_indicator_data():
    raise Exception("Not implemented yet")

###############################
######## CACHING TASKS ########
###############################

@job
def update_existing_api_call_caches():
    from cache.validator import Validator
    v = Validator()
    v.update_cache_calls()

@job
def cache_long_api_calls():
    from cache.validator import Validator
    v = Validator()
    v.update_response_times_and_add_to_cache()

def delete_task_from_queue(job_id):
    from rq import cancel_job
    from rq import Connection
    with Connection():
        cancel_job(job_id)

def delete_all_tasks_from_queue(queue_name):
    if queue_name == "failed":
        q = django_rq.get_failed_queue()
    elif queue_name == "parser":
        q = django_rq.get_queue("parser")
    else:
        q = django_rq.get_queue("default")

    while True:
        job = q.dequeue()
        if not job:
            break
        job.delete()<commit_msg>worker bugfix #3<commit_after># tasks.py
from iati_synchroniser.models import IatiXmlSource
from django_rq import job
import django_rq
from django.http import HttpResponse
import datetime

###############################
######## WORKER TASKS  ########
###############################

@job
def start_worker(queue_name, amount_of_workers):
    from rq import Queue, Connection, Worker
    from redis import Redis

    redis_conn = Redis()
    queue = Queue('low', connection=redis_conn)

    amount_of_workers += 1

    with Connection():
        workername = "oipa-" + queue_name + "-" + str(amount_of_workers)
        w = Worker(queue, workername)
        w.work()

###############################
######## PARSING TASKS ########
###############################

@job
def parse_all_existing_sources():
    for e in IatiXmlSource.objects.all():
        queue = django_rq.get_queue("parser")
        queue.enqueue(parse_source_by_url, e.source_url)

@job
def get_new_sources_from_iati_api():
    from iati_synchroniser.dataset_syncer import DatasetSyncer
    ds = DatasetSyncer()
    ds.synchronize_with_iati_api(1)

@job
def parse_source_by_url(url):
    if IatiXmlSource.objects.filter(source_url=url).exists():
        xml_source = IatiXmlSource.objects.get(source_url=url)
        xml_source.process()

@job
def parse_all_not_parsed_in_x_days(days):
    for source in IatiXmlSource.objects.all():
        curdate = float(datetime.datetime.now().strftime('%s'))
        last_updated = float(source.date_updated.strftime('%s'))

        update_interval_time = 24 * 60 * 60 * int(days)

        if ((curdate - update_interval_time) > last_updated):
            source.save()

@job
def parse_all_over_parse_interval():
    for source in IatiXmlSource.objects.all():
        curdate = float(datetime.datetime.now().strftime('%s'))
        last_updated = float(source.date_updated.strftime('%s'))
        update_interval = source.update_interval

        if update_interval == "day":
            update_interval_time = 24 * 60 * 60
        if update_interval == "week":
            update_interval_time = 24 * 60 * 60 * 7
        if update_interval == "month":
            update_interval_time = 24 * 60 * 60 * 7 * 4.34
        if update_interval == "year":
            update_interval_time = 24 * 60 * 60 * 365

        if ((curdate - update_interval_time) > last_updated):
            source.save()

###############################
#### IATI MANAGEMENT TASKS ####
###############################

@job
def sync_codelist():
    from iati_synchroniser.codelist_importer import CodeListImporter
    syncer = CodeListImporter()

    syncer.synchronise_with_codelists()

###############################
######## GEODATA TASKS ########
###############################

@job
def update_all_geo_data():
    raise Exception("Not implemented yet")

###############################
######## INDICATOR TASKS ########
###############################

@job
def update_all_indicator_data():
    raise Exception("Not implemented yet")

###############################
######## CACHING TASKS ########
###############################

@job
def update_existing_api_call_caches():
    from cache.validator import Validator
    v = Validator()
    v.update_cache_calls()

@job
def cache_long_api_calls():
    from cache.validator import Validator
    v = Validator()
    v.update_response_times_and_add_to_cache()

def delete_task_from_queue(job_id):
    from rq import cancel_job
    from rq import Connection
    with Connection():
        cancel_job(job_id)

def delete_all_tasks_from_queue(queue_name):
    if queue_name == "failed":
        q = django_rq.get_failed_queue()
    elif queue_name == "parser":
        q = django_rq.get_queue("parser")
    else:
        q = django_rq.get_queue("default")

    while True:
        job = q.dequeue()
        if not job:
            break
        job.delete()<|endoftext|>


<commit_before>from django.contrib.admin.views.decorators import staff_member_required
from django.http import HttpResponse
import json
import django_rq
from django_rq import get_connection
from rq import requeue_job
from rq import get_failed_queue
from rq import Worker
from rq_scheduler import Scheduler
from rq.registry import FinishedJobRegistry
from task_queue import tasks
from redis import Redis
from rq.exceptions import NoSuchJobError
from rq.job import Job

from math import ceil

redis_conn = Redis()

# PARSE TASKS
@staff_member_required
def add_task(request):
    task = request.GET.get('task')
    parameters = request.GET.get('parameters')
    queue_to_be_added_to = request.GET.get('queue')
    queue = django_rq.get_queue(queue_to_be_added_to)

    if parameters:
        queue.enqueue(getattr(tasks, task), args=(parameters,), timeout=7200)
    else:
        queue.enqueue(getattr(tasks, task), timeout=7200)
    return HttpResponse('Success')

# TASK QUEUE MANAGEMENT
@staff_member_required
def start_worker_with_supervisor(request):
    from django.core.management import call_command

    action = request.GET.get('action')
    worker_program = request.GET.get('worker_program')

    list = [action, worker_program]
    call_command('supervisor', *list)

    return HttpResponse('Success')

@staff_member_required
def get_workers(request):

    workers = Worker.all(connection=redis_conn)

    workerdata = list()
    # serialize workers
    for w in workers:
        cj = w.get_current_job()

        if cj:
            cjinfo = {
                'id': cj.id,
                'args': cj.args,
                'enqueued_at': cj.enqueued_at.strftime("%a, %d %b %Y %H:%M:%S +0000"),
                'description': cj.description}
        else:
            cjinfo = None

        worker_dict = {
            'pid': w.pid,
            'name': w.name,
            'state': w.get_state(),
            'current_job': cjinfo}

        workerdata.append(worker_dict)
    data = json.dumps(workerdata)
    return HttpResponse(data, content_type='application/json')

@staff_member_required
def delete_task_from_queue(request):
    job_id = request.GET.get('job_id')
    tasks.delete_task_from_queue(job_id)
    return HttpResponse('Success')

@staff_member_required
def delete_all_tasks_from_queue(request):
    queue_name = request.GET.get('queue_name')
    tasks.delete_all_tasks_from_queue(queue_name)
    return HttpResponse('Success')

@staff_member_required
def get_current_job(request):
    from rq import get_current_job
    from rq import Queue

    q = Queue(connection=redis_conn)
    job = get_current_job(q)
    import json
    data = json.dumps(job)
    return HttpResponse(data, content_type='application/json')

# Schedule management
@staff_member_required
def start_scheduler(request):
    from rq_scheduler.scripts import rqscheduler
    rqscheduler.main()
    return HttpResponse('Success')

@staff_member_required
def add_scheduled_task(request):
    from redis import Redis
    from rq_scheduler import Scheduler
    from datetime import datetime

    task = request.GET.get('task')
    period = request.GET.get('period')
    queue = request.GET.get('queue')
    parameters = request.GET.get('parameters')
    scheduler = Scheduler(queue_name=queue, connection=Redis())

    if parameters:
        scheduler.schedule(
            scheduled_time=datetime.utcnow(),   # Time for first execution
            func=getattr(tasks, task),       # Function to be queued
            args=[int(parameters)],
            interval=int(period),                 # Time before the function is called again, in seconds
            repeat=None                      # Repeat this number of times (None means repeat forever)
        )
    else:
        scheduler.schedule(
            scheduled_time=datetime.utcnow(),   # Time for first execution
            func=getattr(tasks, task),       # Function to be queued
            interval=int(period),                 # Time before the function is called again, in seconds
            repeat=None                      # Repeat this number of times (None means repeat forever)
        )
    return HttpResponse('Success')

@staff_member_required
def get_queue(request):

    current_queue = request.GET.get('queue')
    queue = django_rq.get_queue(current_queue)
    jobdata = list()

    count_jobs = 0
    for job in queue.jobs:
        count_jobs += 1
        if count_jobs == 20:
            break

        job_dict = {
            'job_id': job._id,
            'created_at': job.created_at.strftime("%a, %d %b %Y %H:%M:%S +0000"),
            'enqueued_at': job.enqueued_at.strftime("%a, %d %b %Y %H:%M:%S +0000"),
            'status': job.get_status(),
            'function': job.func_name,
            'args': job.args}

        jobdata.append(job_dict)
    data = json.dumps(jobdata)
    return HttpResponse(data, content_type='application/json')

@staff_member_required
def get_scheduled_tasks(request):

    # Use RQ's default Redis connection
    # use_connection()
    # Get a scheduler for the "default" queue
    scheduler = Scheduler(connection=redis_conn)
    list_of_job_instances = scheduler.get_jobs()

    jobdata = list()
    for job in list_of_job_instances:
        if "interval" in job.meta:
            interval = job.meta["interval"]
        else:
            interval = 0

        job_dict = {
            'job_id': job._id,
            'task': job.description,
            'period': interval,
            'args': job.args,
            'queue': "default"}

        jobdata.append(job_dict)

    data = json.dumps(jobdata)
    return HttpResponse(data, content_type='application/json')

@staff_member_required
def cancel_scheduled_task(request):
    job_id = request.GET.get('job_id')
    from rq_scheduler import Scheduler

    scheduler = Scheduler('parser')
    scheduler.cancel(job_id)
    return HttpResponse('Success')

@staff_member_required
def get_failed_tasks(request):

    queue = django_rq.get_failed_queue()

    jobdata = list()
    for job in queue.jobs:

        job_dict = {
            'job_id': job.id,
            'func_name': job.func_name,
            'error_message': job.exc_info,
            'ended_at': job.ended_at.strftime("%a, %d %b %Y %H:%M:%S +0000"),
            'enqueued_at': job.enqueued_at.strftime("%a, %d %b %Y %H:%M:%S +0000")}

        jobdata.append(job_dict)

    data = json.dumps(jobdata)
    return HttpResponse(data, content_type='application/json')

@staff_member_required
def finished_jobs(request):

    current_queue = request.GET.get('queue')
    queue = django_rq.get_queue(current_queue)
    registry = FinishedJobRegistry(queue.name, queue.connection)

    items_per_page = 10
    num_jobs = len(registry)
    jobs = []

    if num_jobs > 0:
        offset = 0
        job_ids = registry.get_job_ids(offset, items_per_page)

        for job_id in job_ids:
            try:
                jobs.append(Job.fetch(job_id, connection=queue.connection))
            except NoSuchJobError:
                pass

    jobdata = list()
    for job in jobs:

        job_dict = {
            'job_id': job.id,
            'func_name': job.func_name,
            'error_message': job.exc_info,
            'ended_at': job.ended_at.strftime("%a, %d %b %Y %H:%M:%S +0000"),
            'enqueued_at': job.enqueued_at.strftime("%a, %d %b %Y %H:%M:%S +0000")}

        jobdata.append(job_dict)

    data = json.dumps(jobdata)
    return HttpResponse(data, content_type='application/json')

@staff_member_required
def reschedule_all_failed(request):

    queue = get_failed_queue(get_connection())

    for job in queue.jobs:
        requeue_job(job.id, connection=queue.connection)

    return HttpResponse('Success')

<commit_msg>make failed/completed tasks show the source url<commit_after>from django.contrib.admin.views.decorators import staff_member_required
from django.http import HttpResponse
import json
import django_rq
from django_rq import get_connection
from rq import requeue_job
from rq import get_failed_queue
from rq import Worker
from rq_scheduler import Scheduler
from rq.registry import FinishedJobRegistry
from task_queue import tasks
from redis import Redis
from rq.exceptions import NoSuchJobError
from rq.job import Job

from math import ceil

redis_conn = Redis()

# PARSE TASKS
@staff_member_required
def add_task(request):
    task = request.GET.get('task')
    parameters = request.GET.get('parameters')
    queue_to_be_added_to = request.GET.get('queue')
    queue = django_rq.get_queue(queue_to_be_added_to)

    if parameters:
        queue.enqueue(getattr(tasks, task), args=(parameters,), timeout=7200)
    else:
        queue.enqueue(getattr(tasks, task), timeout=7200)
    return HttpResponse('Success')

# TASK QUEUE MANAGEMENT
@staff_member_required
def start_worker_with_supervisor(request):
    from django.core.management import call_command

    action = request.GET.get('action')
    worker_program = request.GET.get('worker_program')

    list = [action, worker_program]
    call_command('supervisor', *list)

    return HttpResponse('Success')

@staff_member_required
def get_workers(request):

    workers = Worker.all(connection=redis_conn)

    workerdata = list()
    # serialize workers
    for w in workers:
        cj = w.get_current_job()

        if cj:
            cjinfo = {
                'id': cj.id,
                'args': cj.args,
                'enqueued_at': cj.enqueued_at.strftime("%a, %d %b %Y %H:%M:%S +0000"),
                'description': cj.description}
        else:
            cjinfo = None

        worker_dict = {
            'pid': w.pid,
            'name': w.name,
            'state': w.get_state(),
            'current_job': cjinfo}

        workerdata.append(worker_dict)
    data = json.dumps(workerdata)
    return HttpResponse(data, content_type='application/json')

@staff_member_required
def delete_task_from_queue(request):
    job_id = request.GET.get('job_id')
    tasks.delete_task_from_queue(job_id)
    return HttpResponse('Success')

@staff_member_required
def delete_all_tasks_from_queue(request):
    queue_name = request.GET.get('queue_name')
    tasks.delete_all_tasks_from_queue(queue_name)
    return HttpResponse('Success')

@staff_member_required
def get_current_job(request):
    from rq import get_current_job
    from rq import Queue

    q = Queue(connection=redis_conn)
    job = get_current_job(q)
    import json
    data = json.dumps(job)
    return HttpResponse(data, content_type='application/json')

# Schedule management
@staff_member_required
def start_scheduler(request):
    from rq_scheduler.scripts import rqscheduler
    rqscheduler.main()
    return HttpResponse('Success')

@staff_member_required
def add_scheduled_task(request):
    from redis import Redis
    from rq_scheduler import Scheduler
    from datetime import datetime

    task = request.GET.get('task')
    period = request.GET.get('period')
    queue = request.GET.get('queue')
    parameters = request.GET.get('parameters')
    scheduler = Scheduler(queue_name=queue, connection=Redis())

    if parameters:
        scheduler.schedule(
            scheduled_time=datetime.utcnow(),   # Time for first execution
            func=getattr(tasks, task),       # Function to be queued
            args=[int(parameters)],
            interval=int(period),                 # Time before the function is called again, in seconds
            repeat=None                      # Repeat this number of times (None means repeat forever)
        )
    else:
        scheduler.schedule(
            scheduled_time=datetime.utcnow(),   # Time for first execution
            func=getattr(tasks, task),       # Function to be queued
            interval=int(period),                 # Time before the function is called again, in seconds
            repeat=None                      # Repeat this number of times (None means repeat forever)
        )
    return HttpResponse('Success')

@staff_member_required
def get_queue(request):

    current_queue = request.GET.get('queue')
    queue = django_rq.get_queue(current_queue)
    jobdata = list()

    count_jobs = 0
    for job in queue.jobs:
        count_jobs += 1
        if count_jobs == 20:
            break

        job_dict = {
            'job_id': job._id,
            'created_at': job.created_at.strftime("%a, %d %b %Y %H:%M:%S +0000"),
            'enqueued_at': job.enqueued_at.strftime("%a, %d %b %Y %H:%M:%S +0000"),
            'status': job.get_status(),
            'function': job.func_name,
            'args': job.args}

        jobdata.append(job_dict)
    data = json.dumps(jobdata)
    return HttpResponse(data, content_type='application/json')

@staff_member_required
def get_scheduled_tasks(request):

    # Use RQ's default Redis connection
    # use_connection()
    # Get a scheduler for the "default" queue
    scheduler = Scheduler(connection=redis_conn)
    list_of_job_instances = scheduler.get_jobs()

    jobdata = list()
    for job in list_of_job_instances:
        if "interval" in job.meta:
            interval = job.meta["interval"]
        else:
            interval = 0

        job_dict = {
            'job_id': job._id,
            'task': job.description,
            'period': interval,
            'args': job.args,
            'queue': "default"}

        jobdata.append(job_dict)

    data = json.dumps(jobdata)
    return HttpResponse(data, content_type='application/json')

@staff_member_required
def cancel_scheduled_task(request):
    job_id = request.GET.get('job_id')
    from rq_scheduler import Scheduler

    scheduler = Scheduler('parser')
    scheduler.cancel(job_id)
    return HttpResponse('Success')

@staff_member_required
def get_failed_tasks(request):

    queue = django_rq.get_failed_queue()

    jobdata = list()
    for job in queue.jobs:

        job_dict = {
            'job_id': job.id,
            'func_name': job.description,
            'error_message': job.exc_info,
            'ended_at': job.ended_at.strftime("%a, %d %b %Y %H:%M:%S +0000"),
            'enqueued_at': job.enqueued_at.strftime("%a, %d %b %Y %H:%M:%S +0000")}

        jobdata.append(job_dict)

    data = json.dumps(jobdata)
    return HttpResponse(data, content_type='application/json')

@staff_member_required
def finished_jobs(request):

    current_queue = request.GET.get('queue')
    queue = django_rq.get_queue(current_queue)
    registry = FinishedJobRegistry(queue.name, queue.connection)

    items_per_page = 10
    num_jobs = len(registry)
    jobs = []

    if num_jobs > 0:
        offset = 0
        job_ids = registry.get_job_ids(offset, items_per_page)

        for job_id in job_ids:
            try:
                jobs.append(Job.fetch(job_id, connection=queue.connection))
            except NoSuchJobError:
                pass

    jobdata = list()
    for job in jobs:

        job_dict = {
            'job_id': job.id,
            'func_name': job.description,
            'error_message': job.exc_info,
            'ended_at': job.ended_at.strftime("%a, %d %b %Y %H:%M:%S +0000"),
            'enqueued_at': job.enqueued_at.strftime("%a, %d %b %Y %H:%M:%S +0000")}

        jobdata.append(job_dict)

    data = json.dumps(jobdata)
    return HttpResponse(data, content_type='application/json')

@staff_member_required
def reschedule_all_failed(request):

    queue = get_failed_queue(get_connection())

    for job in queue.jobs:
        requeue_job(job.id, connection=queue.connection)

    return HttpResponse('Success')

<|endoftext|>


<commit_before>from django.utils.cache import add_never_cache_headers
from django.views.generic import View

class RiffMixin(View):
    riff = None
    cacheable = False

    def dispatch(self, request, *args, **kwargs)
        self.request = request
        if not self.permission_check()
            return self.get_unauthorized_response()

        response = super(RiffMixin, self).dispatch(request, *args, **kwargs)
        if not self.cacheable:
            add_never_cache_headers(response)
        return response

    def permission_check(self):
        self.riff.permission_check(self.request)

    def get_unauthorized_response(self):
        return self.riff.get_unauthorized_respnse(self.request)

<commit_msg>Syntax corrections to views.<commit_after>from django.utils.cache import add_never_cache_headers
from django.views.generic import View

class RiffMixin(View):
    riff = None
    cacheable = False

    def dispatch(self, request, *args, **kwargs):
        self.request = request
        if not self.permission_check():
            return self.get_unauthorized_response()

        response = super(RiffMixin, self).dispatch(request, *args, **kwargs)
        if not self.cacheable:
            add_never_cache_headers(response)
        return response

    def permission_check(self):
        self.riff.permission_check(self.request)

    def get_unauthorized_response(self):
        return self.riff.get_unauthorized_respnse(self.request)

<|endoftext|>