<commit_before>#!/usr/bin/env python
# PyChef.py
#
#   Copyright (C) 2009 Diamond Light Source, Graeme Winter
#
#   This code is distributed under the BSD license, a copy of which is
#   included in the root directory of this package.
#
# Main program for PyChef, using the PyChef class. Here is some example input:
#
# labin BASE=DOSE
# range width 5 max 605
# anomalous on
# resolution 1.9
#
# This is the same program interface as used to come from the program CHEF.
#

import os
import sys
import time

from PyChef import PyChef

def banner():
    version = '1.0'
    user = os.environ.get('USER', '')
    now = time.asctime()

    print '#' * 60
    print '#' * 60
    print '#' * 60
    print '### PyCHEF                                           %s ###' % \
          version
    print '#' * 60
    print 'User: %s                  Run at: %s' % (user, now)

    return

def get_hklin_files():
    '''From the command-line, get the list of hklin files. Set up thus
    as it may be useful externally. Assumes that the list of reflection
    files is passed in as HKLIN1 infl.mtz etc. N.B. could well be the case
    that HKLIN infl.mtz HKLIN lrem.mtz works just as well, though thats
    just a side-effect.'''

    hklin_files = []

    for j in range(1, len(sys.argv)):
        if 'HKLIN' in sys.argv[j].upper()[:5]:
            hklin = sys.argv[j + 1]

            if not os.path.abspath(hklin):
                hklin = os.path.join(os.getcwd(), hklin)

            if not os.path.exists(hklin):
                raise RuntimeError, 'hklin %s does not exist' % hklin

            hklin_files.append(hklin)

    for hklin in hklin_files:
        print 'HKLIN: %s' % hklin

    return hklin_files

def get_number_cpus():
    '''Portably get the number of processor cores available.'''

    # Windows NT derived platforms

    if os.name == 'nt':
        return int(os.environ['NUMBER_OF_PROCESSORS'])

    # linux

    if os.path.exists('/proc/cpuinfo'):
        n_cpu = 0

        for record in open('/proc/cpuinfo', 'r').readlines():
            if not record.strip():
                continue
            if 'processor' in record.split()[0]:
                n_cpu += 1

        return n_cpu

    # os X

    output = subprocess.Popen(['system_profiler', 'SPHardwareDataType'],
                              stdout = subprocess.PIPE).communicate()[0]
    for record in output.split('\n'):
        if 'Total Number Of Cores' in record:
            return int(record.split()[-1])

    return -1

def parse_standard_input():
    '''Read and parse the standard input. Return as a dictionary.'''

    range_min = 0.0
    range_max = None
    range_width = None

    anomalous = False

    resolution_low = None
    resolution_high = None

    base_column = None

    title = None

    ncpu = get_number_cpus()

    for record in sys.stdin.readlines():

        record = record.split('!')[0].split('#')[0].strip()

        if not record:
            continue

        print '> %s' % record

        key = record[:4].upper()
        tokens = record.split()

        #### KEYWORD RESOLUTION ####

        if key == 'RESO':
            assert(len(tokens) < 4)
            assert(len(tokens) > 1)

            if len(tokens) == 2:
                resolution_high = float(tokens[1])

            elif len(tokens) == 3:
                resolution_a = float(tokens[1])
                resolution_b = float(tokens[2])
                resolution_high = min(resolution_a, resolution_b)
                resolution_low = max(resolution_a, resolution_b)

        #### KEYWORD RANGE ####

        elif key == 'RANG':
            assert(len(tokens) < 8)
            assert(len(tokens) > 2)

            for j in range(1, len(tokens)):
                subkey = tokens[j][:4].upper()
                if subkey == 'MIN':
                    range_min = float(tokens[j + 1])
                elif subkey == 'MAX':
                    range_max = float(tokens[j + 1])
                if subkey == 'WIDT':
                    range_width = float(tokens[j + 1])

        #### KEYWORD ANOMALOUS ####

        elif key == 'ANOM':
            assert(len(tokens) < 3)

            anomalous = True

            if len(tokens) > 1:
                if tokens[1].upper() == 'OFF' or \
                   tokens[1].upper() == 'FALSE' or \
                   tokens[1].upper() == 'NO':
                    anomalous = False

        #### KEYWORD LABIN ####

        elif key == 'LABI':
            assert(len(tokens) == 2)
            assert('BASE=' in tokens[1])

            base_column = tokens[1].replace('BASE=', '')

        #### KEYWORD TITLE ####

        elif key == 'TITL':
            keyword = tokens[0]
            title = record[len(keyword):].strip()

        #### KEYWORD NCPU

        elif key == 'NCPU':
            ncpu = int(record.split()[-1])

    # check that these values are sound - where they are needed...
    # assert(base_column)

    # if we have BATCH we can guess the limit and range...
    # assert(range_max)
    # assert(range_width)

    # now drop this lot to a dictionary

    results = {
        'range_min':range_min,
        'range_max':range_max,
        'range_width':range_width,
        'anomalous':anomalous,
        'resolution_high':resolution_high,
        'resolution_low':resolution_low,
        'base_column':base_column,
        'title':title,
        'ncpu':ncpu
        }

    return results

def main():
    '''Create and run a PyChef, reading the traditional format input from
    the command-line.'''

    banner()

    start_time = time.time()

    hklin_list = get_hklin_files()
    standard_input = parse_standard_input()

    pychef = PyChef()

    # copy the information across from the standard input dictionary to
    # the PyChef instance.

    pychef.set_ncpu(standard_input['ncpu'])

    print 'Using %d threads' % standard_input['ncpu']

    if standard_input['base_column']:
        pychef.set_base_column(standard_input['base_column'])

    if standard_input['range_max']:
        pychef.set_range(standard_input['range_min'],
                         standard_input['range_max'],
                         standard_input['range_width'])

    pychef.set_resolution(standard_input['resolution_high'],
                          standard_input['resolution_low'])
    pychef.set_anomalous(standard_input['anomalous'])

    if standard_input['title']:
        pychef.set_title(standard_input['title'])

    for hklin in hklin_list:
        pychef.add_hklin(hklin)

    # right, all set up - let's run some analysis - first the completeness
    # vs. dose for each input reflection file

    pychef.init()
    pychef.print_completeness_vs_dose()
    pychef.rcp()
    pychef.scp()
    pychef.rd()
    pychef.print_dose_profile()

    print ' PyChef: ** Normal termination **'
    print ' Times: Elapsed: %.1f' % (time.time() - start_time)

    return

if __name__ == '__main__':

    if False:

        import profile
        profile.run('main()')

    else:
        main()

<commit_msg>Use subprocess on mac, didn't import - D'oh! Thanks to Jim N for spotting this<commit_after>#!/usr/bin/env python
# PyChef.py
#
#   Copyright (C) 2009 Diamond Light Source, Graeme Winter
#
#   This code is distributed under the BSD license, a copy of which is
#   included in the root directory of this package.
#
# Main program for PyChef, using the PyChef class. Here is some example input:
#
# labin BASE=DOSE
# range width 5 max 605
# anomalous on
# resolution 1.9
#
# This is the same program interface as used to come from the program CHEF.
#

import os
import sys
import time
import subprocess

from PyChef import PyChef

def banner():
    version = '1.0'
    user = os.environ.get('USER', '')
    now = time.asctime()

    print '#' * 60
    print '#' * 60
    print '#' * 60
    print '### PyCHEF                                           %s ###' % \
          version
    print '#' * 60
    print 'User: %s                  Run at: %s' % (user, now)

    return

def get_hklin_files():
    '''From the command-line, get the list of hklin files. Set up thus
    as it may be useful externally. Assumes that the list of reflection
    files is passed in as HKLIN1 infl.mtz etc. N.B. could well be the case
    that HKLIN infl.mtz HKLIN lrem.mtz works just as well, though thats
    just a side-effect.'''

    hklin_files = []

    for j in range(1, len(sys.argv)):
        if 'HKLIN' in sys.argv[j].upper()[:5]:
            hklin = sys.argv[j + 1]

            if not os.path.abspath(hklin):
                hklin = os.path.join(os.getcwd(), hklin)

            if not os.path.exists(hklin):
                raise RuntimeError, 'hklin %s does not exist' % hklin

            hklin_files.append(hklin)

    for hklin in hklin_files:
        print 'HKLIN: %s' % hklin

    return hklin_files

def get_number_cpus():
    '''Portably get the number of processor cores available.'''

    # Windows NT derived platforms

    if os.name == 'nt':
        return int(os.environ['NUMBER_OF_PROCESSORS'])

    # linux

    if os.path.exists('/proc/cpuinfo'):
        n_cpu = 0

        for record in open('/proc/cpuinfo', 'r').readlines():
            if not record.strip():
                continue
            if 'processor' in record.split()[0]:
                n_cpu += 1

        return n_cpu

    # os X

    output = subprocess.Popen(['system_profiler', 'SPHardwareDataType'],
                              stdout = subprocess.PIPE).communicate()[0]
    for record in output.split('\n'):
        if 'Total Number Of Cores' in record:
            return int(record.split()[-1])

    return -1

def parse_standard_input():
    '''Read and parse the standard input. Return as a dictionary.'''

    range_min = 0.0
    range_max = None
    range_width = None

    anomalous = False

    resolution_low = None
    resolution_high = None

    base_column = None

    title = None

    ncpu = get_number_cpus()

    for record in sys.stdin.readlines():

        record = record.split('!')[0].split('#')[0].strip()

        if not record:
            continue

        print '> %s' % record

        key = record[:4].upper()
        tokens = record.split()

        #### KEYWORD RESOLUTION ####

        if key == 'RESO':
            assert(len(tokens) < 4)
            assert(len(tokens) > 1)

            if len(tokens) == 2:
                resolution_high = float(tokens[1])

            elif len(tokens) == 3:
                resolution_a = float(tokens[1])
                resolution_b = float(tokens[2])
                resolution_high = min(resolution_a, resolution_b)
                resolution_low = max(resolution_a, resolution_b)

        #### KEYWORD RANGE ####

        elif key == 'RANG':
            assert(len(tokens) < 8)
            assert(len(tokens) > 2)

            for j in range(1, len(tokens)):
                subkey = tokens[j][:4].upper()
                if subkey == 'MIN':
                    range_min = float(tokens[j + 1])
                elif subkey == 'MAX':
                    range_max = float(tokens[j + 1])
                if subkey == 'WIDT':
                    range_width = float(tokens[j + 1])

        #### KEYWORD ANOMALOUS ####

        elif key == 'ANOM':
            assert(len(tokens) < 3)

            anomalous = True

            if len(tokens) > 1:
                if tokens[1].upper() == 'OFF' or \
                   tokens[1].upper() == 'FALSE' or \
                   tokens[1].upper() == 'NO':
                    anomalous = False

        #### KEYWORD LABIN ####

        elif key == 'LABI':
            assert(len(tokens) == 2)
            assert('BASE=' in tokens[1])

            base_column = tokens[1].replace('BASE=', '')

        #### KEYWORD TITLE ####

        elif key == 'TITL':
            keyword = tokens[0]
            title = record[len(keyword):].strip()

        #### KEYWORD NCPU

        elif key == 'NCPU':
            ncpu = int(record.split()[-1])

    # check that these values are sound - where they are needed...
    # assert(base_column)

    # if we have BATCH we can guess the limit and range...
    # assert(range_max)
    # assert(range_width)

    # now drop this lot to a dictionary

    results = {
        'range_min':range_min,
        'range_max':range_max,
        'range_width':range_width,
        'anomalous':anomalous,
        'resolution_high':resolution_high,
        'resolution_low':resolution_low,
        'base_column':base_column,
        'title':title,
        'ncpu':ncpu
        }

    return results

def main():
    '''Create and run a PyChef, reading the traditional format input from
    the command-line.'''

    banner()

    start_time = time.time()

    hklin_list = get_hklin_files()
    standard_input = parse_standard_input()

    pychef = PyChef()

    # copy the information across from the standard input dictionary to
    # the PyChef instance.

    pychef.set_ncpu(standard_input['ncpu'])

    print 'Using %d threads' % standard_input['ncpu']

    if standard_input['base_column']:
        pychef.set_base_column(standard_input['base_column'])

    if standard_input['range_max']:
        pychef.set_range(standard_input['range_min'],
                         standard_input['range_max'],
                         standard_input['range_width'])

    pychef.set_resolution(standard_input['resolution_high'],
                          standard_input['resolution_low'])
    pychef.set_anomalous(standard_input['anomalous'])

    if standard_input['title']:
        pychef.set_title(standard_input['title'])

    for hklin in hklin_list:
        pychef.add_hklin(hklin)

    # right, all set up - let's run some analysis - first the completeness
    # vs. dose for each input reflection file

    pychef.init()
    pychef.print_completeness_vs_dose()
    pychef.rcp()
    pychef.scp()
    pychef.rd()
    pychef.print_dose_profile()

    print ' PyChef: ** Normal termination **'
    print ' Times: Elapsed: %.1f' % (time.time() - start_time)

    return

if __name__ == '__main__':

    if False:

        import profile
        profile.run('main()')

    else:
        main()

<|endoftext|>


<commit_before>#
# Thierry Parmentelat - INRIA
#
from PLC.Faults import *
from PLC.Method import Method
from PLC.Parameter import Parameter, Mixed
from PLC.Filter import Filter
from PLC.Auth import Auth

from PLC.Persons import Person, Persons
from PLC.PersonTags import PersonTag, PersonTags

class GetPersonTags(Method):
    """
    Returns an array of structs containing details about
    persons and related settings.

    If person_tag_filter is specified and is an array of
    person setting identifiers, only person settings matching
    the filter will be returned. If return_fields is specified, only
    the specified details will be returned.
    """

    roles = ['admin', 'pi', 'user', 'tech']

    accepts = [
        Auth(),
        Mixed([PersonTag.fields['person_tag_id']],
              Parameter(int,"Person setting id"),
              Filter(PersonTag.fields)),
        Parameter([str], "List of fields to return", nullok = True)
        ]

    returns = [PersonTag.fields]

    def call(self, auth, person_tag_filter = None, return_fields = None):

        # only persons can call this (as per roles, but..)
        if not isinstance(self.caller,Person):
            return []

        # If we are not admin, make sure to only return viewable accounts
        valid_person_ids=None
        added_fields=[]
        if 'admin' not in self.caller['roles']:
            # Get accounts that we are able to view
            valid_person_ids = [self.caller['person_id']]
            if 'pi' in self.caller['roles'] and self.caller['site_ids']:
                sites = Sites(self.api, self.caller['site_ids'])
                for site in sites:
                    valid_person_ids += site['person_ids']

            if not valid_person_ids:
                return []

            # if we have to filter out on person_id, make sure this is returned from db
            if return_fields:
                added_fields = set(['person_id']).difference(return_fields)
                return_fields += added_fields

        person_tags = PersonTags(self.api, person_tag_filter, return_fields)

        if valid_person_ids is not None:
            person_tags = [ person_tag for person_tag in person_tags
                            if person_tag['person_id'] in valid_person_ids]

        # Remove added fields if not initially specified
        if added_fields:
            for person_tag in person_tags:
                for field in added_fields:
                    if field in person_tag:
                        del person_tag[field]
        return person_tags
<commit_msg>missing import<commit_after>#
# Thierry Parmentelat - INRIA
#
from PLC.Faults import *
from PLC.Method import Method
from PLC.Parameter import Parameter, Mixed
from PLC.Filter import Filter
from PLC.Auth import Auth

from PLC.Persons import Person, Persons
from PLC.PersonTags import PersonTag, PersonTags
from PLC.Sites import Sites, Site

class GetPersonTags(Method):
    """
    Returns an array of structs containing details about
    persons and related settings.

    If person_tag_filter is specified and is an array of
    person setting identifiers, only person settings matching
    the filter will be returned. If return_fields is specified, only
    the specified details will be returned.
    """

    roles = ['admin', 'pi', 'user', 'tech']

    accepts = [
        Auth(),
        Mixed([PersonTag.fields['person_tag_id']],
              Parameter(int,"Person setting id"),
              Filter(PersonTag.fields)),
        Parameter([str], "List of fields to return", nullok = True)
        ]

    returns = [PersonTag.fields]

    def call(self, auth, person_tag_filter = None, return_fields = None):

        # only persons can call this (as per roles, but..)
        if not isinstance(self.caller,Person):
            return []

        # If we are not admin, make sure to only return viewable accounts
        valid_person_ids=None
        added_fields=[]
        if 'admin' not in self.caller['roles']:
            # Get accounts that we are able to view
            valid_person_ids = [self.caller['person_id']]
            if 'pi' in self.caller['roles'] and self.caller['site_ids']:
                sites = Sites(self.api, self.caller['site_ids'])
                for site in sites:
                    valid_person_ids += site['person_ids']

            if not valid_person_ids:
                return []

            # if we have to filter out on person_id, make sure this is returned from db
            if return_fields:
                added_fields = set(['person_id']).difference(return_fields)
                return_fields += added_fields

        person_tags = PersonTags(self.api, person_tag_filter, return_fields)

        if valid_person_ids is not None:
            person_tags = [ person_tag for person_tag in person_tags
                            if person_tag['person_id'] in valid_person_ids]

        # Remove added fields if not initially specified
        if added_fields:
            for person_tag in person_tags:
                for field in added_fields:
                    if field in person_tag:
                        del person_tag[field]
        return person_tags
<|endoftext|>


<commit_before>import os
import re
import json
from datetime import datetime
from datetime import timedelta
from bs4 import BeautifulSoup

from POICrawler.diner import Diner
from POICrawler.requester import Requester
from POICrawler.address import Address

class DinerCrawler(object):

    def __init__(self):
        self.session = Requester()

    # Subclasses must implement this method,
    # return list of Diners
    def crawl(self):
        raise NotImplementedError

class FoodyVNCrawler(DinerCrawler):

    price_re = re.compile('\d+\.\d+đ')

    def __init__(self):
        super(FoodyVNCrawler, self).__init__()

    # Chuyển giờ thành datetime (ví dụ 08:00 AM, 8am, ...)
    def parse_time(self, string):
        time = None
        try:
            time = datetime.strptime(string, '%I:%M %p')
        except ValueError:
            try:
                time = datetime.strptime(string, '%I%p')
            except ValueError:
                try:
                    time = datetime.strptime(string, '%I:%M%p')
                except ValueError:
                    time = datetime.strptime(string, '%I:%M')
        return time

    def parse_price(self, string):
        min_price = None
        max_price = None

        for match in self.price_re.findall(string):
            match = match.replace('.', '').replace('đ', '')

            if min_price is None:
                min_price = int(match)
            elif max_price is None:
                max_price = int(match)
            else:
                break
        return min_price, max_price

    def extract_item_info(self, item):
        foody_id = item['Id']
        name = item['Name']
        address = Address(
            item['Address'], item['District'], item['City'])

        phone_response = self.session.get(
            'http://www.foody.vn/Restaurant/GetPhoneByResId', params={'resId': foody_id})
        phone_json = json.loads(phone_response.text)
        phone = phone_json['phone']

        rating = item['AvgRating']

        try:
            cuisine = item['Cuisines'][0]['Name']
        except IndexError:
            cuisine = 'Việt Nam'

        link = 'http://www.foody.vn' + item['DetailUrl']
        print('Requesting page {}'.format(link))
        response = self.session.get(link)
        soup = BeautifulSoup(response.text)

        try:
            category = soup.find(
                'div', class_='category-items').a['title']
            open_time = self.parse_time(
                soup.find('span', attrs={'itemprop': 'opens'}).text)
            close_time = self.parse_time(
                soup.find('span', attrs={'itemprop': 'closes'}).text)

            if close_time <= open_time:
                close_time += timedelta(days=1)

            min_price, max_price = self.parse_price(soup.find(
                'span', attrs={'itemprop': 'priceRange'}).find('span').text)

        except (AttributeError, ValueError):
            print('Error with ' + link)
            return None

        return Diner(foody_id, rating, name, address, phone,
                     category, cuisine, open_time, close_time, min_price, max_price)

    def crawl(self):

        data = {
            'append': 'true',
            'c': '1,2,3,4,6,11,12,39',
            'page': 1
        }
        headers = {
            'X-Requested-With': 'XMLHttpRequest'
        }
        url = 'http://www.foody.vn/ho-chi-minh/dia-diem'

        while True:
            response = self.session.get(url, headers=headers, params=data)
            if response.status_code != 200:
                return

            items = json.loads(response.text)['restaurants']

            for item in items:
                for sub_item in item['SubItems']:
                    extracted_info = self.extract_item_info(sub_item)
                    if extracted_info is None:
                        continue
                    yield extracted_info
                extracted_main_info = self.extract_item_info(sub_item)
                if extracted_main_info is None:
                    continue
                yield extracted_main_info
            data['page'] += 1

# class DDAOCrawler(DinerCrawler):

#     def __init__(self):
#         super(DDAOCrawler, self).__init__()

# Returns all Diners from URL response.
# It gets link from the 'desc' div,
# goes to link and get diner's details.
#     def extract_page_data(self, response):
#         main_soup = BeautifulSoup(response.text)
#         diners = []

#         for item in main_soup.find_all('div', class_='desc'):
#             if item.h2 is None:
#                 continue

#             anchor = item.h2.a
#             link = anchor['href']

#             details_response = self.session.get(link)
#             yield self.get_diner_details(details_response)

# Returns a Diner from the details page.
#     def get_diner_details(self, details_response):
#         soup = BeautifulSoup(details_response.text)

#         name = soup.find('h1', class_='place-detail-title').string

#         description = soup.find('div', class_='desc')

#         anchor = description.find('a', class_='place-category')
#         category = anchor.string

# Has to call next_sibling 2 times because:
# http://www.crummy.com/software/BeautifulSoup/bs4/doc/#next-sibling-and-previous-sibling
#         anchor = anchor.next_sibling.next_sibling
#         address = anchor.span.next.string.strip()

#         anchor = anchor.next_sibling.next_sibling
#         phone = anchor.span.next.string.strip()

#         anchor = description.find('div', class_='block')
#         open_time = anchor.find('ul', class_='bullet').li.string

# Skip phần comment (<!-- /.block -->)
#         anchor = anchor.next_sibling.next_sibling.next_sibling
#         price_range = anchor.find('strong').string

#         return Diner(name, address, phone,
#                      category, open_time, price_range)

#     def crawl(self):
#         data = {
#             'ajax': 1,
#             'offset': 0,
#             'areaid': 3,
#             'areaseo': 'tp-ho-chi-minh'
#         }

#         offset = 0

#         while True:
#             response = self.session.post(
#                 'http://diadiemanuong.com/location/ajaxLoadMore/', data=data)

#             if response.status_code != 200:
#                 return

#             for diner in self.extract_page_data(response):
#                 yield diner

#             offset += 10
#             data['offset'] = offset
<commit_msg>Fix crawler only processes subitems<commit_after>import os
import re
import json
from datetime import datetime
from datetime import timedelta
from bs4 import BeautifulSoup

from POICrawler.diner import Diner
from POICrawler.requester import Requester
from POICrawler.address import Address

class DinerCrawler(object):

    def __init__(self):
        self.session = Requester()

    # Subclasses must implement this method,
    # return list of Diners
    def crawl(self):
        raise NotImplementedError

class FoodyVNCrawler(DinerCrawler):

    price_re = re.compile('\d+\.\d+đ')

    def __init__(self):
        super(FoodyVNCrawler, self).__init__()

    # Chuyển giờ thành datetime (ví dụ 08:00 AM, 8am, ...)
    def parse_time(self, string):
        time = None
        try:
            time = datetime.strptime(string, '%I:%M %p')
        except ValueError:
            try:
                time = datetime.strptime(string, '%I%p')
            except ValueError:
                try:
                    time = datetime.strptime(string, '%I:%M%p')
                except ValueError:
                    time = datetime.strptime(string, '%I:%M')
        return time

    def parse_price(self, string):
        min_price = None
        max_price = None

        for match in self.price_re.findall(string):
            match = match.replace('.', '').replace('đ', '')

            if min_price is None:
                min_price = int(match)
            elif max_price is None:
                max_price = int(match)
            else:
                break
        return min_price, max_price

    def extract_item_info(self, item):
        foody_id = item['Id']
        name = item['Name']
        address = Address(
            item['Address'], item['District'], item['City'])

        phone_response = self.session.get(
            'http://www.foody.vn/Restaurant/GetPhoneByResId', params={'resId': foody_id})
        phone_json = json.loads(phone_response.text)
        phone = phone_json['phone']

        rating = item['AvgRating']

        try:
            cuisine = item['Cuisines'][0]['Name']
        except IndexError:
            cuisine = 'Việt Nam'

        link = 'http://www.foody.vn' + item['DetailUrl']
        print('Requesting page {}'.format(link))
        response = self.session.get(link)
        soup = BeautifulSoup(response.text)

        try:
            category = soup.find(
                'div', class_='category-items').a['title']
            open_time = self.parse_time(
                soup.find('span', attrs={'itemprop': 'opens'}).text)
            close_time = self.parse_time(
                soup.find('span', attrs={'itemprop': 'closes'}).text)

            if close_time <= open_time:
                close_time += timedelta(days=1)

            min_price, max_price = self.parse_price(soup.find(
                'span', attrs={'itemprop': 'priceRange'}).find('span').text)

        except (AttributeError, ValueError):
            print('Error with ' + link)
            return None

        return Diner(foody_id, rating, name, address, phone,
                     category, cuisine, open_time, close_time, min_price, max_price)

    def crawl(self):

        data = {
            'append': 'true',
            'c': '1,2,3,4,6,11,12,39',
            'page': 1
        }
        headers = {
            'X-Requested-With': 'XMLHttpRequest'
        }
        url = 'http://www.foody.vn/ho-chi-minh/dia-diem'

        while True:
            response = self.session.get(url, headers=headers, params=data)
            if response.status_code != 200:
                return

            items = json.loads(response.text)['restaurants']

            for item in items:
                for sub_item in item['SubItems']:
                    extracted_info = self.extract_item_info(sub_item)
                    if extracted_info is None:
                        continue
                    yield extracted_info
                extracted_main_info = self.extract_item_info(item)
                if extracted_main_info is None:
                    continue
                yield extracted_main_info
            data['page'] += 1

# class DDAOCrawler(DinerCrawler):

#     def __init__(self):
#         super(DDAOCrawler, self).__init__()

# Returns all Diners from URL response.
# It gets link from the 'desc' div,
# goes to link and get diner's details.
#     def extract_page_data(self, response):
#         main_soup = BeautifulSoup(response.text)
#         diners = []

#         for item in main_soup.find_all('div', class_='desc'):
#             if item.h2 is None:
#                 continue

#             anchor = item.h2.a
#             link = anchor['href']

#             details_response = self.session.get(link)
#             yield self.get_diner_details(details_response)

# Returns a Diner from the details page.
#     def get_diner_details(self, details_response):
#         soup = BeautifulSoup(details_response.text)

#         name = soup.find('h1', class_='place-detail-title').string

#         description = soup.find('div', class_='desc')

#         anchor = description.find('a', class_='place-category')
#         category = anchor.string

# Has to call next_sibling 2 times because:
# http://www.crummy.com/software/BeautifulSoup/bs4/doc/#next-sibling-and-previous-sibling
#         anchor = anchor.next_sibling.next_sibling
#         address = anchor.span.next.string.strip()

#         anchor = anchor.next_sibling.next_sibling
#         phone = anchor.span.next.string.strip()

#         anchor = description.find('div', class_='block')
#         open_time = anchor.find('ul', class_='bullet').li.string

# Skip phần comment (<!-- /.block -->)
#         anchor = anchor.next_sibling.next_sibling.next_sibling
#         price_range = anchor.find('strong').string

#         return Diner(name, address, phone,
#                      category, open_time, price_range)

#     def crawl(self):
#         data = {
#             'ajax': 1,
#             'offset': 0,
#             'areaid': 3,
#             'areaseo': 'tp-ho-chi-minh'
#         }

#         offset = 0

#         while True:
#             response = self.session.post(
#                 'http://diadiemanuong.com/location/ajaxLoadMore/', data=data)

#             if response.status_code != 200:
#                 return

#             for diner in self.extract_page_data(response):
#                 yield diner

#             offset += 10
#             data['offset'] = offset
<|endoftext|>


<commit_before>#!/usr/bin/env python

# CytoBandBase.py stores CytoBandIdeo data downloaded from UCSC and
# Allows easy access to the CytoBand information from a Python script
# in a species-independent manner.
# Copyright (C) 2014 DV Klopfenstein;
# note2debra at gmail dot com
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details:
#
#   https://www.gnu.org/licenses/gpl-2.0.txt
#
#
# Drexel University., hereby disclaims all copyright interest in the program
# `CytoBandBase' written by DV Klopfenstein.
#
# Aydin Tozeren, Distinguished Professor
# August 2014

# To cite this script, please use:
#
# Klopfenstein D.V. (2014) Storage and Python Access to UCSC's Cytoband Data In A Species-Independent Manner (Version 2.0) [Computer program]. Available at https://github.com/dklopfenstein/biocode/tree/master/ (Accessed [NN Month 20NN])

class CytoBandBase:
  """Python Interface to UCSC cytoBandIdeo.txt information."""

  def __init__(self, data):
    """Initialize variables using data downloaded from a UCSC Cytoband file."""
    # Number of Chromosomes for this species
    self.num_chr = data['num_chr']
    # Index to Chromosome Name
    self.chr_i2s = data['chr_i2s']
    # Chromosome name (and synonyms) to index
    self.chr_s2i = data['chr_s2i']
    # Length of each Chromosome
    self.lengths = data['lengths']
    # Data
    self.centromeres = data['centromeres']
    self.gvars   = data['gvars']
    self.stalks  = data['stalks']

  def get_sChr_list(self):
    return [sChr for sChr in self.chr_i2s]

  def get_gvarL(self, idx):
    if self.gvars[idx] is not None:
      L = 0
      for E in self.gvars[idx]:
        L += E[2] # Add lengths of each gvar on a chromosome
      return L # return total gvar length
    else:
      return None

  def get_stalkL(self, idx):
    if self.stalks[idx] is not None:
      L = 0
      for E in self.stalks[idx]:
        L += E[2] # Add lengths of each gvar on a chromosome
      return L # return total gvar length
    else:
      return None

  def get_lens(self, idx): return self.lengths

  def get_len(self, idx):
    """Get the length of one particular chromosome."""
    if idx < self.num_chr:
      return self.lengths[idx]

  def get_iChr(self, sChr):
    if sChr in self.chr_s2i:
      return self.chr_s2i[sChr]
    return None

  def get_sChr(self, idx):
    if idx is not None and idx < self.num_chr:
      return self.chr_i2s[idx]
    raise Exception("\n**\n** FATAL: CANNOT DETERMINE CHR NAME WITH ({})\n**\n".format(idx))

  def cen_get_len(self, iChr):
    if iChr < self.num_chr and  \
      self.centromeres[iChr] is not None  and \
      self.centromeres[iChr]['p'] is not None and \
      self.centromeres[iChr]['q'] is not None:
      return self.centromeres[iChr]['p'][1] - self.centromeres[iChr]['q'][0]
    return None

  def cen_get_start(self, iChr):
    if iChr < self.num_chr and \
      self.centromeres[iChr] is not None and \
      self.centromeres[iChr]['q'] is not None:
      return self.centromeres[iChr]['q'][0]
    return None

  def cen_get_end(self, iChr):
    if iChr < self.num_chr and \
      self.centromeres[iChr] is not None and \
      self.centromeres[iChr]['q'] is not None:
      return self.centromeres[iChr]['p'][1]
    return None

<commit_msg>Added location of script in citing text<commit_after>#!/usr/bin/env python

# CytoBandBase.py stores CytoBandIdeo data downloaded from UCSC and
# Allows easy access to the CytoBand information from a Python script
# in a species-independent manner.
# Copyright (C) 2014 DV Klopfenstein;
# note2debra at gmail dot com
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details:
#
#   https://www.gnu.org/licenses/gpl-2.0.txt
#
#
# Drexel University., hereby disclaims all copyright interest in the program
# `CytoBandBase' written by DV Klopfenstein.
#
# Aydin Tozeren, Distinguished Professor
# August 2014

# To cite this script, please use:
#
# Klopfenstein D.V. (2014) Storage and Python Access to UCSC's Cytoband Data In A Species-Independent Manner (Version 2.0) [Computer program]. Available at https://github.com/dklopfenstein/biocode/blob/master/PyDkBio/UCSC/CytoBandBase.py (Accessed [NN Month 20NN])

class CytoBandBase:
  """Python Interface to UCSC cytoBandIdeo.txt information."""

  def __init__(self, data):
    """Initialize variables using data downloaded from a UCSC Cytoband file."""
    # Number of Chromosomes for this species
    self.num_chr = data['num_chr']
    # Index to Chromosome Name
    self.chr_i2s = data['chr_i2s']
    # Chromosome name (and synonyms) to index
    self.chr_s2i = data['chr_s2i']
    # Length of each Chromosome
    self.lengths = data['lengths']
    # Data
    self.centromeres = data['centromeres']
    self.gvars   = data['gvars']
    self.stalks  = data['stalks']

  def get_sChr_list(self):
    return [sChr for sChr in self.chr_i2s]

  def get_gvarL(self, idx):
    if self.gvars[idx] is not None:
      L = 0
      for E in self.gvars[idx]:
        L += E[2] # Add lengths of each gvar on a chromosome
      return L # return total gvar length
    else:
      return None

  def get_stalkL(self, idx):
    if self.stalks[idx] is not None:
      L = 0
      for E in self.stalks[idx]:
        L += E[2] # Add lengths of each gvar on a chromosome
      return L # return total gvar length
    else:
      return None

  def get_lens(self, idx): return self.lengths

  def get_len(self, idx):
    """Get the length of one particular chromosome."""
    if idx < self.num_chr:
      return self.lengths[idx]

  def get_iChr(self, sChr):
    if sChr in self.chr_s2i:
      return self.chr_s2i[sChr]
    return None

  def get_sChr(self, idx):
    if idx is not None and idx < self.num_chr:
      return self.chr_i2s[idx]
    raise Exception("\n**\n** FATAL: CANNOT DETERMINE CHR NAME WITH ({})\n**\n".format(idx))

  def cen_get_len(self, iChr):
    if iChr < self.num_chr and  \
      self.centromeres[iChr] is not None  and \
      self.centromeres[iChr]['p'] is not None and \
      self.centromeres[iChr]['q'] is not None:
      return self.centromeres[iChr]['p'][1] - self.centromeres[iChr]['q'][0]
    return None

  def cen_get_start(self, iChr):
    if iChr < self.num_chr and \
      self.centromeres[iChr] is not None and \
      self.centromeres[iChr]['q'] is not None:
      return self.centromeres[iChr]['q'][0]
    return None

  def cen_get_end(self, iChr):
    if iChr < self.num_chr and \
      self.centromeres[iChr] is not None and \
      self.centromeres[iChr]['q'] is not None:
      return self.centromeres[iChr]['p'][1]
    return None

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

redPins=[20,23]     [12,25]
greenPins=[12,25]   [20,23]
bluePins=[21,18]<commit_msg>*derp*<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

redPins=[12,25]
greenPins=[20,23]
bluePins=[21,18]<|endoftext|>


<commit_before>"""
    Shared generic views used across the modules
"""

import reversion

from django.contrib import messages
from django.http.response import HttpResponseRedirect
from django.views.generic.base import TemplateView
from django.contrib.auth.decorators import login_required
from django.utils.decorators import method_decorator
from django.views.generic.base import View
from django.utils.translation import ugettext as _
from django.views.generic import DetailView
from django.views.generic.edit import UpdateView

from django_tables2 import SingleTableView
from utils import dict_diff, merge_dicts

class LoginMixin(object):
    @method_decorator(login_required)
    def dispatch(self, request, *args, **kwargs):
        return super(LoginMixin, self).dispatch(request, *args, **kwargs)

class MultipleFormView(TemplateView):
    """
    View mixin that handles multiple forms / formsets.
    After the successful data is inserted ``self.forms_valid`` is called.
    """
    form_classes = {}

    def initialize_forms(self, data=None):
        return {name: form(prefix=name, data=data)
                for name, form in self.form_classes.items()}

    def get_context_data(self, **kwargs):
        context = super(MultipleFormView, self).get_context_data(**kwargs)
        return merge_dicts(context, self.initialize_forms())

    def post(self, request):
        forms_initialized = self.initialize_forms(data=request.POST)
        valid = all([form.is_valid() for form in forms_initialized.values()])

        if valid:
            return self.forms_valid(forms_initialized)
        else:
            context = merge_dicts(self.get_context_data(), forms_initialized)
            return self.render_to_response(context)

    def forms_valid(self, form_instances):
        raise NotImplementedError('Implement this in your view!')

class MessageView(object):
    """
        Simple view for making it easier to work with Message framework.
    """

    def add_message(self, message, level=messages.INFO):
        messages.add_message(self.request, level, message)

class ActionView(View, MessageView):
    """
    View for processing actions etc.
    """
    allowed_actions = ()
    # required permission for doing any kind of action
    permission = None

    def process_action(self, action):
        """
        Override this method
        """
        raise NotImplementedError('You must implement this method')

    def get_success_url(self):
        """
        This method will be used when action is successfully performed
        """
        raise NotImplementedError('Implement this!')

    def get_fail_url(self):
        """
        This method will be used when something goes awry.
        """
        raise NotImplementedError('Why would you not implement this?!')

    def get(self, request, *args, **kwargs):
        return HttpResponseRedirect(self.get_fail_url())

    def post(self, request, *args, **kwargs):
        if self.permission and not request.user.has_perm(self.permission):
            self.add_message(_('Insufficient permissions.'), messages.ERROR)
        else:
            action = request.POST.get('action', None)
            if action in self.allowed_actions:
                self.process_action(action)
                return HttpResponseRedirect(self.get_success_url())
            else:
                self.add_message(_('Action {0} not allowed.').format(action),
                                 messages.ERROR)
        return HttpResponseRedirect(self.get_fail_url())

class EditView(UpdateView, MessageView):
    template_name = 'edit_form.html'

    def form_valid(self, form):
        form.save()
        self.add_message(_('Changes successfully saved.'), messages.SUCCESS)
        return HttpResponseRedirect(self.get_object().get_absolute_url())

class HistoryView(DetailView):
    """
        Simple generic view for django reversion history
    """
    template_name = 'history.html'

    def get_context_data(self, **kwargs):
        context = super(HistoryView, self).get_context_data(**kwargs)
        versions = reversion.get_unique_for_object(self.get_object())
        diffs = []
        if len(versions) > 1:
            for i in range(len(versions) - 1):
                new = versions[i]
                old = versions[i+1]
                fields_changed = dict_diff(old.field_dict, new.field_dict)
                if fields_changed:
                    diffs.append({
                        'date': new.revision.date_created,
                        'user': new.revision.user,
                        'comment': new.revision.comment,
                        'fields_changed': fields_changed
                    })
        context['diffs'] = diffs
        context['versions'] = versions
        return context

class FilteredListView(SingleTableView):
    """
        Abstract view class for list views with filters
    """
    template_name = 'filtered_list.html'
    context_object_name = 'objects'
    table_pagination = {"per_page": 20}

    table_class = NotImplemented
    filter_class = NotImplemented

    def get_table_data(self):
        queryset = super(FilteredListView, self).get_table_data()
        return self.filter_class(self.request.GET, queryset=queryset)

    def get_context_data(self, **kwargs):
        context = super(FilteredListView, self).get_context_data(**kwargs)
        context['filter'] = self.filter_class(data=self.request.GET)
        context['filter_active'] = bool(self.request.GET)
        return context
<commit_msg>MultipleFormView was not used<commit_after>"""
    Shared generic views used across the modules
"""

import reversion

from django.contrib import messages
from django.http.response import HttpResponseRedirect
from django.contrib.auth.decorators import login_required
from django.utils.decorators import method_decorator
from django.views.generic.base import View
from django.utils.translation import ugettext as _
from django.views.generic import DetailView
from django.views.generic.edit import UpdateView

from django_tables2 import SingleTableView
from utils import dict_diff

class LoginMixin(object):
    @method_decorator(login_required)
    def dispatch(self, request, *args, **kwargs):
        return super(LoginMixin, self).dispatch(request, *args, **kwargs)

class MessageView(object):
    """
        Simple view for making it easier to work with Message framework.
    """

    def add_message(self, message, level=messages.INFO):
        messages.add_message(self.request, level, message)

class ActionView(View, MessageView):
    """
    View for processing actions etc.
    """
    allowed_actions = ()
    # required permission for doing any kind of action
    permission = None

    def process_action(self, action):
        """
        Override this method
        """
        raise NotImplementedError('You must implement this method')

    def get_success_url(self):
        """
        This method will be used when action is successfully performed
        """
        raise NotImplementedError('Implement this!')

    def get_fail_url(self):
        """
        This method will be used when something goes awry.
        """
        raise NotImplementedError('Why would you not implement this?!')

    def get(self, request, *args, **kwargs):
        return HttpResponseRedirect(self.get_fail_url())

    def post(self, request, *args, **kwargs):
        if self.permission and not request.user.has_perm(self.permission):
            self.add_message(_('Insufficient permissions.'), messages.ERROR)
        else:
            action = request.POST.get('action', None)
            if action in self.allowed_actions:
                self.process_action(action)
                return HttpResponseRedirect(self.get_success_url())
            else:
                self.add_message(_('Action {0} not allowed.').format(action),
                                 messages.ERROR)
        return HttpResponseRedirect(self.get_fail_url())

class EditView(UpdateView, MessageView):
    template_name = 'edit_form.html'

    def form_valid(self, form):
        form.save()
        self.add_message(_('Changes successfully saved.'), messages.SUCCESS)
        return HttpResponseRedirect(self.get_object().get_absolute_url())

class HistoryView(DetailView):
    """
        Simple generic view for django reversion history
    """
    template_name = 'history.html'

    def get_context_data(self, **kwargs):
        context = super(HistoryView, self).get_context_data(**kwargs)
        versions = reversion.get_unique_for_object(self.get_object())
        diffs = []
        if len(versions) > 1:
            for i in range(len(versions) - 1):
                new = versions[i]
                old = versions[i+1]
                fields_changed = dict_diff(old.field_dict, new.field_dict)
                if fields_changed:
                    diffs.append({
                        'date': new.revision.date_created,
                        'user': new.revision.user,
                        'comment': new.revision.comment,
                        'fields_changed': fields_changed
                    })
        context['diffs'] = diffs
        context['versions'] = versions
        return context

class FilteredListView(SingleTableView):
    """
        Abstract view class for list views with filters
    """
    template_name = 'filtered_list.html'
    context_object_name = 'objects'
    table_pagination = {"per_page": 20}

    table_class = NotImplemented
    filter_class = NotImplemented

    def get_table_data(self):
        queryset = super(FilteredListView, self).get_table_data()
        return self.filter_class(self.request.GET, queryset=queryset)

    def get_context_data(self, **kwargs):
        context = super(FilteredListView, self).get_context_data(**kwargs)
        context['filter'] = self.filter_class(data=self.request.GET)
        context['filter_active'] = bool(self.request.GET)
        return context
<|endoftext|>


<commit_before>"""
Routines to get the list of files to run over.
"""

import os
import subprocess
import sys

from Treemaker.Treemaker.dbsapi import dasFileQuery
from Treemaker.Treemaker.dbsapi import constants

def getDASNtuples(directory):
	"""	Given a das 'url', extract the list of files to run over.
		Returns a list of ntuples in xrootd form and the dataset name."""
	_, _, dasName = directory.partition("://")
	dbs, _, dataset = dasName.partition(":")
	if not dbs in constants.instances:
		print "Error: DBS instance " + dbs + " was not recognized!"
		sys.exit(1)
	rawNtuples = dasFileQuery.dasFileQuery(dataset, dbs)
	ntuples = ["root://cmsxrootd.fnal.gov//" + ntuple for ntuple in rawNtuples]
	return ntuples, dataset

def getXRDNtuples(directory):
	"""	Given an xrootd directory, attempt to use xrdfs to get a list of ntuples to
		run over. Returns a list of ntuples in xrootd form."""
	uri, _, path = directory.partition("///")
	path = "/" + path
	try:
		# Expand sub-directories by recursively running this function...
		# assume that directories aren't named .root?
		# this is bad and should be rewritten. XXX
		output = subprocess.check_output(["xrdfs", uri, "ls", path])
		if "[FATAL]" in output or "[ERROR]" in output:
			return []
		else:
			array = output.split('\n')
			results = []
			for string in array:
				searchString = uri + "//" + string
				if '.root' in string:
					results.append(searchString)
				elif string.lstrip().rstrip() == '':
					continue
				else:
					results.extend(getXRDNtuples(searchString))
			return results
	except subprocess.CalledProcessError:
		return []
	except OSError:
		print "Error: the xrdfs program is not installed! Cannot proceed!"
		sys.exit(1)

def getDiskNtuples(directory):
	"""	Given a directory on a proper filesystem, return a list of ntuples to run over."""
		# Accumulate a list of files. Subdirectories are okay.
	ntuples = []
	for path, dirs, files in os.walk(directory):
		for ntuple in files:
			if '.root' in ntuple:
				ntuples.append(os.path.join(path, ntuple))
	return ntuples

def doSplitting(ntuples, index, splitBy, splitInto):
	"""	Do job splitting. Given splitting options, a list of files (strings) and an index,
		splits the files and returns the (index)th set of files to process. Also returns
		the total number of jobs needed."""
	numJobs = 1
	numNtuples = len(ntuples)
	# These cases should be mutually exclusive.
	if splitBy > 0:
		numJobs = splitBy
		startJobs = int(math.ceil(numNtuples / float(splitBy))) * index
		endJobs = int(math.ceil(numNtuples / float(splitBy))) * (index + 1)
		if index + 1 > splitBy:
			print "Error: cannot make this splitting with index + 1 > number of splits!"
			sys.exit(1)
	elif splitInto > 0:
		startJobs = splitInto * index
		endJobs = splitInto * (index + 1)
		numJobs = int(math.ceil(len(ntuples) / float(splitInto)))

	if (splitBy > 0 or splitInto > 0) and index >= 0:
		if endJobs > len(ntuples):
			endJobs = len(ntuples)
		return ntuples[startJobs:endJobs], numJobs
	else:
		return ntuples, numJobs

def getNtuplesAndName(directory, name=""):
	if 'dbs://' in directory or 'das://' in directory:
		# For DAS entries, look them up using dbsapi (or the bit of it we wrote).
		ntuples, dataset = getDASNtuples(directory)

		print "*** Running treemaker over dataset " + dataset
		if name == "":
			name = dataset.split('/')[1]
	elif 'root://' in directory:
		print "*** Running treemaker via xrootd over " + directory
		if name == "":
			name = directory.rstrip('/').rpartition("/")[2]
		ntuples = getXRDNtuples(directory)
	else:
		print "*** Running treemaker over " + directory
		if name == "":
			name = directory.rstrip('/').rpartition("/")[2]
		ntuples = getDiskNtuples(directory)
	return ntuples, name
<commit_msg>Convert DAS strings from unicode to ascii strings.<commit_after>"""
Routines to get the list of files to run over.
"""

import os
import subprocess
import sys

from Treemaker.Treemaker.dbsapi import dasFileQuery
from Treemaker.Treemaker.dbsapi import constants

def getDASNtuples(directory):
	"""	Given a das 'url', extract the list of files to run over.
		Returns a list of ntuples in xrootd form and the dataset name."""
	_, _, dasName = directory.partition("://")
	dbs, _, dataset = dasName.partition(":")
	if not dbs in constants.instances:
		print "Error: DBS instance " + dbs + " was not recognized!"
		sys.exit(1)
	rawNtuples = dasFileQuery.dasFileQuery(dataset, dbs)
	try:
		ntuples = ["root://cmsxrootd.fnal.gov//" + ntuple.encode("utf8") for ntuple in rawNtuples]
	except UnicodeDecodeError:
		print "Error retrieving dataset list from DAS, because of unicode conversion issues? This should never happen."
	return ntuples, dataset

def getXRDNtuples(directory):
	"""	Given an xrootd directory, attempt to use xrdfs to get a list of ntuples to
		run over. Returns a list of ntuples in xrootd form."""
	uri, _, path = directory.partition("///")
	path = "/" + path
	try:
		# Expand sub-directories by recursively running this function...
		# assume that directories aren't named .root?
		# this is bad and should be rewritten. XXX
		output = subprocess.check_output(["xrdfs", uri, "ls", path])
		if "[FATAL]" in output or "[ERROR]" in output:
			return []
		else:
			array = output.split('\n')
			results = []
			for string in array:
				searchString = uri + "//" + string
				if '.root' in string:
					results.append(searchString)
				elif string.lstrip().rstrip() == '':
					continue
				else:
					results.extend(getXRDNtuples(searchString))
			return results
	except subprocess.CalledProcessError:
		return []
	except OSError:
		print "Error: the xrdfs program is not installed! Cannot proceed!"
		sys.exit(1)

def getDiskNtuples(directory):
	"""	Given a directory on a proper filesystem, return a list of ntuples to run over."""
		# Accumulate a list of files. Subdirectories are okay.
	ntuples = []
	for path, dirs, files in os.walk(directory):
		for ntuple in files:
			if '.root' in ntuple:
				ntuples.append(os.path.join(path, ntuple))
	return ntuples

def doSplitting(ntuples, index, splitBy, splitInto):
	"""	Do job splitting. Given splitting options, a list of files (strings) and an index,
		splits the files and returns the (index)th set of files to process. Also returns
		the total number of jobs needed."""
	numJobs = 1
	numNtuples = len(ntuples)
	# These cases should be mutually exclusive.
	if splitBy > 0:
		numJobs = splitBy
		startJobs = int(math.ceil(numNtuples / float(splitBy))) * index
		endJobs = int(math.ceil(numNtuples / float(splitBy))) * (index + 1)
		if index + 1 > splitBy:
			print "Error: cannot make this splitting with index + 1 > number of splits!"
			sys.exit(1)
	elif splitInto > 0:
		startJobs = splitInto * index
		endJobs = splitInto * (index + 1)
		numJobs = int(math.ceil(len(ntuples) / float(splitInto)))

	if (splitBy > 0 or splitInto > 0) and index >= 0:
		if endJobs > len(ntuples):
			endJobs = len(ntuples)
		return ntuples[startJobs:endJobs], numJobs
	else:
		return ntuples, numJobs

def getNtuplesAndName(directory, name=""):
	if 'dbs://' in directory or 'das://' in directory:
		# For DAS entries, look them up using dbsapi (or the bit of it we wrote).
		ntuples, dataset = getDASNtuples(directory)

		print "*** Running treemaker over dataset " + dataset
		if name == "":
			name = dataset.split('/')[1]
	elif 'root://' in directory:
		print "*** Running treemaker via xrootd over " + directory
		if name == "":
			name = directory.rstrip('/').rpartition("/")[2]
		ntuples = getXRDNtuples(directory)
	else:
		print "*** Running treemaker over " + directory
		if name == "":
			name = directory.rstrip('/').rpartition("/")[2]
		ntuples = getDiskNtuples(directory)
	return ntuples, name
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
    zeit.recommend.zonapi
    ~~~~~~~~~~~~~~~~~~~~~

    This module has no description.

    Copyright: (c) 2013 by Nicolas Drebenstedt.
    License: BSD, see LICENSE for more details.
"""

from datetime import datetime
from storm import emit
from storm import log
from storm import Spout
from time import sleep
from urllib import urlencode
from urllib import urlopen
from uuid import uuid4
import json

class ZonAPISpout(Spout):
    def initialize(self, conf, context):
        host = conf.get('zeit.recommend.zonapi.host', 'localhost')
        port = conf.get('zeit.recommend.zonapi.port', 8983)
        self.url = 'http://%s:%s/solr/select' % (host, port)
        self.time_range = [datetime.now(), ] * 2
        self.earliest = datetime(2000, 1, 1)
        self.buffer = {}

    def ack(self, cnt_id):
        tup, retries = self.buffer[cnt_id]
        del self.buffer[cnt_id]
        log('[ZonAPISpout] Acknowledging content id-%s.' % cnt_id)

    def fail(self, cnt_id):
        tup, retries = self.buffer[cnt_id]
        if retries >= 5:
            del self.buffer[cnt_id]
            log('[ZonAPISpout] Message %s failed for good.' % cnt_id)
        else:
            self.buffer[cnt_id] = (tup, retries + 1)
            emit(tup, id=cnt_id)

    def get_docs(self, from_, to, sort):
        # TODO: This is not parallelizable.
        date_range = (from_.isoformat()[:-3], to.isoformat()[:-3])
        params = dict(
            wt='json',
            q='release_date:[%s:00Z TO %s:00Z]' % date_range,
            fl='uuid,href,release_date,title',
            sort='release_date %s' % sort,
            rows='1'
            )
        raw = urlopen(self.url, urlencode(params))
        data = raw.read()
        return json.loads(data)['response']['docs']

    def nextTuple(self):
        docs = self.get_docs(self.time_range[1], datetime.now(), 'asc')
        if docs:
            self.time_range[1] = datetime.now()
        else:
            docs = self.get_docs(self.earliest, self.time_range[0], 'desc')
            if docs:
                args = docs[0]['release_date'], '%Y-%m-%dT%H:%M:%SZ'
                self.time_range[0] = datetime.strptime(*args)
        if docs:
            cnt_id = str(uuid4())
            tup = [docs[0]['href'][18:]]
            self.buffer[cnt_id] = (tup, 0)
            emit(tup, id=cnt_id)

        sleep(1.0)

if __name__ == '__main__':
    ZonAPISpout().run()
<commit_msg>Fix time range management<commit_after># -*- coding: utf-8 -*-

"""
    zeit.recommend.zonapi
    ~~~~~~~~~~~~~~~~~~~~~

    This module has no description.

    Copyright: (c) 2013 by Nicolas Drebenstedt.
    License: BSD, see LICENSE for more details.
"""

from datetime import datetime
from storm import emit
from storm import log
from storm import Spout
from time import sleep
from urllib import urlencode
from urllib import urlopen
import json

class ZonAPISpout(Spout):
    def initialize(self, conf, context):
        host = conf.get('zeit.recommend.zonapi.host', 'localhost')
        port = conf.get('zeit.recommend.zonapi.port', 8983)
        self.url = 'http://%s:%s/solr/select' % (host, port)
        self.time_range = [datetime.now()] * 2
        self.buffer = {}

    def ack(self, cnt_id):
        tup, retries = self.buffer[cnt_id]
        del self.buffer[cnt_id]
        log('[ZonAPISpout] Acknowledging content id-%s.' % cnt_id)

    def fail(self, cnt_id):
        tup, retries = self.buffer[cnt_id]
        if retries >= 5:
            del self.buffer[cnt_id]
            log('[ZonAPISpout] Message %s failed for good.' % cnt_id)
        else:
            self.buffer[cnt_id] = (tup, retries + 1)
            emit(tup, id=cnt_id)

    def get_docs(self):
        upper = '%s:00Z TO NOW' % self.time_range[1].isoformat()[:-3]
        lower = 'NOW-1YEAR TO %s:00Z' % self.time_range[0].isoformat()[:-3]
        params = dict(
            wt='json',
            q='release_date:[%s] OR release_date:[%s]' % (lower, upper),
            fl='uuid,href,release_date,title',
            sort='release_date desc',
            rows='1'
            )
        raw = urlopen(self.url, urlencode(params))
        data = raw.read()
        return json.loads(data)['response']['docs']

    def nextTuple(self):
        docs = self.get_docs()
        if docs:
            args = docs[0]['release_date'], '%Y-%m-%dT%H:%M:%SZ'
            doc_time = datetime.strptime(*args)
            self.time_range[int(doc_time > self.time_range[1])] = doc_time
            uuid = docs[0]['uuid']
            tup = [docs[0]['href'][18:]]
            self.buffer[uuid] = (tup, 0)
            emit(tup, id=uuid)

        sleep(1.0)

if __name__ == '__main__':
    ZonAPISpout().run()
<|endoftext|>


<commit_before>import sys
sys.path.insert(1,"../../")
import h2o
from tests import pyunit_utils
from h2o.estimators.gbm import H2OGradientBoostingEstimator

def pyunit_mean_per_class_error():
    gbm = H2OGradientBoostingEstimator(nfolds=3, fold_assignment="Random", seed=1234)

    ## Binomial
    cars = h2o.import_file("/users/arno/h2o-3/smalldata/junit/cars_20mpg.csv")
    cars["economy_20mpg"] = cars["economy_20mpg"].asfactor()
    r = cars[0].runif(seed=1234)
    train = cars[r > .2]
    valid = cars[r <= .2]
    response_col = "economy_20mpg"
    predictors = ["displacement","power","weight","acceleration","year"]
    gbm.distribution = "bernoulli"
    gbm.train(y=response_col, x=predictors, validation_frame=valid, training_frame=train)
    print(gbm)
    mpce = gbm.mean_per_class_error([0.5,0.8]) ## different thresholds
    assert (abs(mpce[0][1] - 0.004132231404958664) < 1e-5)
    assert (abs(mpce[1][1] - 0.021390374331550777) < 1e-5)

    ## Multinomial
    cars = h2o.import_file("/users/arno/h2o-3/smalldata/junit/cars_20mpg.csv")
    cars["cylinders"] = cars["cylinders"].asfactor()
    r = cars[0].runif(seed=1234)
    train = cars[r > .2]
    valid = cars[r <= .2]
    response_col = "cylinders"
    predictors = ["displacement","power","weight","acceleration","year"]
    gbm.distribution="multinomial"
    gbm.train(x=predictors,y=response_col, training_frame=train, validation_frame=valid)
    print(gbm)
    mpce = gbm.mean_per_class_error(train=True)
    assert( mpce == 0 )
    mpce = gbm.mean_per_class_error(valid=True)
    assert(abs(mpce - 0.207142857143 ) < 1e-5)
    mpce = gbm.mean_per_class_error(xval=True)
    assert(abs(mpce - 0.350071715433 ) < 1e-5)

if __name__ == "__main__":
    pyunit_utils.standalone_test(pyunit_mean_per_class_error)
else:
    pyunit_mean_per_class_error
<commit_msg>PUBDEV-2979: Add more testing of early stopping and grid sorting for mean_per_class_error.<commit_after>import sys
sys.path.insert(1,"../../")
import h2o
import math
from tests import pyunit_utils
from h2o.estimators.gbm import H2OGradientBoostingEstimator
from h2o.grid import H2OGridSearch

def pyunit_mean_per_class_error():
    gbm = H2OGradientBoostingEstimator(nfolds=3, fold_assignment="Random", seed=1234)

    ## Binomial
    cars = h2o.import_file("/users/arno/h2o-3/smalldata/junit/cars_20mpg.csv")
    cars["economy_20mpg"] = cars["economy_20mpg"].asfactor()
    r = cars[0].runif(seed=1234)
    train = cars[r > .2]
    valid = cars[r <= .2]
    response_col = "economy_20mpg"
    predictors = ["displacement","power","weight","acceleration","year"]
    gbm.distribution = "bernoulli"
    gbm.train(y=response_col, x=predictors, validation_frame=valid, training_frame=train)
    print(gbm)
    mpce = gbm.mean_per_class_error([0.5,0.8]) ## different thresholds
    assert (abs(mpce[0][1] - 0.004132231404958664) < 1e-5)
    assert (abs(mpce[1][1] - 0.021390374331550777) < 1e-5)

    ## score on train first
    print(gbm.model_performance(train).mean_per_class_error(thresholds=[0.3,0.5]))

    ## Multinomial
    cars = h2o.import_file("/users/arno/h2o-3/smalldata/junit/cars_20mpg.csv")
    cars["cylinders"] = cars["cylinders"].asfactor()
    r = cars[0].runif(seed=1234)
    train = cars[r > .2]
    valid = cars[r <= .2]
    response_col = "cylinders"
    predictors = ["displacement","power","weight","acceleration","year"]
    gbm.distribution="multinomial"
    gbm.train(x=predictors,y=response_col, training_frame=train, validation_frame=valid)
    print(gbm)
    mpce = gbm.mean_per_class_error(train=True)
    assert( mpce == 0 )
    mpce = gbm.mean_per_class_error(valid=True)
    assert(abs(mpce - 0.207142857143 ) < 1e-5)
    mpce = gbm.mean_per_class_error(xval=True)
    assert(abs(mpce - 0.350071715433 ) < 1e-5)

    ## Early stopping
    gbm.stopping_rounds=2
    gbm.stopping_metric="mean_per_class_error"
    gbm.ntrees=10000
    gbm.max_depth=3
    gbm.min_rows=1
    gbm.learn_rate=0.01
    gbm.score_tree_interval=1
    gbm.nfolds=None
    gbm.fold_assignment=None
    gbm.train(x=predictors,y=response_col, training_frame=train, validation_frame=valid)
    print(gbm)
    print(gbm.scoring_history())

    ## Grid search
    hyper_params_tune = {'max_depth' : list(range(1,10+1,1)),
                  'sample_rate': [x/100. for x in range(20,101)],
                  'col_sample_rate' : [x/100. for x in range(20,101)],
                  'col_sample_rate_per_tree': [x/100. for x in range(20,101)],
                  'col_sample_rate_change_per_level': [x/100. for x in range(90,111)],
                  'min_rows': [2**x for x in range(0,int(math.log(train.nrow,2)-2)+1)],
                  'nbins': [2**x for x in range(4,11)],
                  'nbins_cats': [2**x for x in range(4,13)],
                  'min_split_improvement': [0,1e-8,1e-6,1e-4],
                  'histogram_type': ["UniformAdaptive","QuantilesGlobal","RoundRobin"]}

    search_criteria_tune = {'strategy': "RandomDiscrete",
                     'max_runtime_secs': 600,  ## limit the runtime to 10 minutes
                     'max_models': 10,
                     'seed' : 1234,
                     'stopping_rounds' : 5,
                     'stopping_metric' : "mean_per_class_error",
                     'stopping_tolerance': 1e-3
                     }

    grid = H2OGridSearch(H2OGradientBoostingEstimator, hyper_params=hyper_params_tune, search_criteria=search_criteria_tune)
    grid.train(x=predictors, y=response_col, training_frame=train, validation_frame=valid,distribution="multinomial", seed=1234,
        stopping_rounds = 10, stopping_metric = "mean_per_class_error", stopping_tolerance=1e-3)

    print(grid) ## sorted by logloss
    print(grid.get_grid("mean_per_class_error"))

if __name__ == "__main__":
    pyunit_utils.standalone_test(pyunit_mean_per_class_error)
else:
    pyunit_mean_per_class_error
<|endoftext|>


<commit_before># Copyright 2019 Canonical, Ltd.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import asyncio
import enum
import logging
import os

import requests.exceptions

from subiquitycore.async_helpers import (
    schedule_task,
    SingleInstanceTask,
    )
from subiquitycore.context import with_context
from subiquitycore.controller import (
    Skip,
    )

from subiquity.controller import (
    SubiquityController,
    )

log = logging.getLogger('subiquity.controllers.refresh')

class CheckState(enum.IntEnum):
    UNKNOWN = enum.auto()
    AVAILABLE = enum.auto()
    UNAVAILABLE = enum.auto()

class RefreshController(SubiquityController):

    autoinstall_key = "refresh-installer"
    autoinstall_schema = {
        'type': 'object',
        'properties': {
            'update': {'type': 'boolean'},
            'channel': {'type': 'string'},
            },
        'additionalProperties': False,
        }

    signals = [
        ('snapd-network-change', 'snapd_network_changed'),
    ]

    def __init__(self, app):
        self.ai_data = {}
        super().__init__(app)
        self.snap_name = os.environ.get("SNAP_NAME", "subiquity")
        self.configure_task = None
        self.check_task = None

        self.current_snap_version = "unknown"
        self.new_snap_version = ""

        self.offered_first_time = False
        if 'update' in self.ai_data:
            self.active = self.ai_data['update']
        else:
            self.active = self.interactive()

    def load_autoinstall_data(self, data):
        if data is not None:
            self.ai_data = data

    def start(self):
        if not self.active:
            return
        self.configure_task = schedule_task(self.configure_snapd())
        self.check_task = SingleInstanceTask(
            self.check_for_update, propagate_errors=False)
        self.check_task.start_sync()

    @with_context()
    async def apply_autoinstall_config(self, context, index=1):
        if not self.active:
            return
        try:
            await asyncio.wait_for(self.check_task.wait(), 60)
        except asyncio.TimeoutError:
            return
        if self.check_state != CheckState.AVAILABLE:
            return
        change_id = await self.start_update(context=context)
        while True:
            try:
                change = await self.get_progress(change_id)
            except requests.exceptions.RequestException as e:
                raise e
            if change['status'] == 'Done':
                # Clearly if we got here we didn't get restarted by
                # snapd/systemctl (dry-run mode or logged in via SSH)
                self.app.restart(remove_last_screen=False)
            if change['status'] not in ['Do', 'Doing']:
                raise Exception("update failed")
            await asyncio.sleep(0.1)

    @property
    def check_state(self):
        if not self.active:
            return CheckState.UNAVAILABLE
        task = self.check_task.task
        if not task.done() or task.cancelled():
            return CheckState.UNKNOWN
        if task.exception():
            return CheckState.UNAVAILABLE
        return task.result()

    @with_context()
    async def configure_snapd(self, context):
        with context.child("get_details") as subcontext:
            try:
                r = await self.app.snapd.get(
                    'v2/snaps/{snap_name}'.format(
                        snap_name=self.snap_name))
            except requests.exceptions.RequestException:
                log.exception("getting snap details")
                return
            self.current_snap_version = r['result']['version']
            for k in 'channel', 'revision', 'version':
                self.app.note_data_for_apport(
                    "Snap" + k.title(), r['result'][k])
            subcontext.description = "current version of snap is: %r" % (
                self.current_snap_version)
        channel = self.get_refresh_channel()
        desc = "switching {} to {}".format(self.snap_name, channel)
        with context.child("switching", desc) as subcontext:
            try:
                await self.app.snapd.post_and_wait(
                    'v2/snaps/{}'.format(self.snap_name),
                    {'action': 'switch', 'channel': channel})
            except requests.exceptions.RequestException:
                log.exception("switching channels")
                return
            subcontext.description = "switched to " + channel

    def get_refresh_channel(self):
        """Return the channel we should refresh subiquity to."""
        if 'channel' in self.answers:
            return self.answers['channel']
        prefix = "subiquity-channel="
        for arg in self.app.kernel_cmdline:
            if arg.startswith(prefix):
                log.debug(
                    "get_refresh_channel: found %s on kernel cmdline", arg)
                return arg[len(prefix):]
        if 'channel' in self.ai_data:
            return self.ai_data['channel']

        info_file = '/cdrom/.disk/info'
        try:
            fp = open(info_file)
        except FileNotFoundError:
            if self.opts.dry_run:
                info = (
                    'Ubuntu-Server 18.04.2 LTS "Bionic Beaver" - '
                    'Release amd64 (20190214.3)')
            else:
                log.debug(
                    "get_refresh_channel: failed to find .disk/info file")
                return
        else:
            with fp:
                info = fp.read()
        release = info.split()[1]
        return 'stable/ubuntu-' + release

    def snapd_network_changed(self):
        if self.check_state == CheckState.UNKNOWN:
            self.check_task.start_sync()

    @with_context()
    async def check_for_update(self, context):
        await asyncio.shield(self.configure_task)
        if self.app.updated:
            context.description = "not offered update when already updated"
            return CheckState.UNAVAILABLE
        try:
            result = await self.app.snapd.get('v2/find', select='refresh')
        except requests.exceptions.RequestException:
            log.exception("checking for snap update failed")
            context.description = "checking for snap update failed"
            return CheckState.UNAVAILABLE
        log.debug("check_for_update received %s", result)
        for snap in result["result"]:
            if snap["name"] == self.snap_name:
                self.new_snap_version = snap["version"]
                context.description = (
                    "new version of snap available: %r"
                    % self.new_snap_version)
                return CheckState.AVAILABLE
        else:
            context.description = "no new version of snap available"
        return CheckState.UNAVAILABLE

    @with_context()
    async def start_update(self, context):
        open(self.app.state_path('updating'), 'w').close()
        change = await self.app.snapd.post(
            'v2/snaps/{}'.format(self.snap_name),
            {'action': 'refresh'})
        context.description = "change id: {}".format(change)
        return change

    async def get_progress(self, change):
        result = await self.app.snapd.get('v2/changes/{}'.format(change))
        return result['result']

    def start_ui(self, index=1):
        from subiquity.ui.views.refresh import RefreshView
        if self.app.updated:
            raise Skip()
        show = False
        if index == 1:
            if self.check_state == CheckState.AVAILABLE:
                show = True
                self.offered_first_time = True
        elif index == 2:
            if not self.offered_first_time:
                if self.check_state in [CheckState.UNKNOWN,
                                        CheckState.AVAILABLE]:
                    show = True
        else:
            raise AssertionError("unexpected index {}".format(index))
        if show:
            self.ui.set_body(RefreshView(self))
        else:
            raise Skip()

    def done(self, sender=None):
        log.debug("RefreshController.done next_screen")
        self.app.next_screen()

    def cancel(self, sender=None):
        self.app.prev_screen()
<commit_msg>a snap refresh failure means the refresh state is unknown<commit_after># Copyright 2019 Canonical, Ltd.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import asyncio
import enum
import logging
import os

import requests.exceptions

from subiquitycore.async_helpers import (
    schedule_task,
    SingleInstanceTask,
    )
from subiquitycore.context import with_context
from subiquitycore.controller import (
    Skip,
    )

from subiquity.controller import (
    SubiquityController,
    )

log = logging.getLogger('subiquity.controllers.refresh')

class CheckState(enum.IntEnum):
    UNKNOWN = enum.auto()
    AVAILABLE = enum.auto()
    UNAVAILABLE = enum.auto()

class RefreshController(SubiquityController):

    autoinstall_key = "refresh-installer"
    autoinstall_schema = {
        'type': 'object',
        'properties': {
            'update': {'type': 'boolean'},
            'channel': {'type': 'string'},
            },
        'additionalProperties': False,
        }

    signals = [
        ('snapd-network-change', 'snapd_network_changed'),
    ]

    def __init__(self, app):
        self.ai_data = {}
        super().__init__(app)
        self.snap_name = os.environ.get("SNAP_NAME", "subiquity")
        self.configure_task = None
        self.check_task = None

        self.current_snap_version = "unknown"
        self.new_snap_version = ""

        self.offered_first_time = False
        if 'update' in self.ai_data:
            self.active = self.ai_data['update']
        else:
            self.active = self.interactive()

    def load_autoinstall_data(self, data):
        if data is not None:
            self.ai_data = data

    def start(self):
        if not self.active:
            return
        self.configure_task = schedule_task(self.configure_snapd())
        self.check_task = SingleInstanceTask(
            self.check_for_update, propagate_errors=False)
        self.check_task.start_sync()

    @with_context()
    async def apply_autoinstall_config(self, context, index=1):
        if not self.active:
            return
        try:
            await asyncio.wait_for(self.check_task.wait(), 60)
        except asyncio.TimeoutError:
            return
        if self.check_state != CheckState.AVAILABLE:
            return
        change_id = await self.start_update(context=context)
        while True:
            try:
                change = await self.get_progress(change_id)
            except requests.exceptions.RequestException as e:
                raise e
            if change['status'] == 'Done':
                # Clearly if we got here we didn't get restarted by
                # snapd/systemctl (dry-run mode or logged in via SSH)
                self.app.restart(remove_last_screen=False)
            if change['status'] not in ['Do', 'Doing']:
                raise Exception("update failed")
            await asyncio.sleep(0.1)

    @property
    def check_state(self):
        if not self.active:
            return CheckState.UNAVAILABLE
        task = self.check_task.task
        if not task.done() or task.cancelled():
            return CheckState.UNKNOWN
        if task.exception():
            return CheckState.UNAVAILABLE
        return task.result()

    @with_context()
    async def configure_snapd(self, context):
        with context.child("get_details") as subcontext:
            try:
                r = await self.app.snapd.get(
                    'v2/snaps/{snap_name}'.format(
                        snap_name=self.snap_name))
            except requests.exceptions.RequestException:
                log.exception("getting snap details")
                return
            self.current_snap_version = r['result']['version']
            for k in 'channel', 'revision', 'version':
                self.app.note_data_for_apport(
                    "Snap" + k.title(), r['result'][k])
            subcontext.description = "current version of snap is: %r" % (
                self.current_snap_version)
        channel = self.get_refresh_channel()
        desc = "switching {} to {}".format(self.snap_name, channel)
        with context.child("switching", desc) as subcontext:
            try:
                await self.app.snapd.post_and_wait(
                    'v2/snaps/{}'.format(self.snap_name),
                    {'action': 'switch', 'channel': channel})
            except requests.exceptions.RequestException:
                log.exception("switching channels")
                return
            subcontext.description = "switched to " + channel

    def get_refresh_channel(self):
        """Return the channel we should refresh subiquity to."""
        if 'channel' in self.answers:
            return self.answers['channel']
        prefix = "subiquity-channel="
        for arg in self.app.kernel_cmdline:
            if arg.startswith(prefix):
                log.debug(
                    "get_refresh_channel: found %s on kernel cmdline", arg)
                return arg[len(prefix):]
        if 'channel' in self.ai_data:
            return self.ai_data['channel']

        info_file = '/cdrom/.disk/info'
        try:
            fp = open(info_file)
        except FileNotFoundError:
            if self.opts.dry_run:
                info = (
                    'Ubuntu-Server 18.04.2 LTS "Bionic Beaver" - '
                    'Release amd64 (20190214.3)')
            else:
                log.debug(
                    "get_refresh_channel: failed to find .disk/info file")
                return
        else:
            with fp:
                info = fp.read()
        release = info.split()[1]
        return 'stable/ubuntu-' + release

    def snapd_network_changed(self):
        if self.check_state == CheckState.UNKNOWN:
            self.check_task.start_sync()

    @with_context()
    async def check_for_update(self, context):
        await asyncio.shield(self.configure_task)
        if self.app.updated:
            context.description = "not offered update when already updated"
            return CheckState.UNAVAILABLE
        try:
            result = await self.app.snapd.get('v2/find', select='refresh')
        except requests.exceptions.RequestException:
            log.exception("checking for snap update failed")
            context.description = "checking for snap update failed"
            return CheckState.UNKNOWN
        log.debug("check_for_update received %s", result)
        for snap in result["result"]:
            if snap["name"] == self.snap_name:
                self.new_snap_version = snap["version"]
                context.description = (
                    "new version of snap available: %r"
                    % self.new_snap_version)
                return CheckState.AVAILABLE
        else:
            context.description = "no new version of snap available"
        return CheckState.UNAVAILABLE

    @with_context()
    async def start_update(self, context):
        open(self.app.state_path('updating'), 'w').close()
        change = await self.app.snapd.post(
            'v2/snaps/{}'.format(self.snap_name),
            {'action': 'refresh'})
        context.description = "change id: {}".format(change)
        return change

    async def get_progress(self, change):
        result = await self.app.snapd.get('v2/changes/{}'.format(change))
        return result['result']

    def start_ui(self, index=1):
        from subiquity.ui.views.refresh import RefreshView
        if self.app.updated:
            raise Skip()
        show = False
        if index == 1:
            if self.check_state == CheckState.AVAILABLE:
                show = True
                self.offered_first_time = True
        elif index == 2:
            if not self.offered_first_time:
                if self.check_state in [CheckState.UNKNOWN,
                                        CheckState.AVAILABLE]:
                    show = True
        else:
            raise AssertionError("unexpected index {}".format(index))
        if show:
            self.ui.set_body(RefreshView(self))
        else:
            raise Skip()

    def done(self, sender=None):
        log.debug("RefreshController.done next_screen")
        self.app.next_screen()

    def cancel(self, sender=None):
        self.app.prev_screen()
<|endoftext|>