<commit_before>__author__ = 'mengpeng'
import sys
import re
import time
import math
import urllib
from unidecode import unidecode
from pycrawler.handler import Handler
from pycrawler.scraper import DefaultScraper
from pycrawler.utils.tools import gethash
from pycrawler.spider import Driver
from mongojuice.document import Document

SETTINGS = {'name': 'WSJCrawler',
            'spiders': [
                {'name': 'IncSpider',
                 'scraper': {'name': 'DefaultScraper'},
                 'frontier': {'name': 'BFSFrontier'},
                 'handlers': [{'name': 'WSJHandler'}]},
                {'name': 'WordSpider',
                 'scraper': {'name': 'DefaultScraper'},
                 'frontier': {'name': 'BFSFrontier'},
                 'handlers': [{'name': 'WSJHandler'}]}]}

def loadkeywords(filename):
    result = []
    with open(filename, 'r') as f:
        for word in f.readlines():
            word = word.strip()
            result.append(word)
    return result

def lastday(year, month):
    if month in [1, 3, 5, 7, 8, 10, 12]:
        return 31
    elif month in [4, 6, 9, 11]:
        return 30
    else:
        if (year % 100 != 0 and year % 4 == 0) or year % 400 == 0:
            return 29
        else:
            return 28

def date2num(date):
    """Parse date format 01/17/14 into int 20140117"""
    parts = date.split('/')
    if len(parts) != 3:
        parts = time.strftime("%m/%d/%y", time.localtime()).split('/')
    return ((2000 + int(parts[2])) * 100 + int(parts[0])) * 100 + int(parts[1])

class Phase(Document):
    structure = {'keyword': str,
                 'total': int,
                 'pages': int,
                 'start': int,
                 'end': int,
                 'year': int,
                 'month': int,
                 '_id': int}
    given = ['start', 'end', 'keyword', 'total']
    database = 'pycrawler'
    collection = 'wsj'

    def __init__(self, start, end, keyword, total):
        super(Phase, self).__init__()
        self.keyword = keyword
        self.total = total
        self.pages = int(math.ceil(float(total)/15))
        self.start = date2num(start)
        self.end = date2num(end)
        self.year = self.start / 10000
        self.month = self.start % 10000 / 100
        self._id = gethash(str(self.year) + str(self.month) + str(keyword))

class Article(Document):
    structure = {'_id': int,
                 'title': str,
                 'url': str,
                 'date': str,
                 'datenum': int,
                 'tag': str,
                 'keyword': str}
    given = ['title', 'url', 'date', 'tag', 'keyword']
    database = 'pycrawler'
    collection = 'wsj'

    def __init__(self, title, url, date, tag, keyword):
        super(Article, self).__init__()
        self._id = gethash(title + date + keyword)
        self.title = title
        self.url = url
        self.date = self._cleandate(date)
        self.datenum = date2num(date)
        self.tag = tag
        self.keyword = keyword

    def _cleandate(self, date):
        if date.count('/') == 2:
            return date
        else:
            return time.strftime("%m/%d/%y", time.localtime())

@Handler.register
class WSJHandler(Handler):
    def __init__(self, spider):
        super(WSJHandler, self).__init__(spider)
        self._spider = spider
        self.logger = spider.logger
        self.name = spider.name+'-Handler'
        self.args = {'debug': True,
                     'log': True,
                     'logfile': 'WSJHandler.log'}

    def setargs(self, args):
        for key, value in args.iteritems():
            self.args[key] = value

    def parse(self, html, url):
        notfound = 'class="src_no_results"'
        if notfound in html:
            self.logger.warning(self.name, 'No results in '+url)
            return None
        pagep = re.compile('
[0-9of,\-\s]*
')
        titlep = re.compile('<a class="mjLinkItem.*')
        tagp = re.compile('<li class="metadataType-section.*
')
        datep = re.compile('
[0-9/]*')
        page = pagep.findall(html)
        if len(page) != 1:
            self.logger.error(self.name, 'Page index not found in '+url)
            return None
        titles = titlep.findall(html)
        dates = datep.findall(html)
        tags = tagp.findall(html)
        if len(titles) != len(tags):
            tags = ['']*len(titles)
        if len(titles) == len(dates) == len(tags):
            phase = self._parsepage(page[0], url)
            result = [phase] if phase else []
            keyword = self._parsekeyword(url)
            for i in xrange(len(titles)):
                href, title = self._parsetitle(titles[i])
                date = self._parsedate(dates[i])
                tag = self._parsetag(tags[i])
                article = Article(title, href, date, tag, keyword)
                result.append(article)
            return result
        else:
            self.logger.error(self.name, 'Item numbers not match in '+url)
            return None

    def _parsepage(self, page, oriurl):
        page = page[page.index('> ')+2:-5].strip()
        if ',' in page:
            page = page.replace(',', '')
        if page.startswith('1-'):
            url, data = DefaultScraper.parseurl(oriurl)
            keyword = data['KEYWORDS']
            total = int(page.split(' of ')[1])
            phase = Phase(data['fromDate'], data['toDate'], keyword, total)
            urls = []
            for i in xrange(2, phase.pages+1):
                data['page_no'] = i
                urls.append(DefaultScraper.encodeurl('POST', url, data))
            self._spider.addtask(urls)
            return phase

    def _parsekeyword(self, oriurl):
        _, data = DefaultScraper.parseurl(oriurl)
        return data['KEYWORDS']

    def _parsetitle(self, ori):
        href = ori[ori.index('href="')+6:ori.index('">')]
        title = ori[ori.index('">')+2:-4]
        title = unidecode(title.decode('utf-8'))
        return href, title

    def _parsedate(self, ori):
        return ori[-8:]

    def _parsetag(self, ori):
        return ori[ori.index('">')+2:-5] if ori != '' else ori

def main(option, *args):
    if option not in ['start', 'report', 'extract', 'recover']:
        print('Option not supported.')
        return
    if option == 'extract':
        if len(args) != 2:
            print('Usage: python WSJCrawler.py extract logfile outfile')
        else:
            extractlog(args[0], args[1])
        return
    driver = Driver(SETTINGS)
    if option == 'start':
        inckey = loadkeywords('./data/inc.txt')
        url1, url2 = [], []
        years = list(range(2005, 2015))
        for each in iter(inckey):
            url1.extend(generateseeds(each, years))
            url1.extend(generateseeds(each, [2015], [1, 2, 3, 4]))
        wordkey = loadkeywords('./data/word.txt')
        for each in iter(wordkey):
            url2.extend(generateseeds(each, years))
            url2.extend(generateseeds(each, [2015], [1, 2, 3, 4]))
        driver.addtask('IncSpider', url1)
        driver.addtask('WordSpider', url2)
        driver.start()
    elif option == 'report':
        driver.report()
    elif option == 'recover':
        if len(args) != 2:
            print('Usage: python WSJCrawler.py recover spidername urlfile')
        else:
            driver.recover(args[0], args[1])

def generateseeds(keyword, year, month=None):
    base = 'http://online.wsj.com/search/term.html?KEYWORDS=' + urllib.quote(keyword)
    data = {'KEYWORDS': keyword,
            'fromDate': '',
            'toDate': '',
            'source': 'WSJ.com',
            'media': 'All',
            'page_no': '',
            'sorted_by': 'relevance',
            'date_range': '90 days',
            'adv_search': 'open'}
    urls = []
    if not month:
        month = list(xrange(1, 13))
    for y in year:
        for m in month:
            ys = str(y % 100) if y % 100 >= 10 else ('0' + str(y % 100))
            ms = str(m) if m >= 10 else ('0' + str(m))
            d = lastday(y, m)
            ds = str(d) if d >= 10 else ('0' + str(d))
            data['fromDate'] = ms+'/01/'+ys
            data['toDate'] = ms+'/'+ds+'/'+ys
            urls.append(DefaultScraper.encodeurl('POST', base, data))
    return urls

def extractlog(logfile, outfile):
    print('Extracting urls from '+logfile)
    with open(outfile, 'w') as outf:
        with open(logfile, 'r') as inf:
            for each in inf.readlines():
                if '[ERROR]' in each and 'http://' in each:
                    each = each[each.index('http://'):]
                    outf.write(each)
        outf.flush()
    print('Done')

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print('Usage:')
        print('   start:\t python WSJCrawler.py start')
        print('   report:\t python WSJCrawler.py report')
        print('   extract:\t python WSJCrawler.py extract logfile outfile')
        print('   recover:\t python WSJCrawler.py recover spidername urlfile')
    elif len(sys.argv) == 2:
        main(sys.argv[1])
    else:
        main(sys.argv[1], *sys.argv[2:])<commit_msg>add debug option<commit_after>__author__ = 'mengpeng'
import sys
import re
import time
import math
import urllib
from unidecode import unidecode
from pycrawler.handler import Handler
from pycrawler.scraper import DefaultScraper
from pycrawler.utils.tools import gethash
from pycrawler.spider import Driver
from pycrawler.monitor import Monitor
from mongojuice.document import Document

SETTINGS = {'name': 'WSJCrawler',
            'spiders': [
                {'name': 'IncSpider',
                 'scraper': {'name': 'DefaultScraper'},
                 'frontier': {'name': 'BFSFrontier'},
                 'handlers': [{'name': 'WSJHandler'}]},
                {'name': 'WordSpider',
                 'scraper': {'name': 'DefaultScraper'},
                 'frontier': {'name': 'BFSFrontier'},
                 'handlers': [{'name': 'WSJHandler'}]}]}

def loadkeywords(filename):
    result = []
    with open(filename, 'r') as f:
        for word in f.readlines():
            word = word.strip()
            result.append(word)
    return result

def lastday(year, month):
    if month in [1, 3, 5, 7, 8, 10, 12]:
        return 31
    elif month in [4, 6, 9, 11]:
        return 30
    else:
        if (year % 100 != 0 and year % 4 == 0) or year % 400 == 0:
            return 29
        else:
            return 28

def date2num(date):
    """Parse date format 01/17/14 into int 20140117"""
    parts = date.split('/')
    if len(parts) != 3:
        parts = time.strftime("%m/%d/%y", time.localtime()).split('/')
    return ((2000 + int(parts[2])) * 100 + int(parts[0])) * 100 + int(parts[1])

class Phase(Document):
    structure = {'keyword': str,
                 'total': int,
                 'pages': int,
                 'start': int,
                 'end': int,
                 'year': int,
                 'month': int,
                 '_id': int}
    given = ['start', 'end', 'keyword', 'total']
    database = 'pycrawler'
    collection = 'wsj'

    def __init__(self, start, end, keyword, total):
        super(Phase, self).__init__()
        self.keyword = keyword
        self.total = total
        self.pages = int(math.ceil(float(total)/15))
        self.start = date2num(start)
        self.end = date2num(end)
        self.year = self.start / 10000
        self.month = self.start % 10000 / 100
        self._id = gethash(str(self.year) + str(self.month) + str(keyword))

class Article(Document):
    structure = {'_id': int,
                 'title': str,
                 'url': str,
                 'date': str,
                 'datenum': int,
                 'tag': str,
                 'keyword': str}
    given = ['title', 'url', 'date', 'tag', 'keyword']
    database = 'pycrawler'
    collection = 'wsj'

    def __init__(self, title, url, date, tag, keyword):
        super(Article, self).__init__()
        self._id = gethash(title + date + keyword)
        self.title = title
        self.url = url
        self.date = self._cleandate(date)
        self.datenum = date2num(date)
        self.tag = tag
        self.keyword = keyword

    def _cleandate(self, date):
        if date.count('/') == 2:
            return date
        else:
            return time.strftime("%m/%d/%y", time.localtime())

@Handler.register
class WSJHandler(Handler):
    def __init__(self, spider):
        super(WSJHandler, self).__init__(spider)
        self._spider = spider
        self.logger = spider.logger
        self.name = spider.name+'-Handler'
        self.args = {'debug': True,
                     'log': True,
                     'logfile': 'WSJHandler.log'}

    def setargs(self, args):
        for key, value in args.iteritems():
            self.args[key] = value

    def parse(self, html, url):
        notfound = 'class="src_no_results"'
        if notfound in html:
            self.logger.warning(self.name, 'No results in '+url)
            return None
        pagep = re.compile('
[0-9of,\-\s]*
')
        titlep = re.compile('<a class="mjLinkItem.*')
        tagp = re.compile('<li class="metadataType-section.*
')
        datep = re.compile('
[0-9/]*')
        page = pagep.findall(html)
        if len(page) != 1:
            self.logger.error(self.name, 'Page index not found in '+url)
            return None
        titles = titlep.findall(html)
        dates = datep.findall(html)
        tags = tagp.findall(html)
        if len(titles) != len(tags):
            tags = ['']*len(titles)
        if len(titles) == len(dates) == len(tags):
            phase = self._parsepage(page[0], url)
            result = [phase] if phase else []
            keyword = self._parsekeyword(url)
            for i in xrange(len(titles)):
                href, title = self._parsetitle(titles[i])
                date = self._parsedate(dates[i])
                tag = self._parsetag(tags[i])
                article = Article(title, href, date, tag, keyword)
                result.append(article)
            return result
        else:
            self.logger.error(self.name, 'Item numbers not match in '+url)
            return None

    def _parsepage(self, page, oriurl):
        page = page[page.index('> ')+2:-5].strip()
        if ',' in page:
            page = page.replace(',', '')
        if page.startswith('1-'):
            url, data = DefaultScraper.parseurl(oriurl)
            keyword = data['KEYWORDS']
            total = int(page.split(' of ')[1])
            phase = Phase(data['fromDate'], data['toDate'], keyword, total)
            urls = []
            for i in xrange(2, phase.pages+1):
                data['page_no'] = i
                urls.append(DefaultScraper.encodeurl('POST', url, data))
            self._spider.addtask(urls)
            return phase

    def _parsekeyword(self, oriurl):
        _, data = DefaultScraper.parseurl(oriurl)
        return data['KEYWORDS']

    def _parsetitle(self, ori):
        href = ori[ori.index('href="')+6:ori.index('">')]
        title = ori[ori.index('">')+2:-4]
        title = unidecode(title.decode('utf-8'))
        return href, title

    def _parsedate(self, ori):
        return ori[-8:]

    def _parsetag(self, ori):
        return ori[ori.index('">')+2:-5] if ori != '' else ori

def main(option, *args):
    if option not in ['start', 'report', 'extract', 'recover', 'debug']:
        print('Option not supported.')
        return
    if option == 'extract':
        if len(args) != 2:
            print('Usage: python WSJCrawler.py extract logfile outfile')
        else:
            extractlog(args[0], args[1])
        return
    driver = Driver(SETTINGS)
    if option == 'start':
        inckey = loadkeywords('./data/inc.txt')
        url1, url2 = [], []
        years = list(range(2005, 2015))
        for each in iter(inckey):
            url1.extend(generateseeds(each, years))
            url1.extend(generateseeds(each, [2015], [1, 2, 3, 4]))
        wordkey = loadkeywords('./data/word.txt')
        for each in iter(wordkey):
            url2.extend(generateseeds(each, years))
            url2.extend(generateseeds(each, [2015], [1, 2, 3, 4]))
        driver.addtask('IncSpider', url1)
        driver.addtask('WordSpider', url2)
        driver.start()
    elif option == 'report':
        driver.report()
    elif option == 'recover':
        if len(args) != 2:
            print('Usage: python WSJCrawler.py recover spidername urlfile')
        else:
            driver.recover(args[0], args[1])
    elif option == 'debug':
        monitor = Monitor(driver, 'localhost', 6270)
        monitor.start()

def generateseeds(keyword, year, month=None):
    base = 'http://online.wsj.com/search/term.html?KEYWORDS=' + urllib.quote(keyword)
    data = {'KEYWORDS': keyword,
            'fromDate': '',
            'toDate': '',
            'source': 'WSJ.com',
            'media': 'All',
            'page_no': '',
            'sorted_by': 'relevance',
            'date_range': '90 days',
            'adv_search': 'open'}
    urls = []
    if not month:
        month = list(xrange(1, 13))
    for y in year:
        for m in month:
            ys = str(y % 100) if y % 100 >= 10 else ('0' + str(y % 100))
            ms = str(m) if m >= 10 else ('0' + str(m))
            d = lastday(y, m)
            ds = str(d) if d >= 10 else ('0' + str(d))
            data['fromDate'] = ms+'/01/'+ys
            data['toDate'] = ms+'/'+ds+'/'+ys
            urls.append(DefaultScraper.encodeurl('POST', base, data))
    return urls

def extractlog(logfile, outfile):
    print('Extracting urls from '+logfile)
    with open(outfile, 'w') as outf:
        with open(logfile, 'r') as inf:
            for each in inf.readlines():
                if '[ERROR]' in each and 'http://' in each:
                    each = each[each.index('http://'):]
                    outf.write(each)
        outf.flush()
    print('Done')

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print('Usage:')
        print('   start:\t python WSJCrawler.py start')
        print('   report:\t python WSJCrawler.py report')
        print('   extract:\t python WSJCrawler.py extract logfile outfile')
        print('   recover:\t python WSJCrawler.py recover spidername urlfile')
        print('   debug:\t python WSJCrawler.py debug')
    elif len(sys.argv) == 2:
        main(sys.argv[1])
    else:
        main(sys.argv[1], *sys.argv[2:])<|endoftext|>


<commit_before>import h5py
import os
import warnings
import re
import numpy as np
from datetime import datetime
import argparse
import subprocess
import colored
from colored import stylize
parser = argparse.ArgumentParser(description='Quick overview of batch simulation')
parser.add_argument('-S', '--summary', action='store_true', help='Summary only')
parser.add_argument('-s', '--save', action='store_true', help='Save to file')
parser.add_argument('-f', '--filename', type=str, help='Save to file with filename', default='experiment')
parser.add_argument('-t', '--timestamp', action='store_true', help='Add timestamp to filename')
parser.add_argument('-d', '--directory', type=str, help='Search for hdf5 files in directory', default='output')
parser.add_argument('-o', '--outdir', type=str, help='Save output to directory', default='experiments')
parser.add_argument('-x', '--suffix', type=str, help='Append suffix', default='.log')
args = parser.parse_args()

if args.timestamp:
    args.filename = args.filename + '-' + datetime.today().strftime('%Y-%m-%d-%H:%M:%S')

if args.filename != 'experiment' or args.outdir != 'experiments'  :
    args.save = True

try:
    git_rev = "GIT INFO:\n" + str(subprocess.check_output(['git', 'log', '-1']).decode('ascii'))
    print(git_rev)
except:
    git_rev=''

regex = re.compile(r'\d+')

if args.save:
    if not os.path.exists(args.outdir):
        os.makedirs(args.outdir)
    file = open(args.outdir + '/'+ args.filename + args.suffix, 'w')
    file.write(git_rev + '\n')

for dirName, subdirList, fileList in os.walk(args.directory):
    if subdirList:
        subdirList.sort()
    if not fileList:
        continue
    fileList.sort()
    chainlen  = []
    realization_num = []
    variance  = []
    walltime  = []
    resets    = []
    got_stuck = []
    saturated = []
    converged = []
    succeeded = []
    finished  = []

    if not args.summary:
        header = "{:8} {:15} {:>12} {:>12} {:>12} {:>12} {:>12} {:>12} {:>12} {:>12}".format("Length", "Realization", "Variance", "Time",
                                                                                "Resets", "Got Stuck", "Saturated" ,"Converged",
                                                                                "Succeeded", "Finished")
        print(header)
        if args.save:
            file.write(header + '\n')
    for h5path in fileList:
        # print("Filepath: ", h5path)
        try:
            h5file = h5py.File(dirName + '/' + h5path, 'r', swmr=True)
            if not isinstance(h5file, h5py.File):
                raise IOError("File not readable")
        except Exception as er:
            print("Could not open file [", h5path, "] Reason: ", er)
            continue

        try:
            realization_name = h5path.replace('.h5', '')
            chainlen.append(h5file['xDMRG/measurements/length'][-1])
            realization_num.append([int(x) for x in regex.findall(realization_name)][-1])
            variance.append(h5file['xDMRG/measurements/energy_variance_per_site'][-1])
            walltime.append(h5file['xDMRG/sim_status/wall_time'][-1])
            resets.append(h5file['xDMRG/sim_status/num_resets'][-1])
            saturated.append(h5file['xDMRG/sim_status/simulation_has_saturated'][-1])
            converged.append(h5file['xDMRG/sim_status/simulation_has_converged'][-1])
            succeeded.append(h5file['xDMRG/sim_status/simulation_has_succeeded'][-1])
            try:
                finished.append(h5file['common/finOK'][-1])
            except:
                finished.append(0)

            try:
                got_stuck.append(h5file['xDMRG/sim_status/simulation_has_got_stuck'][-1])
            except:
                if finished[-1] == 0 and converged[-1] == 0 and saturated[-1] == 1:
                    got_stuck.append(1)
                else:
                    got_stuck.append(0)

            h5file.close()

            style = ''
            if finished[-1] == 1 and succeeded[-1] == 1:
                style = colored.bg("green_4")
            elif finished[-1] == 1 and converged[-1] == 1:
                style = colored.bg("dark_orange")
            elif finished[-1] == 1 and got_stuck[-1] == 1:
                style = colored.bg("red_3b")
            elif finished[-1] == 0 and got_stuck[-1] == 1:
                style = colored.fg("sandy_brown")
            elif finished[-1] == 0 and got_stuck[-1] == 0 and converged[-1] == 0:
                style = ''
            elif finished[-1] == 0 and got_stuck[-1] == 0 and converged[-1] == 1:
                style = colored.fg("green_4")

            if not args.summary:
                entry = "{:<8} {:<15} {:>12.4f} {:>12.4f} {:>12} {:>12} {:>12} {:>12} {:>12} {:>12}".format(
                    chainlen[-1],
                    realization_num[-1],
                    np.log10(variance[-1]),
                    walltime[-1] / 60,
                    resets[-1],
                    got_stuck[-1],
                    saturated[-1],
                    converged[-1],
                    succeeded[-1],
                    finished[-1])
                print(stylize(entry, style))

                if args.save:
                    file.write(entry + '\n')

        except Exception as er:
            print("Could not read dataset. Reason: ", er)
            continue
    header = "{:8} {:6} {:>12} {:<12} {:>8} {:>12} {:>12} {:>12} {:>12} {:>12} {:>12}".format("Length","Sims", "log10 <Var>", "<log10 Var>", "Avg Time",
                                                                                "Avg Resets","Sum Stk", "Sum Sat", "Sum Con",
                                                                                "Sum Suc", "Sum Fin")
    entry = "{:<8} {:<6} {:>12.4f} {:<12.4f} {:>8.4f} {:>12.3f} {:>12} {:>12} {:>12} {:>12} {:>12}".format(
        np.int(np.mean(chainlen)),
        len(realization_num),
        np.log10(np.nanmean(variance)),
        np.nanmean(np.log10(variance)),
        np.mean(walltime) / 60,
        np.mean(resets),
        np.sum(got_stuck),
        np.sum(saturated),
        np.sum(converged),
        np.sum(succeeded),
        np.sum(finished))

    print(header)
    print(entry)
    if args.save:
        file.write(header + '\n')
        file.write(entry + '\n')
if args.save:
    file.close()
<commit_msg>Updating coloring in quick overview<commit_after>import h5py
import os
import warnings
import re
import numpy as np
from datetime import datetime
import argparse
import subprocess
import colored
from colored import stylize
parser = argparse.ArgumentParser(description='Quick overview of batch simulation')
parser.add_argument('-S', '--summary', action='store_true', help='Summary only')
parser.add_argument('-s', '--save', action='store_true', help='Save to file')
parser.add_argument('-f', '--filename', type=str, help='Save to file with filename', default='experiment')
parser.add_argument('-t', '--timestamp', action='store_true', help='Add timestamp to filename')
parser.add_argument('-d', '--directory', type=str, help='Search for hdf5 files in directory', default='output')
parser.add_argument('-o', '--outdir', type=str, help='Save output to directory', default='experiments')
parser.add_argument('-x', '--suffix', type=str, help='Append suffix', default='.log')
args = parser.parse_args()

if args.timestamp:
    args.filename = args.filename + '-' + datetime.today().strftime('%Y-%m-%d-%H:%M:%S')

if args.filename != 'experiment' or args.outdir != 'experiments'  :
    args.save = True

try:
    git_rev = "GIT INFO:\n" + str(subprocess.check_output(['git', 'log', '-1']).decode('ascii'))
    print(git_rev)
except:
    git_rev=''

regex = re.compile(r'\d+')

if args.save:
    if not os.path.exists(args.outdir):
        os.makedirs(args.outdir)
    file = open(args.outdir + '/'+ args.filename + args.suffix, 'w')
    file.write(git_rev + '\n')

for dirName, subdirList, fileList in os.walk(args.directory):
    if subdirList:
        subdirList.sort()
    if not fileList:
        continue
    fileList.sort()
    chainlen  = []
    realization_num = []
    variance  = []
    walltime  = []
    resets    = []
    got_stuck = []
    saturated = []
    converged = []
    succeeded = []
    finished  = []

    if not args.summary:
        header = "{:8} {:15} {:>12} {:>12} {:>12} {:>12} {:>12} {:>12} {:>12} {:>12}".format("Length", "Realization", "Variance", "Time",
                                                                                "Resets", "Got Stuck", "Saturated" ,"Converged",
                                                                                "Succeeded", "Finished")
        print(header)
        if args.save:
            file.write(header + '\n')
    for h5path in fileList:
        # print("Filepath: ", h5path)
        try:
            h5file = h5py.File(dirName + '/' + h5path, 'r', swmr=True)
            if not isinstance(h5file, h5py.File):
                raise IOError("File not readable")
        except Exception as er:
            print("Could not open file [", h5path, "] Reason: ", er)
            continue

        try:
            realization_name = h5path.replace('.h5', '')
            chainlen.append(h5file['xDMRG/measurements/length'][-1])
            realization_num.append([int(x) for x in regex.findall(realization_name)][-1])
            variance.append(h5file['xDMRG/measurements/energy_variance_per_site'][-1])
            walltime.append(h5file['xDMRG/sim_status/wall_time'][-1])
            resets.append(h5file['xDMRG/sim_status/num_resets'][-1])
            saturated.append(h5file['xDMRG/sim_status/simulation_has_saturated'][-1])
            converged.append(h5file['xDMRG/sim_status/simulation_has_converged'][-1])
            succeeded.append(h5file['xDMRG/sim_status/simulation_has_succeeded'][-1])
            try:
                finished.append(h5file['common/finOK'][-1])
            except:
                finished.append(0)

            try:
                got_stuck.append(h5file['xDMRG/sim_status/simulation_has_got_stuck'][-1])
            except:
                if finished[-1] == 0 and converged[-1] == 0 and saturated[-1] == 1:
                    got_stuck.append(1)
                else:
                    got_stuck.append(0)

            h5file.close()

            style = ''
            if finished[-1] == 1 and succeeded[-1] == 1:
                style = colored.bg("green_4")
            elif finished[-1] == 1 and converged[-1] == 1:
                style = colored.bg("dark_orange")
            elif finished[-1] == 1 and got_stuck[-1] == 1:
                style = colored.bg("red_3b")
            elif finished[-1] == 0 and got_stuck[-1] == 1:
                style = colored.fg("dark_orange")
            elif finished[-1] == 0 and got_stuck[-1] == 0 and converged[-1] == 0:
                style = ''
            elif finished[-1] == 0 and got_stuck[-1] == 0 and converged[-1] == 1:
                style = colored.fg("green_4")

            if not args.summary:
                entry = "{:<8} {:<15} {:>12.4f} {:>12.4f} {:>12} {:>12} {:>12} {:>12} {:>12} {:>12}".format(
                    chainlen[-1],
                    realization_num[-1],
                    np.log10(variance[-1]),
                    walltime[-1] / 60,
                    resets[-1],
                    got_stuck[-1],
                    saturated[-1],
                    converged[-1],
                    succeeded[-1],
                    finished[-1])
                print(stylize(entry, style))

                if args.save:
                    file.write(entry + '\n')

        except Exception as er:
            print("Could not read dataset. Reason: ", er)
            continue
    header = "{:8} {:6} {:>12} {:<12} {:>8} {:>12} {:>12} {:>12} {:>12} {:>12} {:>12}".format("Length","Sims", "log10 <Var>", "<log10 Var>", "Avg Time",
                                                                                "Avg Resets","Sum Stk", "Sum Sat", "Sum Con",
                                                                                "Sum Suc", "Sum Fin")
    entry = "{:<8} {:<6} {:>12.4f} {:<12.4f} {:>8.4f} {:>12.3f} {:>12} {:>12} {:>12} {:>12} {:>12}".format(
        np.int(np.mean(chainlen)),
        len(realization_num),
        np.log10(np.nanmean(variance)),
        np.nanmean(np.log10(variance)),
        np.mean(walltime) / 60,
        np.mean(resets),
        np.sum(got_stuck),
        np.sum(saturated),
        np.sum(converged),
        np.sum(succeeded),
        np.sum(finished))

    print(header)
    print(entry)
    if args.save:
        file.write(header + '\n')
        file.write(entry + '\n')
if args.save:
    file.close()
<|endoftext|>


<commit_before>import string
import random
import socket
import shutil
import os
import sys
import hashlib
import time
import Constant as const
import Function as func
import Package as pack

####### STRINGHE

# Format string completa text con char per ottenere una stringa di lunghezza length
# Tested, fondamentale che il text passato sia stringa e la length un numero.
def format_string(text, length, char):
	l = len(text)
	dif = length - l
	return char * dif + text

def reverse_format_string(text, length, char):
	l = len(text)
	dif = length - l
	return text + char * dif

def reformat_string(text):
	return text.strip()

def	write_right_text(text):
	print("")
	print(str(text).rjust(shutil.get_terminal_size((80, 20))[0]))

def write_daemon_error(host, addr, text):
	write_right_text(">>> " + host + " [" + addr + "]: " + const.START_RED + "ERROR: " + text + const.END_RED)

def write_daemon_success(host, addr, text):
	write_right_text(">>> " + host + " [" + addr + "]: " + const.START_GREEN + "SUCCESS: " + text + const.END_GREEN)

def write_daemon_text(host, addr, text):
	write_right_text(">>>  " + host + " [" + addr + "]: " + text)

def error(text):
	print (const.START_RED + "Error: " + text + const.END_RED)

def success(text):
	print (const.START_GREEN + "Success: " + text + const.END_GREEN)

def warning(text):
	print (const.START_YELLOW + text + const.END_YELLOW)

def gtext(text):
	print (const.START_GREY + text + const.END_GREY)

# Random Functions
def random_pktid(length):
   return ''.join(random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789') for i in range(length))

def random_sessionID(length):
   return ''.join(random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789') for i in range(length))

def writeHelp():
	func.warning("\nPOSSIBILI ARGOMENTI:")
	print("Super Nodo\t-sn")
	print("Set Default Ip\t-ip group identifier")
	print("Change Port\t-p port")
	print("Change Port SN\t-pSN port")
	print("Change time\t-t time")
	print("Change ttl\t-ttl ttl")
	print("")
	sys.exit(-1)

####### SOCKET

def create_socket_server(myHost, port):
	s = None
	for res in socket.getaddrinfo(None, int(port), socket.AF_UNSPEC,socket.SOCK_STREAM, 0, socket.AI_PASSIVE):
	    af, socktype, proto, canonname, sa = res
	    try:
	    	s = socket.socket(af, socktype, proto)
	    except socket.error as msg:
	    	s = None
	    	continue
	    try:
	    	s.bind(sa)
	    	s.listen(10)
	    except socket.error as msg:
	    	s.close()
	    	s = None
	    	continue
	    break
	return s

def create_socket_client(myHost, port):
	s = None
	for res in socket.getaddrinfo(myHost, int(port), socket.AF_UNSPEC, socket.SOCK_STREAM):
	    af, socktype, proto, canonname, sa = res
	    try:
	        s = socket.socket(af, socktype, proto)
	    except socket.error as msg:
	        s = None
	        continue
	    try:
	        s.connect(sa)
	    except socket.error as msg:
	        s.close()
	        s = None
	        continue
	    break
	return s

def forward(pk, addr, l): # Non andrebbe fatto generico?
	if pk != bytes(const.ERROR_PKT, "ascii"):
		for x in l:
			if addr != x[0]:
				s = func.create_socket_client(func.roll_the_dice(x[0]), x[1])
				if not(s is None):
					s.sendall(pk)
					write_daemon_success("Daemon", "-", "Forward da " + addr + " a " + x[0])
					s.close()

###### IP

def roll_the_dice(ip):
	return random.choice([ip[0:15], ip[16:55]])

def get_ipv4(ip):
	return ip[0:15]

def get_ipv6(ip):
	return ip[16:55]

###### SEARCH FILE

def search_file(query, listFiles, listUsers):
	listResultQuery = []
	for f in listFiles:
		if query in f[1]:
			for i in listUsers:
				if i[2] == f[2]:
					listResultQuery.append([f[0], bytes(func.format_string(str(f[1],"ascii"), const.LENGTH_FILENAME, " "),"ascii"), i[0], i[1]])
					break
	return listResultQuery

def add_pktid(pktid, list_pkt, port):
	list_pkt = clear_pktid(list_pkt)
	for lista in list_pkt:
		if (pktid == lista[0]):# and (port == lista[2]):
			return False
	pkTime = time.time() * 1000
	add_list = [pktid, pkTime, port]
	list_pkt.append(add_list)
	return True

def clear_pktid(list_pkt):
	x = 0
	for i in list_pkt:
		pkTime = i[1]
		nowtime = time.time() * 1000
		diff = nowtime - pkTime
		if diff >= const.MAX_TIME:
			del list_pkt[x]
			x -= 1
		x += 1
	return list_pkt

def check_query(pktid, list_pkt, port):
	list_pkt = clear_pktid(list_pkt)
	for lista in list_pkt:
		if (pktid == lista[0]):# and (port == lista[2]):
			return True
	return False

def check_sn(pktid, list_pkt):
	list_pkt = clear_pktid(list_pkt)
	for lista in list_pkt:
		if pktid == lista[0]:
			return True
	return False

def remove_pktid(pktid, list_pkt):
	i = 0
	for lista in list_pkt:
		if pktid == lista[0]:
			del list_pkt[i]
			i -= 1
		i += 1

###### UPLOAD FILE

def upload(nomeFile, ss):
	f = open((const.FILE_COND + nomeFile), 'rb')

	fileLength = os.stat(const.FILE_COND + nomeFile).st_size
	nChunk = int(fileLength / const.LENGTH_PACK) + 1

	nChunk = format_string(str(nChunk), const.LENGTH_NCHUNKS, "0")
	pk = bytes(const.CODE_ANSWER_DOWNLOAD, "ascii") + bytes(nChunk, "ascii")
	ss.sendall(pk)

	i = 0
	while True:
		line = f.read(const.LENGTH_PACK)
		dimLine = format_string(str(len(line)), const.LENGTH_NCHUNK, "0")
		pk = bytes(dimLine, "ascii") + line
		ss.sendall(pk)
		#print(pack)
		i = i + 1
		if i == int(nChunk):
			break

def find_file_by_md5(md5):
	file_list = os.listdir(const.FILE_COND)
	for filef in file_list:
		if not filef.endswith('~'):
			md5File = hashlib.md5(open(const.FILE_COND + filef,'rb').read()).hexdigest()
			if str(md5, "ascii") == md5File:
				return filef
	return const.ERROR_FILE

###### DOWNLOAD FILE

# Funzione di download
def download(selectFile):
	print ("\n>>> DOWNLOAD")

	md5 = selectFile[1]
	nomeFile = selectFile[2].decode("ascii").strip()
	ip = selectFile[3]
	port = selectFile[4]

	# Con probabilità 0.5 invio su IPv4, else IPv6
	ip = func.roll_the_dice(ip.decode("ascii"))
	print("Connessione con:", ip)

	# Mi connetto al peer

	sP = func.create_socket_client(ip, port)
	if sP is None:
	    print ('Error: could not open socket in download')
	else:
		pk = pack.request_download(md5)
		sP.sendall(pk)

		nChunk = int(sP.recv(const.LENGTH_HEADER)[4:10])

		ricevutoByte = b''

		i = 0

		while i != nChunk:
			ricevutoLen = sP.recv(const.LENGTH_NCHUNK)
			while (len(ricevutoLen) < const.LENGTH_NCHUNK):
				ricevutoLen = ricevutoLen + sP.recv(const.LENGTH_NCHUNK - len(ricevutoLen))
			buff = sP.recv(int(ricevutoLen))
			while(len(buff) < int(ricevutoLen)):
				buff = buff + sP.recv(int(ricevutoLen) - len(buff))
			ricevutoByte = ricevutoByte + buff
			i = i + 1

		sP.close()

		# Salvare il file data
		open((const.FILE_COND + nomeFile),'wb').write(ricevutoByte)
		print("File scaricato correttamente, apertura in corso...")
		try:
			os.system("open " + const.FILE_COND + nomeFile)
		except:
			try:
				os.system("start " + const.FILE_COND + nomeFile)
			except:
				print("Apertura non riuscita")

# Scelta random tra Supernodi

def choose_SN(sn_network):
	choose = random.sample(sn_network, 1)
	choose = choose[0]
	for n in sn_network:
		if not n == choose:
			del n
	return choose

####### USERS
def countUserFile(sessionID, listFiles):
	i = 0
	for f in listFiles:
		if f[2] == sessionID:
			i += 1
	return i

def isUserLogged(sessionID, listUsers):
	for user in listUsers:
		if sessionID == user[2]:
			return True
	return False

def reconnect_user(ip, listUsers):
	pk = const.ERROR_PKT
	for user in listUsers:
		if user[0] == ip:
			pk = pack.answer_login()[:4] + user[2]
			break
	return pk

def check_file(listFiles, ricevutoByte):
	fileConfronto = [ricevutoByte[20:52], ricevutoByte[52:152], ricevutoByte[4:20]]
	for file in listFiles:
		if (file[0] == fileConfronto[0]) and (file[2] == fileConfronto[2]):
			return False
	return True

def send_afin(conn, listResultQuery):
	if len(listResultQuery) != 0:
		nMd5 = 0

		listResultQuery.sort()

		md5 = b''
		for x in listResultQuery:
			if md5 != x[0]:
				md5 = x[0]
				nMd5 += 1

		pk = bytes(const.CODE_ANSWER_SEARCH, "ascii") + bytes(func.format_string(str(nMd5), const.LENGTH_NIDMD5, "0"), "ascii")

		actualMd5 = b''

		print(listResultQuery)

		for i in listResultQuery:
			if i[0] != actualMd5:
				actualMd5 = listResultQuery[0][0]
				# calcolo numero copie
				copy = 0
				for x in listResultQuery:
					if x[0] == actualMd5:
						copy += 1

				pk = pk + i[0] + i[1] + bytes(func.format_string(str(copy), const.LENGTH_NCOPY, "0"), "ascii")
			pk = pk + i[2] + i[3]

	else:
		pk = bytes(const.CODE_ANSWER_SEARCH, "ascii") + bytes("0" * const.LENGTH_NIDMD5, "ascii")

	conn.sendall(pk)

<commit_msg>Testing 1.27:<commit_after>import string
import random
import socket
import shutil
import os
import sys
import hashlib
import time
import Constant as const
import Function as func
import Package as pack

####### STRINGHE

# Format string completa text con char per ottenere una stringa di lunghezza length
# Tested, fondamentale che il text passato sia stringa e la length un numero.
def format_string(text, length, char):
	l = len(text)
	dif = length - l
	return char * dif + text

def reverse_format_string(text, length, char):
	l = len(text)
	dif = length - l
	return text + char * dif

def reformat_string(text):
	return text.strip()

def	write_right_text(text):
	print("")
	print(str(text).rjust(shutil.get_terminal_size((80, 20))[0]))

def write_daemon_error(host, addr, text):
	write_right_text(">>> " + host + " [" + addr + "]: " + const.START_RED + "ERROR: " + text + const.END_RED)

def write_daemon_success(host, addr, text):
	write_right_text(">>> " + host + " [" + addr + "]: " + const.START_GREEN + "SUCCESS: " + text + const.END_GREEN)

def write_daemon_text(host, addr, text):
	write_right_text(">>>  " + host + " [" + addr + "]: " + text)

def error(text):
	print (const.START_RED + "Error: " + text + const.END_RED)

def success(text):
	print (const.START_GREEN + "Success: " + text + const.END_GREEN)

def warning(text):
	print (const.START_YELLOW + text + const.END_YELLOW)

def gtext(text):
	print (const.START_GREY + text + const.END_GREY)

# Random Functions
def random_pktid(length):
   return ''.join(random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789') for i in range(length))

def random_sessionID(length):
   return ''.join(random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789') for i in range(length))

def writeHelp():
	func.warning("\nPOSSIBILI ARGOMENTI:")
	print("Super Nodo\t-sn")
	print("Set Default Ip\t-ip group identifier")
	print("Change Port\t-p port")
	print("Change Port SN\t-pSN port")
	print("Change time\t-t time")
	print("Change ttl\t-ttl ttl")
	print("")
	sys.exit(-1)

####### SOCKET

def create_socket_server(myHost, port):
	s = None
	for res in socket.getaddrinfo(None, int(port), socket.AF_UNSPEC,socket.SOCK_STREAM, 0, socket.AI_PASSIVE):
	    af, socktype, proto, canonname, sa = res
	    try:
	    	s = socket.socket(af, socktype, proto)
	    except socket.error as msg:
	    	s = None
	    	continue
	    try:
	    	s.bind(sa)
	    	s.listen(10)
	    except socket.error as msg:
	    	s.close()
	    	s = None
	    	continue
	    break
	return s

def create_socket_client(myHost, port):
	s = None
	for res in socket.getaddrinfo(myHost, int(port), socket.AF_UNSPEC, socket.SOCK_STREAM):
	    af, socktype, proto, canonname, sa = res
	    try:
	        s = socket.socket(af, socktype, proto)
	    except socket.error as msg:
	        s = None
	        continue
	    try:
	        s.connect(sa)
	    except socket.error as msg:
	        s.close()
	        s = None
	        continue
	    break
	return s

def forward(pk, addr, l): # Non andrebbe fatto generico?
	if pk != bytes(const.ERROR_PKT, "ascii"):
		for x in l:
			if addr != x[0]:
				s = func.create_socket_client(func.roll_the_dice(x[0]), x[1])
				if not(s is None):
					s.sendall(pk)
					#write_daemon_success("Daemon", "-", "Forward da " + addr + " a " + x[0])
					s.close()

###### IP

def roll_the_dice(ip):
	return random.choice([ip[0:15], ip[16:55]])

def get_ipv4(ip):
	return ip[0:15]

def get_ipv6(ip):
	return ip[16:55]

###### SEARCH FILE

def search_file(query, listFiles, listUsers):
	listResultQuery = []
	for f in listFiles:
		if query in f[1]:
			for i in listUsers:
				if i[2] == f[2]:
					listResultQuery.append([f[0], bytes(func.format_string(str(f[1],"ascii"), const.LENGTH_FILENAME, " "),"ascii"), i[0], i[1]])
					break
	return listResultQuery

def add_pktid(pktid, list_pkt, port):
	list_pkt = clear_pktid(list_pkt)
	for lista in list_pkt:
		if (pktid == lista[0]):# and (port == lista[2]):
			return False
	pkTime = time.time() * 1000
	add_list = [pktid, pkTime, port]
	list_pkt.append(add_list)
	return True

def clear_pktid(list_pkt):
	x = 0
	for i in list_pkt:
		pkTime = i[1]
		nowtime = time.time() * 1000
		diff = nowtime - pkTime
		if diff >= const.MAX_TIME:
			del list_pkt[x]
			x -= 1
		x += 1
	return list_pkt

def check_query(pktid, list_pkt, port):
	list_pkt = clear_pktid(list_pkt)
	for lista in list_pkt:
		if (pktid == lista[0]):# and (port == lista[2]):
			return True
	return False

def check_sn(pktid, list_pkt):
	list_pkt = clear_pktid(list_pkt)
	for lista in list_pkt:
		if pktid == lista[0]:
			return True
	return False

def remove_pktid(pktid, list_pkt):
	i = 0
	for lista in list_pkt:
		if pktid == lista[0]:
			del list_pkt[i]
			i -= 1
		i += 1

###### UPLOAD FILE

def upload(nomeFile, ss):
	f = open((const.FILE_COND + nomeFile), 'rb')

	fileLength = os.stat(const.FILE_COND + nomeFile).st_size
	nChunk = int(fileLength / const.LENGTH_PACK) + 1

	nChunk = format_string(str(nChunk), const.LENGTH_NCHUNKS, "0")
	pk = bytes(const.CODE_ANSWER_DOWNLOAD, "ascii") + bytes(nChunk, "ascii")
	ss.sendall(pk)

	i = 0
	while True:
		line = f.read(const.LENGTH_PACK)
		dimLine = format_string(str(len(line)), const.LENGTH_NCHUNK, "0")
		pk = bytes(dimLine, "ascii") + line
		ss.sendall(pk)
		#print(pack)
		i = i + 1
		if i == int(nChunk):
			break

def find_file_by_md5(md5):
	file_list = os.listdir(const.FILE_COND)
	for filef in file_list:
		if not filef.endswith('~'):
			md5File = hashlib.md5(open(const.FILE_COND + filef,'rb').read()).hexdigest()
			if str(md5, "ascii") == md5File:
				return filef
	return const.ERROR_FILE

###### DOWNLOAD FILE

# Funzione di download
def download(selectFile):
	print ("\n>>> DOWNLOAD")

	md5 = selectFile[1]
	nomeFile = selectFile[2].decode("ascii").strip()
	ip = selectFile[3]
	port = selectFile[4]

	# Con probabilità 0.5 invio su IPv4, else IPv6
	ip = func.roll_the_dice(ip.decode("ascii"))
	print("Connessione con:", ip)

	# Mi connetto al peer

	sP = func.create_socket_client(ip, port)
	if sP is None:
	    print ('Error: could not open socket in download')
	else:
		pk = pack.request_download(md5)
		sP.sendall(pk)

		nChunk = int(sP.recv(const.LENGTH_HEADER)[4:10])

		ricevutoByte = b''

		i = 0

		while i != nChunk:
			ricevutoLen = sP.recv(const.LENGTH_NCHUNK)
			while (len(ricevutoLen) < const.LENGTH_NCHUNK):
				ricevutoLen = ricevutoLen + sP.recv(const.LENGTH_NCHUNK - len(ricevutoLen))
			buff = sP.recv(int(ricevutoLen))
			while(len(buff) < int(ricevutoLen)):
				buff = buff + sP.recv(int(ricevutoLen) - len(buff))
			ricevutoByte = ricevutoByte + buff
			i = i + 1

		sP.close()

		# Salvare il file data
		open((const.FILE_COND + nomeFile),'wb').write(ricevutoByte)
		print("File scaricato correttamente, apertura in corso...")
		try:
			os.system("open " + const.FILE_COND + nomeFile)
		except:
			try:
				os.system("start " + const.FILE_COND + nomeFile)
			except:
				print("Apertura non riuscita")

# Scelta random tra Supernodi

def choose_SN(sn_network):
	choose = random.sample(sn_network, 1)
	choose = choose[0]
	for n in sn_network:
		if not n == choose:
			del n
	return choose

####### USERS
def countUserFile(sessionID, listFiles):
	i = 0
	for f in listFiles:
		if f[2] == sessionID:
			i += 1
	return i

def isUserLogged(sessionID, listUsers):
	for user in listUsers:
		if sessionID == user[2]:
			return True
	return False

def reconnect_user(ip, listUsers):
	pk = const.ERROR_PKT
	for user in listUsers:
		if user[0] == ip:
			pk = pack.answer_login()[:4] + user[2]
			break
	return pk

def check_file(listFiles, ricevutoByte):
	fileConfronto = [ricevutoByte[20:52], ricevutoByte[52:152], ricevutoByte[4:20]]
	for file in listFiles:
		if (file[0] == fileConfronto[0]) and (file[2] == fileConfronto[2]):
			return False
	return True

def send_afin(conn, listResultQuery):
	if len(listResultQuery) != 0:
		nMd5 = 0

		listResultQuery.sort()

		md5 = b''
		for x in listResultQuery:
			if md5 != x[0]:
				md5 = x[0]
				nMd5 += 1

		pk = bytes(const.CODE_ANSWER_SEARCH, "ascii") + bytes(func.format_string(str(nMd5), const.LENGTH_NIDMD5, "0"), "ascii")

		actualMd5 = b''

		for i in listResultQuery:
			if i[0] != actualMd5:
				actualMd5 = listResultQuery[0][0]
				# calcolo numero copie
				copy = 0
				for x in listResultQuery:
					if x[0] == actualMd5:
						copy += 1

				pk = pk + i[0] + i[1] + bytes(func.format_string(str(copy), const.LENGTH_NCOPY, "0"), "ascii")
			pk = pk + i[2] + i[3]

	else:
		pk = bytes(const.CODE_ANSWER_SEARCH, "ascii") + bytes("0" * const.LENGTH_NIDMD5, "ascii")

	conn.sendall(pk)

<|endoftext|>


<commit_before>import unittest, time, sys, os
sys.path.extend(['.','..','py'])
import h2o, h2o_cmd, h2o_hosts, h2o_rf, h2o_util, h2o_import as h2i

USE_LOCAL = True

# RF train parameters

# RF test parameters

trainDS1 = {
    'bucket'      : 'home-0xdiag-datasets',
    'pathname'    : 'standard/covtype.shuffled.90pct.sorted.data',
    'timeoutSecs' : 60,
    'header'      : 0
    }

scoreDS1 = {
    'bucket'      : 'home-0xdiag-datasets',
    'pathname'    : 'standard/covtype.shuffled.10pct.sorted.data',
    'timeoutSecs' : 60,
    'header'      : 0
    }

trainDS2 = {
    'bucket'      : 'home-0xdiag-datasets',
    'pathname'    : 'standard/covtype.shuffled.90pct.data',
    'timeoutSecs' : 60,
    'header'      : 0
    }

scoreDS2 = {
    'bucket'      : 'home-0xdiag-datasets',
    'pathname'    : 'standard/covtype.shuffled.10pct.data',
    'timeoutSecs' : 60,
    'header'      : 0
    }

class Basic(unittest.TestCase):

    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        global localhost
        localhost = h2o.decide_if_localhost()
        if (localhost):
            h2o.build_cloud(2, java_heap_GB=7)
        else:
            h2o_hosts.build_cloud_with_hosts()

    @classmethod
    def tearDownClass(cls):
        h2o.tear_down_cloud()

    def parseFile(self, bucket, pathname, timeoutSecs, header, **kwargs):
        # this can get redirected
        if USE_LOCAL:
            schema = None
        else:
            schema = 's3n'

        start = time.time()
        parseResult = h2i.import_parse(bucket=bucket, path=pathname, schema='local', timeoutSecs=180)
        parse_time = time.time() - start
        h2o.verboseprint("parse took {0} sec".format(parse_time))
        parseResult['python_call_timer'] = parse_time
        return parseResult

    def loadData(self, params):
        kwargs   = params.copy()
        trainKey = self.parseFile(**kwargs)
        return trainKey

    def test_RF(self):
        h2o.beta_features = True
        paramsTrainRF = {
            'seed': '1234567890',
            # if I use 100, and just one tree, I should get same results for sorted/shuffled?
            # i.e. the bagging always sees everything. Means oobe will be messed up
            # so will specify validation = the 10pct holdout data (could reuse the training data?)
            'sample_rate': 1.0,
            'ntrees': 3,
            'max_depth': 300,
            'nbins': 200,
            'timeoutSecs': 600,
            'response': 'C55',
        }

        paramsScoreRF = {
            'vactual': 'C55',
            'timeoutSecs': 600,
        }

        # 90% data
        trainKey1 = self.loadData(trainDS1)
        scoreKey1 = self.loadData(scoreDS1)
        kwargs   = paramsTrainRF.copy()
        trainResult1 = h2o_rf.trainRF(trainKey1, scoreKey1, **kwargs)
        (classification_error1, classErrorPctList1, totalScores1) = h2o_rf.simpleCheckRFView(rfv=trainResult1)
        self.assertEqual(4.29, classification_error1)
        self.assertEqual([4.17, 2.98, 4.09, 14.91, 21.12, 15.38, 5.22], classErrorPctList1)
        self.assertEqual(58101, totalScores1)

        kwargs   = paramsScoreRF.copy()
        scoreResult1 = h2o_rf.scoreRF(scoreKey1, trainResult1, **kwargs)

        # 10% data
        trainKey2 = self.loadData(trainDS2)
        scoreKey2 = self.loadData(scoreDS2)
        kwargs   = paramsTrainRF.copy()
        trainResult2 = h2o_rf.trainRF(trainKey2, scoreKey2, **kwargs)
        (classification_error2, classErrorPctList2, totalScores2) = h2o_rf.simpleCheckRFView(rfv=trainResult2)
        self.assertEqual(4.29, classification_error2)
        self.assertEqual([4.17, 2.98, 4.09, 14.91, 21.12, 15.38, 5.22], classErrorPctList2)
        self.assertEqual(58101, totalScores2)

        kwargs   = paramsScoreRF.copy()
        scoreResult2 = h2o_rf.scoreRF(scoreKey2, trainResult2, **kwargs)

        print "\nTraining: JsonDiff sorted data results, to non-sorted results (json responses)"
        df = h2o_util.JsonDiff(trainResult1, trainResult2, with_values=True)
        print "df.difference:", h2o.dump_json(df.difference)

        print "\nScoring: JsonDiff sorted data results, to non-sorted results (json responses)"
        df = h2o_util.JsonDiff(scoreResult1, scoreResult2, with_values=True)
        print "df.difference:", h2o.dump_json(df.difference)

        # should only be two diffs
        if len(df.difference) > 2:
            raise Exception ("Too many diffs in JsonDiff sorted vs non-sorted %s" % len(df.difference))
        # Scoring: JsonDiff sorted data results, to non-sorted results (json responses)
        # df.difference: [
        # "diff: response_info.time - 28 | 11",
        # "diff: python_call_timer - 0.526123046875 | 0.498980998993"
        # ]

if __name__ == '__main__':
    h2o.unit_main()
<commit_msg>change expected RF error to match results with latest RNG change<commit_after>import unittest, time, sys, os
sys.path.extend(['.','..','py'])
import h2o, h2o_cmd, h2o_hosts, h2o_rf, h2o_util, h2o_import as h2i

USE_LOCAL = True

# RF train parameters

# RF test parameters

trainDS1 = {
    'bucket'      : 'home-0xdiag-datasets',
    'pathname'    : 'standard/covtype.shuffled.90pct.sorted.data',
    'timeoutSecs' : 60,
    'header'      : 0
    }

scoreDS1 = {
    'bucket'      : 'home-0xdiag-datasets',
    'pathname'    : 'standard/covtype.shuffled.10pct.sorted.data',
    'timeoutSecs' : 60,
    'header'      : 0
    }

trainDS2 = {
    'bucket'      : 'home-0xdiag-datasets',
    'pathname'    : 'standard/covtype.shuffled.90pct.data',
    'timeoutSecs' : 60,
    'header'      : 0
    }

scoreDS2 = {
    'bucket'      : 'home-0xdiag-datasets',
    'pathname'    : 'standard/covtype.shuffled.10pct.data',
    'timeoutSecs' : 60,
    'header'      : 0
    }

class Basic(unittest.TestCase):

    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        global localhost
        localhost = h2o.decide_if_localhost()
        if (localhost):
            h2o.build_cloud(2, java_heap_GB=7)
        else:
            h2o_hosts.build_cloud_with_hosts()

    @classmethod
    def tearDownClass(cls):
        h2o.tear_down_cloud()

    def parseFile(self, bucket, pathname, timeoutSecs, header, **kwargs):
        # this can get redirected
        if USE_LOCAL:
            schema = None
        else:
            schema = 's3n'

        start = time.time()
        parseResult = h2i.import_parse(bucket=bucket, path=pathname, schema='local', timeoutSecs=180)
        parse_time = time.time() - start
        h2o.verboseprint("parse took {0} sec".format(parse_time))
        parseResult['python_call_timer'] = parse_time
        return parseResult

    def loadData(self, params):
        kwargs   = params.copy()
        trainKey = self.parseFile(**kwargs)
        return trainKey

    def test_RF(self):
        h2o.beta_features = True
        paramsTrainRF = {
            'seed': '1234567890',
            # if I use 100, and just one tree, I should get same results for sorted/shuffled?
            # i.e. the bagging always sees everything. Means oobe will be messed up
            # so will specify validation = the 10pct holdout data (could reuse the training data?)
            'sample_rate': 1.0,
            'ntrees': 3,
            'max_depth': 300,
            'nbins': 200,
            'timeoutSecs': 600,
            'response': 'C55',
        }

        paramsScoreRF = {
            'vactual': 'C55',
            'timeoutSecs': 600,
        }

        # 90% data
        trainKey1 = self.loadData(trainDS1)
        scoreKey1 = self.loadData(scoreDS1)
        kwargs   = paramsTrainRF.copy()
        trainResult1 = h2o_rf.trainRF(trainKey1, scoreKey1, **kwargs)
        (classification_error1, classErrorPctList1, totalScores1) = h2o_rf.simpleCheckRFView(rfv=trainResult1)
        # self.assertEqual(4.29, classification_error1)
        # self.assertEqual([4.17, 2.98, 4.09, 14.91, 21.12, 15.38, 5.22], classErrorPctList1)
        # with new RNG 9/26/14
        self.assertEqual(4.4, classification_error1)
        self.assertEqual([3.71, 3.56, 4.32, 18.55, 21.22, 13.51, 5.82], classErrorPctList1)
        self.assertEqual(58101, totalScores1)

        kwargs   = paramsScoreRF.copy()
        scoreResult1 = h2o_rf.scoreRF(scoreKey1, trainResult1, **kwargs)

        # 10% data
        trainKey2 = self.loadData(trainDS2)
        scoreKey2 = self.loadData(scoreDS2)
        kwargs   = paramsTrainRF.copy()
        trainResult2 = h2o_rf.trainRF(trainKey2, scoreKey2, **kwargs)
        (classification_error2, classErrorPctList2, totalScores2) = h2o_rf.simpleCheckRFView(rfv=trainResult2)
        # self.assertEqual(4.29, classification_error2)
        # self.assertEqual([4.17, 2.98, 4.09, 14.91, 21.12, 15.38, 5.22], classErrorPctList2)
        # with new RNG 9/26/14
        self.assertEqual(4.4, classification_error1)
        self.assertEqual([3.71, 3.56, 4.32, 18.55, 21.22, 13.51, 5.82], classErrorPctList1)
        self.assertEqual(58101, totalScores2)

        kwargs   = paramsScoreRF.copy()
        scoreResult2 = h2o_rf.scoreRF(scoreKey2, trainResult2, **kwargs)

        print "\nTraining: JsonDiff sorted data results, to non-sorted results (json responses)"
        df = h2o_util.JsonDiff(trainResult1, trainResult2, with_values=True)
        print "df.difference:", h2o.dump_json(df.difference)

        print "\nScoring: JsonDiff sorted data results, to non-sorted results (json responses)"
        df = h2o_util.JsonDiff(scoreResult1, scoreResult2, with_values=True)
        print "df.difference:", h2o.dump_json(df.difference)

        # should only be two diffs
        if len(df.difference) > 2:
            raise Exception ("Too many diffs in JsonDiff sorted vs non-sorted %s" % len(df.difference))
        # Scoring: JsonDiff sorted data results, to non-sorted results (json responses)
        # df.difference: [
        # "diff: response_info.time - 28 | 11",
        # "diff: python_call_timer - 0.526123046875 | 0.498980998993"
        # ]

if __name__ == '__main__':
    h2o.unit_main()
<|endoftext|>


<commit_before>import logging

from aiohttp import web

from batteries.conf import settings

logger = logging.getLogger('batteries.plataforms')

class BasePlataform:

    def __init__(self, **kw):
        for item, value in kw.items():
            setattr(self, item, value)

    @property
    def webhook_endpoint(self):
        return '/hook/{}'.format(self.plataform)

    @property
    def webhook_url(self):
        return 'https://{}{}'.format(settings.HOSTNAME, self.webhook_endpoint)

    def build_message(self):
        raise NotImplementedError('create_message not implemented')

    @property
    def handler(self):
        async def handler(request):
            logger.debug('[%s] New message', self.plataform)
            data = await request.json()
            logger.debug('[%s] Building message', self.plataform)
            message = self.build_message(data)
            logger.info('[%s] Message from %s', self.plataform, message.sender)
            return web.Response(text='batteries')

        return handler
<commit_msg>Change message.sender to message.user<commit_after>import logging

from aiohttp import web

from batteries.conf import settings

logger = logging.getLogger('batteries.plataforms')

class BasePlataform:

    def __init__(self, **kw):
        for item, value in kw.items():
            setattr(self, item, value)

    @property
    def webhook_endpoint(self):
        return '/hook/{}'.format(self.plataform)

    @property
    def webhook_url(self):
        return 'https://{}{}'.format(settings.HOSTNAME, self.webhook_endpoint)

    def build_message(self):
        raise NotImplementedError('create_message not implemented')

    @property
    def handler(self):
        async def handler(request):
            logger.debug('[%s] New message', self.plataform)
            data = await request.json()
            logger.debug('[%s] Building message', self.plataform)
            message = self.build_message(data)
            logger.info('[%s] Message from %s', self.plataform, message.user)
            return web.Response(text='batteries')

        return handler
<|endoftext|>


<commit_before>#!/usr/bin/env python
import rospy

from flexbe_core.core.preemptable_state import PreemptableState
from flexbe_core.core.operatable_state_machine import OperatableStateMachine

from flexbe_core.proxy import ProxyPublisher, ProxySubscriberCached
from flexbe_msgs.msg import Container, OutcomeRequest
from std_msgs.msg import UInt8, String

from flexbe_core.state_logger import StateLogger

class OperatableState(PreemptableState):
    """
    A state that supports autonomy levels and silent mode.
    Also, it allows being tracked by a GUI or anything else
    as it reports each transition and its initial structure.
    An additional method is provided to report custom status messages to the widget.
    """

    def __init__(self, *args, **kwargs):
        super(OperatableState, self).__init__(*args, **kwargs)
        self.transitions = None
        self.autonomy = None

        self._mute = False  # is set to true when used in silent state machine (don't report transitions)
        self._last_requested_outcome = None

        self._outcome_topic = 'flexbe/mirror/outcome'
        self._request_topic = 'flexbe/outcome_request'
        self._debug_topic = 'flexbe/debug/current_state'
        self._pub = ProxyPublisher()

        self.__execute = self.execute
        self.execute = self._operatable_execute

    def _build_msg(self, prefix, msg):
        """
        Adds this state to the initial structure message.

        @type prefix: string
        @param prefix: A path consisting of the container hierarchy containing this state.

        @type msg: ContainerStructure
        @param msg: The message that will finally contain the structure message.
        """

        # set path
        name = prefix + self.name

        # no children
        children = None

        # set outcomes
        outcomes = self._outcome_list

        # set transitions and autonomy levels
        transitions = []
        autonomy = []
        for i in range(len(outcomes)):
            outcome = outcomes[i]
            if outcome == 'preempted':          # set preempt transition
                transitions.append('preempted')
                autonomy.append(-1)
            else:
                transitions.append(str(self.transitions[outcome]))
                autonomy.append(self.autonomy[outcome])

        # add to message
        msg.containers.append(Container(name, children, outcomes, transitions, autonomy))

    def _operatable_execute(self, *args, **kwargs):
        outcome = self.__execute(*args, **kwargs)

        if self._is_controlled:

            # reset previously requested outcome if applicable
            if self._last_requested_outcome is not None and outcome == OperatableState._loopback_name:
                self._pub.publish(self._request_topic, OutcomeRequest(outcome=255, target=self._parent._get_path() + "/" + self.name))
                self._last_requested_outcome = None

            # request outcome because autonomy level is too low
            if not self._force_transition and (outcome in self.autonomy and self.autonomy[outcome] >= OperatableStateMachine.autonomy_level or
                                               outcome != OperatableState._loopback_name and
                                               self._get_path() in rospy.get_param('/flexbe/breakpoints', [])):
                if outcome != self._last_requested_outcome:
                    self._pub.publish(self._request_topic, OutcomeRequest(outcome=self._outcome_list.index(outcome), target=self._parent._get_path() + "/" + self.name))
                    rospy.loginfo("<-- Want result: %s > %s", self.name, outcome)
                    StateLogger.log_state_execution(self._get_path(), self.__class__.__name__, outcome, not self._force_transition, False)
                    self._last_requested_outcome = outcome
                outcome = OperatableState._loopback_name

            # autonomy level is high enough, report the executed transition
            elif outcome != OperatableState._loopback_name:
                self._sent_outcome_requests = []
                rospy.loginfo("State result: %s > %s", self.name, outcome)
                self._pub.publish(self._outcome_topic, UInt8(self._outcome_list.index(outcome)))
                self._pub.publish(self._debug_topic, String("%s > %s" % (self._get_path(), outcome)))
                StateLogger.log_state_execution(self._get_path(), self.__class__.__name__, outcome, not self._force_transition, True)

        self._force_transition = False

        return outcome

    def _notify_skipped(self):
        super(OperatableState, self)._notify_skipped()

    def _enable_ros_control(self):
        super(OperatableState, self)._enable_ros_control()
        self._pub.createPublisher(self._outcome_topic, UInt8)
        self._pub.createPublisher(self._debug_topic, String)
        self._pub.createPublisher(self._request_topic, OutcomeRequest)

<commit_msg>[flexbe_core] Fix reset of requested outcome when re-visiting the same state and immediately requesting an outcome<commit_after>#!/usr/bin/env python
import rospy

from flexbe_core.core.preemptable_state import PreemptableState
from flexbe_core.core.operatable_state_machine import OperatableStateMachine

from flexbe_core.proxy import ProxyPublisher, ProxySubscriberCached
from flexbe_msgs.msg import Container, OutcomeRequest
from std_msgs.msg import UInt8, String

from flexbe_core.state_logger import StateLogger

class OperatableState(PreemptableState):
    """
    A state that supports autonomy levels and silent mode.
    Also, it allows being tracked by a GUI or anything else
    as it reports each transition and its initial structure.
    An additional method is provided to report custom status messages to the widget.
    """

    def __init__(self, *args, **kwargs):
        super(OperatableState, self).__init__(*args, **kwargs)
        self.transitions = None
        self.autonomy = None

        self._mute = False  # is set to true when used in silent state machine (don't report transitions)
        self._last_requested_outcome = None

        self._outcome_topic = 'flexbe/mirror/outcome'
        self._request_topic = 'flexbe/outcome_request'
        self._debug_topic = 'flexbe/debug/current_state'
        self._pub = ProxyPublisher()

        self.__execute = self.execute
        self.execute = self._operatable_execute

    def _build_msg(self, prefix, msg):
        """
        Adds this state to the initial structure message.

        @type prefix: string
        @param prefix: A path consisting of the container hierarchy containing this state.

        @type msg: ContainerStructure
        @param msg: The message that will finally contain the structure message.
        """

        # set path
        name = prefix + self.name

        # no children
        children = None

        # set outcomes
        outcomes = self._outcome_list

        # set transitions and autonomy levels
        transitions = []
        autonomy = []
        for i in range(len(outcomes)):
            outcome = outcomes[i]
            if outcome == 'preempted':          # set preempt transition
                transitions.append('preempted')
                autonomy.append(-1)
            else:
                transitions.append(str(self.transitions[outcome]))
                autonomy.append(self.autonomy[outcome])

        # add to message
        msg.containers.append(Container(name, children, outcomes, transitions, autonomy))

    def _operatable_execute(self, *args, **kwargs):
        outcome = self.__execute(*args, **kwargs)

        if self._is_controlled:

            # reset previously requested outcome if applicable
            if self._last_requested_outcome is not None and outcome == OperatableState._loopback_name:
                self._pub.publish(self._request_topic, OutcomeRequest(outcome=255, target=self._parent._get_path() + "/" + self.name))
                self._last_requested_outcome = None

            # request outcome because autonomy level is too low
            if not self._force_transition and (outcome in self.autonomy and self.autonomy[outcome] >= OperatableStateMachine.autonomy_level or
                                               outcome != OperatableState._loopback_name and
                                               self._get_path() in rospy.get_param('/flexbe/breakpoints', [])):
                if outcome != self._last_requested_outcome:
                    self._pub.publish(self._request_topic, OutcomeRequest(outcome=self._outcome_list.index(outcome), target=self._parent._get_path() + "/" + self.name))
                    rospy.loginfo("<-- Want result: %s > %s", self.name, outcome)
                    StateLogger.log_state_execution(self._get_path(), self.__class__.__name__, outcome, not self._force_transition, False)
                    self._last_requested_outcome = outcome
                outcome = OperatableState._loopback_name

            # autonomy level is high enough, report the executed transition
            elif outcome != OperatableState._loopback_name:
                self._last_requested_outcome = None
                rospy.loginfo("State result: %s > %s", self.name, outcome)
                self._pub.publish(self._outcome_topic, UInt8(self._outcome_list.index(outcome)))
                self._pub.publish(self._debug_topic, String("%s > %s" % (self._get_path(), outcome)))
                StateLogger.log_state_execution(self._get_path(), self.__class__.__name__, outcome, not self._force_transition, True)

        self._force_transition = False

        return outcome

    def _notify_skipped(self):
        super(OperatableState, self)._notify_skipped()

    def _enable_ros_control(self):
        super(OperatableState, self)._enable_ros_control()
        self._pub.createPublisher(self._outcome_topic, UInt8)
        self._pub.createPublisher(self._debug_topic, String)
        self._pub.createPublisher(self._request_topic, OutcomeRequest)

<|endoftext|>


<commit_before>import unittest, random, sys, time
sys.path.extend(['.','..','py'])
import h2o, h2o_browse as h2b, h2o_exec as h2e, h2o_hosts, h2o_import as h2i

initList = [
        ('r0.hex', 'r0.hex=c(1.3,0,1,2,3,4,5)'),
        ('r1.hex', 'r1.hex=c(2.3,0,1,2,3,4,5)'),
        ('r2.hex', 'r2.hex=c(3.3,0,1,2,3,4,5)'),
        ('r3.hex', 'r3.hex=c(4.3,0,1,2,3,4,5)'),
        ('r4.hex', 'r4.hex=c(5.3,0,1,2,3,4,5)'),
        ('r.hex', 'r.hex=i.hex'),
        ('z.hex', 'z.hex=c(0)'),
        ]

exprListSmall = [
        'x= 3; r.hex[(x > 0) & (x < 4)]',    # all x values between 0 and 1
        'if (any(r3.hex == 0) || any(r4.hex == 0))', "zero encountered"

        'x <- c(NA, FALSE, TRUE)'
        # 'names(x) <- as.character(x)'
        # outer(x, x, "&")## AND table
        # outer(x, x, "|")## OR  table
        "1.23",
        "!1.23",

        "1.23<2.34",
        "!1.23<2.34",
        "!1.23<!2.34",
        "1.23<!2.34",

        "1.23<=2.34",
        "1.23>2.34",
        "1.23>=2.34",
        "1.23==2.34",
        "1.23!=2.34",

        "r.hex",
        "!r.hex",

        # Not supported
        # "+(1.23,2.34)",
        "x=0; x+2",
        "x=!0; !x+2",
        "x=!0; x+!2",

        "x=1",
        "x=!1",

        "x<-1",
        "x<-!1",
        "c(1,3,5)",
        "!c(1,3,5)",
        "!c(!1,3,5)",
        "!c(1,!3,5)",
        "!c(1,3,!5)",

        "a=0; x=0",
        "a=!0; x=!0",

        "r.hex[2,3]",
        "r.hex[!2,3]",
        "r.hex[2,!3]",

        "r.hex[2+4,-4]",
        "r.hex[1,-1]; r.hex[1,-1]; r.hex[1,-1]",
        "r.hex[1,]",
        "r.hex+1",
        "r.hex[,1]",
        "r.hex[,1]+1",

        "r.hex-r.hex",
        "1.23+(r.hex-r.hex)",
        "(1.23+r.hex)-r.hex",

        "is.na(r.hex)",

        "nrow(r.hex)*3",
        "r.hex[nrow(r.hex)-1,ncol(r.hex)-1]",
        "r.hex[nrow(r.hex),]",
        "a=ncol(r.hex); r.hex[,c(a+1,a+2)]=5",
        "r.hex[,ncol(r.hex)+1]=4",
        "r.hex[,1]=3.3; r.hex",
        "r.hex[,1]=r.hex[,1]+1",

        "function(x,y,z){x[]}(r.hex,1,2)",
        "function(x){x+1}(2)",
        "function(x){y=x*2; y+1}(2)",
        "function(x){y=1+2}(2)",
        "function(funy){function(x){funy(x)*funy(x)}}(sgn)(-2)",
        "a=1; a=2; function(x){x=a;a=3}",
        "a=r.hex; function(x){x=a;a=3;nrow(x)*a}(a)",

        "apply(r.hex,2,sum)",
        "apply(r.hex,2,function(x){ifelse(x==-1,1,x)})",

        # doesn't work
        # "cbind(c(1), c(2), c(3))",
        # "cbind(c(1,2,3), c(4,5,6))",
        # "cbind(c(1,2,3), c(4,5,6), c(7,8,9))",
        # "cbind(c(1,2,3,4), c(5,6,7))",
        # "cbind(c(1,2,3), c(4,5,6,7))",
        "cbind(c(1,2,3,4), c(5,6,7,8))",

        "r.hex[c(1,3,5),]",
        "a=c(11,22,33,44,55,66); a[c(2,6,1),]",
        "r.hex[r.hex[,1]>4,]",
        # fails?
        # "a=c(1,2,3); a[a[,1]>10,1]",

        "ifelse(0,1,2)",
        "ifelse(0,r.hex+1,r.hex+2)",
        "ifelse(r.hex>3,99,r.hex)",
        "ifelse(0,+,*)(1,2)",

        "(0 ? + : *)(1,2)",
        "(1 ? r.hex : (r.hex+1))[1,2]",

        "sum(1,2)",
        "sum(1,2,3)",
        "sum(c(1,3,5))",
        "sum(4,c(1,3,5),2,6)",
        "sum(1,r.hex,3)",

        "min(1,2)",
        # doesn't work
        # "min(1,2,3)",
        # doesn't work. only 2 params?
        # "min(c(1,3,5))",
        # doesn't work. only 2 params?
        # "min(4,c(1,3,5),2,6)",
        # doesn't work
        # "min(1,r.hex,3)",

        "max(1,23)",
        # doesn't work
        # Passed 3 args but expected 2
        # "max(1,2,3)",
        # doesn't work
        # "max(c(1,3,5))",

        # doesn't work
        # Passed 4 args but expected 2
        # "max(4,c(1,3,5),2,6)",
        # doesn't work
        # "max(1,r.hex,3)",

        "factor(r.hex[,5])",
        "r0.hex[,1]==1.0",
        "runif(r4.hex[,1])",

        # doesn't work
        "mean=function(x){apply(x,1,sum)/nrow(x)};mean(r.hex)",

        ]

# concatenate a lot of random choices to make life harder
exprList = []
for i in range(10):
    expr = ""
    for j in range(1):
        expr += random.choice(exprListSmall)
    exprList.append(expr)

class Basic(unittest.TestCase):
    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        global SEED, localhost
        SEED = h2o.setup_random_seed()
        localhost = h2o.decide_if_localhost()
        if (localhost):
            h2o.build_cloud(1)
        else:
            h2o_hosts.build_cloud_with_hosts(1)

    @classmethod
    def tearDownClass(cls):
        h2o.tear_down_cloud()

    def test_exec2_operators2(self):
        bucket = 'smalldata'
        csvPathname = 'iris/iris2.csv'
        hexKey = 'i.hex'
        parseResult = h2i.import_parse(bucket=bucket, path=csvPathname, schema='put', hex_key=hexKey)

        for resultKey, execExpr in initList:
            h2e.exec_expr(h2o.nodes[0], execExpr, resultKey=resultKey, timeoutSecs=4)
        start = time.time()
        h2e.exec_expr_list_rand(len(h2o.nodes), exprList, None, maxTrials=200, timeoutSecs=10)

        h2o.check_sandbox_for_errors()
        print "exec end on ", "operators" , 'took', time.time() - start, 'seconds'

if __name__ == '__main__':
    h2o.unit_main()
<commit_msg>a few tweaks<commit_after>import unittest, random, sys, time
sys.path.extend(['.','..','py'])
import h2o, h2o_browse as h2b, h2o_exec as h2e, h2o_hosts, h2o_import as h2i

initList = [
        ('r0.hex', 'r0.hex=c(1.3,0,1,2,3,4,5)'),
        ('r1.hex', 'r1.hex=c(2.3,0,1,2,3,4,5)'),
        ('r2.hex', 'r2.hex=c(3.3,0,1,2,3,4,5)'),
        ('r3.hex', 'r3.hex=c(4.3,0,1,2,3,4,5)'),
        ('r4.hex', 'r4.hex=c(5.3,0,1,2,3,4,5)'),
        ('r.hex', 'r.hex=i.hex'),
        ('z.hex', 'z.hex=c(0)'),
        ]

exprListSmall = [
        'x= 3; r.hex[(x > 0) & (x < 4),]',    # all x values between 0 and 1
        'x= 3; r.hex[,(x > 0) & (x < 4)]',    # all x values between 0 and 1
        'if (any(r3.hex == 0) || any(r4.hex == 0))', "zero encountered"

        'x <- c(NA, FALSE, TRUE)'
        # 'names(x) <- as.character(x)'
        # outer(x, x, "&")## AND table
        # outer(x, x, "|")## OR  table
        "1.23",
        "!1.23",

        "1.23<2.34",
        "!1.23<2.34",
        "!1.23<!2.34",
        "1.23<!2.34",

        "1.23<=2.34",
        "1.23>2.34",
        "1.23>=2.34",
        "1.23==2.34",
        "1.23!=2.34",

        "r.hex",
        "!r.hex",

        # Not supported
        # "+(1.23,2.34)",
        "x=0; x+2",
        "x=!0; !x+2",
        "x=!0; x+!2",

        "x=1",
        "x=!1",

        "x<-1",
        "x<-!1",
        "c(1,3,5)",
        "!c(1,3,5)",
        "!c(!1,3,5)",
        "!c(1,!3,5)",
        "!c(1,3,!5)",

        "a=0; x=0",
        "a=!0; x=!0",

        "r.hex[2,3]",
        "r.hex[!2,3]",
        # no cols selectd
        # "r.hex[2,!3]",

        "r.hex[2+4,-4]",
        "r.hex[1,-1]; r.hex[1,-1]; r.hex[1,-1]",
        "r.hex[1,]",
        "r.hex+1",
        "r.hex[,1]",
        "r.hex[,1]+1",

        "r.hex-r.hex",
        "1.23+(r.hex-r.hex)",
        "(1.23+r.hex)-r.hex",

        "is.na(r.hex)",

        "nrow(r.hex)*3",
        "r.hex[nrow(r.hex)-1,ncol(r.hex)-1]",
        "r.hex[nrow(r.hex),]",
        "a=ncol(r.hex); r.hex[,c(a+1,a+2)]=5",
        "r.hex[,ncol(r.hex)+1]=4",
        "r.hex[,1]=3.3; r.hex",
        "r.hex[,1]=r.hex[,1]+1",

        "function(x,y,z){x[]}(r.hex,1,2)",
        "function(x){x+1}(2)",
        "function(x){y=x*2; y+1}(2)",
        "function(x){y=1+2}(2)",
        "function(funy){function(x){funy(x)*funy(x)}}(sgn)(-2)",
        "a=1; a=2; function(x){x=a;a=3}",
        "a=r.hex; function(x){x=a;a=3;nrow(x)*a}(a)",

        "apply(r.hex,2,sum)",
        "apply(r.hex,2,function(x){ifelse(x==-1,1,x)})",

        # doesn't work
        # "cbind(c(1), c(2), c(3))",
        # "cbind(c(1,2,3), c(4,5,6))",
        # "cbind(c(1,2,3), c(4,5,6), c(7,8,9))",
        # "cbind(c(1,2,3,4), c(5,6,7))",
        # "cbind(c(1,2,3), c(4,5,6,7))",
        "cbind(c(1,2,3,4), c(5,6,7,8))",

        "r.hex[c(1,3,5),]",
        "a=c(11,22,33,44,55,66); a[c(2,6,1),]",
        "r.hex[r.hex[,1]>4,]",
        # fails?
        # "a=c(1,2,3); a[a[,1]>10,1]",

        "ifelse(0,1,2)",
        "ifelse(0,r.hex+1,r.hex+2)",
        "ifelse(r.hex>3,99,r.hex)",
        "ifelse(0,+,*)(1,2)",

        "(0 ? + : *)(1,2)",
        "(1 ? r.hex : (r.hex+1))[1,2]",

        "sum(1,2)",
        "sum(1,2,3)",
        "sum(c(1,3,5))",
        "sum(4,c(1,3,5),2,6)",
        "sum(1,r.hex,3)",

        "min(1,2)",
        # doesn't work
        # "min(1,2,3)",
        # doesn't work. only 2 params?
        # "min(c(1,3,5))",
        # doesn't work. only 2 params?
        # "min(4,c(1,3,5),2,6)",
        # doesn't work
        # "min(1,r.hex,3)",

        "max(1,23)",
        # doesn't work
        # Passed 3 args but expected 2
        # "max(1,2,3)",
        # doesn't work
        # "max(c(1,3,5))",

        # doesn't work
        # Passed 4 args but expected 2
        # "max(4,c(1,3,5),2,6)",
        # doesn't work
        # "max(1,r.hex,3)",

        "factor(r.hex[,5])",
        "r0.hex[,1]==1.0",
        "runif(r4.hex[,1])",

        # doesn't work
        "mean=function(x){apply(x,1,sum)/nrow(x)};mean(r.hex)",

        ]

# concatenate a lot of random choices to make life harder
exprList = []
for i in range(10):
    expr = ""
    for j in range(1):
        expr += random.choice(exprListSmall)
    exprList.append(expr)

class Basic(unittest.TestCase):
    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        global SEED, localhost
        SEED = h2o.setup_random_seed()
        localhost = h2o.decide_if_localhost()
        if (localhost):
            h2o.build_cloud(1)
        else:
            h2o_hosts.build_cloud_with_hosts(1)

    @classmethod
    def tearDownClass(cls):
        h2o.tear_down_cloud()

    def test_exec2_operators2(self):
        bucket = 'smalldata'
        csvPathname = 'iris/iris2.csv'
        hexKey = 'i.hex'
        parseResult = h2i.import_parse(bucket=bucket, path=csvPathname, schema='put', hex_key=hexKey)

        for resultKey, execExpr in initList:
            h2e.exec_expr(h2o.nodes[0], execExpr, resultKey=resultKey, timeoutSecs=4)
        start = time.time()
        h2e.exec_expr_list_rand(len(h2o.nodes), exprList, None, maxTrials=500, timeoutSecs=10)

        h2o.check_sandbox_for_errors()
        print "exec end on ", "operators" , 'took', time.time() - start, 'seconds'

if __name__ == '__main__':
    h2o.unit_main()
<|endoftext|>


<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Load dependencies needed to compile and test the google-cloud-cpp library."""

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")

def google_cloud_cpp_deps():
    """Loads dependencies need to compile the google-cloud-cpp library.

    Application developers can call this function from their WORKSPACE file
    to obtain all the necessary dependencies for google-cloud-cpp, including
    gRPC and its dependencies. This function only loads dependencies that
    have not been previously loaded, allowing application developers to
    override the version of the dependencies they want to use.
    """

    # Load platforms, we use it directly
    if "platforms" not in native.existing_rules():
        http_archive(
            name = "platforms",
            urls = [
                "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.5/platforms-0.0.5.tar.gz",
                "https://github.com/bazelbuild/platforms/releases/download/0.0.5/platforms-0.0.5.tar.gz",
            ],
            sha256 = "379113459b0feaf6bfbb584a91874c065078aa673222846ac765f86661c27407",
        )

    # Load rules_cc, used by googletest
    if "rules_cc" not in native.existing_rules():
        http_archive(
            name = "rules_cc",
            urls = [
                "https://github.com/bazelbuild/rules_cc/releases/download/0.0.1/rules_cc-0.0.1.tar.gz",
            ],
            sha256 = "4dccbfd22c0def164c8f47458bd50e0c7148f3d92002cdb459c2a96a68498241",
        )

    # Load Abseil
    if "com_google_absl" not in native.existing_rules():
        http_archive(
            name = "com_google_absl",
            strip_prefix = "abseil-cpp-20220623.0",
            urls = [
                "https://github.com/abseil/abseil-cpp/archive/20220623.0.tar.gz",
            ],
            sha256 = "4208129b49006089ba1d6710845a45e31c59b0ab6bff9e5788a87f55c5abd602",
        )

    # Load a version of googletest that we know works.
    if "com_google_googletest" not in native.existing_rules():
        http_archive(
            name = "com_google_googletest",
            strip_prefix = "googletest-release-1.11.0",
            urls = [
                "https://github.com/google/googletest/archive/release-1.11.0.tar.gz",
            ],
            sha256 = "b4870bf121ff7795ba20d20bcdd8627b8e088f2d1dab299a031c1034eddc93d5",
        )

    # Load a version of benchmark that we know works.
    if "com_google_benchmark" not in native.existing_rules():
        http_archive(
            name = "com_google_benchmark",
            strip_prefix = "benchmark-1.6.2",
            urls = [
                "https://github.com/google/benchmark/archive/v1.6.2.tar.gz",
            ],
            sha256 = "a9f77e6188c1cd4ebedfa7538bf5176d6acc72ead6f456919e5f464ef2f06158",
        )

    # Load the googleapis dependency.
    if "com_google_googleapis" not in native.existing_rules():
        http_archive(
            name = "com_google_googleapis",
            urls = [
                "https://github.com/googleapis/googleapis/archive/ff8159ec8694fe07a10142c8c8bf9f740bf0a93e.tar.gz",
            ],
            strip_prefix = "googleapis-ff8159ec8694fe07a10142c8c8bf9f740bf0a93e",
            sha256 = "b0989259d95eddabf9684ecb43d3e558e5a18acd3d4a78a372f8152af95bbe71",
            build_file = "@com_github_googleapis_google_cloud_cpp//bazel:googleapis.BUILD",
            # Scaffolding for patching googleapis after download. For example:
            #   patches = ["googleapis.patch"]
            # NOTE: This should only be used while developing with a new
            # protobuf message. No changes to `patches` should ever be
            # committed to the main branch.
            patch_tool = "patch",
            patch_args = ["-p1"],
            patches = [],
        )

    # Load protobuf.
    if "com_google_protobuf" not in native.existing_rules():
        http_archive(
            name = "com_google_protobuf",
            strip_prefix = "protobuf-21.3",
            urls = [
                "https://github.com/protocolbuffers/protobuf/archive/v21.3.tar.gz",
            ],
            sha256 = "1e958b62debbb46ebefa16d848741d17c88dc018dd57b729c0cd58965380f3f8",
        )

    # Load BoringSSL, this is used by gRPC, but as I write this (2021-06-03, circa gRPC-1.37.1), the version used by
    # gRPC does not compile with GCC-11
    if "boringssl" not in native.existing_rules():
        http_archive(
            name = "boringssl",
            # Use github mirror instead of https://boringssl.googlesource.com/boringssl
            # to obtain a boringssl archive with consistent sha256
            sha256 = "f0f433106f98f6f50ed6bbc2169f7c42dd73d13d0f09d431082519b5649903a6",
            strip_prefix = "boringssl-56eb68f64215738552be452e311da12047934ab4",
            urls = [
                "https://github.com/google/boringssl/archive/56eb68f64215738552be452e311da12047934ab4.tar.gz",
            ],
        )

    # Load gRPC and its dependencies, using a similar pattern to this function.
    if "com_github_grpc_grpc" not in native.existing_rules():
        http_archive(
            name = "com_github_grpc_grpc",
            strip_prefix = "grpc-1.47.1",
            urls = [
                "https://github.com/grpc/grpc/archive/v1.47.1.tar.gz",
            ],
            sha256 = "ba7c36989ad8a52c813adeebb0c65869283448f496dbb2868c75f69941dec6a9",
        )

    # We use the cc_proto_library() rule from @com_google_protobuf, which
    # assumes that grpc_cpp_plugin and grpc_lib are in the //external: module
    native.bind(
        name = "grpc_cpp_plugin",
        actual = "@com_github_grpc_grpc//src/compiler:grpc_cpp_plugin",
    )

    native.bind(
        name = "grpc_lib",
        actual = "@com_github_grpc_grpc//:grpc++",
    )

    # We need libcurl for the Google Cloud Storage client.
    if "com_github_curl_curl" not in native.existing_rules():
        http_archive(
            name = "com_github_curl_curl",
            urls = [
                "https://curl.haxx.se/download/curl-7.69.1.tar.gz",
            ],
            strip_prefix = "curl-7.69.1",
            sha256 = "01ae0c123dee45b01bbaef94c0bc00ed2aec89cb2ee0fd598e0d302a6b5e0a98",
            build_file = "@com_github_googleapis_google_cloud_cpp//bazel:curl.BUILD",
        )

    # We need the nlohmann_json library
    if "com_github_nlohmann_json" not in native.existing_rules():
        http_archive(
            name = "com_github_nlohmann_json",
            urls = [
                "https://github.com/nlohmann/json/releases/download/v3.10.5/include.zip",
            ],
            sha256 = "b94997df68856753b72f0d7a3703b7d484d4745c567f3584ef97c96c25a5798e",
            build_file = "@com_github_googleapis_google_cloud_cpp//bazel:nlohmann_json.BUILD",
        )

    # Load google/crc32c, a library to efficiently compute CRC32C checksums.
    if "com_github_google_crc32c" not in native.existing_rules():
        http_archive(
            name = "com_github_google_crc32c",
            strip_prefix = "crc32c-1.1.2",
            urls = [
                "https://github.com/google/crc32c/archive/1.1.2.tar.gz",
            ],
            sha256 = "ac07840513072b7fcebda6e821068aa04889018f24e10e46181068fb214d7e56",
            build_file = "@com_github_googleapis_google_cloud_cpp//bazel:crc32c.BUILD",
        )
<commit_msg>chore(deps): update dependency com_google_benchmark to v1.7.0 (#9559)<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Load dependencies needed to compile and test the google-cloud-cpp library."""

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")

def google_cloud_cpp_deps():
    """Loads dependencies need to compile the google-cloud-cpp library.

    Application developers can call this function from their WORKSPACE file
    to obtain all the necessary dependencies for google-cloud-cpp, including
    gRPC and its dependencies. This function only loads dependencies that
    have not been previously loaded, allowing application developers to
    override the version of the dependencies they want to use.
    """

    # Load platforms, we use it directly
    if "platforms" not in native.existing_rules():
        http_archive(
            name = "platforms",
            urls = [
                "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.5/platforms-0.0.5.tar.gz",
                "https://github.com/bazelbuild/platforms/releases/download/0.0.5/platforms-0.0.5.tar.gz",
            ],
            sha256 = "379113459b0feaf6bfbb584a91874c065078aa673222846ac765f86661c27407",
        )

    # Load rules_cc, used by googletest
    if "rules_cc" not in native.existing_rules():
        http_archive(
            name = "rules_cc",
            urls = [
                "https://github.com/bazelbuild/rules_cc/releases/download/0.0.1/rules_cc-0.0.1.tar.gz",
            ],
            sha256 = "4dccbfd22c0def164c8f47458bd50e0c7148f3d92002cdb459c2a96a68498241",
        )

    # Load Abseil
    if "com_google_absl" not in native.existing_rules():
        http_archive(
            name = "com_google_absl",
            strip_prefix = "abseil-cpp-20220623.0",
            urls = [
                "https://github.com/abseil/abseil-cpp/archive/20220623.0.tar.gz",
            ],
            sha256 = "4208129b49006089ba1d6710845a45e31c59b0ab6bff9e5788a87f55c5abd602",
        )

    # Load a version of googletest that we know works.
    if "com_google_googletest" not in native.existing_rules():
        http_archive(
            name = "com_google_googletest",
            strip_prefix = "googletest-release-1.11.0",
            urls = [
                "https://github.com/google/googletest/archive/release-1.11.0.tar.gz",
            ],
            sha256 = "b4870bf121ff7795ba20d20bcdd8627b8e088f2d1dab299a031c1034eddc93d5",
        )

    # Load a version of benchmark that we know works.
    if "com_google_benchmark" not in native.existing_rules():
        http_archive(
            name = "com_google_benchmark",
            strip_prefix = "benchmark-1.7.0",
            urls = [
                "https://github.com/google/benchmark/archive/v1.7.0.tar.gz",
            ],
            sha256 = "3aff99169fa8bdee356eaa1f691e835a6e57b1efeadb8a0f9f228531158246ac",
        )

    # Load the googleapis dependency.
    if "com_google_googleapis" not in native.existing_rules():
        http_archive(
            name = "com_google_googleapis",
            urls = [
                "https://github.com/googleapis/googleapis/archive/ff8159ec8694fe07a10142c8c8bf9f740bf0a93e.tar.gz",
            ],
            strip_prefix = "googleapis-ff8159ec8694fe07a10142c8c8bf9f740bf0a93e",
            sha256 = "b0989259d95eddabf9684ecb43d3e558e5a18acd3d4a78a372f8152af95bbe71",
            build_file = "@com_github_googleapis_google_cloud_cpp//bazel:googleapis.BUILD",
            # Scaffolding for patching googleapis after download. For example:
            #   patches = ["googleapis.patch"]
            # NOTE: This should only be used while developing with a new
            # protobuf message. No changes to `patches` should ever be
            # committed to the main branch.
            patch_tool = "patch",
            patch_args = ["-p1"],
            patches = [],
        )

    # Load protobuf.
    if "com_google_protobuf" not in native.existing_rules():
        http_archive(
            name = "com_google_protobuf",
            strip_prefix = "protobuf-21.3",
            urls = [
                "https://github.com/protocolbuffers/protobuf/archive/v21.3.tar.gz",
            ],
            sha256 = "1e958b62debbb46ebefa16d848741d17c88dc018dd57b729c0cd58965380f3f8",
        )

    # Load BoringSSL, this is used by gRPC, but as I write this (2021-06-03, circa gRPC-1.37.1), the version used by
    # gRPC does not compile with GCC-11
    if "boringssl" not in native.existing_rules():
        http_archive(
            name = "boringssl",
            # Use github mirror instead of https://boringssl.googlesource.com/boringssl
            # to obtain a boringssl archive with consistent sha256
            sha256 = "f0f433106f98f6f50ed6bbc2169f7c42dd73d13d0f09d431082519b5649903a6",
            strip_prefix = "boringssl-56eb68f64215738552be452e311da12047934ab4",
            urls = [
                "https://github.com/google/boringssl/archive/56eb68f64215738552be452e311da12047934ab4.tar.gz",
            ],
        )

    # Load gRPC and its dependencies, using a similar pattern to this function.
    if "com_github_grpc_grpc" not in native.existing_rules():
        http_archive(
            name = "com_github_grpc_grpc",
            strip_prefix = "grpc-1.47.1",
            urls = [
                "https://github.com/grpc/grpc/archive/v1.47.1.tar.gz",
            ],
            sha256 = "ba7c36989ad8a52c813adeebb0c65869283448f496dbb2868c75f69941dec6a9",
        )

    # We use the cc_proto_library() rule from @com_google_protobuf, which
    # assumes that grpc_cpp_plugin and grpc_lib are in the //external: module
    native.bind(
        name = "grpc_cpp_plugin",
        actual = "@com_github_grpc_grpc//src/compiler:grpc_cpp_plugin",
    )

    native.bind(
        name = "grpc_lib",
        actual = "@com_github_grpc_grpc//:grpc++",
    )

    # We need libcurl for the Google Cloud Storage client.
    if "com_github_curl_curl" not in native.existing_rules():
        http_archive(
            name = "com_github_curl_curl",
            urls = [
                "https://curl.haxx.se/download/curl-7.69.1.tar.gz",
            ],
            strip_prefix = "curl-7.69.1",
            sha256 = "01ae0c123dee45b01bbaef94c0bc00ed2aec89cb2ee0fd598e0d302a6b5e0a98",
            build_file = "@com_github_googleapis_google_cloud_cpp//bazel:curl.BUILD",
        )

    # We need the nlohmann_json library
    if "com_github_nlohmann_json" not in native.existing_rules():
        http_archive(
            name = "com_github_nlohmann_json",
            urls = [
                "https://github.com/nlohmann/json/releases/download/v3.10.5/include.zip",
            ],
            sha256 = "b94997df68856753b72f0d7a3703b7d484d4745c567f3584ef97c96c25a5798e",
            build_file = "@com_github_googleapis_google_cloud_cpp//bazel:nlohmann_json.BUILD",
        )

    # Load google/crc32c, a library to efficiently compute CRC32C checksums.
    if "com_github_google_crc32c" not in native.existing_rules():
        http_archive(
            name = "com_github_google_crc32c",
            strip_prefix = "crc32c-1.1.2",
            urls = [
                "https://github.com/google/crc32c/archive/1.1.2.tar.gz",
            ],
            sha256 = "ac07840513072b7fcebda6e821068aa04889018f24e10e46181068fb214d7e56",
            build_file = "@com_github_googleapis_google_cloud_cpp//bazel:crc32c.BUILD",
        )
<|endoftext|>