<commit_before># -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""BibFormat element - bookmark toolbar using:

    <http://keith-wood.name/bookmark.html>

"""

import cgi

from invenio.config import CFG_SITE_URL, CFG_SITE_RECORD, CFG_CERN_SITE
from invenio.search_engine import record_public_p
from invenio.jsonutils import wash_for_js
from invenio.bibformat_elements.bfe_sciencewise import create_sciencewise_url, \
    get_arxiv_reportnumber
from invenio.webjournal_utils import \
     parse_url_string, \
     make_journal_url, \
     get_journals_ids_and_names

def format_element(bfo, only_public_records=1, sites="linkedin,twitter,facebook,google,delicious,sciencewise"):
    """
    Return a snippet of JavaScript needed for displaying a bookmark toolbar

    @param only_public_records: if set to 1 (the default), prints the box only
        if the record is public (i.e. if it belongs to the root colletion and is
        accessible to the world).

    @param sites: which sites to enable (default is 'linkedin,twitter,facebook,google,delicious,sciencewise'). This should be a
        comma separated list of strings.
        Valid values are available on:
            <http://keith-wood.name/bookmark.html#sites>
        Note that 'sciencewise' is an ad-hoc service that will be displayed
        only in case the record has an arXiv reportnumber and will always
        be displayed last.
    """
    if int(only_public_records) and not record_public_p(bfo.recID):
        return ""

    sitelist = sites.split(',')
    sitelist = [site.strip().lower() for site in sitelist]

    sciencewise = False
    if 'sciencewise' in sitelist:
        sciencewise = True
        sitelist.remove('sciencewise')

    sites_js = ", ".join("'%s'" % site for site in sitelist)

    title = bfo.field('245__a')
    description = bfo.field('520__a')

    sciencewise_script = ""
    if sciencewise:
        reportnumber = get_arxiv_reportnumber(bfo)
        sciencewise_url = ""
        if reportnumber:
            sciencewise_url = create_sciencewise_url(reportnumber)
        if not sciencewise_url and CFG_CERN_SITE:
            sciencewise_url = create_sciencewise_url(bfo.recID, cds=True)
        if sciencewise_url:
            sciencewise_script = """\
$.bookmark.addSite('sciencewise', 'ScienceWise.info', '%(siteurl)s/img/sciencewise.png', 'en', 'bookmark', '%(url)s');
$('#bookmark_sciencewise').bookmark({sites: ['sciencewise']});
""" % {
                'siteurl': CFG_SITE_URL,
                'url': sciencewise_url.replace("'", r"\'"),
            }

    url = '%(siteurl)s/%(record)s/%(recid)s' % \
          {'recid': bfo.recID,
           'record': CFG_SITE_RECORD,
           'siteurl': CFG_SITE_URL}

    args = parse_url_string(bfo.user_info['uri'])
    journal_name = args["journal_name"]
    if journal_name and \
       (journal_name in [info.get('journal_name', '') for info in get_journals_ids_and_names()]):
        # We are displaying a WebJournal article: URL is slightly different
        url = make_journal_url(bfo.user_info['uri'])

    return """\

""" % {
        'siteurl': CFG_SITE_URL,
        'sciencewise': sciencewise_script,
        'title': wash_for_js(cgi.escape(title)),
        'description': wash_for_js(cgi.escape(description)),
        'sites_js': sites_js,
        'url': url,
    }

def escape_values(bfo):
    """
    Called by BibFormat in order to check if output of this element
    should be escaped.
    """
    return 0
<commit_msg>htmlutils: improve js string escaping<commit_after># -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""BibFormat element - bookmark toolbar using:

    <http://keith-wood.name/bookmark.html>

"""

import cgi

from invenio.config import CFG_SITE_URL, CFG_SITE_RECORD, CFG_CERN_SITE
from invenio.search_engine import record_public_p
from invenio.htmlutils import escape_javascript_string
from invenio.bibformat_elements.bfe_sciencewise import create_sciencewise_url, \
    get_arxiv_reportnumber
from invenio.webjournal_utils import \
     parse_url_string, \
     make_journal_url, \
     get_journals_ids_and_names

def format_element(bfo, only_public_records=1, sites="linkedin,twitter,facebook,google,delicious,sciencewise"):
    """
    Return a snippet of JavaScript needed for displaying a bookmark toolbar

    @param only_public_records: if set to 1 (the default), prints the box only
        if the record is public (i.e. if it belongs to the root colletion and is
        accessible to the world).

    @param sites: which sites to enable (default is 'linkedin,twitter,facebook,google,delicious,sciencewise'). This should be a
        comma separated list of strings.
        Valid values are available on:
            <http://keith-wood.name/bookmark.html#sites>
        Note that 'sciencewise' is an ad-hoc service that will be displayed
        only in case the record has an arXiv reportnumber and will always
        be displayed last.
    """
    if int(only_public_records) and not record_public_p(bfo.recID):
        return ""

    sitelist = sites.split(',')
    sitelist = [site.strip().lower() for site in sitelist]

    sciencewise = False
    if 'sciencewise' in sitelist:
        sciencewise = True
        sitelist.remove('sciencewise')

    sites_js = ", ".join("'%s'" % site for site in sitelist)

    title = bfo.field('245__a')
    description = bfo.field('520__a')

    sciencewise_script = ""
    if sciencewise:
        reportnumber = get_arxiv_reportnumber(bfo)
        sciencewise_url = ""
        if reportnumber:
            sciencewise_url = create_sciencewise_url(reportnumber)
        if not sciencewise_url and CFG_CERN_SITE:
            sciencewise_url = create_sciencewise_url(bfo.recID, cds=True)
        if sciencewise_url:
            sciencewise_script = """\
$.bookmark.addSite('sciencewise', 'ScienceWise.info', '%(siteurl)s/img/sciencewise.png', 'en', 'bookmark', '%(url)s');
$('#bookmark_sciencewise').bookmark({sites: ['sciencewise']});
""" % {
                'siteurl': CFG_SITE_URL,
                'url': sciencewise_url.replace("'", r"\'"),
            }

    url = '%(siteurl)s/%(record)s/%(recid)s' % \
          {'recid': bfo.recID,
           'record': CFG_SITE_RECORD,
           'siteurl': CFG_SITE_URL}

    args = parse_url_string(bfo.user_info['uri'])
    journal_name = args["journal_name"]
    if journal_name and \
       (journal_name in [info.get('journal_name', '') for info in get_journals_ids_and_names()]):
        # We are displaying a WebJournal article: URL is slightly different
        url = make_journal_url(bfo.user_info['uri'])

    return """\

""" % {
        'siteurl': CFG_SITE_URL,
        'sciencewise': sciencewise_script,
        'title': escape_javascript_string(title,
                                          escape_for_html=False,
                                          escape_CDATA=True),
        'description': escape_javascript_string(description,
                                                escape_for_html=False,
                                                escape_CDATA=True),
        'sites_js': sites_js,
        'url': url,
    }

def escape_values(bfo):
    """
    Called by BibFormat in order to check if output of this element
    should be escaped.
    """
    return 0
<|endoftext|>


<commit_before>#! /usr/bin/python
import json, cloudconnect, requests, argparse, time, sys

'''
Examples:
# Create www CNAME pointed to root and Orange Cloud
python massDNSupdates.py --key xxxx --email me@you.com --new --proxy --record-name www --record-type CNAME --content @

# Delete www CNAME
python massDNSupdates.py --key xxxx --email me@you.com --delete --record-name www --record-type CNAME

# Modify www CNAME records and Grey cloud
python massDNSupdates.py --key xxxx --email me@you.com --edit --no-proxy --record-name www --record-type CNAME --content sub.domain.com

'''

parser = argparse.ArgumentParser()

# Authentication
parser.add_argument('--key', help='API key', required=True)
parser.add_argument('--email', help='API email', required=True)
parser.add_argument('--identity', help='Use this instead of key and email if you have keyrings') # Future Dev with keyrings

# Actions
parser.add_argument('--new', help='Create new DNS records', action='store_true')
parser.add_argument('--delete', help='Delete DNS records', action='store_true')
parser.add_argument('--edit', help='Modify DNS records', action='store_true')
parser.add_argument('--view', help='View DNS records', action='store_true')
parser.add_argument('--exclude', help='Exclude domains') # Future Dev

# DNS parameters
parser.add_argument('--content', help='Where the record will resolve to')
parser.add_argument('--record-name', dest='recordName', help='If not defined root will be chosen, other wise only provide subdoamain name')
# example --record-name www
parser.add_argument('--record-type', dest='recordType' , help='Only A records will be changed by default. You can use @ point to root domain')
# example --record-type CNAME

# Orange/Grey Cloud
parser.add_argument('--proxy', help='Orange Cloud all records', dest='proxy', action='store_true')
parser.add_argument('--no-proxy', help='Grey Cloudall records', dest='proxy', action='store_false')
parser.set_defaults(proxy=True)

args = parser.parse_args()

cc = cloudconnect.CloudConnect(args.email, args.key)
page = 1

while True:
  listZones = cc.list_zones(page=page, per_page=20)
  for zone in listZones['result']:
    zid = zone['id']
    zName = zone['name']
    content = args.content

    # Point cname record to root
    if args.content == '@':
      content = zName

    # Defaults
    recType = []
    proxied = args.proxy

    # Record Name
    if args.recordName == '@':
      recName = zName
    elif args.recordName:
      recName = '{}.{}'.format(args.recordName, zName)
    else:
      # Default to root
      recName = zName

    if args.recordType:
      recType = args.recordType.upper()

    if args.proxy:
      proxied = args.proxy

    # View DNS records
    if args.view:
      listDNSRecs = cc.list_dns_records(zid, type=recType, name=recName)
      if listDNSRecs['success']:
        for dnsRecs in listDNSRecs['result']:
          did = dnsRecs['id']
          dName = dnsRecs['name']
          dType = dnsRecs['type']
          dContent = dnsRecs['content']
          if dnsRecs['proxied']:
            dProxy = '+'
          else:
            dProxy = '-'
          print '{} {} {} {} {}'.format(dProxy, did, dName, dType, dContent)

    # Create New DNS Records
    if args.new:
      createDNS = cc.create_dns_record(zid, recType, recName, content=content, proxied=proxied)
      if createDNS['success']:
        print 'Created {} to {}'.format(recName, content)
      else:
        print 'Tried creating {} and failed'.format(recName)
        print >> sys.stderr, 'Failure: {} Message: {}'.format(recName, createDNS)
    else:
      dnsRecs = cc.list_dns_records(zid, name=recName, type=recType)

      for dnsRec in dnsRecs['result']:
        recID = dnsRec['id']

        # Delete DNS records
        if args.delete:
          deleteDNS = cc.delete_dns_record(zid, recID)

          if deleteDNS['success']:
            print 'deleted {} {}'.format(recType, recName)
          else:
            print 'Tried deleting {} and Failed'.format(recName)
            print >> sys.stderr, 'Failure: {} Message: {}'.format(recName, deleteDNS)

        # Modify DNS records
        if args.edit:
          updateDNS = cc.update_dns_record(zid, recID, content=content, type=recType, name=recName, proxied=proxied)

          if updateDNS['success']:
            print 'updated {} to {}'.format(recName, content)
          else:
            print 'Tried modifying {} and Failed'.format(recName)
            print >> sys.stderr, 'Failure: {} Message: {}'.format(recName, updateDNS)
  page = page + 1
  if listZones['result_info']['count'] == 0:
    break

<commit_msg>support for content match<commit_after>#! /usr/bin/python
import json, cloudconnect, requests, argparse, time, sys

'''
Examples:
# Create www CNAME pointed to root and Orange Cloud
python massDNSupdates.py --key xxxx --email me@you.com --new --proxy --record-name www --record-type CNAME --content @

# Delete www CNAME
python massDNSupdates.py --key xxxx --email me@you.com --delete --record-name www --record-type CNAME

# Modify www CNAME records and Grey cloud
python massDNSupdates.py --key xxxx --email me@you.com --edit --no-proxy --record-name www --record-type CNAME --content sub.domain.com

# Content match. Only view/edit specific IP address. Useful if some zones don't share the same IP or if you have multiple IPs.
python massDNSupdates.py --key xxxx --email me@you.com --edit --record-name www.test --record-type A --content 4.5.6.9 --content-match 4.5.6.8
'''

parser = argparse.ArgumentParser()

# Authentication
parser.add_argument('--key', help='API key', required=True)
parser.add_argument('--email', help='API email', required=True)
parser.add_argument('--identity', help='Use this instead of key and email if you have keyrings') # Future Dev with keyrings

# Actions
parser.add_argument('--new', help='Create new DNS records', action='store_true')
parser.add_argument('--delete', help='Delete DNS records', action='store_true')
parser.add_argument('--edit', help='Modify DNS records', action='store_true')
parser.add_argument('--view', help='View DNS records', action='store_true')
parser.add_argument('--exclude', help='Exclude domains') # Future Dev

# DNS parameters
parser.add_argument('--content', help='Where the record will resolve to')
parser.add_argument('--record-name', dest='recordName', help='If not defined root will be chosen, other wise only provide subdoamain name')
# example --record-name www
parser.add_argument('--record-type', dest='recordType' , help='Only A records will be changed by default. You can use @ point to root domain')
# example --record-type CNAME
parser.add_argument('--content-match', dest='contentMatch', help='Only match this content. This helps only replace some domains.')
# example --content-match 2.3.4.5

# Orange/Grey Cloud
parser.add_argument('--proxy', help='Orange Cloud all records', dest='proxy', action='store_true')
parser.add_argument('--no-proxy', help='Grey Cloudall records', dest='proxy', action='store_false')
parser.set_defaults(proxy=True)

args = parser.parse_args()

cc = cloudconnect.CloudConnect(args.email, args.key)
page = 1

while True:
  listZones = cc.list_zones(page=page, per_page=20)
  for zone in listZones['result']:
    zid = zone['id']
    zName = zone['name']
    content = args.content

    # Point cname record to root
    if args.content == '@':
      content = zName

    # Defaults
    recType = []
    contentMatch = []
    proxied = args.proxy

    # Record Name
    if args.recordName == '@':
      recName = zName
    elif args.recordName:
      recName = '{}.{}'.format(args.recordName, zName)
    else:
      # Default to root
      recName = zName

    if args.recordType:
      recType = args.recordType.upper()

    if args.proxy:
      proxied = args.proxy

    if args.contentMatch:
      contentMatch = args.contentMatch

    # View DNS records
    if args.view:
      listDNSRecs = cc.list_dns_records(zid, type=recType, name=recName, content=contentMatch)
      if listDNSRecs['success']:
        for dnsRecs in listDNSRecs['result']:
          did = dnsRecs['id']
          dName = dnsRecs['name']
          dType = dnsRecs['type']
          dContent = dnsRecs['content']
          if dnsRecs['proxied']:
            dProxy = '+'
          else:
            dProxy = '-'
          print '{} {} {} {} {}'.format(dProxy, did, dName, dType, dContent)

    # Create New DNS Records
    if args.new:
      createDNS = cc.create_dns_record(zid, recType, recName, content=content, proxied=proxied)
      if createDNS['success']:
        print 'Created {} to {}'.format(recName, content)
      else:
        print 'Tried creating {} and failed'.format(recName)
        print >> sys.stderr, 'Failure: {} Message: {}'.format(recName, createDNS)

    else:
      dnsRecs = cc.list_dns_records(zid, name=recName, type=recType, content=contentMatch)

      for dnsRec in dnsRecs['result']:
        recID = dnsRec['id']

        # Delete DNS records
        if args.delete:
          deleteDNS = cc.delete_dns_record(zid, recID)

          if deleteDNS['success']:
            print 'deleted {} {}'.format(recType, recName)
          else:
            print 'Tried deleting {} and Failed'.format(recName)
            print >> sys.stderr, 'Failure: {} Message: {}'.format(recName, deleteDNS)

        # Modify DNS records
        if args.edit:
          updateDNS = cc.update_dns_record(zid, recID, content=content, type=recType, name=recName, proxied=proxied)

          if updateDNS['success']:
            print 'updated {} to {}'.format(recName, content)
          else:
            print 'Tried modifying {} and Failed'.format(recName)
            print >> sys.stderr, 'Failure: {} Message: {}'.format(recName, updateDNS)
  page = page + 1
  if listZones['result_info']['count'] == 0:
    break

<|endoftext|>


<commit_before>#!/usr/bin/env python

import fileinput
from domain import Domain
from utils import find_domain, gen_ptr, check_ipv4, check_ipv6
from database import db_connect, db_get_domains, db_create_domains, db_delete_domains

typemap = { 'N': 'NS', 'M': 'MX', 'C': 'CNAME' }

cur_ttl = '3600'
cur_domain = None
all_domains = {}
all_db_domains = {}
warning = 0
error = 0

def main():

    row = 0
    global warning, error

    for line in fileinput.input():
        row += 1
        line = line.rstrip('\n\r')
        if not line:
            continue
        s = line.split()
        sl = len(s)
        if s[0] =='#':
            continue
        elif s[0] == 'T':
            if sl == 2:
                if s[1].isdigit() and int(s[1]) > 0:
                    cur_ttl = int(s[1])
                else:
                    print('W: Not a valid TTL value on line %d' % row)
                    warning += 1
            else:
                print('W: No arguments for TTL value on line %d' % row)
                warning += 1
        elif s[0] == 'D':
            if sl == 4:
                if s[1] not in all_domains:
                    d = Domain(s[1], s[2], s[3], cur_ttl)
                    all_domains[s[1]] = d
                    current_domain = d
                else:
                    print('E: duplicate domain %s at line %d' %(s[1], row))
                    error += 1
            else:
                print('E: Wrong number of arguments for Domain on line %d' % row)
                error += 1
        elif s[0] == 'N':
            if sl > 1:
                for ns in s[1:]:
                    d.add_record(d.name, 'NS', ns, 0, cur_ttl)
            else:
                print('W: No arguments for NS on line %d' % row)
                warning += 1
        elif s[0] == 'M':
            if sl > 1:
                prio = 10
                for x in s[1:]:
                    if x.isdigit():
                        prio = int(x)
                    else:
                        d.add_record(d.name, 'MX', x, prio, cur_ttl)
            else:
                print('W: No arguments for MX on line %d' % row)
                warning += 1
        elif s[0] == 'C':
            if sl == 3:
                find_domain(s[1], all_domains).add_record(s[1], 'CNAME', s[2], 0, cur_ttl)
            else:
                print('W: Wrong number of arguments for CNAME on line %d' % row)
                warning += 1
        elif check_ipv4(s[0]):
            if sl > 1:
                for x in s[1:]:
                    d = find_domain(x, all_domains)
                    if d:
                        d.add_record(x, 'A', s[0], 0, cur_ttl)
                        ptr = gen_ptr(s[0])
                        ptrd = find_domain(ptr, all_domains)
                        if ptrd:
                            ptrd.add_record(ptr, 'PTR', x, 0, cur_ttl)
                        else:
                            print('W: Missing domain for PTR %s on line %s' % (ptr, row))
                            warning += 1
                    else:
                        print('W: Missing domain for A %s on line %s' % (x, row))
                        warning += 1
            else:
                print('W: No names for A on line %d' % row)
                warning += 1
        else:
            print('W: Invalid row %d' % row);
            warning += 1

def sync():
    db_connect()
    all_db_domains = db_get_domains()

    list_domains = all_domains.keys()
    list_db_domains = all_db_domains.keys()
    create_list = list(set(list_domains) - set(list_db_domains))
    delete_list = list(set(list_db_domains) - set(list_domains))

    db_create_domains(create_list)
    db_delete_domains(delete_list)

    for i in list_domains:
        d = all_domains[i]
        d.sync_domain()

main()

print('%d error(s) and %d warning(s)' % (error, warning))

if error == 0:
    sync()
else:
    print('Errors found, not syncing')
<commit_msg>Rewrote arghandling<commit_after>#!/usr/bin/env python

import argparse
from domain import Domain
from utils import find_domain, gen_ptr, check_ipv4, check_ipv6
from database import db_connect, db_get_domains, db_create_domains, db_delete_domains

typemap = { 'N': 'NS', 'M': 'MX', 'C': 'CNAME' }

cur_ttl = '3600'
cur_domain = None
all_domains = {}
all_db_domains = {}
warning = 0
error = 0

def parse(fname):
    global warning, error
    row = 0

    try:
        with open(fname) as f:
            for line in f.readlines():
                row += 1
                line = line.rstrip('\n\r')
                if not line:
                    continue
                s = line.split()
                sl = len(s)
                if s[0] =='#':
                    continue
                elif s[0] == 'T':
                    if sl == 2:
                        if s[1].isdigit() and int(s[1]) > 0:
                            cur_ttl = int(s[1])
                        else:
                            print('W: Not a valid TTL value on line %d' % row)
                            warning += 1
                    else:
                        print('W: No arguments for TTL value on line %d' % row)
                        warning += 1
                elif s[0] == 'D':
                    if sl == 4:
                        if s[1] not in all_domains:
                            d = Domain(s[1], s[2], s[3], cur_ttl)
                            all_domains[s[1]] = d
                            current_domain = d
                        else:
                            print('E: duplicate domain %s at line %d' %(s[1], row))
                            error += 1
                    else:
                        print('E: Wrong number of arguments for Domain on line %d' % row)
                        error += 1
                elif s[0] == 'N':
                    if sl > 1:
                        for ns in s[1:]:
                            d.add_record(d.name, 'NS', ns, 0, cur_ttl)
                    else:
                        print('W: No arguments for NS on line %d' % row)
                        warning += 1
                elif s[0] == 'M':
                    if sl > 1:
                        prio = 10
                        for x in s[1:]:
                            if x.isdigit():
                                prio = int(x)
                            else:
                                d.add_record(d.name, 'MX', x, prio, cur_ttl)
                    else:
                        print('W: No arguments for MX on line %d' % row)
                        warning += 1
                elif s[0] == 'C':
                    if sl == 3:
                        find_domain(s[1], all_domains).add_record(s[1], 'CNAME', s[2], 0, cur_ttl)
                    else:
                        print('W: Wrong number of arguments for CNAME on line %d' % row)
                        warning += 1
                elif check_ipv4(s[0]):
                    if sl > 1:
                        for x in s[1:]:
                            d = find_domain(x, all_domains)
                            if d:
                                d.add_record(x, 'A', s[0], 0, cur_ttl)
                                ptr = gen_ptr(s[0])
                                ptrd = find_domain(ptr, all_domains)
                                if ptrd:
                                    ptrd.add_record(ptr, 'PTR', x, 0, cur_ttl)
                                else:
                                    print('W: Missing domain for PTR %s on line %s' % (ptr, row))
                                    warning += 1
                            else:
                                print('W: Missing domain for A %s on line %s' % (x, row))
                                warning += 1
                    else:
                        print('W: No names for A on line %d' % row)
                        warning += 1
                else:
                    print('W: Invalid row %d' % row);
                    warning += 1
    except IOError as e:
        print '%s: %s' % (n, e.strerror)

def sync():
    db_connect()
    all_db_domains = db_get_domains()

    list_domains = all_domains.keys()
    list_db_domains = all_db_domains.keys()
    create_list = list(set(list_domains) - set(list_db_domains))
    delete_list = list(set(list_db_domains) - set(list_domains))

    db_create_domains(create_list)
    db_delete_domains(delete_list)

    for i in list_domains:
        d = all_domains[i]
        d.sync_domain()

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('files', metavar='file', nargs='+', help='the files to parse')
    parser.add_argument("-v", "--verbosity", action="count", default=0, help="increase output verbosity")
    args = parser.parse_args()

    for fname in args.files:
        parse(fname)

    print('%d error(s) and %d warning(s)' % (error, warning))

    if error == 0:
        sync()
    else:
        print('Errors found, not syncing')

main()
<|endoftext|>


<commit_before><commit_msg>Bump required Kube versions as well<commit_after><|endoftext|>


<commit_before>import flask
import functools
import urllib.parse
from .. import auth
from .. import themes
from .. import database
from .. import settings

INVALID_PERMISSIONS_FLASH_MESSAGE = "Sorry, you don't have permission to view that page."

def checkForSession():
	if "uid" in flask.session:
		session = auth.validateSession(flask.session["uid"])
		if session != None:
			return session
	return None

def redirectAndSave(path):
	flask.session["donePage"] = urllib.parse.urlparse(flask.request.url).path
	return flask.redirect(path)

def markRedirectAsRecovered():
	if "donePage" in flask.session:
		del flask.session["donePage"]
	else:
		raise KeyError("No redirect to be recovered from.")

def canRecoverFromRedirect():
	if "donePage" in flask.session:
		return flask.session["donePage"]
	return None

#Decorator which checks if a user logged in and capable of using the specified permissions.
#If redirectPage is equal to none,
#the target funciton MUST have the arguments authed and authMessage defined.
def checkUserPermissions(redirectPage = None, saveRedirect = True, redirectMessage = INVALID_PERMISSIONS_FLASH_MESSAGE, requiredPermissions = None):
	def decorator(function):
		def decorated(*args, **kwargs):
			session = checkForSession()
			if session != None:
				username = session.user.username
				result = True
				#If we don't have any permissions necessary, a login is enough.
				#Otherwise, we're going to check to make sure that all necessary permissions are in place.
				if requiredPermissions != None:
					if type(requiredPermissions) == str:
						result = auth.userHasPermission(username, requiredPermissions)
					else:
						for permission in requiredPermissions:
							if not auth.userHasPermission(username, permission):
								result = False
				#If all permissions is valid, redirect as needed.
				if result:
					if redirectPage != None:
						return function(*args, **kwargs)
					else:
						return function(authed = True, authMessage = redirectMessage, *args, **kwargs)
				else:
					#We don't want to flash on thigns like ajax routes, so we use redirectPage != None
					willFlash = redirectPage != None
					return _permissionRedirect(redirectPage, saveRedirect, redirectMessage, willFlash)
			else:
				return _permissionRedirect(redirectPage, saveRedirect, redirectMessage, False)
		return functools.update_wrapper(decorated, function)
	return decorator

def _permissionRedirect(redirectPage, saveRedirect, redirectMessage, flash):
	if flash:
		flask.flash(redirectMessage)
	if redirectPage != None:
		#We don't want to save the redirect if either the user page doesn't need it or there's one already saved
		if canRecoverFromRedirect() or not saveRedirect:
			return flask.redirect(redirectPage)
		else:
			return redirectAndSave(redirectPage)
	else:
		return function(authed = False, authMessage = redirectMessage, *args, **kwargs)

#Decorator to accomodate for SQLAlchamey session life.
#Creates a session on request, closes it after.
def usesDatabase(connectionName):
	def decorator(function):
		def decorated():
			databaseConnection = database.ConnectionManager.getConnection("main")
			result = function()
			databaseConnection.closeSession()
			return result
		return decorated
	return decorator

#Will return all information that is needed to render a post.
#Prevents fragmentation in various post display methods
def getPostsParameters():
	title = settings.getSettingValue("title")
	subtitle = settings.getSettingValue("subtitle")
	displayName = settings.getSettingValue("display_name")
	theme = themes.getCurrentTheme()
	return {
		"blogTitle": title,
		"blogSubtitle": subtitle,
		"displayName": displayName,
		"theme": theme["theme"]
	}

#Renders the theme's template if the theme contains one
#Otherwise, it renders the default template
def renderPosts(defaultPath, *args, **kwargs):
	theme = themes.getCurrentTheme()
	template = theme["template"]
	if template == None:
		templateFile = open(defaultPath, "r")
		template = templateFile.read()
		templateFile.close()
	return flask.render_template_string(template, *args, **kwargs)
<commit_msg>Remove connectionName parameter from usesDatabase<commit_after>import flask
import functools
import urllib.parse
from .. import auth
from .. import themes
from .. import database
from .. import settings

INVALID_PERMISSIONS_FLASH_MESSAGE = "Sorry, you don't have permission to view that page."

def checkForSession():
	if "uid" in flask.session:
		session = auth.validateSession(flask.session["uid"])
		if session != None:
			return session
	return None

def redirectAndSave(path):
	flask.session["donePage"] = urllib.parse.urlparse(flask.request.url).path
	return flask.redirect(path)

def markRedirectAsRecovered():
	if "donePage" in flask.session:
		del flask.session["donePage"]
	else:
		raise KeyError("No redirect to be recovered from.")

def canRecoverFromRedirect():
	if "donePage" in flask.session:
		return flask.session["donePage"]
	return None

#Decorator which checks if a user logged in and capable of using the specified permissions.
#If redirectPage is equal to none,
#the target funciton MUST have the arguments authed and authMessage defined.
def checkUserPermissions(redirectPage = None, saveRedirect = True, redirectMessage = INVALID_PERMISSIONS_FLASH_MESSAGE, requiredPermissions = None):
	def decorator(function):
		def decorated(*args, **kwargs):
			session = checkForSession()
			if session != None:
				username = session.user.username
				result = True
				#If we don't have any permissions necessary, a login is enough.
				#Otherwise, we're going to check to make sure that all necessary permissions are in place.
				if requiredPermissions != None:
					if type(requiredPermissions) == str:
						result = auth.userHasPermission(username, requiredPermissions)
					else:
						for permission in requiredPermissions:
							if not auth.userHasPermission(username, permission):
								result = False
				#If all permissions is valid, redirect as needed.
				if result:
					if redirectPage != None:
						return function(*args, **kwargs)
					else:
						return function(authed = True, authMessage = redirectMessage, *args, **kwargs)
				else:
					#We don't want to flash on thigns like ajax routes, so we use redirectPage != None
					willFlash = redirectPage != None
					return _permissionRedirect(redirectPage, saveRedirect, redirectMessage, willFlash)
			else:
				return _permissionRedirect(redirectPage, saveRedirect, redirectMessage, False)
		return functools.update_wrapper(decorated, function)
	return decorator

def _permissionRedirect(redirectPage, saveRedirect, redirectMessage, flash):
	if flash:
		flask.flash(redirectMessage)
	if redirectPage != None:
		#We don't want to save the redirect if either the user page doesn't need it or there's one already saved
		if canRecoverFromRedirect() or not saveRedirect:
			return flask.redirect(redirectPage)
		else:
			return redirectAndSave(redirectPage)
	else:
		return function(authed = False, authMessage = redirectMessage, *args, **kwargs)

#Decorator to accomodate for SQLAlchamey session life.
#Creates a session on request, closes it after.
def usesDatabase():
	def decorated():
		databaseConnection = database.ConnectionManager.getConnection("main")
		result = function()
		databaseConnection.closeSession()
		return result
	return decorated

#Will return all information that is needed to render a post.
#Prevents fragmentation in various post display methods
def getPostsParameters():
	title = settings.getSettingValue("title")
	subtitle = settings.getSettingValue("subtitle")
	displayName = settings.getSettingValue("display_name")
	theme = themes.getCurrentTheme()
	return {
		"blogTitle": title,
		"blogSubtitle": subtitle,
		"displayName": displayName,
		"theme": theme["theme"]
	}

#Renders the theme's template if the theme contains one
#Otherwise, it renders the default template
def renderPosts(defaultPath, *args, **kwargs):
	theme = themes.getCurrentTheme()
	template = theme["template"]
	if template == None:
		templateFile = open(defaultPath, "r")
		template = templateFile.read()
		templateFile.close()
	return flask.render_template_string(template, *args, **kwargs)
<|endoftext|>


<commit_before>import json
import os

import datetime
import threading

import gpxpy
import gpxpy.gpx

from Adafruit_BME280 import *
import paho.mqtt.client as paho
import veml6070
import time

from external_tracker import post_update, TRACKER_URL
from read_serial import get_dust_particles, GPS
import pickle
import serial

broker = "127.0.0.1"
port = 1883
filename = datetime.datetime.now().isoformat() + ".log"
data = {}

def on_publish(client, userdata, result):  # create function for callback
    # print("data published \n")
    pass

def new_gpx_file():
    # gpx file
    gpx = gpxpy.gpx.GPX()
    gpx_track = gpxpy.gpx.GPXTrack()
    gpx.tracks.append(gpx_track)
    # gpx.extensions = "temp"

    # Create first segment in our GPX track:
    gpx_segment = gpxpy.gpx.GPXTrackSegment()
    gpx_track.segments.append(gpx_segment)
    return gpx, gpx_segment

def publish_data(client, data):
    client.publish("sensors/temperature", "%0.3f" % (data["temperature"]), retain=True)
    client.publish("sensors/pressure", "%0.3f" % data["pressure"])
    client.publish("sensors/humidity", "%0.3f" % data["humidity"])
    client.publish("sensors/dewpoint", "%0.3f" % data["dew_point"])
    client.publish("sensors/uv", "%0.3f" % data["uv"])
    client.publish("sensors/uv_raw", "%0.3f" % data["uv_raw"])
    client.publish("sensors/dust_particles", "%0.3f" % data["dust_particles"])

def publish_template(client, template):
    client.publish("template/html_template", "
 "
                   + template + "
")

def append_data(d):
    if os.path.exists(filename):
        with open(filename, "rb") as data_file:
            data_list = pickle.load(data_file)
            data_list.append(d)
        with open(filename, "wb") as data_file:
            pickle.dump(data_list, data_file)
    else:
        with open(filename, "wb") as data_file:
            pickle.dump([], data_file)

def generate_template(gps_dat):
    return "Longitude: " + str(gps_dat["longitude"]) + \
           "
 Latitude: " + str(gps_dat["latitude"]) + \
           "
 Speed over ground: " + str(gps_dat["speed"]) + \
           "
 Altitude: " + str(gps_dat["altitude"]) + "
"

gps_data_list = []
gps = GPS()

def main_loop():
    # config bme sensor
    sensor = BME280(p_mode=BME280_OSAMPLE_8, t_mode=BME280_OSAMPLE_2, h_mode=BME280_OSAMPLE_1, filter=BME280_FILTER_16)
    tstart = time.time()
    client1 = paho.Client("control1")  # create client object
    client1.on_publish = on_publish  # assign function to callback
    client1.connect(broker, port)  # establish connection
    veml = veml6070.Veml6070()
    veml.set_integration_time(veml6070.INTEGRATIONTIME_1T)

    # gpx file
    gpx, gpx_segment = new_gpx_file()

    data_list = []
    particles_mean = []

    start_time = datetime.datetime.now()
    data_published_time = datetime.datetime.now()

    while True:
        data["datetime"] = datetime.datetime.now()
        data["temperature"] = round(sensor.read_temperature(), 2)
        data["pressure"] = round(sensor.read_pressure() / 100, 3)
        data["humidity"] = round(sensor.read_humidity(), 2)
        data["dew_point"] = round(sensor.read_dewpoint(), 2)
        print "Data from BME280 read!"
        data["uv_raw"] = round(veml.get_uva_light_intensity_raw(), 3)
        data["uv"] = round(veml.get_uva_light_intensity(), 3)
        print "Data form UV read!"
        data["dust_particles"] = round(get_dust_particles(), 2)
        print "Dust particles read!"

        # Average out dust particles
        particles_mean.append(data["dust_particles"])
        if len(particles_mean) > 9:
            del particles_mean[0]
        particles_set = list(particles_mean)
        particles_set.sort()
        particles = particles_set[len(particles_set) // 2]
        data["dust_particles"] = particles if particles else 0

        data_list.append(data)

        # Publish to MQTT server
        publish_data(client=client1, data=data)

        for gps_dat in GPS.gps_dat_list:
            if gps_dat and not gps_dat["latitude"] == 0:
                data.update(gps_dat)

                # Publish on MQTT server
                publish_template(client=client1,
                                 template=generate_template(gps_dat))
                if gps_dat["speed"] > 0.5:
                    update_interval = 20
                else:
                    update_interval = 120

                # Post to external tracker
                if (datetime.datetime.now() - data_published_time) > datetime.timedelta(seconds=update_interval):
                    data_published_time = datetime.datetime.now()
                    try:
                        post_update(latitude=data["latitude"], longitude=data["longitude"], timestamp=data["datetime"])
                        print "data posted to: " + TRACKER_URL
                    except:
                        print datetime.datetime.now().isoformat() + "\tNo route to host: " + TRACKER_URL

                pld = {"name": "point", "lat": data["latitude"], "lon": data["longitude"], "radius": 10, "command": {
                    "lat": data["latitude"], "lon": data["longitude"], "zoom": 18
                }}
                client1.publish('gps/worldmap', payload=json.dumps(pld),)
                # Create points in GPX file:
                point = gpxpy.gpx.GPXTrackPoint(data["latitude"],
                                                data["longitude"],
                                                elevation=data["altitude"],
                                                time=datetime.datetime.now())
                point.extensions = dict(data)
                gpx_segment.points.append(point)
                if (datetime.datetime.now() - start_time) > datetime.timedelta(minutes=10):
                    start_time = datetime.datetime.now()
                    fname = "../tracks/track" + datetime.datetime.now().strftime("-%H%M-%d%m") + ".gpx"
                    with open(fname, "w") as f:
                        print datetime.datetime.now().isoformat() + "GPX file printed! Fname: " + fname
                        f.write(gpx.to_xml(version="1.1"))
                    gpx, gpx_segment = new_gpx_file()
        GPS.clear_data()

try:
    main_loop()
except:
    gps.stop()
    raise
<commit_msg>gps fix<commit_after>import json
import os

import datetime
import threading

import gpxpy
import gpxpy.gpx

from Adafruit_BME280 import *
import paho.mqtt.client as paho
import veml6070
import time

from external_tracker import post_update, TRACKER_URL
from read_serial import get_dust_particles, GPS
import pickle
import serial

broker = "127.0.0.1"
port = 1883
filename = datetime.datetime.now().isoformat() + ".log"
data = {}

def on_publish(client, userdata, result):  # create function for callback
    # print("data published \n")
    pass

def new_gpx_file():
    # gpx file
    gpx = gpxpy.gpx.GPX()
    gpx_track = gpxpy.gpx.GPXTrack()
    gpx.tracks.append(gpx_track)
    # gpx.extensions = "temp"

    # Create first segment in our GPX track:
    gpx_segment = gpxpy.gpx.GPXTrackSegment()
    gpx_track.segments.append(gpx_segment)
    return gpx, gpx_segment

def publish_data(client, data):
    client.publish("sensors/temperature", "%0.3f" % (data["temperature"]), retain=True)
    client.publish("sensors/pressure", "%0.3f" % data["pressure"])
    client.publish("sensors/humidity", "%0.3f" % data["humidity"])
    client.publish("sensors/dewpoint", "%0.3f" % data["dew_point"])
    client.publish("sensors/uv", "%0.3f" % data["uv"])
    client.publish("sensors/uv_raw", "%0.3f" % data["uv_raw"])
    client.publish("sensors/dust_particles", "%0.3f" % data["dust_particles"])

def publish_template(client, template):
    client.publish("template/html_template", "
 "
                   + template + "
")

def append_data(d):
    if os.path.exists(filename):
        with open(filename, "rb") as data_file:
            data_list = pickle.load(data_file)
            data_list.append(d)
        with open(filename, "wb") as data_file:
            pickle.dump(data_list, data_file)
    else:
        with open(filename, "wb") as data_file:
            pickle.dump([], data_file)

def generate_template(gps_dat):
    return "Longitude: " + str(gps_dat["longitude"]) + \
           "
 Latitude: " + str(gps_dat["latitude"]) + \
           "
 Speed over ground: " + str(gps_dat["speed"]) + \
           "
 Altitude: " + str(gps_dat["altitude"]) + "
"

gps_data_list = []
gps = GPS()

def main_loop():
    # config bme sensor
    sensor = BME280(p_mode=BME280_OSAMPLE_8, t_mode=BME280_OSAMPLE_2, h_mode=BME280_OSAMPLE_1, filter=BME280_FILTER_16)
    tstart = time.time()
    client1 = paho.Client("control1")  # create client object
    client1.on_publish = on_publish  # assign function to callback
    client1.connect(broker, port)  # establish connection
    veml = veml6070.Veml6070()
    veml.set_integration_time(veml6070.INTEGRATIONTIME_1T)

    # gpx file
    gpx, gpx_segment = new_gpx_file()

    data_list = []
    particles_mean = []

    start_time = datetime.datetime.now()
    data_published_time = datetime.datetime.now()

    while True:
        data["datetime"] = datetime.datetime.now()
        data["temperature"] = round(sensor.read_temperature(), 2)
        data["pressure"] = round(sensor.read_pressure() / 100, 3)
        data["humidity"] = round(sensor.read_humidity(), 2)
        data["dew_point"] = round(sensor.read_dewpoint(), 2)
        print "Data from BME280 read!"
        data["uv_raw"] = round(veml.get_uva_light_intensity_raw(), 3)
        data["uv"] = round(veml.get_uva_light_intensity(), 3)
        print "Data form UV read!"
        data["dust_particles"] = round(get_dust_particles(), 2)
        print "Dust particles read!"

        # Average out dust particles
        particles_mean.append(data["dust_particles"])
        if len(particles_mean) > 9:
            del particles_mean[0]
        particles_set = list(particles_mean)
        particles_set.sort()
        particles = particles_set[len(particles_set) // 2]
        data["dust_particles"] = particles if particles else 0

        data_list.append(data)

        # Publish to MQTT server
        publish_data(client=client1, data=data)

        for gps_dat in GPS.gps_dat_list:
            if gps_dat and not gps_dat["latitude"] == 0:
                data.update(gps_dat)

                # Publish on MQTT server
                publish_template(client=client1,
                                 template=generate_template(gps_dat))
                if gps_dat["speed"] > 0.5:
                    update_interval = 20
                else:
                    update_interval = 120

                # Post to external tracker
                if (datetime.datetime.now() - data_published_time) > datetime.timedelta(seconds=update_interval):
                    data_published_time = datetime.datetime.now()
                    try:
                        post_update(latitude=data["latitude"], longitude=data["longitude"], timestamp=data["datetime"])
                        print "data posted to: " + TRACKER_URL
                    except:
                        print datetime.datetime.now().isoformat() + "\tNo route to host: " + TRACKER_URL

                pld = [{"name": "point"}, {"lat": data["latitude"]}, {"lon": data["longitude"]}, {"radius": 10},{"command": [
                    {"lat": data["latitude"]}, {"lon": data["longitude"]}, {"zoom": 18}]
                }]
                print pld
                client1.publish('gps/worldmap', payload=json.dumps(pld, indent=2))
                # Create points in GPX file:
                point = gpxpy.gpx.GPXTrackPoint(data["latitude"],
                                                data["longitude"],
                                                elevation=data["altitude"],
                                                time=datetime.datetime.now())
                point.extensions = dict(data)
                gpx_segment.points.append(point)
                if (datetime.datetime.now() - start_time) > datetime.timedelta(minutes=10):
                    start_time = datetime.datetime.now()
                    fname = "../tracks/track" + datetime.datetime.now().strftime("-%H%M-%d%m") + ".gpx"
                    with open(fname, "w") as f:
                        print datetime.datetime.now().isoformat() + "GPX file printed! Fname: " + fname
                        f.write(gpx.to_xml(version="1.1"))
                    gpx, gpx_segment = new_gpx_file()
        GPS.clear_data()

try:
    main_loop()
except:
    gps.stop()
    raise
<|endoftext|>


<commit_before><commit_msg>Correct bug in last commit<commit_after><|endoftext|>


<commit_before>import os

import datetime

from Adafruit_BME280 import *
import paho.mqtt.client as paho
import veml6070
import time
from read_serial import get_dust_particles
import pickle

broker = "127.0.0.1"
port = 1883
filename = "datalist.dat"

def on_publish(client, userdata, result):  # create function for callback
    # print("data published \n")
    pass

def publish_data(client, data):
    client.publish("sensors/temperature", "%0.3f" % (data["temperature"]), retain=True)
    client.publish("sensors/pressure", "%0.3f" % data["pressure"])
    client.publish("sensors/humidity", "%0.3f" % data["humidity"])
    client.publish("sensors/dewpoint", "%0.3f" % data["dew_point"])
    client.publish("sensors/uv", "%0.3f" % data["uv"])
    client.publish("sensors/uv_raw", "%0.3f" % data["uv_raw"])
    client.publish("sensors/dust_particles", "%0.3f" % data["dust_particles"])

def append_data(data):
    if os.path.exists(filename):
        with open(filename, "rb") as data_file:
            data_list = pickle.load(data_file)
            data_list.append(data)
        with open(filename, "wb") as data_file:
            pickle.dump(data_list, data_file)
    else:
        with open(filename, "wb") as data_file:
            pickle.dump([], data_file)

def main():
    # sensor = BME280(p_mode=BME280_OSAMPLE_8, t_mode=BME280_OSAMPLE_2, h_mode=BME280_OSAMPLE_1, filter=BME280_FILTER_16)
    tstart = time.time()
    client1 = paho.Client("control1")  # create client object
    client1.on_publish = on_publish  # assign function to callback
    client1.connect(broker, port)  # establish connection
    veml = veml6070.Veml6070()
    veml.set_integration_time(veml6070.INTEGRATIONTIME_1T)
    data = {}

    data_list = []
    particles_mean = []
    particles_set = set()
    # if os.path.exists(filename):
    #     with open(filename, "rb") as data_file:
    #         data_list = pickle.load(data_file)
    # for data in data_list:
    #     publish_data(client=client1, data=data)
    while True:
        data["datetime"] = datetime.datetime.now()
        # data["temperature"] = sensor.read_temperature()
        # pascals = sensor.read_pressure()
        data["pressure"] = pascals / 100
        # data["humidity"] = sensor.read_humidity()
        # data["dew_point"] = sensor.read_dewpoint()
        data["uv_raw"] = veml.get_uva_light_intensity_raw()
        data["uv"] = veml.get_uva_light_intensity()
        # data["dust_particles"] = round((get_dust_particles() - 0.55) * 0.227 * 1000, 0)
        data["dust_particles"] = 0
        particles_mean.append(data["dust_particles"])
        if len(particles_mean) > 9:
            del particles_mean[0]
        particles_set = list(particles_mean)
        particles_set.sort()
        print particles_set
        print particles_set[len(particles_set)//2]
        print data["dust_particles"]

        data_list.append(data)
        publish_data(client=client1, data=data)
        # append_data(data)

        with open("/home/pi/temperature.txt", "aw") as file_t:
            file_t.write("%s" % data["datetime"] + " %0.3f" % data["temperature"] + "\n")
        with open("/home/pi/pressure.txt", "aw") as file_p:
            file_p.write("%s" % data["datetime"] + " %0.3f" % data["pressure"] + "\n")
        with open("/home/pi/humidity.txt", "aw") as file_h:
            file_h.write("%s" % data["datetime"] + " %0.3f" % data["humidity"] + "\n")
        with open("/home/pi/uv.txt", "aw") as file_uv:
            file_uv.write("%s" % data["datetime"] + " %0.3f" % data["uv"] + "\n")

        file_t.close()
        file_p.close()
        file_h.close()
        file_uv.close()
        time.sleep(3)

main()
<commit_msg>median of dust particles<commit_after>import os

import datetime

from Adafruit_BME280 import *
import paho.mqtt.client as paho
import veml6070
import time
from read_serial import get_dust_particles
import pickle

broker = "127.0.0.1"
port = 1883
filename = "datalist.dat"

def on_publish(client, userdata, result):  # create function for callback
    # print("data published \n")
    pass

def publish_data(client, data):
    client.publish("sensors/temperature", "%0.3f" % (data["temperature"]), retain=True)
    client.publish("sensors/pressure", "%0.3f" % data["pressure"])
    client.publish("sensors/humidity", "%0.3f" % data["humidity"])
    client.publish("sensors/dewpoint", "%0.3f" % data["dew_point"])
    client.publish("sensors/uv", "%0.3f" % data["uv"])
    client.publish("sensors/uv_raw", "%0.3f" % data["uv_raw"])
    client.publish("sensors/dust_particles", "%0.3f" % data["dust_particles"])

def append_data(data):
    if os.path.exists(filename):
        with open(filename, "rb") as data_file:
            data_list = pickle.load(data_file)
            data_list.append(data)
        with open(filename, "wb") as data_file:
            pickle.dump(data_list, data_file)
    else:
        with open(filename, "wb") as data_file:
            pickle.dump([], data_file)

def main():
    # sensor = BME280(p_mode=BME280_OSAMPLE_8, t_mode=BME280_OSAMPLE_2, h_mode=BME280_OSAMPLE_1, filter=BME280_FILTER_16)
    tstart = time.time()
    client1 = paho.Client("control1")  # create client object
    client1.on_publish = on_publish  # assign function to callback
    client1.connect(broker, port)  # establish connection
    veml = veml6070.Veml6070()
    veml.set_integration_time(veml6070.INTEGRATIONTIME_1T)
    data = {}

    data_list = []
    particles_mean = []
    particles_set = set()
    # if os.path.exists(filename):
    #     with open(filename, "rb") as data_file:
    #         data_list = pickle.load(data_file)
    # for data in data_list:
    #     publish_data(client=client1, data=data)
    while True:
        data["datetime"] = datetime.datetime.now()
        # data["temperature"] = sensor.read_temperature()
        # pascals = sensor.read_pressure()
        # data["pressure"] = pascals / 100
        # data["humidity"] = sensor.read_humidity()
        # data["dew_point"] = sensor.read_dewpoint()
        data["uv_raw"] = veml.get_uva_light_intensity_raw()
        data["uv"] = veml.get_uva_light_intensity()
        # data["dust_particles"] = round((get_dust_particles() - 0.55) * 0.227 * 1000, 0)
        data["dust_particles"] = 0
        particles_mean.append(data["dust_particles"])
        if len(particles_mean) > 9:
            del particles_mean[0]
        particles_set = list(particles_mean)
        particles_set.sort()
        print particles_set
        print particles_set[len(particles_set)//2]
        print data["dust_particles"]

        data_list.append(data)
        publish_data(client=client1, data=data)
        # append_data(data)

        with open("/home/pi/temperature.txt", "aw") as file_t:
            file_t.write("%s" % data["datetime"] + " %0.3f" % data["temperature"] + "\n")
        with open("/home/pi/pressure.txt", "aw") as file_p:
            file_p.write("%s" % data["datetime"] + " %0.3f" % data["pressure"] + "\n")
        with open("/home/pi/humidity.txt", "aw") as file_h:
            file_h.write("%s" % data["datetime"] + " %0.3f" % data["humidity"] + "\n")
        with open("/home/pi/uv.txt", "aw") as file_uv:
            file_uv.write("%s" % data["datetime"] + " %0.3f" % data["uv"] + "\n")

        file_t.close()
        file_p.close()
        file_h.close()
        file_uv.close()
        time.sleep(3)

main()
<|endoftext|>


<commit_before># Add our packagers on various platforms
julia_packagers  = ["package_osx64"] + ["package_win32", "package_win64"]
julia_packagers += ["package_linux%s"%(arch) for arch in ["32", "64", "armv7l", "ppc64le", "aarch64"]]

# Also add builders for Ubuntu and Centos builders, that won't upload anything at the end
julia_packagers += ["build_ubuntu32", "build_ubuntu64", "build_centos64"]

packager_scheduler = schedulers.AnyBranchScheduler(name="Julia binary packaging", change_filter=util.ChangeFilter(project=['JuliaLang/julia','staticfloat/julia'], branch='master'), builderNames=julia_packagers, treeStableTimer=1)
c['schedulers'].append(packager_scheduler)

# Helper function to generate the necessary julia invocation to get metadata
# about this build such as major/minor versions
@util.renderer
def make_julia_version_command(props):
    command = [
        "usr/bin/julia",
        "-e",
        "println(\"$(VERSION.major).$(VERSION.minor).$(VERSION.patch)\\n$(Base.GIT_VERSION_INFO.commit[1:10])\")"
    ]

    if 'win' in props.getProperty('slavename'):
        command[0] += '.exe'
    return command

# Parse out the full julia version generated by make_julia_version_command's command
def parse_julia_version(return_code, stdout, stderr):
    lines = stdout.split('\n')
    return {
        "majmin": lines[0][:lines[0].rfind('.')],
        "version": lines[0].strip(),
        "shortcommit": lines[1].strip(),
    }

def parse_git_log(return_code, stdout, stderr):
    lines = stdout.split('\n')
    return {
        "commitmessage": lines[0],
        "commitname": lines[1],
        "commitemail": lines[2],
        "authorname": lines[3],
        "authoremail": lines[4],
    }

@util.renderer
def gen_filename(props):
    shortcommit = props.getProperty("shortcommit")
    tar_arch = props.getProperty("tar_arch")
    if is_linux(props):
        return "julia-%s-Linux-%s.tar.gz"%(shortcommit, tar_arch)
    if is_win(props):
        return "julia-%s-WINNT-%s.exe"%(shortcommit, tar_arch)
    if is_osx(props):
        return "julia-%s-Darwin-%s.dmg"%(shortcommit, tar_arch)

# Steps to build a `make binary-dist` tarball that should work on just about every linux ever
julia_package_factory = util.BuildFactory()
julia_package_factory.useProgress = True
julia_package_factory.addSteps([
    # Fetch first (allowing failure if no existing clone is present)
    steps.ShellCommand(
        name="git fetch",
        command=["git", "fetch"],
        flunkOnFailure=False
    ),

    # Clone julia
    steps.Git(
        name="Julia checkout",
        repourl=util.Property('repository', default='git://github.com/JuliaLang/julia.git'),
        mode='incremental',
        method='clean',
        submodules=True,
        clobberOnFailure=True,
        progress=True
    ),

    # Ensure gcc and cmake are installed on OSX
    steps.ShellCommand(
        name="Install necessary brew dependencies",
        command=["brew", "install", "gcc", "cmake"],
        doStepIf=is_osx,
        flunkOnFailure=False
    ),

    # make clean first
    steps.ShellCommand(
        name="make cleanall",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s cleanall")],
        env={'CFLAGS':None, 'CPPFLAGS':None}
    ),

    # Make, forcing some degree of parallelism to cut down compile times
    steps.ShellCommand(
        name="make",
        command=["/bin/bash", "-c", util.Interpolate("make -j3 %(prop:flags)s")],
        haltOnFailure = True,
        timeout=3600,
        env={'CFLAGS':None, 'CPPFLAGS':None}
    ),

    # Test this build
    steps.ShellCommand(
        name="make testall",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s testall")],
        haltOnFailure = True,
        timeout=3600,
        env={'CFLAGS':None, 'CPPFLAGS':None}
    ),

    # Make win-extras on windows
    steps.ShellCommand(
        name="make win-extras",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s win-extras")],
        haltOnFailure = True,
        doStepIf=is_windows,
        env={'CFLAGS':None, 'CPPFLAGS':None},
    ),

    # Make binary-dist to package it up
    steps.ShellCommand(
        name="make binary-dist",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s binary-dist")],
        haltOnFailure = True,
        timeout=3600,
        env={'CFLAGS':None, 'CPPFLAGS':None},
    ),

    # Set a bunch of properties that are useful down the line
    steps.SetPropertyFromCommand(
        name="Get julia version/shortcommit",
        command=make_julia_version_command,
        extract_fn=parse_julia_version,
        want_stderr=False
    ),
    steps.SetPropertyFromCommand(
        name="Get commitmessage",
        command=["git", "log", "-1", "--pretty=format:%s%n%cN%n%cE%n%aN%n%aE"],
        extract_fn=parse_git_log,
        want_stderr=False
    ),

    # Transfer the result to the buildmaster for uploading to AWS
    steps.MasterShellCommand(
        name="mkdir julia_package",
        command=["mkdir", "-p", "/tmp/julia_package"]
    ),
    steps.FileUpload(
        workersrc=util.Interpolate("julia-%(prop:shortcommit)s-Linux-%(prop:tar_arch)s.tar.gz"),
        masterdest=util.Interpolate("/tmp/julia_package/julia-%(prop:shortcommit)s-Linux-%(prop:tar_arch)s.tar.gz")
    ),

    # Upload it to AWS and cleanup the master!
    steps.MasterShellCommand(
        name="Upload to AWS",
        command=["/bin/bash", "-c", util.Interpolate("~/bin/try_thrice ~/bin/aws put --fail --public julianightlies/bin/linux/%(prop:up_arch)s/%(prop:majmin)s/julia-%(prop:version)s-%(prop:shortcommit)s-linux%(prop:bits)s.tar.gz /tmp/julia_package/julia-%(prop:shortcommit)s-Linux-%(prop:tar_arch)s.tar.gz")],
        doStepIf=should_upload,
        haltOnFailure=True
    ),
    steps.MasterShellCommand(
        name="Upload to AWS (latest)",
        command=["/bin/bash", "-c", util.Interpolate("~/bin/try_thrice ~/bin/aws put --fail --public julianightlies/bin/linux/%(prop:up_arch)s/julia-latest-linux%(prop:bits)s.tar.gz /tmp/julia_package/julia-%(prop:shortcommit)s-Linux-%(prop:tar_arch)s.tar.gz")],
        doStepIf=should_upload_latest,
        haltOnFailure=True
    ),

    steps.MasterShellCommand(
        name="Cleanup Master",
        command=["rm", "-f", util.Interpolate("/tmp/julia_package/julia-%(prop:shortcommit)s-Linux-%(prop:tar_arch)s.tar.gz")],
        doStepIf=should_upload
    ),

    # Trigger a download of this file onto another slave for coverage purposes
    steps.Trigger(schedulerNames=["Julia Coverage Testing"],
        set_properties={
            'url': util.Interpolate('https://s3.amazonaws.com/julianightlies/bin/linux/%(prop:up_arch)s/%(prop:majmin)s/julia-%(prop:version)s-%(prop:shortcommit)s-linux%(prop:bits)s.tar.gz'),
            'commitmessage': util.Property('commitmessage'),
            'commitname': util.Property('commitname'),
            'commitemail': util.Property('commitemail'),
            'authorname': util.Property('authorname'),
            'authoremail': util.Property('authoremail'),
            'shortcommit': util.Property('shortcommit'),
        },
        waitForFinish=False,
        doStepIf=should_run_coverage
    )
])

# Map each builder to each packager
mapping = {
    "package_osx64": "osx10_10-x64",
    "package_win32": "win6_2-x86",
    "package_win64": "win6_2-x64",
    "package_linux32": "centos5_11-x86",
    "package_linux64": "centos5_11-x64",
    "package_linuxarmv7l": "debian7_11-armv7l",
    "package_linuxppc64le": "centos7_2-ppc64le",
    "package_linuxaarch64": "centos7_2-aarch64",

    # These builders don't get uploaded
    "build_ubuntu32": "ubuntu14_04-x86",
    "build_ubuntu64": "ubuntu14_04-x64",
    "build_centos64": "centos7_1-x64",
}
for packager, slave in mapping.iteritems():
    c['builders'].append(util.BuilderConfig(
        name=packager,
        workernames=[slave],
        tags=["Packaging"],
        factory=julia_package_factory
    ))

# Add a scheduler for building release candidates/triggering builds manually
force_build_scheduler = schedulers.ForceScheduler(
    name="force_julia_package",
    label="Force Julia build/packaging",
    builderNames=julia_packagers,
    reason=util.FixedParameter(name="reason", default=""),
    codebases=[
        util.CodebaseParameter(
            "",
            name="",
            branch=util.FixedParameter(name="branch", default=""),
            repository=util.FixedParameter(name="repository", default=""),
            project=util.FixedParameter(name="project", default="Packaging"),
        )
    ],
    properties=[]
)
c['schedulers'].append(force_build_scheduler)
<commit_msg>Sync over some env changes to packaging, reorganize<commit_after># Add our packagers on various platforms
julia_packagers  = ["package_osx64"] + ["package_win32", "package_win64"]
julia_packagers += ["package_linux%s"%(arch) for arch in ["32", "64", "armv7l", "ppc64le", "aarch64"]]

# Also add builders for Ubuntu and Centos builders, that won't upload anything at the end
julia_packagers += ["build_ubuntu32", "build_ubuntu64", "build_centos64"]

packager_scheduler = schedulers.AnyBranchScheduler(name="Julia binary packaging", change_filter=util.ChangeFilter(project=['JuliaLang/julia','staticfloat/julia'], branch='master'), builderNames=julia_packagers, treeStableTimer=1)
c['schedulers'].append(packager_scheduler)

# Helper function to generate the necessary julia invocation to get metadata
# about this build such as major/minor versions
@util.renderer
def make_julia_version_command(props):
    command = [
        "usr/bin/julia",
        "-e",
        "println(\"$(VERSION.major).$(VERSION.minor).$(VERSION.patch)\\n$(Base.GIT_VERSION_INFO.commit[1:10])\")"
    ]

    if 'win' in props.getProperty('slavename'):
        command[0] += '.exe'
    return command

# Parse out the full julia version generated by make_julia_version_command's command
def parse_julia_version(return_code, stdout, stderr):
    lines = stdout.split('\n')
    return {
        "majmin": lines[0][:lines[0].rfind('.')],
        "version": lines[0].strip(),
        "shortcommit": lines[1].strip(),
    }

def parse_git_log(return_code, stdout, stderr):
    lines = stdout.split('\n')
    return {
        "commitmessage": lines[0],
        "commitname": lines[1],
        "commitemail": lines[2],
        "authorname": lines[3],
        "authoremail": lines[4],
    }

@util.renderer
def gen_filename(props):
    shortcommit = props.getProperty("shortcommit")
    tar_arch = props.getProperty("tar_arch")
    if is_linux(props):
        return "julia-%s-Linux-%s.tar.gz"%(shortcommit, tar_arch)
    if is_win(props):
        return "julia-%s-WINNT-%s.exe"%(shortcommit, tar_arch)
    if is_osx(props):
        return "julia-%s-Darwin-%s.dmg"%(shortcommit, tar_arch)

julia_package_env = {
    'CFLAGS':None,
    'CPPFLAGS': None,
    'LLVM_CMAKE': Property('llvm_cmake', default=None),
}

# Steps to build a `make binary-dist` tarball that should work on just about every linux ever
julia_package_factory = util.BuildFactory()
julia_package_factory.useProgress = True
julia_package_factory.addSteps([
    # Fetch first (allowing failure if no existing clone is present)
    steps.ShellCommand(
        name="git fetch",
        command=["git", "fetch"],
        flunkOnFailure=False
    ),

    # Clone julia
    steps.Git(
        name="Julia checkout",
        repourl=util.Property('repository', default='git://github.com/JuliaLang/julia.git'),
        mode='incremental',
        method='clean',
        submodules=True,
        clobberOnFailure=True,
        progress=True
    ),

    # Ensure gcc and cmake are installed on OSX
    steps.ShellCommand(
        name="Install necessary brew dependencies",
        command=["brew", "install", "gcc", "cmake"],
        doStepIf=is_osx,
        flunkOnFailure=False
    ),

    # make clean first
    steps.ShellCommand(
        name="make cleanall",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s cleanall")],
        env=julia_package_env,
    ),

    # Make, forcing some degree of parallelism to cut down compile times
    steps.ShellCommand(
        name="make",
        command=["/bin/bash", "-c", util.Interpolate("make -j3 %(prop:flags)s")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),

    # Test this build
    steps.ShellCommand(
        name="make testall",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s testall")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),

    # Make win-extras on windows
    steps.ShellCommand(
        name="make win-extras",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s win-extras")],
        haltOnFailure = True,
        doStepIf=is_windows,
        env=julia_package_env,
    ),

    # Make binary-dist to package it up
    steps.ShellCommand(
        name="make binary-dist",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s binary-dist")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),

    # Set a bunch of properties that are useful down the line
    steps.SetPropertyFromCommand(
        name="Get julia version/shortcommit",
        command=make_julia_version_command,
        extract_fn=parse_julia_version,
        want_stderr=False
    ),
    steps.SetPropertyFromCommand(
        name="Get commitmessage",
        command=["git", "log", "-1", "--pretty=format:%s%n%cN%n%cE%n%aN%n%aE"],
        extract_fn=parse_git_log,
        want_stderr=False
    ),

    # Transfer the result to the buildmaster for uploading to AWS
    steps.MasterShellCommand(
        name="mkdir julia_package",
        command=["mkdir", "-p", "/tmp/julia_package"]
    ),
    steps.FileUpload(
        workersrc=util.Interpolate("julia-%(prop:shortcommit)s-Linux-%(prop:tar_arch)s.tar.gz"),
        masterdest=util.Interpolate("/tmp/julia_package/julia-%(prop:shortcommit)s-Linux-%(prop:tar_arch)s.tar.gz")
    ),

    # Upload it to AWS and cleanup the master!
    steps.MasterShellCommand(
        name="Upload to AWS",
        command=["/bin/bash", "-c", util.Interpolate("~/bin/try_thrice ~/bin/aws put --fail --public julianightlies/bin/linux/%(prop:up_arch)s/%(prop:majmin)s/julia-%(prop:version)s-%(prop:shortcommit)s-linux%(prop:bits)s.tar.gz /tmp/julia_package/julia-%(prop:shortcommit)s-Linux-%(prop:tar_arch)s.tar.gz")],
        doStepIf=should_upload,
        haltOnFailure=True
    ),
    steps.MasterShellCommand(
        name="Upload to AWS (latest)",
        command=["/bin/bash", "-c", util.Interpolate("~/bin/try_thrice ~/bin/aws put --fail --public julianightlies/bin/linux/%(prop:up_arch)s/julia-latest-linux%(prop:bits)s.tar.gz /tmp/julia_package/julia-%(prop:shortcommit)s-Linux-%(prop:tar_arch)s.tar.gz")],
        doStepIf=should_upload_latest,
        haltOnFailure=True
    ),

    steps.MasterShellCommand(
        name="Cleanup Master",
        command=["rm", "-f", util.Interpolate("/tmp/julia_package/julia-%(prop:shortcommit)s-Linux-%(prop:tar_arch)s.tar.gz")],
        doStepIf=should_upload
    ),

    # Trigger a download of this file onto another slave for coverage purposes
    steps.Trigger(schedulerNames=["Julia Coverage Testing"],
        set_properties={
            'url': util.Interpolate('https://s3.amazonaws.com/julianightlies/bin/linux/%(prop:up_arch)s/%(prop:majmin)s/julia-%(prop:version)s-%(prop:shortcommit)s-linux%(prop:bits)s.tar.gz'),
            'commitmessage': util.Property('commitmessage'),
            'commitname': util.Property('commitname'),
            'commitemail': util.Property('commitemail'),
            'authorname': util.Property('authorname'),
            'authoremail': util.Property('authoremail'),
            'shortcommit': util.Property('shortcommit'),
        },
        waitForFinish=False,
        doStepIf=should_run_coverage
    )
])

# Map each builder to each worker
mapping = {
    "package_osx64": "osx10_10-x64",
    "package_win32": "win6_2-x86",
    "package_win64": "win6_2-x64",
    "package_linux32": "centos5_11-x86",
    "package_linux64": "centos5_11-x64",
    "package_linuxarmv7l": "debian7_11-armv7l",
    "package_linuxppc64le": "centos7_2-ppc64le",
    "package_linuxaarch64": "centos7_2-aarch64",

    # These builders don't get uploaded
    "build_ubuntu32": "ubuntu14_04-x86",
    "build_ubuntu64": "ubuntu14_04-x64",
    "build_centos64": "centos7_1-x64",
}
for packager, slave in mapping.iteritems():
    c['builders'].append(util.BuilderConfig(
        name=packager,
        workernames=[slave],
        tags=["Packaging"],
        factory=julia_package_factory
    ))

# Add a scheduler for building release candidates/triggering builds manually
force_build_scheduler = schedulers.ForceScheduler(
    name="force_julia_package",
    label="Force Julia build/packaging",
    builderNames=julia_packagers,
    reason=util.FixedParameter(name="reason", default=""),
    codebases=[
        util.CodebaseParameter(
            "",
            name="",
            branch=util.FixedParameter(name="branch", default=""),
            repository=util.FixedParameter(name="repository", default=""),
            project=util.FixedParameter(name="project", default="Packaging"),
        )
    ],
    properties=[]
)
c['schedulers'].append(force_build_scheduler)
<|endoftext|>


<commit_before># Add our packagers on various platforms
julia_packagers  = ["package_osx64"] + ["package_win32", "package_win64"]
julia_packagers += ["package_linux%s"%(arch) for arch in ["32", "64", "armv7l", "ppc64le", "aarch64"]]

# Also add builders for Ubuntu and Centos builders, that won't upload anything at the end
julia_packagers += ["build_ubuntu32", "build_ubuntu64", "build_centos64"]

packager_scheduler = schedulers.AnyBranchScheduler(name="Julia binary packaging", change_filter=util.ChangeFilter(project=['JuliaLang/julia','staticfloat/julia'], branch='master'), builderNames=julia_packagers, treeStableTimer=1)
c['schedulers'].append(packager_scheduler)

# Helper function to generate the necessary julia invocation to get metadata
# about this build such as major/minor versions
@util.renderer
def make_julia_version_command(props):
    command = [
        "usr/bin/julia",
        "-e",
        "println(\"$(VERSION.major).$(VERSION.minor).$(VERSION.patch)\\n$(Base.GIT_VERSION_INFO.commit[1:10])\")"
    ]

    if 'win' in props.getProperty('slavename'):
        command[0] += '.exe'
    return command

# Parse out the full julia version generated by make_julia_version_command's command
def parse_julia_version(return_code, stdout, stderr):
    lines = stdout.split('\n')
    return {
        "majmin": lines[0][:lines[0].rfind('.')],
        "version": lines[0].strip(),
        "shortcommit": lines[1].strip(),
    }

def parse_git_log(return_code, stdout, stderr):
    lines = stdout.split('\n')
    return {
        "commitmessage": lines[0],
        "commitname": lines[1],
        "commitemail": lines[2],
        "authorname": lines[3],
        "authoremail": lines[4],
    }

@util.renderer
def gen_upload_path(props):
    up_arch = props.getProperty("up_arch")
    majmin = props.getProperty("majmin")
    artifact_filename = props.getProperty("artifact_filename")
    os = get_os_name(props)
    return "julianightlies/bin/%s/%s/%s/%s"%(os, up_arch, majmin, artifact_filename)

@util.renderer
def gen_latest_upload_path(props):
    up_arch = props.getProperty("up_arch")
    artifact_filename = props.getProperty("artifact_filename")
    if artifact_filename[:6] == "julia-":
        artifact_filename = "julia-latest-%s"%(artifact_filename[6:])
    os = get_os_name(props)
    return "julianightlies/bin/latest/%s/%s/%s"%(os, up_arch, artifact_filename)

@util.renderer
def gen_upload_command(props):
    upload_path = gen_upload_path(props)
    artifact_filename = props.getProperty("artifact_filename")
    return ["/bin/bash", "-c", "~/bin/try_thrice ~/bin/aws put --fail --public %s /tmp/julia_package/%s"%(upload_path, artifact_filename)]

@util.renderer
def gen_latest_upload_command(props):
    latest_upload_path = gen_latest_upload_path(props)
    artifact_filename = props.getProperty("artifact_filename")
    return ["/bin/bash", "-c", "~/bin/try_thrice ~/bin/aws put --fail --public %s /tmp/julia_package/%s"%(latest_upload_path, artifact_filename)]

@util.renderer
def gen_coverage_properties(props):
    return {
        'url': 'https://s3.amazonaws.com/'+gen_upload_path(props),
        'commitmessage': props.getProperty('commitmessage'),
        'commitname': props.getProperty('commitname'),
        'commitemail': props.getProperty('commitemail'),
        'authorname': props.getProperty('authorname'),
        'authoremail': props.getProperty('authoremail'),
        'shortcommit': props.getProperty('shortcommit'),
    }

julia_package_env = {
    'CFLAGS':None,
    'CPPFLAGS': None,
    'LLVM_CMAKE': util.Property('llvm_cmake', default=None),
}

# Steps to build a `make binary-dist` tarball that should work on just about every linux ever
julia_package_factory = util.BuildFactory()
julia_package_factory.useProgress = True
julia_package_factory.addSteps([
    # Fetch first (allowing failure if no existing clone is present)
    steps.ShellCommand(
        name="git fetch",
        command=["git", "fetch"],
        flunkOnFailure=False
    ),

    # Clone julia
    steps.Git(
        name="Julia checkout",
        repourl=util.Property('repository', default='git://github.com/JuliaLang/julia.git'),
        mode='incremental',
        method='clean',
        submodules=True,
        clobberOnFailure=True,
        progress=True
    ),

    # Ensure gcc and cmake are installed on OSX
    steps.ShellCommand(
        name="Install necessary brew dependencies",
        command=["brew", "install", "gcc", "cmake"],
        doStepIf=is_osx,
        flunkOnFailure=False
    ),

    # make clean first
    steps.ShellCommand(
        name="make cleanall",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s cleanall")],
        env=julia_package_env,
    ),

    # Make, forcing some degree of parallelism to cut down compile times
    # Also build `debug` and `release` in parallel, we should have enough RAM for that now
    steps.ShellCommand(
        name="make",
        command=["/bin/bash", "-c", util.Interpolate("make -j3 %(prop:flags)s debug release")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),

    # Test this build
    steps.ShellCommand(
        name="make testall",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s testall")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),

    # Make win-extras on windows
    steps.ShellCommand(
        name="make win-extras",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s win-extras")],
        haltOnFailure = True,
        doStepIf=is_windows,
        env=julia_package_env,
    ),

    # Make binary-dist to package it up
    steps.ShellCommand(
        name="make binary-dist",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s binary-dist")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),

    # Set a bunch of properties that are useful down the line
    steps.SetPropertyFromCommand(
        name="Get julia version/shortcommit",
        command=make_julia_version_command,
        extract_fn=parse_julia_version,
        want_stderr=False
    ),
    steps.SetPropertyFromCommand(
        name="Get commitmessage",
        command=["git", "log", "-1", "--pretty=format:%s%n%cN%n%cE%n%aN%n%aE"],
        extract_fn=parse_git_log,
        want_stderr=False
    ),
    steps.SetPropertyFromCommand(
        name="Get build artifact filename",
        command=["make", "print-JULIA_BINARYDIST_FILENAME"],
        property="artifact_filename"
    )

    # Transfer the result to the buildmaster for uploading to AWS
    steps.MasterShellCommand(
        name="mkdir julia_package",
        command=["mkdir", "-p", "/tmp/julia_package"]
    ),

    steps.FileUpload(
        workersrc=util.Interpolate("%(prop:artifact_filename)s"),
        masterdest=util.Interpolate("/tmp/julia_package/%(prop:artifact_filename)s")
    ),

    # Upload it to AWS and cleanup the master!
    steps.MasterShellCommand(
        name="Upload to AWS",
        command=gen_upload_command,
        doStepIf=should_upload,
        haltOnFailure=True
    ),
    steps.MasterShellCommand(
        name="Upload to AWS (latest)",
        command=gen_latest_upload_command,
        doStepIf=should_upload_latest,
        haltOnFailure=True
    ),

    steps.MasterShellCommand(
        name="Cleanup Master",
        command=["rm", "-f", util.Interpolate("/tmp/julia_package/%(prop:artifact_filename)s")],
        doStepIf=should_upload
    ),

    # Trigger a download of this file onto another slave for coverage purposes
    steps.Trigger(schedulerNames=["Julia Coverage Testing"],
        set_properties=gen_coverage_properties,
        waitForFinish=False,
        doStepIf=should_run_coverage
    )
])

# Map each builder to each worker
mapping = {
    "package_osx64": "osx10_10-x64",
    "package_win32": "win6_2-x86",
    "package_win64": "win6_2-x64",
    "package_linux32": "centos5_11-x86",
    "package_linux64": "centos5_11-x64",
    "package_linuxarmv7l": "debian7_11-armv7l",
    "package_linuxppc64le": "centos7_2-ppc64le",
    "package_linuxaarch64": "centos7_2-aarch64",

    # These builders don't get uploaded
    "build_ubuntu32": "ubuntu14_04-x86",
    "build_ubuntu64": "ubuntu14_04-x64",
    "build_centos64": "centos7_1-x64",
}
for packager, slave in mapping.iteritems():
    c['builders'].append(util.BuilderConfig(
        name=packager,
        workernames=[slave],
        tags=["Packaging"],
        factory=julia_package_factory
    ))

# Add a scheduler for building release candidates/triggering builds manually
force_build_scheduler = schedulers.ForceScheduler(
    name="force_julia_package",
    label="Force Julia build/packaging",
    builderNames=julia_packagers,
    reason=util.FixedParameter(name="reason", default=""),
    codebases=[
        util.CodebaseParameter(
            "",
            name="",
            branch=util.FixedParameter(name="branch", default=""),
            repository=util.FixedParameter(name="repository", default=""),
            project=util.FixedParameter(name="project", default="Packaging"),
        )
    ],
    properties=[]
)
c['schedulers'].append(force_build_scheduler)
<commit_msg>Add missing `,`<commit_after># Add our packagers on various platforms
julia_packagers  = ["package_osx64"] + ["package_win32", "package_win64"]
julia_packagers += ["package_linux%s"%(arch) for arch in ["32", "64", "armv7l", "ppc64le", "aarch64"]]

# Also add builders for Ubuntu and Centos builders, that won't upload anything at the end
julia_packagers += ["build_ubuntu32", "build_ubuntu64", "build_centos64"]

packager_scheduler = schedulers.AnyBranchScheduler(name="Julia binary packaging", change_filter=util.ChangeFilter(project=['JuliaLang/julia','staticfloat/julia'], branch='master'), builderNames=julia_packagers, treeStableTimer=1)
c['schedulers'].append(packager_scheduler)

# Helper function to generate the necessary julia invocation to get metadata
# about this build such as major/minor versions
@util.renderer
def make_julia_version_command(props):
    command = [
        "usr/bin/julia",
        "-e",
        "println(\"$(VERSION.major).$(VERSION.minor).$(VERSION.patch)\\n$(Base.GIT_VERSION_INFO.commit[1:10])\")"
    ]

    if 'win' in props.getProperty('slavename'):
        command[0] += '.exe'
    return command

# Parse out the full julia version generated by make_julia_version_command's command
def parse_julia_version(return_code, stdout, stderr):
    lines = stdout.split('\n')
    return {
        "majmin": lines[0][:lines[0].rfind('.')],
        "version": lines[0].strip(),
        "shortcommit": lines[1].strip(),
    }

def parse_git_log(return_code, stdout, stderr):
    lines = stdout.split('\n')
    return {
        "commitmessage": lines[0],
        "commitname": lines[1],
        "commitemail": lines[2],
        "authorname": lines[3],
        "authoremail": lines[4],
    }

@util.renderer
def gen_upload_path(props):
    up_arch = props.getProperty("up_arch")
    majmin = props.getProperty("majmin")
    artifact_filename = props.getProperty("artifact_filename")
    os = get_os_name(props)
    return "julianightlies/bin/%s/%s/%s/%s"%(os, up_arch, majmin, artifact_filename)

@util.renderer
def gen_latest_upload_path(props):
    up_arch = props.getProperty("up_arch")
    artifact_filename = props.getProperty("artifact_filename")
    if artifact_filename[:6] == "julia-":
        artifact_filename = "julia-latest-%s"%(artifact_filename[6:])
    os = get_os_name(props)
    return "julianightlies/bin/latest/%s/%s/%s"%(os, up_arch, artifact_filename)

@util.renderer
def gen_upload_command(props):
    upload_path = gen_upload_path(props)
    artifact_filename = props.getProperty("artifact_filename")
    return ["/bin/bash", "-c", "~/bin/try_thrice ~/bin/aws put --fail --public %s /tmp/julia_package/%s"%(upload_path, artifact_filename)]

@util.renderer
def gen_latest_upload_command(props):
    latest_upload_path = gen_latest_upload_path(props)
    artifact_filename = props.getProperty("artifact_filename")
    return ["/bin/bash", "-c", "~/bin/try_thrice ~/bin/aws put --fail --public %s /tmp/julia_package/%s"%(latest_upload_path, artifact_filename)]

@util.renderer
def gen_coverage_properties(props):
    return {
        'url': 'https://s3.amazonaws.com/'+gen_upload_path(props),
        'commitmessage': props.getProperty('commitmessage'),
        'commitname': props.getProperty('commitname'),
        'commitemail': props.getProperty('commitemail'),
        'authorname': props.getProperty('authorname'),
        'authoremail': props.getProperty('authoremail'),
        'shortcommit': props.getProperty('shortcommit'),
    }

julia_package_env = {
    'CFLAGS':None,
    'CPPFLAGS': None,
    'LLVM_CMAKE': util.Property('llvm_cmake', default=None),
}

# Steps to build a `make binary-dist` tarball that should work on just about every linux ever
julia_package_factory = util.BuildFactory()
julia_package_factory.useProgress = True
julia_package_factory.addSteps([
    # Fetch first (allowing failure if no existing clone is present)
    steps.ShellCommand(
        name="git fetch",
        command=["git", "fetch"],
        flunkOnFailure=False
    ),

    # Clone julia
    steps.Git(
        name="Julia checkout",
        repourl=util.Property('repository', default='git://github.com/JuliaLang/julia.git'),
        mode='incremental',
        method='clean',
        submodules=True,
        clobberOnFailure=True,
        progress=True
    ),

    # Ensure gcc and cmake are installed on OSX
    steps.ShellCommand(
        name="Install necessary brew dependencies",
        command=["brew", "install", "gcc", "cmake"],
        doStepIf=is_osx,
        flunkOnFailure=False
    ),

    # make clean first
    steps.ShellCommand(
        name="make cleanall",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s cleanall")],
        env=julia_package_env,
    ),

    # Make, forcing some degree of parallelism to cut down compile times
    # Also build `debug` and `release` in parallel, we should have enough RAM for that now
    steps.ShellCommand(
        name="make",
        command=["/bin/bash", "-c", util.Interpolate("make -j3 %(prop:flags)s debug release")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),

    # Test this build
    steps.ShellCommand(
        name="make testall",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s testall")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),

    # Make win-extras on windows
    steps.ShellCommand(
        name="make win-extras",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s win-extras")],
        haltOnFailure = True,
        doStepIf=is_windows,
        env=julia_package_env,
    ),

    # Make binary-dist to package it up
    steps.ShellCommand(
        name="make binary-dist",
        command=["/bin/bash", "-c", util.Interpolate("make %(prop:flags)s binary-dist")],
        haltOnFailure = True,
        timeout=3600,
        env=julia_package_env,
    ),

    # Set a bunch of properties that are useful down the line
    steps.SetPropertyFromCommand(
        name="Get julia version/shortcommit",
        command=make_julia_version_command,
        extract_fn=parse_julia_version,
        want_stderr=False
    ),
    steps.SetPropertyFromCommand(
        name="Get commitmessage",
        command=["git", "log", "-1", "--pretty=format:%s%n%cN%n%cE%n%aN%n%aE"],
        extract_fn=parse_git_log,
        want_stderr=False
    ),
    steps.SetPropertyFromCommand(
        name="Get build artifact filename",
        command=["make", "print-JULIA_BINARYDIST_FILENAME"],
        property="artifact_filename"
    ),

    # Transfer the result to the buildmaster for uploading to AWS
    steps.MasterShellCommand(
        name="mkdir julia_package",
        command=["mkdir", "-p", "/tmp/julia_package"]
    ),

    steps.FileUpload(
        workersrc=util.Interpolate("%(prop:artifact_filename)s"),
        masterdest=util.Interpolate("/tmp/julia_package/%(prop:artifact_filename)s")
    ),

    # Upload it to AWS and cleanup the master!
    steps.MasterShellCommand(
        name="Upload to AWS",
        command=gen_upload_command,
        doStepIf=should_upload,
        haltOnFailure=True
    ),
    steps.MasterShellCommand(
        name="Upload to AWS (latest)",
        command=gen_latest_upload_command,
        doStepIf=should_upload_latest,
        haltOnFailure=True
    ),

    steps.MasterShellCommand(
        name="Cleanup Master",
        command=["rm", "-f", util.Interpolate("/tmp/julia_package/%(prop:artifact_filename)s")],
        doStepIf=should_upload
    ),

    # Trigger a download of this file onto another slave for coverage purposes
    steps.Trigger(schedulerNames=["Julia Coverage Testing"],
        set_properties=gen_coverage_properties,
        waitForFinish=False,
        doStepIf=should_run_coverage
    )
])

# Map each builder to each worker
mapping = {
    "package_osx64": "osx10_10-x64",
    "package_win32": "win6_2-x86",
    "package_win64": "win6_2-x64",
    "package_linux32": "centos5_11-x86",
    "package_linux64": "centos5_11-x64",
    "package_linuxarmv7l": "debian7_11-armv7l",
    "package_linuxppc64le": "centos7_2-ppc64le",
    "package_linuxaarch64": "centos7_2-aarch64",

    # These builders don't get uploaded
    "build_ubuntu32": "ubuntu14_04-x86",
    "build_ubuntu64": "ubuntu14_04-x64",
    "build_centos64": "centos7_1-x64",
}
for packager, slave in mapping.iteritems():
    c['builders'].append(util.BuilderConfig(
        name=packager,
        workernames=[slave],
        tags=["Packaging"],
        factory=julia_package_factory
    ))

# Add a scheduler for building release candidates/triggering builds manually
force_build_scheduler = schedulers.ForceScheduler(
    name="force_julia_package",
    label="Force Julia build/packaging",
    builderNames=julia_packagers,
    reason=util.FixedParameter(name="reason", default=""),
    codebases=[
        util.CodebaseParameter(
            "",
            name="",
            branch=util.FixedParameter(name="branch", default=""),
            repository=util.FixedParameter(name="repository", default=""),
            project=util.FixedParameter(name="project", default="Packaging"),
        )
    ],
    properties=[]
)
c['schedulers'].append(force_build_scheduler)
<|endoftext|>