<commit_before>#! /usr/bin/python
#
# Basic initialization of a test ICAT.  Setup all the stuff that must
# be there before useroffice may add investigations.
# - setup basic permissions
# - create a facility
# - create an instrument
# - create InvestigationType, DatasetType, DatafileParameter
#
# This script must be run by the ICAT root user as configured in the
# rootUserNames property.
#

import icat
import icat.config
import sys
import logging
import yaml

logging.basicConfig(level=logging.INFO)
#logging.getLogger('suds.client').setLevel(logging.DEBUG)

config = icat.config.Config()
config.add_variable('datafile', ("datafile",),
                    dict(metavar="inputdata.yaml",
                         help="name of the input datafile"))
conf = config.getconfig()

client = icat.Client(conf.url, **conf.client_kwargs)
client.login(conf.auth, conf.credentials)

# ------------------------------------------------------------
# Helper functions
# ------------------------------------------------------------

def initobj(obj, attrs):
    """Initialize an entity object from a dict of attributes."""
    for a in obj.InstAttr:
        if a != 'id' and a in attrs:
            setattr(obj, a, attrs[a])

# ------------------------------------------------------------
# Read input data
# ------------------------------------------------------------

if conf.datafile == "-":
    f = sys.stdin
else:
    f = open(conf.datafile, 'r')
data = yaml.load(f)
f.close()

# ------------------------------------------------------------
# Setup some basic permissions
# ------------------------------------------------------------

# The name of the Group entity type has been changed to Grouping in
# ICAT 4.3.
if client.apiversion < '4.2.99':
    groupname = "Group"
else:
    groupname = "Grouping"

# List of all tables in the schema
alltables = set(client.getEntityNames())

# Public tables that may be read by anybody.  Basically anything thats
# static and not related to any particular investigation.
pubtables = { "Application", "DatafileFormat", "DatasetType",
              "Facility", "FacilityCycle", "Instrument",
              "InvestigationType", "ParameterType",
              "PermissibleStringValue", "SampleType", "User", }

# Objects that useroffice might need to create.  Basically anything
# related to a particular investigation as a whole, but not to
# individual items created during the investigation (Datafiles and
# Datasets).  Plus FacilityCycle and InstrumentScientist.
uotables = { "FacilityCycle", groupname, "InstrumentScientist",
             "Investigation", "InvestigationParameter",
             "InvestigationUser", "Keyword", "Publication", "Shift",
             "Study", "StudyInvestigation", "User", "UserGroup", }
if client.apiversion > '4.2.99':
    uotables.add("InvestigationInstrument")
if client.apiversion > '4.3.99':
    uotables.add("InvestigationGroup")
if client.apiversion < '4.3.99':
    uotables.add("Rule")

# Permit root to read and write everything.  This gives ourselves the
# permissions to continue this script further on.  With ICAT 4.4 and
# newer this is not needed, as the root user already has CRUD
# permission on everything built in.
root = client.createUser("root", fullName="Root")
if client.apiversion < '4.3.99':
    rootgroup = client.createGroup("root", [ root ])
    client.createRules("CRUD", alltables, rootgroup)

# Grant public read permission to some basic tables.  Note that the
# created rules do not refer to any group.  That means they will apply
# to anybody.
client.createRules("R", pubtables)

# Special rule: each user gets the permission to see the groups he
# is in.
client.createRules("R", [ "%s <-> UserGroup <-> User[name=:user]" % groupname ])

# Add a sepcial user to be configured as reader in ids.server.  This
# user needs at least permission to read all datasets, datafiles,
# investigations and facilities.  But well, then we can make live
# simple by giving him read all permissions.
idsreader = client.createUser("idsreader", fullName="IDS reader")
rallgroup = client.createGroup("rall", [ idsreader ])
client.createRules("R", alltables - pubtables - {"Log"}, rallgroup)

# Setup permissions for useroffice.  They need to create
# Investigations and to setup access permissions for them.  Note that
# this requires the useroffice to have write permission to authz
# tables which basically gives useroffice root power.
useroffice = client.createUser("useroffice", fullName="User Office")
uogroup = client.createGroup("useroffice", [ useroffice ])
client.createRules("CRUD", uotables, uogroup)

# Special case SampleType: most tables in the data base are either
# specific to the site as a whole (most public tables) or specific to
# a particular investigation.  The former contain public data that is
# mostly static, created once at initialization and does not change
# too much later on.  The latter is private and created during each
# investigation.  SampleType is something inbetween: it is considered
# public data and shared among unrelated investigations.  But still
# each investigation might come up with a new SampleType and might
# need to create them.
#
# This requires special permissions.  Create a group of SampleType
# writers that have write permissions in SampleType.  We will populate
# this group later.
st_writers = client.createGroup("samplewriter")
client.createRules("CRUD", [ "SampleType" ], st_writers)

# ------------------------------------------------------------
# Create facilities
# ------------------------------------------------------------

facilities = {}
for k in data['facilities'].keys():
    fac = client.new("facility")
    initobj(fac, data['facilities'][k])
    fac.create()
    facilities[k] = fac

# ------------------------------------------------------------
# Create instruments
# ------------------------------------------------------------

instusers = []
for k in data['instruments'].keys():
    inst = client.new("instrument")
    initobj(inst, data['instruments'][k])
    inst.facility = facilities[data['instruments'][k]['facility']]
    inst.create()
    ud = data['users'][data['instruments'][k]['instrumentscientist']]
    instuser = client.createUser(ud['name'], fullName=ud['fullName'],
                                 search=True)
    inst.addInstrumentScientists([instuser])
    instusers.append(instuser)
# As a default rule, instrument scientists are SampleType writers
st_writers.addUsers(instusers)

# ------------------------------------------------------------
# Create InvestigationType, DatasetType, DatafileFormat
# ------------------------------------------------------------

# investigationTypes
investigation_types = []
for k in data['investigation_types'].keys():
    it = client.new("investigationType")
    initobj(it, data['investigation_types'][k])
    it.facility = facilities[data['investigation_types'][k]['facility']]
    investigation_types.append(it)
client.createMany(investigation_types)

# datasetTypes
dataset_types = []
for k in data['dataset_types'].keys():
    dt = client.new("datasetType")
    initobj(dt, data['dataset_types'][k])
    dt.facility = facilities[data['dataset_types'][k]['facility']]
    dataset_types.append(dt)
client.createMany(dataset_types)

# datafileFormats
fileformats = []
for k in data['datafile_formats'].keys():
    ff = client.new("datafileFormat")
    initobj(ff, data['datafile_formats'][k])
    ff.facility = facilities[data['datafile_formats'][k]['facility']]
    fileformats.append(ff)
client.createMany(fileformats)

# applications
applications = []
for k in data['applications'].keys():
    app = client.new("application")
    initobj(app, data['applications'][k])
    app.facility = facilities[data['applications'][k]['facility']]
    applications.append(app)
client.createMany(applications)

# ------------------------------------------------------------
# Access rules for investigations (ICAT 4.4 and newer)
# ------------------------------------------------------------

# ICAT 4.4 introduced InvestigationGroup.  This allows us to setup one
# static set of permissions to access individual investigations rather
# then creating individual rules for each investigation.

if client.apiversion > '4.3.99':

    # Items that are considered to belong to the content of an
    # investigation, where %s represents the investigation itself.
    # The writer group will get CRUD permissions and the reader group
    # R permissions on these items.
    invitems = [ "Sample <-> %s",
                 "Dataset <-> %s",
                 "Datafile <-> Dataset <-> %s",
                 "InvestigationParameter <-> %s",
                 "SampleParameter <-> Sample <-> %s",
                 "DatasetParameter <-> Dataset <-> %s",
                 "DatafileParameter <-> Datafile <-> Dataset <-> %s",
                 "Shift <-> %s",
                 "Keyword <-> %s",
                 "Publication <-> %s", ]

    # set permissions for the writer group
    invcond = ("Investigation <-> InvestigationGroup [role='writer'] "
               "<-> Grouping <-> UserGroup <-> User [name=:user]")
    client.createRules("CRUD", [ s % invcond for s in invitems ])

    # set permissions for the reader group.  Actually, we give read
    # permissions to all groups related to the investigation.
    invcond = ("Investigation <-> InvestigationGroup "
               "<-> Grouping <-> UserGroup <-> User [name=:user]")
    client.createRules("R", [ invcond ])
    client.createRules("R", [ s % invcond for s in invitems ])

    # set permission to grant and to revoke permissions for the owner
    # Would like to add a condition
    # ,
    # but this is too long (have 255 chars max).
    item = ("SELECT tug FROM UserGroup tug "
            "JOIN tug.grouping tg "
            "JOIN tg.investigationGroups tig "
            "JOIN tig.investigation i "
            "JOIN i.investigationGroups uig "
            "JOIN uig.grouping ug "
            "JOIN ug.userGroups uug JOIN uug.user u "
            "WHERE (tig.role = 'writer' OR tig.role = 'reader') "
            "AND uig.role = 'owner' AND u.name = :user")
    client.createRules("CRUD", [ item ])
    item = ("SELECT tg FROM Grouping tg "
            "JOIN tg.investigationGroups tig "
            "JOIN tig.investigation i "
            "JOIN i.investigationGroups uig "
            "JOIN uig.grouping ug "
            "JOIN ug.userGroups uug JOIN uug.user u "
            "WHERE uig.role = 'owner' AND u.name = :user")
    client.createRules("R", [ item ])

# ------------------------------------------------------------
# Public steps
# ------------------------------------------------------------

if client.apiversion > '4.2.99':
    # Compatibility ICAT 4.3.0 vs. ICAT 4.3.1 and later: name of the
    # parameters relation in DataCollection.
    if client.apiversion < '4.3.1':
        datacolparamname = 'dataCollectionParameters'
    else:
        datacolparamname = 'parameters'
    pubsteps = [ ("DataCollection", "dataCollectionDatafiles"),
                 ("DataCollection", "dataCollectionDatasets"),
                 ("DataCollection", datacolparamname),
                 ("Datafile", "dataset"),
                 ("Datafile", "parameters"),
                 ("Dataset", "datafiles"),
                 ("Dataset", "investigation"),
                 ("Dataset", "parameters"),
                 ("Dataset", "sample"),
                 ("Grouping", "userGroups"),
                 ("Instrument", "instrumentScientists"),
                 ("Investigation", "datasets"),
                 ("Investigation", "investigationInstruments"),
                 ("Investigation", "investigationUsers"),
                 ("Investigation", "keywords"),
                 ("Investigation", "parameters"),
                 ("Investigation", "publications"),
                 ("Investigation", "samples"),
                 ("Investigation", "shifts"),
                 ("Job", "inputDataCollection"),
                 ("Job", "outputDataCollection"),
                 ("Sample", "investigation"),
                 ("Sample", "parameters"),
                 ("Study", "studyInvestigations"), ]
    if client.apiversion > '4.3.99':
        pubsteps += [ ("Investigation", "investigationGroups"),
                      ("InvestigationGroup", "grouping"), ]
    objs = [ client.new("publicStep", origin=origin, field=field)
             for (origin, field) in pubsteps ]
    client.createMany(objs)

<commit_msg>Comment.<commit_after>#! /usr/bin/python
#
# Basic initialization of a test ICAT.  Setup all the stuff that must
# be there before useroffice may add investigations.
# - setup basic permissions
# - create a facility
# - create an instrument
# - create InvestigationType, DatasetType, DatafileParameter
#
# This script must be run by the ICAT root user as configured in the
# rootUserNames property.
#

import icat
import icat.config
import sys
import logging
import yaml

logging.basicConfig(level=logging.INFO)
#logging.getLogger('suds.client').setLevel(logging.DEBUG)

config = icat.config.Config()
config.add_variable('datafile', ("datafile",),
                    dict(metavar="inputdata.yaml",
                         help="name of the input datafile"))
conf = config.getconfig()

client = icat.Client(conf.url, **conf.client_kwargs)
client.login(conf.auth, conf.credentials)

# ------------------------------------------------------------
# Helper functions
# ------------------------------------------------------------

def initobj(obj, attrs):
    """Initialize an entity object from a dict of attributes."""
    for a in obj.InstAttr:
        if a != 'id' and a in attrs:
            setattr(obj, a, attrs[a])

# ------------------------------------------------------------
# Read input data
# ------------------------------------------------------------

if conf.datafile == "-":
    f = sys.stdin
else:
    f = open(conf.datafile, 'r')
data = yaml.load(f)
f.close()

# ------------------------------------------------------------
# Setup some basic permissions
# ------------------------------------------------------------

# The name of the Group entity type has been changed to Grouping in
# ICAT 4.3.
if client.apiversion < '4.2.99':
    groupname = "Group"
else:
    groupname = "Grouping"

# List of all tables in the schema
alltables = set(client.getEntityNames())

# Public tables that may be read by anybody.  Basically anything thats
# static and not related to any particular investigation.
pubtables = { "Application", "DatafileFormat", "DatasetType",
              "Facility", "FacilityCycle", "Instrument",
              "InvestigationType", "ParameterType",
              "PermissibleStringValue", "SampleType", "User", }

# Objects that useroffice might need to create.  Basically anything
# related to a particular investigation as a whole, but not to
# individual items created during the investigation (Datafiles and
# Datasets).  Plus FacilityCycle and InstrumentScientist.
#
# With ICAT 4.4 and newer, access permisions on investigations are
# based InvestigationGroup.  In this case, we have a fixed set of
# static rules.  With older ICAT versions, we need per investigation
# rules and thus useroffice need permission to create them.
uotables = { "FacilityCycle", groupname, "InstrumentScientist",
             "Investigation", "InvestigationParameter",
             "InvestigationUser", "Keyword", "Publication", "Shift",
             "Study", "StudyInvestigation", "User", "UserGroup", }
if client.apiversion > '4.2.99':
    uotables.add("InvestigationInstrument")
if client.apiversion > '4.3.99':
    uotables.add("InvestigationGroup")
if client.apiversion < '4.3.99':
    uotables.add("Rule")

# Permit root to read and write everything.  This gives ourselves the
# permissions to continue this script further on.  With ICAT 4.4 and
# newer this is not needed, as the root user already has CRUD
# permission on everything built in.
root = client.createUser("root", fullName="Root")
if client.apiversion < '4.3.99':
    rootgroup = client.createGroup("root", [ root ])
    client.createRules("CRUD", alltables, rootgroup)

# Grant public read permission to some basic tables.  Note that the
# created rules do not refer to any group.  That means they will apply
# to anybody.
client.createRules("R", pubtables)

# Special rule: each user gets the permission to see the groups he
# is in.
client.createRules("R", [ "%s <-> UserGroup <-> User[name=:user]" % groupname ])

# Add a sepcial user to be configured as reader in ids.server.  This
# user needs at least permission to read all datasets, datafiles,
# investigations and facilities.  But well, then we can make live
# simple by giving him read all permissions.
idsreader = client.createUser("idsreader", fullName="IDS reader")
rallgroup = client.createGroup("rall", [ idsreader ])
client.createRules("R", alltables - pubtables - {"Log"}, rallgroup)

# Setup permissions for useroffice.  They need to create
# Investigations and to setup access permissions for them.  Note that
# this requires the useroffice to have write permission to authz
# tables which basically gives useroffice root power.
useroffice = client.createUser("useroffice", fullName="User Office")
uogroup = client.createGroup("useroffice", [ useroffice ])
client.createRules("CRUD", uotables, uogroup)

# Special case SampleType: most tables in the data base are either
# specific to the site as a whole (most public tables) or specific to
# a particular investigation.  The former contain public data that is
# mostly static, created once at initialization and does not change
# too much later on.  The latter is private and created during each
# investigation.  SampleType is something inbetween: it is considered
# public data and shared among unrelated investigations.  But still
# each investigation might come up with a new SampleType and might
# need to create them.
#
# This requires special permissions.  Create a group of SampleType
# writers that have write permissions in SampleType.  We will populate
# this group later.
st_writers = client.createGroup("samplewriter")
client.createRules("CRUD", [ "SampleType" ], st_writers)

# ------------------------------------------------------------
# Create facilities
# ------------------------------------------------------------

facilities = {}
for k in data['facilities'].keys():
    fac = client.new("facility")
    initobj(fac, data['facilities'][k])
    fac.create()
    facilities[k] = fac

# ------------------------------------------------------------
# Create instruments
# ------------------------------------------------------------

instusers = []
for k in data['instruments'].keys():
    inst = client.new("instrument")
    initobj(inst, data['instruments'][k])
    inst.facility = facilities[data['instruments'][k]['facility']]
    inst.create()
    ud = data['users'][data['instruments'][k]['instrumentscientist']]
    instuser = client.createUser(ud['name'], fullName=ud['fullName'],
                                 search=True)
    inst.addInstrumentScientists([instuser])
    instusers.append(instuser)
# As a default rule, instrument scientists are SampleType writers
st_writers.addUsers(instusers)

# ------------------------------------------------------------
# Create InvestigationType, DatasetType, DatafileFormat
# ------------------------------------------------------------

# investigationTypes
investigation_types = []
for k in data['investigation_types'].keys():
    it = client.new("investigationType")
    initobj(it, data['investigation_types'][k])
    it.facility = facilities[data['investigation_types'][k]['facility']]
    investigation_types.append(it)
client.createMany(investigation_types)

# datasetTypes
dataset_types = []
for k in data['dataset_types'].keys():
    dt = client.new("datasetType")
    initobj(dt, data['dataset_types'][k])
    dt.facility = facilities[data['dataset_types'][k]['facility']]
    dataset_types.append(dt)
client.createMany(dataset_types)

# datafileFormats
fileformats = []
for k in data['datafile_formats'].keys():
    ff = client.new("datafileFormat")
    initobj(ff, data['datafile_formats'][k])
    ff.facility = facilities[data['datafile_formats'][k]['facility']]
    fileformats.append(ff)
client.createMany(fileformats)

# applications
applications = []
for k in data['applications'].keys():
    app = client.new("application")
    initobj(app, data['applications'][k])
    app.facility = facilities[data['applications'][k]['facility']]
    applications.append(app)
client.createMany(applications)

# ------------------------------------------------------------
# Access rules for investigations (ICAT 4.4 and newer)
# ------------------------------------------------------------

# ICAT 4.4 introduced InvestigationGroup.  This allows us to setup one
# static set of permissions to access individual investigations rather
# then creating individual rules for each investigation.

if client.apiversion > '4.3.99':

    # Items that are considered to belong to the content of an
    # investigation, where %s represents the investigation itself.
    # The writer group will get CRUD permissions and the reader group
    # R permissions on these items.
    invitems = [ "Sample <-> %s",
                 "Dataset <-> %s",
                 "Datafile <-> Dataset <-> %s",
                 "InvestigationParameter <-> %s",
                 "SampleParameter <-> Sample <-> %s",
                 "DatasetParameter <-> Dataset <-> %s",
                 "DatafileParameter <-> Datafile <-> Dataset <-> %s",
                 "Shift <-> %s",
                 "Keyword <-> %s",
                 "Publication <-> %s", ]

    # set permissions for the writer group
    invcond = ("Investigation <-> InvestigationGroup [role='writer'] "
               "<-> Grouping <-> UserGroup <-> User [name=:user]")
    client.createRules("CRUD", [ s % invcond for s in invitems ])

    # set permissions for the reader group.  Actually, we give read
    # permissions to all groups related to the investigation.
    invcond = ("Investigation <-> InvestigationGroup "
               "<-> Grouping <-> UserGroup <-> User [name=:user]")
    client.createRules("R", [ invcond ])
    client.createRules("R", [ s % invcond for s in invitems ])

    # set permission to grant and to revoke permissions for the owner
    # Would like to add a condition
    # ,
    # but this is too long (have 255 chars max).
    item = ("SELECT tug FROM UserGroup tug "
            "JOIN tug.grouping tg "
            "JOIN tg.investigationGroups tig "
            "JOIN tig.investigation i "
            "JOIN i.investigationGroups uig "
            "JOIN uig.grouping ug "
            "JOIN ug.userGroups uug JOIN uug.user u "
            "WHERE (tig.role = 'writer' OR tig.role = 'reader') "
            "AND uig.role = 'owner' AND u.name = :user")
    client.createRules("CRUD", [ item ])
    item = ("SELECT tg FROM Grouping tg "
            "JOIN tg.investigationGroups tig "
            "JOIN tig.investigation i "
            "JOIN i.investigationGroups uig "
            "JOIN uig.grouping ug "
            "JOIN ug.userGroups uug JOIN uug.user u "
            "WHERE uig.role = 'owner' AND u.name = :user")
    client.createRules("R", [ item ])

# ------------------------------------------------------------
# Public steps
# ------------------------------------------------------------

if client.apiversion > '4.2.99':
    # Compatibility ICAT 4.3.0 vs. ICAT 4.3.1 and later: name of the
    # parameters relation in DataCollection.
    if client.apiversion < '4.3.1':
        datacolparamname = 'dataCollectionParameters'
    else:
        datacolparamname = 'parameters'
    pubsteps = [ ("DataCollection", "dataCollectionDatafiles"),
                 ("DataCollection", "dataCollectionDatasets"),
                 ("DataCollection", datacolparamname),
                 ("Datafile", "dataset"),
                 ("Datafile", "parameters"),
                 ("Dataset", "datafiles"),
                 ("Dataset", "investigation"),
                 ("Dataset", "parameters"),
                 ("Dataset", "sample"),
                 ("Grouping", "userGroups"),
                 ("Instrument", "instrumentScientists"),
                 ("Investigation", "datasets"),
                 ("Investigation", "investigationInstruments"),
                 ("Investigation", "investigationUsers"),
                 ("Investigation", "keywords"),
                 ("Investigation", "parameters"),
                 ("Investigation", "publications"),
                 ("Investigation", "samples"),
                 ("Investigation", "shifts"),
                 ("Job", "inputDataCollection"),
                 ("Job", "outputDataCollection"),
                 ("Sample", "investigation"),
                 ("Sample", "parameters"),
                 ("Study", "studyInvestigations"), ]
    if client.apiversion > '4.3.99':
        pubsteps += [ ("Investigation", "investigationGroups"),
                      ("InvestigationGroup", "grouping"), ]
    objs = [ client.new("publicStep", origin=origin, field=field)
             for (origin, field) in pubsteps ]
    client.createMany(objs)

<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
An upload script for Google Music using https://github.com/simon-weber/Unofficial-Google-Music-API.
You may contact the author (thebigmunch) in #gmusicapi on irc.freenode.net or by e-mail at munchicus+gmusicapi@gmail.com.
"""

from  __future__ import print_function, unicode_literals
import argparse
import mutagen
import os
import re
import sys
from gmusicapi import CallFailure
from gmusicapi.clients import Musicmanager, OAUTH_FILEPATH

formats = ('.mp3', '.flac', '.ogg', '.m4a')

# Pre-compile regex for clean_tag function.
track_slash = re.compile('\/\s*\d+')
lead_zeros = re.compile('^0+([0-9]+)')
track_dots = re.compile('^\d+\.+')
non_word = re.compile('[^\w\s]')
spaces = re.compile('\s+')
lead_space = re.compile('^\s+')
trail_space = re.compile('\s+$')
the = re.compile('^the\s+', re.I)

# Parse command line for arguments.
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('-c', '--cred', default='oauth', help='Specify oauth credential file name to use/create\n(Default: "oauth" -> ' + OAUTH_FILEPATH + ')')
parser.add_argument('-l', '--log', action='store_true', default=False, help='Enable gmusicapi logging')
parser.add_argument('-m', '--match', action='store_true', default=False, help='Enable scan and match')
parser.add_argument('-d', '--dry-run', action='store_true', default=False, help='Output list of songs that would be uploaded')
parser.add_argument('-q', '--quiet', action='store_true', default=False, help='Don\'t output status messages\n-l,--log will display gmusicapi warnings\n-d,--dry-run will display song list')
parser.add_argument('input', nargs='*', default='.', help='Files, directories, or glob patterns to upload\nDefaults to current directory if none given')
opts = parser.parse_args()

MM = Musicmanager(debug_logging=opts.log)

_print = print if not opts.quiet else lambda *a, **k: None

def do_auth():
	"""
	Authenticates the MM client.
	"""

	attempts = 0

	oauth_file = os.path.join(os.path.dirname(OAUTH_FILEPATH), opts.cred + '.cred')

	# Attempt to login. Perform oauth only when necessary.
	while attempts < 3:
		if MM.login(oauth_credentials=oauth_file):
			break
		MM.perform_oauth(storage_filepath=oauth_file)
		attempts += 1

	if not MM.is_authenticated():
		_print("Sorry, login failed.")
		return

	_print("Successfully logged in.\n")

def clean_tag(tag):
	"""
	Cleans up metadata tags to improve matching accuracy.
	"""

	tag = unicode(tag)  # Convert tag to unicode.
	tag = tag.lower()  # Convert to lower case.

	tag = track_slash.sub('', tag)  # Remove "/<totaltracks>" from track number.
	tag = lead_zeros.sub(r'\1', tag)  # Remove leading zero(s) from track number.
	tag = track_dots.sub('', tag)  # Remove dots from track number.
	tag = non_word.sub('', tag)  # Remove any non-words.
	tag = spaces.sub(' ', tag)  # Reduce multiple spaces to a single space.
	tag = lead_space.sub('', tag)  # Remove leading space.
	tag = trail_space.sub('', tag)  # Remove trailing space.
	tag = the.sub('', tag)  # Remove leading "the".

	return tag

def do_upload(files, total):
	"""
	Uploads the files and outputs the upload response with a counter.
	"""

	filenum = 0
	errors = {}
	pad = len(str(total))

	for file in files:
		filenum += 1

		try:
			uploaded, matched, not_uploaded = MM.upload(file, transcode_quality="320k", enable_matching=opts.match)
		except CallFailure as e:
			_print("({num:>{pad}}/{total}) Failed to upload  {file} | {error}".format(num=filenum, total=total, file=file, error=e, pad=pad).encode('utf8'))
			errors[file] = e
		else:
			if uploaded:
				_print("({num:>{pad}}/{total}) Successfully uploaded  {file}".format(num=filenum, total=total, file=file, pad=pad).encode('utf8'))
			elif matched:
				_print("({num:>{pad}}/{total}) Successfully scanned and matched  {file}".format(num=filenum, total=total, file=file, pad=pad).encode('utf8'))
			else:
				check_strings = ["ALREADY_EXISTS", "this song is already uploaded"]
				if any(check_string in not_uploaded[file] for check_string in check_strings):
					response = "ALREADY EXISTS"
				else:
					response = not_uploaded[file]
				_print("({num:>{pad}}/{total}) Failed to upload  {file} | {response}".format(num=filenum, total=total, file=file, response=response, pad=pad).encode('utf8'))

	if errors:
		_print("\n\nThe following errors occurred:\n")
		for file, e in errors.iteritems():
			_print("{file} | {error}".format(file=file, error=e).encode('utf8'))
		_print("\nThese files may need to be synced again.\n")

def filter_tags(song):
	"""
	Filters out a missing artist, album, title, or track tag to improve matching accuracy.
	"""

	# Replace track numbers with 0 if no tag exists.
	if song.get('id'):
		if not song.get('track_number'):
			song['track_number'] = '0'
	else:
		if not song.get('tracknumber'):
			song['tracknumber'] = '0'

	# Need both tracknumber (mutagen) and track_number (Google Music) here.
	return [song[tag] for tag in ['artist', 'album', 'title', 'tracknumber', 'track_number'] if song.get(tag)]

def get_file_list():
	"""
	Creates a list of supported files from user input(s).
	"""

	files = []

	for i in opts.input:
		if os.path.isfile(i) and i.lower().endswith(formats):
			files.append(i)

		if os.path.isdir(i):
			for dirpath, dirnames, filenames in os.walk(i):
				for filename in filenames:
					if filename.lower().endswith(formats):
						file = os.path.join(dirpath, filename)
						files.append(file)

	return files

def get_google_songs():
	"""
	Load song list from Google Music library.
	"""

	_print("Loading Google Music songs...")

	google_songs = {}

	for song in MM.get_all_songs():
		tags = []

		for tag in filter_tags(song):
			tags.append(clean_tag(tag))
			key = '|'.join(tags)

		google_songs[key] = song

	_print("Loaded {0} Google Music songs\n".format(len(google_songs)))

	return google_songs

def get_local_songs():
	"""
	Load song list from local system.
	"""

	_print("Loading local songs...")

	local_songs = {}

	for file in get_file_list():
		song = mutagen.File(file, easy=True)

		tags = []

		for tag in filter_tags(song):
			tags.append(clean_tag(tag[0]))
			key = '|'.join(tags)

		local_songs[key] = file

	_print("Loaded {0} local songs\n".format(len(local_songs)))

	return local_songs

def main():
	do_auth()

	local_songs = get_local_songs()
	google_songs = get_google_songs()

	_print("Scanning for missing songs...")

	files = []

	# Compare local and Google Music library lists.
	for key, song in local_songs.iteritems():
		if key not in google_songs:
			files.append(song)

	# Upload any local songs not in your Google Music library.
	if files:
		# Sort the list for sensible output.
		files.sort()
		total = len(files)

		if opts.dry_run:
			_print("Found {0} songs\n".format(total))
			for file in files:
				print(file)
		else:
			_print("Uploading {0} songs to Google Music\n".format(total))
			do_upload(files, total)
	else:
		_print("No songs to upload")

	# Log out MM session when finished.
	MM.logout()
	_print("\nAll done!")

if __name__ == '__main__':
	main()
<commit_msg>Add check for gmusicapi 3.0.0<commit_after>#!/usr/bin/env python

"""
An upload script for Google Music using https://github.com/simon-weber/Unofficial-Google-Music-API.
You may contact the author (thebigmunch) in #gmusicapi on irc.freenode.net or by e-mail at munchicus+gmusicapi@gmail.com.
"""

from  __future__ import print_function, unicode_literals
import argparse
import mutagen
import os
import re
import sys
from gmusicapi import CallFailure
from gmusicapi.clients import Musicmanager, OAUTH_FILEPATH

formats = ('.mp3', '.flac', '.ogg', '.m4a')

# Pre-compile regex for clean_tag function.
track_slash = re.compile('\/\s*\d+')
lead_zeros = re.compile('^0+([0-9]+)')
track_dots = re.compile('^\d+\.+')
non_word = re.compile('[^\w\s]')
spaces = re.compile('\s+')
lead_space = re.compile('^\s+')
trail_space = re.compile('\s+$')
the = re.compile('^the\s+', re.I)

# Parse command line for arguments.
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('-c', '--cred', default='oauth', help='Specify oauth credential file name to use/create\n(Default: "oauth" -> ' + OAUTH_FILEPATH + ')')
parser.add_argument('-l', '--log', action='store_true', default=False, help='Enable gmusicapi logging')
parser.add_argument('-m', '--match', action='store_true', default=False, help='Enable scan and match')
parser.add_argument('-d', '--dry-run', action='store_true', default=False, help='Output list of songs that would be uploaded')
parser.add_argument('-q', '--quiet', action='store_true', default=False, help='Don\'t output status messages\n-l,--log will display gmusicapi warnings\n-d,--dry-run will display song list')
parser.add_argument('input', nargs='*', default='.', help='Files, directories, or glob patterns to upload\nDefaults to current directory if none given')
opts = parser.parse_args()

MM = Musicmanager(debug_logging=opts.log)

_print = print if not opts.quiet else lambda *a, **k: None

def do_auth():
	"""
	Authenticates the MM client.
	"""

	attempts = 0

	oauth_file = os.path.join(os.path.dirname(OAUTH_FILEPATH), opts.cred + '.cred')

	# Attempt to login. Perform oauth only when necessary.
	while attempts < 3:
		if MM.login(oauth_credentials=oauth_file):
			break
		MM.perform_oauth(storage_filepath=oauth_file)
		attempts += 1

	if not MM.is_authenticated():
		_print("Sorry, login failed.")
		return

	_print("Successfully logged in.\n")

def clean_tag(tag):
	"""
	Cleans up metadata tags to improve matching accuracy.
	"""

	tag = unicode(tag)  # Convert tag to unicode.
	tag = tag.lower()  # Convert to lower case.

	tag = track_slash.sub('', tag)  # Remove "/<totaltracks>" from track number.
	tag = lead_zeros.sub(r'\1', tag)  # Remove leading zero(s) from track number.
	tag = track_dots.sub('', tag)  # Remove dots from track number.
	tag = non_word.sub('', tag)  # Remove any non-words.
	tag = spaces.sub(' ', tag)  # Reduce multiple spaces to a single space.
	tag = lead_space.sub('', tag)  # Remove leading space.
	tag = trail_space.sub('', tag)  # Remove trailing space.
	tag = the.sub('', tag)  # Remove leading "the".

	return tag

def do_upload(files, total):
	"""
	Uploads the files and outputs the upload response with a counter.
	"""

	filenum = 0
	errors = {}
	pad = len(str(total))

	for file in files:
		filenum += 1

		try:
			uploaded, matched, not_uploaded = MM.upload(file, transcode_quality="320k", enable_matching=opts.match)
		except CallFailure as e:
			_print("({num:>{pad}}/{total}) Failed to upload  {file} | {error}".format(num=filenum, total=total, file=file, error=e, pad=pad).encode('utf8'))
			errors[file] = e
		else:
			if uploaded:
				_print("({num:>{pad}}/{total}) Successfully uploaded  {file}".format(num=filenum, total=total, file=file, pad=pad).encode('utf8'))
			elif matched:
				_print("({num:>{pad}}/{total}) Successfully scanned and matched  {file}".format(num=filenum, total=total, file=file, pad=pad).encode('utf8'))
			else:
				check_strings = ["ALREADY_EXISTS", "this song is already uploaded"]
				if any(check_string in not_uploaded[file] for check_string in check_strings):
					response = "ALREADY EXISTS"
				else:
					response = not_uploaded[file]
				_print("({num:>{pad}}/{total}) Failed to upload  {file} | {response}".format(num=filenum, total=total, file=file, response=response, pad=pad).encode('utf8'))

	if errors:
		_print("\n\nThe following errors occurred:\n")
		for file, e in errors.iteritems():
			_print("{file} | {error}".format(file=file, error=e).encode('utf8'))
		_print("\nThese files may need to be synced again.\n")

def filter_tags(song):
	"""
	Filters out a missing artist, album, title, or track tag to improve matching accuracy.
	"""

	# Replace track numbers with 0 if no tag exists.
	if song.get('id'):
		if not song.get('track_number'):
			song['track_number'] = '0'
	else:
		if not song.get('tracknumber'):
			song['tracknumber'] = '0'

	# Need both tracknumber (mutagen) and track_number (Google Music) here.
	return [song[tag] for tag in ['artist', 'album', 'title', 'tracknumber', 'track_number'] if song.get(tag)]

def get_file_list():
	"""
	Creates a list of supported files from user input(s).
	"""

	files = []

	for i in opts.input:
		if os.path.isfile(i) and i.lower().endswith(formats):
			files.append(i)

		if os.path.isdir(i):
			for dirpath, dirnames, filenames in os.walk(i):
				for filename in filenames:
					if filename.lower().endswith(formats):
						file = os.path.join(dirpath, filename)
						files.append(file)

	return files

def get_google_songs():
	"""
	Load song list from Google Music library.
	"""

	_print("Loading Google Music songs...")

	google_songs = {}

	try:
		songs = MM.get_all_songs()
	except:
		songs = MM.get_uploaded_songs()

	for song in songs:
		tags = []

		for tag in filter_tags(song):
			tags.append(clean_tag(tag))
			key = '|'.join(tags)

		google_songs[key] = song

	_print("Loaded {0} Google Music songs\n".format(len(google_songs)))

	return google_songs

def get_local_songs():
	"""
	Load song list from local system.
	"""

	_print("Loading local songs...")

	local_songs = {}

	for file in get_file_list():
		song = mutagen.File(file, easy=True)

		tags = []

		for tag in filter_tags(song):
			tags.append(clean_tag(tag[0]))
			key = '|'.join(tags)

		local_songs[key] = file

	_print("Loaded {0} local songs\n".format(len(local_songs)))

	return local_songs

def main():
	do_auth()

	local_songs = get_local_songs()
	google_songs = get_google_songs()

	_print("Scanning for missing songs...")

	files = []

	# Compare local and Google Music library lists.
	for key, song in local_songs.iteritems():
		if key not in google_songs:
			files.append(song)

	# Upload any local songs not in your Google Music library.
	if files:
		# Sort the list for sensible output.
		files.sort()
		total = len(files)

		if opts.dry_run:
			_print("Found {0} songs\n".format(total))
			for file in files:
				print(file)
		else:
			_print("Uploading {0} songs to Google Music\n".format(total))
			do_upload(files, total)
	else:
		_print("No songs to upload")

	# Log out MM session when finished.
	MM.logout()
	_print("\nAll done!")

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before># License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

{
    "name": "Romania - Account Period Closing",
    "category": "Localization",
    "summary": "Romania - Account Period Closing",
    "depends": ["account", "date_range", "l10n_ro"],
    "data": [
        "views/account_period_close_view.xml",
        "wizards/wizard_account_period_closing_view.xml",
        "security/account_security.xml",
        "security/ir.model.access.csv",
    ],
    "license": "AGPL-3",
    "version": "14.0.1.1.0",
    "author": "NextERP Romania,"
    "Forest and Biomass Romania,"
    "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/l10n-romania",
    "installable": True,
    "development_status": "Mature",
    "external_dependencies": {"python": ["python-dateutil"]},
    "maintainers": ["feketemihai"],
}
<commit_msg>l10n_ro_account_period_close 14.0.1.2.0<commit_after># License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

{
    "name": "Romania - Account Period Closing",
    "category": "Localization",
    "summary": "Romania - Account Period Closing",
    "depends": ["account", "date_range", "l10n_ro"],
    "data": [
        "views/account_period_close_view.xml",
        "wizards/wizard_account_period_closing_view.xml",
        "security/account_security.xml",
        "security/ir.model.access.csv",
    ],
    "license": "AGPL-3",
    "version": "14.0.1.2.0",
    "author": "NextERP Romania,"
    "Forest and Biomass Romania,"
    "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/l10n-romania",
    "installable": True,
    "development_status": "Mature",
    "external_dependencies": {"python": ["python-dateutil"]},
    "maintainers": ["feketemihai"],
}
<|endoftext|>


<commit_before>#!/usr/bin/env python
import subprocess, requests, json, tarfile, os, boto3, argparse
import xml.etree.ElementTree as ET

def make_tarfile(output_filename, source_dir):
    with tarfile.open(output_filename, "w:gz") as tar:
        tar.add(source_dir, arcname=os.path.basename(source_dir))

def backup(config):
    backup_endpoint = "{}/go/api/backups".format(config["host"])
    config_endpoint = "{}/go/api/admin/config.xml".format(config["host"])

    s3_bucket_name = config['bucket']
    prefix = config["prefix"]
    headers = {'Confirm': 'true', 'Accept': 'application/vnd.go.cd.v1+json'}

    r = requests.post(backup_endpoint, auth=(config["username"], config["password"]), headers=headers)
    response_object = json.loads(r.text)

    if not r.status_code == 200:
        print 'Something is wrong'
        print r.text
        exit(2)

    backup_path = response_object['path']  # use this form for immediate fail
    backup_file = "{}/{}.tgz".format(config["backup_path"], os.path.basename(backup_path))

    make_tarfile(backup_file, backup_path)
    os.chmod(backup_file, 0600)
    s3 = boto3.resource('s3')
    bucket = s3.Bucket(s3_bucket_name)
    bucket.upload_file(backup_file, '{}/config/{}'.format(prefix, os.path.basename(backup_file)))

    r = requests.post(config_endpoint, auth=(config["username"], config["password"]), headers=headers)
    artifacts_dir = None
    for node in ET.ElementTree(ET.fromstring(r.text)).getroot().findall('server'):
        artifacts_dir = node.get('artifactsdir')
    if artifacts_dir is None:
        print "Could not find artifacts dir"
        exit(2)
    command = ['service', 'go-server', 'stop']
    subprocess.call(command, shell=False)

    command = ['aws', 's3', 'sync', '{}/pipelines'.format(artifacts_dir),
               's3://{}/go_server/backups/pipelines'.format(s3_bucket_name)]
    subprocess.call(command, shell=False)

    command = ['service', 'go-server', 'start']
    subprocess.call(command, shell=False)

parser = argparse.ArgumentParser()

parser.add_argument("--config-file", help="Config file location", default="/etc/go-ctl.conf")

subparsers = parser.add_subparsers(help='commands')

set_parser = subparsers.add_parser("set", help="Sets config values")
set_parser.add_argument("--key", help="Key")
set_parser.add_argument("--value", help="Value")
set_parser.set_defaults(which='set')

get_parser = subparsers.add_parser("get", help="Get config value associated with key")
get_parser.set_defaults(which='get')
get_parser.add_argument('key', help='an integer for the accumulator')
args = parser.parse_args()

print args
config = {
    "backup_path": "/tmp/go-backups",
    "prefix": "go_server/backups",
    "host": "http://localhost:8153",
    "bucket": "rvbgo-s3bucket-10utfkgtekakz"
}
if os.path.isfile(args.config_file):
    with open(args.config_file) as data_file:
        config = json.load(data_file)

if args.which == "set":
    print "We now set {} = {}".format(args.key, args.value)
    config[args.key] = args.value
    with open(args.config_file, 'w') as outfile:
        json.dump(config, outfile)
    os.chmod(args.config_file, 0600)
elif args.which == "get":
    print "Now we get the value of {}".format(args.key)
    print config.get(args.key)
<commit_msg>Added backup command<commit_after>#!/usr/bin/env python
import subprocess, requests, json, tarfile, os, boto3, argparse
import xml.etree.ElementTree as ET

def make_tarfile(output_filename, source_dir):
    with tarfile.open(output_filename, "w:gz") as tar:
        tar.add(source_dir, arcname=os.path.basename(source_dir))

def backup(config, s3_bucket_name=None, prefix=None):
    backup_endpoint = "{}/go/api/backups".format(config["host"])
    config_endpoint = "{}/go/api/admin/config.xml".format(config["host"])

    if s3_bucket_name is None:
        s3_bucket_name = config['bucket']
    if prefix is None:
        prefix = config["prefix"]
    headers = {'Confirm': 'true', 'Accept': 'application/vnd.go.cd.v1+json'}

    r = requests.post(backup_endpoint, auth=(config["username"], config["password"]), headers=headers)
    response_object = json.loads(r.text)

    if not r.status_code == 200:
        print 'Something is wrong'
        print r.text
        exit(2)

    backup_path = response_object['path']  # use this form for immediate fail
    backup_file = "{}/{}.tgz".format(config["backup_path"], os.path.basename(backup_path))

    make_tarfile(backup_file, backup_path)
    os.chmod(backup_file, 0600)
    s3 = boto3.resource('s3')
    bucket = s3.Bucket(s3_bucket_name)
    bucket.upload_file(backup_file, '{}/config/{}'.format(prefix, os.path.basename(backup_file)))
    os.remove(backup_file)
    r = requests.post(config_endpoint, auth=(config["username"], config["password"]), headers=headers)
    artifacts_dir = None
    for node in ET.ElementTree(ET.fromstring(r.text)).getroot().findall('server'):
        artifacts_dir = node.get('artifactsdir')
    if artifacts_dir is None:
        print "Could not find artifacts dir"
        exit(2)
    command = ['service', 'go-server', 'stop']
    subprocess.call(command, shell=False)

    command = ['aws', 's3', 'sync', '{}/pipelines'.format(artifacts_dir),
               's3://{}/go_server/backups/pipelines'.format(s3_bucket_name)]
    subprocess.call(command, shell=False)

    command = ['service', 'go-server', 'start']
    subprocess.call(command, shell=False)

parser = argparse.ArgumentParser()

parser.add_argument("--config-file", help="Config file location", default="/etc/go-ctl.conf")

subparsers = parser.add_subparsers(help='commands')

set_parser = subparsers.add_parser("set", help="Sets config values")
set_parser.add_argument("--key", help="Key")
set_parser.add_argument("--value", help="Value")
set_parser.set_defaults(which='set')

get_parser = subparsers.add_parser("get", help="Get config value associated with key")
get_parser.set_defaults(which='get')
get_parser.add_argument('key', help='an integer for the accumulator')

backup_parser = subparsers.add_parser("backup", help="Backups the pipelines and config")
backup_parser.add_argument("--bucket", default=None, help="Bucket to use")
backup_parser.add_argument("--prefix", default=None, help="Bucket prefix")
backup_parser.set_defaults(which="backup")

args = parser.parse_args()

print args
config = {
    "backup_path": "/tmp/go-backups",
    "prefix": "go_server/backups",
    "host": "http://localhost:8153",
    "bucket": "rvbgo-s3bucket-10utfkgtekakz"
}
if os.path.isfile(args.config_file):
    with open(args.config_file) as data_file:
        config = json.load(data_file)

if args.which == "set":
    print "We now set {} = {}".format(args.key, args.value)
    config[args.key] = args.value
    with open(args.config_file, 'w') as outfile:
        json.dump(config, outfile)
    os.chmod(args.config_file, 0600)
elif args.which == "get":
    print "Now we get the value of {}".format(args.key)
    print config.get(args.key)
elif args.which == "backup":
    backup(config, s3_bucket_name=args.bucket, prefix=args.prefix)
<|endoftext|>


<commit_before>import subprocess
from genes.apt import commands as apt

def test_install_package(monkeypatch):
    """Test calling apt.install with one package
    :param monkeypatch: This is the py.test monkeypatching for mock.
    """
    def mocked_subprocess_call(*args, **kwargs):
        cmd = " ".join(args[0])
        assert(cmd == 'sudo -E apt-get -y install test')
        env = kwargs.get('env', None)
        assert(env is not None)
        assert(env.get('DEBIAN_FRONTEND', None) == 'noninteractive')
        return 0

    monkeypatch.setattr(subprocess, 'call', mocked_subprocess_call)
    apt.install("test")

def test_install_packages(monkeypatch):
    """Test calling apt install with multiple packages
    :param monkeypatch: This is the py.test monkeypatching for mock.
    """
    def mocked_subprocess_call(*args, **kwargs):
        cmd = " ".join(args[0])
        assert(cmd == 'sudo -E apt-get -y install test1 test2')
        env = kwargs.get('env', None)
        assert(env is not None)
        assert(env.get('DEBIAN_FRONTEND', None) == 'noninteractive')
        return 0

    monkeypatch.setattr(subprocess, 'call', mocked_subprocess_call)
    apt.install("test1", "test2")

def test_install_nothing(monkeypatch):
    """Test calling apt install with no packages (failure)
    :param monkeypatch: This is the py.test monkeypatching for mock.
    """
    def mocked_subprocess_call(*args, **kwargs):
        # This should never be called
        assert not args and not kwargs
        assert False

    monkeypatch.setattr(subprocess, 'call', mocked_subprocess_call)
    apt.install()
<commit_msg>Update test_apt.py<commit_after>import subprocess
from genes.apt import commands as apt

def test_install_package(monkeypatch):
    """Test calling apt.install with one package
    :param monkeypatch: This is the py.test monkeypatching for mock.
    """
    # FIXME: this test should fail but passes
    def mocked_subprocess_call(*args, **kwargs):
        cmd = " ".join(args[0])
        assert(cmd == 'sudo -E apt-get -y install test')
        env = kwargs.get('env', None)
        assert(env is not None)
        assert(env.get('DEBIAN_FRONTEND', None) == 'noninteractive')
        return 0

    monkeypatch.setattr(subprocess, 'call', mocked_subprocess_call)
    apt.install("test")

def test_install_packages(monkeypatch):
    """Test calling apt install with multiple packages
    :param monkeypatch: This is the py.test monkeypatching for mock.
    """
    # FIXME: this test should fail
    def mocked_subprocess_call(*args, **kwargs):
        cmd = " ".join(args[0])
        assert(cmd == 'sudo -E apt-get -y install test1 test2')
        env = kwargs.get('env', None)
        assert(env is not None)
        assert(env.get('DEBIAN_FRONTEND', None) == 'noninteractive')
        return 0

    monkeypatch.setattr(subprocess, 'call', mocked_subprocess_call)
    apt.install("test1", "test2")

def test_install_nothing(monkeypatch):
    """Test calling apt install with no packages (failure)
    :param monkeypatch: This is the py.test monkeypatching for mock.
    """
    def mocked_subprocess_call(*args, **kwargs):
        # This should never be called
        assert not args and not kwargs
        assert False

    monkeypatch.setattr(subprocess, 'call', mocked_subprocess_call)
    apt.install()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
from shipper import Shipper, run, command
import subprocess

# DOCKER_URL = "/var/run/docker.sock"
DOCKER_HOST = "localhost:4243"
DOCKER_URL = "tcp://127.0.0.1:4243"
s = Shipper([DOCKER_HOST])

def pmk(t):
    rootdir = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
    return os.path.join(rootdir, t)

def splitimage(name):
    return name.split(':', 1)

@command
def cleanup_containers():
    try:
      subprocess.check_call("docker -H {} ps -a -q | grep ''").format(DOCKER_URL)
    except subprocess.CalledProcessError, e:
      s.log.debug("No docker containers found, skipping cleanup")
    else:
      cmd = "docker -H={} rm `docker -H {} ps -a -q`".format(DOCKER_URL, DOCKER_URL)
      s.log.debug("About to execute: {}".format(cmd))
      subprocess.check_output(cmd, shell=True)

@command
def cleanup_images():
    try:
      subprocess.check_call("docker -H {} images -q | grep ''").format(DOCKER_URL)
    except subprocess.CalledProcessError, e:
      s.log.debug("No docker images found, skipping cleanup")
    else:
      cmd = "docker -H={} rmi `docker -H {} images -q`".format(DOCKER_URL, DOCKER_URL)
      s.log.debug("About to execute: {}".format(cmd))
      subprocess.check_output(cmd, shell=True)

@command
def precise(stamp):
    s.build(
        tag="precise-with-updates:{}".format(stamp),
        path=pmk("precise"))

@command
def export(image, exportto):
    container = s.run(image, "/bin/bash")[0]
    cmd = "docker -H={} export {} | gzip > {}".format(DOCKER_URL, container.id, exportto)
    s.log.debug("About to execute: {}".format(cmd))
    subprocess.check_output(cmd, shell=True)
    s.stop(container)

@command
def flatten(image):
    repo, tag = splitimage(image)
    container = s.run(image, "/bin/bash")[0]

    cmd = "docker -H={} export {} > /tmp/{}.tar".format(DOCKER_URL, container.id, container.id)
    s.log.debug("About to execute: {}".format(cmd))
    subprocess.check_output(cmd, shell=True)

    s.stop(container)

    cmd = "cat /tmp/{}.tar | docker -H=tcp://127.0.0.1:4243 import - {} {}".format(container.id, repo, tag)
    s.log.debug("About to execute: {}".format(cmd))
    subprocess.check_output(cmd, shell=True)

    cmd = "rm -f /tmp/{}.tar".format(container.id)
    s.log.debug("About to execute: {}".format(cmd))
    subprocess.check_output(cmd, shell=True)

@command
def retag(image, newname):
    repo, tag = splitimage(newname)
    cmd = "docker -H={} tag -f {} {} {}".format(DOCKER_URL, image, repo, tag)
    s.log.debug("About to execute: {}".format(cmd))
    subprocess.check_output(cmd, shell=True)

@command
def push(repo):
    s.log.debug("About to upload: {}".format(repo))
    cmd = "docker -H={} push {}".format(DOCKER_URL, repo)
    s.log.debug("About to execute: {}".format(cmd))
    subprocess.check_output(cmd, shell=True)

run()
<commit_msg>fix(subprocess.check_call): Set shell=True<commit_after>#!/usr/bin/env python

import os
from shipper import Shipper, run, command
import subprocess

# DOCKER_URL = "/var/run/docker.sock"
DOCKER_HOST = "localhost:4243"
DOCKER_URL = "tcp://127.0.0.1:4243"
s = Shipper([DOCKER_HOST])

def pmk(t):
    rootdir = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
    return os.path.join(rootdir, t)

def splitimage(name):
    return name.split(':', 1)

@command
def cleanup_containers():
    try:
      subprocess.check_call("docker -H {} ps -a -q | grep ''", shell=True).format(DOCKER_URL)
    except subprocess.CalledProcessError, e:
      s.log.debug("No docker containers found, skipping cleanup")
    else:
      cmd = "docker -H={} rm `docker -H {} ps -a -q`".format(DOCKER_URL, DOCKER_URL)
      s.log.debug("About to execute: {}".format(cmd))
      subprocess.check_output(cmd, shell=True)

@command
def cleanup_images():
    try:
      subprocess.check_call("docker -H {} images -q | grep ''", shell=True).format(DOCKER_URL)
    except subprocess.CalledProcessError, e:
      s.log.debug("No docker images found, skipping cleanup")
    else:
      cmd = "docker -H={} rmi `docker -H {} images -q`".format(DOCKER_URL, DOCKER_URL)
      s.log.debug("About to execute: {}".format(cmd))
      subprocess.check_output(cmd, shell=True)

@command
def precise(stamp):
    s.build(
        tag="precise-with-updates:{}".format(stamp),
        path=pmk("precise"))

@command
def export(image, exportto):
    container = s.run(image, "/bin/bash")[0]
    cmd = "docker -H={} export {} | gzip > {}".format(DOCKER_URL, container.id, exportto)
    s.log.debug("About to execute: {}".format(cmd))
    subprocess.check_output(cmd, shell=True)
    s.stop(container)

@command
def flatten(image):
    repo, tag = splitimage(image)
    container = s.run(image, "/bin/bash")[0]

    cmd = "docker -H={} export {} > /tmp/{}.tar".format(DOCKER_URL, container.id, container.id)
    s.log.debug("About to execute: {}".format(cmd))
    subprocess.check_output(cmd, shell=True)

    s.stop(container)

    cmd = "cat /tmp/{}.tar | docker -H=tcp://127.0.0.1:4243 import - {} {}".format(container.id, repo, tag)
    s.log.debug("About to execute: {}".format(cmd))
    subprocess.check_output(cmd, shell=True)

    cmd = "rm -f /tmp/{}.tar".format(container.id)
    s.log.debug("About to execute: {}".format(cmd))
    subprocess.check_output(cmd, shell=True)

@command
def retag(image, newname):
    repo, tag = splitimage(newname)
    cmd = "docker -H={} tag -f {} {} {}".format(DOCKER_URL, image, repo, tag)
    s.log.debug("About to execute: {}".format(cmd))
    subprocess.check_output(cmd, shell=True)

@command
def push(repo):
    s.log.debug("About to upload: {}".format(repo))
    cmd = "docker -H={} push {}".format(DOCKER_URL, repo)
    s.log.debug("About to execute: {}".format(cmd))
    subprocess.check_output(cmd, shell=True)

run()
<|endoftext|>


<commit_before>from django.core.management import BaseCommand

class Command(BaseCommand):
    help = 'Assign wholesaler case IDs to child cases.'

    def handle(self, *args, **options):
        pass
<commit_msg>implement load_wholesaler_case_ids<commit_after>from corehq.apps.commtrack.models import SupplyPointCase
from corehq.apps.locations.models import Location
from django.core.management import BaseCommand

class Command(BaseCommand):
    help = 'Assign wholesaler case IDs to child cases.'

    def handle(self, *args, **options):
        for retailer_location in Location.filter_by_type(
                "colalifezambia", "retailer"):
            retailer_case = SupplyPointCase.get_by_location(retailer_location)
            wholesaler_location = retailer_location.parent
            wholesaler_case = SupplyPointCase.get_by_location(
                wholesaler_location
            )
            retailer_case.wholesaler_case_id = wholesaler_case._id
            retailer_case.save()
<|endoftext|>


<commit_before>#!/bin/python

# Copyright (c) 2013, Arvid Norberg
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in
#       the documentation and/or other materials provided with the distribution.
#     * Neither the name of the author nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

# This is meant to be run from the root directory of the repo. It will
# look for the .regression.yml file and expect a regression_tests directory
# with results from test runs previously produced by run_tests.py

import os
import sys
import glob
import json

# TODO: different parsers could be run on output from different actions
# if we would use the xml output in stead of stdout/stderr
def style_output(o):
	ret = ''
	subtle = False
	for l in o.split('\n'):
		l = l.replace('<', '<')
		l = l.replace('>', '>')
		if 'TEST_CHECK' in l or 'TEST_EQUAL_ERROR' in l or l.startswith('EXIT STATUS: ') or \
			' second time limit exceeded' in l or l.startswith('signal: SIG'):
			ret += '%s\n' % l
		elif '**passed**' in l:
			ret += '%s\n' % l
		elif ': error: ' in l or ': fatal error: ' in l or ' : fatal error ' in l or \
			'failed to write output file' in l or ') : error C' in l or \
			' : error LNK' in l or ': undefined reference to ' in l or \
			'jump or move depends on uninitialised value(s)' in l or \
			'Invalid read of size' in l or \
			'Invalid write of size' in l or \
			'Use of uninitialised value of size' in l or \
			'Uninitialised byte(s) found during' in l:
			ret += '%s\n' % l
		elif ': warning: ' in l or ') : warning C' in or l \
			'Uninitialised value was created by a' in l or \
			'bytes after a block of size' in l:
			ret += '%s\n' % l
		elif l == '====== END OUTPUT ======' and not subtle:
			ret += '%s\n' % l
			subtle = True
		else:
			ret += '%s\n' % l
	if subtle: ret += ''
	return ret

def modification_time(file):
	mtime = 0
	try:
		mtime = os.stat(file).st_mtime
	except: pass
	return mtime

def save_log_file(log_name, project_name, branch_name, test_name, timestamp, data):

	if not os.path.exists(os.path.split(log_name)[0]):
		os.mkdir(os.path.split(log_name)[0])

	try:
		# if the log file already exists, and it's newer than
		# the source, no need to re-parse it
		mtime = os.stat(log_name).st_mtime
		if mtime >= timestamp: return
	except: pass

	html = open(log_name, 'w+')
	print >>html, '''<title>%s - %s</title>

%s - %s
''' % (project_name, branch_name, project_name, branch_name)
	print >>html, '
%s

%s
' % \
		(test_name.encode('utf-8'), style_output(data).encode('utf-8'))

	print >>html, ''
	html.close()
	sys.stdout.write('.')
	sys.stdout.flush()

def parse_tests(rev_dir):

	# this contains mappings from platforms to
	# the next layer of dictionaries. The next
	# layer contains a mapping of toolsets to
	# dictionaries the next layer of dictionaries.
	# those dictionaries contain a mapping from
	# feature-sets to the next layer of dictionaries.
	# the next layer contains a mapping from
	# tests to information about those tests, such
	# as whether it passed and the output from the
	# command
	# example:

	# {
	#   darwin: {
	#     clang-4.2.1: {
	#       ipv6=off: {
	#         test_primitives: {
	#           output: ...
	#           status: 1
	#         }
	#       }
	#     }
	#   }
	# }

	platforms = {}

	tests = {}

	for f in glob.glob(os.path.join(rev_dir, '*.json')):
		platform_toolset = os.path.split(f)[1].split('.json')[0].split('#')
		try:
			j = json.loads(open(f, 'rb').read())
			timestamp = os.stat(f).st_mtime
		except:
			print '\nFAILED TO LOAD "%s"\n' %f
			continue

		platform = platform_toolset[0]
		toolset = platform_toolset[1]

		for cfg in j:
			test_name = cfg.split('|')[0]
			features = cfg.split('|')[1]

			if not features in tests:
				tests[features] = set()

			tests[features].add(test_name)

			if not platform in platforms:
				platforms[platform] = {}

			if not toolset in platforms[platform]:
				platforms[platform][toolset] = {}

			if not features in platforms[platform][toolset]:
				platforms[platform][toolset][features] = {}

			platforms[platform][toolset][features][test_name] = j[cfg]
			platforms[platform][toolset][features][test_name]['timestamp'] = timestamp

	return (platforms, tests)

# TODO: remove this dependency by encoding it in the output files
# this script should work from outside of the repo, just having
# access to the shared folder
project_name = 'libtorrent'

# maps branch name to latest rev
revs = {}

os.chdir('regression_tests')

for rev in os.listdir('.'):
	try:
		branch = rev.split('-')[0]
		if branch == 'logs': continue
		r = int(rev.split('-')[1])
		if not branch in revs:
			revs[branch] = r
		else:
			if r > revs[branch]:
				revs[branch] = r
	except:
		print 'ignoring %s' % rev

if revs == {}:
	print 'no test files found'
	sys.exit(1)

print 'latest versions'
for b in revs:
	print '%s\t%d' % (b, revs[b])

for branch_name in revs:

	latest_rev = revs[branch_name]

	html_file = '%s.html' % branch_name

	html = open(html_file, 'w+')

	print >>html, '''<title>regression tests, %s</title>

%s - %s
''' % (project_name, project_name, branch_name)

	print >>html, '
'

	num_printed_revs = 0;
	for r in range(latest_rev, latest_rev - 40, -1):
		sys.stdout.write('.')
		sys.stdout.flush()

		rev_dir = '%s-%d' % (branch_name, r)
		(platforms, tests) = parse_tests(rev_dir)

		if len(tests) + len(platforms) == 0: continue

		print >>html, '
revision %d	' % r

		for f in tests:
			print >>html, '%s	' % (len(tests[f]), len(tests[f])*9 - 5, f)
		print >>html, '
'

		for p in platforms:
			print >>html, '
%s	' % (len(platforms[p]), p)
			idx = 0
			for toolset in platforms[p]:
				if idx > 0: print >>html, '
'
				print >>html, '%s	' % toolset
				for f in platforms[p][toolset]:
					for t in platforms[p][toolset][f]:
						details = platforms[p][toolset][f][t]
						if details['status'] == 0: c = 'passed'
						else: c = 'failed'
						log_name = os.path.join('logs-%s-%d' % (branch_name, r), p + '~' + toolset + '~' + t + '~' + f.replace(' ', '.') + '.html')
						print >>html, '	' % (t, f, c, log_name)
						save_log_file(log_name, project_name, branch_name, '%s - %s' % (t, f), int(details['timestamp']), details['output'])

				print >>html, '
'
				idx += 1
		num_printed_revs += 1
		if num_printed_revs >= 20: break

	print >>html, '
'
	html.close()

	print ''

<commit_msg>fix parse_test_results.py typo<commit_after>#!/bin/python

# Copyright (c) 2013, Arvid Norberg
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in
#       the documentation and/or other materials provided with the distribution.
#     * Neither the name of the author nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

# This is meant to be run from the root directory of the repo. It will
# look for the .regression.yml file and expect a regression_tests directory
# with results from test runs previously produced by run_tests.py

import os
import sys
import glob
import json

# TODO: different parsers could be run on output from different actions
# if we would use the xml output in stead of stdout/stderr
def style_output(o):
	ret = ''
	subtle = False
	for l in o.split('\n'):
		l = l.replace('<', '<')
		l = l.replace('>', '>')
		if 'TEST_CHECK' in l or 'TEST_EQUAL_ERROR' in l or l.startswith('EXIT STATUS: ') or \
			' second time limit exceeded' in l or l.startswith('signal: SIG'):
			ret += '%s\n' % l
		elif '**passed**' in l:
			ret += '%s\n' % l
		elif ': error: ' in l or ': fatal error: ' in l or ' : fatal error ' in l or \
			'failed to write output file' in l or ') : error C' in l or \
			' : error LNK' in l or ': undefined reference to ' in l or \
			'jump or move depends on uninitialised value(s)' in l or \
			'Invalid read of size' in l or \
			'Invalid write of size' in l or \
			'Use of uninitialised value of size' in l or \
			'Uninitialised byte(s) found during' in l:
			ret += '%s\n' % l
		elif ': warning: ' in l or ') : warning C' in l or \
			'Uninitialised value was created by a' in l or \
			'bytes after a block of size' in l:
			ret += '%s\n' % l
		elif l == '====== END OUTPUT ======' and not subtle:
			ret += '%s\n' % l
			subtle = True
		else:
			ret += '%s\n' % l
	if subtle: ret += ''
	return ret

def modification_time(file):
	mtime = 0
	try:
		mtime = os.stat(file).st_mtime
	except: pass
	return mtime

def save_log_file(log_name, project_name, branch_name, test_name, timestamp, data):

	if not os.path.exists(os.path.split(log_name)[0]):
		os.mkdir(os.path.split(log_name)[0])

	try:
		# if the log file already exists, and it's newer than
		# the source, no need to re-parse it
		mtime = os.stat(log_name).st_mtime
		if mtime >= timestamp: return
	except: pass

	html = open(log_name, 'w+')
	print >>html, '''<title>%s - %s</title>

%s - %s
''' % (project_name, branch_name, project_name, branch_name)
	print >>html, '
%s

%s
' % \
		(test_name.encode('utf-8'), style_output(data).encode('utf-8'))

	print >>html, ''
	html.close()
	sys.stdout.write('.')
	sys.stdout.flush()

def parse_tests(rev_dir):

	# this contains mappings from platforms to
	# the next layer of dictionaries. The next
	# layer contains a mapping of toolsets to
	# dictionaries the next layer of dictionaries.
	# those dictionaries contain a mapping from
	# feature-sets to the next layer of dictionaries.
	# the next layer contains a mapping from
	# tests to information about those tests, such
	# as whether it passed and the output from the
	# command
	# example:

	# {
	#   darwin: {
	#     clang-4.2.1: {
	#       ipv6=off: {
	#         test_primitives: {
	#           output: ...
	#           status: 1
	#         }
	#       }
	#     }
	#   }
	# }

	platforms = {}

	tests = {}

	for f in glob.glob(os.path.join(rev_dir, '*.json')):
		platform_toolset = os.path.split(f)[1].split('.json')[0].split('#')
		try:
			j = json.loads(open(f, 'rb').read())
			timestamp = os.stat(f).st_mtime
		except:
			print '\nFAILED TO LOAD "%s"\n' %f
			continue

		platform = platform_toolset[0]
		toolset = platform_toolset[1]

		for cfg in j:
			test_name = cfg.split('|')[0]
			features = cfg.split('|')[1]

			if not features in tests:
				tests[features] = set()

			tests[features].add(test_name)

			if not platform in platforms:
				platforms[platform] = {}

			if not toolset in platforms[platform]:
				platforms[platform][toolset] = {}

			if not features in platforms[platform][toolset]:
				platforms[platform][toolset][features] = {}

			platforms[platform][toolset][features][test_name] = j[cfg]
			platforms[platform][toolset][features][test_name]['timestamp'] = timestamp

	return (platforms, tests)

# TODO: remove this dependency by encoding it in the output files
# this script should work from outside of the repo, just having
# access to the shared folder
project_name = 'libtorrent'

# maps branch name to latest rev
revs = {}

os.chdir('regression_tests')

for rev in os.listdir('.'):
	try:
		branch = rev.split('-')[0]
		if branch == 'logs': continue
		r = int(rev.split('-')[1])
		if not branch in revs:
			revs[branch] = r
		else:
			if r > revs[branch]:
				revs[branch] = r
	except:
		print 'ignoring %s' % rev

if revs == {}:
	print 'no test files found'
	sys.exit(1)

print 'latest versions'
for b in revs:
	print '%s\t%d' % (b, revs[b])

for branch_name in revs:

	latest_rev = revs[branch_name]

	html_file = '%s.html' % branch_name

	html = open(html_file, 'w+')

	print >>html, '''<title>regression tests, %s</title>

%s - %s
''' % (project_name, project_name, branch_name)

	print >>html, '
'

	num_printed_revs = 0;
	for r in range(latest_rev, latest_rev - 40, -1):
		sys.stdout.write('.')
		sys.stdout.flush()

		rev_dir = '%s-%d' % (branch_name, r)
		(platforms, tests) = parse_tests(rev_dir)

		if len(tests) + len(platforms) == 0: continue

		print >>html, '
revision %d	' % r

		for f in tests:
			print >>html, '%s	' % (len(tests[f]), len(tests[f])*9 - 5, f)
		print >>html, '
'

		for p in platforms:
			print >>html, '
%s	' % (len(platforms[p]), p)
			idx = 0
			for toolset in platforms[p]:
				if idx > 0: print >>html, '
'
				print >>html, '%s	' % toolset
				for f in platforms[p][toolset]:
					for t in platforms[p][toolset][f]:
						details = platforms[p][toolset][f][t]
						if details['status'] == 0: c = 'passed'
						else: c = 'failed'
						log_name = os.path.join('logs-%s-%d' % (branch_name, r), p + '~' + toolset + '~' + t + '~' + f.replace(' ', '.') + '.html')
						print >>html, '	' % (t, f, c, log_name)
						save_log_file(log_name, project_name, branch_name, '%s - %s' % (t, f), int(details['timestamp']), details['output'])

				print >>html, '
'
				idx += 1
		num_printed_revs += 1
		if num_printed_revs >= 20: break

	print >>html, '
'
	html.close()

	print ''

<|endoftext|>


<commit_before>"""
Radiation engine and geometry handler for CEA
"""
from __future__ import division
import pandas as pd
import time
from cea.resources.radiation_daysim import settings
from cea.resources.radiation_daysim import daysim_main, geometry_generator
import multiprocessing as mp

import pyliburo.py3dmodel.fetch as fetch
import pyliburo.py2radiance as py2radiance

from geopandas import GeoDataFrame as gpdf

import cea.globalvar
import cea.inputlocator

__author__ = "Paul Neitzel, Kian Wee Chen"
__copyright__ = "Copyright 2016, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Paul Neitzel", "Kian Wee Chen", "Jimeno A. Fonseca"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

def create_radiance_srf(occface, srfname, srfmat, rad):
    bface_pts = fetch.pyptlist_frm_occface(occface)
    py2radiance.RadSurface(srfname, bface_pts, srfmat, rad)

def add_rad_mat(daysim_mat_file, ageometry_table):
    file_path = daysim_mat_file

    with open(file_path, 'w') as write_file:
        #first write the material use for the terrain and surrounding buildings
        string = "void plastic reflectance0.2\n0\n0\n5 0.5360 0.1212 0.0565 0 0"
        write_file.writelines(string + '\n')

        written_mat_name_list = []
        for geo in ageometry_table.index.values:
            mat_name = "wall" + str(ageometry_table['type_wall'][geo])
            if mat_name not in written_mat_name_list:
                mat_value1 = ageometry_table['r_wall'][geo]
                mat_value2 = ageometry_table['g_wall'][geo]
                mat_value3 = ageometry_table['b_wall'][geo]
                mat_value4 = ageometry_table['spec_wall'][geo]
                mat_value5 = ageometry_table['rough_wall'][geo]
                string = "void plastic " + mat_name + "\n0\n0\n5 " + str(mat_value1) + " " + str(mat_value2) + " " + str(mat_value3) \
                         + " " + str(mat_value4) + " " + str(mat_value5)

                write_file.writelines('\n' + string + '\n')

                written_mat_name_list.append(mat_name)

            mat_name = "win" + str(ageometry_table['type_win'][geo])
            if mat_name not in written_mat_name_list:
                mat_value1 = ageometry_table['rtn_win'][geo]
                mat_value2 = ageometry_table['gtn_win'][geo]
                mat_value3 = ageometry_table['btn_win'][geo]

                string = "void glass " + mat_name + "\n0\n0\n3 " + str(mat_value1) + " " + str(mat_value2) + " " + str(mat_value3)
                write_file.writelines('\n' + string + '\n')
                written_mat_name_list.append(mat_name)

            mat_name = "roof" + str(ageometry_table['type_roof'][geo])
            if mat_name not in written_mat_name_list:
                mat_value1 = ageometry_table['r_roof'][geo]
                mat_value2 = ageometry_table['g_roof'][geo]
                mat_value3 = ageometry_table['b_roof'][geo]
                mat_value4 = ageometry_table['spec_roof'][geo]
                mat_value5 = ageometry_table['rough_roof'][geo]

                string = "void plastic " + mat_name + "\n0\n0\n5 " + str(mat_value1) + " " + str(mat_value2) + " " + str(mat_value3) \
                         + " " + str(mat_value4) + " " + str(mat_value5)
                write_file.writelines('\n' + string + '\n')
                written_mat_name_list.append(mat_name)

        write_file.close()

def terrain2radiance(rad, tin_occface_terrain):
    for id, face in enumerate(tin_occface_terrain):
        create_radiance_srf(face, "terrain_srf"+ str(id), "reflectance0.2", rad)

def buildings2radiance(rad, ageometry_table, geometry_3D_zone, geometry_3D_surroundings):

    #translate buildings into radiance surface
    fcnt = 0
    for bcnt, building_surfaces in enumerate(geometry_3D_zone):
        building_name = building_surfaces['name']
        for pypolygon in building_surfaces['windows']:
            create_radiance_srf(pypolygon, "win" + str(bcnt) + str(fcnt),
                                "win" + str(ageometry_table['type_win'][building_name]), rad)
            fcnt+=1
        for pypolygon in building_surfaces['walls']:
            create_radiance_srf(pypolygon, "wall" + str(bcnt) + str(fcnt),
                               "wall" + str(ageometry_table['type_wall'][building_name]), rad)
            fcnt+= 1
        for pypolygon in building_surfaces['roofs']:
            create_radiance_srf(pypolygon, "roof" + str(bcnt) + str(fcnt),
                                "roof" + str(ageometry_table['type_roof'][building_name]), rad)
            fcnt+= 1

    for building_surfaces in geometry_3D_surroundings:
        ## for the surrounding buildings only, walls and roofs
        id = 0
        for pypolygon in building_surfaces['walls']:
            create_radiance_srf(pypolygon, "surroundingbldgs" + str(id), "reflectance0.2" , rad)
            id += 1
        for pypolygon in building_surfaces['roofs']:
            create_radiance_srf(pypolygon, "surroundingbldgs" + str(id), "reflectance0.2", rad)
            id += 1

    return

def reader_surface_properties(locator, input_shp):
    """
    This function returns a dataframe with the emissivity values of walls, roof, and windows
    of every building in the scene
    :param input_shp:
    :return:
    """

    # local variables
    architectural_properties = gpdf.from_file(input_shp).drop('geometry', axis=1)
    surface_database_windows = pd.read_excel(locator.get_envelope_systems(), "WINDOW")
    surface_database_roof = pd.read_excel(locator.get_envelope_systems(), "ROOF")
    surface_database_walls = pd.read_excel(locator.get_envelope_systems(), "WALL")

    # querry data
    df = architectural_properties.merge(surface_database_windows, left_on='type_win', right_on='code')
    df2 = architectural_properties.merge(surface_database_roof, left_on='type_roof', right_on='code')
    df3 = architectural_properties.merge(surface_database_walls, left_on='type_wall', right_on='code')
    fields = ['Name', 'G_win', 'rtn_win', 'gtn_win', 'btn_win', "type_win"]
    fields2 = ['Name', 'r_roof', 'g_roof', 'b_roof', 'spec_roof', 'rough_roof', "type_roof"]
    fields3 = ['Name', 'r_wall', 'g_wall', 'b_wall', 'spec_wall', 'rough_wall', "type_wall"]
    surface_properties = df[fields].merge(df2[fields2], on='Name').merge(df3[fields3], on='Name')

    return surface_properties.set_index('Name').round(decimals=2)

def radiation_multiprocessing(rad, bldg_dict_list, aresults_path, aweatherfile_path):

    # get chunks of buildings to iterate
    simul_params = settings.SIMUL_PARAMS
    rad_params = settings.RAD_PARMS
    # get chunks to iterate and start multiprocessing
    chunks = [bldg_dict_list[i:i +simul_params['n_build_in_chunk']] for i in range(0, len(bldg_dict_list),
                                                                                simul_params['n_build_in_chunk'])]

    # pool = mp.Pool()
    # joblist = []
    # for chunk_n, bldg_dict in enumerate(chunks):
    #     job = pool.apply_async(daysim_main.isolation_daysim,
    #                                 [chunk_n, rad, bldg_dict, aresults_path, rad_params, aweatherfile_path])
    #     joblist.append(job)
    # for i, job in enumerate(joblist):
    #     job.get(240)
    # pool.close()

    processes = []
    for chunk_n, bldg_dict in enumerate(chunks):
        process = mp.Process(target=daysim_main.isolation_daysim, args=(
            chunk_n, rad, bldg_dict, aresults_path, rad_params, aweatherfile_path,))
        process.start()
        processes.append(process)
    for process in processes:
        process.join()

def radiation_singleprocessing(rad, bldg_dict_list, locator, aweatherfile_path):

    # get chunks of buildings to iterate
    simul_params = settings.SIMUL_PARAMS
    chunks = [bldg_dict_list[i:i + simul_params['n_build_in_chunk']] for i in range(0, len(bldg_dict_list),
                                                                                simul_params['n_build_in_chunk'])]

    print bldg_dict_list
    for chunk_n, bldg_dict in enumerate(chunks):
        daysim_main.isolation_daysim(chunk_n, rad, bldg_dict, locator, settings.RAD_PARMS, aweatherfile_path)

def radiation_daysim_main(weatherfile_path, locator, zone_shp, district_shp,
                          input_terrain_raster, architecture_dbf):
    """
        This function makes the calculation of solar insolation in X sensor points for every building in the zone
    of interest. the number of sensor points depends on the size of the grid selected in the SETTINGS.py file and
    are generated automatically.
    :param weatherfile_path:
    :param locator:
    :param zone_shp:
    :param district_shp:
    :param input_terrain_raster:
    :param architecture_dbf:
    :return:
    """

    # import material properties of buildings
    building_surface_properties = reader_surface_properties(locator=locator,
                                                            input_shp=locator.get_building_architecture())

    print "creating 3D geometry and surfaces"
    # create geometrical faces of terrain and buildings
    simplification_params = settings.SIMPLIFICATION_PARAMS
    geometry_terrain, geometry_3D_zone, geometry_3D_surroundings = geometry_generator.geometry_main(zone_shp, district_shp,
                                                                            input_terrain_raster, architecture_dbf,
                                                                                simplification_params)

    print "Sending the scene: geometry and materials to daysim"
    # send materials
    daysim_mat = locator.get_temporary_file('default_materials.rad')
    rad = py2radiance.Rad(daysim_mat, locator.get_temporary_folder())
    add_rad_mat(daysim_mat, building_surface_properties)
    # send terrain
    terrain2radiance(rad, geometry_terrain)
    # send buildings
    buildings2radiance(rad, building_surface_properties, geometry_3D_zone, geometry_3D_surroundings)
    # create scene out of all this
    rad.create_rad_input_file()

    time1 = time.time()
    radiation_singleprocessing(rad, geometry_3D_zone, locator, weatherfile_path)

    if (settings.SIMUL_PARAMS['multiprocessing'] and mp.cpu_count() > 1):
        radiation_multiprocessing(rad, geometry_3D_zone, locator, weatherfile_path)
    else:
        radiation_singleprocessing(rad, geometry_3D_zone, locator, weatherfile_path)

    print "Daysim simulation finished in ", (time.time() - time1) / 60.0, " mins"

def main(locator, weather_path):

    district_shp = locator.get_district()
    zone_shp = locator.get_building_geometry()
    architecture_dbf = locator.get_building_architecture()
    input_terrain_raster = locator.get_terrain()

    # calculate solar radiation
    radiation_daysim_main(weather_path, locator, zone_shp=zone_shp, district_shp=district_shp,
         input_terrain_raster=input_terrain_raster, architecture_dbf=architecture_dbf)

if __name__ == '__main__':

    gv = cea.globalvar.GlobalVariables()
    scenario_path = gv.scenario_reference
    locator = cea.inputlocator.InputLocator(scenario_path=scenario_path)
    weather_path = locator.get_default_weather()
    main(locator=locator, weather_path=weather_path)<commit_msg>erasing print<commit_after>"""
Radiation engine and geometry handler for CEA
"""
from __future__ import division
import pandas as pd
import time
from cea.resources.radiation_daysim import settings
from cea.resources.radiation_daysim import daysim_main, geometry_generator
import multiprocessing as mp

import pyliburo.py3dmodel.fetch as fetch
import pyliburo.py2radiance as py2radiance

from geopandas import GeoDataFrame as gpdf

import cea.globalvar
import cea.inputlocator

__author__ = "Paul Neitzel, Kian Wee Chen"
__copyright__ = "Copyright 2016, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Paul Neitzel", "Kian Wee Chen", "Jimeno A. Fonseca"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

def create_radiance_srf(occface, srfname, srfmat, rad):
    bface_pts = fetch.pyptlist_frm_occface(occface)
    py2radiance.RadSurface(srfname, bface_pts, srfmat, rad)

def add_rad_mat(daysim_mat_file, ageometry_table):
    file_path = daysim_mat_file

    with open(file_path, 'w') as write_file:
        #first write the material use for the terrain and surrounding buildings
        string = "void plastic reflectance0.2\n0\n0\n5 0.5360 0.1212 0.0565 0 0"
        write_file.writelines(string + '\n')

        written_mat_name_list = []
        for geo in ageometry_table.index.values:
            mat_name = "wall" + str(ageometry_table['type_wall'][geo])
            if mat_name not in written_mat_name_list:
                mat_value1 = ageometry_table['r_wall'][geo]
                mat_value2 = ageometry_table['g_wall'][geo]
                mat_value3 = ageometry_table['b_wall'][geo]
                mat_value4 = ageometry_table['spec_wall'][geo]
                mat_value5 = ageometry_table['rough_wall'][geo]
                string = "void plastic " + mat_name + "\n0\n0\n5 " + str(mat_value1) + " " + str(mat_value2) + " " + str(mat_value3) \
                         + " " + str(mat_value4) + " " + str(mat_value5)

                write_file.writelines('\n' + string + '\n')

                written_mat_name_list.append(mat_name)

            mat_name = "win" + str(ageometry_table['type_win'][geo])
            if mat_name not in written_mat_name_list:
                mat_value1 = ageometry_table['rtn_win'][geo]
                mat_value2 = ageometry_table['gtn_win'][geo]
                mat_value3 = ageometry_table['btn_win'][geo]

                string = "void glass " + mat_name + "\n0\n0\n3 " + str(mat_value1) + " " + str(mat_value2) + " " + str(mat_value3)
                write_file.writelines('\n' + string + '\n')
                written_mat_name_list.append(mat_name)

            mat_name = "roof" + str(ageometry_table['type_roof'][geo])
            if mat_name not in written_mat_name_list:
                mat_value1 = ageometry_table['r_roof'][geo]
                mat_value2 = ageometry_table['g_roof'][geo]
                mat_value3 = ageometry_table['b_roof'][geo]
                mat_value4 = ageometry_table['spec_roof'][geo]
                mat_value5 = ageometry_table['rough_roof'][geo]

                string = "void plastic " + mat_name + "\n0\n0\n5 " + str(mat_value1) + " " + str(mat_value2) + " " + str(mat_value3) \
                         + " " + str(mat_value4) + " " + str(mat_value5)
                write_file.writelines('\n' + string + '\n')
                written_mat_name_list.append(mat_name)

        write_file.close()

def terrain2radiance(rad, tin_occface_terrain):
    for id, face in enumerate(tin_occface_terrain):
        create_radiance_srf(face, "terrain_srf"+ str(id), "reflectance0.2", rad)

def buildings2radiance(rad, ageometry_table, geometry_3D_zone, geometry_3D_surroundings):

    #translate buildings into radiance surface
    fcnt = 0
    for bcnt, building_surfaces in enumerate(geometry_3D_zone):
        building_name = building_surfaces['name']
        for pypolygon in building_surfaces['windows']:
            create_radiance_srf(pypolygon, "win" + str(bcnt) + str(fcnt),
                                "win" + str(ageometry_table['type_win'][building_name]), rad)
            fcnt+=1
        for pypolygon in building_surfaces['walls']:
            create_radiance_srf(pypolygon, "wall" + str(bcnt) + str(fcnt),
                               "wall" + str(ageometry_table['type_wall'][building_name]), rad)
            fcnt+= 1
        for pypolygon in building_surfaces['roofs']:
            create_radiance_srf(pypolygon, "roof" + str(bcnt) + str(fcnt),
                                "roof" + str(ageometry_table['type_roof'][building_name]), rad)
            fcnt+= 1

    for building_surfaces in geometry_3D_surroundings:
        ## for the surrounding buildings only, walls and roofs
        id = 0
        for pypolygon in building_surfaces['walls']:
            create_radiance_srf(pypolygon, "surroundingbldgs" + str(id), "reflectance0.2" , rad)
            id += 1
        for pypolygon in building_surfaces['roofs']:
            create_radiance_srf(pypolygon, "surroundingbldgs" + str(id), "reflectance0.2", rad)
            id += 1

    return

def reader_surface_properties(locator, input_shp):
    """
    This function returns a dataframe with the emissivity values of walls, roof, and windows
    of every building in the scene
    :param input_shp:
    :return:
    """

    # local variables
    architectural_properties = gpdf.from_file(input_shp).drop('geometry', axis=1)
    surface_database_windows = pd.read_excel(locator.get_envelope_systems(), "WINDOW")
    surface_database_roof = pd.read_excel(locator.get_envelope_systems(), "ROOF")
    surface_database_walls = pd.read_excel(locator.get_envelope_systems(), "WALL")

    # querry data
    df = architectural_properties.merge(surface_database_windows, left_on='type_win', right_on='code')
    df2 = architectural_properties.merge(surface_database_roof, left_on='type_roof', right_on='code')
    df3 = architectural_properties.merge(surface_database_walls, left_on='type_wall', right_on='code')
    fields = ['Name', 'G_win', 'rtn_win', 'gtn_win', 'btn_win', "type_win"]
    fields2 = ['Name', 'r_roof', 'g_roof', 'b_roof', 'spec_roof', 'rough_roof', "type_roof"]
    fields3 = ['Name', 'r_wall', 'g_wall', 'b_wall', 'spec_wall', 'rough_wall', "type_wall"]
    surface_properties = df[fields].merge(df2[fields2], on='Name').merge(df3[fields3], on='Name')

    return surface_properties.set_index('Name').round(decimals=2)

def radiation_multiprocessing(rad, bldg_dict_list, aresults_path, aweatherfile_path):

    # get chunks of buildings to iterate
    simul_params = settings.SIMUL_PARAMS
    rad_params = settings.RAD_PARMS
    # get chunks to iterate and start multiprocessing
    chunks = [bldg_dict_list[i:i +simul_params['n_build_in_chunk']] for i in range(0, len(bldg_dict_list),
                                                                                simul_params['n_build_in_chunk'])]

    # pool = mp.Pool()
    # joblist = []
    # for chunk_n, bldg_dict in enumerate(chunks):
    #     job = pool.apply_async(daysim_main.isolation_daysim,
    #                                 [chunk_n, rad, bldg_dict, aresults_path, rad_params, aweatherfile_path])
    #     joblist.append(job)
    # for i, job in enumerate(joblist):
    #     job.get(240)
    # pool.close()

    processes = []
    for chunk_n, bldg_dict in enumerate(chunks):
        process = mp.Process(target=daysim_main.isolation_daysim, args=(
            chunk_n, rad, bldg_dict, aresults_path, rad_params, aweatherfile_path,))
        process.start()
        processes.append(process)
    for process in processes:
        process.join()

def radiation_singleprocessing(rad, bldg_dict_list, locator, aweatherfile_path):

    # get chunks of buildings to iterate
    simul_params = settings.SIMUL_PARAMS
    chunks = [bldg_dict_list[i:i + simul_params['n_build_in_chunk']] for i in range(0, len(bldg_dict_list),
                                                                                simul_params['n_build_in_chunk'])]

    for chunk_n, bldg_dict in enumerate(chunks):
        daysim_main.isolation_daysim(chunk_n, rad, bldg_dict, locator, settings.RAD_PARMS, aweatherfile_path)

def radiation_daysim_main(weatherfile_path, locator, zone_shp, district_shp,
                          input_terrain_raster, architecture_dbf):
    """
        This function makes the calculation of solar insolation in X sensor points for every building in the zone
    of interest. the number of sensor points depends on the size of the grid selected in the SETTINGS.py file and
    are generated automatically.
    :param weatherfile_path:
    :param locator:
    :param zone_shp:
    :param district_shp:
    :param input_terrain_raster:
    :param architecture_dbf:
    :return:
    """

    # import material properties of buildings
    building_surface_properties = reader_surface_properties(locator=locator,
                                                            input_shp=locator.get_building_architecture())

    print "creating 3D geometry and surfaces"
    # create geometrical faces of terrain and buildings
    simplification_params = settings.SIMPLIFICATION_PARAMS
    geometry_terrain, geometry_3D_zone, geometry_3D_surroundings = geometry_generator.geometry_main(zone_shp, district_shp,
                                                                            input_terrain_raster, architecture_dbf,
                                                                                simplification_params)

    print "Sending the scene: geometry and materials to daysim"
    # send materials
    daysim_mat = locator.get_temporary_file('default_materials.rad')
    rad = py2radiance.Rad(daysim_mat, locator.get_temporary_folder())
    add_rad_mat(daysim_mat, building_surface_properties)
    # send terrain
    terrain2radiance(rad, geometry_terrain)
    # send buildings
    buildings2radiance(rad, building_surface_properties, geometry_3D_zone, geometry_3D_surroundings)
    # create scene out of all this
    rad.create_rad_input_file()

    time1 = time.time()
    radiation_singleprocessing(rad, geometry_3D_zone, locator, weatherfile_path)

    if (settings.SIMUL_PARAMS['multiprocessing'] and mp.cpu_count() > 1):
        radiation_multiprocessing(rad, geometry_3D_zone, locator, weatherfile_path)
    else:
        radiation_singleprocessing(rad, geometry_3D_zone, locator, weatherfile_path)

    print "Daysim simulation finished in ", (time.time() - time1) / 60.0, " mins"

def main(locator, weather_path):

    district_shp = locator.get_district()
    zone_shp = locator.get_building_geometry()
    architecture_dbf = locator.get_building_architecture()
    input_terrain_raster = locator.get_terrain()

    # calculate solar radiation
    radiation_daysim_main(weather_path, locator, zone_shp=zone_shp, district_shp=district_shp,
         input_terrain_raster=input_terrain_raster, architecture_dbf=architecture_dbf)

if __name__ == '__main__':

    gv = cea.globalvar.GlobalVariables()
    scenario_path = gv.scenario_reference
    locator = cea.inputlocator.InputLocator(scenario_path=scenario_path)
    weather_path = locator.get_default_weather()
    main(locator=locator, weather_path=weather_path)<|endoftext|>