<commit_before>import time
import json
import math
import ephem
import redis
import urllib2
import requests
import ConfigParser
import httplib2
import os
import iso8601
import datetime

from apiclient         import discovery
from oauth2client      import client
from oauth2client      import tools
from oauth2client.file import Storage
from requests.auth     import HTTPBasicAuth
from imapclient        import IMAPClient

class InformationFetcher():

    def _readConfig(self):
        update = False

        if not os.path.isdir(self._homeDir):
            print "Creating homeDir"
            os.makedirs(self._homeDir)

        if os.path.isfile(self._configFileName):
            self._config.read(self._configFileName)
        else:
            print "Config file not found"
            update = True

        if not self._config.has_section('INFORMATION'):
            print "Adding MQTT part"
            update = True
            self._config.add_section("INFORMATION")

        if not self._config.has_option("INFORMATION", "IMAPServer"):
            print "No Server Address"
            update = True
            self._config.set("INFORMATION", "IMAPServer", "<ServerAddress>")

        if not self._config.has_option("INFORMATION", "IMAPUser"):
            print "No Username"
            update = True
            self._config.set("INFORMATION", "IMAPUser", "<IMAPUser>")

        if not self._config.has_option("INFORMATION", "IMAPPasswd"):
            print "No Passwd"
            update = True
            self._config.set("INFORMATION", "IMAPPasswd", "<IMAPPasswd>")

        if not self._config.has_option("INFORMATION", "IMAPFolder"):
            print "No Folder"
            update = True
            self._config.set("INFORMATION", "IMAPFolder", "<IMAPFolder>")

        if not self._config.has_option("INFORMATION", "WUCurrentURL"):
            print "No URL"
            update = True
            self._config.set("INFORMATION", "WUCurrentURL", "<WU Current>")

        if not self._config.has_option("INFORMATION", "WUPredictionURL"):
            print "No URL"
            update = True
            self._config.set("INFORMATION", "WUPredictionURL", "<WU Predict>")

        if not self._config.has_option("INFORMATION", "RadiationURL"):
            print "No RadiationURL"
            update = True
            self._config.set("INFORMATION", "RadiationURL", "https://odlinfo.bfs.de/daten/json/")

        if not self._config.has_option("INFORMATION", "RadiationUser"):
            print "No RadiationURL"
            update = True
            self._config.set("INFORMATION", "RadiationUser", "USERNAME")

        if not self._config.has_option("INFORMATION", "RadiationPasswd"):
            print "No RadiationURL"
            update = True
            self._config.set("INFORMATION", "RadiationPasswd", "USERPASSWD")

        if not self._config.has_option("INFORMATION", "RadiationStation"):
            print "No RadiationURL"
            update = True
            self._config.set("INFORMATION", "RadiationStation", "110000006")

        if not self._config.has_option("INFORMATION", "Latitude"):
            print "No Latitude"
            update = True
            self._config.set("INFORMATION", "Latitude", "52.5148453")

        if not self._config.has_option("INFORMATION", "Logitude"):
            print "No Logitude"
            update = True
            self._config.set("INFORMATION", "Logitude", "13.4389504")

        if not self._config.has_option("INFORMATION", "Elevation"):
            print "No Elevation"
            update = True
            self._config.set("INFORMATION", "Elevation", "42.23")

        if not self._config.has_option("INFORMATION", "NASAurl"):
            print "No NASAurl"
            update = True
            self._config.set("INFORMATION", "NASAurl", "https://api.nasa.gov/")

        if not self._config.has_option("INFORMATION", "NASAKey"):
            print "No NASAKey"
            update = True
            self._config.set("INFORMATION", "NASAKey", "magic")

        if update:
            with open(self._configFileName, 'w') as f:
                self._config.write(f)

    def __init__(self):
        self._homeDir        = os.path.expanduser("~/.sensomatic")
        self._configFileName = self._homeDir + '/config.ini'
        self._config         = ConfigParser.ConfigParser()
        self._readConfig()

    def getEarthRotationTime(self):
        seconds  = time.gmtime().tm_sec         *  1.0
        seconds += time.gmtime().tm_min         * 60.0
        seconds += time.gmtime().tm_hour * 60.0 * 60.0
        fraction = 360.0 * (seconds / (60.0 * 60.0 * 24.0 * 1.0) )

        degrees = int(   fraction                                    )
        minutes = int(  (fraction - degrees) * 60.0                  )
        seconds = int( ((fraction - degrees) * 60.0 - minutes) * 60.0)
        return fraction, degrees, minutes, seconds

    def getSunPosition(self):
        observer = ephem.Observer()
        observer.lon       =       self._config.get("INFORMATION", "Logitude" )
        observer.lat       =       self._config.get("INFORMATION", "Latitude" )
        observer.elevation = float(self._config.get("INFORMATION", "Elevation"))
        sun = ephem.Sun(observer)
        alt = sun.alt * (180 / math.pi)
        az  = sun.az  * (180 / math.pi)
        return alt,az

    def getOutsideLightLevel(self):
        # For now just calculated
        sun, _ = self.getSunPosition()
        if sun > 0:
            return (10.0 / 9.0) * sun
        else:
            return 0

    def getNumEmailMessages(self):
        try:
            server = IMAPClient(self._config.get("INFORMATION","IMAPServer"), use_uid=True, ssl=True)
            server.login(self._config.get("INFORMATION", "IMAPUser"), self._config.get("INFORMATION", "IMAPPasswd"))
            select_info = server .select_folder(self._config.get("INFORMATION", "IMAPFolder"))
            #print json.dumps(select_info, sort_keys=True, indent=4, separators=(',', ': '))
            return select_info['EXISTS']
        except Exception as inst:
            print str(inst)
            return 0

    def getRoomTemp(self, room):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists(room + "/temperature"):
            return float(r.get(room + "/temperature"))
        else:
            return None

    def getRoomDefinedTemp(self, room):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists(room + "/settemperature"):
            return float(r.get(room + "/settemperature"))
        else:
            return None

    def getRoomHumidity(self, room):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists(room + "/humidity"):
            return float(r.get(room + "/humidity"))
        else:
            return None

    def getRoomDefinedHumidity(self, room):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists(room + "/sethumidity"):
            return float(r.get(room + "/sethumidity"))
        else:
            return None

    def getRoomCo2Level(self, room):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists(room + "/co2"):
            return int(r.get(room + "/co2"))
        else:
            return None

    def getPlantSoilLevel(self):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists("ansiroom/bed/plant"):
            return r.get("ansiroom/bed/plant")
        else:
            return None

    def getOutdoor(self):
        '''
            temp, hum, feelslike,  ..... 7
        '''
        try:
            f = urllib2.urlopen(self._config.get("INFORMATION", "WUCurrentURL"))
            json_string = f.read()
            j = json.loads(json_string)
            #print json.dumps(j, sort_keys=True, indent=4, separators=(',', ': '))
            return j["current_observation"]["temp_c"], j["current_observation"]["relative_humidity"], j["current_observation"]["feelslike_c"], j["current_observation"]["weather"], j["current_observation"]["wind_dir"], j["current_observation"]["wind_kph"], j["current_observation"]["wind_gust_kph"]
        except:
            return 0,0,0,0,0,0,0

    def getPrediction(self):
        f = urllib2.urlopen(self._config.get("INFORMATION","WUPredictionURL"))
        json_string = f.read()
        j = json.loads(json_string)
        #print json.dumps(j, sort_keys=True, indent=4, separators=(',', ': '))
        return j['forecast']['txt_forecast']['forecastday'][0]['fcttext_metric']

    def getNextISSPass(self):
        f = urllib2.urlopen("http://api.open-notify.org/iss-pass.json?lat=52.5147&lon=13.4394&alt=80&n=10")
        j = json.loads(f.read())
        #print json.dumps(j, sort_keys=True, indent=4, separators=(',', ': '))
        passes = []
        for i in j['response']:
            if int(i['duration']) > 600:
                passes.append(int(i['risetime']))
        return passes

    def getAstronauts(self):
        try:
            f = urllib2.urlopen("http://api.open-notify.org/astros.json")
            j = json.loads(f.read())
            #print json.dumps(j, sort_keys=True, indent=4, separators=(',', ': '))
            num = int(j['number'])
            ast = ""
            for i in range(num - 1):
                    ast = ast +  j['people'][i]['name'] + ', '
            ast = ast[:-2]
            ast = ast + " and "
            ast = ast + j['people'][num - 1]['name']
            return ast
        except:
            return "No valid data"

    def getTimeInBathShower(self):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists("bath"):
            start = float(r.get("bath"))
        if r.exists("shower"):
            start = float(r.get("shower"))
        return int((time.time()-start)/60.0)

    def getWhoIsInBathShower(self):
        return "Ansi"

    def getBathOrShower(self):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists("bath"):
            return "bath"
        if r.exists("shower"):
            return "shower"

    def isSomeoneInTheRoom(self, room):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists(room+"/populated"):
            last = float(r.get(room+"/populated"))
            if (time.time()) - last < (5 * 60):
                return True
        return False

    def getNextWackeuptime(self):
        credential_path     = os.path.join(self._homeDir, 'calendar-python.json')
        secret_path         = os.path.join(self._homeDir, 'client_secret.json'  )
        store               = Storage(credential_path)
        credentials         = store.get()
        if not credentials or credentials.invalid:
            flow            = client.flow_from_clientsecrets(secret_path, 'https://www.googleapis.com/auth/calendar.readonly')
            flow.user_agent = 'Sensomatic Home Automatisation'
            credentials     = tools.run_flow(flow, store)
        http                = credentials.authorize(httplib2.Http())
        service             = discovery.build('calendar', 'v3', http=http)
        now                 = datetime.datetime.utcnow().isoformat() + 'Z'
        eventsResult        = service.events().list(calendarId='hrl7gpmm5u34379e2757kanuhg@group.calendar.google.com', timeMin=now, maxResults=10, singleEvents=True, orderBy='startTime').execute()
        events              = eventsResult.get('items', [])

        if not events:
            return None, None
        event = events[0]
        start = event['start'].get('dateTime', event['start'].get('date'))
        end   = event['end'].get('dateTime', event['start'].get('date'))
        #print(start, event['summary'])
        return iso8601.parse_date(start), iso8601.parse_date(end)

    def getRadiationAverage(self):
        # https://odlinfo.bfs.de/daten/Datenbereitstellung-2016-04-21.pdf
        try:
            auth = HTTPBasicAuth(self._config.get("INFORMATION","RadiationUser"), self._config.get("INFORMATION","RadiationPasswd"))
            url = self._config.get("INFORMATION","RadiationURL") + "stat.json"
            j = json.loads(requests.get(url, auth=auth).content)
            return j['mwavg']['mw']
        except:
            return 0

    def getRadiationForOneStation(self):
        # https://odlinfo.bfs.de/daten/Datenbereitstellung-2016-04-21.pdf
        try:
            auth = HTTPBasicAuth(self._config.get("INFORMATION","RadiationUser"), self._config.get("INFORMATION","RadiationPasswd"))
            url = self._config.get("INFORMATION","RadiationURL") + self._config.get("INFORMATION","RadiationStation") + "ct.json"
            j = json.loads(requests.get(url, auth=auth).content)
            status = j['stamm']['status']  # 0 defekt, 1 in Betrieb, 128 Testbetrieb, 2048 Wartung
            lastmw = j['stamm']['mw']      # Letzter verfuegbarer (ungepruefter) 1h-Messwert
            mw1h = j['mw1h']
            mw24h = j['mw24h']
            return lastmw
        except:
            return 0

if __name__ == '__main__':

    print "Testing"
    i = InformationFetcher()
    #print i.getEarthRotationTime()
    #print i.getNumEmailMessages()
    #print i.getRoomTemp(Room.BATH_ROOM)
    #print i.getRoomHumidity(Room.BATH_ROOM)
    #print i.getOutdoor()
    #print i.getPrediction()
    #print i.getNextISSPass()
    #print i.getAstronauts()
    #print i.getRoomCo2Level(Room.ANSI_ROOM)
    #print i.getRadiationAverage()
    #print i.getRadiationForOneStation()
    #print i.getSunPosition()
    #print i.getOutsideLightLevel()
    print i.getNextWackeuptime()<commit_msg>Workaround<commit_after>import time
import json
import math
import ephem
import redis
import urllib2
import requests
import ConfigParser
import httplib2
import os
import iso8601
import datetime

from apiclient         import discovery
from oauth2client      import client
from oauth2client      import tools
from oauth2client.file import Storage
from requests.auth     import HTTPBasicAuth
from imapclient        import IMAPClient

class InformationFetcher():

    def _readConfig(self):
        update = False

        if not os.path.isdir(self._homeDir):
            print "Creating homeDir"
            os.makedirs(self._homeDir)

        if os.path.isfile(self._configFileName):
            self._config.read(self._configFileName)
        else:
            print "Config file not found"
            update = True

        if not self._config.has_section('INFORMATION'):
            print "Adding MQTT part"
            update = True
            self._config.add_section("INFORMATION")

        if not self._config.has_option("INFORMATION", "IMAPServer"):
            print "No Server Address"
            update = True
            self._config.set("INFORMATION", "IMAPServer", "<ServerAddress>")

        if not self._config.has_option("INFORMATION", "IMAPUser"):
            print "No Username"
            update = True
            self._config.set("INFORMATION", "IMAPUser", "<IMAPUser>")

        if not self._config.has_option("INFORMATION", "IMAPPasswd"):
            print "No Passwd"
            update = True
            self._config.set("INFORMATION", "IMAPPasswd", "<IMAPPasswd>")

        if not self._config.has_option("INFORMATION", "IMAPFolder"):
            print "No Folder"
            update = True
            self._config.set("INFORMATION", "IMAPFolder", "<IMAPFolder>")

        if not self._config.has_option("INFORMATION", "WUCurrentURL"):
            print "No URL"
            update = True
            self._config.set("INFORMATION", "WUCurrentURL", "<WU Current>")

        if not self._config.has_option("INFORMATION", "WUPredictionURL"):
            print "No URL"
            update = True
            self._config.set("INFORMATION", "WUPredictionURL", "<WU Predict>")

        if not self._config.has_option("INFORMATION", "RadiationURL"):
            print "No RadiationURL"
            update = True
            self._config.set("INFORMATION", "RadiationURL", "https://odlinfo.bfs.de/daten/json/")

        if not self._config.has_option("INFORMATION", "RadiationUser"):
            print "No RadiationURL"
            update = True
            self._config.set("INFORMATION", "RadiationUser", "USERNAME")

        if not self._config.has_option("INFORMATION", "RadiationPasswd"):
            print "No RadiationURL"
            update = True
            self._config.set("INFORMATION", "RadiationPasswd", "USERPASSWD")

        if not self._config.has_option("INFORMATION", "RadiationStation"):
            print "No RadiationURL"
            update = True
            self._config.set("INFORMATION", "RadiationStation", "110000006")

        if not self._config.has_option("INFORMATION", "Latitude"):
            print "No Latitude"
            update = True
            self._config.set("INFORMATION", "Latitude", "52.5148453")

        if not self._config.has_option("INFORMATION", "Logitude"):
            print "No Logitude"
            update = True
            self._config.set("INFORMATION", "Logitude", "13.4389504")

        if not self._config.has_option("INFORMATION", "Elevation"):
            print "No Elevation"
            update = True
            self._config.set("INFORMATION", "Elevation", "42.23")

        if not self._config.has_option("INFORMATION", "NASAurl"):
            print "No NASAurl"
            update = True
            self._config.set("INFORMATION", "NASAurl", "https://api.nasa.gov/")

        if not self._config.has_option("INFORMATION", "NASAKey"):
            print "No NASAKey"
            update = True
            self._config.set("INFORMATION", "NASAKey", "magic")

        if update:
            with open(self._configFileName, 'w') as f:
                self._config.write(f)

    def __init__(self):
        self._homeDir        = os.path.expanduser("~/.sensomatic")
        self._configFileName = self._homeDir + '/config.ini'
        self._config         = ConfigParser.ConfigParser()
        self._readConfig()

    def getEarthRotationTime(self):
        seconds  = time.gmtime().tm_sec         *  1.0
        seconds += time.gmtime().tm_min         * 60.0
        seconds += time.gmtime().tm_hour * 60.0 * 60.0
        fraction = 360.0 * (seconds / (60.0 * 60.0 * 24.0 * 1.0) )

        degrees = int(   fraction                                    )
        minutes = int(  (fraction - degrees) * 60.0                  )
        seconds = int( ((fraction - degrees) * 60.0 - minutes) * 60.0)
        return fraction, degrees, minutes, seconds

    def getSunPosition(self):
        observer = ephem.Observer()
        observer.lon       =       self._config.get("INFORMATION", "Logitude" )
        observer.lat       =       self._config.get("INFORMATION", "Latitude" )
        observer.elevation = float(self._config.get("INFORMATION", "Elevation"))
        sun = ephem.Sun(observer)
        alt = sun.alt * (180 / math.pi)
        az  = sun.az  * (180 / math.pi)
        return alt,az

    def getOutsideLightLevel(self):
        # For now just calculated
        sun, _ = self.getSunPosition()
        if sun > 0:
            return (10.0 / 9.0) * sun
        else:
            return 0

    def getNumEmailMessages(self):
        try:
            server = IMAPClient(self._config.get("INFORMATION","IMAPServer"), use_uid=True, ssl=True)
            server.login(self._config.get("INFORMATION", "IMAPUser"), self._config.get("INFORMATION", "IMAPPasswd"))
            select_info = server .select_folder(self._config.get("INFORMATION", "IMAPFolder"))
            #print json.dumps(select_info, sort_keys=True, indent=4, separators=(',', ': '))
            return select_info['EXISTS']
        except Exception as inst:
            print str(inst)
            return 0

    def getRoomTemp(self, room):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists(room + "/temperature"):
            return float(r.get(room + "/temperature"))
        else:
            return None

    def getRoomDefinedTemp(self, room):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists(room + "/settemperature"):
            return float(r.get(room + "/settemperature"))
        else:
            return None

    def getRoomHumidity(self, room):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists(room + "/humidity"):
            return float(r.get(room + "/humidity"))
        else:
            return None

    def getRoomDefinedHumidity(self, room):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists(room + "/sethumidity"):
            return float(r.get(room + "/sethumidity"))
        else:
            return None

    def getRoomCo2Level(self, room):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists(room + "/co2"):
            return int(r.get(room + "/co2"))
        else:
            return None

    def getPlantSoilLevel(self):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists("ansiroom/bed/plant"):
            return r.get("ansiroom/bed/plant")
        else:
            return None

    def getOutdoor(self):
        '''
            temp, hum, feelslike,  ..... 7
        '''
        try:
            f = urllib2.urlopen(self._config.get("INFORMATION", "WUCurrentURL"))
            json_string = f.read()
            j = json.loads(json_string)
            #print json.dumps(j, sort_keys=True, indent=4, separators=(',', ': '))
            return j["current_observation"]["temp_c"], j["current_observation"]["relative_humidity"], j["current_observation"]["feelslike_c"], j["current_observation"]["weather"], j["current_observation"]["wind_dir"], j["current_observation"]["wind_kph"], j["current_observation"]["wind_gust_kph"]
        except:
            return 0,0,0,0,0,0,0

    def getPrediction(self):
        f = urllib2.urlopen(self._config.get("INFORMATION","WUPredictionURL"))
        json_string = f.read()
        j = json.loads(json_string)
        #print json.dumps(j, sort_keys=True, indent=4, separators=(',', ': '))
        return j['forecast']['txt_forecast']['forecastday'][0]['fcttext_metric']

    def getNextISSPass(self):
        f = urllib2.urlopen("http://api.open-notify.org/iss-pass.json?lat=52.5147&lon=13.4394&alt=80&n=10")
        j = json.loads(f.read())
        #print json.dumps(j, sort_keys=True, indent=4, separators=(',', ': '))
        passes = []
        for i in j['response']:
            if int(i['duration']) > 600:
                passes.append(int(i['risetime']))
        return passes

    def getAstronauts(self):
        try:
            f = urllib2.urlopen("http://api.open-notify.org/astros.json")
            j = json.loads(f.read())
            #print json.dumps(j, sort_keys=True, indent=4, separators=(',', ': '))
            num = int(j['number'])
            ast = ""
            for i in range(num - 1):
                    ast = ast +  j['people'][i]['name'] + ', '
            ast = ast[:-2]
            ast = ast + " and "
            ast = ast + j['people'][num - 1]['name']
            return ast
        except:
            return "No valid data"

    def getTimeInBathShower(self):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists("bath"):
            start = float(r.get("bath"))
        if r.exists("shower"):
            start = float(r.get("shower"))
        return int((time.time()-start)/60.0)

    def getWhoIsInBathShower(self):
        return "Ansi or Phawx" # TODO fix name

    def getBathOrShower(self):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists("bath"):
            return "bath"
        if r.exists("shower"):
            return "shower"

    def isSomeoneInTheRoom(self, room):
        r = redis.StrictRedis(host=self._config.get("REDIS","ServerAddress"), port=self._config.get("REDIS","ServerPort"), db=0)
        if r.exists(room+"/populated"):
            last = float(r.get(room+"/populated"))
            if (time.time()) - last < (5 * 60): # 5 min
                return True
        return False

    def getNextWackeuptime(self):
        credential_path     = os.path.join(self._homeDir, 'calendar-python.json')
        secret_path         = os.path.join(self._homeDir, 'client_secret.json'  )
        store               = Storage(credential_path)
        credentials         = store.get()
        if not credentials or credentials.invalid:
            flow            = client.flow_from_clientsecrets(secret_path, 'https://www.googleapis.com/auth/calendar.readonly')
            flow.user_agent = 'Sensomatic Home Automatisation'
            credentials     = tools.run_flow(flow, store)
        http                = credentials.authorize(httplib2.Http())
        service             = discovery.build('calendar', 'v3', http=http)
        now                 = datetime.datetime.utcnow().isoformat() + 'Z'
        eventsResult        = service.events().list(calendarId='hrl7gpmm5u34379e2757kanuhg@group.calendar.google.com', timeMin=now, maxResults=10, singleEvents=True, orderBy='startTime').execute()
        events              = eventsResult.get('items', [])

        if not events:
            return None, None
        event = events[0]
        start = event['start'].get('dateTime', event['start'].get('date'))
        end   = event['end'].get('dateTime', event['start'].get('date'))
        #print(start, event['summary'])
        return iso8601.parse_date(start), iso8601.parse_date(end)

    def getRadiationAverage(self):
        # https://odlinfo.bfs.de/daten/Datenbereitstellung-2016-04-21.pdf
        try:
            auth = HTTPBasicAuth(self._config.get("INFORMATION","RadiationUser"), self._config.get("INFORMATION","RadiationPasswd"))
            url = self._config.get("INFORMATION","RadiationURL") + "stat.json"
            j = json.loads(requests.get(url, auth=auth).content)
            return j['mwavg']['mw']
        except:
            return 0

    def getRadiationForOneStation(self):
        # https://odlinfo.bfs.de/daten/Datenbereitstellung-2016-04-21.pdf
        try:
            auth = HTTPBasicAuth(self._config.get("INFORMATION","RadiationUser"), self._config.get("INFORMATION","RadiationPasswd"))
            url = self._config.get("INFORMATION","RadiationURL") + self._config.get("INFORMATION","RadiationStation") + "ct.json"
            j = json.loads(requests.get(url, auth=auth).content)
            status = j['stamm']['status']  # 0 defekt, 1 in Betrieb, 128 Testbetrieb, 2048 Wartung
            lastmw = j['stamm']['mw']      # Letzter verfuegbarer (ungepruefter) 1h-Messwert
            mw1h = j['mw1h']
            mw24h = j['mw24h']
            return lastmw
        except:
            return 0

if __name__ == '__main__':

    print "Testing"
    i = InformationFetcher()
    #print i.getEarthRotationTime()
    #print i.getNumEmailMessages()
    #print i.getRoomTemp(Room.BATH_ROOM)
    #print i.getRoomHumidity(Room.BATH_ROOM)
    #print i.getOutdoor()
    #print i.getPrediction()
    #print i.getNextISSPass()
    #print i.getAstronauts()
    #print i.getRoomCo2Level(Room.ANSI_ROOM)
    #print i.getRadiationAverage()
    #print i.getRadiationForOneStation()
    #print i.getSunPosition()
    #print i.getOutsideLightLevel()
    print i.getNextWackeuptime()<|endoftext|>


<commit_before>"""`jacquard` command-line tool handling."""

import sys
import pathlib
import argparse
import pkg_resources

from jacquard.config import load_config

def argument_parser():
    """
    Generate an argparse `ArgumentParser` for the CLI.

    This will look through all defined `jacquard.commands` entry points for
    subcommands; these are subclasses of `jacquard.commands.BaseCommand`.
    Using this mechanism, plugins can add their own subcommands.
    """
    parser = argparse.ArgumentParser(description="Split testing server")
    parser.add_argument(
        '-v',
        '--verbose',
        help="enable verbose output",
        action='store_true',
    )
    parser.add_argument(
        '-c',
        '--config',
        help="config file",
        type=pathlib.Path,
        default=pathlib.Path('config.cfg'),
    )
    parser.set_defaults(func=None)

    subparsers = parser.add_subparsers(metavar='subcommand')

    for entry_point in pkg_resources.iter_entry_points('jacquard.commands'):
        command = entry_point.load()()

        command_help = getattr(command, 'help', entry_point.name)

        subparser = subparsers.add_parser(
            entry_point.name,
            help=command_help,
            description=command_help,
        )

        subparser.set_defaults(func=command.handle)

        command.add_arguments(subparser)

    return parser

def main(args=sys.argv[1:]):
    """
    Run as if from the command line, with the given arguments.

    If the arguments in `args` are not given they default to using `sys.argv`.

    Note that this function is permitted to raise SystemExit; users who do not
    want exciting exiting behaviour should be prepared to catch this.
    """
    parser = argument_parser()
    options = parser.parse_args(args)

    if options.func is None:
        parser.print_help()
        return

    # Parse options
    config = load_config(options.config)

    # Run subcommand
    options.func(config, options)

if '__name__' == '__main__':
    main()
<commit_msg>Add version print command<commit_after>"""`jacquard` command-line tool handling."""

import sys
import pathlib
import argparse
import pkg_resources

from jacquard.config import load_config

def argument_parser():
    """
    Generate an argparse `ArgumentParser` for the CLI.

    This will look through all defined `jacquard.commands` entry points for
    subcommands; these are subclasses of `jacquard.commands.BaseCommand`.
    Using this mechanism, plugins can add their own subcommands.
    """
    parser = argparse.ArgumentParser(description="Split testing server")
    parser.add_argument(
        '-v',
        '--verbose',
        help="enable verbose output",
        action='store_true',
    )
    parser.add_argument(
        '-c',
        '--config',
        help="config file",
        type=pathlib.Path,
        default=pathlib.Path('config.cfg'),
    )
    parser.add_argument(
        '-V',
        '--version',
        help="show version and exit",
        action='version',
        version=str(pkg_resources.working_set.by_key['jacquard-split']),
    )
    parser.set_defaults(func=None)

    subparsers = parser.add_subparsers(metavar='subcommand')

    for entry_point in pkg_resources.iter_entry_points('jacquard.commands'):
        command = entry_point.load()()

        command_help = getattr(command, 'help', entry_point.name)

        subparser = subparsers.add_parser(
            entry_point.name,
            help=command_help,
            description=command_help,
        )

        subparser.set_defaults(func=command.handle)

        command.add_arguments(subparser)

    return parser

def main(args=sys.argv[1:]):
    """
    Run as if from the command line, with the given arguments.

    If the arguments in `args` are not given they default to using `sys.argv`.

    Note that this function is permitted to raise SystemExit; users who do not
    want exciting exiting behaviour should be prepared to catch this.
    """
    parser = argument_parser()
    options = parser.parse_args(args)

    if options.func is None:
        parser.print_help()
        return

    # Parse options
    config = load_config(options.config)

    # Run subcommand
    options.func(config, options)

if '__name__' == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import logging
import os.path
import unittest

from app import app
from datetime import datetime
from jobs import yr, hackernews, nsb, ping, calendar, avinor
from multiprocessing import Process
from werkzeug.serving import run_simple
from xml.etree import ElementTree as etree
from requests import Session
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter

class App(unittest.TestCase):

    def setUp(self):
        app.testing = True
        app.debug = True
        app.config['JOBS'] = {
            'mock': {
                'enabled': True,
                'interval': 60
            }
        }
        app.logger.setLevel(logging.WARN)
        logging.getLogger('werkzeug').setLevel(logging.WARN)
        self.p = Process(target=run_simple, args=('127.0.0.1', 8080, app),
                         kwargs={'use_debugger': app.debug})
        self.p.start()

    def url(self, path):
        return 'http://127.0.0.1:8080' + path

    def session(self):
        s = Session()
        s.mount('http://', HTTPAdapter(
            max_retries=Retry(total=100, backoff_factor=0.1))
        )
        return s

    def get(self, path, **kwargs):
        return self.session().get(self.url(path), **kwargs)

    def post(self, path, **kwargs):
        return self.session().post(self.url(path), **kwargs)

    def test_widget(self):
        r = self.get('/widget/foo')
        self.assertEqual(404, r.status_code)

        r = self.get('/widget/mock')
        self.assertEqual(200, r.status_code)
        self.assertEqual('text/html; charset=utf-8', r.headers['content-type'])

        r = self.get('/w/mock')
        self.assertEqual(200, r.status_code)
        self.assertEqual('text/html; charset=utf-8', r.headers['content-type'])

    def test_dashboard(self):
        r = self.get('/dashboard/foo')
        self.assertEqual(404, r.status_code)

        r = self.get('/')
        self.assertEqual(200, r.status_code)
        self.assertEqual('text/html; charset=utf-8', r.headers['content-type'])

    def test_events(self):
        r = self.get('/events', stream=True)
        with r:
            data = next(r.iter_lines(chunk_size=1)).decode('utf-8')
        self.assertEqual(200, r.status_code)
        self.assertEqual('text/event-stream; charset=utf-8',
                         r.headers['content-type'])
        self.assertEqual('data: {"body":{"data":"spam"},"job":"mock"}', data)

    def test_events_post(self):
        r = self.post('/events/foo')
        self.assertEqual(404, r.status_code)

        r = self.post('/events/mock', json={'data': 'eggs'})
        self.assertEqual(201, r.status_code)

        r = self.get('/events', stream=True)
        with r:
            data = next(r.iter_lines(chunk_size=1)).decode('utf-8')
        self.assertEqual(200, r.status_code)
        self.assertEqual('text/event-stream; charset=utf-8',
                         r.headers['content-type'])
        self.assertEqual('data: {"body":{"data":"eggs"},"job":"mock"}', data)

    def tearDown(self):
        self.p.terminate()
        self.p.join()

class Yr(unittest.TestCase):

    def setUp(self):
        xml_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                'test_data', 'varsel.xml'))
        with open(xml_path, 'rb') as f:
            self.tree = etree.fromstring(f.read())

    def test_parse_tree(self):
        y = yr.Yr({'interval': None, 'url': None})
        data = y._parse_tree(self.tree)

        self.assertEqual('Delvis skyet', data['description'])
        self.assertEqual('Trondheim', data['location'])
        self.assertEqual('16.3', data['temperature'])
        self.assertEqual('Nord', data['wind']['direction'])
        self.assertEqual('0.7', data['wind']['speed'])
        self.assertEqual('Flau vind', data['wind']['description'])

    def test_parse_tree_date(self):
        y = yr.Yr({'interval': None, 'url': None})
        data = y._parse_tree(self.tree, datetime(2013, 7, 1))

        self.assertEqual('Regn', data['description'])
        self.assertEqual('Trondheim', data['location'])
        self.assertEqual('23', data['temperature'])
        self.assertEqual(u'Sør', data['wind']['direction'])
        self.assertEqual('3.6', data['wind']['speed'])
        self.assertEqual('Lett bris', data['wind']['description'])

    def test_parse_tree_missing_wind(self):
        xml_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                'test_data', 'varsel2.xml'))
        with open(xml_path, 'rb') as f:
            tree = etree.fromstring(f.read())
        y = yr.Yr({'interval': None, 'url': None})
        data = y._parse_tree(tree)
        self.assertTrue(data['wind'] is None)

class HackerNews(unittest.TestCase):

    def setUp(self):
        html_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                 'test_data', 'hn.html'))
        with open(html_path, 'r') as f:
            self.html = f.read()
        self.hn = hackernews.HackerNews({'interval': None})

    def test_parse(self):
        data = self.hn._parse(self.html)

        self.assertEqual(30, len(data['items']))
        self.assertTrue('points' in data['items'][3])
        self.assertTrue('title' in data['items'][3])
        self.assertEqual(76, data['items'][3]['points'])
        self.assertEqual('Building an OpenBSD Router',
                         data['items'][3]['title'])

class Nsb(unittest.TestCase):

    def setUp(self):
        test_data = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                 'test_data'))
        with open(os.path.join(test_data, 'nsb.html')) as f:
            self.html = f.read()

        self.nsb = nsb.Nsb({'interval': None,
                            'from': 'Skansen',
                            'to': 'V\xc3\xa6rnes (Trondheim Lufthavn)'})

    def test_parse(self):
        # Test data contains multiple dates and varying durations
        data = self.nsb._parse(self.html)

        self.assertEqual('02.10.2016', data['date'])
        self.assertEqual('V\xc3\xa6rnes (Trondheim Lufthavn)', data['to'])
        self.assertEqual('Skansen', data['from'])
        self.assertEqual(5, len(data['departures']))
        self.assertEqual('20:56', data['departures'][0]['departure'])
        self.assertEqual('21:36', data['departures'][0]['arrival'])
        self.assertEqual('40min', data['departures'][0]['duration'])

class Ping(unittest.TestCase):

    def setUp(self):
        self.ping = ping.Ping({'interval': None, 'hosts': None})

    def test_parse_time(self):
        s = ('PING google.com (173.194.69.139): 56 data bytes\n'
             '64 bytes from 173.194.69.139: icmp_seq=0 ttl=46 time=57.478 ms\n'
             '\n--- google.com ping statistics ---\n'
             '1 packets transmitted, 1 packets received, 0.0% packet loss\n'
             'round-trip min/avg/max/stddev = 57.478/57.478/57.478/0.000 ms\n')

        self.assertEqual(57.478, self.ping._parse_time(s))
        self.assertEqual(0, self.ping._parse_time('foo bar'))

class Calendar(unittest.TestCase):

    def setUp(self):
        self.calendar = calendar.Calendar({'interval': None})

    def test_parse(self):
        items = [
            {'id': 'foo1', 'summary': 'Foo bar 1',
             'start': {'dateTime': '2013-07-01T15:00:00+02:00'}},
            {'id': 'foo2', 'summary': 'Foo bar 2',
             'start': {'dateTime': '2013-08-01T15:00:00+02:00'}},
        ]
        expected = [{'date': '2013-07-01T15:00:00+02:00', 'id': 'foo1',
                     'summary': 'Foo bar 1'},
                    {'date': '2013-08-01T15:00:00+02:00', 'id': 'foo2',
                     'summary': 'Foo bar 2'}]
        self.assertEqual(self.calendar._parse(items), expected)

class Avinor(unittest.TestCase):

    def setUp(self):
        xml_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                'test_data', 'flights.xml'))
        with open(xml_path, 'rb') as f:
            self.xml = f.read()

    def test_parse(self):
        a = avinor.Avinor({'interval': None,
                           'url': None,
                           'from': 'TRD',
                           'to': 'OSL'})
        data = a._parse(self.xml)
        self.assertEqual(34, len(data['flights']))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Use app.run to start app in test<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import logging
import os.path
import unittest

from app import app
from datetime import datetime
from jobs import yr, hackernews, nsb, ping, calendar, avinor
from multiprocessing import Process
from xml.etree import ElementTree as etree
from requests import Session
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter

class App(unittest.TestCase):

    def setUp(self):
        app.testing = True
        app.debug = True
        app.config['JOBS'] = {
            'mock': {
                'enabled': True,
                'interval': 60
            }
        }
        app.logger.setLevel(logging.WARN)
        logging.getLogger('werkzeug').setLevel(logging.WARN)
        # Run the application in a separate process so that streamed responses
        # are not blocked on running the client in the same thread
        self.p = Process(target=app.run, kwargs={
            'host': '127.0.0.1',
            'port': 8080,
            'use_reloader': False
        })
        self.p.start()

    def url(self, path):
        return 'http://127.0.0.1:8080' + path

    def session(self):
        s = Session()
        s.mount('http://', HTTPAdapter(
            max_retries=Retry(total=100, backoff_factor=0.1))
        )
        return s

    def get(self, path, **kwargs):
        return self.session().get(self.url(path), **kwargs)

    def post(self, path, **kwargs):
        return self.session().post(self.url(path), **kwargs)

    def test_widget(self):
        r = self.get('/widget/foo')
        self.assertEqual(404, r.status_code)

        r = self.get('/widget/mock')
        self.assertEqual(200, r.status_code)
        self.assertEqual('text/html; charset=utf-8', r.headers['content-type'])

        r = self.get('/w/mock')
        self.assertEqual(200, r.status_code)
        self.assertEqual('text/html; charset=utf-8', r.headers['content-type'])

    def test_dashboard(self):
        r = self.get('/dashboard/foo')
        self.assertEqual(404, r.status_code)

        r = self.get('/')
        self.assertEqual(200, r.status_code)
        self.assertEqual('text/html; charset=utf-8', r.headers['content-type'])

    def test_events(self):
        r = self.get('/events', stream=True)
        with r:
            data = next(r.iter_lines(chunk_size=1)).decode('utf-8')
        self.assertEqual(200, r.status_code)
        self.assertEqual('text/event-stream; charset=utf-8',
                         r.headers['content-type'])
        self.assertEqual('data: {"body":{"data":"spam"},"job":"mock"}', data)

    def test_events_post(self):
        r = self.post('/events/foo')
        self.assertEqual(404, r.status_code)

        r = self.post('/events/mock', json={'data': 'eggs'})
        self.assertEqual(201, r.status_code)

        r = self.get('/events', stream=True)
        with r:
            data = next(r.iter_lines(chunk_size=1)).decode('utf-8')
        self.assertEqual(200, r.status_code)
        self.assertEqual('text/event-stream; charset=utf-8',
                         r.headers['content-type'])
        self.assertEqual('data: {"body":{"data":"eggs"},"job":"mock"}', data)

    def tearDown(self):
        self.p.terminate()
        self.p.join()

class Yr(unittest.TestCase):

    def setUp(self):
        xml_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                'test_data', 'varsel.xml'))
        with open(xml_path, 'rb') as f:
            self.tree = etree.fromstring(f.read())

    def test_parse_tree(self):
        y = yr.Yr({'interval': None, 'url': None})
        data = y._parse_tree(self.tree)

        self.assertEqual('Delvis skyet', data['description'])
        self.assertEqual('Trondheim', data['location'])
        self.assertEqual('16.3', data['temperature'])
        self.assertEqual('Nord', data['wind']['direction'])
        self.assertEqual('0.7', data['wind']['speed'])
        self.assertEqual('Flau vind', data['wind']['description'])

    def test_parse_tree_date(self):
        y = yr.Yr({'interval': None, 'url': None})
        data = y._parse_tree(self.tree, datetime(2013, 7, 1))

        self.assertEqual('Regn', data['description'])
        self.assertEqual('Trondheim', data['location'])
        self.assertEqual('23', data['temperature'])
        self.assertEqual(u'Sør', data['wind']['direction'])
        self.assertEqual('3.6', data['wind']['speed'])
        self.assertEqual('Lett bris', data['wind']['description'])

    def test_parse_tree_missing_wind(self):
        xml_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                'test_data', 'varsel2.xml'))
        with open(xml_path, 'rb') as f:
            tree = etree.fromstring(f.read())
        y = yr.Yr({'interval': None, 'url': None})
        data = y._parse_tree(tree)
        self.assertTrue(data['wind'] is None)

class HackerNews(unittest.TestCase):

    def setUp(self):
        html_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                 'test_data', 'hn.html'))
        with open(html_path, 'r') as f:
            self.html = f.read()
        self.hn = hackernews.HackerNews({'interval': None})

    def test_parse(self):
        data = self.hn._parse(self.html)

        self.assertEqual(30, len(data['items']))
        self.assertTrue('points' in data['items'][3])
        self.assertTrue('title' in data['items'][3])
        self.assertEqual(76, data['items'][3]['points'])
        self.assertEqual('Building an OpenBSD Router',
                         data['items'][3]['title'])

class Nsb(unittest.TestCase):

    def setUp(self):
        test_data = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                 'test_data'))
        with open(os.path.join(test_data, 'nsb.html')) as f:
            self.html = f.read()

        self.nsb = nsb.Nsb({'interval': None,
                            'from': 'Skansen',
                            'to': 'V\xc3\xa6rnes (Trondheim Lufthavn)'})

    def test_parse(self):
        # Test data contains multiple dates and varying durations
        data = self.nsb._parse(self.html)

        self.assertEqual('02.10.2016', data['date'])
        self.assertEqual('V\xc3\xa6rnes (Trondheim Lufthavn)', data['to'])
        self.assertEqual('Skansen', data['from'])
        self.assertEqual(5, len(data['departures']))
        self.assertEqual('20:56', data['departures'][0]['departure'])
        self.assertEqual('21:36', data['departures'][0]['arrival'])
        self.assertEqual('40min', data['departures'][0]['duration'])

class Ping(unittest.TestCase):

    def setUp(self):
        self.ping = ping.Ping({'interval': None, 'hosts': None})

    def test_parse_time(self):
        s = ('PING google.com (173.194.69.139): 56 data bytes\n'
             '64 bytes from 173.194.69.139: icmp_seq=0 ttl=46 time=57.478 ms\n'
             '\n--- google.com ping statistics ---\n'
             '1 packets transmitted, 1 packets received, 0.0% packet loss\n'
             'round-trip min/avg/max/stddev = 57.478/57.478/57.478/0.000 ms\n')

        self.assertEqual(57.478, self.ping._parse_time(s))
        self.assertEqual(0, self.ping._parse_time('foo bar'))

class Calendar(unittest.TestCase):

    def setUp(self):
        self.calendar = calendar.Calendar({'interval': None})

    def test_parse(self):
        items = [
            {'id': 'foo1', 'summary': 'Foo bar 1',
             'start': {'dateTime': '2013-07-01T15:00:00+02:00'}},
            {'id': 'foo2', 'summary': 'Foo bar 2',
             'start': {'dateTime': '2013-08-01T15:00:00+02:00'}},
        ]
        expected = [{'date': '2013-07-01T15:00:00+02:00', 'id': 'foo1',
                     'summary': 'Foo bar 1'},
                    {'date': '2013-08-01T15:00:00+02:00', 'id': 'foo2',
                     'summary': 'Foo bar 2'}]
        self.assertEqual(self.calendar._parse(items), expected)

class Avinor(unittest.TestCase):

    def setUp(self):
        xml_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                'test_data', 'flights.xml'))
        with open(xml_path, 'rb') as f:
            self.xml = f.read()

    def test_parse(self):
        a = avinor.Avinor({'interval': None,
                           'url': None,
                           'from': 'TRD',
                           'to': 'OSL'})
        data = a._parse(self.xml)
        self.assertEqual(34, len(data['flights']))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import re
import os
import json
import sys

import cfscrape
import demjson
import jsbeautifier

from bs4 import BeautifulSoup as bs

WATCHANIME = "watchanime.me"
ANIMELAND = "animeland.tv"

def identify_website(url):
    watchanime = re.compile(r"^(http:\/\/|https:\/\/)*([a-z0-9][a-z0-9\-]*\.)*(watchanime)\.me(\/.*)?$")
    animeland = re.compile(r"^(http:\/\/|https:\/\/)*([a-z0-9][a-z0-9\-]*\.)*(animeland)\.tv(\/.*)?$")

    if watchanime.match(url):
        return WATCHANIME

    if animeland.match(url):
        return ANIMELAND

    return False

def _get_webpages(episodes_dict, start, end):
    webpages = []

    if start > 0 and end > 0:
        for i in range(start, end + 1):
            try:
                # Matching episodes
                for episode in episodes_dict.keys():
                    ep_index = re.search(r"Episode (\d+)", episode).group(1)
                    if i == int(ep_index):
                        webpages.append(episodes_dict["Episode " + ep_index])
            except:
                print("No Episode " + str(i))
    else:
        keys = list(episodes_dict.keys())

        if re.match(r"^Episode \d+$", keys[0]):
            keys.sort(key = lambda episode: episode.split()[1])

        for episode in keys:
            webpages.append(episodes_dict[episode])

    return webpages

def _is_episode_missing(ep):
    if ep not in [os.path.splitext(f)[0] for f in os.listdir()]:
        return True
    return False

def _scrape_episodes(url, start, end, find_missing):
    if not (url[:7] == "http://" or url[:8] == "https://"):
        url = "http://" + url

    page_url = url
    START_EPISODE = start
    END_EPISODE = end
    episodes_dict = {}
    repisodes_dict = {}
    webpages = []
    failed_episodes = []
    hash_map = {}

    scraper = cfscrape.create_scraper()

    if identify_website(url) == ANIMELAND:
        # Animeland
        QUALITY = ["360p", "480p", "720p"][0]   # Select quality
        website_base_url = "http://www.animeland.tv/"
        source = scraper.get(page_url).content
        soup = bs(source, "html.parser")

        # Fetch the list of episodes
        for script in soup.find_all("script"):
            match = re.search(r'\$\("#load"\)\.load\(\'(.+)\'\)', str(script))
            if match:
                soup = bs(scraper.get(website_base_url + match.group(1)[1:]).content, "html.parser")
                for a in soup.find_all("a", {"class": "play"}):
                    ep = a.getText()
                    if find_missing:
                        if _is_episode_missing(ep):
                            episodes_dict[ep] = website_base_url + a["href"][1:]
                        continue
                    episodes_dict[ep] = website_base_url + a["href"][1:]

        webpages = _get_webpages(episodes_dict, START_EPISODE, END_EPISODE)

        # Reverse episodes_dict
        for key in episodes_dict.keys():
            repisodes_dict[episodes_dict[key]] = key

        downloads = []

        for url in webpages:
            episode = repisodes_dict[url]
            failed = False

            try:
                source = scraper.get(url).content
                soup = bs(source, "html.parser")
                iframe = soup.find("iframe", {"id": "video"})

                vidurl = iframe["src"]
                vidurl_source = scraper.get(vidurl).content
                vidurl_soup = bs(vidurl_source, "html.parser")

                new_player = False
                for meta in vidurl_soup.find_all("meta"):
                    if meta.has_attr("http-equiv"):
                        if meta["http-equiv"].lower() == "refresh":
                            new_player = True
                            vidurl = re.search(r'url=(.+)', meta["content"].split(";")[1]).group(1)
                            vidurl_source = scraper.get(vidurl).content
                            vidurl_soup = bs(vidurl_source, "html.parser")

                if new_player:
                    vidjs = vidurl_soup.find_all("script", {"type": "text/javascript"})[1].text.replace("\n", "")
                else:
                    vidjs = jsbeautifier.beautify(vidurl_soup.body.find("script", {"type": "text/javascript"}).text.replace("\n", "")).replace("\n", "")
                sources = demjson.decode("{" + re.search(r'playerInstance\.setup\(.+(sources:.+?\[.+?\])', vidjs).group(1).replace(" ", "") + "}")
            except:
                print(sys.exc_info())
                print("Failed to get " + episode)
                failed = True

            if not failed:
                for source in sources["sources"]:
                    download_url = ""
                    if source["label"] == QUALITY:
                        download_url = source["file"]

                if not download_url:
                    download_url = sources["sources"][0]["file"]  # Sorry ;p
                downloads.append(download_url)
                print(episode + ":", download_url, end="\n\n")
                hash_map[download_url] = episode

            else:
                failed_episodes.append(episode)

            '''
            # The website has 3 kinds of DOM structures for their videos
                try:
                    # Method 1
                    video = iframe_soup.find("video", {"id": "video"})
                    sources = video.find_all("source")
                    method = 1
                except:
                    try:
                        # Method 2
                        parent_div = iframe_soup.find("div", {"id": "videop"})
                        script = str(parent_div.script).replace("\n", "")
                        json_string = "{" + re.search(r"\bsources:.*\]", script).group(0) + "}"
                        sources = demjson.decode(json_string)
                        sources = sources["sources"]
                        method = 2
                    except:
                        try:
                            # Method 3
                            sources = [{"file": iframe_soup.find("div", {"id": "vid"}).source["src"], "label": QUALITY}]  # Sorry for lying :(
                            method = 3
                        except:
                            #print(sys.exc_info())
                            print("Failed to get " + episode)
                            failed = True'''
    else:
        # Watchanime
        QUALITY = ["360", "480", "720"][1]
        website_base_url = "http://watchanime.me/"
        sp = bs(scraper.get(page_url).content, "html.parser")
        ep_divs = []
        for div in sp.find_all("div"):
            try:
                if re.match(r"episodes_\d+-\d+", div["id"]):
                    ep_divs.append(div)
            except KeyError:
                pass

        for ep_div in ep_divs:
            for a in ep_div.find_all("a"):
                #print(a.getText())
                ep = "Episode " + re.search(r"Ep\. (\d+(\.\d+)?) \[.+\]", a.getText()).group(1)
                if find_missing:
                    if _is_episode_missing(ep):
                        episodes_dict[ep] = a["href"].strip()
                    continue
                episodes_dict[ep] = a["href"].strip()

        webpages = _get_webpages(episodes_dict, START_EPISODE, END_EPISODE)

        # Reverse episodes_dict
        for key in episodes_dict.keys():
            repisodes_dict[episodes_dict[key]] = key

        downloads = []
        for url in webpages:
            episode = repisodes_dict[url]
            source = scraper.get(url).content
            soup = bs(source, "html.parser")
            download_url = ""
            mp4upload = re.compile(r"^(http:\/\/|https:\/\/)*([a-z0-9][a-z0-9\-]*\.)*(mp4upload)\.com(\/.*)?$")
            yourupload = re.compile(r"^(http:\/\/|https:\/\/)*([a-z0-9][a-z0-9\-]*\.)*(yourupload)\.com(\/.*)?$")
            video_frames = soup.find("div", {"class": "autosize-container"}).find_all("iframe")
            try:
                #print("method 1")
                # Method 1 (YourUpload)
                #raise ValueError("")  # For when I need only Mp4Upload
                for frame in video_frames:
                    if yourupload.match(frame["src"]):
                        yourup_iframe_source = scraper.get(frame["src"]).content
                        yourup_soup = bs(yourup_iframe_source, "html.parser")
                        path = yourup_soup.find("video", {"id": "player"}).source["src"]
                        if path.lower() == "undefined":
                            raise ValueError("")  # Failing method 1.
                        download_url = "https://yourupload.com" + path
                if not download_url:
                    raise ValueError("")
            except:
                #print(sys.exc_info())
                # Method 2 (MP4Upload)
                try:
                    #print("method 2")
                    for frame in video_frames:
                        if mp4upload.match(frame["src"]):
                            mp4up_iframe_source = scraper.get(frame["src"]).content
                            mp4up_soup = bs(mp4up_iframe_source, "html.parser")
                            mp4up_js = jsbeautifier.beautify(mp4up_soup.body.find("script", {"type": "text/javascript"}).text)
                            download_url = re.search(r'src:"(.+\.mp4")', mp4up_js).group(1).replace('"', "")
                            #print(download_url, url)
                    if not download_url:
                        raise ValueError("")
                except:
                    print(sys.exc_info())
                    failed_episodes.append(episode)
                    print("Failed to get", episode)

            if download_url:
                print(episode + ": " + download_url, end="\n\n")
                downloads.append(download_url)
                hash_map[download_url] = episode

    return hash_map, failed_episodes

def get_episodes_dictionary(url, start=0, end=0, find_missing=False):
    """
    Returns a dictionary with episode download URLs mapped to their named and a list of episodes that couldn't be fetched.
        start: Episode to start fetching from
        end: Episode to stop fetching at
    """
    if start < 0 or end < 0:
        raise Exception("Invalid argument(s) for start and/or end")
    if identify_website(url):
        return _scrape_episodes(url, start, end, find_missing)
    else:
        raise Exception("Given URL not supported")
<commit_msg>remove scraper.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

import logging
import collections
import card_manager

""" Utility functions for GTR.
"""

def print_line(symbol='-'):
  """ Print a line for visual division of info """
  logging.info(symbol*80)

def print_header(title, symbol='-'):
  n_symbol = 80 - len(title)
  logging.info('{0}: {1}'.format(title, symbol*n_symbol))

def get_card_from_zone(card, zone):
  """ Wrapper around the possible exception caused by trying to
  find a non-existent card in a list. Prints an error and
  re-raises the exception.
  """
  logging.debug('getting card {0!s} from zone {1!s}'.format(card, zone))
  try:
    return zone.pop(zone.index(card))
  except ValueError as e:
    logging.error('Error! card {0!s} not found'.format(card))
    raise

def add_card_to_zone(card, zone):
  """
  """
  logging.debug('adding card {0!s} to zone {1!s}'.format(card, zone))
  zone.append(card)

def get_short_zone_summary(card_list):
  """ Return a single-line string with 4-letter card abbreviations and numbers
  of card instances.
  """
  counter = collections.Counter(card_list)
  cards_string = ''
  for card, count in counter.items():
    cards_string += '{0}[{1:d}], '.format(card[:4], count)
  cards_string.rstrip(', ')
  return cards_string

def get_detailed_card_summary(card, count=None):
  """ return a single-line string to describe one card in detail """

  card_string = '{0}'.format(card[:4])
  if count:
    card_string += '[{0:d}]'.format(count)
  if card != 'Jack':
    material = card_manager.get_material_of_card(card)
    role = card_manager.get_role_of_card(card)
    value = card_manager.get_value_of_card(card)
    function = card_manager.get_function_of_card(card)
    card_string += ' | {0}'.format(material[:3])
    card_string += ' | {0}'.format(role[:3])
    card_string += ' | {0}'.format(value)
    card_string += ' | {0}'.format(function)
  card_string += '\n'
  return card_string

def get_detailed_zone_summary(zone):
  """ return a multi-line description of cards in zone """

  counter = collections.Counter(zone)
  counter_dict = dict(counter)
  cards = counter_dict.keys()
  cards.sort() # alphabetize
  zone_string = 'CARD    | MAT | ROL | $ | DESCRIPTION \n'
  for card in cards:
    count = counter_dict[card]
    zone_string += get_detailed_card_summary(card, count)
  return zone_string

def get_building_info(building):
    """ Return a string to describe a building """

    # each building is a list where the 0th card defines the building, the other
    # orders cards are the materials, and there may be a site card

    # the 0th card is the title card
    if not building:
        return ''
    title_card = building[0]
    function = card_manager.get_function_of_card(title_card)

    has_site = False
    has_materials = False
    materials_string = ''
    for card in building[1:]:
      try:
        material = card_manager.get_material_of_card(card)
        materials_string += material[0]
        has_materials = True
      except:
        site_material = card
        has_site = True

    info = '{0} | '.format(title_card)
    if has_site:
        info += '{0} site '.format(site_material[:3])
    if has_materials:
        info += '+ '
    info += '{0} '.format(materials_string)
    info +=  ' | {0}'.format(function)
    return info

if __name__ == '__main__':

  # simple tests...
  logging.basicConfig(level=logging.DEBUG, format='%(message)s')

  test_zone = ['a', 'a', 'b', 'c']
  card_one = get_card_from_zone('a', test_zone)
  card_two = get_card_from_zone('c', test_zone)
  add_card_to_zone(card_one, test_zone)

  print get_short_zone_summary(test_zone)

<commit_msg>removed newline from detailed card description<commit_after>#!/usr/bin/env python

import logging
import collections
import card_manager

""" Utility functions for GTR.
"""

def print_line(symbol='-'):
  """ Print a line for visual division of info """
  logging.info(symbol*80)

def print_header(title, symbol='-'):
  n_symbol = 80 - len(title)
  logging.info('{0}: {1}'.format(title, symbol*n_symbol))

def get_card_from_zone(card, zone):
  """ Wrapper around the possible exception caused by trying to
  find a non-existent card in a list. Prints an error and
  re-raises the exception.
  """
  logging.debug('getting card {0!s} from zone {1!s}'.format(card, zone))
  try:
    return zone.pop(zone.index(card))
  except ValueError as e:
    logging.error('Error! card {0!s} not found'.format(card))
    raise

def add_card_to_zone(card, zone):
  """
  """
  logging.debug('adding card {0!s} to zone {1!s}'.format(card, zone))
  zone.append(card)

def get_short_zone_summary(card_list):
  """ Return a single-line string with 4-letter card abbreviations and numbers
  of card instances.
  """
  counter = collections.Counter(card_list)
  cards_string = ''
  for card, count in counter.items():
    cards_string += '{0}[{1:d}], '.format(card[:4], count)
  cards_string.rstrip(', ')
  return cards_string

def get_detailed_card_summary(card, count=None):
  """ return a single-line string to describe one card in detail """

  card_string = '{0}'.format(card[:4])
  if count:
    card_string += '[{0:d}]'.format(count)
  if card != 'Jack':
    material = card_manager.get_material_of_card(card)
    role = card_manager.get_role_of_card(card)
    value = card_manager.get_value_of_card(card)
    function = card_manager.get_function_of_card(card)
    card_string += ' | {0}'.format(material[:3])
    card_string += ' | {0}'.format(role[:3])
    card_string += ' | {0}'.format(value)
    card_string += ' | {0}'.format(function)
  return card_string

def get_detailed_zone_summary(zone):
  """ return a multi-line description of cards in zone """

  counter = collections.Counter(zone)
  counter_dict = dict(counter)
  cards = counter_dict.keys()
  cards.sort() # alphabetize
  zone_string = 'CARD    | MAT | ROL | $ | DESCRIPTION \n'
  for card in cards:
    count = counter_dict[card]
    zone_string += get_detailed_card_summary(card, count)
    zone_string += '\n'
  return zone_string

def get_building_info(building):
    """ Return a string to describe a building """

    # each building is a list where the 0th card defines the building, the other
    # orders cards are the materials, and there may be a site card

    # the 0th card is the title card
    if not building:
        return ''
    title_card = building[0]
    function = card_manager.get_function_of_card(title_card)

    has_site = False
    has_materials = False
    materials_string = ''
    for card in building[1:]:
      try:
        material = card_manager.get_material_of_card(card)
        materials_string += material[0]
        has_materials = True
      except:
        site_material = card
        has_site = True

    info = '{0} | '.format(title_card)
    if has_site:
        info += '{0} site '.format(site_material[:3])
    if has_materials:
        info += '+ '
    info += '{0} '.format(materials_string)
    info +=  ' | {0}'.format(function)
    return info

if __name__ == '__main__':

  # simple tests...
  logging.basicConfig(level=logging.DEBUG, format='%(message)s')

  test_zone = ['a', 'a', 'b', 'c']
  card_one = get_card_from_zone('a', test_zone)
  card_two = get_card_from_zone('c', test_zone)
  add_card_to_zone(card_one, test_zone)

  print get_short_zone_summary(test_zone)

<|endoftext|>


<commit_before># Copyright 2017 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import logging
import os
import re
import shlex
import subprocess
import sys

import guild.plugin
import guild.run

from guild import deps
from guild import opref
from guild import util
from guild import var

log = logging.getLogger("core")

OP_RUNFILES = [
    "org_click",
    "org_psutil"
]

class InvalidCmd(ValueError):
    pass

class Operation(object):

    def __init__(self, model, opdef, rundir=None):
        self.model = model
        self.opdef = opdef
        self.cmd_args = _init_cmd_args(opdef)
        self.cmd_env = _init_cmd_env(opdef)
        self._rundir = rundir
        self._running = False
        self._started = None
        self._stopped = None
        self._run = None
        self._proc = None
        self._exit_status = None

    @property
    def rundir(self):
        return self._rundir or (self._run.path if self._run else None)

    def run(self):
        assert not self._running
        self._running = True
        self._started = guild.run.timestamp()
        self._init_run()
        self._init_attrs()
        self._resolve_deps()
        self._start_proc()
        self._wait_for_proc()
        self._finalize_attrs()
        return self._exit_status

    def _init_run(self):
        if self._rundir:
            run_id = _rundir_hash(self._rundir)
            path = self._rundir
        else:
            run_id = guild.run.mkid()
            path = os.path.join(var.runs_dir(), run_id)
        self._run = guild.run.Run(run_id, path)
        log.debug("initializing run in %s", path)
        self._run.init_skel()

    def _init_attrs(self):
        assert self._run is not None
        self._run.write_attr("opref", self._opref_attr())
        self._run.write_attr("flags", self.opdef.flag_values())
        self._run.write_attr("cmd", self.cmd_args)
        self._run.write_attr("env", self.cmd_env)
        self._run.write_attr("started", self._started)

    def _opref_attr(self):
        ref = opref.OpRef.from_op(self.opdef.name, self.model.reference)
        return str(ref)

    def _resolve_deps(self):
        assert self._run is not None
        ctx = deps.ResolutionContext(
            target_dir=self._run.path,
            opdef=self.opdef)
        deps.resolve(self.opdef.dependencies, ctx)

    def _start_proc(self):
        assert self._proc is None
        assert self._run is not None
        args = self._proc_args()
        env = self._proc_env()
        cwd = self._run.path
        log.debug("starting operation run %s", self._run.id)
        log.debug("operation command: %s", args)
        log.debug("operation env: %s", env)
        log.debug("operation cwd: %s", cwd)
        self._proc = subprocess.Popen(args, env=env, cwd=cwd)
        _write_proc_lock(self._proc, self._run)

    def _proc_args(self):
        return self.cmd_args

    def _proc_env(self):
        assert self._run is not None
        env = {}
        env.update(self.cmd_env)
        env["RUNDIR"] = self._run.path
        env["RUNID"] = self._run.id
        return env

    def _wait_for_proc(self):
        assert self._proc is not None
        self._exit_status = self._proc.wait()
        self._stopped = guild.run.timestamp()
        _delete_proc_lock(self._run)

    def _finalize_attrs(self):
        assert self._run is not None
        assert self._exit_status is not None
        assert self._stopped is not None
        self._run.write_attr("exit_status", self._exit_status)
        self._run.write_attr("stopped", self._stopped)

def _init_cmd_args(opdef):
    python_args = [_python_cmd(opdef), "-u", _run_script_path()]
    cmd_args = _split_cmd(opdef.cmd)
    if not cmd_args:
        raise InvalidCmd(opdef.cmd)
    flag_args = _flag_args(opdef.flag_values(), cmd_args)
    return python_args + cmd_args + flag_args

def _python_cmd(_opdef):
    # TODO: This is an important operation that should be controlled
    # by the model (e.g. does it run under Python 2 or 3, etc.) and
    # not by whatever Python runtime is configured in the user env.
    return sys.executable

def _run_script_path():
    return os.path.join(os.path.dirname(__file__), "scripts", "run")

def _split_cmd(cmd):
    if isinstance(cmd, list):
        return cmd
    else:
        return shlex.split(cmd)

def _flag_args(flags, cmd_args):
    flag_args = []
    cmd_options = _cmd_options(cmd_args)
    for name in sorted(flags):
        value = flags[name]
        if name in cmd_options:
            log.warning(
                "ignoring flag '%s = %s' because it's shadowed "
                "in the operation cmd", name, value)
            continue
        flag_args.extend(_cmd_option_args(name, value))
    return flag_args

def _cmd_options(args):
    p = re.compile("--([^=]+)")
    return [m.group(1) for m in [p.match(arg) for arg in args] if m]

def _cmd_option_args(name, val):
    opt = "--%s" % name
    if val is None:
        return []
    elif val is True:
        return [opt]
    else:
        return [opt, str(val)]

def _init_cmd_env(opdef):
    env = {}
    env.update(util.safe_osenv())
    env["GUILD_PLUGINS"] = _op_plugins(opdef)
    env["LOG_LEVEL"] = str(logging.getLogger().getEffectiveLevel())
    env["PYTHONPATH"] = _python_path(opdef)
    return env

def _op_plugins(opdef):
    op_plugins = []
    for name, plugin in guild.plugin.iter_plugins():
        if _plugin_disabled_in_project(name, opdef):
            plugin_enabled = False
            reason = "explicitly disabled by model or user config"
        else:
            plugin_enabled, reason = plugin.enabled_for_op(opdef)
        log.debug(
            "plugin '%s' %s%s",
            name,
            "enabled" if plugin_enabled else "disabled",
            " (%s)" % reason if reason else "")
        if plugin_enabled:
            op_plugins.append(name)
    return ",".join(sorted(op_plugins))

def _plugin_disabled_in_project(name, opdef):
    disabled = (opdef.disabled_plugins +
                opdef.modeldef.disabled_plugins)
    return any([disabled_name in (name, "all") for disabled_name in disabled])

def _python_path(opdef):
    paths = _model_paths(opdef) + _guild_paths()
    return os.path.pathsep.join(paths)

def _model_paths(opdef):
    return [os.path.abspath(opdef.modelfile.dir)]

def _guild_paths():
    guild_path = os.path.dirname(os.path.dirname(__file__))
    abs_guild_path = os.path.abspath(guild_path)
    return [abs_guild_path] + _runfile_paths()

def _runfile_paths():
    return [
        os.path.abspath(path)
        for path in sys.path if _is_runfile_pkg(path)
    ]

def _is_runfile_pkg(path):
    return os.path.split(path)[-1] in OP_RUNFILES

def _rundir_hash(path):
    import md5
    return md5.md5(path).hexdigest()

def _write_proc_lock(proc, run):
    with open(run.guild_path("LOCK"), "w") as f:
        f.write(str(proc.pid))

def _delete_proc_lock(run):
    try:
        os.remove(run.guild_path("LOCK"))
    except OSError:
        pass
<commit_msg>Generate a unique run ID even for specified run dirs<commit_after># Copyright 2017 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import logging
import os
import re
import shlex
import subprocess
import sys

import guild.plugin
import guild.run

from guild import deps
from guild import opref
from guild import util
from guild import var

log = logging.getLogger("core")

OP_RUNFILES = [
    "org_click",
    "org_psutil"
]

class InvalidCmd(ValueError):
    pass

class Operation(object):

    def __init__(self, model, opdef, rundir=None):
        self.model = model
        self.opdef = opdef
        self.cmd_args = _init_cmd_args(opdef)
        self.cmd_env = _init_cmd_env(opdef)
        self._rundir = rundir
        self._running = False
        self._started = None
        self._stopped = None
        self._run = None
        self._proc = None
        self._exit_status = None

    @property
    def rundir(self):
        return self._rundir or (self._run.path if self._run else None)

    def run(self):
        assert not self._running
        self._running = True
        self._started = guild.run.timestamp()
        self._init_run()
        self._init_attrs()
        self._resolve_deps()
        self._start_proc()
        self._wait_for_proc()
        self._finalize_attrs()
        return self._exit_status

    def _init_run(self):
        run_id = guild.run.mkid()
        if self._rundir:
            path = self._rundir
        else:
            path = os.path.join(var.runs_dir(), run_id)
        self._run = guild.run.Run(run_id, path)
        log.debug("initializing run in %s", path)
        self._run.init_skel()

    def _init_attrs(self):
        assert self._run is not None
        self._run.write_attr("opref", self._opref_attr())
        self._run.write_attr("flags", self.opdef.flag_values())
        self._run.write_attr("cmd", self.cmd_args)
        self._run.write_attr("env", self.cmd_env)
        self._run.write_attr("started", self._started)

    def _opref_attr(self):
        ref = opref.OpRef.from_op(self.opdef.name, self.model.reference)
        return str(ref)

    def _resolve_deps(self):
        assert self._run is not None
        ctx = deps.ResolutionContext(
            target_dir=self._run.path,
            opdef=self.opdef)
        deps.resolve(self.opdef.dependencies, ctx)

    def _start_proc(self):
        assert self._proc is None
        assert self._run is not None
        args = self._proc_args()
        env = self._proc_env()
        cwd = self._run.path
        log.debug("starting operation run %s", self._run.id)
        log.debug("operation command: %s", args)
        log.debug("operation env: %s", env)
        log.debug("operation cwd: %s", cwd)
        self._proc = subprocess.Popen(args, env=env, cwd=cwd)
        _write_proc_lock(self._proc, self._run)

    def _proc_args(self):
        return self.cmd_args

    def _proc_env(self):
        assert self._run is not None
        env = {}
        env.update(self.cmd_env)
        env["RUNDIR"] = self._run.path
        env["RUNID"] = self._run.id
        return env

    def _wait_for_proc(self):
        assert self._proc is not None
        self._exit_status = self._proc.wait()
        self._stopped = guild.run.timestamp()
        _delete_proc_lock(self._run)

    def _finalize_attrs(self):
        assert self._run is not None
        assert self._exit_status is not None
        assert self._stopped is not None
        self._run.write_attr("exit_status", self._exit_status)
        self._run.write_attr("stopped", self._stopped)

def _init_cmd_args(opdef):
    python_args = [_python_cmd(opdef), "-u", _run_script_path()]
    cmd_args = _split_cmd(opdef.cmd)
    if not cmd_args:
        raise InvalidCmd(opdef.cmd)
    flag_args = _flag_args(opdef.flag_values(), cmd_args)
    return python_args + cmd_args + flag_args

def _python_cmd(_opdef):
    # TODO: This is an important operation that should be controlled
    # by the model (e.g. does it run under Python 2 or 3, etc.) and
    # not by whatever Python runtime is configured in the user env.
    return sys.executable

def _run_script_path():
    return os.path.join(os.path.dirname(__file__), "scripts", "run")

def _split_cmd(cmd):
    if isinstance(cmd, list):
        return cmd
    else:
        return shlex.split(cmd)

def _flag_args(flags, cmd_args):
    flag_args = []
    cmd_options = _cmd_options(cmd_args)
    for name in sorted(flags):
        value = flags[name]
        if name in cmd_options:
            log.warning(
                "ignoring flag '%s = %s' because it's shadowed "
                "in the operation cmd", name, value)
            continue
        flag_args.extend(_cmd_option_args(name, value))
    return flag_args

def _cmd_options(args):
    p = re.compile("--([^=]+)")
    return [m.group(1) for m in [p.match(arg) for arg in args] if m]

def _cmd_option_args(name, val):
    opt = "--%s" % name
    if val is None:
        return []
    elif val is True:
        return [opt]
    else:
        return [opt, str(val)]

def _init_cmd_env(opdef):
    env = {}
    env.update(util.safe_osenv())
    env["GUILD_PLUGINS"] = _op_plugins(opdef)
    env["LOG_LEVEL"] = str(logging.getLogger().getEffectiveLevel())
    env["PYTHONPATH"] = _python_path(opdef)
    return env

def _op_plugins(opdef):
    op_plugins = []
    for name, plugin in guild.plugin.iter_plugins():
        if _plugin_disabled_in_project(name, opdef):
            plugin_enabled = False
            reason = "explicitly disabled by model or user config"
        else:
            plugin_enabled, reason = plugin.enabled_for_op(opdef)
        log.debug(
            "plugin '%s' %s%s",
            name,
            "enabled" if plugin_enabled else "disabled",
            " (%s)" % reason if reason else "")
        if plugin_enabled:
            op_plugins.append(name)
    return ",".join(sorted(op_plugins))

def _plugin_disabled_in_project(name, opdef):
    disabled = (opdef.disabled_plugins +
                opdef.modeldef.disabled_plugins)
    return any([disabled_name in (name, "all") for disabled_name in disabled])

def _python_path(opdef):
    paths = _model_paths(opdef) + _guild_paths()
    return os.path.pathsep.join(paths)

def _model_paths(opdef):
    return [os.path.abspath(opdef.modelfile.dir)]

def _guild_paths():
    guild_path = os.path.dirname(os.path.dirname(__file__))
    abs_guild_path = os.path.abspath(guild_path)
    return [abs_guild_path] + _runfile_paths()

def _runfile_paths():
    return [
        os.path.abspath(path)
        for path in sys.path if _is_runfile_pkg(path)
    ]

def _is_runfile_pkg(path):
    return os.path.split(path)[-1] in OP_RUNFILES

def _write_proc_lock(proc, run):
    with open(run.guild_path("LOCK"), "w") as f:
        f.write(str(proc.pid))

def _delete_proc_lock(run):
    try:
        os.remove(run.guild_path("LOCK"))
    except OSError:
        pass
<|endoftext|>


<commit_before>import logging
import os
import re
import shlex
import subprocess
import sys
import time
import uuid

import guild.run
import guild.util
import guild.var

OS_ENVIRON_WHITELIST = [
    "LD_LIBRARY_PATH",
]

GUILD_PACKAGES = [
    "org_guildai_guild",
    "org_psutil",
]

class InvalidCmd(ValueError):
    pass

class Operation(object):

    def __init__(self, name, cmd_args, cmd_env, attrs={}):
        self.name = name
        self.cmd_args = cmd_args
        self.cmd_env = cmd_env
        self.attrs = attrs
        self._running = False
        self._run = None
        self._proc = None
        self._exit_status = None

    def run(self):
        assert not self._running:
        self._running = True
        self._started = int(time.time())
        self._init_run()
        self._init_attrs()
        self._start_proc()
        self._wait_for_proc()
        self._finalize_attrs()
        return self._exit_status

    def _init_run(self):
        id = unique_run_id()
        path = os.path.join(guild.var.runs_dir(), id)
        self._run = guild.run.Run(id, path)
        logging.debug("Initializing run in %s", path)
        self._run.init_skel()

    def _init_attrs(self):
        assert self._run is not None
        for name, val in self.attrs.items():
            self._run.write_attr(name, val)
        self._run.write_attr("cmd", self.cmd_args)
        self._run.write_attr("env", self.cmd_env)
        self._run.write_attr("started", self._started)
        self._run.write_attr("op", self.name)

    def _start_proc(self):
        assert self._proc is None
        assert self._run is not None
        args = self._proc_args()
        env = self._proc_env()
        cwd = self._run.path
        logging.debug("Starting process %s" % (args,))
        self._proc = subprocess.Popen(args, env=env, cwd=cwd)
        _write_proc_lock(self._proc, self._run)

    def _proc_args(self):
        assert self._run
        return self.cmd_args + ["--rundir", self._run.path]

    def _proc_env(self):
        assert self._run
        env = {}
        env.update(self.cmd_env)
        env["RUNDIR"] = self._run.path
        return env

    def _wait_for_proc(self):
        assert self._proc is not None
        self._exit_status = self._proc.wait()
        self._stopped = int(time.time())
        _delete_proc_lock(self._run)

    def _finalize_attrs(self):
        assert self._run is not None
        assert self._exit_status is not None
        assert self._stopped is not None
        self._run.write_attr("exit_status", self._exit_status)
        self._run.write_attr("stopped", self._stopped)

def unique_run_id():
    return uuid.uuid1().get_hex()

def _write_proc_lock(proc, run):
    with open(run.guild_path("LOCK"), "w") as f:
        f.write(str(proc.pid))

def _delete_proc_lock(run):
    try:
        os.remove(run.guild_path("LOCK"))
    except OSError:
        pass

def from_project_op(project_op):
    flags = _op_flags(project_op)
    cmd_args = _op_cmd_args(project_op, flags)
    cmd_env = _op_cmd_env(project_op)
    attrs = {
        "flags": flags
    }
    return Operation(
        project_op.full_name,
        cmd_args,
        cmd_env,
        attrs)

def _op_flags(op):
    flags = {}
    _acc_flags(op.model.flags, flags)
    _acc_flags(op.flags, flags)
    return flags

def _acc_flags(project_flags, flags):
    for name, val in project_flags.items():
        if isinstance(val, dict):
            val = val.get("value", None)
        flags[name] = val

def _op_cmd_args(project_op, flags):
    python_args = [_python_cmd(project_op), "-um", "guild.op_main"]
    cmd_args = _cmd_args(project_op)
    if not cmd_args:
        raise InvalidCmd(project_op.cmd)
    flag_args = _flag_args(flags)
    return python_args + cmd_args + flag_args

def _python_cmd(_project_op):
    # TODO: This is an important operation that should be controlled
    # by the model (e.g. does it run under Python 2 or 3, etc.) and
    # not by whatever Python runtime is configured in the user env.
    return "python"

def _cmd_args(project_op):
    cmd = project_op.cmd
    if isinstance(cmd, str):
        return shlex.split(cmd)
    elif isinstance(cmd, list):
        return cmd
    else:
        raise AssertionError(cmd)

def _flag_args(flags):
    return [
        arg for args in
        [_opt_args(name, val) for name, val in _sorted_flags(flags)]
        for arg in args
    ]

def _sorted_flags(flags):
    flags = flags or {}
    return [(key, flags[key]) for key in sorted(flags)]

def _opt_args(name, val):
    opt = "--%s" % name
    return [opt] if val is None else [opt, str(val)]

def _op_cmd_env(project_op):
    env = {}
    env.update(_op_os_env())
    env["GUILD_PLUGINS"] = _op_plugins(project_op)
    env["LOG_LEVEL"] = str(logging.getLogger().getEffectiveLevel())
    env["PYTHONPATH"] = _python_path(project_op)
    return env

def _op_os_env():
    return {
        name: val
        for name, val in os.environ.items()
        if name in OS_ENVIRON_WHITELIST
    }

def _op_plugins(project_op):
    op_plugins = []
    for name, plugin in guild.plugin.iter_plugins():
        if _plugin_disabled(name, project_op):
            logging.debug("plugin '%s' disabled for operation", name)
            continue
        if plugin.enabled_for_op(project_op):
            logging.debug("plugin '%s' enabled for operation", name)
            op_plugins.append(name)
    return ",".join(op_plugins)

def _plugin_disabled(name, project_op):
    return (name in project_op.disabled_plugins
            or name in project_op.model.disabled_plugins)

def _python_path(project_op):
    paths = _model_paths(project_op) + _guild_paths()
    return os.path.pathsep.join(paths)

def _model_paths(project_op):
    return [os.path.dirname(project_op.project.src)]

def _guild_paths():
    return [path for path in sys.path if _is_guild_path(path)]

def _is_guild_path(path):
    for pkg in GUILD_PACKAGES:
        if path.endswith(pkg):
            return True
    return False
<commit_msg>Syntax error<commit_after>import logging
import os
import re
import shlex
import subprocess
import sys
import time
import uuid

import guild.run
import guild.util
import guild.var

OS_ENVIRON_WHITELIST = [
    "LD_LIBRARY_PATH",
]

GUILD_PACKAGES = [
    "org_guildai_guild",
    "org_psutil",
]

class InvalidCmd(ValueError):
    pass

class Operation(object):

    def __init__(self, name, cmd_args, cmd_env, attrs={}):
        self.name = name
        self.cmd_args = cmd_args
        self.cmd_env = cmd_env
        self.attrs = attrs
        self._running = False
        self._run = None
        self._proc = None
        self._exit_status = None

    def run(self):
        assert not self._running
        self._running = True
        self._started = int(time.time())
        self._init_run()
        self._init_attrs()
        self._start_proc()
        self._wait_for_proc()
        self._finalize_attrs()
        return self._exit_status

    def _init_run(self):
        id = unique_run_id()
        path = os.path.join(guild.var.runs_dir(), id)
        self._run = guild.run.Run(id, path)
        logging.debug("Initializing run in %s", path)
        self._run.init_skel()

    def _init_attrs(self):
        assert self._run is not None
        for name, val in self.attrs.items():
            self._run.write_attr(name, val)
        self._run.write_attr("cmd", self.cmd_args)
        self._run.write_attr("env", self.cmd_env)
        self._run.write_attr("started", self._started)
        self._run.write_attr("op", self.name)

    def _start_proc(self):
        assert self._proc is None
        assert self._run is not None
        args = self._proc_args()
        env = self._proc_env()
        cwd = self._run.path
        logging.debug("Starting process %s" % (args,))
        self._proc = subprocess.Popen(args, env=env, cwd=cwd)
        _write_proc_lock(self._proc, self._run)

    def _proc_args(self):
        assert self._run
        return self.cmd_args + ["--rundir", self._run.path]

    def _proc_env(self):
        assert self._run
        env = {}
        env.update(self.cmd_env)
        env["RUNDIR"] = self._run.path
        return env

    def _wait_for_proc(self):
        assert self._proc is not None
        self._exit_status = self._proc.wait()
        self._stopped = int(time.time())
        _delete_proc_lock(self._run)

    def _finalize_attrs(self):
        assert self._run is not None
        assert self._exit_status is not None
        assert self._stopped is not None
        self._run.write_attr("exit_status", self._exit_status)
        self._run.write_attr("stopped", self._stopped)

def unique_run_id():
    return uuid.uuid1().get_hex()

def _write_proc_lock(proc, run):
    with open(run.guild_path("LOCK"), "w") as f:
        f.write(str(proc.pid))

def _delete_proc_lock(run):
    try:
        os.remove(run.guild_path("LOCK"))
    except OSError:
        pass

def from_project_op(project_op):
    flags = _op_flags(project_op)
    cmd_args = _op_cmd_args(project_op, flags)
    cmd_env = _op_cmd_env(project_op)
    attrs = {
        "flags": flags
    }
    return Operation(
        project_op.full_name,
        cmd_args,
        cmd_env,
        attrs)

def _op_flags(op):
    flags = {}
    _acc_flags(op.model.flags, flags)
    _acc_flags(op.flags, flags)
    return flags

def _acc_flags(project_flags, flags):
    for name, val in project_flags.items():
        if isinstance(val, dict):
            val = val.get("value", None)
        flags[name] = val

def _op_cmd_args(project_op, flags):
    python_args = [_python_cmd(project_op), "-um", "guild.op_main"]
    cmd_args = _cmd_args(project_op)
    if not cmd_args:
        raise InvalidCmd(project_op.cmd)
    flag_args = _flag_args(flags)
    return python_args + cmd_args + flag_args

def _python_cmd(_project_op):
    # TODO: This is an important operation that should be controlled
    # by the model (e.g. does it run under Python 2 or 3, etc.) and
    # not by whatever Python runtime is configured in the user env.
    return "python"

def _cmd_args(project_op):
    cmd = project_op.cmd
    if isinstance(cmd, str):
        return shlex.split(cmd)
    elif isinstance(cmd, list):
        return cmd
    else:
        raise AssertionError(cmd)

def _flag_args(flags):
    return [
        arg for args in
        [_opt_args(name, val) for name, val in _sorted_flags(flags)]
        for arg in args
    ]

def _sorted_flags(flags):
    flags = flags or {}
    return [(key, flags[key]) for key in sorted(flags)]

def _opt_args(name, val):
    opt = "--%s" % name
    return [opt] if val is None else [opt, str(val)]

def _op_cmd_env(project_op):
    env = {}
    env.update(_op_os_env())
    env["GUILD_PLUGINS"] = _op_plugins(project_op)
    env["LOG_LEVEL"] = str(logging.getLogger().getEffectiveLevel())
    env["PYTHONPATH"] = _python_path(project_op)
    return env

def _op_os_env():
    return {
        name: val
        for name, val in os.environ.items()
        if name in OS_ENVIRON_WHITELIST
    }

def _op_plugins(project_op):
    op_plugins = []
    for name, plugin in guild.plugin.iter_plugins():
        if _plugin_disabled(name, project_op):
            logging.debug("plugin '%s' disabled for operation", name)
            continue
        if plugin.enabled_for_op(project_op):
            logging.debug("plugin '%s' enabled for operation", name)
            op_plugins.append(name)
    return ",".join(op_plugins)

def _plugin_disabled(name, project_op):
    return (name in project_op.disabled_plugins
            or name in project_op.model.disabled_plugins)

def _python_path(project_op):
    paths = _model_paths(project_op) + _guild_paths()
    return os.path.pathsep.join(paths)

def _model_paths(project_op):
    return [os.path.dirname(project_op.project.src)]

def _guild_paths():
    return [path for path in sys.path if _is_guild_path(path)]

def _is_guild_path(path):
    for pkg in GUILD_PACKAGES:
        if path.endswith(pkg):
            return True
    return False
<|endoftext|>