<commit_before>import datetime
import glob
import os
import shutil
import tempfile
import traceback
import zipfile

from django.conf import settings
from django.contrib.gis.gdal import GDALRaster
from django.db import connection
from django.dispatch import Signal
from raster import tiler
from raster.const import WEB_MERCATOR_SRID, WEB_MERCATOR_TILESIZE
from raster.models import RasterLayerMetadata, RasterTile

rasterlayers_parser_started = Signal(providing_args=['instance'])

class RasterLayerParser(object):
    """
    Class to parse raster layers.
    """
    def __init__(self, rasterlayer):
        self.rasterlayer = rasterlayer

        self.rastername = os.path.basename(rasterlayer.rasterfile.name)

        # Set raster tilesize
        self.tilesize = int(getattr(settings, 'RASTER_TILESIZE', WEB_MERCATOR_TILESIZE))
        self.zoomdown = getattr(settings, 'RASTER_ZOOM_NEXT_HIGHER', True)

    def log(self, msg, reset=False):
        """
        Write a message to the parse log of the rasterlayer instance.
        """
        # Prepare datetime stamp for log
        now = '[{0}] '.format(datetime.datetime.now().strftime('%Y-%m-%d %T'))

        # Write log, reset if requested
        if reset:
            self.rasterlayer.parse_log = now + msg
        else:
            self.rasterlayer.parse_log += '\n' + now + msg

        self.rasterlayer.save()

    def get_raster_file(self):
        """
        Make local copy of rasterfile, which is needed if files are stored on
        remote storage, and unzip it if necessary.
        """
        self.log('Getting raster file from storage')

        self.tmpdir = tempfile.mkdtemp()

        # Access rasterfile and store in a temp folder
        rasterfile = open(os.path.join(self.tmpdir, self.rastername), 'wb')
        for chunk in self.rasterlayer.rasterfile.chunks():
            rasterfile.write(chunk)
        rasterfile.close()

        # If the raster file is compress, decompress it
        fileName, fileExtension = os.path.splitext(self.rastername)

        if fileExtension == '.zip':

            # Open and extract zipfile
            zf = zipfile.ZipFile(os.path.join(self.tmpdir, self.rastername))
            zf.extractall(self.tmpdir)

            # Remove zipfile
            os.remove(os.path.join(self.tmpdir, self.rastername))

            # Get filelist from directory
            raster_list = glob.glob(os.path.join(self.tmpdir, "*.*"))

            # Check if only one file is found in zipfile
            if len(raster_list) > 1:
                self.log(
                    'WARNING: Found more than one file in zipfile '
                    'using only first file found. This might lead '
                    'to problems if its not a raster file.'
                )

            # Return first one as raster file
            self.rastername = os.path.basename(raster_list[0])

    def open_raster_file(self):
        """
        Open the raster file as GDALRaster and set nodata-values.
        """
        self.log('Opening raster file as GDALRaster.')

        # Open raster file
        self.dataset = GDALRaster(os.path.join(self.tmpdir, self.rastername), write=True)

        # Make sure nodata value is set from input
        if self.rasterlayer.nodata is not None:
            for band in self.dataset.bands:
                band.nodata_value = float(self.rasterlayer.nodata)

        # Store original metadata for this raster
        self.store_original_metadata()

    def store_original_metadata(self):
        """
        Exports the input raster meta data to a RasterLayerMetadata object.
        """
        lyrmeta = RasterLayerMetadata.objects.get_or_create(rasterlayer=self.rasterlayer)[0]
        lyrmeta.uperleftx = self.dataset.origin.x
        lyrmeta.uperlefty = self.dataset.origin.y
        lyrmeta.width = self.dataset.width
        lyrmeta.height = self.dataset.height
        lyrmeta.scalex = self.dataset.scale.x
        lyrmeta.scaley = self.dataset.scale.y
        lyrmeta.skewx = self.dataset.skew.x
        lyrmeta.skewy = self.dataset.skew.y
        lyrmeta.numbands = len(self.dataset.bands)
        lyrmeta.srs_wkt = self.dataset.srs.wkt
        lyrmeta.srid = self.dataset.srs.srid
        lyrmeta.save()

    def create_tiles(self, zoom):
        """
        Create tiles for this raster at the given zoomlevel.

        This routine first snaps the raster to the grid of the zoomlevel,
        then creates  the tiles from the snapped raster.
        """
        # Compute the tile x-y-z index range for the rasterlayer for this zoomlevel
        bbox = self.rasterlayer.extent()
        indexrange = tiler.tile_index_range(bbox, zoom)

        # Compute scale of tiles for this zoomlevel
        tilescale = tiler.tile_scale(zoom)

        # Count the number of tiles that are required to cover the raster at this zoomlevel
        nr_of_tiles = (indexrange[2] - indexrange[0] + 1) * (indexrange[3] - indexrange[1] + 1)

        self.log('Creating {0} tiles for zoom {1}.'.format(nr_of_tiles, zoom))

        for tilex in range(indexrange[0], indexrange[2] + 1):
            for tiley in range(indexrange[1], indexrange[3] + 1):
                # Calculate raster tile origin
                bounds = tiler.tile_bounds(tilex, tiley, zoom)

                # Warp source raster into this tile (in memory)
                dest = self.dataset.warp({
                    'driver': 'MEM',
                    'width': self.tilesize,
                    'height': self.tilesize,
                    'scale': [tilescale, -tilescale],
                    'origin': [bounds[0], bounds[3]],
                })

                # Store tile
                RasterTile.objects.create(
                    rast=dest,
                    rasterlayer=self.rasterlayer,
                    filename=self.rastername,
                    tilex=tilex,
                    tiley=tiley,
                    tilez=zoom
                )

    def drop_empty_rasters(self):
        """
        Remove rasters that are only no-data from the current rasterlayer.
        """
        self.log('Dropping empty raster tiles.')

        # Setup SQL command
        sql = (
            "DELETE FROM raster_rastertile "
            "WHERE ST_Count(rast)=0 "
            "AND rasterlayer_id={0}"
        ).format(self.rasterlayer.id)

        # Run SQL to drop empty tiles
        cursor = connection.cursor()
        cursor.execute(sql)

    def parse_raster_layer(self):
        """
        This function pushes the raster data from the Raster Layer into the
        RasterTile table.
        """
        try:
            # Clean previous parse log
            self.log('Started parsing raster file', reset=True)

            # Send signal for start of parsing
            rasterlayers_parser_started.send(sender=self.rasterlayer.__class__, instance=self.rasterlayer)

            # Download, unzip and open raster file
            self.get_raster_file()
            self.open_raster_file()

            # Remove existing tiles for this layer before loading new ones
            self.rasterlayer.rastertile_set.all().delete()

            # Transform raster to global srid
            if self.dataset.srs.srid == WEB_MERCATOR_SRID:
                self.log('Dataset already in SRID {0}, skipping transform'.format(WEB_MERCATOR_SRID))
            else:
                self.log('Transforming raster to SRID {0}'.format(WEB_MERCATOR_SRID))
                self.dataset = self.dataset.transform(WEB_MERCATOR_SRID)

            # Compute max zoom based on scale of input raster
            max_zoom = tiler.closest_zoomlevel(
                abs(self.dataset.scale.x),
                self.zoomdown
            )

            # Loop through all lower zoom levels and create tiles to
            # setup TMS aligned tiles in world mercator
            for iz in range(max_zoom + 1):
                self.create_tiles(iz)

            self.drop_empty_rasters()

            # Log success of parsing
            self.log('Successfully finished parsing raster')
        except:
            self.log(traceback.format_exc())
            raise
        finally:
            shutil.rmtree(self.tmpdir)
<commit_msg>Reverted back to file based parser to compare speed<commit_after>import datetime
import glob
import os
import shutil
import tempfile
import traceback
import zipfile

from django.conf import settings
from django.contrib.gis.gdal import GDALRaster
from django.db import connection
from django.dispatch import Signal
from raster import tiler
from raster.const import WEB_MERCATOR_SRID, WEB_MERCATOR_TILESIZE
from raster.models import RasterLayerMetadata, RasterTile

rasterlayers_parser_started = Signal(providing_args=['instance'])

class RasterLayerParser(object):
    """
    Class to parse raster layers.
    """
    def __init__(self, rasterlayer):
        self.rasterlayer = rasterlayer

        self.rastername = os.path.basename(rasterlayer.rasterfile.name)

        # Set raster tilesize
        self.tilesize = int(getattr(settings, 'RASTER_TILESIZE', WEB_MERCATOR_TILESIZE))
        self.zoomdown = getattr(settings, 'RASTER_ZOOM_NEXT_HIGHER', True)

    def log(self, msg, reset=False):
        """
        Write a message to the parse log of the rasterlayer instance.
        """
        # Prepare datetime stamp for log
        now = '[{0}] '.format(datetime.datetime.now().strftime('%Y-%m-%d %T'))

        # Write log, reset if requested
        if reset:
            self.rasterlayer.parse_log = now + msg
        else:
            self.rasterlayer.parse_log += '\n' + now + msg

        self.rasterlayer.save()

    def get_raster_file(self):
        """
        Make local copy of rasterfile, which is needed if files are stored on
        remote storage, and unzip it if necessary.
        """
        self.log('Getting raster file from storage')

        self.tmpdir = tempfile.mkdtemp()

        # Access rasterfile and store in a temp folder
        rasterfile = open(os.path.join(self.tmpdir, self.rastername), 'wb')
        for chunk in self.rasterlayer.rasterfile.chunks():
            rasterfile.write(chunk)
        rasterfile.close()

        # If the raster file is compress, decompress it
        fileName, fileExtension = os.path.splitext(self.rastername)

        if fileExtension == '.zip':

            # Open and extract zipfile
            zf = zipfile.ZipFile(os.path.join(self.tmpdir, self.rastername))
            zf.extractall(self.tmpdir)

            # Remove zipfile
            os.remove(os.path.join(self.tmpdir, self.rastername))

            # Get filelist from directory
            raster_list = glob.glob(os.path.join(self.tmpdir, "*.*"))

            # Check if only one file is found in zipfile
            if len(raster_list) > 1:
                self.log(
                    'WARNING: Found more than one file in zipfile '
                    'using only first file found. This might lead '
                    'to problems if its not a raster file.'
                )

            # Return first one as raster file
            self.rastername = os.path.basename(raster_list[0])

    def open_raster_file(self):
        """
        Open the raster file as GDALRaster and set nodata-values.
        """
        self.log('Opening raster file as GDALRaster.')

        # Open raster file
        self.dataset = GDALRaster(os.path.join(self.tmpdir, self.rastername), write=True)

        # Make sure nodata value is set from input
        if self.rasterlayer.nodata is not None:
            for band in self.dataset.bands:
                band.nodata_value = float(self.rasterlayer.nodata)

        # Store original metadata for this raster
        self.store_original_metadata()

    def store_original_metadata(self):
        """
        Exports the input raster meta data to a RasterLayerMetadata object.
        """
        lyrmeta = RasterLayerMetadata.objects.get_or_create(rasterlayer=self.rasterlayer)[0]
        lyrmeta.uperleftx = self.dataset.origin.x
        lyrmeta.uperlefty = self.dataset.origin.y
        lyrmeta.width = self.dataset.width
        lyrmeta.height = self.dataset.height
        lyrmeta.scalex = self.dataset.scale.x
        lyrmeta.scaley = self.dataset.scale.y
        lyrmeta.skewx = self.dataset.skew.x
        lyrmeta.skewy = self.dataset.skew.y
        lyrmeta.numbands = len(self.dataset.bands)
        lyrmeta.srs_wkt = self.dataset.srs.wkt
        lyrmeta.srid = self.dataset.srs.srid
        lyrmeta.save()

    def create_tiles(self, zoom):
        """
        Create tiles for this raster at the given zoomlevel.

        This routine first snaps the raster to the grid of the zoomlevel,
        then creates  the tiles from the snapped raster.
        """
        # Compute the tile x-y-z index range for the rasterlayer for this zoomlevel
        bbox = self.rasterlayer.extent()
        indexrange = tiler.tile_index_range(bbox, zoom)

        # Compute scale of tiles for this zoomlevel
        tilescale = tiler.tile_scale(zoom)

        # Count the number of tiles that are required to cover the raster at this zoomlevel
        nr_of_tiles = (indexrange[2] - indexrange[0] + 1) * (indexrange[3] - indexrange[1] + 1)

        # Create destination raster file
        self.log('Snapping dataset to zoom level {0}'.format(zoom))

        bounds = tiler.tile_bounds(indexrange[0], indexrange[1], zoom)
        sizex = (indexrange[2] - indexrange[0] + 1) * self.tilesize
        sizey = (indexrange[3] - indexrange[1] + 1) * self.tilesize
        dest_file = os.path.join(self.tmpdir, 'djangowarpedraster' + str(zoom) + '.tif')

        snapped_dataset = self.dataset.warp({
            'name': dest_file,
            'origin': [bounds[0], bounds[3]],
            'scale': [tilescale, -tilescale],
            'width': sizex,
            'height': sizey,
        })

        self.log('Creating {0} tiles for zoom {1}.'.format(nr_of_tiles, zoom))

        counter = 0
        for tilex in range(indexrange[0], indexrange[2] + 1):
            for tiley in range(indexrange[1], indexrange[3] + 1):
                # Log progress
                counter += 1
                if counter % 250 == 0:
                    self.log('{0} tiles created at zoom {1}'.format(counter, zoom))

                # Calculate raster tile origin
                bounds = tiler.tile_bounds(tilex, tiley, zoom)

                # Warp source raster into this tile (in memory)
                dest = snapped_dataset.warp({
                    'driver': 'MEM',
                    'width': self.tilesize,
                    'height': self.tilesize,
                    'origin': [bounds[0], bounds[3]],
                })

                # Store tile
                RasterTile.objects.create(
                    rast=dest,
                    rasterlayer=self.rasterlayer,
                    filename=self.rastername,
                    tilex=tilex,
                    tiley=tiley,
                    tilez=zoom
                )

    def drop_empty_rasters(self):
        """
        Remove rasters that are only no-data from the current rasterlayer.
        """
        self.log('Dropping empty raster tiles.')

        # Setup SQL command
        sql = (
            "DELETE FROM raster_rastertile "
            "WHERE ST_Count(rast)=0 "
            "AND rasterlayer_id={0}"
        ).format(self.rasterlayer.id)

        # Run SQL to drop empty tiles
        cursor = connection.cursor()
        cursor.execute(sql)

    def parse_raster_layer(self):
        """
        This function pushes the raster data from the Raster Layer into the
        RasterTile table.
        """
        try:
            # Clean previous parse log
            self.log('Started parsing raster file', reset=True)

            # Send signal for start of parsing
            rasterlayers_parser_started.send(sender=self.rasterlayer.__class__, instance=self.rasterlayer)

            # Download, unzip and open raster file
            self.get_raster_file()
            self.open_raster_file()

            # Remove existing tiles for this layer before loading new ones
            self.rasterlayer.rastertile_set.all().delete()

            # Transform raster to global srid
            if self.dataset.srs.srid == WEB_MERCATOR_SRID:
                self.log('Dataset already in SRID {0}, skipping transform'.format(WEB_MERCATOR_SRID))
            else:
                self.log('Transforming raster to SRID {0}'.format(WEB_MERCATOR_SRID))
                self.dataset = self.dataset.transform(WEB_MERCATOR_SRID)

            # Compute max zoom based on scale of input raster
            max_zoom = tiler.closest_zoomlevel(
                abs(self.dataset.scale.x),
                self.zoomdown
            )

            # Loop through all lower zoom levels and create tiles to
            # setup TMS aligned tiles in world mercator
            for iz in range(max_zoom + 1):
                self.create_tiles(iz)

            self.drop_empty_rasters()

            # Log success of parsing
            self.log('Successfully finished parsing raster')
        except:
            self.log(traceback.format_exc())
            raise
        finally:
            shutil.rmtree(self.tmpdir)
<|endoftext|>


<commit_before>import json
from urllib.parse import urljoin

import requests_cache
import lxml.html as html

BASE_URL = {
    "en": "http://na.finalfantasyxiv.com/lodestone/playguide/db/recipe/",
    "ja": "http://jp.finalfantasyxiv.com/lodestone/playguide/db/recipe/",
    "fr": "http://fr.finalfantasyxiv.com/lodestone/playguide/db/recipe/",
    "de": "http://de.finalfantasyxiv.com/lodestone/playguide/db/recipe/",
}

CLASSES = [
    "Carpenter",
    "Blacksmith",
    "Armorer",
    "Goldsmith",
    "Leatherworker",
    "Weaver",
    "Alchemist",
    "Culinarian",
]

LEVEL_RANGE = ["{0}-{1}".format(start, start + 4) for start in range(1, 60, 5)]
NUM_LEVEL_RANGES = len(LEVEL_RANGE)

session = requests_cache.CachedSession()

def parse_recipe_links_page(base_url, r):
    tree = html.fromstring(r.text)
    rel_links = tree.xpath("//div/@data-ldst-href")
    links = [urljoin(base_url, rel_link) for rel_link in rel_links]
    show_end = int(tree.xpath("//span[@class='show_end']/text()")[0])
    total = int(tree.xpath("//span[@class='total']/text()")[0])
    return links, show_end, total

def fetch_recipe_links_page(cls, level_range, page):
    base_url = BASE_URL["en"]

    params = {
        "category2": CLASSES.index(cls),
        "category3": level_range,
        "page": page,
    }

    print("Fetching {0} {1} page {2}".format(cls, LEVEL_RANGE[level_range], page))

    return parse_recipe_links_page(base_url, session.get(base_url, params=params))

def fetch_recipe_links(cls):
    links = []

    for level_range in range(0, NUM_LEVEL_RANGES):
        page = 1
        while True:
            page_links, show_end, total = fetch_recipe_links_page(cls, level_range, page)
            links += page_links
            if show_end < total:
                page += 1
            else:
                break

    return links

def fetch_recipe(url):
    print("Fetching recipe: {0}".format(url))
    r = session.get(url)
    tree = html.fromstring(r.text)
    return {
        "name" : tree.xpath("//h2[contains(@class,'item_name')]/text()")[0],
        "level" : int(tree.xpath("//div[@class='recipe_level']/span/text()")[0]),
        "difficulty" : int(tree.xpath("//dl/dt[text()='Difficulty']/following-sibling::dd[1]/text()")[0]),
        "durability" : int(tree.xpath("//dl/dt[text()='Durability']/following-sibling::dd[1]/text()")[0]),
        "maxQuality" : int(tree.xpath("//dl/dt[text()='Maximum Quality']/following-sibling::dd[1]/text()")[0]),
    }

def fetch(cls):
    return [fetch_recipe(link) for link in fetch_recipe_links(cls)]

def scrape_to_file(cls):
    recipes = fetch(cls)
    with open("out/" + cls + ".json", "wt") as db_file:
        json.dump(recipes, db_file, indent="  ")
    print("Wrote {0} recipes for {1}".format(len(recipes), cls))

def main():
    for cls in CLASSES:
        scrape_to_file(cls)

if __name__ == '__main__':
    main()
<commit_msg>Fetch recipe names in other languages.<commit_after>import json
from urllib.parse import urljoin, urlparse, urlunparse

import requests_cache
import lxml.html as html

BASE_URL = "http://na.finalfantasyxiv.com/lodestone/playguide/db/recipe/"

LANGUAGES = {
    "ja": "jp.finalfantasyxiv.com",
    "fr": "fr.finalfantasyxiv.com",
    "de": "de.finalfantasyxiv.com",
}

CLASSES = [
    "Carpenter",
    "Blacksmith",
    "Armorer",
    "Goldsmith",
    "Leatherworker",
    "Weaver",
    "Alchemist",
    "Culinarian",
]

LEVEL_RANGE = ["{0}-{1}".format(start, start + 4) for start in range(1, 60, 5)]
NUM_LEVEL_RANGES = len(LEVEL_RANGE)

session = requests_cache.CachedSession()

def parse_recipe_links_page(base_url, r):
    tree = html.fromstring(r.text)
    rel_links = tree.xpath("//div/@data-ldst-href")
    links = [urljoin(base_url, rel_link) for rel_link in rel_links]
    show_end = int(tree.xpath("//span[@class='show_end']/text()")[0])
    total = int(tree.xpath("//span[@class='total']/text()")[0])
    return links, show_end, total

def fetch_recipe_links_page(cls, level_range, page):
    params = {
        "category2": CLASSES.index(cls),
        "category3": level_range,
        "page": page,
    }

    return parse_recipe_links_page(BASE_URL, session.get(BASE_URL, params=params))

def fetch_recipe_links(cls):
    links = []

    for level_range in range(0, NUM_LEVEL_RANGES):
        page = 1
        while True:
            print("\rFetching {0} links... {1} page {2}".format(cls, LEVEL_RANGE[level_range], page), end="")
            page_links, show_end, total = fetch_recipe_links_page(cls, level_range, page)
            links += page_links
            if show_end < total:
                page += 1
            else:
                break

    print("\rFetching {0} links... done.".format(cls))

    return links

def switch_url_host(url, host):
    parsed_url = urlparse(url)
    return urlunparse((parsed_url.scheme, host, parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment))

def fetch_recipe(url):
    r = session.get(url)
    tree = html.fromstring(r.text)
    recipe = {
        "name" : { "en": tree.xpath("//h2[contains(@class,'item_name')]/text()")[0] },
        "level" : int(tree.xpath("//div[@class='recipe_level']/span/text()")[0]),
        "difficulty" : int(tree.xpath("//dl/dt[text()='Difficulty']/following-sibling::dd[1]/text()")[0]),
        "durability" : int(tree.xpath("//dl/dt[text()='Durability']/following-sibling::dd[1]/text()")[0]),
        "maxQuality" : int(tree.xpath("//dl/dt[text()='Maximum Quality']/following-sibling::dd[1]/text()")[0]),
    }
    for lang in LANGUAGES:
        url = switch_url_host(url, LANGUAGES[lang])
        r = session.get(url)
        tree = html.fromstring(r.text)
        recipe["name"][lang] = tree.xpath("//h2[contains(@class,'item_name')]/text()")[0]
    return recipe

def fetch(cls):
    recipes = []
    links = fetch_recipe_links(cls)
    for i in range(0, len(links)):
        print("\rFetching {0} recipe... {1} of {2}".format(cls, i+1, len(links)), end="")
        recipes.append(fetch_recipe(links[i]))
    print("\rFetching {0} recipes... done.".format(cls))
    return recipes

def scrape_to_file(cls):
    recipes = fetch(cls)
    with open("out/" + cls + ".json", "wt") as db_file:
        json.dump(recipes, db_file, indent="  ")
    print("Wrote {0} recipes for {1}".format(len(recipes), cls))

def main():
    for cls in CLASSES:
        scrape_to_file(cls)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># =====================================================================
#  HPClusterJobConfigurator
#  Copyright (C) 2014 by Gabriel Nützi <gnuetzi (at) gmail (dot) com>
#
#  This Source Code Form is subject to the terms of the Mozilla Public
#  License, v. 2.0. If a copy of the MPL was not distributed with this
#  file, You can obtain one at http://mozilla.org/MPL/2.0/.
# =====================================================================

import sys
if sys.version_info[0] != 3:
    print("This script is only python3 compatible!")
    exit(1)

import os, subprocess, ctypes,re,traceback,ast,copy
import json

from argparse import ArgumentParser
from attrdict import AttrMap
import xml.etree.ElementTree as ET

from HPCJobConfigurator.jobGenerators.importHelpers import ImportHelpers as iH
from HPCJobConfigurator.jobGenerators.commonFunctions import CommonFunctions as cf

class MyOptParser(ArgumentParser):
    def error(self,msg):
        self.print_help()
        raise ValueError("Error occured: " + msg)

def validateFile(validationCommands,fileList):
    valid = []
    if not isinstance(fileList,list):
        fileList = list(fileList)

    for f in fileList:
        ext = os.path.splitext(f)[1]
        if ext in validationCommand:
            command += validationCommand[ext] + " %s" % f
            try:
                cf.callProcess(command)
                valid.append(True)
            except subprocess.CalledProcessError:
                valid.append(False)
        else:
            valid.append(False)

    return valid

def main():

    parser = MyOptParser()

    parser.add_argument("-s", "--searchDir", dest="searchDir",
            help="""This is the search directory where it is looked for render output files (.tiff,.exr,.rib.gz). """, metavar="<path>", required=True)

    parser.add_argument("--pipelineSpecs", dest="pipelineSpecs", default="",
            help="""Json file with info about the pipeline, fileValidation, fileValidationTools.
                 """, metavar="<string>", required=True)

    parser.add_argument("--statusFolder", dest="statusFolder", default="",
            help="""The output status folder which contains links to files which are finished, or can be recovered.
                 """, metavar="<string>", required=False)

    parser.add_argument("--validateOnlyLastModified", dest="validateOnlyLastModified", type=cf.toBool, default=True,
            help="""The file with the moset recent modified time is only validated, all others are set to finished!.""", required=False)

    parser.add_argument("-v", "--validationFileInfo", dest="validationFileInfo",
            help="""The input xml with file status which is updated and a new info is saved under --output.""", default="", metavar="<path>", required=False)

    parser.add_argument("-o", "--output", dest="output",
            help="""The output xml which is written, which proivides validation info for each file found""", metavar="<path>", required=True)

    try:

        print("================== FileValidation ==================== ")

        opts= AttrMap(vars(parser.parse_args()))

        print("validationInput: %s" % opts.validationFileInfo)
        print("validationOutput: %s" % opts.output)

        d = cf.jsonLoad(opts.pipelineSpecs)
        pipelineTools = d["pipelineTools"]
        fileValidationSpecs = d["fileValidationSpecs"]
        fileValidationTools = d["fileValidationTools"]

        # compile all regexes
        regexes = {}
        for i,specs in enumerate(fileValidationSpecs):
            try:
                regexes[i] = re.compile(specs["regex"])
            except:
                raise ValueError("Could not compile regex: %s" % specs["regex"])

        allFiles = []
        allFilesHash = {}
        filesPerProc = {}
        # walk directory and
        for dirpath, dirs, files in os.walk(opts.searchDir,followlinks=True):

            for file in files:

                filePath = os.path.join(dirpath, file)
                # try to match path with all regexes till one matches:
                for specIdx, specs in enumerate(fileValidationSpecs):

                    m=regexes[specIdx].search(filePath)

                    # we have a file match
                    if m:

                        try:
                            processId = int(m.group("processId"))
                        except:
                            raise ValueError("Non convertable processId found in filePath %s" % filePath)

                        if processId not in filesPerProc:
                            filesPerProc[processId] = {"all" : [] , "tool" : { tool:[] for tool in pipelineTools.keys() } };

                        #make dict for this file
                        f = {}
                        # add regex groups
                        f.update(m.groupdict())

                        # add all values from d (deep copy since we want for each one a different)
                        f.update(copy.deepcopy(specs))

                        # set file status on finished, (as initial guess, validation fully determines this value)
                        f.update({"status":"finished"})

                        # format all values again with the regex results
                        f = cf.formatAll(f,m.groupdict(),exceptKeys={"regex":None})

                        # get tool of this file
                        if "tool" in f:
                            tool = f["tool"]
                            if tool not in pipelineTools.keys():
                                raise ValueError("The tool %s is not in %s!" % (tool,str(pipelineTools.keys())) )
                        else:
                            raise ValueError("You need to define a 'tool' key for %s " % str(specs))

                        # make hashes
                        if "hashString" in specs:
                            h = cf.makeUUID( specs["hashString"].format(**m.groupdict()) )
                            f["hash"] = h
                            if h not in allFilesHash:
                                allFilesHash[h] = f
                            else:
                                raise ValueError("Found files with the same hash %s, %s, this should not happen!" % (f["absPath"], allFilesHash[h]["absPath"] ) )
                        else:
                            raise ValueError("You need to define a 'hash' key for file %s " % str(specs))

                        # convert frameIdx
                        if "frameIdx" in f:
                             f["frameIdx"] = int(f["frameIdx"])
                        else:
                             raise ValueError("You need to define a 'frameIdx' key for %s (or in regex!) " % str(specs))

                        # add file to the lists
                        filesPerProc[processId]["all"].append( f )
                        filesPerProc[processId]["tool"][ tool ].append(f)

                        allFiles.append(f)

                        break

        if not allFiles:
            print("We found no files in folder: %s to validate!" % opts.searchDir)
            return 0

        # sort files according to maximal modified time of the output files for each tool and each process
        for procId, procFiles in filesPerProc.items():
            for tool,files in procFiles["tool"].items():
                filesPerProc[procId]["tool"][tool] =  sorted( files , key= lambda file : os.path.getmtime(file["absPath"]) );

        #determine files to validate
        filesToValidate = []
        for procid, procFiles in filesPerProc.items():
                if opts.validateOnlyLastModified:
                   # validate last file of all tools for each processor, to see if its ok or not, all others are valid
                   for tool, toolFiles in procFiles["tool"].items():
                       if toolFiles:
                           filesToValidate.append(toolFiles[-1])
                else:
                   filesToValidate += procFiles["all"]

        # Validate all files with the appropriate command

        for fIdx, file in enumerate(filesToValidate):
            try:
                ext = os.path.splitext(file["absPath"])[1];
                try:
                    validateCmd = fileValidationTools[ext]
                except:
                    print("No validation command found for extentsion of file: %s" % file["absPath"])
                    raise

                validateCmd = validateCmd.format(**{"file":file["absPath"]})

                try:
                    out = subprocess.check_output(validateCmd.split(" ")).decode('utf-8')
                except:
                    print("Validation command %s failed!" % validateCmd)
                    raise

                if out not in ["finished","recover"]:
                    print("Validation output %s not in list ['finished','recover']" % out)
                    raise
                else:
                    validationAttributes = {"status":out}

                filesToValidate[fIdx].update(validationAttributes);
            except:

                # file is invalid, clear this file from the list
                filesToValidate[fIdx]["status"] = "invalid";

        print("Validated files: ", "\n".join([ f["absPath"] + " --> " + f["status"] for f in filesToValidate ]) )

        # filter all empty stuff from lists:
        allFiles = list(filter(lambda x: x["status"] != "invalid" ,allFiles))
        allFilesHash = dict(filter(lambda pair: pair[1]["status"] != "invalid", allFilesHash.items()))
        del filesPerProc

        # compare with old fileInfo if we have one and update necessary files

        finalFiles = []
        deleteFiles = []

        if opts.validationFileInfo:
            f = open(opts.validationFileInfo)
            oldFileInfo = json.load(f)

            for oldFile in oldFileInfo:
                h = oldFile["hash"]
                if h not in allFilesHash:
                    # copy element to new file info
                    finalFiles.append(oldFile)
                else:
                    # we have the same hash in the new info
                    # take our new one which is better!
                    # added afterwards
                    newFile = allFilesHash[h]

                    # TODO, symbolic link and so on
                    # old one can be moved on the delete list
                    # if the new file is no symbolic link to the old file (hardlinks does not matter!)
                    if not os.path.islink(newFile["absPath"]):
                        deleteFiles.append(oldFile)

        # write out all our produced files which
        for h,file in allFilesHash.items():
            finalFiles.append(file)

        f = open(opts.output,"w+")
        cf.jsonDump(finalFiles,f,indent=4, sort_keys=True)
        f.close();

        # Renew status folder, move over new xml info
        if opts.statusFolder:

            finished = os.path.join(opts.statusFolder,"finished")
            recover = os.path.join(opts.statusFolder,"recover")

            cf.makeDirectory(finished,interact=False, defaultMakeEmpty=True)
            cf.makeDirectory(recover ,interact=False, defaultMakeEmpty=True)
            # make symlinks for all files in the apropriate folder:
            paths = {"recover": recover, "finished": finished}

            for f in finalFiles:
                h = f["hash"]
                p = os.path.relpath(f["absPath"],start=finished)
                filename = os.path.basename(p)
                head,ext = os.path.splitext(filename)

                os.symlink(p, os.path.join( paths[f["status"]] , head+"-uuid-"+h+ext ) );

        print("============================================================ ")

    except Exception as e:
        print("====================================================================")
        print("Exception occured: " + str(e))
        print("====================================================================")
        traceback.print_exc(file=sys.stdout)
        parser.print_help()
        return 1

if __name__ == "__main__":
   sys.exit(main());
<commit_msg>small add<commit_after># =====================================================================
#  HPClusterJobConfigurator
#  Copyright (C) 2014 by Gabriel Nützi <gnuetzi (at) gmail (dot) com>
#
#  This Source Code Form is subject to the terms of the Mozilla Public
#  License, v. 2.0. If a copy of the MPL was not distributed with this
#  file, You can obtain one at http://mozilla.org/MPL/2.0/.
# =====================================================================

import sys
if sys.version_info[0] != 3:
    print("This script is only python3 compatible!")
    exit(1)

import os, subprocess, ctypes,re,traceback,ast,copy
import json

from argparse import ArgumentParser
from attrdict import AttrMap
import xml.etree.ElementTree as ET

from HPCJobConfigurator.jobGenerators.importHelpers import ImportHelpers as iH
from HPCJobConfigurator.jobGenerators.commonFunctions import CommonFunctions as cf

class MyOptParser(ArgumentParser):
    def error(self,msg):
        self.print_help()
        raise ValueError("Error occured: " + msg)

def validateFile(validationCommands,fileList):
    valid = []
    if not isinstance(fileList,list):
        fileList = list(fileList)

    for f in fileList:
        ext = os.path.splitext(f)[1]
        if ext in validationCommand:
            command += validationCommand[ext] + " %s" % f
            try:
                cf.callProcess(command)
                valid.append(True)
            except subprocess.CalledProcessError:
                valid.append(False)
        else:
            valid.append(False)

    return valid

def main():

    parser = MyOptParser()

    parser.add_argument("-s", "--searchDir", dest="searchDir",
            help="""This is the search directory where it is looked for render output files (.tiff,.exr,.rib.gz). """, metavar="<path>", required=True)

    parser.add_argument("--pipelinespec", dest="pipelinespec", default="",
            help="""Json file with info about the pipeline, fileValidation, fileValidationTools.
                 """, metavar="<string>", required=True)

    parser.add_argument("--statusFolder", dest="statusFolder", default="",
            help="""The output status folder which contains links to files which are finished, or can be recovered.
                 """, metavar="<string>", required=False)

    parser.add_argument("--validateOnlyLastModified", dest="validateOnlyLastModified", type=cf.toBool, default=True,
            help="""The file with the moset recent modified time is only validated, all others are set to finished!.""", required=False)

    parser.add_argument("-v", "--validationFileInfo", dest="validationFileInfo",
            help="""The input xml with file status which is updated and a new info is saved under --output.""", default="", metavar="<path>", required=False)

    parser.add_argument("-o", "--output", dest="output",
            help="""The output xml which is written, which proivides validation info for each file found""", metavar="<path>", required=True)

    try:

        print("================== FileValidation ==================== ")

        opts= AttrMap(vars(parser.parse_args()))
        print("searchDir: %s" % opts.searchDir)
        print("validationInput: %s" % opts.validationFileInfo)
        print("validationOutput: %s" % opts.output)

        d = cf.jsonLoad(opts.pipelinespec)
        pipelineTools = d["pipelineTools"]
        fileValidationspec = d["fileValidationspec"]
        fileValidationTools = d["fileValidationTools"]

        # compile all regexes
        regexes = {}
        for i,spec in enumerate(fileValidationspec):
            try:
                regexes[i] = re.compile(spec["regex"])
            except:
                raise ValueError("Could not compile regex: %s" % spec["regex"])

        allFiles = []
        allFilesHash = {}
        filesPerProc = {}
        # walk directory and
        for dirpath, dirs, files in os.walk(opts.searchDir,followlinks=True):

            for file in files:

                filePath = os.path.join(dirpath, file)
                # try to match path with all regexes till one matches:
                for specIdx, spec in enumerate(fileValidationspec):

                    m=regexes[specIdx].search(filePath)

                    # we have a file match
                    if m:

                        try:
                            processId = int(m.group("processId"))
                        except:
                            raise ValueError("Non convertable processId found in filePath %s" % filePath)

                        if processId not in filesPerProc:
                            filesPerProc[processId] = {"allFiles" : [] , "tools" : { tool:[] for tool in pipelineTools.keys() } };

                        #make dict for this file
                        f = {}
                        # add regex groups
                        f.update(m.groupdict())

                        # add all values from the validation spec (deep copy since we want for each one a different)
                        f.update(copy.deepcopy(spec))

                        # set file status on finished, (as initial guess, validation fully determines this value)
                        f.update({"status":"finished"})

                        # format all values again with the regex results
                        f = cf.formatAll(f,m.groupdict(),exceptKeys={"regex":None})

                        # get tool of this file
                        if "tool" in f:
                            tool = f["tool"]
                            if tool not in pipelineTools.keys():
                                raise ValueError("The tool %s is not in %s!" % (tool,str(pipelineTools.keys())) )
                        else:
                            raise ValueError("You need to define a 'tool' key for %s " % str(spec))

                        # make hashes
                        if "hashString" in spec:
                            h = cf.makeUUID( spec["hashString"].format(**m.groupdict()) )
                            f["hash"] = h
                            if h not in allFilesHash:
                                allFilesHash[h] = f
                            else:
                                raise ValueError("Found files with the same hash %s, %s, this should not happen!" % (f["absPath"], allFilesHash[h]["absPath"] ) )
                        else:
                            raise ValueError("You need to define a 'hash' key for file %s " % str(spec))

                        # convert frameIdx
                        if "frameIdx" in f:
                             f["frameIdx"] = int(f["frameIdx"])
                        else:
                             raise ValueError("You need to define a 'frameIdx' key for %s (or in regex!) " % str(spec))

                        # add file to the lists
                        filesPerProc[processId]["allFiles"].append( f )
                        filesPerProc[processId]["tools"][ tool ].append(f)

                        allFiles.append(f)

                        break

        if not allFiles:
            print("We found no files in folder: %s to validate!" % opts.searchDir)
            return 0

        # sort files according to maximal modified time of the output files for each tool and each process
        for procId, procFiles in filesPerProc.items():
            for tool,files in procFiles["tools"].items():
                filesPerProc[procId]["tools"][tool] =  sorted( files , key= lambda file : os.path.getmtime(file["absPath"]) );

        #determine files to validate
        filesToValidate = []
        for procid, procFiles in filesPerProc.items():
                if opts.validateOnlyLastModified:
                   # validate last file of all tools for each processor, to see if its ok or not, all others are valid
                   for tool, toolFiles in procFiles["tools"].items():
                       if toolFiles:
                           filesToValidate.append(toolFiles[-1])
                else:
                   filesToValidate += procFiles["allFiles"]

        # Validate all files with the appropriate command

        for fIdx, file in enumerate(filesToValidate):
            try:
                ext = os.path.splitext(file["absPath"])[1];
                try:
                    validateCmd = fileValidationTools[ext]
                except:
                    print("No validation command found for extentsion of file: %s" % file["absPath"])
                    raise

                validateCmd = validateCmd.format(**{"file":file["absPath"]})

                try:
                    out = subprocess.check_output(validateCmd.split(" ")).decode('utf-8')
                except:
                    print("Validation command %s failed!" % validateCmd)
                    raise

                if out not in ["finished","recover"]:
                    print("Validation output %s not in list ['finished','recover']" % out)
                    raise
                else:
                    validationAttributes = {"status":out}

                filesToValidate[fIdx].update(validationAttributes);
            except:

                # file is invalid, clear this file from the list
                filesToValidate[fIdx]["status"] = "invalid";

        print("Validated last files of each tool in the pipeline: ", "\n".join([ f["absPath"] + " --> " + f["status"] for f in filesToValidate ]) )

        # filter all empty stuff from lists:
        allFiles = list(filter(lambda x: x["status"] != "invalid" ,allFiles))
        allFilesHash = dict(filter(lambda pair: pair[1]["status"] != "invalid", allFilesHash.items()))
        del filesPerProc

        # compare with old fileInfo if we have one and update necessary files

        finalFiles = []
        deleteFiles = []

        if opts.validationFileInfo:
            f = open(opts.validationFileInfo)
            oldFileInfo = json.load(f)

            for oldFile in oldFileInfo:
                h = oldFile["hash"]
                if h not in allFilesHash:
                    # copy element to new file info
                    finalFiles.append(oldFile)
                else:
                    # we have the same hash in the new info
                    # take our new one which is better!
                    # added afterwards
                    newFile = allFilesHash[h]

                    # TODO, symbolic link and so on
                    # old one can be moved on the delete list
                    # if the new file is no symbolic link to the old file (hardlinks does not matter!)
                    if not os.path.islink(newFile["absPath"]):
                        deleteFiles.append(oldFile)

        # write out all our produced files which
        for h,file in allFilesHash.items():
            finalFiles.append(file)

        f = open(opts.output,"w+")
        cf.jsonDump(finalFiles,f,indent=4, sort_keys=True)
        f.close();

        # Renew status folder, move over new xml info
        if opts.statusFolder:

            finished = os.path.join(opts.statusFolder,"finished")
            recover = os.path.join(opts.statusFolder,"recover")

            cf.makeDirectory(finished,interact=False, defaultMakeEmpty=True)
            cf.makeDirectory(recover ,interact=False, defaultMakeEmpty=True)
            # make symlinks for all files in the apropriate folder:
            paths = {"recover": recover, "finished": finished}

            for f in finalFiles:
                h = f["hash"]
                p = os.path.relpath(f["absPath"],start=finished)
                filename = os.path.basename(p)
                head,ext = os.path.splitext(filename)

                os.symlink(p, os.path.join( paths[f["status"]] , head+"-uuid-"+h+ext ) );

        print("============================================================ ")

    except Exception as e:
        print("====================================================================")
        print("Exception occured: " + str(e))
        print("====================================================================")
        traceback.print_exc(file=sys.stdout)
        parser.print_help()
        return 1

if __name__ == "__main__":
   sys.exit(main());
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""\
Common utilities used in the rbit extensions.

Author: Michael Mulich
Copyright (c) 2013 Rice University

This software is subject to the provisions of the GNU Lesser General
Public License Version 2.1 (LGPL).  See LICENSE.txt for details.
"""
import os
import logging
import subprocess
import requests

logger = logging.getLogger('rbit-ext')

def unpack_zip(file, working_dir=None):
    """Unpacks a zip file and returns the contents file path."""
    # Get a listing of the current directory if we are working in
    #   one. This is used later to differentiate the unpacked contents
    #   from those that were previously there.
    directory_listing = os.listdir(working_dir)

    command = ['unzip', '-n', file]
    if working_dir is not None:
        command.extend(['-d', working_dir])
    process = subprocess.Popen(command,
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               cwd=working_dir)
    logger.debug("Running: " + ' '.join(command))
    stdout, stderr = process.communicate()
    if process.returncode != 0:
        raise RuntimeError(stdout + "\n\n" + stderr)

    return [x for x in os.listdir(working_dir) if x not in directory_listing]

def get_completezip(pkg_name, version, base_uri, working_dir):
    """"Acquire the collection data from a (Plone based) Connexions
    repository in the completezip format.

    An assumption is made that the working_dir is empty. This is so that
    the unpacked contents can be discovered.

    """
    filename = "{0}-{1}.complete.zip".format(pkg_name, version)
    url = '{0}/content/{1}/{2}/complete'.format(base_uri, pkg_name, version)

    # Download the completezip
    response = requests.get(url)
    if response.status_code != 200:
        raise RuntimeError("Could not download file at '{0}' with "
                           "response ({1}):\n{2}".format(url,
                                                         response.status_code,
                                                         response.text))
    filepath = os.path.join(working_dir, filename)
    with open(filepath, 'wb') as f:
        f.write(response.content)

    unpacked_file_list = unpack_zip(filename, working_dir)
    unpacked_filename = unpacked_file_list[0]
    return unpacked_filename
<commit_msg>Opitonally unpack the zip. This doesn't affect any logic already using the function.<commit_after># -*- coding: utf-8 -*-
"""\
Common utilities used in the rbit extensions.

Author: Michael Mulich
Copyright (c) 2013 Rice University

This software is subject to the provisions of the GNU Lesser General
Public License Version 2.1 (LGPL).  See LICENSE.txt for details.
"""
import os
import logging
import subprocess
import requests

logger = logging.getLogger('rbit-ext')

def unpack_zip(file, working_dir=None):
    """Unpacks a zip file and returns the contents file path."""
    # Get a listing of the current directory if we are working in
    #   one. This is used later to differentiate the unpacked contents
    #   from those that were previously there.
    directory_listing = os.listdir(working_dir)

    command = ['unzip', '-n', file]
    if working_dir is not None:
        command.extend(['-d', working_dir])
    process = subprocess.Popen(command,
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               cwd=working_dir)
    logger.debug("Running: " + ' '.join(command))
    stdout, stderr = process.communicate()
    if process.returncode != 0:
        raise RuntimeError(stdout + "\n\n" + stderr)

    return [x for x in os.listdir(working_dir) if x not in directory_listing]

def get_completezip(pkg_name, version, base_uri, working_dir, unpack=True):
    """"Acquire the collection data from a (Plone based) Connexions
    repository in the completezip format.

    An assumption is made that the working_dir is empty. This is so that
    the unpacked contents can be discovered.

    """
    filename = "{0}-{1}.complete.zip".format(pkg_name, version)
    url = '{0}/content/{1}/{2}/complete'.format(base_uri, pkg_name, version)

    # Download the completezip
    response = requests.get(url)
    if response.status_code != 200:
        raise RuntimeError("Could not download file at '{0}' with "
                           "response ({1}):\n{2}".format(url,
                                                         response.status_code,
                                                         response.text))
    filepath = os.path.join(working_dir, filename)
    with open(filepath, 'wb') as f:
        f.write(response.content)

    if unpack is True:
        unpacked_file_list = unpack_zip(filename, working_dir)
        unpacked_filename = unpacked_file_list[0]
        return unpacked_filename
    else:
        return os.path.join(working_dir, filename)
<|endoftext|>


<commit_before>#!/usr/bin/python

try:
    import RPi.GPIO as GPIO
    PI = True
except ImportError:
    PI = False

import os
import sys
import logging
import time
import traceback

import settings
from rotary.rotary_thread import RotaryThread
#from httpd.server import Server

logger = logging.getLogger('piradio')

######################################################################
# Callbacks
######################################################################

def rotary1_left():
    logger.debug("rotary1_left")

def rotary1_right():
    logger.debug("rotary1_right")

def rotary1_push():
    logger.debug("rotary1_push")

def rotary2_left():
    logger.debug("rotary2_left")

def rotary2_right():
    logger.debug("rotary2_right")

def rotary2_push():
    logger.debug("rotary2_push")

######################################################################
# Main Loop
######################################################################

def main():
    # Setup Logging
    hdlr = logging.FileHandler(settings.LOG)
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    hdlr = logging.StreamHandler(sys.stdout)
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    if settings.DEBUG:
        logger.setLevel(logging.DEBUG)
        logger.info("DEBUG level logging")
    else:
        logger.setLevel(logging.INFO)
        logger.info("INFO level logging")
        logger.info("Starting up...")

    if PI:
        GPIO.setwarnings(False)
        GPIO.setmode(GPIO.BCM)
        # Rotary Encoder 1
        GPIO.setup(settings.ROTARY1_APin, GPIO.IN, pull_up_down=GPIO.PUD_UP)
        GPIO.setup(settings.ROTARY1_BPin, GPIO.IN, pull_up_down=GPIO.PUD_UP)
        GPIO.setup(settings.ROTARY1_SPin, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)
        # Rotary Encoder 2
        GPIO.setup(settings.ROTARY2_APin, GPIO.IN, pull_up_down=GPIO.PUD_UP)
        GPIO.setup(settings.ROTARY2_BPin, GPIO.IN, pull_up_down=GPIO.PUD_UP)
        GPIO.setup(settings.ROTARY2_SPin, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)

        rotary1_thread = None
        rotary2_thread = None
        # server = None

        loops = 0
        while True:
            try:
                loops = loops + 1
                logger.debug("Main Loop " + str(loops))

                # Make sure our rotary threads are alive and happy
                if not rotary1_thread or not rotary1_thread.is_alive():
                    logger.info("Starting rotary1 thread")
                    rotary1_thread = RotaryThread(settings.ROTARY1_APin, settings.ROTARY1_BPin, settings.ROTARY1_SPin)
                    rotary1_thread.setLeftCallback(rotary1_left)
                    rotary1_thread.setRightCallback(rotary1_right)
                    rotary1_thread.setPushCallback(rotary1_push)
                    rotary1_thread.setDaemon(True)
                    rotary1_thread.start()

                    logger.info("Starting rotary2 thread")
                    rotary2_thread = RotaryThread(settings.ROTARY2_APin, settings.ROTARY2_BPin, settings.ROTARY2_SPin)
                    rotary2_thread.setLeftCallback(rotary2_left)
                    rotary2_thread.setRightCallback(rotary2_right)
                    rotary2_thread.setPushCallback(rotary2_push)
                    rotary2_thread.setDaemon(True)
                    rotary2_thread.start()

                    # if settings.HTTP_SERVER:
                    #     if not server or not server.is_alive():
                    #         logger.info("Starting HTTP server")
                    #         server = Server(queue, logger)
                    #         server.setDaemon(True)
                    #         server.start()

                    time.sleep(15)

                    logger.warn("Exiting main thread")
            except KeyboardInterrupt:
                logger.debug("Shutting Down...")
            finally:
                # Release resource
                GPIO.cleanup()

if __name__ == '__main__':
    main()
<commit_msg>better error handeling<commit_after>#!/usr/bin/python

try:
    import RPi.GPIO as GPIO
    PI = True
except ImportError:
    PI = False

import os
import sys
import logging
import time
import traceback

import settings
from rotary.rotary_thread import RotaryThread
#from httpd.server import Server

logger = logging.getLogger('piradio')

######################################################################
# Callbacks
######################################################################

def rotary1_left():
    logger.debug("rotary1_left")

def rotary1_right():
    logger.debug("rotary1_right")

def rotary1_push():
    logger.debug("rotary1_push")

def rotary2_left():
    logger.debug("rotary2_left")

def rotary2_right():
    logger.debug("rotary2_right")

def rotary2_push():
    logger.debug("rotary2_push")

######################################################################
# Main Loop
######################################################################

def main():
    # Setup Logging
    hdlr = logging.FileHandler(settings.LOG)
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    hdlr = logging.StreamHandler(sys.stdout)
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    if settings.DEBUG:
        logger.setLevel(logging.DEBUG)
        logger.info("DEBUG level logging")
    else:
        logger.setLevel(logging.INFO)
        logger.info("INFO level logging")
        logger.info("Starting up...")

    if PI:
        GPIO.setwarnings(False)
        GPIO.setmode(GPIO.BCM)
        # Rotary Encoder 1
        GPIO.setup(settings.ROTARY1_APin, GPIO.IN, pull_up_down=GPIO.PUD_UP)
        GPIO.setup(settings.ROTARY1_BPin, GPIO.IN, pull_up_down=GPIO.PUD_UP)
        GPIO.setup(settings.ROTARY1_SPin, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)
        # Rotary Encoder 2
        GPIO.setup(settings.ROTARY2_APin, GPIO.IN, pull_up_down=GPIO.PUD_UP)
        GPIO.setup(settings.ROTARY2_BPin, GPIO.IN, pull_up_down=GPIO.PUD_UP)
        GPIO.setup(settings.ROTARY2_SPin, GPIO.IN, pull_up_down=GPIO.PUD_DOWN)

        rotary1_thread = None
        rotary2_thread = None
        # server = None

        loops = 0
        while True:
            try:
                loops = loops + 1
                logger.debug("Main Loop " + str(loops))

                # Make sure our rotary threads are alive and happy
                if not rotary1_thread or not rotary1_thread.is_alive():
                    logger.info("Starting rotary1 thread")
                    rotary1_thread = RotaryThread(settings.ROTARY1_APin, settings.ROTARY1_BPin, settings.ROTARY1_SPin)
                    rotary1_thread.setLeftCallback(rotary1_left)
                    rotary1_thread.setRightCallback(rotary1_right)
                    rotary1_thread.setPushCallback(rotary1_push)
                    rotary1_thread.setDaemon(True)
                    rotary1_thread.start()

                    logger.info("Starting rotary2 thread")
                    rotary2_thread = RotaryThread(settings.ROTARY2_APin, settings.ROTARY2_BPin, settings.ROTARY2_SPin)
                    rotary2_thread.setLeftCallback(rotary2_left)
                    rotary2_thread.setRightCallback(rotary2_right)
                    rotary2_thread.setPushCallback(rotary2_push)
                    rotary2_thread.setDaemon(True)
                    rotary2_thread.start()

                    # if settings.HTTP_SERVER:
                    #     if not server or not server.is_alive():
                    #         logger.info("Starting HTTP server")
                    #         server = Server(queue, logger)
                    #         server.setDaemon(True)
                    #         server.start()

                    time.sleep(15)

                    logger.warn("Exiting main thread")
            except KeyboardInterrupt:
                logger.debug("Shutting Down...")
    		except Exception as e:
    			logger.exception("Exception in main thread: " + str(e))
            finally:
                # Release resource
                try:
                    GPIO.cleanup()
                except Exception:
                    pass

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#
# Copyright (c) 2006-2008 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

"""
rBuild-specific errors.
"""
import os
from conary.lib import util

# make ParseError available from here as well
# pylint: disable-msg=W0611
from conary.errors import ParseError

class RbuildBaseError(RuntimeError):
    """
    B{C{RbuildBaseError}} - Base class for all rBuild exceptions.

    New exceptions should derive from a subclass of this one, for
    example C{RbuildPluginError}.

    @cvar template: A template string used when displaying the
                    exception. Must use only keyword substitution.
    @cvar params: A list of parameter names used in the above template.
    """
    template = 'An unknown error has occurred'
    params = []

    def __init__(self, *args, **kwargs):
        RuntimeError.__init__(self)
        name = self.__class__.__name__

        # Sanity check
        for param in self.params:
            assert not hasattr(self.__class__, param), ("Parameter %r "
                    "conflicts with class dictionary, name it "
                    "something else." % param)
            assert param not in self.__dict__, ("Parameter %r "
                    "conflicts with instance dictionary, name it "
                    "something else." % param)

        self._values = dict()
        if kwargs:
            # Use keyword arguments
            if args:
                raise TypeError("Exception %s cannot take both positional "
                    "and keyword arguments" % name)

            missing = set(self.params) - set(kwargs)
            if missing:
                missingStr = ", ".join("%r" % x for x in missing)
                raise TypeError("Expected argument%s %s to exception %s" % (
                    len(missing) != 1 and 's' or '', missingStr, name))

            for key, value in kwargs.iteritems():
                if key not in self.params:
                    raise TypeError("Exception %s got an unexpected "
                        "argument %r" % (name, key))
                self._values[key] = value
        else:
            # Use positional arguments
            if len(self.params) != len(args):
                raise TypeError("Exception %s takes exactly %d argument%s "
                    "(%d given)" % (name, len(self.params),
                    len(self.params) != 1 and "s" or "", len(args)))

            for name, value in zip(self.params, args):
                self._values[name] = value

    def __getattr__(self, name):
        if name in self._values:
            return self._values[name]
        raise AttributeError(name)

    def __str__(self):
        return self.template % self._values

    def __repr__(self):
        params = ', '.join('%s=%r' % (x, self._values[x])
                for x in self.params)
        return '%s(%s)' % (self.__class__.__name__, params)

class RbuildInternalError(RbuildBaseError):
    """
    B{C{RbuildInternalError}} - superclass for all errors that are not meant to be
    seen.

    Errors deriving from InternalError should never occur, but when they do
    they indicate a failure within the code to handle some unexpected case.

    Do not raise this exception directly.
    """

class RbuildError(RbuildBaseError):
    """
    B{C{RbuildError}} - Internal rBuild errors

    This error may be raised only by rBuild internals, not by plugins.
    It may be raised directly but creating a new subclass for each
    specific case is recommended.
    """
    template = "%(message)s"
    params = ['message']

class PluginError(RbuildBaseError):
    """
    B{C{RbuildPluginError}} - rBuild plugin errors

    This error may be raised only by rBuild plugins, not by rBuild
    internals, and is the superclass for more specific errors raised
    by rBuild plugins. It may be raised directly but creating a new
    subclass for each specific case is recommended.
    """
    template = "%(message)s"
    params = ['message']

#Uncomment this, possibly modifying it, when we add deprecation
#infrastructure -- https://issues.rpath.com/browse/RBLD-147
#class DeprecatedInterfaceError(RbuildBaseError):
#    """
#    B{C{DeprecatedInterfaceError}} - Interface deprecated
#
#    This interface has been deprecated and may be removed in the
#    specified version.
#    """
#    Unsupportedtemplate = ('Deprecated interface %(name)s:'
#                           ' will be removed in rBuild %(version)s')
#    params = ['name', 'version' ]

class IncompleteInterfaceError(RbuildBaseError):
    """
    B{C{IncompleteInterfaceError}} - Interface unavailable in this configuration

    This error is raised by stub functions that are not filled in;
    for example, a function that makes sense only in the context
    of filesystem operations might raise this error if called in
    the context of repository operations.
    """
    template = "Unsupported interface: %(message)s"
    params = ['message']

class BadParameterError(RbuildBaseError):
    """
    Raised when a command is given bad parameters at the command line.
    """
    template = "%(message)s"
    params = ['message']

## BEGIN Internal Errors
class InvalidAPIMethodError(RbuildInternalError):
    "Raised when an unknown API method is referenced by an internal call."
    template = "No such API method %(method)r"
    params = ['method']

class InvalidHookReturnError(RbuildInternalError):
    "Raised when a prehook returns something unexpected."
    template = ("Invalid return value from prehook %(hook)r "
            "for function %(method)r")
    params = ['hook', 'method']

class MissingPluginError(RbuildInternalError):
    """
    Raised on attempts to access a plugin that is not known to the
    C{RbuildHandle} object.
    """
    template = "Plugin %(pluginName)r is not loaded"
    params = ['pluginName']

class InternalRmakeFacadeError(RbuildError):
    """
    Raised when rMake returns results that rBuild does not know how
    to interpret.
    """
    template = "Unexpected results from rMake: %(message)r"
    params = ['message']

## END Internal Errors

## BEGIN rBuild Errors
class MissingProductStoreError(RbuildError):
    template = "Directory %(path)r does not contain a product checkout"
    params = ['path']

class MissingGroupSearchPathElementError(RbuildError):
    template = ("Group search path element "
            "%(name)s=%(version)s[%(flavor)s] was not found")
    params = ['name', 'version', 'flavor']

class MissingImageDefinitionError(RbuildError):
    template = '\n'.join((
        'Product Definition %(name)s contains no image definitions',
        'Please add at least one image to your product definition'))
    params = ['name']

class RbuilderError(RbuildError):
    template = "rBuilder error %(error)s: %(frozen)r"
    params = ['error', 'frozen']

class RbuilderUserError(RbuilderError):
    template = 'Error retrieving user details: %(error)s: %(frozen)r'

## END rBuild Errors

#: error that is output when a Python exception makes it to the command
#: line.
_ERROR_MESSAGE = '''
ERROR: An unexpected condition has occurred in rBuild.  This is
most likely due to insufficient handling of erroneous input, but
may be some other bug.  In either case, please report the error at
https://issues.rpath.com/ and attach to the issue the file
%(stackfile)s

To get a debug prompt, rerun the command with the --debug-all argument.

For more information, go to:
http://wiki.rpath.com/wiki/Conary:How_To_File_An_Effective_Bug_Report
For more debugging help, please go to #conary on freenode.net
or email conary-list@lists.rpath.com.

Error details follow:

%(filename)s:%(lineno)s
%(errtype)s: %(errmsg)s

The complete related traceback has been saved as %(stackfile)s
'''

def _findCheckoutRoot():
    """
    Find the top-level directory of the current checkout, if any.
    @return: directory name, or None if no checkout found
    """
    dirName = os.getcwd()
    for _ in range(dirName.count(os.path.sep)+1):
        if os.path.isdir(os.path.join(dirName, '.rbuild')):
            return dirName
        dirName = os.path.dirname(dirName)
    return None

def genExcepthook(*args, **kw):
    #pylint: disable-msg=C0999
    # just passes arguments through
    """
    Generates an exception handling hook that brings up a debugger.

    If the current working directory is underneath a product checkout,
    a full traceback will be output in
    C{<checkout root>/.rbuild/tracebacks/}, otherwise one will be
    output in C{/tmp}.

    Example::
        sys.excepthook = genExceptHook(debugAll=True)
    """

    #pylint: disable-msg=C0103
    # follow external convention
    def excepthook(e_type, e_value, e_traceback):
        """Exception hook wrapper"""
        checkoutRoot = _findCheckoutRoot()
        outputDir = None
        if checkoutRoot:
            #pylint: disable-msg=W0702
            # No exception type(s) specified - a generic handler is
            # warranted as this is in an exception handler.
            try:
                outputDir = checkoutRoot + '/.rbuild/tracebacks'
                if not os.path.exists(outputDir):
                    os.mkdir(outputDir, 0700)
                elif not os.access(outputDir, os.W_OK):
                    outputDir = None
            except:
                # fall back gracefully if we can't create the directory
                outputDir = None

        if outputDir:
            baseHook = util.genExcepthook(error=_ERROR_MESSAGE,
                prefix=outputDir + '/rbuild-error-', *args, **kw)
        else:
            baseHook = util.genExcepthook(error=_ERROR_MESSAGE,
                prefix='rbuild-error-', *args, **kw)

        baseHook(e_type, e_value, e_traceback)

    return excepthook

#pylint: disable-msg=C0103
# this shouldn't be upper case.
_uncatchableExceptions = (KeyboardInterrupt, SystemExit)
<commit_msg>Change exception arguments to "msg" to avoid colliding with py2.6<commit_after>#
# Copyright (c) 2006-2008 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

"""
rBuild-specific errors.
"""
import os
from conary.lib import util

# make ParseError available from here as well
# pylint: disable-msg=W0611
from conary.errors import ParseError

class RbuildBaseError(RuntimeError):
    """
    B{C{RbuildBaseError}} - Base class for all rBuild exceptions.

    New exceptions should derive from a subclass of this one, for
    example C{RbuildPluginError}.

    @cvar template: A template string used when displaying the
                    exception. Must use only keyword substitution.
    @cvar params: A list of parameter names used in the above template.
    """
    template = 'An unknown error has occurred'
    params = []

    def __init__(self, *args, **kwargs):
        RuntimeError.__init__(self)
        name = self.__class__.__name__

        # Sanity check
        for param in self.params:
            assert not hasattr(self.__class__, param), ("Parameter %r "
                    "conflicts with class dictionary, name it "
                    "something else." % param)
            assert param not in self.__dict__, ("Parameter %r "
                    "conflicts with instance dictionary, name it "
                    "something else." % param)

        self._values = dict()
        if kwargs:
            # Use keyword arguments
            if args:
                raise TypeError("Exception %s cannot take both positional "
                    "and keyword arguments" % name)

            missing = set(self.params) - set(kwargs)
            if missing:
                missingStr = ", ".join("%r" % x for x in missing)
                raise TypeError("Expected argument%s %s to exception %s" % (
                    len(missing) != 1 and 's' or '', missingStr, name))

            for key, value in kwargs.iteritems():
                if key not in self.params:
                    raise TypeError("Exception %s got an unexpected "
                        "argument %r" % (name, key))
                self._values[key] = value
        else:
            # Use positional arguments
            if len(self.params) != len(args):
                raise TypeError("Exception %s takes exactly %d argument%s "
                    "(%d given)" % (name, len(self.params),
                    len(self.params) != 1 and "s" or "", len(args)))

            for name, value in zip(self.params, args):
                self._values[name] = value

    def __getattr__(self, name):
        if name in self._values:
            return self._values[name]
        raise AttributeError(name)

    def __str__(self):
        return self.template % self._values

    def __repr__(self):
        params = ', '.join('%s=%r' % (x, self._values[x])
                for x in self.params)
        return '%s(%s)' % (self.__class__.__name__, params)

class RbuildInternalError(RbuildBaseError):
    """
    B{C{RbuildInternalError}} - superclass for all errors that are not meant to be
    seen.

    Errors deriving from InternalError should never occur, but when they do
    they indicate a failure within the code to handle some unexpected case.

    Do not raise this exception directly.
    """

class RbuildError(RbuildBaseError):
    """
    B{C{RbuildError}} - Internal rBuild errors

    This error may be raised only by rBuild internals, not by plugins.
    It may be raised directly but creating a new subclass for each
    specific case is recommended.
    """
    template = "%(msg)s"
    params = ['msg']

class PluginError(RbuildBaseError):
    """
    B{C{RbuildPluginError}} - rBuild plugin errors

    This error may be raised only by rBuild plugins, not by rBuild
    internals, and is the superclass for more specific errors raised
    by rBuild plugins. It may be raised directly but creating a new
    subclass for each specific case is recommended.
    """
    template = "%(msg)s"
    params = ['msg']

#Uncomment this, possibly modifying it, when we add deprecation
#infrastructure -- https://issues.rpath.com/browse/RBLD-147
#class DeprecatedInterfaceError(RbuildBaseError):
#    """
#    B{C{DeprecatedInterfaceError}} - Interface deprecated
#
#    This interface has been deprecated and may be removed in the
#    specified version.
#    """
#    Unsupportedtemplate = ('Deprecated interface %(name)s:'
#                           ' will be removed in rBuild %(version)s')
#    params = ['name', 'version' ]

class IncompleteInterfaceError(RbuildBaseError):
    """
    B{C{IncompleteInterfaceError}} - Interface unavailable in this configuration

    This error is raised by stub functions that are not filled in;
    for example, a function that makes sense only in the context
    of filesystem operations might raise this error if called in
    the context of repository operations.
    """
    template = "Unsupported interface: %(msg)s"
    params = ['msg']

class BadParameterError(RbuildBaseError):
    """
    Raised when a command is given bad parameters at the command line.
    """
    template = "%(msg)s"
    params = ['msg']

## BEGIN Internal Errors
class InvalidAPIMethodError(RbuildInternalError):
    "Raised when an unknown API method is referenced by an internal call."
    template = "No such API method %(method)r"
    params = ['method']

class InvalidHookReturnError(RbuildInternalError):
    "Raised when a prehook returns something unexpected."
    template = ("Invalid return value from prehook %(hook)r "
            "for function %(method)r")
    params = ['hook', 'method']

class MissingPluginError(RbuildInternalError):
    """
    Raised on attempts to access a plugin that is not known to the
    C{RbuildHandle} object.
    """
    template = "Plugin %(pluginName)r is not loaded"
    params = ['pluginName']

class InternalRmakeFacadeError(RbuildError):
    """
    Raised when rMake returns results that rBuild does not know how
    to interpret.
    """
    template = "Unexpected results from rMake: %(msg)r"
    params = ['msg']

## END Internal Errors

## BEGIN rBuild Errors
class MissingProductStoreError(RbuildError):
    template = "Directory %(path)r does not contain a product checkout"
    params = ['path']

class MissingGroupSearchPathElementError(RbuildError):
    template = ("Group search path element "
            "%(name)s=%(version)s[%(flavor)s] was not found")
    params = ['name', 'version', 'flavor']

class MissingImageDefinitionError(RbuildError):
    template = '\n'.join((
        'Product Definition %(name)s contains no image definitions',
        'Please add at least one image to your product definition'))
    params = ['name']

class RbuilderError(RbuildError):
    template = "rBuilder error %(error)s: %(frozen)r"
    params = ['error', 'frozen']

class RbuilderUserError(RbuilderError):
    template = 'Error retrieving user details: %(error)s: %(frozen)r'

## END rBuild Errors

#: error that is output when a Python exception makes it to the command
#: line.
_ERROR_MESSAGE = '''
ERROR: An unexpected condition has occurred in rBuild.  This is
most likely due to insufficient handling of erroneous input, but
may be some other bug.  In either case, please report the error at
https://issues.rpath.com/ and attach to the issue the file
%(stackfile)s

To get a debug prompt, rerun the command with the --debug-all argument.

For more information, go to:
http://wiki.rpath.com/wiki/Conary:How_To_File_An_Effective_Bug_Report
For more debugging help, please go to #conary on freenode.net
or email conary-list@lists.rpath.com.

Error details follow:

%(filename)s:%(lineno)s
%(errtype)s: %(errmsg)s

The complete related traceback has been saved as %(stackfile)s
'''

def _findCheckoutRoot():
    """
    Find the top-level directory of the current checkout, if any.
    @return: directory name, or None if no checkout found
    """
    dirName = os.getcwd()
    for _ in range(dirName.count(os.path.sep)+1):
        if os.path.isdir(os.path.join(dirName, '.rbuild')):
            return dirName
        dirName = os.path.dirname(dirName)
    return None

def genExcepthook(*args, **kw):
    #pylint: disable-msg=C0999
    # just passes arguments through
    """
    Generates an exception handling hook that brings up a debugger.

    If the current working directory is underneath a product checkout,
    a full traceback will be output in
    C{<checkout root>/.rbuild/tracebacks/}, otherwise one will be
    output in C{/tmp}.

    Example::
        sys.excepthook = genExceptHook(debugAll=True)
    """

    #pylint: disable-msg=C0103
    # follow external convention
    def excepthook(e_type, e_value, e_traceback):
        """Exception hook wrapper"""
        checkoutRoot = _findCheckoutRoot()
        outputDir = None
        if checkoutRoot:
            #pylint: disable-msg=W0702
            # No exception type(s) specified - a generic handler is
            # warranted as this is in an exception handler.
            try:
                outputDir = checkoutRoot + '/.rbuild/tracebacks'
                if not os.path.exists(outputDir):
                    os.mkdir(outputDir, 0700)
                elif not os.access(outputDir, os.W_OK):
                    outputDir = None
            except:
                # fall back gracefully if we can't create the directory
                outputDir = None

        if outputDir:
            baseHook = util.genExcepthook(error=_ERROR_MESSAGE,
                prefix=outputDir + '/rbuild-error-', *args, **kw)
        else:
            baseHook = util.genExcepthook(error=_ERROR_MESSAGE,
                prefix='rbuild-error-', *args, **kw)

        baseHook(e_type, e_value, e_traceback)

    return excepthook

#pylint: disable-msg=C0103
# this shouldn't be upper case.
_uncatchableExceptions = (KeyboardInterrupt, SystemExit)
<|endoftext|>


<commit_before>import os
import sys
import logging
import time
import traceback
import RPi.GPIO as GPIO

import mpd
from PIL import Image
from PIL import ImageDraw
from Adafruit_LED_Backpack import BicolorMatrix8x8

import settings
from rotary.rotary_thread import RotaryThread
#from httpd.server import Server

logger = logging.getLogger('piradio')

GPIO.setmode(GPIO.BCM)
GPIO.setwarnings(False)

# display = BicolorMatrix8x8.BicolorMatrix8x8()
# display.begin()

######################################################################
# Display Functions
######################################################################

def fill_display(r, g):
    display.clear()
    image = Image.new('RGB', (8, 8))
    draw = ImageDraw.Draw(image)
    draw.rectangle((0, 0, 7, 7), fill=(r, g, 0))
    display.set_image(image)
    display.write_display()

def drawPixel(x, y, c):
    display.clear()
    display.set_pixel(x, y, c)
    display.write_display()

def splash():
    fill_display(255, 0)
    time.sleep(1)
    fill_display(0, 255)
    time.sleep(1)
    fill_display(255, 255)
    time.sleep(1)
    display.clear()
    display.write_display()

######################################################################
# MPD Functions
######################################################################

def getMPDClient():
    client = mpd.MPDClient(use_unicode=True)
    client.connect("localhost", 6600)
    return client

######################################################################
# Callbacks
######################################################################

def rotary1_left():
    logger.debug("rotary1_left")
    client = getMPDClient()
    logger.debug("[ PREVIOUS ]")
    client.previous()
    client.close()

def rotary1_right():
    logger.debug("rotary1_right")
    client = getMPDClient()
    logger.debug("[ NEXT ]")
    client.next()
    client.close()

def rotary1_push(ev=None):
    logger.debug("rotary1_push")
    client = getMPDClient()
    logger.debug("[ PAUSE ]")
    client.pause()
    client.close()
    time.sleep(1)

def rotary2_left():
    logger.debug("rotary2_left")
    client = getMPDClient()
    status = client.status()
    volume = int(status['volume']) - 1
    logger.debug("[ VOLUME %d ]" % volume)
    client.setvol(volume)
    client.close()

def rotary2_right():
    logger.debug("rotary2_right")
    client = getMPDClient()
    status = client.status()
    volume = int(status['volume']) + 1
    logger.debug("[ VOLUME %d ]" % volume)
    client.setvol(volume)
    client.close()

def rotary2_push(ev=None):
    logger.debug("rotary2_push")
    client = getMPDClient()
    logger.debug("[ PAUSE ]")
    client.pause()
    client.close()
    time.sleep(1)

######################################################################
# Main Loop
######################################################################

def main():
    # Setup Logging
    hdlr = logging.FileHandler(settings.LOG)
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    hdlr = logging.StreamHandler(sys.stdout)
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    if settings.DEBUG:
        logger.setLevel(logging.DEBUG)
        logger.info("DEBUG level logging")
    else:
        logger.setLevel(logging.INFO)
        logger.info("INFO level logging")
        logger.info("Starting up...")

    # Splash the display
    # splash()

    rotary1_thread = None
    rotary2_thread = None
    # server = None

    loops = 0
    try:
        while True:
            loops = loops + 1
            logger.debug("Main Loop " + str(loops))

            # Make sure our rotary threads are alive and happy
            if not rotary1_thread or not rotary1_thread.is_alive():
                logger.info("Starting rotary1 thread")
                rotary1_thread = RotaryThread("Rotary1", settings.ROTARY1_APin, settings.ROTARY1_BPin, settings.ROTARY1_SPin, logger)
                rotary1_thread.setLeftCallback(rotary1_left)
                rotary1_thread.setRightCallback(rotary1_right)
                rotary1_thread.setPushCallback(rotary1_push)
                rotary1_thread.start()

            if not rotary2_thread or not rotary2_thread.is_alive():
                logger.info("Starting rotary2 thread")
                rotary2_thread = RotaryThread("Rotary2", settings.ROTARY2_APin, settings.ROTARY2_BPin, settings.ROTARY2_SPin, logger)
                rotary2_thread.setLeftCallback(rotary2_left)
                rotary2_thread.setRightCallback(rotary2_right)
                rotary2_thread.setPushCallback(rotary2_push)
                rotary2_thread.start()

            # if settings.HTTP_SERVER:
            #     if not server or not server.is_alive():
            #         logger.info("Starting HTTP server")
            #         server = Server(queue, logger)
            #         server.setDaemon(True)
            #         server.start()

            time.sleep(15)
    except KeyboardInterrupt:
        logger.info("Keyboard Interuption!")
    except Exception as e:
        logger.exception(e)
    finally:
        logger.debug("Shutting Down...")
        if rotary1_thread:
            rotary1_thread.stop()
            rotary1_thread.join()
        if rotary2_thread:
            rotary2_thread.stop()
            rotary2_thread.join()

    # Release GPIO
    GPIO.cleanup()

if __name__ == '__main__':
    main()
<commit_msg>reversed volume<commit_after>import os
import sys
import logging
import time
import traceback
import RPi.GPIO as GPIO

import mpd
from PIL import Image
from PIL import ImageDraw
from Adafruit_LED_Backpack import BicolorMatrix8x8

import settings
from rotary.rotary_thread import RotaryThread
#from httpd.server import Server

logger = logging.getLogger('piradio')

GPIO.setmode(GPIO.BCM)
GPIO.setwarnings(False)

# display = BicolorMatrix8x8.BicolorMatrix8x8()
# display.begin()

######################################################################
# Display Functions
######################################################################

def fill_display(r, g):
    display.clear()
    image = Image.new('RGB', (8, 8))
    draw = ImageDraw.Draw(image)
    draw.rectangle((0, 0, 7, 7), fill=(r, g, 0))
    display.set_image(image)
    display.write_display()

def drawPixel(x, y, c):
    display.clear()
    display.set_pixel(x, y, c)
    display.write_display()

def splash():
    fill_display(255, 0)
    time.sleep(1)
    fill_display(0, 255)
    time.sleep(1)
    fill_display(255, 255)
    time.sleep(1)
    display.clear()
    display.write_display()

######################################################################
# MPD Functions
######################################################################

def getMPDClient():
    client = mpd.MPDClient(use_unicode=True)
    client.connect("localhost", 6600)
    return client

######################################################################
# Callbacks
######################################################################

def rotary1_left():
    logger.debug("rotary1_left")
    # client = getMPDClient()
    # logger.debug("[ PREVIOUS ]")
    # client.previous()
    # client.close()

def rotary1_right():
    logger.debug("rotary1_right")
    # client = getMPDClient()
    # logger.debug("[ NEXT ]")
    # client.next()
    # client.close()

def rotary1_push(ev=None):
    logger.debug("rotary1_push")
    client = getMPDClient()
    logger.debug("[ PAUSE ]")
    client.pause()
    client.close()
    time.sleep(1)

def rotary2_left():
    logger.debug("rotary2_left")
    client = getMPDClient()
    status = client.status()
    volume = int(status['volume']) + 1
    logger.debug("[ VOLUME %d ]" % volume)
    client.setvol(volume)
    client.close()

def rotary2_right():
    logger.debug("rotary2_right")
    client = getMPDClient()
    status = client.status()
    volume = int(status['volume']) - 1
    logger.debug("[ VOLUME %d ]" % volume)
    client.setvol(volume)
    client.close()

def rotary2_push(ev=None):
    logger.debug("rotary2_push")
    client = getMPDClient()
    logger.debug("[ PAUSE ]")
    client.pause()
    client.close()
    time.sleep(1)

######################################################################
# Main Loop
######################################################################

def main():
    # Setup Logging
    hdlr = logging.FileHandler(settings.LOG)
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    hdlr = logging.StreamHandler(sys.stdout)
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    if settings.DEBUG:
        logger.setLevel(logging.DEBUG)
        logger.info("DEBUG level logging")
    else:
        logger.setLevel(logging.INFO)
        logger.info("INFO level logging")
        logger.info("Starting up...")

    # Splash the display
    # splash()

    rotary1_thread = None
    rotary2_thread = None
    # server = None

    loops = 0
    try:
        while True:
            loops = loops + 1
            logger.debug("Main Loop " + str(loops))

            # Make sure our rotary threads are alive and happy
            if not rotary1_thread or not rotary1_thread.is_alive():
                logger.info("Starting rotary1 thread")
                rotary1_thread = RotaryThread("Rotary1", settings.ROTARY1_APin, settings.ROTARY1_BPin, settings.ROTARY1_SPin, logger)
                rotary1_thread.setLeftCallback(rotary1_left)
                rotary1_thread.setRightCallback(rotary1_right)
                rotary1_thread.setPushCallback(rotary1_push)
                rotary1_thread.start()

            if not rotary2_thread or not rotary2_thread.is_alive():
                logger.info("Starting rotary2 thread")
                rotary2_thread = RotaryThread("Rotary2", settings.ROTARY2_APin, settings.ROTARY2_BPin, settings.ROTARY2_SPin, logger)
                rotary2_thread.setLeftCallback(rotary2_left)
                rotary2_thread.setRightCallback(rotary2_right)
                rotary2_thread.setPushCallback(rotary2_push)
                rotary2_thread.start()

            # if settings.HTTP_SERVER:
            #     if not server or not server.is_alive():
            #         logger.info("Starting HTTP server")
            #         server = Server(queue, logger)
            #         server.setDaemon(True)
            #         server.start()

            time.sleep(15)
    except KeyboardInterrupt:
        logger.info("Keyboard Interuption!")
    except Exception as e:
        logger.exception(e)
    finally:
        logger.debug("Shutting Down...")
        if rotary1_thread:
            rotary1_thread.stop()
            rotary1_thread.join()
        if rotary2_thread:
            rotary2_thread.stop()
            rotary2_thread.join()

    # Release GPIO
    GPIO.cleanup()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import sublime
import sublime_plugin
import os
import re

class RBoxMainMenuListener(sublime_plugin.EventListener):

    def should_show_menu(self, view):

        sel = [s for s in view.sel()]
        point = sel[0].end() if len(sel) > 0 else 0
        score = view.score_selector(
            point,
            "source.r, "
            "text.tex.latex.rsweave, "
            "text.html.markdown.rmarkdown, "
            "source.c++.rcpp")

        if score > 0:
            return True

        r = re.compile(".*\.Rproj$")
        try:
            pd = view.window().project_data()
            first_folder = pd["folders"][0]["path"]
            for f in os.listdir(first_folder):
                if r.match(f):
                    return True
        except:
            pass

        return False

    def on_activated_async(self, view):
        if view.is_scratch() or view.settings().get('is_widget'):
            return

        targetpath = os.path.join(
            sublime.packages_path(),
            'User', 'R-Box', 'Main.sublime-menu')
        targetdir = os.path.dirname(targetpath)

        if self.should_show_menu(view):

            if not os.path.exists(targetdir):
                os.makedirs(targetdir, 0o755)

            if not os.path.exists(targetpath):
                data = sublime.load_binary_resource(
                    "Packages/R-Box/support/R-Box.sublime-menu").decode("utf-8")
                with open(targetpath, 'w') as binfile:
                    binfile.write(data)
                    binfile.close()
        else:
            if os.path.exists(targetpath):
                os.remove(targetpath)

class RBoxMainMenuClearWorkspace(sublime_plugin.TextCommand):
    def run(self, edit):
        ok = sublime.ok_cancel_dialog("Clear R Workspace?")
        if ok:
            self.view.run_command("send_code", {"cmd": "rm(list=ls())"})
<commit_msg>use match_selector<commit_after>import sublime
import sublime_plugin
import os
import re

class RBoxMainMenuListener(sublime_plugin.EventListener):

    def should_show_menu(self, view):

        try:
            pt = view.sel()[0].end()
        except:
            pt = 0

        if view.match_selector(
                pt,
                "source.r, "
                "text.tex.latex.rsweave, "
                "text.html.markdown.rmarkdown, "
                "source.c++.rcpp"):
            return True

        r = re.compile(".*\.Rproj$")
        try:
            pd = view.window().project_data()
            first_folder = pd["folders"][0]["path"]
            for f in os.listdir(first_folder):
                if r.match(f):
                    return True
        except:
            pass

        return False

    def on_activated_async(self, view):
        if view.settings().get('is_widget'):
            return

        targetpath = os.path.join(
            sublime.packages_path(),
            'User', 'R-Box', 'Main.sublime-menu')
        targetdir = os.path.dirname(targetpath)

        if self.should_show_menu(view):

            if not os.path.exists(targetdir):
                os.makedirs(targetdir, 0o755)

            if not os.path.exists(targetpath):
                data = sublime.load_binary_resource(
                    "Packages/R-Box/support/R-Box.sublime-menu").decode("utf-8")
                with open(targetpath, 'w') as binfile:
                    binfile.write(data)
                    binfile.close()
        else:
            if os.path.exists(targetpath):
                os.remove(targetpath)

class RBoxMainMenuClearWorkspace(sublime_plugin.TextCommand):
    def run(self, edit):
        ok = sublime.ok_cancel_dialog("Clear R Workspace?")
        if ok:
            self.view.run_command("send_code", {"cmd": "rm(list=ls())"})
<|endoftext|>