<commit_before>#!/usr/bin/python

# This code takes nested data structures and flattens them into groups, keyed
# on id fields, and linked to each other by id values. You must first set some
# values to define the output keys and any entities that you want to flatten.
# Nested data that is not defined to be flattened will be left in place

# TODO:
# - support field rename and remove
# - license
# - benchmark performance
# - add unit tests

class Normalize_Base:

    def __init__(self):
        '''init data'''

        self.entities = {}
        self.entity_order = []
        self.remove_fldvals = {}
        self.rename_fldvals = {}

    def _set_nested_id(self, data, key, idval):
        '''recursively replace nested data with an id'''

        for index in data:
            if index == key:
                data[index] = idval;
                return
            if isinstance(data[index], dict):
                return self._set_nested_id(data[index], key, idval)
        return

    def _search_dict(self, data, key):
        '''recursive search a dict for a key'''

        for index in data:
            if index == key:
                return data[index]
            if isinstance(data[index], dict):
                return self._search_dict(data[index], key)
        return None

    def _get_entity_depth(self, entity, data, depth=1):
        '''determine the nesting depth of a key'''

        for index in data:
            if index == entity:
                return depth
            if isinstance(data[index], dict):
                depth += 1
                return self._get_entity_depth(entity, data[index], depth)
        return None

    def _base_data(self):
        '''setup the basic wrapper around the newly normalized data'''

        name = self.entities.keys()[0]
        id_key = self.entities[name]['id']
        new_data = {'results': [], 'entities': {name: {}}}
        for entity in self.entities[name]['entities']:
            new_data['entities'][entity] = {}
        return (name, id_key, new_data)

    def _get_entity_order(self, name, row):
        '''determine entity depth order from the first row of data'''

        data = []
        for entity in self.entities[name]['entities']:
            key = self.entities[name]['entities'][entity]['key']
            level = self._get_entity_depth(key, row)
            data.append([level, entity])
        data.sort(key=lambda x:x[0])
        self.entity_order = [v[1] for v in data]
        self.entity_order.reverse()

    def _process_data_changes(self, entity, data):
        data = self._process_remove(entity, data)
        data = self._process_rename(entity, data)
        return data

    def _process_rename(self, entity, data):
        if entity in self.rename_fldvals:
            for fld in self.rename_fldvals[entity]:
                if fld[0] in data:
                    data[fld[1]] = data[fld[0]]
                    del data[fld[0]]
        return data

    def _process_remove(self, entity, data):
        if entity in self.remove_fldvals:
            for fld in self.remove_fldvals[entity]:
                if fld in data:
                    del data[fld]
        return data

class Normalize(Normalize_Base):

    def define_primary(self, name, id_fld='id'):
        '''used to define the top level entity name used in the results'''

        if len(self.entities):
            raise ValueError('Only one primary entity is allowed')
        self.entities[name] = {'id': id_fld, 'entities': {}}

    def set_entity_order(self, order):
        '''set the nested depth order (deepest first)'''

        self.entity_order = order

    def define_nested_entity(self, name, keyval, id_fld='id'):
        '''set a nested entity to be flattend'''

        if not len(self.entities):
            raise ValueError('You must set the primary first')
        self.entities[self.entities.keys()[0]]['entities'][name] = {
            'id': id_fld, 'key': keyval}

    def remove_flds(self, entity, flds):
        '''remove an array of fields from a defined entity'''

        if entity in self.remove_fldvals:
            self.remove_fldvals[entity].append(flds)
        else:
            self.remove_fldvals[entity] = [flds]

    def rename_flds(self, entity, name, new_name):
        '''rename a field for an entity'''

        if entity in self.rename_fldvals:
            self.rename_fldvals[entity].append((name, new_name))
        else:
            self.rename_fldvals[entity] = [(name, new_name)]

    def parse(self, data):
        '''convert data'''

        if not data:
            return None
        name, id_key, new_data = self._base_data()
        if not self.entity_order:
            self._get_entity_order(name, data[0])
        for entry in data:
            if id_key not in entry:
                raise ValueError('Id key "%s" missing from data' % id_key)

            for entity in self.entity_order:
                entity_id = self.entities[name]['entities'][entity]['id']
                entity_key = self.entities[name]['entities'][entity]['key']
                match = self._search_dict(entry, entity_key)
                if not match or entity_id not in match:
                    continue

                match = self._process_data_changes(entity, match)
                new_data['entities'][entity][match[entity_id]] = match
                self._set_nested_id(entry, entity_key, match[entity_id])

            entry = self._process_data_changes(name, entry)
            new_data['entities'][name][entry[id_key]] = entry
            new_data['results'].append(entry[id_key])
        return new_data
<commit_msg>todo update<commit_after>#!/usr/bin/python

# This code takes nested data structures and flattens them into groups, keyed
# on id fields, and linked to each other by id values. You must first set some
# values to define the output keys and any entities that you want to flatten.
# Nested data that is not defined to be flattened will be left in place

class Normalize_Base:

    def __init__(self):
        '''init data'''

        self.entities = {}
        self.entity_order = []
        self.remove_fldvals = {}
        self.rename_fldvals = {}

    def _set_nested_id(self, data, key, idval):
        '''recursively replace nested data with an id'''

        for index in data:
            if index == key:
                data[index] = idval;
                return
            if isinstance(data[index], dict):
                return self._set_nested_id(data[index], key, idval)
        return

    def _search_dict(self, data, key):
        '''recursive search a dict for a key'''

        for index in data:
            if index == key:
                return data[index]
            if isinstance(data[index], dict):
                return self._search_dict(data[index], key)
        return None

    def _get_entity_depth(self, entity, data, depth=1):
        '''determine the nesting depth of a key'''

        for index in data:
            if index == entity:
                return depth
            if isinstance(data[index], dict):
                depth += 1
                return self._get_entity_depth(entity, data[index], depth)
        return None

    def _base_data(self):
        '''setup the basic wrapper around the newly normalized data'''

        name = self.entities.keys()[0]
        id_key = self.entities[name]['id']
        new_data = {'results': [], 'entities': {name: {}}}
        for entity in self.entities[name]['entities']:
            new_data['entities'][entity] = {}
        return (name, id_key, new_data)

    def _get_entity_order(self, name, row):
        '''determine entity depth order from the first row of data'''

        data = []
        for entity in self.entities[name]['entities']:
            key = self.entities[name]['entities'][entity]['key']
            level = self._get_entity_depth(key, row)
            data.append([level, entity])
        data.sort(key=lambda x:x[0])
        self.entity_order = [v[1] for v in data]
        self.entity_order.reverse()

    def _process_data_changes(self, entity, data):
        data = self._process_remove(entity, data)
        data = self._process_rename(entity, data)
        return data

    def _process_rename(self, entity, data):
        if entity in self.rename_fldvals:
            for fld in self.rename_fldvals[entity]:
                if fld[0] in data:
                    data[fld[1]] = data[fld[0]]
                    del data[fld[0]]
        return data

    def _process_remove(self, entity, data):
        if entity in self.remove_fldvals:
            for fld in self.remove_fldvals[entity]:
                if fld in data:
                    del data[fld]
        return data

class Normalize(Normalize_Base):

    def define_primary(self, name, id_fld='id'):
        '''used to define the top level entity name used in the results'''

        if len(self.entities):
            raise ValueError('Only one primary entity is allowed')
        self.entities[name] = {'id': id_fld, 'entities': {}}

    def set_entity_order(self, order):
        '''set the nested depth order (deepest first)'''

        self.entity_order = order

    def define_nested_entity(self, name, keyval, id_fld='id'):
        '''set a nested entity to be flattend'''

        if not len(self.entities):
            raise ValueError('You must set the primary first')
        self.entities[self.entities.keys()[0]]['entities'][name] = {
            'id': id_fld, 'key': keyval}

    def remove_flds(self, entity, flds):
        '''remove an array of fields from a defined entity'''

        if entity in self.remove_fldvals:
            self.remove_fldvals[entity].append(flds)
        else:
            self.remove_fldvals[entity] = [flds]

    def rename_flds(self, entity, name, new_name):
        '''rename a field for an entity'''

        if entity in self.rename_fldvals:
            self.rename_fldvals[entity].append((name, new_name))
        else:
            self.rename_fldvals[entity] = [(name, new_name)]

    def parse(self, data):
        '''convert data'''

        if not data:
            return None
        name, id_key, new_data = self._base_data()
        if not self.entity_order:
            self._get_entity_order(name, data[0])
        for entry in data:
            if id_key not in entry:
                raise ValueError('Id key "%s" missing from data' % id_key)

            for entity in self.entity_order:
                entity_id = self.entities[name]['entities'][entity]['id']
                entity_key = self.entities[name]['entities'][entity]['key']
                match = self._search_dict(entry, entity_key)
                if not match or entity_id not in match:
                    continue

                match = self._process_data_changes(entity, match)
                new_data['entities'][entity][match[entity_id]] = match
                self._set_nested_id(entry, entity_key, match[entity_id])

            entry = self._process_data_changes(name, entry)
            new_data['entities'][name][entry[id_key]] = entry
            new_data['results'].append(entry[id_key])
        return new_data
<|endoftext|>


<commit_before>class Null(object):
    from inspect import isclass
    #test! TESTING!
    def __init__(self, *args, **kwargs):
        """Ignore parameters."""
        return None

    # object calling
    def __call__(self, *args, **kwargs):
        """Ignore method calls."""
        return self

    # attribute handling
    def __getattr__(self, name):
        """Ignore attribute requests."""
        return self
    def __setattr__(self, name, value):
        """Ignore attribute setting."""
        return self
    def __delattr__(self, name):
        """Ignore deleting attributes."""
        return self

    # misc.
    def __repr__(self):
        """Return a string representation."""
        return "<Null>"
    def __str__(self):
        """Convert to a string and return it."""
        return """Null"""
    def __dir__(self):
        return self

    # math
    def __add__(self, other):
        """Return only other"""
        return other
    def __sub__(self, other):
        """Return only other"""
        return other
    def __radd__(self, other):
        """Return only other"""
        return other
    def __rsub__(self, other):
        """Return only other"""
        return other
    def __cmp__(self, other):
        """Only return True if other is Null"""
        if isinstance(self, type(other)):
            return 0
        else:
            return -1
    def __nonzero__(self):
        """Always return False"""
        return False<commit_msg>Testing Keys<commit_after>class Null(object):
    from inspect import isclass
    #test!
    def __init__(self, *args, **kwargs):
        """Ignore parameters."""
        return None

    # object calling
    def __call__(self, *args, **kwargs):
        """Ignore method calls."""
        return self

    # attribute handling
    def __getattr__(self, name):
        """Ignore attribute requests."""
        return self
    def __setattr__(self, name, value):
        """Ignore attribute setting."""
        return self
    def __delattr__(self, name):
        """Ignore deleting attributes."""
        return self

    # misc.
    def __repr__(self):
        """Return a string representation."""
        return "<Null>"
    def __str__(self):
        """Convert to a string and return it."""
        return """Null"""
    def __dir__(self):
        return self

    # math
    def __add__(self, other):
        """Return only other"""
        return other
    def __sub__(self, other):
        """Return only other"""
        return other
    def __radd__(self, other):
        """Return only other"""
        return other
    def __rsub__(self, other):
        """Return only other"""
        return other
    def __cmp__(self, other):
        """Only return True if other is Null"""
        if isinstance(self, type(other)):
            return 0
        else:
            return -1
    def __nonzero__(self):
        """Always return False"""
        return False<|endoftext|>


<commit_before>__author__ = 'zelgadis'

import bpy
import sys

UPDATE = "update"
WIDTH = "width"
HEIGHT = "height"
STEREO_CAMERA = "camera"
AUDIOFILE = "audiofile"
FORMAT = "format"
CYCLES_SAMPLES = "cycles_samples"
PRERENDER_COUNT = "prerender_count"
GPU_DEVICE = "gpu_device"

params = {UPDATE: False,
          WIDTH: 480,
          HEIGHT: 270,
          STEREO_CAMERA: "",
          AUDIOFILE:"/tmp/renderchan-test.wav",
          FORMAT: "png",
          CYCLES_SAMPLES: None,
          PRERENDER_COUNT: 0,
          GPU_DEVICE: ""}

def find_camera(cameras, base, prefixes, suffixes, side_index = 0):
    for camera in cameras:
        for prefix in prefixes:
            if camera.name.lower().startswith(prefix[side_index]) and camera.name[len(prefix[side_index]):] == base:
                return camera
        for suffix in suffixes:
            if camera.name.lower().endswith(suffix[side_index]) and camera.name[:-len(suffix[side_index])] == base:
                return camera
    return None

def main():

    update = params[UPDATE]

    sce = bpy.context.scene

    sce.frame_current=sce.frame_current+1
    sce.frame_current=sce.frame_current-1

    uses_builtin_stereo = "use_multiview" in dir(sce.render) and sce.render.use_multiview

    if params[STEREO_CAMERA] != "" and not uses_builtin_stereo:
        found=False

        for ob in sce.objects:
            if ob.name == params[STEREO_CAMERA] and ob.type == 'CAMERA':
                sce.camera = ob
                found=True
                break

        if not found:
            separators = ['_', '.', '-', ' ']

            side = params[STEREO_CAMERA]
            alt_side = "left" if side == "right" else "right"
            sides = [side, side[0]]
            alt_sides = [alt_side, alt_side[0]]

            prefixes = [(sides[side_index] + sep, alt_sides[side_index] + sep) for sep in separators for side_index in range(0, len(sides))]
            prefixes.append((side, alt_side))

            suffixes = [(sep + sides[side_index], sep + alt_sides[side_index]) for sep in separators for side_index in range(0, len(sides))]
            suffixes.append((side, alt_side))

            cameras = [obj for obj in sce.objects if obj.type == "CAMERA"]

            if len(cameras) < 1:
                print("Error: Cannot render, no camera in file " + bpy.data.filepath, file=sys.stderr)
                exit(1)
            elif len(cameras) == 1:
                print("Warning: Stereo specified, but only one camera found. Using camera for both sides.")
                sce.camera = cameras[0]
                found = True
            else:
                selected_camera = find_camera(cameras, sce.camera.name, prefixes, suffixes)

                base = None
                if not selected_camera:
                    for prefix in prefixes:
                        if sce.camera.name.lower().startswith(prefix[0]):
                            base = sce.camera.name[len(prefix[0]):]
                            break
                        if sce.camera.name.lower().startswith(prefix[1]):
                            base = sce.camera.name[len(prefix[1]):]
                            break
                    if base:
                        selected_camera = find_camera(cameras, base, prefixes, suffixes)
                if not selected_camera:
                    for suffix in suffixes:
                        if sce.camera.name.lower().endswith(suffix[0]):
                            base = sce.camera.name[:-len(suffix[0])]
                            break
                        if sce.camera.name.lower().endswith(suffix[1]):
                            base = sce.camera.name[:-len(suffix[1])]
                            break
                    if base:
                        selected_camera = find_camera(cameras, base, prefixes, suffixes)

                if selected_camera:
                    sce.camera = selected_camera
                    found = True

        if not found:
            print(prefixes)
            print(suffixes)
            print([c.name for c in cameras])
            print(sce.camera.name)
            print("Error: Could not find " + params[STEREO_CAMERA] + " camera for the stereo render of " + bpy.data.filepath, file=sys.stderr)
            exit(1)

    elif params[STEREO_CAMERA] != "" and uses_builtin_stereo:

        side = params[STEREO_CAMERA]
        alt_side = "left" if side == "right" else "right"

        sce.render.views[side].use = True
        sce.render.views[alt_side].use = False
        sce.render.views[side].file_suffix = ""
        sce.render.views[alt_side].file_suffix = "_"+alt_side
        sce.render.views_format == 'INDIVIDUAL'

    rend = sce.render

    rend.resolution_percentage = 100

    #rend.alpha_mode = "PREMUL"

    #rend.color_mode = "RGBA"

    # Cycles special tweaks
    if sce.render.engine == 'CYCLES':

        # Allow to override smples from .conf file
        if params[CYCLES_SAMPLES]!=None:
            sce.cycles.samples = params[CYCLES_SAMPLES]

        # Allow to set GPU device from RenderChan module settings
        # For information how to identify your GPU device from
        # a cluster console, see http://www.dalaifelinto.com/?p=746
        if params[GPU_DEVICE]==None:
            # That means we have to explicitly force CPU rendering
            sce.cycles.device = 'CPU'
        elif params[GPU_DEVICE]!="":
            print("Cycles: GPU configuration found")
            error=False

            if 'CUDA' in bpy.context.user_preferences.system.compute_device_type:
                bpy.context.user_preferences.system.compute_device_type = 'CUDA'
            else:
                error = True
                print("ERROR: Cannot activate CUDA.")

            if not error and params[GPU_DEVICE] in bpy.context.user_preferences.system.bl_rna.properties['compute_device'].enum_items.keys():
                bpy.context.user_preferences.system.compute_device = params[GPU_DEVICE]
            else:
                error = True
                # FIXME: This test probably should go somewhere else (in modules's CheckRequirements?)
                print("ERROR: Cannot set GPU device (%s) - not found." % params[GPU_DEVICE])
                print()
                print("Available devices:")
                for device in bpy.context.user_preferences.system.bl_rna.properties['compute_device'].enum_items.keys():
                    print("   * %s\n" % device)
                print()

            if not error:
                sce.cycles.device = 'GPU'
            else:
                sce.cycles.device = 'CPU'

        # Optimize tiles for speed depending on rendering device
        # See tip #3 at http://www.blenderguru.com/4-easy-ways-to-speed-up-cycles/
        if sce.cycles.device == 'GPU':
            print("Cycles: GPU device used")
            sce.render.tile_x = 256
            sce.render.tile_y = 256
            sce.cycles.debug_use_spatial_splits = False
        else:
            print("Cycles: CPU device used")
            bpy.context.user_preferences.system.compute_device_type = 'NONE'
            sce.render.tile_x = 64
            sce.render.tile_y = 64
            sce.cycles.debug_use_spatial_splits = True
        print()

        #sce.cycles.use_cache = True   # Cache BVH
        sce.cycles.debug_bvh_type = 'STATIC_BVH'
        #sce.render.use_persistent_data = True   # Persistent Images

    # Suff for updating file
    size_x = rend.resolution_x
    size_y = rend.resolution_y
    fps = rend.fps

    # This is a dirty hack to make  objects initialized properly.
    # Sometimes (especially when using linked groups with armatures and complex expressions)
    # the objects are not properly initialized and appear correctly only on second-third render.
    # With the trick below you can instruct blender to make some count of pre-renders to
    # ensure that all objects are properly initialized.
    # Just put a relevant option for .conf file, like this:
    #      blender_prerender_count=1
    for _ in range(params[PRERENDER_COUNT]):
        rend.resolution_x = 32
        rend.resolution_y = 32
        bpy.ops.render.render()

    rend.resolution_x = params[WIDTH]
    rend.resolution_y = params[HEIGHT]
    #rend.fps = $FPS

    # OPENEXR stuff
    #rend.exr_zbuf = False
    #rend.use_exr_half = True
    #rend.exr_preview = False

    rend.use_placeholder = False
    rend.use_overwrite = True

    # Force format here
    if params[FORMAT] == "png":
        rend.image_settings.file_format = "PNG"
    elif params[FORMAT] == "avi":
        rend.image_settings.file_format = "H264"
        rend.ffmpeg.format = "H264"
        rend.ffmpeg.use_lossless_output=True
        #rend.ffmpeg.audio_codec="AAC"

    # Update .blend file if permitted and we have width or height changed
    if update and ( size_x != rend.resolution_x or size_y != rend.resolution_y or fps != rend.fps ):
        bpy.ops.wm.save_mainfile("EXEC_DEFAULT", filepath=bpy.data.filepath)

    # Dump audio track if any
    #audio_found = False
    #for path in bpy.utils.blend_paths(0):
    #    if path.endswith(".wav"):
    #        bpy.ops.sound.mixdown(filepath=params[AUDIOFILE], check_existing=False, container="WAV")
    #        break

if __name__ == '__main__':
    main()
<commit_msg>Blender: Fix regular rendering when  multiview is enabled<commit_after>__author__ = 'zelgadis'

import bpy
import sys

UPDATE = "update"
WIDTH = "width"
HEIGHT = "height"
STEREO_CAMERA = "camera"
AUDIOFILE = "audiofile"
FORMAT = "format"
CYCLES_SAMPLES = "cycles_samples"
PRERENDER_COUNT = "prerender_count"
GPU_DEVICE = "gpu_device"

params = {UPDATE: False,
          WIDTH: 480,
          HEIGHT: 270,
          STEREO_CAMERA: "",
          AUDIOFILE:"/tmp/renderchan-test.wav",
          FORMAT: "png",
          CYCLES_SAMPLES: None,
          PRERENDER_COUNT: 0,
          GPU_DEVICE: ""}

def find_camera(cameras, base, prefixes, suffixes, side_index = 0):
    for camera in cameras:
        for prefix in prefixes:
            if camera.name.lower().startswith(prefix[side_index]) and camera.name[len(prefix[side_index]):] == base:
                return camera
        for suffix in suffixes:
            if camera.name.lower().endswith(suffix[side_index]) and camera.name[:-len(suffix[side_index])] == base:
                return camera
    return None

def main():

    update = params[UPDATE]

    sce = bpy.context.scene

    sce.frame_current=sce.frame_current+1
    sce.frame_current=sce.frame_current-1

    uses_builtin_stereo = "use_multiview" in dir(sce.render) and sce.render.use_multiview

    if params[STEREO_CAMERA] != "" and not uses_builtin_stereo:
        found=False

        for ob in sce.objects:
            if ob.name == params[STEREO_CAMERA] and ob.type == 'CAMERA':
                sce.camera = ob
                found=True
                break

        if not found:
            separators = ['_', '.', '-', ' ']

            side = params[STEREO_CAMERA]
            alt_side = "left" if side == "right" else "right"
            sides = [side, side[0]]
            alt_sides = [alt_side, alt_side[0]]

            prefixes = [(sides[side_index] + sep, alt_sides[side_index] + sep) for sep in separators for side_index in range(0, len(sides))]
            prefixes.append((side, alt_side))

            suffixes = [(sep + sides[side_index], sep + alt_sides[side_index]) for sep in separators for side_index in range(0, len(sides))]
            suffixes.append((side, alt_side))

            cameras = [obj for obj in sce.objects if obj.type == "CAMERA"]

            if len(cameras) < 1:
                print("Error: Cannot render, no camera in file " + bpy.data.filepath, file=sys.stderr)
                exit(1)
            elif len(cameras) == 1:
                print("Warning: Stereo specified, but only one camera found. Using camera for both sides.")
                sce.camera = cameras[0]
                found = True
            else:
                selected_camera = find_camera(cameras, sce.camera.name, prefixes, suffixes)

                base = None
                if not selected_camera:
                    for prefix in prefixes:
                        if sce.camera.name.lower().startswith(prefix[0]):
                            base = sce.camera.name[len(prefix[0]):]
                            break
                        if sce.camera.name.lower().startswith(prefix[1]):
                            base = sce.camera.name[len(prefix[1]):]
                            break
                    if base:
                        selected_camera = find_camera(cameras, base, prefixes, suffixes)
                if not selected_camera:
                    for suffix in suffixes:
                        if sce.camera.name.lower().endswith(suffix[0]):
                            base = sce.camera.name[:-len(suffix[0])]
                            break
                        if sce.camera.name.lower().endswith(suffix[1]):
                            base = sce.camera.name[:-len(suffix[1])]
                            break
                    if base:
                        selected_camera = find_camera(cameras, base, prefixes, suffixes)

                if selected_camera:
                    sce.camera = selected_camera
                    found = True

        if not found:
            print(prefixes)
            print(suffixes)
            print([c.name for c in cameras])
            print(sce.camera.name)
            print("Error: Could not find " + params[STEREO_CAMERA] + " camera for the stereo render of " + bpy.data.filepath, file=sys.stderr)
            exit(1)

    elif params[STEREO_CAMERA] != "" and uses_builtin_stereo:

        side = params[STEREO_CAMERA]
        alt_side = "left" if side == "right" else "right"

        sce.render.views[side].use = True
        sce.render.views[alt_side].use = False
        sce.render.views[side].file_suffix = ""
        sce.render.views[alt_side].file_suffix = "_"+alt_side
        sce.render.views_format == 'INDIVIDUAL'

    else:
        sce.render.use_multiview = False

    rend = sce.render

    rend.resolution_percentage = 100

    #rend.alpha_mode = "PREMUL"

    #rend.color_mode = "RGBA"

    # Cycles special tweaks
    if sce.render.engine == 'CYCLES':

        # Allow to override smples from .conf file
        if params[CYCLES_SAMPLES]!=None:
            sce.cycles.samples = params[CYCLES_SAMPLES]

        # Allow to set GPU device from RenderChan module settings
        # For information how to identify your GPU device from
        # a cluster console, see http://www.dalaifelinto.com/?p=746
        if params[GPU_DEVICE]==None:
            # That means we have to explicitly force CPU rendering
            sce.cycles.device = 'CPU'
        elif params[GPU_DEVICE]!="":
            print("Cycles: GPU configuration found")
            error=False

            if 'CUDA' in bpy.context.user_preferences.system.compute_device_type:
                bpy.context.user_preferences.system.compute_device_type = 'CUDA'
            else:
                error = True
                print("ERROR: Cannot activate CUDA.")

            if not error and params[GPU_DEVICE] in bpy.context.user_preferences.system.bl_rna.properties['compute_device'].enum_items.keys():
                bpy.context.user_preferences.system.compute_device = params[GPU_DEVICE]
            else:
                error = True
                # FIXME: This test probably should go somewhere else (in modules's CheckRequirements?)
                print("ERROR: Cannot set GPU device (%s) - not found." % params[GPU_DEVICE])
                print()
                print("Available devices:")
                for device in bpy.context.user_preferences.system.bl_rna.properties['compute_device'].enum_items.keys():
                    print("   * %s\n" % device)
                print()

            if not error:
                sce.cycles.device = 'GPU'
            else:
                sce.cycles.device = 'CPU'

        # Optimize tiles for speed depending on rendering device
        # See tip #3 at http://www.blenderguru.com/4-easy-ways-to-speed-up-cycles/
        if sce.cycles.device == 'GPU':
            print("Cycles: GPU device used")
            sce.render.tile_x = 256
            sce.render.tile_y = 256
            sce.cycles.debug_use_spatial_splits = False
        else:
            print("Cycles: CPU device used")
            bpy.context.user_preferences.system.compute_device_type = 'NONE'
            sce.render.tile_x = 64
            sce.render.tile_y = 64
            sce.cycles.debug_use_spatial_splits = True
        print()

        #sce.cycles.use_cache = True   # Cache BVH
        sce.cycles.debug_bvh_type = 'STATIC_BVH'
        #sce.render.use_persistent_data = True   # Persistent Images

    # Suff for updating file
    size_x = rend.resolution_x
    size_y = rend.resolution_y
    fps = rend.fps

    # This is a dirty hack to make  objects initialized properly.
    # Sometimes (especially when using linked groups with armatures and complex expressions)
    # the objects are not properly initialized and appear correctly only on second-third render.
    # With the trick below you can instruct blender to make some count of pre-renders to
    # ensure that all objects are properly initialized.
    # Just put a relevant option for .conf file, like this:
    #      blender_prerender_count=1
    for _ in range(params[PRERENDER_COUNT]):
        rend.resolution_x = 32
        rend.resolution_y = 32
        bpy.ops.render.render()

    rend.resolution_x = params[WIDTH]
    rend.resolution_y = params[HEIGHT]
    #rend.fps = $FPS

    # OPENEXR stuff
    #rend.exr_zbuf = False
    #rend.use_exr_half = True
    #rend.exr_preview = False

    rend.use_placeholder = False
    rend.use_overwrite = True

    # Force format here
    if params[FORMAT] == "png":
        rend.image_settings.file_format = "PNG"
    elif params[FORMAT] == "avi":
        rend.image_settings.file_format = "H264"
        rend.ffmpeg.format = "H264"
        rend.ffmpeg.use_lossless_output=True
        #rend.ffmpeg.audio_codec="AAC"

    # Update .blend file if permitted and we have width or height changed
    if update and ( size_x != rend.resolution_x or size_y != rend.resolution_y or fps != rend.fps ):
        bpy.ops.wm.save_mainfile("EXEC_DEFAULT", filepath=bpy.data.filepath)

    # Dump audio track if any
    #audio_found = False
    #for path in bpy.utils.blend_paths(0):
    #    if path.endswith(".wav"):
    #        bpy.ops.sound.mixdown(filepath=params[AUDIOFILE], check_existing=False, container="WAV")
    #        break

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># opts.py - module to parse command line options and output what parameters will be used for this run

import os
import os.path
import json
import sys
import common
from common import OK, NOTOK, FsDriftException
import argparse
import parser_data_types
from parser_data_types import boolean, positive_integer, non_negative_integer, bitmask
from parser_data_types import positive_float, non_negative_float, positive_percentage
from parser_data_types import host_set, file_access_distrib, directory_list

# command line parameter variables here

class FsDriftOpts:
    def __init__(self):
        self.top_directory = '/tmp/foo'
        self.output_json_path = None  # filled in later
        self.host_set = [] # default is local test
        self.threads = 2 #  number of subprocesses per host
        self.is_slave = False
        self.duration = 1
        self.max_files = 200
        self.max_file_size_kb = 10
        self.max_record_size_kb = 1
        self.max_random_reads = 2
        self.max_random_writes = 2
        self.fdatasync_probability_pct = 10
        self.fsync_probability_pct = 20
        self.levels = 2
        self.subdirs_per_dir = 3
        self.rsptimes = False
        self.workload_table_csv_path = None
        self.stats_report_interval = 0
        self.pause_between_ops = 100
        self.pause_secs = self.pause_between_ops / float(common.USEC_PER_SEC)
        self.incompressible = False
        # new parameters related to gaussian filename distribution
        self.rand_distr_type = common.FileAccessDistr.uniform
        self.mean_index_velocity = 0.0  # default is a fixed mean for the distribution
        self.gaussian_stddev = 1000.0  # just a guess, means most of accesses within 1000 files?
        # just a guess, most files will be created before they are read
        self.create_stddevs_ahead = 3.0
        self.drift_time = -1
        self.mount_command = None
        # not settable
        self.is_slave = False
        self.as_host = None  # filled in by worker host
        self.verbosity = 0

    def kvtuplelist(self):
        return [
            ('top directory', self.top_directory),
            ('JSON output file', self.output_json_path),
            ('save response times?', self.rsptimes),
            ('stats report interval', self.stats_report_interval),
            ('workload table csv path', self.workload_table_csv_path),
            ('host set', ','.join(self.host_set)),
            ('threads', self.threads),
            ('test duration', self.duration),
            ('maximum file count', self.max_files),
            ('maximum file size (KB)', self.max_file_size_kb),
            ('maximum record size (KB)', self.max_record_size_kb),
            ('maximum random reads per op', self.max_random_reads),
            ('maximum random writes per op', self.max_random_writes),
            ('fsync probability pct', self.fsync_probability_pct),
            ('fdatasync probability pct', self.fdatasync_probability_pct),
            ('directory levels', self.levels),
            ('subdirectories per directory', self.subdirs_per_dir),
            ('incompressible data', self.incompressible),
            ('pause between opts (usec)', self.pause_between_ops),
            ('distribution', common.FileAccessDistr2str(self.random_distribution)),
            ('mean index velocity', self.mean_index_velocity),
            ('gaussian std. dev.', self.gaussian_stddev),
            ('create stddevs ahead', self.create_stddevs_ahead),
            ('mount command', self.mount_command),
            ('verbosity', self.verbosity),
            ('pause path', self.pause_path),
            ]

    def __str__(self, use_newline=True, indentation='  '):
        kvlist = [ '%s = %s' % (k, str(v)) for (k, v) in self.kvtuplelist() ]
        if use_newline:
            return ('\n%s' % indentation).join(kvlist)
        else:
            return ' , '.join(kvlist)

    def to_json_obj(self):
        d = {}
        for (k, v) in self.kvtuplelist():
            d[k] = v
        return d

    def validate(self):

        if len(self.top_directory) < 6:
            raise FsDriftException(
                'top directory %s too short, may be system directory' %
                self.top_directory)

        if not os.path.isdir(self.top_directory):
            raise FsDriftException(
                'top directory %s does not exist, so please create it' %
                self.top_directory)

        if self.workload_table_csv_path == None:
            self.workload_table_csv_path = os.path.join(self.top_directory,
                                                        'example_workload_table.csv')
            workload_table = [
                    'read, 2',
                    'random_read, 1',
                    'random_write, 1',
                    'append, 4',
                    'delete, 0.1',
                    'hardlink, 0.01',
                    'softlink, 0.02',
                    'truncate, 0.05',
                    'rename, 1',
                    'create, 4']
            with open(self.workload_table_csv_path, 'w') as w_f:
                w_f.write( '\n'.join(workload_table))

def parseopts():
    o = FsDriftOpts()

    parser = argparse.ArgumentParser(description='parse fs-drift parameters')
    add = parser.add_argument
    add('--top', help='directory containing all file accesses',
            default=o.top_directory)
    add('--output-json', help='output file containing results in JSON format',
            default=None)
    add('--workload-table', help='.csv file containing workload mix',
            default=None)
    add('--duration', help='seconds to run test',
            type=positive_integer,
            default=o.duration)
    add('--host-set', help='comma-delimited list of host names/ips',
            type=host_set,
            default=o.host_set)
    add('--threads', help='number of subprocesses per host',
            type=positive_integer,
            default=o.threads)
    add('--max-files', help='maximum number of files to access',
            type=positive_integer,
            default=o.max_files)
    add('--max-file-size-kb', help='maximum file size in KB',
            type=positive_integer,
            default=o.max_file_size_kb)
    add('--pause-between-ops', help='delay between ops in microsec',
            type=non_negative_integer,
            default=o.pause_between_ops)
    add('--max-record-size-kb', help='maximum read/write size in KB',
            type=positive_integer,
            default=o.max_record_size_kb)
    add('--max-random-reads', help='maximum consecutive random reads',
            type=positive_integer,
            default=o.max_random_reads)
    add('--max-random-writes', help='maximum consecutive random writes',
            type=positive_integer,
            default=o.max_random_writes)
    add('--fdatasync-pct', help='probability of fdatasync after write',
            type=positive_percentage,
            default=o.fdatasync_probability_pct)
    add('--fsync-pct', help='probability of fsync after write',
            type=positive_percentage,
            default=o.fsync_probability_pct)
    add('--levels', help='number of directory levels in tree',
            type=non_negative_integer,
            default=o.levels)
    add('--dirs-per-level', help='number of subdirectories per directory',
            type=non_negative_integer,
            default=o.subdirs_per_dir)
    add('--report-interval', help='seconds between counter output',
            type=positive_integer,
            default=o.stats_report_interval)
    add('--response-times', help='if True then save response times to CSV file',
            type=boolean,
            default=o.rsptimes)
    add('--incompressible', help='if True then write incompressible data',
            type=boolean,
            default=o.incompressible)
    add('--random-distribution', help='either "uniform" or "gaussian"',
            type=file_access_distrib,
            default=common.FileAccessDistr.uniform)
    add('--mean-velocity', help='rate at which mean advances through files',
            type=float,
            default=o.mean_index_velocity)
    add('--gaussian-stddev', help='std. dev. of file number',
            type=float,
            default=o.gaussian_stddev)
    add('--create-stddevs-ahead', help='file creation ahead of other opts by this many stddevs',
            type=float,
            default=o.create_stddevs_ahead)
    add('--mount-command', help='command to mount the filesystem containing top directory',
            default=o.mount_command)
    add('--verbosity', help='decimal or hexadecimal integer bitmask controlling debug logging',
            type=bitmask,
            default=o.verbosity)

    # parse the command line and update opts
    args = parser.parse_args()
    o.top_directory = args.top
    o.output_json_path = args.output_json
    o.rsptimes = args.response_times
    o.host_set = args.host_set
    o.threads = args.threads
    o.report_interval = args.report_interval
    o.workload_table_csv_path = args.workload_table
    o.duration = args.duration
    o.max_files = args.max_files
    o.max_file_size_kb = args.max_file_size_kb
    o.max_record_size_kb = args.max_record_size_kb
    o.max_random_reads = args.max_random_reads
    o.max_random_writes = args.max_random_writes
    o.fdatasync_probability_pct = args.fdatasync_pct
    o.fsync_probability_pct = args.fsync_pct
    o.levels = args.levels
    o.subdirs_per_dir = args.dirs_per_level
    o.incompressible = args.incompressible
    o.pause_between_ops = args.pause_between_ops
    o.pause_secs = o.pause_between_ops / float(common.USEC_PER_SEC)
    o.response_times = args.response_times
    o.random_distribution = args.random_distribution
    o.mean_index_velocity = args.mean_velocity
    o.gaussian_stddev = args.gaussian_stddev
    o.create_stddevs_ahead = args.create_stddevs_ahead
    o.mount_command = args.mount_command
    o.verbosity = args.verbosity

    # some fields derived from user inputs

    o.network_shared_path = os.path.join(o.top_directory, 'network-shared')

    nsjoin = lambda fn : os.path.join(o.network_shared_path, fn)
    o.starting_gun_path     = nsjoin('starting-gun.tmp')
    o.stop_file_path        = nsjoin('stop-file.tmp')
    o.param_pickle_path     = nsjoin('params.pickle')
    o.rsptime_path          = nsjoin('host-%s_thrd-%s_rsptimes.csv')
    o.abort_path            = nsjoin('abort.tmp')
    o.pause_path            = nsjoin('pause.tmp')
    o.checkerflag_path      = nsjoin('checkered_flag.tmp')

    o.remote_pgm_dir = os.path.dirname(sys.argv[0])
    if o.remote_pgm_dir == '.':
        o.remote_pgm_dir = os.getcwd()

    o.is_slave = sys.argv[0].endswith('fs-drift-remote.py')

    return o

# unit test

if __name__ == "__main__":
    options = parseopts()
    options.validate()
    print(options)
    print(json.dumps(options.to_json_obj(), indent=2, sort_keys=True))
<commit_msg>add --tolerate-stale-file-handles boolean option<commit_after># opts.py - module to parse command line options and output what parameters will be used for this run

import os
import os.path
import json
import sys
import common
from common import OK, NOTOK, FsDriftException
import argparse
import parser_data_types
from parser_data_types import boolean, positive_integer, non_negative_integer, bitmask
from parser_data_types import positive_float, non_negative_float, positive_percentage
from parser_data_types import host_set, file_access_distrib, directory_list

# command line parameter variables here

class FsDriftOpts:
    def __init__(self):
        self.top_directory = '/tmp/foo'
        self.output_json_path = None  # filled in later
        self.host_set = [] # default is local test
        self.threads = 2 #  number of subprocesses per host
        self.is_slave = False
        self.duration = 1
        self.max_files = 200
        self.max_file_size_kb = 10
        self.max_record_size_kb = 1
        self.max_random_reads = 2
        self.max_random_writes = 2
        self.fdatasync_probability_pct = 10
        self.fsync_probability_pct = 20
        self.levels = 2
        self.subdirs_per_dir = 3
        self.rsptimes = False
        self.workload_table_csv_path = None
        self.stats_report_interval = 0
        self.pause_between_ops = 100
        self.pause_secs = self.pause_between_ops / float(common.USEC_PER_SEC)
        self.incompressible = False
        # new parameters related to gaussian filename distribution
        self.rand_distr_type = common.FileAccessDistr.uniform
        self.mean_index_velocity = 0.0  # default is a fixed mean for the distribution
        self.gaussian_stddev = 1000.0  # just a guess, means most of accesses within 1000 files?
        # just a guess, most files will be created before they are read
        self.create_stddevs_ahead = 3.0
        self.drift_time = -1
        self.mount_command = None
        # not settable
        self.is_slave = False
        self.as_host = None  # filled in by worker host
        self.verbosity = 0
        self.tolerate_stale_fh = False

    def kvtuplelist(self):
        return [
            ('top directory', self.top_directory),
            ('JSON output file', self.output_json_path),
            ('save response times?', self.rsptimes),
            ('stats report interval', self.stats_report_interval),
            ('workload table csv path', self.workload_table_csv_path),
            ('host set', ','.join(self.host_set)),
            ('threads', self.threads),
            ('test duration', self.duration),
            ('maximum file count', self.max_files),
            ('maximum file size (KB)', self.max_file_size_kb),
            ('maximum record size (KB)', self.max_record_size_kb),
            ('maximum random reads per op', self.max_random_reads),
            ('maximum random writes per op', self.max_random_writes),
            ('fsync probability pct', self.fsync_probability_pct),
            ('fdatasync probability pct', self.fdatasync_probability_pct),
            ('directory levels', self.levels),
            ('subdirectories per directory', self.subdirs_per_dir),
            ('incompressible data', self.incompressible),
            ('pause between opts (usec)', self.pause_between_ops),
            ('distribution', common.FileAccessDistr2str(self.random_distribution)),
            ('mean index velocity', self.mean_index_velocity),
            ('gaussian std. dev.', self.gaussian_stddev),
            ('create stddevs ahead', self.create_stddevs_ahead),
            ('mount command', self.mount_command),
            ('verbosity', self.verbosity),
            ('pause path', self.pause_path),
            ('tolerate stale file handles', self.tolerate_stale_fh),
            ]

    def __str__(self, use_newline=True, indentation='  '):
        kvlist = [ '%s = %s' % (k, str(v)) for (k, v) in self.kvtuplelist() ]
        if use_newline:
            return ('\n%s' % indentation).join(kvlist)
        else:
            return ' , '.join(kvlist)

    def to_json_obj(self):
        d = {}
        for (k, v) in self.kvtuplelist():
            d[k] = v
        return d

    def validate(self):

        if len(self.top_directory) < 6:
            raise FsDriftException(
                'top directory %s too short, may be system directory' %
                self.top_directory)

        if not os.path.isdir(self.top_directory):
            raise FsDriftException(
                'top directory %s does not exist, so please create it' %
                self.top_directory)

        if self.workload_table_csv_path == None:
            self.workload_table_csv_path = os.path.join(self.top_directory,
                                                        'example_workload_table.csv')
            workload_table = [
                    'read, 2',
                    'random_read, 1',
                    'random_write, 1',
                    'append, 4',
                    'delete, 0.1',
                    'hardlink, 0.01',
                    'softlink, 0.02',
                    'truncate, 0.05',
                    'rename, 1',
                    'create, 4']
            with open(self.workload_table_csv_path, 'w') as w_f:
                w_f.write( '\n'.join(workload_table))

def parseopts():
    o = FsDriftOpts()

    parser = argparse.ArgumentParser(description='parse fs-drift parameters')
    add = parser.add_argument
    add('--top', help='directory containing all file accesses',
            default=o.top_directory)
    add('--output-json', help='output file containing results in JSON format',
            default=None)
    add('--workload-table', help='.csv file containing workload mix',
            default=None)
    add('--duration', help='seconds to run test',
            type=positive_integer,
            default=o.duration)
    add('--host-set', help='comma-delimited list of host names/ips',
            type=host_set,
            default=o.host_set)
    add('--threads', help='number of subprocesses per host',
            type=positive_integer,
            default=o.threads)
    add('--max-files', help='maximum number of files to access',
            type=positive_integer,
            default=o.max_files)
    add('--max-file-size-kb', help='maximum file size in KB',
            type=positive_integer,
            default=o.max_file_size_kb)
    add('--pause-between-ops', help='delay between ops in microsec',
            type=non_negative_integer,
            default=o.pause_between_ops)
    add('--max-record-size-kb', help='maximum read/write size in KB',
            type=positive_integer,
            default=o.max_record_size_kb)
    add('--max-random-reads', help='maximum consecutive random reads',
            type=positive_integer,
            default=o.max_random_reads)
    add('--max-random-writes', help='maximum consecutive random writes',
            type=positive_integer,
            default=o.max_random_writes)
    add('--fdatasync-pct', help='probability of fdatasync after write',
            type=positive_percentage,
            default=o.fdatasync_probability_pct)
    add('--fsync-pct', help='probability of fsync after write',
            type=positive_percentage,
            default=o.fsync_probability_pct)
    add('--levels', help='number of directory levels in tree',
            type=non_negative_integer,
            default=o.levels)
    add('--dirs-per-level', help='number of subdirectories per directory',
            type=non_negative_integer,
            default=o.subdirs_per_dir)
    add('--report-interval', help='seconds between counter output',
            type=positive_integer,
            default=o.stats_report_interval)
    add('--response-times', help='if True then save response times to CSV file',
            type=boolean,
            default=o.rsptimes)
    add('--incompressible', help='if True then write incompressible data',
            type=boolean,
            default=o.incompressible)
    add('--random-distribution', help='either "uniform" or "gaussian"',
            type=file_access_distrib,
            default=common.FileAccessDistr.uniform)
    add('--mean-velocity', help='rate at which mean advances through files',
            type=float,
            default=o.mean_index_velocity)
    add('--gaussian-stddev', help='std. dev. of file number',
            type=float,
            default=o.gaussian_stddev)
    add('--create-stddevs-ahead', help='file creation ahead of other opts by this many stddevs',
            type=float,
            default=o.create_stddevs_ahead)
    add('--mount-command', help='command to mount the filesystem containing top directory',
            default=o.mount_command)
    add('--tolerate-stale-file-handles', help='if true, do not throw exception on ESTALE',
            type=boolean,
            default=o.tolerate_stale_fh)
    add('--verbosity', help='decimal or hexadecimal integer bitmask controlling debug logging',
            type=bitmask,
            default=o.verbosity)

    # parse the command line and update opts
    args = parser.parse_args()
    o.top_directory = args.top
    o.output_json_path = args.output_json
    o.rsptimes = args.response_times
    o.host_set = args.host_set
    o.threads = args.threads
    o.report_interval = args.report_interval
    o.workload_table_csv_path = args.workload_table
    o.duration = args.duration
    o.max_files = args.max_files
    o.max_file_size_kb = args.max_file_size_kb
    o.max_record_size_kb = args.max_record_size_kb
    o.max_random_reads = args.max_random_reads
    o.max_random_writes = args.max_random_writes
    o.fdatasync_probability_pct = args.fdatasync_pct
    o.fsync_probability_pct = args.fsync_pct
    o.levels = args.levels
    o.subdirs_per_dir = args.dirs_per_level
    o.incompressible = args.incompressible
    o.pause_between_ops = args.pause_between_ops
    o.pause_secs = o.pause_between_ops / float(common.USEC_PER_SEC)
    o.response_times = args.response_times
    o.random_distribution = args.random_distribution
    o.mean_index_velocity = args.mean_velocity
    o.gaussian_stddev = args.gaussian_stddev
    o.create_stddevs_ahead = args.create_stddevs_ahead
    o.mount_command = args.mount_command
    o.tolerate_stale_fh = args.tolerate_stale_file_handles
    o.verbosity = args.verbosity

    # some fields derived from user inputs

    o.network_shared_path = os.path.join(o.top_directory, 'network-shared')

    nsjoin = lambda fn : os.path.join(o.network_shared_path, fn)
    o.starting_gun_path     = nsjoin('starting-gun.tmp')
    o.stop_file_path        = nsjoin('stop-file.tmp')
    o.param_pickle_path     = nsjoin('params.pickle')
    o.rsptime_path          = nsjoin('host-%s_thrd-%s_rsptimes.csv')
    o.abort_path            = nsjoin('abort.tmp')
    o.pause_path            = nsjoin('pause.tmp')
    o.checkerflag_path      = nsjoin('checkered_flag.tmp')

    o.remote_pgm_dir = os.path.dirname(sys.argv[0])
    if o.remote_pgm_dir == '.':
        o.remote_pgm_dir = os.getcwd()

    o.is_slave = sys.argv[0].endswith('fs-drift-remote.py')

    return o

# unit test

if __name__ == "__main__":
    options = parseopts()
    options.validate()
    print(options)
    print(json.dumps(options.to_json_obj(), indent=2, sort_keys=True))
<|endoftext|>


<commit_before># Copyright (C) 2012, Nikola Kovcevic <nikolak@outlook.com>

# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:

# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT
# OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.

from xmlrpclib import ServerProxy
import StringIO
import argparse
import gzip
import os
import struct
import sys
import urllib2

try:
    import guessit
except ImportError:
    print "Can't import guessit module, only subtitle searches based on file hashes can be preformed"

FILE_EXT = [
    '.3g2', '.3gp', '.3gp2', '.3gpp', '.60d', '.ajp', '.asf', '.asx', '.avchd', '.avi',
    '.bik', '.bix', '.box', '.cam', '.dat', '.divx', '.dmf', '.dv', '.dvr-ms', '.evo',
    'flc', '.fli', '.flic', '.flv', '.flx', '.gvi', '.gvp', '.h264', '.m1v', '.m2p',
    '.m2ts', '.m2v', '.m4e', '.m4v', '.mjp', '.mjpeg', '.mjpg', '.mkv', '.moov', '.mov',
    '.movhd', '.movie', '.movx', '.mp4', '.mpe', '.mpeg', '.mpg', '.mpv', '.mpv2', '.mxf',
    '.nsv', '.nut', '.ogg', '.ogm', '.omf', '.ps', '.qt', '.ram', '.rm', '.rmvb',
    '.swf', '.ts', '.vfw', '.vid', '.video', '.viv', '.vivo', '.vob', '.vro', '.wm',
    '.wmv', '.wmx', '.wrap', '.wvx', '.wx', '.x264', '.xvid'
]

sub_language = 'eng'
opensub_domain = "http://api.opensubtitles.org/xml-rpc"
useragent = "ossubd"

server = ServerProxy(opensub_domain)

# overwrite = False
hash_search = False
AUTO_DOWNLOAD= False
SUBFOLDER="Subs"

def get_tests(file_name):
    session = server.LogIn("", "", sub_language, useragent)
    token = session["token"]
    ep_info = guessit.guess_episode_info(file_name)
    tv_show = ep_info['series']
    season = ep_info['season']
    episode = ep_info['episodeNumber']
    print ep_info
    query_info = ("%s S%.2dE%.2d" % (tv_show,
                                     int(season),
                                     int(episode),)
    ).replace(" ", "+")
    searchlist = []
    searchlist.append({'sublanguageid': sub_language, 'query': query_info})
    moviesList = server.SearchSubtitles(token, searchlist)
    print moviesList
    with open("query.log", "a") as q:
        q.write(file_name + "\n")
        q.write(str(moviesList) + "\n")
        q.write("\n\n\n")

# noinspection PyBroadException
def search_subtitles(file_list):
    # session = server.LogIn("", "", sub_language, useragent)
    # token = session["token"]
    count = 0
    done_count = 0
    for file_name in file_list:
        count += 1
        do_download = True
        file_name, file_extension = os.path.splitext(file_name)
        # print file_name, file_xtension
        print "-"*50
        print 'Searching subtitle for "{0}" | ({1}/{2})'.format(os.path.basename(file_name), count, len(file_list))

        # if os.path.exists(file_name + '.srt'):
        #     if overwrite:
        #         print 'Subtitle exist, but new one will be downloaded'
        #     else:
        #         print 'Subtitle already exists, skipping'
        #         do_download = False

        if do_download and hash_search:
            current_hash = get_hash(file_name + file_extension)
            file_size = os.path.getsize(file_name + file_extension)

            if current_hash is None:
                print "Can't calculate hash for {}".format(file_name)
                do_download = False
        elif do_download and not hash_search:
            ep_info = guessit.guess_episode_info(file_name)
            ep_info["filename"]=file_name
            tv_show = ep_info['series']
            season = ep_info['season']
            episode = ep_info['episodeNumber']

            query_info = ("%s S%.2dE%.2d" % (tv_show,
                                             int(season),
                                             int(episode),)
                                            ).replace(" ", "+")
        if do_download:
            searchlist = []
            if hash_search:
                searchlist.append({'sublanguageid': sub_language, 'moviehash': current_hash,
                                   'moviebytesize': str(file_size)})
            else:
                searchlist.append({'sublanguageid': sub_language, 'query': query_info})
            # moviesList = server.SearchSubtitles(token, searchlist)
            import json#XXX: Debug stuff
            moviesList = json.load(open("data.json"))
            if moviesList["status"]!="200 OK":
                print "Error searching for subtitles..."
            else:
                if moviesList['data']:
                    download_prompt(moviesList["data"],ep_info)
                #http://trac.opensubtitles.org/projects/opensubtitles/wiki/XMLRPC#SearchSubtitles
                else:
                    print 'Couldn\'t find subtitles in {0} for {1}'.format(sub_language, file_name)
    print '-' * 30 + \
          '\nDownloaded subtitles for {0} out of {1} files'.format(done_count, len(file_list))

def download_prompt(subtitles_list, episode_info):
    if AUTO_DOWNLOAD:
        auto_download(subtitles_list,episode_info)
        return
    user_choice=""
    possible_choices=["a","q","s"]
    sub_dict={}
    count=1
    for subtitle in subtitles_list:
        print "{}: {} | Downloads: {:^}".format(count,
                                                subtitle["SubFileName"],
                                                subtitle["SubDownloadsCnt"],)
        sub_dict[count]={"SubFileName":subtitle["SubFileName"],
                         "SubDownloadLink":subtitle["SubDownloadLink"]}
        count+=1
    possible_choices.extend(sub_dict.keys())

    while user_choice not in possible_choices:
        inp=raw_input("Enter subtitle # to download or 's' - skip this file, 'a' - auto download, 'q' - quit\n>>")
        try:#Faster than .isdigit()
            user_choice=int(inp)
        except:
            user_choice=inp

        if user_choice not in possible_choices:
            print "invalid input"
    if user_choice in possible_choices: # if needed?
        if type(user_choice) is int:
            if sub_dict.get(user_choice,False) is not False:
                download_subtitle(sub_dict[user_choice], episode_info)
            else:
                print "Invalid input only subtitle choices from {} to {} are available".format(1,count)
        elif user_choice.lower()=="a":
            auto_download(subtitles_list,episode_info)
        elif user_choice.lower()=="q":
            print 'Quitting'
            exit()
        elif user_choice.lower()=="s":
            print "skipping..."
        else:
            print "Invalid input"

def auto_download(subtitles_list, ep_info):
    """
    episode_info:
    {u'mimetype': u'video/x-flv', u'episodeNumber': 11, u'container': u'flv',
    u'title': u'Adventures in Babysitting', u'series': u'Supernatural',
    u'type': u'episode', u'season': 7, u'filename':<full file path>}
    """
    if len(subtitles_list)<2:
        print "Only one subtitle found, this may be wrong one..."
        download_subtitle(subtitles_list[0])

    possible_matches=[]
    best_choice={"best":None,"downloads":0}

    for subtitle in subtitles_list:
        # if subtitle['SeriesSeason']==str(ep_info['season']) and \
        #                 subtitle['SeriesEpisode']==str(ep_info['episodeNumber']):
        #     # Subtitle is for same season and episode number, it's probably something we want
        #     # Wrong subtitles in most cases don't have same season/episode numbers
        #     # FIXME: Yeah, this isn't working as described above; too much "false positives"
        #     possible_matches.append(subtitle)
        #     print 'first if'
        subtitle_title_name=subtitle['MovieName'].replace("'","").replace('"','').lower()
        episode_title_name="{} {}".format(ep_info['series'].lower(),ep_info['title'].lower())
        if subtitle_title_name==episode_title_name:
            # Subtitle has same series name+episode name as our file.
            possible_matches.append(subtitle)

    for sub in possible_matches:
        # print sub['SubFileName']
        if int(sub["SubDownloadsCnt"])>best_choice["downloads"]:
            best_choice["best"]=sub
            best_choice["downloads"]=sub["SubDownloadsCnt"]
    print best_choice["best"],best_choice["downloads"]
    if best_choice["best"] is not None:
        download_subtitle(best_choice["best"],ep_info)
    else:
        print "Can't match subtitle..."

def download_subtitle(subtitle_info,ep_info):
    """
    episode_info:
    {u'mimetype': u'video/x-flv', u'episodeNumber': 11, u'container': u'flv',
    u'title': u'Adventures in Babysitting', u'series': u'Supernatural',
    u'type': u'episode', u'season': 7, u'filename':<full file path>}
    """
    download_url=subtitle_info["SubDownloadLink"]
    subtitle_folder=os.path.dirname(ep_info['filename'])
    subtitle_folder+="/" if SUBFOLDER is None else "/"+SUBFOLDER.replace("/","")+"/"
    subtitle_name=subtitle_folder+os.path.basename(ep_info["filename"])
    print "Downloading from {} and saving as {}.srt ".format(download_url,subtitle_name)

    if not os.path.isdir(subtitle_folder):
        os.mkdir(subtitle_folder)
        # TODO: Add exception handling, if we can't create folder we won't be able to save sub there (probably)

    sub_zip_file = urllib2.urlopen(download_url)
    try:
        sub_gzip = gzip.GzipFile(fileobj=StringIO.StringIO(sub_zip_file.read()))
        subtitle_content = sub_gzip.read()
        # TODO: Save subtitle as original filename.srt
        # TODO: Add support for subfolder to save ot
        with open(subtitle_name + '.srt', 'wb') as subtitle_output:
            subtitle_output.write(subtitle_content)
            # FIXME: This should be in try/except loop and notify of possible permission issues on fail
        print 'Done!'
    except:
        print "Couldn't save subtitle, permissions issue?"
    pass

def get_hash(name):
    """
    :param name: File path for which to calculate hash
    :return: Hash or None
    """
    try:
        longlongformat = 'q'  # long long
        bytesize = struct.calcsize(longlongformat)

        f = open(name, "rb")

        filesize = os.path.getsize(name)
        file_hash = filesize

        if filesize < 65536 * 2:
            return "SizeError"

        for x in range(65536 / bytesize):
            file_buffer = f.read(bytesize)
            (l_value,) = struct.unpack(longlongformat, file_buffer)
            file_hash += l_value
            file_hash &= 0xFFFFFFFFFFFFFFFF# to remain as 64bit number

        f.seek(max(0, filesize - 65536), 0)
        for x in range(65536 / bytesize):
            file_buffer = f.read(bytesize)
            (l_value,) = struct.unpack(longlongformat, file_buffer)
            file_hash += l_value
            file_hash &= 0xFFFFFFFFFFFFFFFF

        f.close()
        return "%016x" % file_hash

    except IOError:
        return None

if __name__ == '__main__':
    # filename="/media/8C82817682816614/TV/Supernatural/Season 7/Supernatural - [07x11] - Adventures in Babysitting.flv"
    # search_subtitles([filename])
    valid_files = []

    parser = argparse.ArgumentParser()
    parser.add_argument("folder", type=str,
                        help="Folder which will be scanned for allowed video files, and subtitles for those files will be downloaded")
    parser.add_argument("-o", "--output", type=str,
                         help="Subfolder to save subtitles to, relative to original video file path")
    parser.add_argument("-l", "--language", type=str,
                        help="Subtitle language, must be an ISO 639-1 Code i.e. (eng,fre,deu) Default English(eng); Full list http://en.wikipedia.org/wiki/List_of_ISO_639-1_codes")
    parser.add_argument("-t","--type", type=str,
                        help="Subtitle search type 'hash' or 'filename'")
    parser.add_argument("-a","--auto", action="store_true",
                        help="Auto download subtitles for all files without prompt (Overwrites subtitles with same filename)")
    args = parser.parse_args()

    directory = args.folder
    if not directory.endswith('/'):
        directory += '/'
    if args.output:
        SUBFOLDER=args.output
    if args.type:
        if args.type=="hash":
            hash_search=True
        elif args.type=="filename":
            hash_search=False
        else:
            print "Invalid search type 'hash' or 'filename' are available options\n defaulting to hash search"
    if args.language:
        if len(args.language) == 3:
            sub_language = args.language.lower()
        else:
            print 'Argument not  ISO 639-1 Code check this for list of valid codes http://en.wikipedia.org/wiki/List_of_ISO_639-1_codes'
            sys.exit()

    if args.auto:
        AUTO_DOWNLOAD=True
    print "Dir: {}, subfolder: {}, hash search type: {}, language: {}, auto download: {} ".format(
        directory,SUBFOLDER,hash_search, sub_language, AUTO_DOWNLOAD
    )
    names = os.listdir(directory)

    for name in names:
        file_name, file_extension = os.path.splitext(name)
        if file_extension in FILE_EXT:
            valid_files.append(directory + file_name + file_extension)

    search_subtitles(valid_files)
<commit_msg>Code reformatting<commit_after># Copyright (C) 2012, Nikola Kovcevic <nikolak@outlook.com>

# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:

# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT
# OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.

from xmlrpclib import ServerProxy
import StringIO
import argparse
import gzip
import os
import struct
import sys
import urllib2

try:
    import guessit
except ImportError:
    print "Can't import guessit module, only subtitle searches based on file hashes can be preformed"

FILE_EXT = [
    '.3g2', '.3gp', '.3gp2', '.3gpp', '.60d', '.ajp', '.asf', '.asx', '.avchd', '.avi',
    '.bik', '.bix', '.box', '.cam', '.dat', '.divx', '.dmf', '.dv', '.dvr-ms', '.evo',
    'flc', '.fli', '.flic', '.flv', '.flx', '.gvi', '.gvp', '.h264', '.m1v', '.m2p',
    '.m2ts', '.m2v', '.m4e', '.m4v', '.mjp', '.mjpeg', '.mjpg', '.mkv', '.moov', '.mov',
    '.movhd', '.movie', '.movx', '.mp4', '.mpe', '.mpeg', '.mpg', '.mpv', '.mpv2', '.mxf',
    '.nsv', '.nut', '.ogg', '.ogm', '.omf', '.ps', '.qt', '.ram', '.rm', '.rmvb',
    '.swf', '.ts', '.vfw', '.vid', '.video', '.viv', '.vivo', '.vob', '.vro', '.wm',
    '.wmv', '.wmx', '.wrap', '.wvx', '.wx', '.x264', '.xvid'
]

sub_language = 'eng'
opensub_domain = "http://api.opensubtitles.org/xml-rpc"
useragent = "ossubd"

server = ServerProxy(opensub_domain)

# overwrite = False
hash_search = False
AUTO_DOWNLOAD = False
SUBFOLDER = "Subs"

def get_tests(file_name):
    session = server.LogIn("", "", sub_language, useragent)
    token = session["token"]
    ep_info = guessit.guess_episode_info(file_name)
    tv_show = ep_info['series']
    season = ep_info['season']
    episode = ep_info['episodeNumber']
    print ep_info
    query_info = ("%s S%.2dE%.2d" % (tv_show,
                                     int(season),
                                     int(episode),)
    ).replace(" ", "+")
    searchlist = []
    searchlist.append({'sublanguageid': sub_language, 'query': query_info})
    moviesList = server.SearchSubtitles(token, searchlist)
    print moviesList
    with open("query.log", "a") as q:
        q.write(file_name + "\n")
        q.write(str(moviesList) + "\n")
        q.write("\n\n\n")

# noinspection PyBroadException
def search_subtitles(file_list):
    # session = server.LogIn("", "", sub_language, useragent)
    # token = session["token"]
    count = 0
    done_count = 0
    for file_name in file_list:
        count += 1
        do_download = True
        file_name, file_extension = os.path.splitext(file_name)
        # print file_name, file_xtension
        print "-" * 50
        print 'Searching subtitle for "{0}" | ({1}/{2})'.format(os.path.basename(file_name), count, len(file_list))

        # if os.path.exists(file_name + '.srt'):
        #     if overwrite:
        #         print 'Subtitle exist, but new one will be downloaded'
        #     else:
        #         print 'Subtitle already exists, skipping'
        #         do_download = False

        if do_download and hash_search:
            current_hash = get_hash(file_name + file_extension)
            file_size = os.path.getsize(file_name + file_extension)

            if current_hash is None:
                print "Can't calculate hash for {}".format(file_name)
                do_download = False
        elif do_download and not hash_search:
            ep_info = guessit.guess_episode_info(file_name)
            ep_info["filename"] = file_name
            tv_show = ep_info['series']
            season = ep_info['season']
            episode = ep_info['episodeNumber']

            query_info = ("%s S%.2dE%.2d" % (tv_show,
                                             int(season),
                                             int(episode),)
            ).replace(" ", "+")
        if do_download:
            searchlist = []
            if hash_search:
                searchlist.append({'sublanguageid': sub_language, 'moviehash': current_hash,
                                   'moviebytesize': str(file_size)})
            else:
                searchlist.append({'sublanguageid': sub_language, 'query': query_info})
                # moviesList = server.SearchSubtitles(token, searchlist)
            import json#XXX: Debug stuff

            moviesList = json.load(open("data.json"))
            if moviesList["status"] != "200 OK":
                print "Error searching for subtitles..."
            else:
                if moviesList['data']:
                    download_prompt(moviesList["data"], ep_info)
                #http://trac.opensubtitles.org/projects/opensubtitles/wiki/XMLRPC#SearchSubtitles
                else:
                    print 'Couldn\'t find subtitles in {0} for {1}'.format(sub_language, file_name)
    print '-' * 30 + \
          '\nDownloaded subtitles for {0} out of {1} files'.format(done_count, len(file_list))

def download_prompt(subtitles_list, episode_info):
    if AUTO_DOWNLOAD:
        auto_download(subtitles_list, episode_info)
        return
    user_choice = ""
    possible_choices = ["a", "q", "s"]
    sub_dict = {}
    count = 1
    for subtitle in subtitles_list:
        print "{}: {} | Downloads: {:^}".format(count,
                                                subtitle["SubFileName"],
                                                subtitle["SubDownloadsCnt"], )
        sub_dict[count] = {"SubFileName": subtitle["SubFileName"],
                           "SubDownloadLink": subtitle["SubDownloadLink"]}
        count += 1
    possible_choices.extend(sub_dict.keys())

    while user_choice not in possible_choices:
        inp = raw_input("Enter subtitle # to download or 's' - skip this file, 'a' - auto download, 'q' - quit\n>>")
        try:#Faster than .isdigit()
            user_choice = int(inp)
        except:
            user_choice = inp

        if user_choice not in possible_choices:
            print "invalid input"
    if user_choice in possible_choices: # if needed?
        if type(user_choice) is int:
            if sub_dict.get(user_choice, False) is not False:
                download_subtitle(sub_dict[user_choice], episode_info)
            else:
                print "Invalid input only subtitle choices from {} to {} are available".format(1, count)
        elif user_choice.lower() == "a":
            auto_download(subtitles_list, episode_info)
        elif user_choice.lower() == "q":
            print 'Quitting'
            exit()
        elif user_choice.lower() == "s":
            print "skipping..."
        else:
            print "Invalid input"

def auto_download(subtitles_list, ep_info):
    """
    episode_info:
    {u'mimetype': u'video/x-flv', u'episodeNumber': 11, u'container': u'flv',
    u'title': u'Adventures in Babysitting', u'series': u'Supernatural',
    u'type': u'episode', u'season': 7, u'filename':<full file path>}
    """
    if len(subtitles_list) < 2:
        print "Only one subtitle found, this may be wrong one..."
        download_subtitle(subtitles_list[0])

    possible_matches = []
    best_choice = {"best": None, "downloads": 0}

    for subtitle in subtitles_list:
        # if subtitle['SeriesSeason']==str(ep_info['season']) and \
        #                 subtitle['SeriesEpisode']==str(ep_info['episodeNumber']):
        #     # Subtitle is for same season and episode number, it's probably something we want
        #     # Wrong subtitles in most cases don't have same season/episode numbers
        #     # FIXME: Yeah, this isn't working as described above; too much "false positives"
        #     possible_matches.append(subtitle)
        #     print 'first if'
        subtitle_title_name = subtitle['MovieName'].replace("'", "").replace('"', '').lower()
        episode_title_name = "{} {}".format(ep_info['series'].lower(), ep_info['title'].lower())
        if subtitle_title_name == episode_title_name:
            # Subtitle has same series name+episode name as our file.
            possible_matches.append(subtitle)

    for sub in possible_matches:
        # print sub['SubFileName']
        if int(sub["SubDownloadsCnt"]) > best_choice["downloads"]:
            best_choice["best"] = sub
            best_choice["downloads"] = sub["SubDownloadsCnt"]
    print best_choice["best"], best_choice["downloads"]
    if best_choice["best"] is not None:
        download_subtitle(best_choice["best"], ep_info)
    else:
        print "Can't match subtitle..."

def download_subtitle(subtitle_info, ep_info):
    """
    episode_info:
    {u'mimetype': u'video/x-flv', u'episodeNumber': 11, u'container': u'flv',
    u'title': u'Adventures in Babysitting', u'series': u'Supernatural',
    u'type': u'episode', u'season': 7, u'filename':<full file path>}
    """
    download_url = subtitle_info["SubDownloadLink"]
    subtitle_folder = os.path.dirname(ep_info['filename'])
    subtitle_folder += "/" if SUBFOLDER is None else "/" + SUBFOLDER.replace("/", "") + "/"
    subtitle_name = subtitle_folder + os.path.basename(ep_info["filename"])
    print "Downloading from {} and saving as {}.srt ".format(download_url, subtitle_name)

    if not os.path.isdir(subtitle_folder):
        os.mkdir(subtitle_folder)
        # TODO: Add exception handling, if we can't create folder we won't be able to save sub there (probably)

    sub_zip_file = urllib2.urlopen(download_url)
    try:
        sub_gzip = gzip.GzipFile(fileobj=StringIO.StringIO(sub_zip_file.read()))
        subtitle_content = sub_gzip.read()
        # TODO: Save subtitle as original filename.srt
        # TODO: Add support for subfolder to save ot
        with open(subtitle_name + '.srt', 'wb') as subtitle_output:
            subtitle_output.write(subtitle_content)
            # FIXME: This should be in try/except loop and notify of possible permission issues on fail
        print 'Done!'
    except:
        print "Couldn't save subtitle, permissions issue?"
    pass

def get_hash(name):
    """
    :param name: File path for which to calculate hash
    :return: Hash or None
    """
    try:
        longlongformat = 'q'  # long long
        bytesize = struct.calcsize(longlongformat)

        f = open(name, "rb")

        filesize = os.path.getsize(name)
        file_hash = filesize

        if filesize < 65536 * 2:
            return "SizeError"

        for x in range(65536 / bytesize):
            file_buffer = f.read(bytesize)
            (l_value,) = struct.unpack(longlongformat, file_buffer)
            file_hash += l_value
            file_hash &= 0xFFFFFFFFFFFFFFFF# to remain as 64bit number

        f.seek(max(0, filesize - 65536), 0)
        for x in range(65536 / bytesize):
            file_buffer = f.read(bytesize)
            (l_value,) = struct.unpack(longlongformat, file_buffer)
            file_hash += l_value
            file_hash &= 0xFFFFFFFFFFFFFFFF

        f.close()
        return "%016x" % file_hash

    except IOError:
        return None

if __name__ == '__main__':
    # filename="/media/8C82817682816614/TV/Supernatural/Season 7/Supernatural - [07x11] - Adventures in Babysitting.flv"
    # search_subtitles([filename])
    valid_files = []

    parser = argparse.ArgumentParser()
    parser.add_argument("folder", type=str,
                        help="Folder which will be scanned for allowed video files, and subtitles for those files will be downloaded")
    parser.add_argument("-o", "--output", type=str,
                        help="Subfolder to save subtitles to, relative to original video file path")
    parser.add_argument("-l", "--language", type=str,
                        help="Subtitle language, must be an ISO 639-1 Code i.e. (eng,fre,deu) Default English(eng); Full list http://en.wikipedia.org/wiki/List_of_ISO_639-1_codes")
    parser.add_argument("-t", "--type", type=str,
                        help="Subtitle search type 'hash' or 'filename'")
    parser.add_argument("-a", "--auto", action="store_true",
                        help="Auto download subtitles for all files without prompt (Overwrites subtitles with same filename)")
    args = parser.parse_args()

    directory = args.folder
    if not directory.endswith('/'):
        directory += '/'
    if args.output:
        SUBFOLDER = args.output
    if args.type:
        if args.type == "hash":
            hash_search = True
        elif args.type == "filename":
            hash_search = False
        else:
            print "Invalid search type 'hash' or 'filename' are available options\n defaulting to hash search"
    if args.language:
        if len(args.language) == 3:
            sub_language = args.language.lower()
        else:
            print 'Argument not  ISO 639-1 Code check this for list of valid codes http://en.wikipedia.org/wiki/List_of_ISO_639-1_codes'
            sys.exit()

    if args.auto:
        AUTO_DOWNLOAD = True
    print "Dir: {}, subfolder: {}, hash search type: {}, language: {}, auto download: {} ".format(
        directory, SUBFOLDER, hash_search, sub_language, AUTO_DOWNLOAD
    )
    names = os.listdir(directory)

    for name in names:
        file_name, file_extension = os.path.splitext(name)
        if file_extension in FILE_EXT:
            valid_files.append(directory + file_name + file_extension)

    search_subtitles(valid_files)
<|endoftext|>