<commit_before>import os
import re
from collections import namedtuple
from urllib import urlencode

from flask import (
    Flask,
    render_template,
    request,
    redirect,
    url_for,
    send_from_directory
)
import requests

app = Flask(__name__)
app.config.from_object('config')

# Special rule for old browsers to correctly handle favicon.
@app.route('/favicon.ico')
def favicon():
    return send_from_directory(
        os.path.join(app.root_path, 'static'),
        'favicon.ico',
        mimetype='image/vnd.microsoft.icon')

def error(code, message, template):
    """A generic error handler"""
    return render_template(template, message=message), code

def not_found(message=None):
    """A generic 404 handler"""
    message = message or 'Page not found.'
    return error(404, message, 'errors/404.html')

def internal_server_error(message=None):
    """A generic 500 handler"""
    message = message or 'Something went wrong.'
    return error(500, message, 'errors/500.html')

app.register_error_handler(404, not_found)
app.register_error_handler(500, internal_server_error)

@app.route('/')
def index():
    projects = app.config.get('PROJECTS', {}).keys()

    return render_template(
        'index.html',
        projects=projects)

StringError = namedtuple('StringError', ['locale', 'fn', 'errors'])

def parse_postatus(url):
    def new_section(line):
        return (line.startswith('dennis ')
                or line.startswith('Totals')
                or line.startswith('BUSTED')
                or line.startswith('COMPILED'))

    # Download the postatus file
    postatus = requests.get(url)

    # Parse it to see which locales have issues
    lines = postatus.content.splitlines()
    datestamp = lines.pop(0)

    errordata = []

    locale = None
    fn = None
    errors = []

    while lines:
        line = lines.pop(0)
        if line.startswith('>>> '):
            # total ew.
            fn = line.split(': ')[1]
            locale = fn[fn.find('locale/')+7:fn.find('/LC_MESSAGES')]

            line = lines.pop(0)
            while lines and not new_section(line):
                errors.append(line)
                line = lines.pop(0)

            errordata.append(
                StringError(locale, fn, '\n'.join(errors))
            )
            locale = None
            fn = None
            errors = []

    if errors:
        errordata.append(
            StringError(locale, fn, '\n'.join(errors))
        )

    return datestamp, errordata

L10N_COMPONENTS_CACHE = None

def get_component_for_locale(locale):
    global L10N_COMPONENTS_CACHE
    if not L10N_COMPONENTS_CACHE:
        resp = requests.get('https://bugzilla.mozilla.org/rest/product/Mozilla%20Localizations')
        data = resp.json()
        comps = [
            comp['name'] for comp in data['products'][0]['components']
        ]
        L10N_COMPONENTS_CACHE = dict(
            [(comp.split(' ')[0], comp) for comp in comps]
        )

    return L10N_COMPONENTS_CACHE[locale.replace('_', '-')]

SUMMARY = "%(product)s: errors in strings"
DESC = """\
We found errors in the translated strings for %(product)s
<%(url)s>. The errors are as follows:

%(errors)s

%(product)s strings can be fixed in the %(product)s project in Verbatim
<%(verbatim_locale_url)s>.

Once that is fixed we can pick up all recent changes in %(locale)s. Thanks
a lot for your work on %(product)s!

If you have any questions, please let us know.
"""

def generate_bug_url(project, locale, errortext):
    context = {
        'url': project['url'],
        'product': project['name'],
        'verbatim_locale_url': project['verbatim_locale_url'] % locale,
        'locale': locale,
        'errors': errortext,
    }

    data = {
        'product': 'Mozilla Localizations',
        'component': get_component_for_locale(locale),
        'format': '__standard__',
        'short_desc': SUMMARY % context,
        'comment': DESC % context,
        'rep_platform': 'all',
        'op_sys': 'all'
    }

    return 'https://bugzilla.mozilla.org/enter_bug.cgi?' + urlencode(data)

@app.route('/p/<project>')
def view_postatus(project):
    projdata = app.config.get('PROJECTS', {}).get(project, {})

    datestamp = ''
    errors = []
    serrors = []

    if not projdata:
        errors.append('Project "%s" does not exist.' % project)

    if 'postatus_url' not in projdata:
        errors.append('Project has no postatus_url configured.')

    else:
        datestamp, serrors = parse_postatus(projdata['postatus_url'])

    return render_template(
        'postatus.html',
        project=projdata,
        errors=errors,
        datestamp=datestamp,
        serrors=serrors,
        generate_bug_url=generate_bug_url)
<commit_msg>Fix Request-URI too large problem<commit_after>import os
import re
from collections import namedtuple
from urllib import urlencode

from flask import (
    Flask,
    render_template,
    request,
    redirect,
    url_for,
    send_from_directory
)
import requests

app = Flask(__name__)
app.config.from_object('config')

# Special rule for old browsers to correctly handle favicon.
@app.route('/favicon.ico')
def favicon():
    return send_from_directory(
        os.path.join(app.root_path, 'static'),
        'favicon.ico',
        mimetype='image/vnd.microsoft.icon')

def error(code, message, template):
    """A generic error handler"""
    return render_template(template, message=message), code

def not_found(message=None):
    """A generic 404 handler"""
    message = message or 'Page not found.'
    return error(404, message, 'errors/404.html')

def internal_server_error(message=None):
    """A generic 500 handler"""
    message = message or 'Something went wrong.'
    return error(500, message, 'errors/500.html')

app.register_error_handler(404, not_found)
app.register_error_handler(500, internal_server_error)

@app.route('/')
def index():
    projects = app.config.get('PROJECTS', {}).keys()

    return render_template(
        'index.html',
        projects=projects)

StringError = namedtuple('StringError', ['locale', 'fn', 'errors'])

def parse_postatus(url):
    def new_section(line):
        return (line.startswith('dennis ')
                or line.startswith('Totals')
                or line.startswith('BUSTED')
                or line.startswith('COMPILED'))

    # Download the postatus file
    postatus = requests.get(url)

    # Parse it to see which locales have issues
    lines = postatus.content.splitlines()
    datestamp = lines.pop(0)

    errordata = []

    locale = None
    fn = None
    errors = []

    while lines:
        line = lines.pop(0)
        if line.startswith('>>> '):
            # total ew.
            fn = line.split(': ')[1]
            locale = fn[fn.find('locale/')+7:fn.find('/LC_MESSAGES')]

            line = lines.pop(0)
            while lines and not new_section(line):
                errors.append(line)
                line = lines.pop(0)

            errordata.append(
                StringError(locale, fn, '\n'.join(errors))
            )
            locale = None
            fn = None
            errors = []

    if errors:
        errordata.append(
            StringError(locale, fn, '\n'.join(errors))
        )

    return datestamp, errordata

L10N_COMPONENTS_CACHE = None

def get_component_for_locale(locale):
    global L10N_COMPONENTS_CACHE
    if not L10N_COMPONENTS_CACHE:
        resp = requests.get('https://bugzilla.mozilla.org/rest/product/Mozilla%20Localizations')
        data = resp.json()
        comps = [
            comp['name'] for comp in data['products'][0]['components']
        ]
        L10N_COMPONENTS_CACHE = dict(
            [(comp.split(' ')[0], comp) for comp in comps]
        )

    return L10N_COMPONENTS_CACHE[locale.replace('_', '-')]

SUMMARY = "%(product)s: errors in strings"
DESC = """\
We found errors in the translated strings for %(product)s
<%(url)s>. The errors are as follows:

%(errors)s

%(product)s strings can be fixed in the %(product)s project in Verbatim
<%(verbatim_locale_url)s>.

Once that is fixed we can pick up all recent changes in %(locale)s. Thanks
a lot for your work on %(product)s!

If you have any questions, please let us know.
"""

def generate_bug_url(project, locale, errortext):
    if len(errortext) > 2000:
        errortext = 'TOO MANY ERRORS: PLEASE COPY AND PASTE ERRORS HERE'

    context = {
        'url': project['url'],
        'product': project['name'],
        'verbatim_locale_url': project['verbatim_locale_url'] % locale,
        'locale': locale,
        'errors': errortext,
    }

    data = {
        'product': 'Mozilla Localizations',
        'component': get_component_for_locale(locale),
        'format': '__standard__',
        'short_desc': SUMMARY % context,
        'comment': DESC % context,
        'rep_platform': 'all',
        'op_sys': 'all'
    }

    return 'https://bugzilla.mozilla.org/enter_bug.cgi?' + urlencode(data)

@app.route('/p/<project>')
def view_postatus(project):
    projdata = app.config.get('PROJECTS', {}).get(project, {})

    datestamp = ''
    errors = []
    serrors = []

    if not projdata:
        errors.append('Project "%s" does not exist.' % project)

    if 'postatus_url' not in projdata:
        errors.append('Project has no postatus_url configured.')

    else:
        datestamp, serrors = parse_postatus(projdata['postatus_url'])

    return render_template(
        'postatus.html',
        project=projdata,
        errors=errors,
        datestamp=datestamp,
        serrors=serrors,
        generate_bug_url=generate_bug_url)
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function
import Adafruit_BBIO.UART as uart
import Adafruit_BBIO.GPIO as gpio
from Adafruit_Thermal import *
import requests
import time
import atexit
import ssd1351 as ssd

BG_COLOR = 0x000000
TEXT_COLOR = 0x99f0f0
button = 'P8_37'
LINEWIDTH = 21 # with this font, we can fit 21 chars on a line
to_print = False
r = ' '

class the_screen(object):

    def __init__(self):
        self.text = 'awaiting update'

    def update_screens(self):
        req = requests.get(curriculum, params=payload)
        # print(r.text)
        resp = req.json()
        print('encoding is',req.encoding)
        keyword = resp['data']['noun_phrase']
        # print(keyword, 'utf-8')
        print(t)
        try:
            t = keyword.decode('utf-8','ignore')
        except UnicodeEncodeError:
            print(keyword)
            print('ERROR IN ABOVE LINE')
            t = keyword.encode('utf-8','ignore')

        disp.clear_display()
        disp.fillScreen(BG_COLOR)
        i=3 # initial vertical offset
        k = t.split('\n')
        # a = ["really long sentence with lots of words","short thing", "really long thing again really long hello hello", "cat pajamas"]
        print(k)
        colorweird = False # we differently-color multiline entries
        for p in k:
            buff = ""
            # disp.draw_text_bg(5, 5+i, p, TEXT_COLOR, BG_COLOR)
            for j in p.split(' '):
                j += " "
                if len(buff + j) > LINEWIDTH:
                    print('should wrap ', buff+j)
                    disp.draw_text_bg(0, i, buff, TEXT_COLOR+30, BG_COLOR)
                    colorweird = True
                    print(i)
                    i += 10
                    buff = " " + j
                else:
                    buff = buff + j
            if len(buff):
                print(i)
                if colorweird:
                    disp.draw_text_bg(0, i, buff, TEXT_COLOR+30, BG_COLOR)
                    colorweird = False
                else:
                    disp.draw_text_bg(0, i, buff, TEXT_COLOR, BG_COLOR)
                i+=10
            i+=10
        self.text = t

oled = the_screen()

def exit_handler():
    gpio.cleanup()

def ppp(a):
    # print('printing! %s' % r)
    # printer.print(r)
    # printer.print(oled.text)
    # printer.feed(4)
    pass

curriculum = 'http://curriculum.local/api'
payload = {'n':3} # three phrases
# print('about to create disp')
disp = ssd.SSD1351()
atexit.register(exit_handler)
disp.begin()
gpio.setup(button, gpio.IN)
gpio.add_event_detect(button, gpio.FALLING, callback=ppp)
uart.setup('UART1')
# print('just began UART')
printer = Adafruit_Thermal('/dev/ttyO1', 19200, timeout=5)
printer.begin()
keyword = u"" # does this solve the ridic encoding prob

# def update_screens():
#     req = requests.get(curriculum, params=payload)
#     # print(r.text)
#     resp = req.json()
#     print('encoding is',req.encoding)
#     keyword = resp['data']
#     # print(keyword, 'utf-8')
#     try:
#         t = keyword.decode('utf-8','ignore')
#     except UnicodeEncodeError:
#         print(keyword)
#         print('ERROR IN ABOVE LINE')
#         t = keyword.encode('utf-8','ignore')

#     disp.clear_display()
#     disp.fillScreen(BG_COLOR)
#     i=3 # initial vertical offset
#     k = t.split('\n')
#     # a = ["really long sentence with lots of words","short thing", "really long thing again really long hello hello", "cat pajamas"]
#     print(k)
#     colorweird = False # we differently-color multiline entries
#     for p in k:
#         buff = ""
#         # disp.draw_text_bg(5, 5+i, p, TEXT_COLOR, BG_COLOR)
#         for j in p.split(' '):
#             j += " "
#             if len(buff + j) > LINEWIDTH:
#                 print('should wrap ', buff+j)
#                 disp.draw_text_bg(0, i, buff, TEXT_COLOR+30, BG_COLOR)
#                 colorweird = True
#                 print(i)
#                 i += 10
#                 buff = " " + j
#             else:
#                 buff = buff + j
#         if len(buff):
#             print(i)
#             if colorweird:
#                 disp.draw_text_bg(0, i, buff, TEXT_COLOR+30, BG_COLOR)
#                 colorweird = False
#             else:
#                 disp.draw_text_bg(0, i, buff, TEXT_COLOR, BG_COLOR)
#             i+=10
#         i+=10
#     return t

while True:
    oled.update_screens()
    time.sleep(10)<commit_msg>testing now<commit_after>#!/usr/bin/env python
from __future__ import print_function
import Adafruit_BBIO.UART as uart
import Adafruit_BBIO.GPIO as gpio
from Adafruit_Thermal import *
import requests
import time
import atexit
import ssd1351 as ssd

BG_COLOR = 0x000000
TEXT_COLOR = 0x99f0f0
button = 'P8_37'
LINEWIDTH = 21 # with this font, we can fit 21 chars on a line
to_print = False
r = ' '

class the_screen(object):

    def __init__(self):
        self.text = 'awaiting update'

    def update_screens(self):
        req = requests.get(curriculum, params=payload)
        # print(r.text)
        resp = req.json()
        print('encoding is',req.encoding)
        keyword = resp['data']['noun_phrase']
        print(keyword, 'utf-8')
        try:
            t = keyword.decode('utf-8','ignore')
        except UnicodeEncodeError:
            print(keyword)
            print('ERROR IN ABOVE LINE')
            t = keyword.encode('utf-8','ignore')

        disp.clear_display()
        disp.fillScreen(BG_COLOR)
        i=3 # initial vertical offset
        k = t.split('\n')
        # a = ["really long sentence with lots of words","short thing", "really long thing again really long hello hello", "cat pajamas"]
        print(k)
        colorweird = False # we differently-color multiline entries
        for p in k:
            buff = ""
            # disp.draw_text_bg(5, 5+i, p, TEXT_COLOR, BG_COLOR)
            for j in p.split(' '):
                j += " "
                if len(buff + j) > LINEWIDTH:
                    print('should wrap ', buff+j)
                    disp.draw_text_bg(0, i, buff, TEXT_COLOR+30, BG_COLOR)
                    colorweird = True
                    print(i)
                    i += 10
                    buff = " " + j
                else:
                    buff = buff + j
            if len(buff):
                print(i)
                if colorweird:
                    disp.draw_text_bg(0, i, buff, TEXT_COLOR+30, BG_COLOR)
                    colorweird = False
                else:
                    disp.draw_text_bg(0, i, buff, TEXT_COLOR, BG_COLOR)
                i+=10
            i+=10
        self.text = t

oled = the_screen()

def exit_handler():
    gpio.cleanup()

def ppp(a):
    # print('printing! %s' % r)
    # printer.print(r)
    # printer.print(oled.text)
    # printer.feed(4)
    pass

curriculum = 'http://curriculum.local/api'
payload = {'n':3} # three phrases
# print('about to create disp')
disp = ssd.SSD1351()
atexit.register(exit_handler)
disp.begin()
gpio.setup(button, gpio.IN)
gpio.add_event_detect(button, gpio.FALLING, callback=ppp)
uart.setup('UART1')
# print('just began UART')
printer = Adafruit_Thermal('/dev/ttyO1', 19200, timeout=5)
printer.begin()
keyword = u"" # does this solve the ridic encoding prob

# def update_screens():
#     req = requests.get(curriculum, params=payload)
#     # print(r.text)
#     resp = req.json()
#     print('encoding is',req.encoding)
#     keyword = resp['data']
#     # print(keyword, 'utf-8')
#     try:
#         t = keyword.decode('utf-8','ignore')
#     except UnicodeEncodeError:
#         print(keyword)
#         print('ERROR IN ABOVE LINE')
#         t = keyword.encode('utf-8','ignore')

#     disp.clear_display()
#     disp.fillScreen(BG_COLOR)
#     i=3 # initial vertical offset
#     k = t.split('\n')
#     # a = ["really long sentence with lots of words","short thing", "really long thing again really long hello hello", "cat pajamas"]
#     print(k)
#     colorweird = False # we differently-color multiline entries
#     for p in k:
#         buff = ""
#         # disp.draw_text_bg(5, 5+i, p, TEXT_COLOR, BG_COLOR)
#         for j in p.split(' '):
#             j += " "
#             if len(buff + j) > LINEWIDTH:
#                 print('should wrap ', buff+j)
#                 disp.draw_text_bg(0, i, buff, TEXT_COLOR+30, BG_COLOR)
#                 colorweird = True
#                 print(i)
#                 i += 10
#                 buff = " " + j
#             else:
#                 buff = buff + j
#         if len(buff):
#             print(i)
#             if colorweird:
#                 disp.draw_text_bg(0, i, buff, TEXT_COLOR+30, BG_COLOR)
#                 colorweird = False
#             else:
#                 disp.draw_text_bg(0, i, buff, TEXT_COLOR, BG_COLOR)
#             i+=10
#         i+=10
#     return t

while True:
    oled.update_screens()
    time.sleep(10)<|endoftext|>


<commit_before>import ConfigParser, os
import dircache
import dbus

"""
The registry of managers takes the form of any number of .manager files, which
are searched for in /usr/share/telepathy/services or in ~/.telepathy.

.manager files should have an initial stanza of the form:

[ConnectionManager]
Name = value
ServiceName = value
ManagerObject = value

where:
'Name' field sets the name of connection manager.
'ServiceName' sets the DBus bus name of this connetion manager.
'ManagerObject' sets the Dbus object path to the ConnectionManager object under this service.

Then any number of proctol support declarators of the form:

[Proto (name of supported protocol)]
MandatoryParams = list of values
OptionalParams = list of values
default-(paramater name) = value

Where:
MandatoryParams is a comma-seperated list of mandatory parameters that must be passed to Connections created by this ConnectionManager for this protocol. Each should be of the form dbus type signature:name, eg s:password, indicating a paramater called 'password' of type string.
OptionalParams is a comma-separated list of optional params, of same form as MandatoryParams.
default-(paramater name) sets the default value for that parameter. e.g. default-port=522 sets te default value of the 'port' parameter to 522.

All connection managers should register as activatable dbus services. They should also close themselves down after an idle time with no open connections.

Clients should use the Proto sections to query the user for necessary informatoin.

Telepathy defines a common subset of paramter names to facilitate GUI design.

s:server - a fully qualified domain name or numeric IPv4 or IPv6 address. Using the fully-qualified domain name form is RECOMMENDED whenever possible. If this paramter is specified and the user id for that service also specifies a server, this parameter should override that in the user id.

q:port - a TCP or UDP port number. If this paramter is specified and the user id for that service also specifies a port, this parameter should override that in the user id.

s:password - A password associated with the user.

s:proxy-server - a uri for a proxyserver to use for this connection

b:require-encryption - require encryption for this connection. A connection
should fail if require-encryption is set and encryption is not possible.

UIs should display any default values, but should *not* store them.
"""

class ManagerRegistry:
    def __init__(self):
        self.services = {}

    def LoadManagers(self):
        """
        Searches local and system wide configurations

        Can raise all ConfigParser errors. Generally filename member will be
        set to the name of the erronous file.
        """
        config = ConfigParser.SafeConfigParser()
        all_services=[]
        if os.path.exists("/usr/share/telepathy/services/"):
            all_services += dircache.listdir("/usr/share/telepathy/services/")
        if os.path.exists(os.path.expanduser("~/.telepathy")):
            all_services += dircache.listdir(os.path.expanduser("~/.telepathy"))
        for service in all_services:
            config.read(service)
            connection_manager =dict(config.items("ConnectionManager"))
            if "name" not in connection_manager.keys():
                raise ConfigParser.NoOptionError("name","ConnectionManager")
            self.services[connection_manager["name"]]=connection_manager
            for section in set(config.sections()) - set(["ConnectionManager"]):
                if section[:6]=="Proto ":
                    self.services[connection_manager["name"]]["protos"]={section[6:]:dict(config.items(section))}

    def GetProtos(self):
        """
        returns a list of protocols supported on this system
        """
        protos=[]
        for service in self.services.keys():
            if self.services[service].has_key("protos"):
                protos.extend(self.services[service]["protos"].keys())
        return protos

    def GetManagers(self, proto):
        """
        Returns names of managers that can handle the given protocol.
        """
        managers = []
        for service in self.services.keys():
            if self.services[service].has_key("protos"):
                if self.services[service]["protos"][proto]:
                    managers.append(service)
        return managers

    def GetParams(self, manager, proto):
        """
        Returns two dicts of paramters for the given proto on the given manager.
        One dict of mandatory parameters, one of optional.
        The keys will be the parameters names, and the values a tuple of
        (dbus type, default value). If no default value is specified, the second
        item in the tuple will be None.
        """
        ret=()

        for field in ["MandatoryParams","OptionalParams"]:
            params={}
            for item in split(self.services[manager][proto][field],','):
                type, name = split(strip(item),':')
                default=None
                for key in self.services[manager][proto].keys():
                    strip(key)
                    if key=="Default-"+name:
                        default=key[8:]
                        break
                if default:
                    params[name]=(type, dbus.Variant(default,signature=type))
                else:
                    params[name]=(type, None)
            ret.append(params)

        return ret

<commit_msg>lot of fixes in view of trying to use this with two manager definitions<commit_after>import ConfigParser, os
import dircache
import dbus

"""
The registry of managers takes the form of any number of .manager files, which
are searched for in /usr/share/telepathy/services or in ~/.telepathy.

.manager files should have an initial stanza of the form:

[ConnectionManager]
Name = value
ServiceName = value
ManagerObject = value

where:
'Name' field sets the name of connection manager.
'ServiceName' sets the DBus bus name of this connetion manager.
'ManagerObject' sets the Dbus object path to the ConnectionManager object under this service.

Then any number of proctol support declarators of the form:

[Proto (name of supported protocol)]
MandatoryParams = list of values
OptionalParams = list of values
default-(paramater name) = value

Where:
MandatoryParams is a comma-seperated list of mandatory parameters that must be passed to Connections created by this ConnectionManager for this protocol. Each should be of the form dbus type signature:name, eg s:password, indicating a paramater called 'password' of type string.
OptionalParams is a comma-separated list of optional params, of same form as MandatoryParams.
default-(paramater name) sets the default value for that parameter. e.g. default-port=522 sets te default value of the 'port' parameter to 522.

All connection managers should register as activatable dbus services. They should also close themselves down after an idle time with no open connections.

Clients should use the Proto sections to query the user for necessary informatoin.

Telepathy defines a common subset of paramter names to facilitate GUI design.

s:server - a fully qualified domain name or numeric IPv4 or IPv6 address. Using the fully-qualified domain name form is RECOMMENDED whenever possible. If this paramter is specified and the user id for that service also specifies a server, this parameter should override that in the user id.

q:port - a TCP or UDP port number. If this paramter is specified and the user id for that service also specifies a port, this parameter should override that in the user id.

s:password - A password associated with the user.

s:proxy-server - a uri for a proxyserver to use for this connection

b:require-encryption - require encryption for this connection. A connection
should fail if require-encryption is set and encryption is not possible.

UIs should display any default values, but should *not* store them.
"""

class ManagerRegistry:
    def __init__(self):
        self.services = {}

    def LoadManagers(self):
        """
        Searches local and system wide configurations

        Can raise all ConfigParser errors. Generally filename member will be
        set to the name of the erronous file.
        """
        all_services=[]
        if os.path.exists("/usr/share/telepathy/services/"):
            all_services += map( lambda dir: "/usr/share/telepathy/services/"+dir, dircache.listdir("/usr/share/telepathy/services/"))
        local_path=os.path.expanduser("~/.telepathy")
        if os.path.exists(local_path):
            all_services += map( lambda dir: local_path +'/'+dir, dircache.listdir(local_path))
        print all_services
        for service in all_services:
            print service
            config = ConfigParser.SafeConfigParser()
            config.read(service)
            print config.sections()
            connection_manager =dict(config.items("ConnectionManager"))
            if "name" not in connection_manager.keys():
                print "no name"
                raise ConfigParser.NoOptionError("name","ConnectionManager")
            self.services[connection_manager["name"]]=connection_manager
            for section in set(config.sections()) - set(["ConnectionManager"]):
                print connection_manager["name"]
                print section
                if section[:6]=="Proto ":
                    self.services[connection_manager["name"]]["protos"]={section[6:]:dict(config.items(section))}
                    print self.services[connection_manager["name"]]["protos"]
            del config

    def GetProtos(self):
        """
        returns a list of protocols supported on this system
        """
        protos=[]
        for service in self.services.keys():
            if self.services[service].has_key("protos"):
                protos.extend(self.services[service]["protos"].keys())
        return protos

    def GetManagers(self, proto):
        """
        Returns names of managers that can handle the given protocol.
        """
        managers = []
        for service in self.services.keys():
            if self.services[service].has_key("protos"):
                if self.services[service]["protos"].has_key(proto):
                    managers.append(service)
        return managers

    def GetParams(self, manager, proto):
        """
        Returns two dicts of paramters for the given proto on the given manager.
        One dict of mandatory parameters, one of optional.
        The keys will be the parameters names, and the values a tuple of
        (dbus type, default value). If no default value is specified, the second
        item in the tuple will be None.
        """
        ret=()

        for field in ["MandatoryParams","OptionalParams"]:
            params={}
            for item in split(self.services[manager][proto][field],','):
                type, name = split(strip(item),':')
                default=None
                for key in self.services[manager][proto].keys():
                    strip(key)
                    if key=="Default-"+name:
                        default=key[8:]
                        break
                if default:
                    params[name]=(type, dbus.Variant(default,signature=type))
                else:
                    params[name]=(type, None)
            ret.append(params)

        return ret

<|endoftext|>


<commit_before>
import os
from urlparse import urlparse

import click

import adapters
import utils
from filters import BasicFilterer

@click.command()
@click.argument('sources', type=click.Path(exists=True), required=True)
@click.argument('output', type=click.Path(exists=True), required=True)
@click.option('--force', is_flag=True)
def process(sources, output, force):
    """Download sources and process the file to the output directory.

    \b
    SOURCES: Source JSON file or directory of files. Required.
    OUTPUT: Destination directory for generated data. Required.
    """
    catalog_features = []
    for path in utils.get_files(sources):
        pathparts = utils.get_path_parts(path)
        pathparts[0] = output.strip(os.sep)
        pathparts[-1] = pathparts[-1].replace('.json', '.geojson')

        outdir = os.sep.join(pathparts[:-1])
        outfile = os.sep.join(pathparts)

        source = utils.read_json(path)
        urlfile = urlparse(source['url']).path.split('/')[-1]

        if not hasattr(adapters, source['filetype']):
            utils.error('Unknown filetype', source['filetype'], '\n')
            continue

        if os.path.isfile(outfile) and not force:
            utils.error('Skipping', path, 'since generated file exists.',
                        'Use --force to regenerate.', '\n')
            continue

        utils.info('Downloading', source['url'])

        try:
            fp = utils.download(source['url'])
        except IOError:
            utils.error('Failed to download', source['url'], '\n')
            continue

        utils.info('Reading', urlfile)

        if 'filter' in source:
            filterer = BasicFilterer(source['filter'], source.get('filter_operator', 'and'))
        else:
            filterer = None

        try:
            geojson = getattr(adapters, source['filetype'])\
                .read(fp, source['properties'],
                    filterer=filterer,
                    source_filename=source.get("filenameInZip", None))
        except IOError:
            utils.error('Failed to read', urlfile)
            continue
        finally:
            os.remove(fp.name)

        excluded_keys = ['filetype', 'url', 'properties', 'filter', 'filenameInZip']
        properties = {k:v for k,v in source.iteritems() if k not in excluded_keys}
        properties['source_url'] = source['url']

        geojson['properties'] = properties

        utils.make_sure_path_exists(outdir)
        utils.write_json(outfile, geojson)

        properties['path'] = outfile
        catalog_entry = {
            'type': 'Feature',
            'properties': properties,
            'geometry': {
                'type': 'Polygon',
                'coordinates': utils.polygon_from_bbox(geojson['bbox'])
            }
        }
        catalog_features.append(catalog_entry)

        utils.success('Done. Processed to', outfile, '\n')

    catalog = {
        'type': 'FeatureCollection',
        'features': catalog_features
    }
    utils.write_json(os.path.join(output,'catalog.geojson'), catalog)

if __name__ == '__main__':
    process()
<commit_msg>catch exceptions for invalid zip file<commit_after>
import os
from urlparse import urlparse
import zipfile

import click

import adapters
import utils
from filters import BasicFilterer

@click.command()
@click.argument('sources', type=click.Path(exists=True), required=True)
@click.argument('output', type=click.Path(exists=True), required=True)
@click.option('--force', is_flag=True)
def process(sources, output, force):
    """Download sources and process the file to the output directory.

    \b
    SOURCES: Source JSON file or directory of files. Required.
    OUTPUT: Destination directory for generated data. Required.
    """
    catalog_features = []
    for path in utils.get_files(sources):
        pathparts = utils.get_path_parts(path)
        pathparts[0] = output.strip(os.sep)
        pathparts[-1] = pathparts[-1].replace('.json', '.geojson')

        outdir = os.sep.join(pathparts[:-1])
        outfile = os.sep.join(pathparts)

        source = utils.read_json(path)
        urlfile = urlparse(source['url']).path.split('/')[-1]

        if not hasattr(adapters, source['filetype']):
            utils.error('Unknown filetype', source['filetype'], '\n')
            continue

        if os.path.isfile(outfile) and not force:
            utils.error('Skipping', path, 'since generated file exists.',
                        'Use --force to regenerate.', '\n')
            continue

        utils.info('Downloading', source['url'])

        try:
            fp = utils.download(source['url'])
        except IOError:
            utils.error('Failed to download', source['url'], '\n')
            continue

        utils.info('Reading', urlfile)

        if 'filter' in source:
            filterer = BasicFilterer(source['filter'], source.get('filter_operator', 'and'))
        else:
            filterer = None

        try:
            geojson = getattr(adapters, source['filetype'])\
                .read(fp, source['properties'],
                    filterer=filterer,
                    source_filename=source.get("filenameInZip", None))
        except IOError:
            utils.error('Failed to read', urlfile)
            continue
        except zipfile.BadZipfile, e:
            utils.error('Unable to open zip file', source['url'])
            continue
        finally:
            os.remove(fp.name)

        excluded_keys = ['filetype', 'url', 'properties', 'filter', 'filenameInZip']
        properties = {k:v for k,v in source.iteritems() if k not in excluded_keys}
        properties['source_url'] = source['url']

        geojson['properties'] = properties

        utils.make_sure_path_exists(outdir)
        utils.write_json(outfile, geojson)

        properties['path'] = outfile
        catalog_entry = {
            'type': 'Feature',
            'properties': properties,
            'geometry': {
                'type': 'Polygon',
                'coordinates': utils.polygon_from_bbox(geojson['bbox'])
            }
        }
        catalog_features.append(catalog_entry)

        utils.success('Done. Processed to', outfile, '\n')

    catalog = {
        'type': 'FeatureCollection',
        'features': catalog_features
    }
    utils.write_json(os.path.join(output,'catalog.geojson'), catalog)

if __name__ == '__main__':
    process()
<|endoftext|>


<commit_before>from driver import ensure_path
import numpy as np
from fitstools import combine, save_2darr
from astropy.io import fits
from find_fibers import find_fibers
from throughput import make_throughput_map
from sys import stdout

def get_recipes(dname, recipe=None, rtype=None):
    if recipe == None:
        recipe = 'recipes/'+dname+'.recipe'

    #Extract lines from recipe file that pertain to flat fields.
    recipe_lines = [line for line in filter(None, open(recipe).read().split('\n')) if line[0] != '#']
    if rtype != None:
        rtype_recipes = [line for line in recipe_lines if len(line.split(',')) > 2 and line.split(',')[1] == rtype]
        return rtype_recipes
    return recipe_lines

def process_bias(dname, recipe=None, output=stdout):
    bias_recipes = get_recipes(dname, recipe, rtype='zero')

    filenames = []
    for r in bias_recipes:
        info = r.split(',')
        fnames = info[2].split(' ')
        filenames.extend(fnames)

    calib_imgs_dir = 'calib/'+dname+'/cal_images/'
    ensure_path(calib_imgs_dir)

    #Load files and create master bias
    biases = [fits.open('indata/'+dname+'/'+fname) for fname in filenames]
    master_bias = combine(*biases, method='mean')
    master_bias.writeto(calib_imgs_dir+'master_bias.fits', clobber=True)
    for f in biases:
        f.close()

    #Load all other files make a bias-corrected version
    all_recipes = get_recipes(dname, recipe)
    all_filenames = []
    for r in all_recipes:
        info = r.split(',')
        fnames = filter(None, info[2].split(' '))
        for fname in fnames:
            if not fname in all_filenames:
                all_filenames.append(fname)
    master_bias_data = master_bias[0].data
    num_imgs = len(all_filenames)
    for i,fname in enumerate(all_filenames):
        output.write('\rBias correcting '+dname+' images: '+str(i+1)+'/'+str(num_imgs))
        output.flush()
        f = fits.open('indata/'+dname+'/'+fname)
        f[0].data  = f[0].data - master_bias_data
        f[0].header['COMMENT'] = 'This image is bias corrected.'
        new_fname = fname.split('.')[0]+'_b.fits'
        f.writeto('calib/'+dname+'/cal_images/'+new_fname, clobber=True)
        f.close()
    output.write('\n')

    #Save bias-corrected images in calib/dname/calib_images

def process_flat(dname, recipe=None, output=stdout):
    flat_recipes = get_recipes(dname, recipe, rtype='flat')

    num_r = len(flat_recipes)
    for i,r in enumerate(flat_recipes):
        output.write('\rProcessing '+dname+' flats: '+str(i+1)+'/'+str(num_r))
        output.flush()
        flat_dorecipe(r, dname)
    output.write('\n')

def flat_dorecipe(r, dname):
    info = r.split(',')
    group_num = info[0]
    rtype = info[1]
    recipe_filenames = info[2].split(' ')
    filenames = [fname.split('.')[0]+'_b.fits' for fname in recipe_filenames]
    use_fibers = [int(f_num) for f_num in info[3].split(' ')]

    #Make directory in calib for this group number
    group_dir = 'calib/'+dname+'/group'+group_num+'/'
    ensure_path(group_dir)

    #Median combine flats, save to calib directory
    flats = [fits.open('calib/'+dname+'/cal_images/'+filename) for filename in filenames]
    master_flat = combine(*flats)
    master_flat.writeto(group_dir+'master_flat.fits', clobber=True)

    #Find fibers, make fiber mask, save to directory
    fiber_mask = find_fibers(master_flat, use_fibers)
    fits.writeto(group_dir+'fiber_mask.fits', fiber_mask, clobber=True)

    #Generate a fiber thoughput map.
    throughput_map = make_throughput_map(fiber_mask, master_flat)
    fits.writeto(group_dir+'throughput_map.fits', throughput_map, clobber=True)
<commit_msg>Reorganized functions for dealing with recipes.<commit_after>from driver import ensure_path
import numpy as np
from fitstools import combine, save_2darr, mask_fits
from astropy.io import fits
from find_fibers import find_fibers
from throughput import make_throughput_map
from find_wvlsol import wvlsol
from sys import stdout

#Define suffixes for bias-corrected and throghput-corrected images.
bc_suffix = '_bc'
tc_suffix = '_tc'

def get_recipes(dname, recipe=None, gnum=None, rtype=None):
    if recipe == None:
        recipe = 'recipes/'+dname+'.recipe'

    #Extract lines from recipe file that pertain to flat fields.
    recipe_lines = [line for line in filter(None, open(recipe).read().split('\n')) if line[0] != '#']
    if rtype != None:
        recipe_lines = [line for line in recipe_lines if len(line.split(',')) > 2 and line.split(',')[1] == rtype]
    if gnum != None:
        gnum = int(gnum)
        recipe_lines = [line for line in recipe_lines if len(line.split(',')) > 1 and int(line.split(',')[0]) == gnum]
    return recipe_lines

def get_num_groups(dname, recipe):
    if recipe == None:
        recipe = 'recipes/'+dname+'.recipe'
    f = open(recipe)
    recipe_lines = [line for line in filter(None, f.read().split('\n')) if line[0] != '#']
    f.close()
    group_nums = [int(line.split(',')[0]) for line in recipe_lines]
    num_groups = max(group_nums)
    return num_groups

def process_bias(dname, recipe=None, output=stdout):
    num_groups = get_num_groups(dname, recipe)
    bias_recipes = get_recipes(dname, recipe, rtype='zero')

    calib_dir = 'calib/'+dname
    ensure_path(calib_dir+'/')

    filenames = []
    for r in bias_recipes:
        info = r.split(',')
        fnames = info[2].split(' ')
        filenames.extend(fnames)

    #Load files and create master bias
    biases = [fits.open('indata/'+dname+'/'+fname) for fname in filenames]
    master_bias = combine(*biases, method='mean')
    master_bias.writeto(calib_dir+'/master_bias.fits', clobber=True)
    for f in biases:
        f.close()

    #Load all other files make a bias-corrected version
    master_bias_data = master_bias[0].data
    for gnum in range(1, num_groups+1):
        group_recipes = get_recipes(dname, recipe, gnum=gnum)
        group_dir = calib_dir+'/group'+str(gnum)+'/'
        ensure_path(group_dir)
        fnames = []
        for r in group_recipes:
            fnames.extend(filter(None, r.split(',')[2].split(' ')))
        num_images = len(fnames)
        for i,fname in enumerate(fnames):
            output.write('\rBias correcting: Group '+str(gnum)+'/'+str(num_groups)+', Image '+str(i+1)+'/'+str(num_images))
            output.flush()
            f = fits.open('indata/'+dname+'/'+fname)
            f[0].data  = f[0].data - master_bias_data
            f[0].header['COMMENT'] = 'This image is bias corrected.'
            new_fname = fname.split('.')[0]+bc_suffix+'.fits'
            f.writeto(group_dir+'/'+new_fname, clobber=True)
            f.close()
    output.write('\n')

def process_flat(dname, recipe=None, output=stdout):
    flat_recipes = get_recipes(dname, recipe, rtype='flat')

    num_r = len(flat_recipes)
    for i,r in enumerate(flat_recipes):
        output.write('\rProcessing '+dname+' flats: '+str(i+1)+'/'+str(num_r))
        output.flush()
        flat_dorecipe(r, dname, recipe)
    output.write('\n')

def flat_dorecipe(r, dname, recipe):
    info = r.split(',')
    group_num = info[0]
    rtype = info[1]
    recipe_filenames = info[2].split(' ')
    filenames = [fname.split('.')[0]+bc_suffix+'.fits' for fname in recipe_filenames]
    use_fibers = [int(f_num) for f_num in info[3].split(' ')]

    #Make directory in calib for this group number
    group_dir = 'calib/'+dname+'/group'+group_num
    ensure_path(group_dir)

    #Median combine flats, save to calib directory
    flats = [fits.open(group_dir+'/'+filename) for filename in filenames]
    master_flat = combine(*flats)
    master_flat.writeto(group_dir+'/master_flat.fits', clobber=True)

    #Find fibers, make fiber mask, save to directory
    fiber_mask = find_fibers(master_flat, use_fibers)
    fits.writeto(group_dir+'/fiber_mask.fits', fiber_mask, clobber=True)

    #Generate a fiber thoughput map.
    throughput_map = make_throughput_map(fiber_mask, master_flat)
    fits.writeto(group_dir+'/throughput_map.fits', throughput_map, clobber=True)

    #Throughput correct all images from this pointing.
    #recipe_lines = get_recipes(dname, recipe, gnum=group_num)
    #for r in recipe_lines:
    #    info = r.split(',')
    #    recipe_filenames = info[2].split(' ')
    #    filenames = [fname.split('.')[0]+bc_suffix+'.fits' for fname in recipe_filenames]
    #    for fname in filenames:
    #        #Load file
    #        f = fits.open(group_dir+'/'+fname)
    #        f[0].data  = f[0].data / throughput_map
    #        f[0].header['COMMENT'] = 'This image is throughput corrected.'
    #        new_fname = fname.split(bc_suffix)[0]+tc_suffix+'.fits'
    #        f.writeto(group_dir+'/'+new_fname, clobber=True)

def process_thar(dname, recipe=None, output=stdout):
    thar_recipes = get_recipes(dname, recipe, rtype='comp')

    num_r = len(thar_recipes)
    for i,r in enumerate(thar_recipes):
        output.write('Processing '+dname+' thar: '+str(i+1)+'/'+str(num_r))
        output.flush()
        thar_dorecipe(r, dname)
    output.write('n')

def thar_dorecipe(r, dname):
    info = r.split(',')
    group_num = info[0]
    group_dir = 'calib/'+dname+'/'+group_num
    rtype = info[1]
    recipe_filenames = info[2].split(' ')
    filenames = [fname.split('.')[0]+bc_suffix+'.fits' for fname in recipe_filenames]
    use_fibers = [int(f_num) for f_num in info[3].split(' ')]

    #Make directory in calib for this group number
    group_dir = 'calib/'+dname+'/group'+group_num
    ensure_path(group_dir+'/')

    tm = fits.open(group_dir+'/throughput_map.fits')
    throughput_map = tm[0].data
    tm.close()
    fm = fits.open(group_dir+'/fiber_mask.fits')
    fiber_mask = fm[0].data
    fm.close()
    for fname in filenames:
       comp = fits.open(group_dir+'/'+fname)
       comp[0].data = comp[0].data / throughput_map
       new_fname = fname.split(bc_suffix)[0]+tc_suffix+'.fits'
       comp.writeto(group_dir+'/'+new_fname, clobber=True)
       wvlsol(comp, fiber_mask, use_fibers)
       exit()
<|endoftext|>


<commit_before>'''
Created on May 26, 2011

@author: oberman
'''

import os
import datetime
from itertools import izip

import cmd_line_io as geo_io

from parse_geo import get_parser
import argparse
from grid_geo import ValidProjections
from map_geo import ValidMaps

    """
    Process a series of files, generating some kind of output for each

    By default, files in the current folder are processed using the generic
    filetype appropriate for their extension.  Griddef is obtained
    interactively from the user if no griddef instance is passed.
    Similarly, map is obtained from user if no mapping function is passed.

    OutFuncs are obtained from the user as well.  OutFuncs must be a LIST
    of outFuncs, even if only one function is desired.  Outfuncs passed
    into the function are expected to be properly initialized.  When
    the user specifies outfunctions, the outfunctions are responsible
    for obtaining fieldnames and parameters from the user.

    OutFileNames are expected to passed as a LIST of the same length
    as OutFuncs, if the user provides them.
    If no outFileNames are given, any files generated are saved into
    the current directory under the name "output<num>" where <num>
    is the index of the outFunc associated with that output.

    Ordinarily, directory signifies the directory and filelist the subset
    of files within that directory that should be processed.  If no
    argument is passed for filelist, all files in the directory are
    processed.  If no directory is passed, the current working directory
    is assumed.

    Alternately, a list of pre-instantiated parsers may be passed.  Note that
    if parsers are passed, the files in the filelist are IGNORED and the files
    with which the parsers are associated are used instead.

    If verbose is set to True, all default status updates will be printed.
    If set to False, the program will run silently
    """

parser = argparse.ArgumentParser("Process a series of files, generating some kind of output for each")

class ProjArgsAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        for string in values:
            pair = string.strip('()').split(',')
            setattr(namespace, pair[0], pair[1])
def double(string):
    if len(string.split(',')) != 2:
        msg = "%r is not correctly formatted.  Correct format: 'varName,varVal'" % string
        raise argparse.ArgumentTypeError(msg)
    return string
parser.add_argument('--directory', help='The directory containing the files to process (default: current working directory)')
parser.add_argument('--filelist', help='The list of files in the directory to process (default: process all files')
parser.add_argument('--gridProj', help='Supply a valid grid projection type with which to form a grid', choices=ValidProjections())
parser.add_argument('--progAttrs', nargs='*', action= ProjArgsAction, type=double, help='Supply the attributes required for the projection')
parser.add_argument('--mapFunc', help='Supply a valid mapping function.', choices = ValidMaps())
parser.add_argument('--outFuncs', nargs='+', help='Supply desired output function(s)', choices=ValidOutfuncs())
parser.add_argument('--outFileNames', nargs='*', help='Optionally, supply the names of the respective output files to be used for each output function.  If none are provided, the output files will be named \'output1\', \'output2\', ...')
parser.add_argument('--extension', help='Optional override extension')
parser.add_argument('--subtype', help='Optional file subtype', default='')
parser.add_argument('--verbose', help='Supply False here to disable verbose execution', default=True)
parser.add_argument('--interactive', help='Supply True here to enable interactive error handling in the event that the program encounters an invalid input file. (default behavior ignores any invalid files and continues processing all requested files)', default=False)

gnomespice = parser.parse_args()

directory = gnomespice.directory or os.getcwd()
filelist = gnomespice.filelist or os.listdir(directory)
gridDict = dict()
for attr in gnomespice.gridProj.requiredParms():
    try:
        gridDict[attr] = getattr(gnomespice, attr)
    except AttributeError:
        raise argparse.ArgumentError('Missing argument %r, which is required for selected projection. Please include a value for %r.' % (attr, attr))
griddef = projClass(gridDict)
outFileNames = gnomespice.outFileNames or \
    [os.path.join(directory, 'output%s' % i) for i in range(len(outFuncs))]
badfile = gnomespice.interactive and bad_file or bad_file_default

if parserList:
    if verbose: print 'Ignoring filelist, using parserList'
    parsers = parserList
else:
    if verbose: print('building filelist '+str(datetime.datetime.now()))
    files = [os.path.join(directory, f) for f in filelist]
    parsers = []
    if verbose: print('getting parsers '+str(datetime.datetime.now()))
    for f in files:
        try:
            parsers.append(\
                get_parser(f, gnomespice.subtype, gnomespice.extension))
        except(IOError):
            answer = badfile(f)
            if answer is 1:
                return 'continue'
            elif answer is 2:
                return 'break'
            elif answer if 3:
                raise SystemExit

if verbose: print('calculating maps '+str(datetime.datetime.now()))
maps = [mapFunc(p, griddef) for p in parsers]
if verbose: print('creating outfiles '+str(datetime.datetime.now()))
outputs = [out(maps, griddef, fnames) for (out, fnames) in izip(outFuncs, outFileNames)]

# eventually, we may want to do stuff to outputs, but for now...
del(outputs)

def bad_file_default(filename):
    return 1

def bad_file(filename):
    '''
    Determine what the user wants to do when one fo the
    files turns out to be invalid.
    '''
    valid = False
    first = True
    while not valid:
        if not first:
            print("Invalid answer.  Please try again.")
        first = False
        prompt = "File %s couldn't be read properly.  What \n\
        do you wish to do?  Enter (1) to skip this file, \n\
        but continue processing other files.  Enter (2) to \n\
        stop reading files but continue with data \n\
        processing.  Enter (3) to quit this program.  Enter \n\
        your selection here: " % filename
        answer = raw_input(prompt)
        try:
            answer = int(answer)
        except(ValueError, TypeError):
            continue
        valid = answer in [1,2,3]
    return answer
<commit_msg>Processes a runnin' hey, hey\nMessages a printin'\nOn that Friday afternoon with\nOur hearts a thumpin' and all from the U\nnix terminal\nUnix Terminal\nDo you remember when we used to sing\nSha la la la la la la la la la la te da<commit_after>'''
Created on May 26, 2011

@author: oberman
'''

import os
import datetime
from itertools import izip

import cmd_line_io as geo_io

from parse_geo import get_parser
import argparse
from grid_geo import *
from map_geo import *
from out_geo import ValidOutfuncs

def bad_file_default(filename):
    return 1

def bad_file(filename):
    '''
    Determine what the user wants to do when one fo the
    files turns out to be invalid.
    '''
    valid = False
    first = True
    while not valid:
        if not first:
            print("Invalid answer.  Please try again.")
        first = False
        prompt = "File %s couldn't be read properly.  What \n\
        do you wish to do?  Enter (1) to skip this file, \n\
        but continue processing other files.  Enter (2) to \n\
        stop reading files but continue with data \n\
        processing.  Enter (3) to quit this program.  Enter \n\
        your selection here: " % filename
        answer = raw_input(prompt)
        try:
            answer = int(answer)
        except(ValueError, TypeError):
            continue
        valid = answer in [1,2,3]
    return answer

"""
Process a series of files, generating some kind of output for each

By default, files in the current folder are processed using the generic
filetype appropriate for their extension.  Griddef is obtained
interactively from the user if no griddef instance is passed.
Similarly, map is obtained from user if no mapping function is passed.

OutFuncs are obtained from the user as well.  OutFuncs must be a LIST
of outFuncs, even if only one function is desired.  Outfuncs passed
into the function are expected to be properly initialized.  When
the user specifies outfunctions, the outfunctions are responsible
for obtaining fieldnames and parameters from the user.

OutFileNames are expected to passed as a LIST of the same length
as OutFuncs, if the user provides them.
If no outFileNames are given, any files generated are saved into
the current directory under the name "output<num>" where <num>
is the index of the outFunc associated with that output.

Ordinarily, directory signifies the directory and filelist the subset
of files within that directory that should be processed.  If no
argument is passed for filelist, all files in the directory are
processed.  If no directory is passed, the current working directory
is assumed.

Alternately, a list of pre-instantiated parsers may be passed.  Note that
if parsers are passed, the files in the filelist are IGNORED and the files
with which the parsers are associated are used instead.

If verbose is set to True, all default status updates will be printed.
If set to False, the program will run silently
"""

parser = argparse.ArgumentParser("Process a series of files, generating some kind of output for each")

class ProjArgsAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        for string in values:
            pair = string.strip('()').split(',')
            setattr(namespace, pair[0], pair[1])
def double(string):
    if len(string.split(',')) != 2:
        msg = "%r is not correctly formatted.  Correct format: 'varName,varVal'" % string
        raise argparse.ArgumentTypeError(msg)
    return string
parser.add_argument('--directory', help='The directory containing the files to process (default: current working directory)')
parser.add_argument('--filelist', help='The list of files in the directory to process (default: process all files')
parser.add_argument('--gridProj', help='Supply a valid grid projection type with which to form a grid', choices=ValidProjections())
parser.add_argument('--progAttrs', nargs='*', action= ProjArgsAction, type=double, help='Supply the attributes required for the projection')
parser.add_argument('--mapFunc', help='Supply a valid mapping function.', choices = ValidMaps())
parser.add_argument('--outFuncs', nargs='+', help='Supply desired output function(s)', choices=ValidOutfuncs())
parser.add_argument('--outFileNames', nargs='*', help='Optionally, supply the names of the respective output files to be used for each output function.  If none are provided, the output files will be named \'output1\', \'output2\', ...')
parser.add_argument('--extension', help='Optional override extension')
parser.add_argument('--subtype', help='Optional file subtype', default='')
parser.add_argument('--verbose', help='Supply False here to disable verbose execution', default=True)
parser.add_argument('--interactive', help='Supply True here to enable interactive error handling in the event that the program encounters an invalid input file. (default behavior ignores any invalid files and continues processing all requested files)', default=False)

parserList = None

gnomespice = parser.parse_args()

directory = gnomespice.directory or os.getcwd()
filelist = gnomespice.filelist or os.listdir(directory)
gridDict = dict()
gridDef =  eval(gnomespice.gridProj + '_GridDef')
for attr in gridDef.requiredParms():
    try:
        gridDict[attr] = getattr(gnomespice, attr)
    except AttributeError:
        raise argparse.ArgumentError('Missing argument %r, which is required for selected projection. Please include a value for %r.' % (attr, attr))
griddef = gridDef(gridDict)
mapFunc = gnomespice.mapFunc + '_map_geo'
outFuncs = gnomespice.outFuncs
outFileNames = gnomespice.outFileNames or \
    [os.path.join(directory, 'output%s' % i) for i in range(len(outFuncs))]
verbose = gnomespice.verbose
badfile = gnomespice.interactive and bad_file or bad_file_default

if parserList:
    if verbose: print 'Ignoring filelist, using parserList'
    parsers = parserList
else:
    if verbose: print('building filelist '+str(datetime.datetime.now()))
    files = [os.path.join(directory, f) for f in filelist]
    parsers = []
    if verbose: print('getting parsers '+str(datetime.datetime.now()))
    for f in files:
        try:
            parsers.append(\
                get_parser(f, gnomespice.subtype, gnomespice.extension))
        except(IOError):
            answer = badfile(f)
            if answer is 1:
                continue
            elif answer is 2:
                break
            elif answer is 3:
                raise SystemExit

print(mapFunc)
if verbose: print('calculating maps '+str(datetime.datetime.now()))
maps = eval('[' + mapFunc + '(p, griddef) for p in parsers]')
if verbose: print('creating outfiles '+str(datetime.datetime.now()))
outputs = [out(maps, griddef, fnames) for (out, fnames) in izip(outFuncs, outFileNames)]

# eventually, we may want to do stuff to outputs, but for now...
del(outputs)

<|endoftext|>


<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

"""
Preprocessing program

This program will:
- check if there are a new file to process (not implemented yet)
- process the file getting informations (not implemented yet)
- move the file to the proper directory (not implemented yet)

"""

# preprocessing.py
# This file is part of limestonedb
#
# Copyright (C) 2014 - Enrico Polesel
#
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import hashlib
import pymongo
import datetime
import os
import importlib

debug = 5

def print_debug(level,msg):
    if level <= debug:
        print("[limestonedb preprocessing][debug] "+str(msg))

def get_file_hash_size(file_name):
    file_hash = hashlib.sha512()
    chunk_size = 1024*1024  # 1MiB
    try:
        with open(file_name, 'rb') as f:
            byte = f.read(chunk_size)
            previous_byte = byte
            byte_size = len(byte)
            while byte:
                file_hash.update(byte)
                previous_byte = byte
                byte = f.read(chunk_size)
                byte_size += len(byte)
    except IOError:
        raise ("Unable to read "+file_name)
    except:
        raise ("Error while hashing "+file_name)
    return file_hash.hexdigest() , byte_size

def create_stone(file_name):
    hashsum , size = get_file_hash_size(file_name)
    stone = {
        'name' : os.path.basename(file_name) ,
        'hash' : hashsum,
        'size' : size,
        # TODO: location,
        'insertion_date' : datetime.datetime.utcnow()
        }
    return stone

def insert_in_db(mongodb, stone):
    try:
        stones = mongodb['stones']
        insertion_id = stones.insert(stone)
    except:
        raise ("Error inserting in the db")
    else:
        return insertion_id

def scan_directory(mongodb,path):
    found = []
    for dirpath, dirs, files in os.walk(path):
        for file_name in files:
            print("Scanning "+file_name)
            try:
                stone = create_stone(os.path.join(dirpath,file_name))
            except:
                print ("Unable to process "+ os.path.join(dirpath,file_name))
            else:
                found.append(insert_in_db(mongodb,stone))
    return found

def initialize_db(url):
    try:
        client = pymongo.MongoClient(url)
        database = client['limestone']
    except:
        raise ('Database error')
    else:
        return database

def get_substone(moudule_name, file_name):
    try:
        module = importlib.import_module(module_name)
    except:
        raise("Error importing "+module_name+" module")
    try:
        substone = module.get_substone(file_name)
    except:
        raise("Error getting the "+module_name+" substone from "+file_name)
    else:
        return substone

<commit_msg>get_substone function and scan_dir modifications to handle substones<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

"""
Preprocessing program

This program will:
- check if there are a new file to process (not implemented yet)
- process the file getting informations (not implemented yet)
- move the file to the proper directory (not implemented yet)

"""

# preprocessing.py
# This file is part of limestonedb
#
# Copyright (C) 2014 - Enrico Polesel
#
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import hashlib
import pymongo
import datetime
import os
import importlib

debug = 5

def print_debug(level,msg):
    if level <= debug:
        print("[limestonedb preprocessing][debug] "+str(msg))

def get_file_hash_size(file_name):
    file_hash = hashlib.sha512()
    chunk_size = 1024*1024  # 1MiB
    try:
        with open(file_name, 'rb') as f:
            byte = f.read(chunk_size)
            previous_byte = byte
            byte_size = len(byte)
            while byte:
                file_hash.update(byte)
                previous_byte = byte
                byte = f.read(chunk_size)
                byte_size += len(byte)
    except IOError:
        raise ("Unable to read "+file_name)
    except:
        raise ("Error while hashing "+file_name)
    return file_hash.hexdigest() , byte_size

def create_stone(file_name):
    hashsum , size = get_file_hash_size(file_name)
    stone = {
        'name' : os.path.basename(file_name) ,
        'hash' : hashsum,
        'size' : size,
        # TODO: location,
        'insertion_date' : datetime.datetime.utcnow()
        }
    return stone

def insert_in_db(mongodb, stone):
    try:
        stones = mongodb['stones']
        insertion_id = stones.insert(stone)
    except:
        raise ("Error inserting in the db")
    else:
        return insertion_id

def scan_directory(mongodb,path,modules = []):
    found = []
    for dirpath, dirs, files in os.walk(path):
        for file_name in files:
            print("Scanning "+file_name)
            try:
                stone = create_stone(os.path.join(dirpath,file_name))
                for module in modules:
                    try:
                        substone = get_substone(module,os.path.join(dirpath,file_name))
                    except:
                        pass
                    else:
                        if substone != {}:
                            stone[module] = substone
            except:
                print ("Unable to process "+ os.path.join(dirpath,file_name))
            else:
                found.append(insert_in_db(mongodb,stone))
    return found

def initialize_db(url):
    try:
        client = pymongo.MongoClient(url)
        database = client['limestone']
    except:
        raise ('Database error')
    else:
        return database

def get_substone(module_name, file_name):
    try:
        module = importlib.import_module(module_name)
    except:
        raise("Error importing "+module_name+" module")
    try:
        substone = module.get_substone(file_name)
    except:
        raise("Error getting the "+module_name+" substone from "+file_name)
    else:
        return substone

<|endoftext|>


<commit_before>"""
Get the version string from versioneer and print it to stdout
"""
import versioneer

versioneer.VCS = 'git'
versioneer.tag_prefix = 'v'
versioneer.versionfile_source = 'version.py' # This line is useless
versioneer.parentdir_prefix = 'tesseroids-'
version = versioneer.get_version()
if version == 'master':
    # When downloading a zip from Github, versioneer gets the version from the
    # directory name in the zip file. That would endup being 'master'. More
    # useful would be the commit hash. The pattern below will be replaced by
    # git when 'git archive' is called
    version = '$Format:%H$'

if __name__ == '__main__':
    print(version)
<commit_msg>Make sure versioneer is imported from here<commit_after>"""
Get the version string from versioneer and print it to stdout
"""
import sys
import os

# Make sure versioneer is imported from here
here = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, here)
import versioneer

versioneer.VCS = 'git'
versioneer.tag_prefix = 'v'
versioneer.versionfile_source = 'version.py' # This line is useless
versioneer.parentdir_prefix = 'tesseroids-'
version = versioneer.get_version()
if version == 'master':
    # When downloading a zip from Github, versioneer gets the version from the
    # directory name in the zip file. That would endup being 'master'. More
    # useful would be the commit hash. The pattern below will be replaced by
    # git when 'git archive' is called
    version = '$Format:%H$'

if __name__ == '__main__':
    print(version)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    product.py

Implementing Add listing wizard for downstream modules:

* In the __setup__ method of `product.listing.add.start` in downstream
  module, add the type as a valid channel type. Since this is non trivial
  a convenience method `add_source` is provided which will add the source
  type in an idempotent fashion.
* Implement a StateView in the `product.listing.add` wizard with the name
  `start_<source_name>`. This StateView can change the state to other state
  views or transitions. Eventually it should end with the `end` state.

"""
from trytond.pool import PoolMeta
from trytond.wizard import Wizard, Button, StateTransition, StateView
from trytond.transaction import Transaction
from trytond.model import ModelView, fields, ModelSQL

__metaclass__ = PoolMeta
__all__ = [
    'ProductSaleChannelListing', 'Product', 'AddProductListing',
    'AddProductListingStart',
]

class AddProductListingStart(ModelView):
    "Add listing form start"
    __name__ = 'product.listing.add.start'

    product = fields.Many2One(
        'product.product', 'Product', readonly=True
    )

    channel = fields.Many2One(
        'sale.channel', 'Channel', required=True,
        domain=[('source', 'in', [])]
    )

    @classmethod
    def add_source(cls, source):
        """
        A convenience method for downstream modules to add channel
        source types once they have implemented the step in the wizard
        below.

        This method must be called from `__setup__` method of downstream
        module.
        """
        source_leaf = cls.channel.domain[0][2]
        if source not in source_leaf:
            source_leaf.append(source)

class AddProductListing(Wizard):
    "Add product Channel Listing Wizard"
    __name__ = 'product.listing.add'

    start = StateView(
        'product.listing.add.start',
        'sale_channel.listing_add_start_form', [
            Button('Cancel', 'end', 'tryton-cancel'),
            Button('Next', 'next', 'tryton-go-next', default=True),
        ]
    )
    next = StateTransition()

    def default_start(self, fields):
        return {
            'product': Transaction().context['active_id']
        }

    def transition_next(self):
        return 'start_%s' % self.start.channel.source

class Product:
    "Product"
    __name__ = "product.product"

    channel_listings = fields.One2Many(
        'product.product.channel_listing', 'product', 'Channel Listings',
    )

    @classmethod
    def __setup__(cls):
        super(Product, cls).__setup__()
        cls._buttons.update({
            'add_listing': {},
        })

    @classmethod
    @ModelView.button_action('sale_channel.wizard_add_listing')
    def add_listing(cls, products):
        pass

    @classmethod
    def create_from(cls, channel, product_data):
        """
        Create the product for the channel
        """
        raise NotImplementedError(
            "create_from is not implemented in product for %s channels"
            % channel.source
        )

class ProductSaleChannelListing(ModelSQL, ModelView):
    '''Product - Sale Channel
    This model keeps a record of a product's association with Sale Channels.
    A product can be listed on multiple marketplaces
    '''
    __name__ = 'product.product.channel_listing'

    # TODO: Only show channels where this ability is there. For example
    # showing a manual channel is pretty much useless
    channel = fields.Many2One(
        'sale.channel', 'Sale Channel',
        domain=[('source', '!=', 'manual')],
        required=True, select=True,
        ondelete='RESTRICT'
    )
    product = fields.Many2One(
        'product.product', 'Product', required=True, select=True,
        ondelete='CASCADE'
    )
    product_identifier = fields.Char("Product Identifier")
    state = fields.Selection([
        ('active', 'Active'),
        ('disabled', 'Disabled'),
    ], 'State', required=True, select=True)
    channel_source = fields.Function(
        fields.Char("Channel Source"),
        getter="on_change_with_channel_source"
    )

    @fields.depends('channel')
    def on_change_with_channel_source(self, name=None):
        return self.channel and self.channel.source

    @classmethod
    def __setup__(cls):
        '''
        Setup the class and define constraints
        '''
        super(ProductSaleChannelListing, cls).__setup__()
        cls._sql_constraints += [
            (
                'channel_product_unique',
                'UNIQUE(channel, product)',
                'Each product can be linked to only one Sale Channel!'
            )
        ]

    @staticmethod
    def default_state():
        return 'active'

    @classmethod
    def create_from(cls, channel, product_data):
        """
        Create a listing for the product from channel and data
        """
        raise NotImplementedError(
            "create_from is not implemented in channel listing for %s channels"
            % channel.source
        )

    def export_inventory(self):
        """
        Export listing.product inventory to listing.channel

        Since external channels are implemented by downstream modules, it is
        the responsibility of those channels to implement exporting or call
        super to delegate.
        """
        raise NotImplementedError(
            "Export inventory is not implemented for %s channels"
            % self.channel.source
        )

    @classmethod
    def export_bulk_inventory(cls, listings):
        """
        Export listing.product inventory to listing.channel in bulk

        Since external channels are implemented by downstream modules, it is
        the responsibility of those channels to implement bulk exporting for
        respective channels.
        Default behaviour is to export inventory individually.
        """
        for listing in listings:
            listing.export_inventory()

    def get_availability_context(self):
        """
        Allow overriding the context used to compute availability of
        products.
        """
        return self.channel.get_availability_context()

    def get_availability(self):
        """
        Return the availability of the product for this listing
        """
        with Transaction().set_context(**self.get_availability_context()):
            rv = {'type': 'bucket'}
            quantity = Product.get_quantity(
                [self.product], 'quantity'
            )[self.product.id]
            if quantity > 0:
                rv['value'] = 'in_stock'
            else:
                rv['value'] = 'out_of_stock'
            return rv
<commit_msg>Change error message<commit_after># -*- coding: utf-8 -*-
"""
    product.py

Implementing Add listing wizard for downstream modules:

* In the __setup__ method of `product.listing.add.start` in downstream
  module, add the type as a valid channel type. Since this is non trivial
  a convenience method `add_source` is provided which will add the source
  type in an idempotent fashion.
* Implement a StateView in the `product.listing.add` wizard with the name
  `start_<source_name>`. This StateView can change the state to other state
  views or transitions. Eventually it should end with the `end` state.

"""
from trytond.pool import PoolMeta
from trytond.wizard import Wizard, Button, StateTransition, StateView
from trytond.transaction import Transaction
from trytond.model import ModelView, fields, ModelSQL

__metaclass__ = PoolMeta
__all__ = [
    'ProductSaleChannelListing', 'Product', 'AddProductListing',
    'AddProductListingStart',
]

class AddProductListingStart(ModelView):
    "Add listing form start"
    __name__ = 'product.listing.add.start'

    product = fields.Many2One(
        'product.product', 'Product', readonly=True
    )

    channel = fields.Many2One(
        'sale.channel', 'Channel', required=True,
        domain=[('source', 'in', [])]
    )

    @classmethod
    def add_source(cls, source):
        """
        A convenience method for downstream modules to add channel
        source types once they have implemented the step in the wizard
        below.

        This method must be called from `__setup__` method of downstream
        module.
        """
        source_leaf = cls.channel.domain[0][2]
        if source not in source_leaf:
            source_leaf.append(source)

class AddProductListing(Wizard):
    "Add product Channel Listing Wizard"
    __name__ = 'product.listing.add'

    start = StateView(
        'product.listing.add.start',
        'sale_channel.listing_add_start_form', [
            Button('Cancel', 'end', 'tryton-cancel'),
            Button('Next', 'next', 'tryton-go-next', default=True),
        ]
    )
    next = StateTransition()

    def default_start(self, fields):
        return {
            'product': Transaction().context['active_id']
        }

    def transition_next(self):
        return 'start_%s' % self.start.channel.source

class Product:
    "Product"
    __name__ = "product.product"

    channel_listings = fields.One2Many(
        'product.product.channel_listing', 'product', 'Channel Listings',
    )

    @classmethod
    def __setup__(cls):
        super(Product, cls).__setup__()
        cls._buttons.update({
            'add_listing': {},
        })

    @classmethod
    @ModelView.button_action('sale_channel.wizard_add_listing')
    def add_listing(cls, products):
        pass

    @classmethod
    def create_from(cls, channel, product_data):
        """
        Create the product for the channel
        """
        raise NotImplementedError(
            "create_from is not implemented in product for %s channels"
            % channel.source
        )

class ProductSaleChannelListing(ModelSQL, ModelView):
    '''Product - Sale Channel
    This model keeps a record of a product's association with Sale Channels.
    A product can be listed on multiple marketplaces
    '''
    __name__ = 'product.product.channel_listing'

    # TODO: Only show channels where this ability is there. For example
    # showing a manual channel is pretty much useless
    channel = fields.Many2One(
        'sale.channel', 'Sale Channel',
        domain=[('source', '!=', 'manual')],
        required=True, select=True,
        ondelete='RESTRICT'
    )
    product = fields.Many2One(
        'product.product', 'Product', required=True, select=True,
        ondelete='CASCADE'
    )
    product_identifier = fields.Char("Product Identifier")
    state = fields.Selection([
        ('active', 'Active'),
        ('disabled', 'Disabled'),
    ], 'State', required=True, select=True)
    channel_source = fields.Function(
        fields.Char("Channel Source"),
        getter="on_change_with_channel_source"
    )

    @fields.depends('channel')
    def on_change_with_channel_source(self, name=None):
        return self.channel and self.channel.source

    @classmethod
    def __setup__(cls):
        '''
        Setup the class and define constraints
        '''
        super(ProductSaleChannelListing, cls).__setup__()
        cls._sql_constraints += [
            (
                'channel_product_unique',
                'UNIQUE(channel, product)',
                'Product is already mapped to this channel'
            )
        ]

    @staticmethod
    def default_state():
        return 'active'

    @classmethod
    def create_from(cls, channel, product_data):
        """
        Create a listing for the product from channel and data
        """
        raise NotImplementedError(
            "create_from is not implemented in channel listing for %s channels"
            % channel.source
        )

    def export_inventory(self):
        """
        Export listing.product inventory to listing.channel

        Since external channels are implemented by downstream modules, it is
        the responsibility of those channels to implement exporting or call
        super to delegate.
        """
        raise NotImplementedError(
            "Export inventory is not implemented for %s channels"
            % self.channel.source
        )

    @classmethod
    def export_bulk_inventory(cls, listings):
        """
        Export listing.product inventory to listing.channel in bulk

        Since external channels are implemented by downstream modules, it is
        the responsibility of those channels to implement bulk exporting for
        respective channels.
        Default behaviour is to export inventory individually.
        """
        for listing in listings:
            listing.export_inventory()

    def get_availability_context(self):
        """
        Allow overriding the context used to compute availability of
        products.
        """
        return self.channel.get_availability_context()

    def get_availability(self):
        """
        Return the availability of the product for this listing
        """
        with Transaction().set_context(**self.get_availability_context()):
            rv = {'type': 'bucket'}
            quantity = Product.get_quantity(
                [self.product], 'quantity'
            )[self.product.id]
            if quantity > 0:
                rv['value'] = 'in_stock'
            else:
                rv['value'] = 'out_of_stock'
            return rv
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Thu Jun 26 14:43:19 2014

@author: Kyle Ellefsen
"""
from __future__ import (absolute_import, division,print_function, unicode_literals)
from future.builtins import (bytes, dict, int, list, object, range, str, ascii, chr, hex, input, next, oct, open, pow, round, super, filter, map, zip)

from PyQt4.QtCore import *
from PyQt4.QtGui import *
import pyqtgraph as pg
import pyqtgraph.exporters
import time
import os.path
import numpy as np
from skimage.io import imread, imsave
from window import Window
import global_vars as g
import codecs
import shutil, subprocess
import tifffile
import json
import re

__all__ = ['open_file_gui','open_file','save_file_gui','save_file', 'close', 'simulate_distances', 'exportMSD', 'export_distances', 'import_coords', 'export_track_lengths']

def open_file_gui(func, filetypes, prompt='Open File'):
	filename=g.m.settings['filename']
	if filename is not None and os.path.isfile(filename):
		filename= QFileDialog.getOpenFileName(g.m, prompt, filename, filetypes)
	else:
		filename= QFileDialog.getOpenFileName(g.m, prompt, '', filetypes)
	filename=str(filename)
	if filename != '':
		func(filename)
	else:
		g.m.statusBar().showMessage('No File Selected')

def save_file_gui(func, filetypes, prompt = 'Save File'):
	filename=g.m.settings['filename']
	directory=os.path.dirname(filename)
	if filename is not None and directory != '':
		filename= QFileDialog.getSaveFileName(g.m, prompt, directory, filetypes)
	else:
		filename= QFileDialog.getSaveFileName(g.m, prompt, filetypes)
	filename=str(filename)
	if filename != '':
		func(filename)
	else:
		g.m.statusBar().showMessage('Save Cancelled')

def open_file(filename):
	g.m.statusBar().showMessage('Loading {}'.format(os.path.basename(filename)))
	t=time.time()
	Tiff=tifffile.TiffFile(filename)
	try:
		metadata=Tiff[0].image_description
		metadata = txt2dict(metadata)
	except AttributeError:
		metadata=dict()
	tif=Tiff.asarray().astype(g.m.settings['internal_data_type'])
	Tiff.close()
	#tif=imread(filename,plugin='tifffile').astype(g.m.settings['internal_data_type'])
	if len(tif.shape)>3: # WARNING THIS TURNS COLOR movies TO BLACK AND WHITE BY AVERAGING ACROSS THE THREE CHANNELS
		if 'channels' in metadata.keys() and 'ImageJ' in metadata.keys():
			tif=np.transpose(tif,(0,3,2,1))
		tif=np.mean(tif,3)
	tif=np.squeeze(tif) #this gets rid of the meaningless 4th dimention in .stk files
	if len(tif.shape)==3: #this could either be a movie or a colored still frame
		if tif.shape[2]==3: #this is probably a colored still frame
			tif=np.mean(tif,2)
			tif=np.transpose(tif,(1,0)) # This keeps the x and y the same as in FIJI.
		else:
			tif=np.transpose(tif,(0,2,1)) # This keeps the x and y the same as in FIJI.
	elif len(tif.shape)==2: # I haven't tested whether this preserved the x y and keeps it the same as in FIJI.  TEST THIS!!
		tif=np.transpose(tif,(1,0))
	g.m.statusBar().showMessage('{} successfully loaded ({} s)'.format(os.path.basename(filename), time.time()-t))
	g.m.settings['filename']=filename
	return tif

def set_background_image(filename):
	tif = open_file(filename)
	if np.ndim(tif) == 3:
		tif = np.average(tif, 0)
	g.m.trackView.imageview.setImage(tif)

def save_file(filename):
	if os.path.dirname(filename)=='': #if the user didn't specify a directory
		directory=os.path.normpath(os.path.dirname(g.m.settings['filename']))
		filename=os.path.join(directory,filename)
	g.m.statusBar().showMessage('Saving {}'.format(os.path.basename(filename)))
	tif=g.m.currentWindow.image.astype(g.m.settings['data_type'])
	metadata=json.dumps(g.m.currentWindow.metadata)
	if len(tif.shape)==3:
		tif=np.transpose(tif,(0,2,1)) # This keeps the x and the y the same as in FIJI
	elif len(tif.shape)==2:
		tif=np.transpose(tif,(1,0))
	tifffile.imsave(filename, tif, description=metadata) #http://stackoverflow.com/questions/20529187/what-is-the-best-way-to-save-image-metadata-alongside-a-tif-with-python
	g.m.statusBar().showMessage('Successfully saved {}'.format(os.path.basename(filename)))

def txt2dict(metadata):
	meta=dict()
	try:
		metadata=json.loads(metadata.decode('utf-8'))
		return metadata
	except ValueError: #if the metadata isn't in JSON
		pass
	for line in metadata.splitlines():
		line=re.split('[:=]',line.decode())
		if len(line)==1:
			meta[line[0]]=''
		else:
			meta[line[0].lstrip().rstrip()]=line[1].lstrip().rstrip()
	return meta

def close(windows=None):
	'''Will close a window or a set of windows.  Takes several types as its argument:
		| 'all' (str) -- closes all windows
		| windows (list) - closes each window in the list
		| (Window) - closes individual window
		| (None) - closes current window
	'''
	if isinstance(windows,basestring):
		if windows=='all':
			windows=[window for window in g.m.windows]
			for window in windows:
				window.close()
	elif isinstance(windows,list):
		for window in windows:
			if isinstance(window,Window):
				window.close()
	elif isinstance(windows,Window):
		windows.close()
	elif windows is None:
		if g.m.currentWindow is not None:
			g.m.currentWindow.close()

def simulate_distances(filename):
	#export all distances from randomly generated XY coordinates to random Track means
	if not hasattr(g.m.trackView, 'imported'):
		print('No coordinates imported')
	g.m.statusBar().showMessage('Saving tracks to {}'.format(os.path.basename(filename)))
	coord_count = len(g.m.trackView.imported.getData()[1])
	means = g.m.trackPlot.means.getData()
	minP = (min(means[0]), min(means[1]))
	maxP = (max(means[0]), max(means[1]))
	coords = np.random.random(len(means[1]), 2) * maxP + minP
	rand_means = np.random.random(coord_count, 2) * maxP + minP
	export_distances(filenamecoords, rand_means)

def import_coords(filename):
	data = np.loadtxt(filename, skiprows=1, delimiter='\t')
	try:
		x, y = np.transpose(data)
	except:
		print('%s has more than 2 columns of values. Must import X,Y coordinates only' % filename)
	if not hasattr(g.m.trackView, 'imported'):
		plotItem = pg.ScatterPlotItem(x=x, y=y)
		g.m.trackView.addItem(plotItem)
		g.m.trackView.imported = plotItem
	else:
		g.m.trackView.imported.setData(x=x, y=y)

def export_distances(filename, ptsA, ptsB):
	t = time.time()
	with open(filename, 'w') as outf:
		outf.write('Distances\n')
		for i, pt in enumerate(coords):
			for j in range(len(rand_means)):
				outf.write('%.3f\n' % np.linalg.norm(np.subtract(pt, rand_means[j])))
	g.m.statusBar().showMessage('Successfully saved simulated distances in {}'.format(time.time() - t))

def exportMSD(filename):
	t = time.time()
	g.m.statusBar().showMessage('Exporting Track Lengths to {}'.format(os.path.basename(filename)))
	d = g.m.MSDWidget.plot_data
	data = np.transpose([d['x'], d['y'], d['er']])
	np.savetxt(filename, data, header='X\tY\tError', comments='', delimiter='\t')
	g.m.statusBar().showMessage('Track Lengths successfully exported ({} s)'.format(os.path.basename(filename), time.time()-t))

def export_track_lengths(filename):
	t = time.time()
	g.m.statusBar().showMessage('Exporting MSD to {}'.format(os.path.basename(filename)))
	d = g.m.trackPlot.filtered_tracks
	data = np.array([tr['fr_length'] for tr in tracks])
	np.savetxt(filename, data, header='X\tY\tError', comments='', delimiter='\t')
	g.m.statusBar().showMessage('MSD successfully exported ({} s)'.format(os.path.basename(filename), time.time()-t))

<commit_msg>TOO MANY MIXUPS<commit_after># -*- coding: utf-8 -*-
"""
Created on Thu Jun 26 14:43:19 2014

@author: Kyle Ellefsen
"""
from __future__ import (absolute_import, division,print_function, unicode_literals)
from future.builtins import (bytes, dict, int, list, object, range, str, ascii, chr, hex, input, next, oct, open, pow, round, super, filter, map, zip)

from PyQt4.QtCore import *
from PyQt4.QtGui import *
import pyqtgraph as pg
import pyqtgraph.exporters
import time
import os.path
import numpy as np
from skimage.io import imread, imsave
from window import Window
import global_vars as g
import codecs
import shutil, subprocess
import tifffile
import json
import re

__all__ = ['open_file_gui','open_file','save_file_gui','save_file', 'close', 'simulate_distances', 'exportMSD', 'export_distances', 'import_coords', 'export_track_lengths']

def open_file_gui(func, filetypes, prompt='Open File'):
	filename=g.m.settings['filename']
	if filename is not None and os.path.isfile(filename):
		filename= QFileDialog.getOpenFileName(g.m, prompt, filename, filetypes)
	else:
		filename= QFileDialog.getOpenFileName(g.m, prompt, '', filetypes)
	filename=str(filename)
	if filename != '':
		func(filename)
	else:
		g.m.statusBar().showMessage('No File Selected')

def save_file_gui(func, filetypes, prompt = 'Save File'):
	filename=g.m.settings['filename']
	directory=os.path.dirname(filename)
	if filename is not None and directory != '':
		filename= QFileDialog.getSaveFileName(g.m, prompt, directory, filetypes)
	else:
		filename= QFileDialog.getSaveFileName(g.m, prompt, filetypes)
	filename=str(filename)
	if filename != '':
		func(filename)
	else:
		g.m.statusBar().showMessage('Save Cancelled')

def open_file(filename):
	g.m.statusBar().showMessage('Loading {}'.format(os.path.basename(filename)))
	t=time.time()
	Tiff=tifffile.TiffFile(filename)
	try:
		metadata=Tiff[0].image_description
		metadata = txt2dict(metadata)
	except AttributeError:
		metadata=dict()
	tif=Tiff.asarray().astype(g.m.settings['internal_data_type'])
	Tiff.close()
	#tif=imread(filename,plugin='tifffile').astype(g.m.settings['internal_data_type'])
	if len(tif.shape)>3: # WARNING THIS TURNS COLOR movies TO BLACK AND WHITE BY AVERAGING ACROSS THE THREE CHANNELS
		if 'channels' in metadata.keys() and 'ImageJ' in metadata.keys():
			tif=np.transpose(tif,(0,3,2,1))
		tif=np.mean(tif,3)
	tif=np.squeeze(tif) #this gets rid of the meaningless 4th dimention in .stk files
	if len(tif.shape)==3: #this could either be a movie or a colored still frame
		if tif.shape[2]==3: #this is probably a colored still frame
			tif=np.mean(tif,2)
			tif=np.transpose(tif,(1,0)) # This keeps the x and y the same as in FIJI.
		else:
			tif=np.transpose(tif,(0,2,1)) # This keeps the x and y the same as in FIJI.
	elif len(tif.shape)==2: # I haven't tested whether this preserved the x y and keeps it the same as in FIJI.  TEST THIS!!
		tif=np.transpose(tif,(1,0))
	g.m.statusBar().showMessage('{} successfully loaded ({} s)'.format(os.path.basename(filename), time.time()-t))
	g.m.settings['filename']=filename
	return tif

def set_background_image(filename):
	tif = open_file(filename)
	if np.ndim(tif) == 3:
		tif = np.average(tif, 0)
	g.m.trackView.imageview.setImage(tif)

def save_file(filename):
	if os.path.dirname(filename)=='': #if the user didn't specify a directory
		directory=os.path.normpath(os.path.dirname(g.m.settings['filename']))
		filename=os.path.join(directory,filename)
	g.m.statusBar().showMessage('Saving {}'.format(os.path.basename(filename)))
	tif=g.m.currentWindow.image.astype(g.m.settings['data_type'])
	metadata=json.dumps(g.m.currentWindow.metadata)
	if len(tif.shape)==3:
		tif=np.transpose(tif,(0,2,1)) # This keeps the x and the y the same as in FIJI
	elif len(tif.shape)==2:
		tif=np.transpose(tif,(1,0))
	tifffile.imsave(filename, tif, description=metadata) #http://stackoverflow.com/questions/20529187/what-is-the-best-way-to-save-image-metadata-alongside-a-tif-with-python
	g.m.statusBar().showMessage('Successfully saved {}'.format(os.path.basename(filename)))

def txt2dict(metadata):
	meta=dict()
	try:
		metadata=json.loads(metadata.decode('utf-8'))
		return metadata
	except ValueError: #if the metadata isn't in JSON
		pass
	for line in metadata.splitlines():
		line=re.split('[:=]',line.decode())
		if len(line)==1:
			meta[line[0]]=''
		else:
			meta[line[0].lstrip().rstrip()]=line[1].lstrip().rstrip()
	return meta

def close(windows=None):
	'''Will close a window or a set of windows.  Takes several types as its argument:
		| 'all' (str) -- closes all windows
		| windows (list) - closes each window in the list
		| (Window) - closes individual window
		| (None) - closes current window
	'''
	if isinstance(windows,basestring):
		if windows=='all':
			windows=[window for window in g.m.windows]
			for window in windows:
				window.close()
	elif isinstance(windows,list):
		for window in windows:
			if isinstance(window,Window):
				window.close()
	elif isinstance(windows,Window):
		windows.close()
	elif windows is None:
		if g.m.currentWindow is not None:
			g.m.currentWindow.close()

def simulate_distances(filename):
	#export all distances from randomly generated XY coordinates to random Track means
	if not hasattr(g.m.trackView, 'imported'):
		print('No coordinates imported')
	g.m.statusBar().showMessage('Saving tracks to {}'.format(os.path.basename(filename)))
	coord_count = len(g.m.trackView.imported.getData()[1])
	means = g.m.trackPlot.means.getData()
	minP = (min(means[0]), min(means[1]))
	maxP = (max(means[0]), max(means[1]))
	coords = np.random.random(len(means[1]), 2) * maxP + minP
	rand_means = np.random.random(coord_count, 2) * maxP + minP
	export_distances(filenamecoords, rand_means)

def import_coords(filename):
	data = np.loadtxt(filename, skiprows=1, delimiter='\t')
	try:
		x, y = np.transpose(data)
	except:
		print('%s has more than 2 columns of values. Must import X,Y coordinates only' % filename)
	if not hasattr(g.m.trackView, 'imported'):
		plotItem = pg.ScatterPlotItem(x=x, y=y)
		g.m.trackView.addItem(plotItem)
		g.m.trackView.imported = plotItem
	else:
		g.m.trackView.imported.setData(x=x, y=y)

def export_distances(filename, ptsA, ptsB):
	t = time.time()
	with open(filename, 'w') as outf:
		outf.write('Distances\n')
		for i, pt in enumerate(coords):
			for j in range(len(rand_means)):
				outf.write('%.3f\n' % np.linalg.norm(np.subtract(pt, rand_means[j])))
	g.m.statusBar().showMessage('Successfully saved simulated distances in {}'.format(time.time() - t))

def export_track_lengths(filename):
	t = time.time()
	g.m.statusBar().showMessage('Exporting Track Lengths to {}'.format(os.path.basename(filename)))
	d = g.m.trackPlot.filtered_tracks
	data = np.array([tr['fr_length'] for tr in tracks])
	np.savetxt(filename, data, header='X\tY\tError', comments='', delimiter='\t')
	g.m.statusBar().showMessage('Track Lengths successfully exported ({} s)'.format(os.path.basename(filename), time.time()-t))

def exportMSD(filename):
	t = time.time()
	g.m.statusBar().showMessage('Exporting MSD to {}'.format(os.path.basename(filename)))
	d = g.m.MSDWidget.plot_data
	data = np.transpose([d['x'], d['y'], d['er']])
	np.savetxt(filename, data, header='X\tY\tError', comments='', delimiter='\t')
	g.m.statusBar().showMessage('MSD successfully exported ({} s)'.format(os.path.basename(filename), time.time()-t))

<|endoftext|>