<commit_before>#!/usr/bin/python

import glob
import re
import os
from myparser import MyHTMLParser, filesnotfound, css

# by default, the file search is performed recursively
RECURSIVE = True

# set to True when we reach the end of the <head> section no more css files to find?
STOPPARSING = False

#dictionary of all html classes and ids found in (html) files
html = {}

#list of all (html) files found not in the 'correct' encoding (utf-8)
encodingerrors = []

#userdir = input("Enter a directory to search:\n")
userdir = "/home/elanman/www/"

while not os.path.isdir(userdir):
    print("Can't find the specified directory\nMake sure you entered the path correctly\n")
    userdir = input("Enter a directory to search:\n")

#userexts = input("Enter a comma separated list of file extensions to search in e.g php,html\n")
userexts = "html"

exts = userexts.split(",")

for ext in exts:
    print ("Searching file extension... ", ext)

    if RECURSIVE:
        files = [os.path.join(dirpath, f)
                 for dirpath, dirnames, files in os.walk(userdir)
                 for f in files if f.endswith('.' + ext)]
    else:
        files = glob.glob(userdir + "*." + ext)

    print("number of files found:", len(files))
    for file in files:

        parser = MyHTMLParser(file, css, strict=False)

        filecss = {}

        handle = open(file, "r")
        try:
            content = handle.readlines()
            linenum = 1
            cssfound = False

            for line in content:

                if not STOPPARSING:
                    parser.feed(line)

                classes = re.findall(r'class=\"(.+?)\"', line)
                if classes:
                    cssfound = True
                    for htmlclass in classes:
                        htmlclass = "." + htmlclass
                        filecss[htmlclass] = linenum

                ids = re.findall(r'id=\"(.+?)\"', line)
                if ids:
                    cssfound = True
                    for htmlid in ids:
                        htmlid = "#" + htmlid
                        filecss[htmlid] = linenum

                linenum = linenum + 1

            if cssfound:
                html[file] = filecss

        except UnicodeDecodeError:
            if not file in encodingerrors:
                encodingerrors.append(file)

for file in html:

    print("processing ...", file)
    used = []
    unused = []

    if file in css:

        for c,s in css[file].items():
            print("    css file is", c)
            #need to add css file for each used /unused
            for i,l in s["ids"].items():

                if i in html[file]:
                    used.append({i:l})

                else:
                    unused.append({i:l})

            for i,l in s["classes"].items():

                if i in html[file]:
                    used.append({i:l})

                else:
                    unused.append({i:l})

    if used:
        print("        used css:")
        for l in used:
            for key in l:
                lines = str(l[key]).strip("[]")
                print("            ", key, "on line(s)", lines)

    if unused:
        print("        unused css:")
        for l in unused:
            for key in l:
                lines = str(l[key]).strip("[]")
                print("            ", key, "on line(s)", lines)

if filesnotfound:

    print("Files not found:")

    for l in filesnotfound:
        print("    ",l)
        for files in filesnotfound[l]:
            print("        ",files)

if encodingerrors:
    print("Files found with encoding errors (not utf-8): ")
    for f in encodingerrors:
        print(f)<commit_msg>Moved used/unused list initialisation inside loop for each linked css file. Easier to display ids/classes by relevant css file and line number.<commit_after>#!/usr/bin/python

import glob
import re
import os
from myparser import MyHTMLParser, filesnotfound, css

# by default, the file search is performed recursively
RECURSIVE = True

# set to True when we reach the end of the <head> section no more css files to find?
STOPPARSING = False

#dictionary of all html classes and ids found in (html) files
html = {}

#list of all (html) files found not in the 'correct' encoding (utf-8)
encodingerrors = []

#userdir = input("Enter a directory to search:\n")
userdir = "/home/elanman/www/"

# if we can't find the directory, prompt the user for another
while not os.path.isdir(userdir):
    print("Can't find the specified directory\nMake sure you entered the path correctly\n")
    userdir = input("Enter a directory to search:\n")

#userexts = input("Enter a comma separated list of file extensions to search in e.g php,html\n")
userexts = "html"

exts = userexts.split(",")

for ext in exts:
    print ("Searching file extension... ", ext)

    #redursive search of user directory or not
    if RECURSIVE:
        files = [os.path.join(dirpath, f)
                 for dirpath, dirnames, files in os.walk(userdir)
                 for f in files if f.endswith('.' + ext)]
    else:
        files = glob.glob(userdir + "*." + ext)

    print("number of files found:", len(files))
    for file in files:

        parser = MyHTMLParser(file, css, strict=False)

        filecss = {}

        handle = open(file, "r")
        try:
            content = handle.readlines()
            linenum = 1
            cssfound = False

            for line in content:

                if not STOPPARSING:
                    parser.feed(line)

                classes = re.findall(r'class=\"(.+?)\"', line)
                if classes:
                    cssfound = True
                    for htmlclass in classes:
                        htmlclass = "." + htmlclass
                        filecss[htmlclass] = linenum

                ids = re.findall(r'id=\"(.+?)\"', line)
                if ids:
                    cssfound = True
                    for htmlid in ids:
                        htmlid = "#" + htmlid
                        filecss[htmlid] = linenum

                linenum = linenum + 1

            if cssfound:
                html[file] = filecss

        except UnicodeDecodeError:
            if not file in encodingerrors:
                encodingerrors.append(file)

for file in html:

    print("processing ...", file)

    if file in css:

        for c,s in css[file].items():

            used = []
            unused = []

            #l is the list of line numbers
            for i,l in s["ids"].items():

                if i in html[file]:
                    used.append({i:l})

                else:
                    unused.append({i:l})

            for i,l in s["classes"].items():

                if i in html[file]:
                    used.append({i:l})

                else:
                    unused.append({i:l})

            if used or unused:
                print("    searching css file:", c)

            if used:
                print("        used css:")
                for l in used:
                    for key in l:
                        lines = str(l[key]).strip("[]")
                        print("            ", key, "on line(s)", lines)

            if unused:
                print("        unused css:")
                for l in unused:
                    for key in l:
                        lines = str(l[key]).strip("[]")
                        print("            ", key, "on line(s)", lines)

if filesnotfound:

    print("Files not found:")

    for l in filesnotfound:
        print("    ",l)
        for files in filesnotfound[l]:
            print("        ",files)

if encodingerrors:
    print("Files found with encoding errors (not utf-8): ")
    for f in encodingerrors:
        print(f)<|endoftext|>


<commit_before>"""
Functions to interrogate and extract CTD data from the ctd.db SQLite3 database.

"""

from __future__ import print_function

import inspect
from pathlib import Path
from datetime import datetime

import numpy as np

from warnings import warn

try:
    import sqlite3
    use_sqlite = True
except ImportError:
    warn('No sqlite standard library found in this python '
         'installation. Some functions will be disabled.')
    use_sqlite = False

def get_CTD_metadata(db):
    """
    Extracts the meta data from the CTD database.

    Parameters
    ----------
    db : str
        Full path to the CTD data SQLite database.

    Returns
    -------
    meta_info : list
        List of dicts with keys based on the field names from the Stations
        table. Returns [False] if there is an error.

    """

    if not use_sqlite:
        raise RuntimeError('No sqlite standard library found in this python'
                           ' installation. This function (get_CTD_metadata)'
                           ' is unavailable.')

    def _dict_factory(cursor, row):
        d = {}
        for idx, col in enumerate(cursor.description):
            d[col[0]] = row[idx]
        return d

    try:
        con = sqlite3.connect(db)

        con.row_factory = _dict_factory

        c = con.cursor()

        out = c.execute('SELECT * from Stations')

        meta_info = out.fetchall()

    except sqlite3.Error as e:
        if con:
            con.close()
            print('Error {}:'.format(e.args[0]))
            meta_info = [False]

    finally:
        if con:
            con.close()

    return meta_info

def get_CTD_data(db, table, fields, noisy=False):
    """
    Extract the CTD from the SQLite database for a given site. Specify the
    database (db), the table name (table) of the station of interest.

    Parameters
    ----------
    db : str
        Full path to the CTD data SQLite database.
    table : str
        Name of the table to be extracted (e.g. 'b0000001').
    fields : list
        List of names of fields to extract for the given table, such as
        ['Depth', 'Temperature']. Where no data exists, a column of NaNs will
        be returned (actually Nones, but numpy does the conversion for you).
    noisy : bool, optional
        Set to True to enable verbose output.

    Returns
    -------
    data : ndarray
        Array of the fields requested from the table specified.

    See Also
    --------
    ctd.get_observed_metadata : extract metadata for a CTD cast.

    Notes
    -----
    Search is case insensitive (b0737327 is equal to B0737327).

    """

    if not use_sqlite:
        raise RuntimeError('No sqlite standard library found in this python'
                           ' installation. This function (get_CTD_data)'
                           ' is unavailable.')

    if noisy:
        print('Getting data for {} from the database...'.format(table),
              end=' ')

    try:
        con = sqlite3.connect(db)

        with con:
            c = con.cursor()
            # I know, using a string is Bad. But it works and it's only me
            # working with this.
            c.execute('SELECT {} FROM {}'.format(','.join(fields), table))

            # Now get the data in a format we might actually want to use
            data = np.asarray(c.fetchall())

        if noisy:
            print('done.')

    except sqlite3.Error as e:
        if con:
            con.close()
            print('Error {}:'.format(e.args[0]))
            data = np.asarray([False])

    finally:
        if con:
            con.close()

    return data

def get_ferrybox_data(db, fields, table='PrideOfBilbao', noisy=False):
    """
    Extract the Ferry Box data from the SQLite database for a given route
    (defaults to PrideOfBilbao). Specify the database (db), the table name
    (table) of the station of interest.

    Parameters
    ----------
    db : str
        Full path to the CTD data SQLite database.
    fields : list
        List of names of fields to extract for the given table, such as
        ['salinity', 'temp']. Where no data exists, a column of NaNs will
        be returned (actually Nones, but numpy does the conversion for you).
    table : str, optional
        Name of the table to be extracted (defaults to 'PrideOfBilbao').
    noisy : bool, optional
        Set to True to enable verbose output.

    Returns
    -------
    data : ndarray
        Array of the fields requested from the table specified.

    Notes
    -----
    Search is case insensitive (b0737327 is equal to B0737327).

    """

    if noisy:
        print('Getting data for {} from the database...'.format(table),
              end=' ')

    try:
        con = sqlite3.connect(db)

        with con:
            c = con.cursor()
            # I know, using a string is Bad. But it works and it's only me
            # working with this.
            c.execute('SELECT {} FROM {}'.format(','.join(fields), table))

            # Now get the data in a format we might actually want to use
            data = np.asarray(c.fetchall())

        if noisy:
            print('done.')

    except sqlite3.Error as e:
        if con:
            con.close()
            print('Error {}:'.format(e.args[0]))
            data = np.asarray([False])

    finally:
        if con:
            con.close()

    return data

class BODC(object):
    """ Work with BODC data. Mainly focused around CTD data. """

    def __init__(self):
        """ Initialise a BODC object. """

        self.debug = False

        self.time = type('time', (), {})()
        self.data = type('data', (), {})()
        self.position = type('position', (), {})()
        self.metadata = type('metadata', (), {})()  # maybe one day this will be used.

    def _read_lst(self, filename):
        """
        Read the given LST-formatted file and process the data accordingly.

        Parameters
        ----------
        filename : str, pathlib.Path
            The file name to read in.

        Provides
        --------
        self.data, self.time and self.position are populated accordingly.

        """

        # Initialise things which we expect to exist later. For example, self.position.sensor needn't exist,
        # but if it gets created, then it's used when writing out a file.
        self.data.names = []
        self.data.units = {}
        self.data.long_name = {}
        self.position.lon = None
        self.position.lat = None
        self.position.depth = None

        # Make something to tidy up the lines.
        clean = lambda x: [i.strip() for i in line.split(' ') if i.strip()]

        filename = Path(filename)
        with filename.open('r') as f:
            for line in f:
                line = line.strip()
                if line:
                    line_list = clean(line)
                    if self.debug:
                        print('headers {}'.format(line))
                    if line.startswith('BODC'):
                        header_length = int(line_list[6])
                        num_records = int(line_list[-1])
                    elif line.startswith('Series'):
                        name = line_list[1]
                    elif 'start:' in line:
                        raw_position = line_list[0]
                        # Extract degrees, minutes, hemisphere for latitude and longitude.
                        nice_position = [float(raw_position[:3]), float(raw_position[4:7]), raw_position[8],
                                         float(raw_position[9:12]), float(raw_position[13:17]), raw_position[18]]
                        self.position.lon = nice_position[3] + (nice_position[4] / 60)
                        if nice_position[5] == 'W':
                            self.position.lon = -self.position.lon
                        self.position.lat = nice_position[0] + (nice_position[1] / 60)
                        if nice_position[2] == 'S':
                            self.position.lat = -self.position.lat

                        start = line_list[1].split(':')[1]
                        self.time.datetime = [datetime.strptime(start, '%Y%m%d%H%M%S')]
                    elif line.startswith('Dep'):
                        if 'floor' in line:
                            self.position.depth = float(line_list[2])
                            self.position.sensor = [float(i) for i in line_list[4:6]]
                        else:
                            self.position.depth = float(line_list[1])
                        self.time.interval = float(line_list[-2])
                        self.time.units = line_list[-1]
                    elif 'included' in line:
                        # Get the number of parameters in this file.
                        num_fields = int(line_list[0])
                        # Read one more line here and then break out here as we handle reading the metadata in a
                        # separate loop.
                        next(f)
                        break

            # Extract the information about each variable.
            for record in range(num_fields):
                line = next(f).strip()
                if self.debug:
                    print('metadata {}'.format(line))
                if line:
                    line_list = clean(line)
                    self.data.names.append(line_list[0])
                    # Preallocate the data array ready for the data.
                    setattr(self.data, self.data.names[-1], np.zeros((num_records)))
                    self.data.units[self.data.names[-1]] = line_list[-1]
                    # Skip to the next line now so we can get the long_name value.
                    line = next(f).strip()
                    self.data.long_name[self.data.names[-1]] = line

            # Now ignore the rest until we hit the data itself.
            at_data = False
            for line in f:
                line = line.strip()
                if self.debug:
                    print('data {}'.format(line))
                line_list = clean(line)
                if line:
                    if line_list[0] == 'Cycle':
                        # Next line is the start of the data. Grab the headers so we know where to put what.
                        columns = line_list[1:]  # skip the cycle column
                        at_data = True
                        continue
                    if line_list[0] == 'Number':
                        # The second data header line. We could use this to figure out the formats of the data,
                        # but doesn't seem worth the effort right now. Skip it.
                        continue
                    if at_data:
                        for name_index, name in enumerate(columns):
                            # Use the cycle number to get the index for the data. Offset the name_index by 1 to
                            # account for the missing Cycle column.
                            data_index = int(line_list[0].replace(')', '')) - 1
                            getattr(self.data, name)[data_index] = float(line_list[name_index + 1])

    def _read_qxf(self, filename):
        """
        Read the given QXF-formatted file and process the data accordingly.

        Parameters
        ----------
        filename : str, pathlib.Path
            The file name to read in.

        Provides
        --------
        self.data, self.time and self.position are populated accordingly.

        """
        pass

    def read(self, filename):
        """
        Generic wrapper around the two main BODC file formats I'm interested in (QXF. vs LST, essentially netCDF vs.
        ASCII).

        Parameters
        ----------
        filename : str, pathlib.Path
            The file name to read in.

        Provides
        --------
        self.data, self.time and self.position are populated accordingly.

        """
        try:
            suffix = Path(filename).suffix
        except ValueError:
            suffix = filename.suffix

        if suffix == '.qxf':
            self._read_qxf(filename)
        elif suffix == '.lst':
            self._read_lst(filename)
        else:
            raise ValueError('Unsupported file extension: {}, supply either .lst or .qxf'.format(suffix))

    def _write_lst(self, filename):
        """
        Write the given data to LST-formatted file.

        Parameters
        ----------
        filename : str, pathlib.Path
            The file name to write out to.

        """

        # Parse the data for relevant metadata.
        num_fields = len(self.data.names)
        num_samples = len(getattr(self.data, self.data.names[0]))

        start = np.min(self.time.datetime).strftime('%Y%m%d%H%M%S')
        lon = np.mean(self.position.lon)
        lat = np.mean(self.position.lat)
        northsouth = 'N'
        westeast = 'E'
        if lat < 0:
            northsouth = 'S'
        if lon < 0:
            westeast = 'W'

        # Some stuff, we'll just ignore/make up for now.
        try:
            sensor = self.position.sensor
        except AttributeError:
            sensor = [np.min(self.position.depth), np.max(self.position.depth)]

        # The BODC format is... unique. Try to replicate it to the extent which is useful for me. May not be 100%
        # compatible with actual BODC formatting.
        filename = Path(filename)
        with filename.open('w') as f:
            # Naturally, we start with three blank lines.
            f.write('\n\n\n')
            f.write('BODC Request Format Std. V1.0           Headers=  {} Data Cycles=   {}\n'.format(14, num_samples))
            f.write('Series=      {}                     Produced: {}\n'.format('AAAAAAAA', datetime.now().strftime('%d-%b-%Y')))  # dummy data for now
            f.write('Id                       AAAAAAAA PML\n')  # more dummy data
            position = '{deglat:03d}d{minlat:.1f}m{hemilat}{deglon:03d}d{minlon:.1f}m{hemilon}'.format(deglat=int(lat),
                                                                                minlat=lat - int(lat),
                                                                                hemilat=northsouth,
                                                                                deglon=int(lon),
                                                                                minlon=lon - int(lon),
                                                                                hemilon=westeast)
            f.write('{position}                     start:{start}\n'.format(position=position, start=start))
            format_string = 'Dep: floor {depth:.1f} sensor    {sensor_1:.1f}  {sensor_2:.1f} Nom. sample int.:    {interval:.1f} {units}\n'
            f.write(format_string.format(depth=self.position.depth,
                                         sensor_1=sensor[0],
                                         sensor_2=sensor[1],
                                         interval=self.time.interval,
                                         units=self.time.units))
            f.write('    {} Parameters included:\n'.format(num_fields))
            # Now we're into the headers for each variable.
            f.write('Parameter f    P    Q Absent Data Value Minimum Value  Maximum Value       Units\n')
            for name in self.data.names:
                header_string = '{name:8s}  {f} {P} {Q} {missing:.2f} {min:.2f} {max:.2f} {unit}\n'
                f.write(header_string.format(name=name[:8],
                                             f=0,
                                             P=0,
                                             Q=0,
                                             missing=-1,
                                             min=np.min(getattr(self.data, name)),
                                             max=np.max(getattr(self.data, name)),
                                             unit=self.data.units[name]))
                f.write('{}\n'.format(self.data.long_name[name][:80]))  # maximum line length is 80 characters
            f.write('\n')
            f.write('Format Record\n')
            f.write('(some nonesense for now)\n')
            # A few more new lines in case we're missing some.
            f.write('\n\n\n')
            # The data header.
            f.write('  Cycle {}\n'.format('  '.join([i[:8] for i in self.data.names])))
            f.write('Number          {}\n'.format('       '.join('f' * num_fields)))
            # Now add the data.
            cycle = np.arange(num_samples) + 1
            data = []
            for name in self.data.names:
                data.append(getattr(self.data, name))
            # data = np.column_stack((cycle, np.asarray(data)))

            # Must be possible to dump a whole numpy array in one shot...
            # np.savetxt(f, data, fmt='%f')
            # f.writelines(data)

# Add for backwards compatibility.
def getCTDMetadata(*args, **kwargs):
    warn('{} is deprecated. Use get_CTD_metadata instead.'.format(inspect.stack()[0][3]))
    return get_CTD_metadata(*args, **kwargs)

def getCTDData(*args, **kwargs):
    warn('{} is deprecated. Use get_CTD_data instead.'.format(inspect.stack()[0][3]))
    return get_CTD_data(*args, **kwargs)

def getFerryBoxData(*args, **kwargs):
    warn('{} is deprecated. Use get_ferrybox_data instead.'.format(inspect.stack()[0][3]))
    return get_ferrybox_data(*args, **kwargs)
<commit_msg>Tidy imports.<commit_after>"""
Functions to interrogate and extract CTD data from the ctd.db SQLite3 database.

"""

from __future__ import print_function

import inspect
from pathlib import Path
from warnings import warn
from datetime import datetime

import numpy as np

use_sqlite = True
try:
    import sqlite3
except ImportError:
    warn('No sqlite standard library found in this python installation. Some functions will be disabled.')
    use_sqlite = False

def get_CTD_metadata(db):
    """
    Extracts the meta data from the CTD database.

    Parameters
    ----------
    db : str
        Full path to the CTD data SQLite database.

    Returns
    -------
    meta_info : list
        List of dicts with keys based on the field names from the Stations
        table. Returns [False] if there is an error.

    """

    if not use_sqlite:
        raise RuntimeError('No sqlite standard library found in this python'
                           ' installation. This function (get_CTD_metadata)'
                           ' is unavailable.')

    def _dict_factory(cursor, row):
        d = {}
        for idx, col in enumerate(cursor.description):
            d[col[0]] = row[idx]
        return d

    try:
        con = sqlite3.connect(db)

        con.row_factory = _dict_factory

        c = con.cursor()

        out = c.execute('SELECT * from Stations')

        meta_info = out.fetchall()

    except sqlite3.Error as e:
        if con:
            con.close()
            print('Error {}:'.format(e.args[0]))
            meta_info = [False]

    finally:
        if con:
            con.close()

    return meta_info

def get_CTD_data(db, table, fields, noisy=False):
    """
    Extract the CTD from the SQLite database for a given site. Specify the
    database (db), the table name (table) of the station of interest.

    Parameters
    ----------
    db : str
        Full path to the CTD data SQLite database.
    table : str
        Name of the table to be extracted (e.g. 'b0000001').
    fields : list
        List of names of fields to extract for the given table, such as
        ['Depth', 'Temperature']. Where no data exists, a column of NaNs will
        be returned (actually Nones, but numpy does the conversion for you).
    noisy : bool, optional
        Set to True to enable verbose output.

    Returns
    -------
    data : ndarray
        Array of the fields requested from the table specified.

    See Also
    --------
    ctd.get_observed_metadata : extract metadata for a CTD cast.

    Notes
    -----
    Search is case insensitive (b0737327 is equal to B0737327).

    """

    if not use_sqlite:
        raise RuntimeError('No sqlite standard library found in this python'
                           ' installation. This function (get_CTD_data)'
                           ' is unavailable.')

    if noisy:
        print('Getting data for {} from the database...'.format(table),
              end=' ')

    try:
        con = sqlite3.connect(db)

        with con:
            c = con.cursor()
            # I know, using a string is Bad. But it works and it's only me
            # working with this.
            c.execute('SELECT {} FROM {}'.format(','.join(fields), table))

            # Now get the data in a format we might actually want to use
            data = np.asarray(c.fetchall())

        if noisy:
            print('done.')

    except sqlite3.Error as e:
        if con:
            con.close()
            print('Error {}:'.format(e.args[0]))
            data = np.asarray([False])

    finally:
        if con:
            con.close()

    return data

def get_ferrybox_data(db, fields, table='PrideOfBilbao', noisy=False):
    """
    Extract the Ferry Box data from the SQLite database for a given route
    (defaults to PrideOfBilbao). Specify the database (db), the table name
    (table) of the station of interest.

    Parameters
    ----------
    db : str
        Full path to the CTD data SQLite database.
    fields : list
        List of names of fields to extract for the given table, such as
        ['salinity', 'temp']. Where no data exists, a column of NaNs will
        be returned (actually Nones, but numpy does the conversion for you).
    table : str, optional
        Name of the table to be extracted (defaults to 'PrideOfBilbao').
    noisy : bool, optional
        Set to True to enable verbose output.

    Returns
    -------
    data : ndarray
        Array of the fields requested from the table specified.

    Notes
    -----
    Search is case insensitive (b0737327 is equal to B0737327).

    """

    if noisy:
        print('Getting data for {} from the database...'.format(table),
              end=' ')

    try:
        con = sqlite3.connect(db)

        with con:
            c = con.cursor()
            # I know, using a string is Bad. But it works and it's only me
            # working with this.
            c.execute('SELECT {} FROM {}'.format(','.join(fields), table))

            # Now get the data in a format we might actually want to use
            data = np.asarray(c.fetchall())

        if noisy:
            print('done.')

    except sqlite3.Error as e:
        if con:
            con.close()
            print('Error {}:'.format(e.args[0]))
            data = np.asarray([False])

    finally:
        if con:
            con.close()

    return data

class BODC(object):
    """ Work with BODC data. Mainly focused around CTD data. """

    def __init__(self):
        """ Initialise a BODC object. """

        self.debug = False

        self.time = type('time', (), {})()
        self.data = type('data', (), {})()
        self.position = type('position', (), {})()
        self.metadata = type('metadata', (), {})()  # maybe one day this will be used.

    def _read_lst(self, filename):
        """
        Read the given LST-formatted file and process the data accordingly.

        Parameters
        ----------
        filename : str, pathlib.Path
            The file name to read in.

        Provides
        --------
        self.data, self.time and self.position are populated accordingly.

        """

        # Initialise things which we expect to exist later. For example, self.position.sensor needn't exist,
        # but if it gets created, then it's used when writing out a file.
        self.data.names = []
        self.data.units = {}
        self.data.long_name = {}
        self.position.lon = None
        self.position.lat = None
        self.position.depth = None

        # Make something to tidy up the lines.
        clean = lambda x: [i.strip() for i in line.split(' ') if i.strip()]

        filename = Path(filename)
        with filename.open('r') as f:
            for line in f:
                line = line.strip()
                if line:
                    line_list = clean(line)
                    if self.debug:
                        print('headers {}'.format(line))
                    if line.startswith('BODC'):
                        header_length = int(line_list[6])
                        num_records = int(line_list[-1])
                    elif line.startswith('Series'):
                        name = line_list[1]
                    elif 'start:' in line:
                        raw_position = line_list[0]
                        # Extract degrees, minutes, hemisphere for latitude and longitude.
                        nice_position = [float(raw_position[:3]), float(raw_position[4:7]), raw_position[8],
                                         float(raw_position[9:12]), float(raw_position[13:17]), raw_position[18]]
                        self.position.lon = nice_position[3] + (nice_position[4] / 60)
                        if nice_position[5] == 'W':
                            self.position.lon = -self.position.lon
                        self.position.lat = nice_position[0] + (nice_position[1] / 60)
                        if nice_position[2] == 'S':
                            self.position.lat = -self.position.lat

                        start = line_list[1].split(':')[1]
                        self.time.datetime = [datetime.strptime(start, '%Y%m%d%H%M%S')]
                    elif line.startswith('Dep'):
                        if 'floor' in line:
                            self.position.depth = float(line_list[2])
                            self.position.sensor = [float(i) for i in line_list[4:6]]
                        else:
                            self.position.depth = float(line_list[1])
                        self.time.interval = float(line_list[-2])
                        self.time.units = line_list[-1]
                    elif 'included' in line:
                        # Get the number of parameters in this file.
                        num_fields = int(line_list[0])
                        # Read one more line here and then break out here as we handle reading the metadata in a
                        # separate loop.
                        next(f)
                        break

            # Extract the information about each variable.
            for record in range(num_fields):
                line = next(f).strip()
                if self.debug:
                    print('metadata {}'.format(line))
                if line:
                    line_list = clean(line)
                    self.data.names.append(line_list[0])
                    # Preallocate the data array ready for the data.
                    setattr(self.data, self.data.names[-1], np.zeros((num_records)))
                    self.data.units[self.data.names[-1]] = line_list[-1]
                    # Skip to the next line now so we can get the long_name value.
                    line = next(f).strip()
                    self.data.long_name[self.data.names[-1]] = line

            # Now ignore the rest until we hit the data itself.
            at_data = False
            for line in f:
                line = line.strip()
                if self.debug:
                    print('data {}'.format(line))
                line_list = clean(line)
                if line:
                    if line_list[0] == 'Cycle':
                        # Next line is the start of the data. Grab the headers so we know where to put what.
                        columns = line_list[1:]  # skip the cycle column
                        at_data = True
                        continue
                    if line_list[0] == 'Number':
                        # The second data header line. We could use this to figure out the formats of the data,
                        # but doesn't seem worth the effort right now. Skip it.
                        continue
                    if at_data:
                        for name_index, name in enumerate(columns):
                            # Use the cycle number to get the index for the data. Offset the name_index by 1 to
                            # account for the missing Cycle column.
                            data_index = int(line_list[0].replace(')', '')) - 1
                            getattr(self.data, name)[data_index] = float(line_list[name_index + 1])

    def _read_qxf(self, filename):
        """
        Read the given QXF-formatted file and process the data accordingly.

        Parameters
        ----------
        filename : str, pathlib.Path
            The file name to read in.

        Provides
        --------
        self.data, self.time and self.position are populated accordingly.

        """
        pass

    def read(self, filename):
        """
        Generic wrapper around the two main BODC file formats I'm interested in (QXF. vs LST, essentially netCDF vs.
        ASCII).

        Parameters
        ----------
        filename : str, pathlib.Path
            The file name to read in.

        Provides
        --------
        self.data, self.time and self.position are populated accordingly.

        """
        try:
            suffix = Path(filename).suffix
        except ValueError:
            suffix = filename.suffix

        if suffix == '.qxf':
            self._read_qxf(filename)
        elif suffix == '.lst':
            self._read_lst(filename)
        else:
            raise ValueError('Unsupported file extension: {}, supply either .lst or .qxf'.format(suffix))

    def _write_lst(self, filename):
        """
        Write the given data to LST-formatted file.

        Parameters
        ----------
        filename : str, pathlib.Path
            The file name to write out to.

        """

        # Parse the data for relevant metadata.
        num_fields = len(self.data.names)
        num_samples = len(getattr(self.data, self.data.names[0]))

        start = np.min(self.time.datetime).strftime('%Y%m%d%H%M%S')
        lon = np.mean(self.position.lon)
        lat = np.mean(self.position.lat)
        northsouth = 'N'
        westeast = 'E'
        if lat < 0:
            northsouth = 'S'
        if lon < 0:
            westeast = 'W'

        # Some stuff, we'll just ignore/make up for now.
        try:
            sensor = self.position.sensor
        except AttributeError:
            sensor = [np.min(self.position.depth), np.max(self.position.depth)]

        # The BODC format is... unique. Try to replicate it to the extent which is useful for me. May not be 100%
        # compatible with actual BODC formatting.
        filename = Path(filename)
        with filename.open('w') as f:
            # Naturally, we start with three blank lines.
            f.write('\n\n\n')
            f.write('BODC Request Format Std. V1.0           Headers=  {} Data Cycles=   {}\n'.format(14, num_samples))
            f.write('Series=      {}                     Produced: {}\n'.format('AAAAAAAA', datetime.now().strftime('%d-%b-%Y')))  # dummy data for now
            f.write('Id                       AAAAAAAA PML\n')  # more dummy data
            position = '{deglat:03d}d{minlat:.1f}m{hemilat}{deglon:03d}d{minlon:.1f}m{hemilon}'.format(deglat=int(lat),
                                                                                minlat=lat - int(lat),
                                                                                hemilat=northsouth,
                                                                                deglon=int(lon),
                                                                                minlon=lon - int(lon),
                                                                                hemilon=westeast)
            f.write('{position}                     start:{start}\n'.format(position=position, start=start))
            format_string = 'Dep: floor {depth:.1f} sensor    {sensor_1:.1f}  {sensor_2:.1f} Nom. sample int.:    {interval:.1f} {units}\n'
            f.write(format_string.format(depth=self.position.depth,
                                         sensor_1=sensor[0],
                                         sensor_2=sensor[1],
                                         interval=self.time.interval,
                                         units=self.time.units))
            f.write('    {} Parameters included:\n'.format(num_fields))
            # Now we're into the headers for each variable.
            f.write('Parameter f    P    Q Absent Data Value Minimum Value  Maximum Value       Units\n')
            for name in self.data.names:
                header_string = '{name:8s}  {f} {P} {Q} {missing:.2f} {min:.2f} {max:.2f} {unit}\n'
                f.write(header_string.format(name=name[:8],
                                             f=0,
                                             P=0,
                                             Q=0,
                                             missing=-1,
                                             min=np.min(getattr(self.data, name)),
                                             max=np.max(getattr(self.data, name)),
                                             unit=self.data.units[name]))
                f.write('{}\n'.format(self.data.long_name[name][:80]))  # maximum line length is 80 characters
            f.write('\n')
            f.write('Format Record\n')
            f.write('(some nonesense for now)\n')
            # A few more new lines in case we're missing some.
            f.write('\n\n\n')
            # The data header.
            f.write('  Cycle {}\n'.format('  '.join([i[:8] for i in self.data.names])))
            f.write('Number          {}\n'.format('       '.join('f' * num_fields)))
            # Now add the data.
            cycle = np.arange(num_samples) + 1
            data = []
            for name in self.data.names:
                data.append(getattr(self.data, name))
            # data = np.column_stack((cycle, np.asarray(data)))

            # Must be possible to dump a whole numpy array in one shot...
            # np.savetxt(f, data, fmt='%f')
            # f.writelines(data)

# Add for backwards compatibility.
def getCTDMetadata(*args, **kwargs):
    warn('{} is deprecated. Use get_CTD_metadata instead.'.format(inspect.stack()[0][3]))
    return get_CTD_metadata(*args, **kwargs)

def getCTDData(*args, **kwargs):
    warn('{} is deprecated. Use get_CTD_data instead.'.format(inspect.stack()[0][3]))
    return get_CTD_data(*args, **kwargs)

def getFerryBoxData(*args, **kwargs):
    warn('{} is deprecated. Use get_ferrybox_data instead.'.format(inspect.stack()[0][3]))
    return get_ferrybox_data(*args, **kwargs)
<|endoftext|>


<commit_before># encoding: utf-8

"""Basic file controller."""

import webob

import web.core

from web.extras.contentment.api import action, view
from web.extras.contentment.components.asset.controller import AssetController
from web.extras.contentment.components.file.scaler import scale, get_cache, save_cache
from web.extras.contentment.components.file.reflection import add_reflection

from cStringIO import StringIO

log = __import__('logging').getLogger(__name__)
__all__ = ['FileController']

class FileController(AssetController):
    @view('Preview', "Preview uploaded content.")
    def view_preview(self):
        return 'preview', None

    @view('Download', "Download this file.")
    def view_download(self, filename=None, inline=False):
        asset = self.asset
        response = webob.Response(request=web.core.request, conditional_response=True)
        filename = asset.filename if filename is None else filename

        response.content_type = unicode(asset.mimetype, 'ascii').encode('ascii')
        response.content_length = asset.size
        response.last_modified = asset.modified if asset.modified else asset.created
        response.accept_ranges = 'bytes'
        response.etag = '%s-%s-%s' % (asset.modified if asset.modified else asset.created, asset.size, hash(filename))
        response.cache_control = 'public'
        response.content_transfer_encoding = 'binary'
        response.content_disposition = "inline" if inline else ('attachment; filename=' + filename)
        response.range = (0, asset.size)

        response.app_iter = asset.content.get()

        return response

    # TODO: Caching.
    @view('Scale', "Display a scaled version of this file.")
    def view_scale(self, filename=None, cache=True, inline=True, reflection=False, color="000000", amount=0.75, opacity=0.4, **kw):
        asset = self.asset

        color = '#' + color
        cache = bool(cache)
        inline = bool(inline)
        reflection = bool(reflection)
        amount = float(amount)
        opacity = float(opacity)

        if cache:
            id_ = str(asset.id)

            url = get_cache(str(asset.id), asset.modified, style="standard" if not reflection else "reflection", **kw)

            if url is not None:
                raise web.core.http.HTTPSeeOther(location=url)

            with open(save_cache(str(asset.id), asset.modified, style="standard" if not reflection else "reflection", **kw), 'wb') as fh:
                result, jquality = scale(asset.content.get(), fh, raw=reflection, **kw)

                if reflection:
                    result = add_reflection(result, bgcolor=color, amount=float(amount), opacity=float(opacity))
                    result.save(fh, "JPEG", optimize=True, quality=jquality)

            raise web.core.http.HTTPSeeOther(location=get_cache(id_, asset.modified, style="standard" if not reflection else "reflection", **kw))

        response = webob.Response(request=web.core.request, conditional_response=True)
        filename = asset.filename if filename is None else filename

        response.content_type = "image/jpeg"
        response.content_length = asset.size
        response.last_modified = asset.modified if asset.modified else asset.created
        response.accept_ranges = 'bytes'
        response.etag = '%s-%s-%s' % (asset.modified if asset.modified else asset.created, asset.size, hash(filename))
        response.cache_control = 'public'
        response.content_transfer_encoding = 'binary'
        response.content_disposition = "inline" if inline else ('attachment; filename=' + filename)

        target = StringIO()

        result, jquality = scale(asset.content.get(), target, raw=reflection, **kw)

        if reflection:
            result = add_reflection(result, bgcolor=color, amount=float(amount), opacity=float(opacity))
            result.save(target, "JPEG", optimize=True, quality=jquality)

        response.body = target.getvalue()

        return response
<commit_msg>Simpler solution.<commit_after># encoding: utf-8

"""Basic file controller."""

import webob

import web.core

from web.extras.contentment.api import action, view
from web.extras.contentment.components.asset.controller import AssetController
from web.extras.contentment.components.file.scaler import scale, get_cache, save_cache
from web.extras.contentment.components.file.reflection import add_reflection

from cStringIO import StringIO

log = __import__('logging').getLogger(__name__)
__all__ = ['FileController']

class FileController(AssetController):
    @view('Preview', "Preview uploaded content.")
    def view_preview(self):
        return 'preview', None

    @view('Download', "Download this file.")
    def view_download(self, filename=None, inline=False):
        asset = self.asset
        response = webob.Response(request=web.core.request, conditional_response=True)
        filename = asset.filename if filename is None else filename

        response.content_type = asset.mimetype.encode('ascii')
        response.content_length = asset.size
        response.last_modified = asset.modified if asset.modified else asset.created
        response.accept_ranges = 'bytes'
        response.etag = '%s-%s-%s' % (asset.modified if asset.modified else asset.created, asset.size, hash(filename))
        response.cache_control = 'public'
        response.content_transfer_encoding = 'binary'
        response.content_disposition = "inline" if inline else ('attachment; filename=' + filename)
        response.range = (0, asset.size)

        response.app_iter = asset.content.get()

        return response

    # TODO: Caching.
    @view('Scale', "Display a scaled version of this file.")
    def view_scale(self, filename=None, cache=True, inline=True, reflection=False, color="000000", amount=0.75, opacity=0.4, **kw):
        asset = self.asset

        color = '#' + color
        cache = bool(cache)
        inline = bool(inline)
        reflection = bool(reflection)
        amount = float(amount)
        opacity = float(opacity)

        if cache:
            id_ = str(asset.id)

            url = get_cache(str(asset.id), asset.modified, style="standard" if not reflection else "reflection", **kw)

            if url is not None:
                raise web.core.http.HTTPSeeOther(location=url)

            with open(save_cache(str(asset.id), asset.modified, style="standard" if not reflection else "reflection", **kw), 'wb') as fh:
                result, jquality = scale(asset.content.get(), fh, raw=reflection, **kw)

                if reflection:
                    result = add_reflection(result, bgcolor=color, amount=float(amount), opacity=float(opacity))
                    result.save(fh, "JPEG", optimize=True, quality=jquality)

            raise web.core.http.HTTPSeeOther(location=get_cache(id_, asset.modified, style="standard" if not reflection else "reflection", **kw))

        response = webob.Response(request=web.core.request, conditional_response=True)
        filename = asset.filename if filename is None else filename

        response.content_type = "image/jpeg"
        response.content_length = asset.size
        response.last_modified = asset.modified if asset.modified else asset.created
        response.accept_ranges = 'bytes'
        response.etag = '%s-%s-%s' % (asset.modified if asset.modified else asset.created, asset.size, hash(filename))
        response.cache_control = 'public'
        response.content_transfer_encoding = 'binary'
        response.content_disposition = "inline" if inline else ('attachment; filename=' + filename)

        target = StringIO()

        result, jquality = scale(asset.content.get(), target, raw=reflection, **kw)

        if reflection:
            result = add_reflection(result, bgcolor=color, amount=float(amount), opacity=float(opacity))
            result.save(target, "JPEG", optimize=True, quality=jquality)

        response.body = target.getvalue()

        return response
<|endoftext|>


<commit_before>import numpy as np
import scipy.ndimage
import statsmodels.api as stats
import matplotlib.pyplot as plt
import sep

def make_pixel_mask(image, saturation, input_mask=None):
    """Make a pixel mask that marks saturated pixels; optionally join with input_mask"""

    if input_mask is not None:
        new_mask = input_mask
    else:
        new_mask = np.zeros(image.shape)

    new_mask[image >= saturation] = 1
    return new_mask

def center_psf(psf):
    """Center psf at (0,0) based on max value"""

    peak = np.unravel_index(psf.argmax(), psf.shape)

    psf = np.roll(psf, -peak[0], 0)
    psf = np.roll(psf, -peak[1], 1)
    return psf

def fit_noise(data, n_stamps=1, mode='iqr'):
    """Find the standard deviation of the image background; returns standard deviation, median"""

    median_small = np.zeros([n_stamps, n_stamps])
    std_small = np.zeros([n_stamps, n_stamps])
    for y_stamp in range(n_stamps):
        for x_stamp in range(n_stamps):
            y_index = [y_stamp * data.shape[0] // n_stamps, (y_stamp + 1) * data.shape[0] // n_stamps]
            x_index = [x_stamp * data.shape[1] // n_stamps, (x_stamp + 1) * data.shape[1] // n_stamps]
            stamp_data = data[y_index[0]: y_index[1], x_index[0]: x_index[1]].compressed()
            if mode == 'gaussian':
                trimmed_stamp_data = stamp_data[stamp_data < np.percentile(stamp_data, 90)]
                trimmed_stamp_data = trimmed_stamp_data[trimmed_stamp_data != 0]
                histogram_data = np.histogram(trimmed_stamp_data, bins=100)
                x = histogram_data[1][:-1]
                y = histogram_data[0]
                guess = [np.max(y), np.median(trimmed_stamp_data), np.std(trimmed_stamp_data)]
                parameters, covariance = scipy.optimize.curve_fit(gauss, x, y, p0=guess, maxfev=1600)
                median_small[y_stamp, x_stamp] = parameters[1]
                std_small[y_stamp, x_stamp] = parameters[2]
            elif mode == 'iqr':
                quartile25, median, quartile75 = np.percentile(stamp_data, (25, 50, 75))
                median_small[y_stamp, x_stamp] = median
                # 0.741301109 is a tuning parameter that scales iqr to std
                std_small[y_stamp, x_stamp] = 0.741301109 * (quartile75 - quartile25)
            elif mode == 'mad':
                median = np.median(stamp_data)
                absdev = np.abs(stamp_data - median)
                mad = np.median(absdev)
                median_small[y_stamp, x_stamp] = median
                std_small[y_stamp, x_stamp] = 1.4826 * mad

    median = scipy.ndimage.zoom(median_small, [data.shape[0] / float(n_stamps), data.shape[1] / float(n_stamps)])
    std = scipy.ndimage.zoom(std_small, [data.shape[0] / float(n_stamps), data.shape[1] / float(n_stamps)])

    return std, median

def gauss(position, amplitude, median, std):
    """Return a gaussian function"""

    return amplitude * np.exp(-(position - median) ** 2 / (2 * std ** 2))

def interpolate_bad_pixels(image, median_size=6):
    """Interpolate over bad pixels using a global median"""

    # from http://stackoverflow.com/questions/18951500/automatically-remove-hot-dead-pixels-from-an-image-in-python
    pix = np.transpose(np.where(image.mask))
    blurred = scipy.ndimage.median_filter(image, size=median_size)
    interpolated_image = np.copy(image)
    for y, x in pix:
        interpolated_image[y, x] = blurred[y, x]

    return interpolated_image

def join_images(science_raw, science_mask, reference_raw, reference_mask, sigma_cut, min_elements, use_pixels=False, show=False):
    """Join two images to fittable vectors"""

    science = np.ma.array(science_raw, mask=science_mask, copy=True)
    reference = np.ma.array(reference_raw, mask=reference_mask, copy=True)
    science_std = np.ma.std(science)
    reference_std = np.ma.std(reference)
    if use_pixels:
        # remove pixels less than sigma_cut above sky level to speed fitting
        science.mask[science < sigma_cut * science_std] = True
        reference.mask[reference_raw < sigma_cut * reference_std] = True

        # flatten into 1d arrays of good pixels
        science.mask |= reference.mask
        reference.mask |= science.mask
        science_flatten = science.compressed()
        reference_flatten = reference.compressed()
    else:
        science_sources = sep.extract(np.ascontiguousarray(science.data), thresh=sigma_cut, err=science_std, mask=np.ascontiguousarray(science.mask))
        reference_sources = sep.extract(np.ascontiguousarray(reference.data), thresh=sigma_cut, err=reference_std, mask=np.ascontiguousarray(reference.mask))
        dx = science_sources['x'] - np.atleast_2d(reference_sources['x']).T
        dy = science_sources['y'] - np.atleast_2d(reference_sources['y']).T
        sep2 = dx**2 + dy**2
        matches = np.min(sep2, axis=1) < 1.
        inds = np.argmin(sep2, axis=1)
        science_flatten = science_sources['flux'][inds][matches]
        reference_flatten = reference_sources['flux'][matches]

    if show:
        plt.ion()
        plt.figure(1)
        plt.clf()
        vmin, vmax = np.percentile(science, (1, 99))
        plt.imshow(science, vmin=vmin, vmax=vmax)
        if not use_pixels:
            plt.plot(reference_sources['x'][matches], reference_sources['y'][matches], 'o', mfc='none', mec='r')

        plt.figure(2)
        plt.clf()
        vmin, vmax = np.percentile(reference, (1, 99))
        plt.imshow(reference, vmin=vmin, vmax=vmax)
        if not use_pixels:
            plt.plot(reference_sources['x'][matches], reference_sources['y'][matches], 'o', mfc='none', mec='r')

        plt.figure(3)
        plt.clf()
        plt.plot(reference_flatten, science_flatten, '.')
        plt.xlabel('Reference')
        plt.ylabel('Science')

    return reference_flatten, science_flatten

def resize_psf(psf, shape):
    """Resize centered (0,0) psf to larger shape"""

    psf_extended = np.pad(psf, ((0, shape[0] - psf.shape[0]), (0, shape[1] - psf.shape[1])),
                          mode='constant', constant_values=0.)
    return psf_extended

def pad_to_power2(data, value='median'):
    """Pad arrays to the nearest power of two"""

    if value == 'median':
        constant = np.median(data)
    elif value == 'zero':
        constant = 0.
    n = 0
    defecit = [0, 0]
    while (data.shape[0] > (2 ** n)) or (data.shape[1] > (2 ** n)):
        n += 1
        defecit = [(2 ** n) - data.shape[0], (2 ** n) - data.shape[1]]
    padded_data = np.pad(data, ((0, defecit[0]), (0, defecit[1])), mode='constant', constant_values=constant)
    return padded_data

def solve_iteratively(science, reference, mask_tolerance=10e-5, gain_tolerance=10e-6,
                      max_iterations=5, sigma_cut=5, min_elements=800, use_pixels=False, show=False):
    """Solve for linear fit iteratively"""

    gain = 1.
    gain0 = 10e5
    i = 0
    # pad image to power of two to speed fft
    old_size = science.image_data.shape
    science_image = pad_to_power2(science.image_data)
    reference_image = pad_to_power2(reference.image_data)

    science_psf = center_psf(resize_psf(science.raw_psf_data, science_image.shape))
    science_psf /= np.sum(science.raw_psf_data)
    reference_psf = center_psf(resize_psf(reference.raw_psf_data, reference_image.shape))
    reference_psf /= np.sum(reference.raw_psf_data)

    science_std = pad_to_power2(science.background_std)
    reference_std = pad_to_power2(reference.background_std)

    science_mask = pad_to_power2(science.pixel_mask)
    reference_mask = pad_to_power2(reference.pixel_mask)

    # fft arrays
    science_image_fft = np.fft.fft2(science_image)
    reference_image_fft = np.fft.fft2(reference_image)
    science_psf_fft = np.fft.fft2(science_psf)
    reference_psf_fft = np.fft.fft2(reference_psf)

    while abs(gain - gain0) > gain_tolerance:

        # calculate the psf in the difference image to convolve masks
        # not a simple convolution of the two PSF's; see the paper for details
        difference_zero_point = gain / np.sqrt(science_std ** 2 + reference_std ** 2 * gain ** 2)
        denominator = science_std ** 2 * abs(reference_psf_fft) ** 2
        denominator += reference_std ** 2 * gain ** 2 * abs(science_psf_fft) ** 2
        difference_psf_fft = gain * science_psf_fft * reference_psf_fft / (difference_zero_point * np.sqrt(denominator))

        # convolve masks with difference psf to mask all pixels within a psf radius
        # this is important to prevent convolutions of saturated pixels from affecting the fit
        science_mask_convolved = np.fft.ifft2(difference_psf_fft * np.fft.fft2(science_mask))
        science_mask_convolved[science_mask_convolved > mask_tolerance] = 1
        science_mask_convolved = np.real(science_mask_convolved).astype(int)
        reference_mask_convolved = np.fft.ifft2(difference_psf_fft * np.fft.fft2(reference_mask))
        reference_mask_convolved[reference_mask_convolved > mask_tolerance] = 1
        reference_mask_convolved = np.real(reference_mask_convolved).astype(int)

        # do the convolutions on the images
        denominator = science_std ** 2 * abs(reference_psf_fft) ** 2
        denominator += gain ** 2 * reference_std ** 2 * abs(science_psf_fft) ** 2

        science_convolved_image_fft = reference_psf_fft * science_image_fft / np.sqrt(denominator)
        reference_convolved_image_fft = science_psf_fft * reference_image_fft / np.sqrt(denominator)

        science_convolved_image = np.real(np.fft.ifft2(science_convolved_image_fft))
        reference_convolved_image = np.real(np.fft.ifft2(reference_convolved_image_fft))

        # remove power of 2 padding
        science_convolved_image = science_convolved_image[: old_size[0], : old_size[1]]
        reference_convolved_image = reference_convolved_image[: old_size[0], : old_size[1]]
        science_mask_convolved = science_mask_convolved[: old_size[0], : old_size[1]]
        reference_mask_convolved = reference_mask_convolved[: old_size[0], : old_size[1]]

        # do a linear robust regression between convolved images
        gain0 = gain
        sigma = sigma_cut
        x, y = join_images(science_convolved_image, science_mask_convolved, reference_convolved_image,
                           reference_mask_convolved, sigma, min_elements, use_pixels, show)
        while (x.size < min_elements) or (y.size < min_elements):
            sigma -= 0.5
            x, y = join_images(science_convolved_image, science_mask_convolved, reference_convolved_image,
                               reference_mask_convolved, sigma, min_elements, use_pixels, show)
            if sigma == 0.:
                break
        robust_fit = stats.RLM(y, x).fit()
        parameters = robust_fit.params
        gain = parameters[0]
        gain1 = np.median(y / x)
        if show:
            xfit = np.arange(np.max(x))
            plt.plot(xfit, gain*xfit, label='gain')
            plt.plot(xfit, gain1*xfit, label='gain1')
            plt.legend()
            raw_input('Press enter to continue to next iteration')

        if i == max_iterations:
            break
        i += 1
        print('Iteration {0}:'.format(i))
        print('Gain = {0}'.format(gain))

    print('Fit done in {} iterations'.format(i))

    return gain
<commit_msg>remove worse way of calculating gain<commit_after>import numpy as np
import scipy.ndimage
import statsmodels.api as stats
import matplotlib.pyplot as plt
import sep

def make_pixel_mask(image, saturation, input_mask=None):
    """Make a pixel mask that marks saturated pixels; optionally join with input_mask"""

    if input_mask is not None:
        new_mask = input_mask
    else:
        new_mask = np.zeros(image.shape)

    new_mask[image >= saturation] = 1
    return new_mask

def center_psf(psf):
    """Center psf at (0,0) based on max value"""

    peak = np.unravel_index(psf.argmax(), psf.shape)

    psf = np.roll(psf, -peak[0], 0)
    psf = np.roll(psf, -peak[1], 1)
    return psf

def fit_noise(data, n_stamps=1, mode='iqr'):
    """Find the standard deviation of the image background; returns standard deviation, median"""

    median_small = np.zeros([n_stamps, n_stamps])
    std_small = np.zeros([n_stamps, n_stamps])
    for y_stamp in range(n_stamps):
        for x_stamp in range(n_stamps):
            y_index = [y_stamp * data.shape[0] // n_stamps, (y_stamp + 1) * data.shape[0] // n_stamps]
            x_index = [x_stamp * data.shape[1] // n_stamps, (x_stamp + 1) * data.shape[1] // n_stamps]
            stamp_data = data[y_index[0]: y_index[1], x_index[0]: x_index[1]].compressed()
            if mode == 'gaussian':
                trimmed_stamp_data = stamp_data[stamp_data < np.percentile(stamp_data, 90)]
                trimmed_stamp_data = trimmed_stamp_data[trimmed_stamp_data != 0]
                histogram_data = np.histogram(trimmed_stamp_data, bins=100)
                x = histogram_data[1][:-1]
                y = histogram_data[0]
                guess = [np.max(y), np.median(trimmed_stamp_data), np.std(trimmed_stamp_data)]
                parameters, covariance = scipy.optimize.curve_fit(gauss, x, y, p0=guess, maxfev=1600)
                median_small[y_stamp, x_stamp] = parameters[1]
                std_small[y_stamp, x_stamp] = parameters[2]
            elif mode == 'iqr':
                quartile25, median, quartile75 = np.percentile(stamp_data, (25, 50, 75))
                median_small[y_stamp, x_stamp] = median
                # 0.741301109 is a tuning parameter that scales iqr to std
                std_small[y_stamp, x_stamp] = 0.741301109 * (quartile75 - quartile25)
            elif mode == 'mad':
                median = np.median(stamp_data)
                absdev = np.abs(stamp_data - median)
                mad = np.median(absdev)
                median_small[y_stamp, x_stamp] = median
                std_small[y_stamp, x_stamp] = 1.4826 * mad

    median = scipy.ndimage.zoom(median_small, [data.shape[0] / float(n_stamps), data.shape[1] / float(n_stamps)])
    std = scipy.ndimage.zoom(std_small, [data.shape[0] / float(n_stamps), data.shape[1] / float(n_stamps)])

    return std, median

def gauss(position, amplitude, median, std):
    """Return a gaussian function"""

    return amplitude * np.exp(-(position - median) ** 2 / (2 * std ** 2))

def interpolate_bad_pixels(image, median_size=6):
    """Interpolate over bad pixels using a global median"""

    # from http://stackoverflow.com/questions/18951500/automatically-remove-hot-dead-pixels-from-an-image-in-python
    pix = np.transpose(np.where(image.mask))
    blurred = scipy.ndimage.median_filter(image, size=median_size)
    interpolated_image = np.copy(image)
    for y, x in pix:
        interpolated_image[y, x] = blurred[y, x]

    return interpolated_image

def join_images(science_raw, science_mask, reference_raw, reference_mask, sigma_cut, min_elements, use_pixels=False, show=False):
    """Join two images to fittable vectors"""

    science = np.ma.array(science_raw, mask=science_mask, copy=True)
    reference = np.ma.array(reference_raw, mask=reference_mask, copy=True)
    science_std = np.ma.std(science)
    reference_std = np.ma.std(reference)
    if use_pixels:
        # remove pixels less than sigma_cut above sky level to speed fitting
        science.mask[science < sigma_cut * science_std] = True
        reference.mask[reference_raw < sigma_cut * reference_std] = True

        # flatten into 1d arrays of good pixels
        science.mask |= reference.mask
        reference.mask |= science.mask
        science_flatten = science.compressed()
        reference_flatten = reference.compressed()
    else:
        science_sources = sep.extract(np.ascontiguousarray(science.data), thresh=sigma_cut, err=science_std, mask=np.ascontiguousarray(science.mask))
        reference_sources = sep.extract(np.ascontiguousarray(reference.data), thresh=sigma_cut, err=reference_std, mask=np.ascontiguousarray(reference.mask))
        dx = science_sources['x'] - np.atleast_2d(reference_sources['x']).T
        dy = science_sources['y'] - np.atleast_2d(reference_sources['y']).T
        sep2 = dx**2 + dy**2
        matches = np.min(sep2, axis=1) < 1.
        inds = np.argmin(sep2, axis=1)
        science_flatten = science_sources['flux'][inds][matches]
        reference_flatten = reference_sources['flux'][matches]

    if show:
        plt.ion()
        plt.figure(1)
        plt.clf()
        vmin, vmax = np.percentile(science, (1, 99))
        plt.imshow(science, vmin=vmin, vmax=vmax)
        if not use_pixels:
            plt.plot(reference_sources['x'][matches], reference_sources['y'][matches], 'o', mfc='none', mec='r')

        plt.figure(2)
        plt.clf()
        vmin, vmax = np.percentile(reference, (1, 99))
        plt.imshow(reference, vmin=vmin, vmax=vmax)
        if not use_pixels:
            plt.plot(reference_sources['x'][matches], reference_sources['y'][matches], 'o', mfc='none', mec='r')

        plt.figure(3)
        plt.clf()
        plt.plot(reference_flatten, science_flatten, '.')
        plt.xlabel('Reference')
        plt.ylabel('Science')

    return reference_flatten, science_flatten

def resize_psf(psf, shape):
    """Resize centered (0,0) psf to larger shape"""

    psf_extended = np.pad(psf, ((0, shape[0] - psf.shape[0]), (0, shape[1] - psf.shape[1])),
                          mode='constant', constant_values=0.)
    return psf_extended

def pad_to_power2(data, value='median'):
    """Pad arrays to the nearest power of two"""

    if value == 'median':
        constant = np.median(data)
    elif value == 'zero':
        constant = 0.
    n = 0
    defecit = [0, 0]
    while (data.shape[0] > (2 ** n)) or (data.shape[1] > (2 ** n)):
        n += 1
        defecit = [(2 ** n) - data.shape[0], (2 ** n) - data.shape[1]]
    padded_data = np.pad(data, ((0, defecit[0]), (0, defecit[1])), mode='constant', constant_values=constant)
    return padded_data

def solve_iteratively(science, reference, mask_tolerance=10e-5, gain_tolerance=10e-6,
                      max_iterations=5, sigma_cut=5, min_elements=800, use_pixels=False, show=False):
    """Solve for linear fit iteratively"""

    gain = 1.
    gain0 = 10e5
    i = 0
    # pad image to power of two to speed fft
    old_size = science.image_data.shape
    science_image = pad_to_power2(science.image_data)
    reference_image = pad_to_power2(reference.image_data)

    science_psf = center_psf(resize_psf(science.raw_psf_data, science_image.shape))
    science_psf /= np.sum(science.raw_psf_data)
    reference_psf = center_psf(resize_psf(reference.raw_psf_data, reference_image.shape))
    reference_psf /= np.sum(reference.raw_psf_data)

    science_std = pad_to_power2(science.background_std)
    reference_std = pad_to_power2(reference.background_std)

    science_mask = pad_to_power2(science.pixel_mask)
    reference_mask = pad_to_power2(reference.pixel_mask)

    # fft arrays
    science_image_fft = np.fft.fft2(science_image)
    reference_image_fft = np.fft.fft2(reference_image)
    science_psf_fft = np.fft.fft2(science_psf)
    reference_psf_fft = np.fft.fft2(reference_psf)

    while abs(gain - gain0) > gain_tolerance:

        # calculate the psf in the difference image to convolve masks
        # not a simple convolution of the two PSF's; see the paper for details
        difference_zero_point = gain / np.sqrt(science_std ** 2 + reference_std ** 2 * gain ** 2)
        denominator = science_std ** 2 * abs(reference_psf_fft) ** 2
        denominator += reference_std ** 2 * gain ** 2 * abs(science_psf_fft) ** 2
        difference_psf_fft = gain * science_psf_fft * reference_psf_fft / (difference_zero_point * np.sqrt(denominator))

        # convolve masks with difference psf to mask all pixels within a psf radius
        # this is important to prevent convolutions of saturated pixels from affecting the fit
        science_mask_convolved = np.fft.ifft2(difference_psf_fft * np.fft.fft2(science_mask))
        science_mask_convolved[science_mask_convolved > mask_tolerance] = 1
        science_mask_convolved = np.real(science_mask_convolved).astype(int)
        reference_mask_convolved = np.fft.ifft2(difference_psf_fft * np.fft.fft2(reference_mask))
        reference_mask_convolved[reference_mask_convolved > mask_tolerance] = 1
        reference_mask_convolved = np.real(reference_mask_convolved).astype(int)

        # do the convolutions on the images
        denominator = science_std ** 2 * abs(reference_psf_fft) ** 2
        denominator += gain ** 2 * reference_std ** 2 * abs(science_psf_fft) ** 2

        science_convolved_image_fft = reference_psf_fft * science_image_fft / np.sqrt(denominator)
        reference_convolved_image_fft = science_psf_fft * reference_image_fft / np.sqrt(denominator)

        science_convolved_image = np.real(np.fft.ifft2(science_convolved_image_fft))
        reference_convolved_image = np.real(np.fft.ifft2(reference_convolved_image_fft))

        # remove power of 2 padding
        science_convolved_image = science_convolved_image[: old_size[0], : old_size[1]]
        reference_convolved_image = reference_convolved_image[: old_size[0], : old_size[1]]
        science_mask_convolved = science_mask_convolved[: old_size[0], : old_size[1]]
        reference_mask_convolved = reference_mask_convolved[: old_size[0], : old_size[1]]

        # do a linear robust regression between convolved images
        gain0 = gain
        sigma = sigma_cut
        x, y = join_images(science_convolved_image, science_mask_convolved, reference_convolved_image,
                           reference_mask_convolved, sigma, min_elements, use_pixels, show)
        while (x.size < min_elements) or (y.size < min_elements):
            sigma -= 0.5
            x, y = join_images(science_convolved_image, science_mask_convolved, reference_convolved_image,
                               reference_mask_convolved, sigma, min_elements, use_pixels, show)
            if sigma == 0.:
                break
        robust_fit = stats.RLM(y, x).fit()
        parameters = robust_fit.params
        gain = parameters[0]
        if show:
            xfit = np.arange(np.max(x))
            plt.plot(xfit, gain*xfit)
            plt.legend()
            raw_input('Press enter to continue to next iteration')

        if i == max_iterations:
            break
        i += 1
        print('Iteration {0}:'.format(i))
        print('Gain = {0}'.format(gain))

    print('Fit done in {} iterations'.format(i))

    return gain
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright(C) 2011  Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

import time
from weboob.capabilities.messages import ICapMessages, Message, Thread
from weboob.tools.backend import BaseBackend
from weboob.tools.newsfeed import Newsfeed

class GenericNewspaperBackend(BaseBackend, ICapMessages):
    """
    GenericNewspaperBackend class
    """
    MAINTAINER = u'Julien Hebert'
    EMAIL = 'juke@free.fr'
    VERSION = '0.i'
    LICENSE = 'AGPLv3+'
    STORAGE = {'seen': {}}
    RSS_FEED = None
    RSSID = None
    URL2ID = None
    RSSSIZE = 0

    def _get_thread(self, id):
        for thread in self.iter_threads():
            if thread.id == id:
                return thread

    def get_thread(self, _id):
        if isinstance(_id, Thread):
            thread = _id
            id = thread.id
        else:
            thread = self._get_thread(_id)
            id = _id

        with self.browser:
            content = self.browser.get_content(id)

        if content is None:
            return None

        if not thread:
            thread = Thread(id)

        flags = Message.IS_HTML
        if not thread.id in self.storage.get('seen', default={}):
            flags |= Message.IS_UNREAD
        thread.title = content.title
        if not thread.date:
            thread.date = content.date

        thread.root = Message(
            thread=thread,
            id=0,
            title=content.title,
            sender=content.author,
            receivers=None,
            date=thread.date,
            parent=None,
            content=content.body,
            signature= u'URL \n' % content.url,
            flags=flags,
            children=[])
        return thread

    def iter_threads(self):
        for article in Newsfeed(self.RSS_FEED, GenericNewspaperBackend.RSSID).iter_entries():
            thread = Thread(article.id)
            thread.title = article.title
            thread.date = article.datetime
            yield(thread)

    def fill_thread(self, thread, fields):
        "fill the thread"
        t = self.get_thread(thread)
        return t or thread

    def iter_unread_messages(self):
        for thread in self.iter_threads():
            if thread.id in self.storage.get('seen', default={}):
                continue
            self.fill_thread(thread, 'root')
            for msg in thread.iter_all_messages():
                yield msg

    def set_message_read(self, message):
        self.storage.set(
            'seen',
            message.thread.id,
            'comments',
            self.storage.get(
                'seen',
                message.thread.id,
                'comments',
                default=[]) + [message.id])

        if self.URL2ID and self.RSSSIZE != 0:
            url2id = self.URL2ID
            lastpurge = self.storage.get('lastpurge', default=0)
            l = []
            if time.time() - lastpurge > 7200:
                self.storage.set('lastpurge', time.time())
                for id in self.storage.get('seen', default={}):
                    l.append((int(url2id(id)), id))
                l.sort()
                l.reverse()
                tosave = [v[1] for v in l[0:self.RSSSIZE + 10]]
                toremove = set([v for v in self.storage.get('seen', default={})]).difference(tosave)
                for id in toremove:
                    self.storage.delete('seen', id)

        self.storage.save()

    OBJECTS = {Thread: fill_thread}
<commit_msg>One more use of find_object<commit_after># -*- coding: utf-8 -*-

# Copyright(C) 2011  Julien Hebert
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

import time
from weboob.capabilities.messages import ICapMessages, Message, Thread
from weboob.capabilities.base import find_object
from weboob.tools.backend import BaseBackend
from weboob.tools.newsfeed import Newsfeed

class GenericNewspaperBackend(BaseBackend, ICapMessages):
    """
    GenericNewspaperBackend class
    """
    MAINTAINER = u'Julien Hebert'
    EMAIL = 'juke@free.fr'
    VERSION = '0.i'
    LICENSE = 'AGPLv3+'
    STORAGE = {'seen': {}}
    RSS_FEED = None
    RSSID = None
    URL2ID = None
    RSSSIZE = 0

    def get_thread(self, _id):
        if isinstance(_id, Thread):
            thread = _id
            id = thread.id
        else:
            thread = find_object(self.iter_threads(), id=_id)
            id = _id

        with self.browser:
            content = self.browser.get_content(id)

        if content is None:
            return None

        if not thread:
            thread = Thread(id)

        flags = Message.IS_HTML
        if not thread.id in self.storage.get('seen', default={}):
            flags |= Message.IS_UNREAD
        thread.title = content.title
        if not thread.date:
            thread.date = content.date

        thread.root = Message(
            thread=thread,
            id=0,
            title=content.title,
            sender=content.author,
            receivers=None,
            date=thread.date,
            parent=None,
            content=content.body,
            signature= u'URL \n' % content.url,
            flags=flags,
            children=[])
        return thread

    def iter_threads(self):
        for article in Newsfeed(self.RSS_FEED, GenericNewspaperBackend.RSSID).iter_entries():
            thread = Thread(article.id)
            thread.title = article.title
            thread.date = article.datetime
            yield(thread)

    def fill_thread(self, thread, fields):
        "fill the thread"
        t = self.get_thread(thread)
        return t or thread

    def iter_unread_messages(self):
        for thread in self.iter_threads():
            if thread.id in self.storage.get('seen', default={}):
                continue
            self.fill_thread(thread, 'root')
            for msg in thread.iter_all_messages():
                yield msg

    def set_message_read(self, message):
        self.storage.set(
            'seen',
            message.thread.id,
            'comments',
            self.storage.get(
                'seen',
                message.thread.id,
                'comments',
                default=[]) + [message.id])

        if self.URL2ID and self.RSSSIZE != 0:
            url2id = self.URL2ID
            lastpurge = self.storage.get('lastpurge', default=0)
            l = []
            if time.time() - lastpurge > 7200:
                self.storage.set('lastpurge', time.time())
                for id in self.storage.get('seen', default={}):
                    l.append((int(url2id(id)), id))
                l.sort()
                l.reverse()
                tosave = [v[1] for v in l[0:self.RSSSIZE + 10]]
                toremove = set([v for v in self.storage.get('seen', default={})]).difference(tosave)
                for id in toremove:
                    self.storage.delete('seen', id)

        self.storage.save()

    OBJECTS = {Thread: fill_thread}
<|endoftext|>


<commit_before>#DronePool module which handles interaction with SITLs

from dronekit import Vehicle, VehicleMode, connect
from dronekit_sitl import SITL
import node, time
import mavparser

drone_pool = {}
instance_count = 0

class Sim(SITL, object):
	def __init__(self, instance=1, home=None):
		super(Sim, self).download("copter", "3.3", verbose=True)
		self.instance = instance

		if home:
			self.home = home
		else:
			self.home = {"lat":6.9271, "lon":79.8612, "alt": 1}

		self.p = None
		return

	def connection_string(self):
		return super(Sim, self).connection_string()[:-4] + str(5760 + self.instance * 10)

	def launch(self):
		home_str = str(self.home['lat']) + ',' + str(self.home['lon']) + ',' + str(self.home['alt']) + ',353'
		super(Sim, self).launch(["--instance", str(self.instance), "--home", home_str], await_ready=True, verbose=True)

	def get_sitl_status(self):
		return { 'id': self.instance, 'home': self.home }

def initialize():

	drones = node.get_drones()['drones']

	if not drones:
		return

	for drone_id in drones:
		if drone_id not in drone_pool.keys():
			location = node.get_drone_by_id(drone_id)['location']
			create_new_drone(db_key=drone_id, home=location)

def create_new_drone(home=None, db_key=None):
	global instance_count
	drone = Sim(instance_count, home)
	drone.launch()
	drone_conn = connect(drone.connection_string(), wait_ready=True)

	drone_pool[db_key] = drone_conn
	instance_count += 1

	res = { "status" : "OK", "id" : db_key }
	return res

def run_mission(drone, target_height, waypoints):
	while True:
		if drone.location.global_relative_frame.alt >= target_height * 0.9:
			break

	print 'target alt reached'

	mavparser.create_mission(drone, waypoints)
	print 'mission acquired'

	drone.mode = VehicleMode('AUTO')
	print 'initiating sequence'

	print 'in mission'

def takeoff_drone(drone_id, target_height=10, waypoints=None):
	try:
		drone = drone_pool[drone_id]
	except:
		raise

	drone.initialize()

	while not drone.is_armable:
		time.sleep(1)

	drone.mode = VehicleMode('GUIDED')
	drone.armed = True

	while not drone.armed:
		time.sleep(1)

	drone.simple_takeoff(target_height)

	if waypoints:
		run_mission(drone, target_height, waypoints)

	@drone.on_attribute('location')
	def update_location(self, attr_name, value):
		node.update_drone(drone_id, { "location" : {"lat": value.global_relative_frame.lat, "lon": value.global_relative_frame.lon, "alt": value.global_relative_frame.alt}})

	@drone.on_attribute('airspeed')
	def update_airspeed(self, attr_name, value):
		node.update_drone(drone_id, {"airspeed": value})

	@drone.on_attribute('attitude')
	def udpate_attitude(self, attr_name, value):
		node.update_drone(drone_id, { "pitch": value.pitch, 'roll': value.roll, 'yaw': value.yaw })

	@drone.on_attribute('heading')
	def update_heading(self, attr_name, value):
		node.update_drone(drone_id, { "heading": value })

	print 'took off'

	return True

def land_drone(drone_id):
	try:
		drone = drone_pool[drone_id]
	except:
		raise

	if not drone.armed:
		return False

	cmds = drone.commands
	cmds.wait_ready()
	cmds.clear()

	drone.mode = VehicleMode('LAND')

	return True
<commit_msg>Initialize sims with altitude of 0<commit_after>#DronePool module which handles interaction with SITLs

from dronekit import Vehicle, VehicleMode, connect
from dronekit_sitl import SITL
import node, time
import mavparser

drone_pool = {}
instance_count = 0

class Sim(SITL, object):
	def __init__(self, instance=1, home=None):
		super(Sim, self).download("copter", "3.3", verbose=True)
		self.instance = instance

		if home:
			self.home = home
		else:
			self.home = {"lat":6.9271, "lon":79.8612, "alt": 1}

		self.p = None
		return

	def connection_string(self):
		return super(Sim, self).connection_string()[:-4] + str(5760 + self.instance * 10)

	def launch(self):
		home_str = str(self.home['lat']) + ',' + str(self.home['lon']) + ',0,353'
		super(Sim, self).launch(["--instance", str(self.instance), "--home", home_str], await_ready=True, verbose=True)

	def get_sitl_status(self):
		return { 'id': self.instance, 'home': self.home }

def initialize():

	drones = node.get_drones()['drones']

	if not drones:
		return

	for drone_id in drones:
		if drone_id not in drone_pool.keys():
			location = node.get_drone_by_id(drone_id)['location']
			create_new_drone(db_key=drone_id, home=location)

def create_new_drone(home=None, db_key=None):
	global instance_count
	drone = Sim(instance_count, home)
	drone.launch()
	drone_conn = connect(drone.connection_string(), wait_ready=True)

	drone_pool[db_key] = drone_conn
	instance_count += 1

	res = { "status" : "OK", "id" : db_key }
	return res

def run_mission(drone, target_height, waypoints):
	while True:
		if drone.location.global_relative_frame.alt >= target_height * 0.9:
			break

	print 'target alt reached'

	mavparser.create_mission(drone, waypoints)
	print 'mission acquired'

	drone.mode = VehicleMode('AUTO')
	print 'initiating sequence'

	print 'in mission'

def takeoff_drone(drone_id, target_height=10, waypoints=None):
	try:
		drone = drone_pool[drone_id]
	except:
		raise

	drone.initialize()

	while not drone.is_armable:
		time.sleep(1)

	drone.mode = VehicleMode('GUIDED')
	drone.armed = True

	while not drone.armed:
		time.sleep(1)

	drone.simple_takeoff(target_height)

	if waypoints:
		run_mission(drone, target_height, waypoints)

	@drone.on_attribute('location')
	def update_location(self, attr_name, value):
		node.update_drone(drone_id, { "location" : {"lat": value.global_relative_frame.lat, "lon": value.global_relative_frame.lon, "alt": value.global_relative_frame.alt}})

	@drone.on_attribute('airspeed')
	def update_airspeed(self, attr_name, value):
		node.update_drone(drone_id, {"airspeed": value})

	@drone.on_attribute('attitude')
	def udpate_attitude(self, attr_name, value):
		node.update_drone(drone_id, { "pitch": value.pitch, 'roll': value.roll, 'yaw': value.yaw })

	@drone.on_attribute('heading')
	def update_heading(self, attr_name, value):
		node.update_drone(drone_id, { "heading": value })

	print 'took off'

	return True

def land_drone(drone_id):
	try:
		drone = drone_pool[drone_id]
	except:
		raise

	if not drone.armed:
		return False

	cmds = drone.commands
	cmds.wait_ready()
	cmds.clear()

	drone.mode = VehicleMode('LAND')

	return True
<|endoftext|>


<commit_before>import re
from django import template

register = template.Library()

def user_dweet_to_link(m):
    text = m.group('text')
    dweet_id = m.group('dweet_id')
    username = m.group('username')

    if username is None:
        url = 'd/' + dweet_id
    else:
        url = 'u/' + username

    result = '{0}'.format(url)
    return text.replace(url, result)

def hashtag_to_link(m):
    text = m.group('text')
    hashtag = m.group('hashtag')

    url = 'h/' + hashtag
    tag = '#' + hashtag

    result = '{1}'.format(url, tag)
    return text.replace(tag, result)

@register.filter(is_safe=True)
def insert_magic_links(text):
    text = re.sub(
        r'(?:^|(?<=\s))'                                        # start of string or whitespace
        r'/?'                                                   # optional /
        r'(?P<text>'                                            # capture original pattern
        r'[^a-zA-Z\d]?d/(?P<dweet_id>\d+)[^a-zA-Z]?'            # dweet reference
        r'|'                                                    # or
        r'[^a-zA-Z\d]?u/(?P<username>[\w.@+-]+)[^a-zA-Z\d]?)'   # user reference
        r'(?=$|\s)',                                            # end of string or whitespace
        user_dweet_to_link,
        text
    )
    text = re.sub(
        r'(?P<text>'                                            # capture original pattern
        r'#(?P<hashtag>[a-zA-Z\d]+)[^a-zA-Z\d]?)',                 # hashtag
        hashtag_to_link,
        text
    )

    return text
<commit_msg>Allow hashtag right after u/username or d/123#link<commit_after>import re
from django import template

register = template.Library()

def user_dweet_to_link(m):
    text = m.group('text')
    dweet_id = m.group('dweet_id')
    username = m.group('username')

    if username is None:
        url = 'd/' + dweet_id
    else:
        url = 'u/' + username

    result = '{0}'.format(url)
    return text.replace(url, result)

def hashtag_to_link(m):
    text = m.group('text')
    hashtag = m.group('hashtag')

    url = 'h/' + hashtag
    tag = '#' + hashtag

    result = '{1}'.format(url, tag)
    return text.replace(tag, result)

@register.filter(is_safe=True)
def insert_magic_links(text):
    text = re.sub(
        r'(?:^|(?<=\s))'                                        # start of string or whitespace
        r'/?'                                                   # optional /
        r'(?P<text>'                                            # capture original pattern
        r'[^a-zA-Z\d]?d/(?P<dweet_id>\d+)[^a-zA-Z]?'            # dweet reference
        r'|'                                                    # or
        r'[^a-zA-Z\d]?u/(?P<username>[\w.@+-]+)[^a-zA-Z\d]?)'   # user reference
        r'(?=$|\s|#)',                                          # end of string, whitespace or hashtag
        user_dweet_to_link,
        text
    )
    text = re.sub(
        r'(?P<text>'                                            # capture original pattern
        r'#(?P<hashtag>[a-zA-Z\d]+)[^a-zA-Z\d]?)',              # hashtag
        hashtag_to_link,
        text
    )

    return text
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright 2015 by Ecpy Authors, see AUTHORS for more details.
#
# Distributed under the terms of the BSD license.
#
# The full license is in the file LICENCE, distributed with this software.
# -----------------------------------------------------------------------------
"""Base classes to handle driver settings edition.

Settings are architecture specific information. Then can allow to select a
if several are available for example.

"""
from __future__ import (division, unicode_literals, print_function,
                        absolute_import)

from atom.api import Unicode, ForwardTyped, Bool
from enaml.core.api import d_, Declarative, d_func
from enaml.widgets.api import GroupBox

class BaseSettings(GroupBox):
    """Base widget for creating settings.

    """
    #: Id of this settings (different from the declaration one as multiple
    #: settings of the same type can exist for a single instrument).
    name = d_(Unicode())

    #: Reference to the declaration that created this object.
    declaration = d_(ForwardTyped(lambda: Settings))

    #: Whether or not to make the settings editable
    read_only = d_(Bool())

    @d_func
    def gather_infos(self):
        """Return the current values as a dictionary.

        """
        return {'id': self.declaration.id, 'name': self.name}

    def _default_title(self):
        return self.name

class Settings(Declarative):
    """A declarative class for contributing a driver settings.

    Settings object can be contributed as extensions child to the
    'settings' extension point of the 'ecpy.instruments' plugin.

    """
    #: Unique name used to identify the editor.
    id = d_(Unicode())

    #: Connection description.
    description = d_(Unicode())

    @d_func
    def new(self, workbench, defaults):
        """Create a new setting and instantiate it properly.

        Defaults should be used to update the created setting.

        """
        raise NotImplementedError()
<commit_msg>instruments.settings : fix title and update title when name changes<commit_after># -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright 2015 by Ecpy Authors, see AUTHORS for more details.
#
# Distributed under the terms of the BSD license.
#
# The full license is in the file LICENCE, distributed with this software.
# -----------------------------------------------------------------------------
"""Base classes to handle driver settings edition.

Settings are architecture specific information. Then can allow to select a
if several are available for example.

"""
from __future__ import (division, unicode_literals, print_function,
                        absolute_import)

from atom.api import Unicode, ForwardTyped, Bool
from enaml.core.api import d_, Declarative, d_func
from enaml.widgets.api import GroupBox

class BaseSettings(GroupBox):
    """Base widget for creating settings.

    """
    #: Id of this settings (different from the declaration one as multiple
    #: settings of the same type can exist for a single instrument).
    name = d_(Unicode())

    #: Reference to the declaration that created this object.
    declaration = d_(ForwardTyped(lambda: Settings))

    #: Whether or not to make the settings editable
    read_only = d_(Bool())

    @d_func
    def gather_infos(self):
        """Return the current values as a dictionary.

        """
        return {'id': self.declaration.id, 'name': self.name}

    def _default_title(self):
        return self.name + '(' + self.declaration.id + ')'

    def _post_setattr_name(self, old, new):
        self.title = new + '(' + self.declaration.id + ')'

class Settings(Declarative):
    """A declarative class for contributing a driver settings.

    Settings object can be contributed as extensions child to the
    'settings' extension point of the 'ecpy.instruments' plugin.

    """
    #: Unique name used to identify the editor.
    id = d_(Unicode())

    #: Connection description.
    description = d_(Unicode())

    @d_func
    def new(self, workbench, defaults):
        """Create a new setting and instantiate it properly.

        Defaults should be used to update the created setting.

        """
        raise NotImplementedError()
<|endoftext|>