<commit_before>#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at http://mozilla.org/MPL/2.0/.

import logging
import os
import subprocess
import time
import uuid

import stoneridge

class StoneRidgeMaster(stoneridge.QueueListener):
    def setup(self):
        self.queues = {
            'broadband': stoneridge.QueueWriter(
                stoneridge.NETCONFIG_QUEUES['broadband']),
            'umts': stoneridge.QueueWriter(
                stoneridge.NETCONFIG_QUEUES['umts']),
            'gsm': stoneridge.QueueWriter(
                stoneridge.NETCONFIG_QUEUES['gsm'])
        }
        self.logdir = stoneridge.get_config('stoneridge', 'logs')
        self.config = stoneridge.get_config_file()

    def handle(self, nightly, ldap, sha, netconfigs, operating_systems,
            srid=None, attempt=1):
        if srid is None:
            srid = str(uuid.uuid4())

        logging.debug('Got request')
        logging.debug('Nightly: %s' % (nightly,))
        logging.debug('LDAP: %s' % (ldap,))
        logging.debug('SHA: %s' % (sha,))
        logging.debug('Netconfigs: %s' % (' '.join(netconfigs),))
        logging.debug('Operating systems: %s' % (' '.join(operating_systems),))
        logging.debug('Attempt: %s' % (attempt,))
        logging.debug('SRID: %s' % (srid,))

        logfile = 'cloner_%s.log' % (srid,)
        cloner_log = os.path.join(self.logdir, logfile)
        args = ['srcloner.py', '--config', self.config, '--srid', srid,
                '--log', cloner_log, '--attempt', attempt]
        if nightly:
            args.append('--nightly')

            # Make sure the list of netconfigs and operating systems is right
            netconfigs = stoneridge.NETCONFIGS
            operating_systems = stoneridge.OPERATING_SYSTEMS
        else:
            if not ldap or not sha:
                logging.error('Missing ldap/sha for non-nightly build')
                return

            args.extend(['--ldap', ldap])
            args.extend(['--sha', sha])
            for ops in operating_systems:
                args.append('--%s' % (ops,))
            for nc in netconfigs:
                args.append('--%s' % (nc,))

        try:
            stoneridge.run_process(*args)
        except subprocess.CalledProcessError as e:
            # Either we will retry this later, at the cloner's request, or the
            # error has already been logged by run_process and there's no
            # recovery we can do.
            return

        # In order to have the points for each OS/netconfig match up with each
        # other for a particular test run (good for graphing), we set the
        # timestamp once we know we're going to actually run the test (which is
        # right now, after we've cloned the builds).
        # We also sleep for one second, so we don't accidentally have 2
        # different runs show up at the same time as each other on the graphs.
        # Sure, it's unlikely, but sleeping for a second won't kill us, and
        # better safe than sorry!
        tstamp = int(time.time())
        time.sleep(1)

        for nc in netconfigs:
            queue = self.queues.get(nc, None)
            if queue is None:
                logging.warning('Got request for invalid netconfig %s' % (nc,))
                continue

            queue.enqueue(operating_systems=operating_systems, srid=srid,
                    tstamp=tstamp)

def daemon():
    master = StoneRidgeMaster(stoneridge.INCOMING_QUEUE)
    master.run()

@stoneridge.main
def main():
    parser = stoneridge.DaemonArgumentParser()
    args = parser.parse_args()

    parser.start_daemon(daemon)
<commit_msg>Fix lint warnings in master<commit_after>#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at http://mozilla.org/MPL/2.0/.

import logging
import os
import subprocess
import time
import uuid

import stoneridge

class StoneRidgeMaster(stoneridge.QueueListener):
    def setup(self):
        self.queues = {
            'broadband': stoneridge.QueueWriter(
                stoneridge.NETCONFIG_QUEUES['broadband']),
            'umts': stoneridge.QueueWriter(
                stoneridge.NETCONFIG_QUEUES['umts']),
            'gsm': stoneridge.QueueWriter(
                stoneridge.NETCONFIG_QUEUES['gsm'])
        }
        self.logdir = stoneridge.get_config('stoneridge', 'logs')
        self.config = stoneridge.get_config_file()

    def handle(self, nightly, ldap, sha, netconfigs, operating_systems,
               srid=None, attempt=1):
        if srid is None:
            srid = str(uuid.uuid4())

        logging.debug('Got request')
        logging.debug('Nightly: %s' % (nightly,))
        logging.debug('LDAP: %s' % (ldap,))
        logging.debug('SHA: %s' % (sha,))
        logging.debug('Netconfigs: %s' % (' '.join(netconfigs),))
        logging.debug('Operating systems: %s' % (' '.join(operating_systems),))
        logging.debug('Attempt: %s' % (attempt,))
        logging.debug('SRID: %s' % (srid,))

        logfile = 'cloner_%s.log' % (srid,)
        cloner_log = os.path.join(self.logdir, logfile)
        args = ['srcloner.py', '--config', self.config, '--srid', srid,
                '--log', cloner_log, '--attempt', attempt]
        if nightly:
            args.append('--nightly')

            # Make sure the list of netconfigs and operating systems is right
            netconfigs = stoneridge.NETCONFIGS
            operating_systems = stoneridge.OPERATING_SYSTEMS
        else:
            if not ldap or not sha:
                logging.error('Missing ldap/sha for non-nightly build')
                return

            args.extend(['--ldap', ldap])
            args.extend(['--sha', sha])
            for ops in operating_systems:
                args.append('--%s' % (ops,))
            for nc in netconfigs:
                args.append('--%s' % (nc,))

        try:
            stoneridge.run_process(*args)
        except subprocess.CalledProcessError:
            # Either we will retry this later, at the cloner's request, or the
            # error has already been logged by run_process and there's no
            # recovery we can do.
            return

        # In order to have the points for each OS/netconfig match up with each
        # other for a particular test run (good for graphing), we set the
        # timestamp once we know we're going to actually run the test (which is
        # right now, after we've cloned the builds).
        # We also sleep for one second, so we don't accidentally have 2
        # different runs show up at the same time as each other on the graphs.
        # Sure, it's unlikely, but sleeping for a second won't kill us, and
        # better safe than sorry!
        tstamp = int(time.time())
        time.sleep(1)

        for nc in netconfigs:
            queue = self.queues.get(nc, None)
            if queue is None:
                logging.warning('Got request for invalid netconfig %s' % (nc,))
                continue

            queue.enqueue(operating_systems=operating_systems, srid=srid,
                          tstamp=tstamp)

def daemon():
    master = StoneRidgeMaster(stoneridge.INCOMING_QUEUE)
    master.run()

@stoneridge.main
def main():
    parser = stoneridge.DaemonArgumentParser()
    parser.parse_args()

    parser.start_daemon(daemon)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import datetime
from inspiderweb.log import logger
from inspiderweb.database import Database
from inspiderweb.dotgraph import DotGraph
import sys
import argparse
from argparse import RawTextHelpFormatter

""" Main file of inspiderweb: Tool to analyze paper reference networks.
Currently hosted at: https://github.com/klieret/inspiderweb

Run this file via `python3 inspiderweb.py --help` for instructions on
usage. """

description = r"""
    INSPIDERWEB
 `.,-'\_____/`-.,'     Tool to analyze networks papers referencing and citing each
  /`..'\ _ /`.,'\      other. It acts as a web-crawler, extracting information from
 /  /`.,' `.,'\  \     inspirehep, then uses the dot languageto describe the
/__/__/     \__\__\__  network. The result can then be plotted by the graphviz
\  \  \     /  /  /    Package and similar programs.
 \  \,'`._,'`./  /     More info on the github page
  \,'`./___\,'`./      https://github.com/klieret/inspiderweb
 ,'`-./_____\,-'`.
     /       \
"""

# Use the RawTextHelpFormatter in order to allow linebreaks in the description
parser = argparse.ArgumentParser(description=description,
                                 prog="inspiderweb.py",
                                 formatter_class=RawTextHelpFormatter,
                                 add_help=False)

setup_options = parser.add_argument_group('Setup/Configure Options',
                                          'Supply in/output paths...')
action_options = parser.add_argument_group('Action Options',
                                           'What do you want to do?')
misc_options = parser.add_argument_group('Misc', 'Misc Options')

# todo: update tutorial
setup_options.add_argument("-d", "--database", required=True,
                           help="Pickle database (db) file. Multiple db files "
                                "are supported. In this case the first one "
                                "will be used to save the resulting merged db",
                           type=str, nargs="+")

setup_options.add_argument("-o", "--output", required=False,
                           help="Output dot file.",
                           type=str)

# todo: update tutorial
setup_options.add_argument("-s", "--seeds", required=False,
                           help="Input seed file. Multiple seed files are "
                                "supported.",
                           type=str, nargs="+", default=[])

action_options.add_argument("-p", "--plot", required=False,
                            action="store_true",
                            help="Generate dot output (i.e. plot).",
                            default=False)
# fixme: update tutorial
action_options.add_argument("-u", "--updateseeds", required=False,
                            help="Get specified information for the seeds "
                                 "records Multiple arguments are supported. "
                                 "See 'usage' for the allowed arguments. ",
                            type=str, default="", nargs="+",
                            choices=["refs", "cites", "bib"])
# fixme: update tutorial
action_options.add_argument("-t", "--updatedb", required=False,
                            help="Get specified information for the records in"
                                 " the database. "
                                 "Multiple arguments are supported. "
                                 "See 'usage' for the allowed arguments. ",
                            type=str, default="", nargs="+",
                            choices=["refs", "cites", "bib"])

misc_options.add_argument("-h", "--help",
                          help="Print this help message.", action="help")
misc_options.add_argument("--rank", required=False,
                          help="Rank by [year]", default="",
                          type=str,
                          choices=["year"])
misc_options.add_argument("--maxseeds", required=False, type=int,
                          help="Maximum number of seeds (for testing "
                               "purposes).",
                          default=0)
misc_options.add_argument("--forceupdate", action="store_true",
                          help="For all information that we get from the "
                               "database: Force redownload")

args = parser.parse_args()

if args.plot and not args.seeds:
    logger.error("We need seeds to plot. Exiting.")
    sys.exit(10)
if args.plot and not args.output:
    logger.error("We need output filename to plot. Exiting.")
    sys.exit(20)
if args.updateseeds and not args.seeds:
    logger.error("We need seeds to update seeds. Exiting.")
    sys.exit(30)

# todo: Use API instead of parsing. But I don't see where there is a clear ...
# ...link to cited documents....
# todo: add docstrings
# todo: maybe use a proper format to save the record data or at least allow ...
# .... to export into such
# todo: add clusters
# todo: extract more infomration; add title as tooltip
# todo: control verbosity

db = Database(args.database[0])
db.load()

for path in args.database[1:]:
    db.load(path)

db.statistics()

seeds = []
for seedfile in args.seeds:
    with open(seedfile, "r") as seedfile_stream:
        for i, line in enumerate(seedfile_stream):
            if (i + 1) == args.maxseeds:
                # if args.maxseeds == 0 (default), this will never run
                break
            line = line.replace('\n', "")
            line = line.strip()
            if not line:
                continue
            seeds.append(line)
    logger.info("Read {} seeds from file(s) {}.".format(len(seeds),
                                                        ', '.join(args.seeds)))

db.autocomplete_records(args.updateseeds, force=args.forceupdate, mids=seeds)
db.autocomplete_records(args.updatedb, force=args.forceupdate)

if args.plot:

    dg = DotGraph(db)

    # ALWAYS END EVERYTHING WITH A SEMICOLON
    # EXCEPT THINGS IN SQUARE BRACKETS: USE COMMA
    # todo: move that to config or something
    graph_style = \
        "graph [label=\"inspiderweb {date} {time}\", fontsize=40];".format(
            date=str(datetime.date.today()),
            time=str(datetime.datetime.now().time()))
    node_style = "node[fontsize=20, fontcolor=black, fontname=Arial, " \
                 "style=filled, color=green];"
    # size = 'ratio="0.3";'#''size="14,10";'
    # size = 'overlap=prism; overlap_scaling=0.01; ratio=0.7'
    size = ";"
    style = '\n'.join([graph_style, node_style, size])
    dg._style = style
    # "//ratio=\"1:1\";\n"
    #      "//ratio=\"fill\";\n"
    #      "//size=\"11.692,8.267\"; \n"
    #      "//size=\"16.53,11.69\"; //a3\n"
    #      "//size=\"33.06,11.69\"\n"

    # todo: Add otion to customize this
    def valid_connection(source, target):
        sr = db.get_record(source)
        tr = db.get_record(target)
        return sr.bibkey and tr.bibkey and source in seeds and target in seeds

    # fixme: use getter to get all records
    for mid, record in db._records.items():
        for reference_mid in record.references:
            if not valid_connection(record.mid, reference_mid):
                continue
            dg.add_connection(record.mid, reference_mid)
        for citation_mid in record.citations:
            if not valid_connection(record.mid, citation_mid):
                continue
            dg.add_connection(citation_mid, record.mid)

    dg.generate_dot_str(rank=args.rank)
    dg.write_to_file(args.output)

db.save()
<commit_msg>proper formatting of help message<commit_after>#!/usr/bin/env python3

import datetime
from inspiderweb.log import logger
from inspiderweb.database import Database
from inspiderweb.dotgraph import DotGraph
import sys
import argparse
from argparse import RawDescriptionHelpFormatter

""" Main file of inspiderweb: Tool to analyze paper reference networks.
Currently hosted at: https://github.com/klieret/inspiderweb

Run this file via `python3 inspiderweb.py --help` for instructions on
usage. """

description = r"""
    INSPIDERWEB
 `.,-'\_____/`-.,'     Tool to analyze networks papers referencing and citing each
  /`..'\ _ /`.,'\      other. It acts as a web-crawler, extracting information from
 /  /`.,' `.,'\  \     inspirehep, then uses the dot languageto describe the
/__/__/     \__\__\__  network. The result can then be plotted by the graphviz
\  \  \     /  /  /    Package and similar programs.
 \  \,'`._,'`./  /     More info on the github page
  \,'`./___\,'`./      https://github.com/klieret/inspiderweb
 ,'`-./_____\,-'`.
     /       \
"""

# Use the RawTextHelpFormatter in order to allow linebreaks in the description
parser = argparse.ArgumentParser(description=description,
                                 prog="inspiderweb.py",
                                 formatter_class=RawDescriptionHelpFormatter,
                                 add_help=False)

setup_options = parser.add_argument_group('Setup/Configure Options',
                                          'Supply in/output paths...')
action_options = parser.add_argument_group('Action Options',
                                           'What do you want to do?')
misc_options = parser.add_argument_group('Misc', 'Misc Options')

# todo: update tutorial
setup_options.add_argument("-d", "--database", required=True,
                           help="Pickle database (db) file. Multiple db files "
                                "are supported. In this case the first one "
                                "will be used to save the resulting merged db",
                           type=str, nargs="+")

setup_options.add_argument("-o", "--output", required=False,
                           help="Output dot file.",
                           type=str)

# todo: update tutorial
setup_options.add_argument("-s", "--seeds", required=False,
                           help="Input seed file. Multiple seed files are "
                                "supported.",
                           type=str, nargs="+", default=[])

action_options.add_argument("-p", "--plot", required=False,
                            action="store_true",
                            help="Generate dot output (i.e. plot).",
                            default=False)
# fixme: update tutorial
action_options.add_argument("-u", "--updateseeds", required=False,
                            help="Get specified information for the seeds. "
                                 "records Multiple arguments are supported. ",
                            type=str, default="", nargs="+",
                            choices=["refs", "cites", "bib"])
# fixme: update tutorial
action_options.add_argument("-t", "--updatedb", required=False,
                            help="Get specified information for the records in"
                                 " the database. "
                                 "Multiple arguments are supported. ",
                            type=str, default="", nargs="+",
                            choices=["refs", "cites", "bib"])

misc_options.add_argument("-h", "--help",
                          help="Print this help message.", action="help")
misc_options.add_argument("--rank", required=False,
                          help="Rank by [year]", default="",
                          type=str,
                          choices=["year"])
misc_options.add_argument("--maxseeds", required=False, type=int,
                          help="Maximum number of seeds (for testing "
                               "purposes).",
                          default=0)
misc_options.add_argument("--forceupdate", action="store_true",
                          help="For all information that we get from the "
                               "database: Force redownload")

args = parser.parse_args()

if args.plot and not args.seeds:
    logger.error("We need seeds to plot. Exiting.")
    sys.exit(10)
if args.plot and not args.output:
    logger.error("We need output filename to plot. Exiting.")
    sys.exit(20)
if args.updateseeds and not args.seeds:
    logger.error("We need seeds to update seeds. Exiting.")
    sys.exit(30)

# todo: Use API instead of parsing. But I don't see where there is a clear ...
# ...link to cited documents....
# todo: add docstrings
# todo: maybe use a proper format to save the record data or at least allow ...
# .... to export into such
# todo: add clusters
# todo: extract more infomration; add title as tooltip
# todo: control verbosity

db = Database(args.database[0])
db.load()

for path in args.database[1:]:
    db.load(path)

db.statistics()

seeds = []
for seedfile in args.seeds:
    with open(seedfile, "r") as seedfile_stream:
        for i, line in enumerate(seedfile_stream):
            if (i + 1) == args.maxseeds:
                # if args.maxseeds == 0 (default), this will never run
                break
            line = line.replace('\n', "")
            line = line.strip()
            if not line:
                continue
            seeds.append(line)
    logger.info("Read {} seeds from file(s) {}.".format(len(seeds),
                                                        ', '.join(args.seeds)))

db.autocomplete_records(args.updateseeds, force=args.forceupdate, mids=seeds)
db.autocomplete_records(args.updatedb, force=args.forceupdate)

if args.plot:

    dg = DotGraph(db)

    # ALWAYS END EVERYTHING WITH A SEMICOLON
    # EXCEPT THINGS IN SQUARE BRACKETS: USE COMMA
    # todo: move that to config or something
    graph_style = \
        "graph [label=\"inspiderweb {date} {time}\", fontsize=40];".format(
            date=str(datetime.date.today()),
            time=str(datetime.datetime.now().time()))
    node_style = "node[fontsize=20, fontcolor=black, fontname=Arial, " \
                 "style=filled, color=green];"
    # size = 'ratio="0.3";'#''size="14,10";'
    # size = 'overlap=prism; overlap_scaling=0.01; ratio=0.7'
    size = ";"
    style = '\n'.join([graph_style, node_style, size])
    dg._style = style
    # "//ratio=\"1:1\";\n"
    #      "//ratio=\"fill\";\n"
    #      "//size=\"11.692,8.267\"; \n"
    #      "//size=\"16.53,11.69\"; //a3\n"
    #      "//size=\"33.06,11.69\"\n"

    # todo: Add otion to customize this
    def valid_connection(source, target):
        sr = db.get_record(source)
        tr = db.get_record(target)
        return sr.bibkey and tr.bibkey and source in seeds and target in seeds

    # fixme: use getter to get all records
    for mid, record in db._records.items():
        for reference_mid in record.references:
            if not valid_connection(record.mid, reference_mid):
                continue
            dg.add_connection(record.mid, reference_mid)
        for citation_mid in record.citations:
            if not valid_connection(record.mid, citation_mid):
                continue
            dg.add_connection(citation_mid, record.mid)

    dg.generate_dot_str(rank=args.rank)
    dg.write_to_file(args.output)

db.save()
<|endoftext|>


<commit_before><commit_msg>opt: Adding vectorize_damping function.<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

__version__ = '0.9.2'<commit_msg>version add<commit_after>#!/usr/bin/env python

__version__ = '0.9.3'<|endoftext|>


<commit_before>"""
Utilities for making finder charts and overlay images for ALMA proposing
"""
import string

import numpy as np

from astropy import wcs
from astropy import log
from astropy import units as u
from astropy.io import fits

from astroquery.skyview import SkyView
from astroquery.alma import Alma

def pyregion_subset(region, data, mywcs):
    """
    Return a subset of an image (`data`) given a region.

    Parameters
    ----------
    region : `pyregion.parser_helper.Shape`
        A Shape from a pyregion-parsed region file
    data : np.ndarray
        An array with shape described by WCS
    mywcs : `astropy.wcs.WCS`
        A world coordinate system describing the data
    """
    import pyregion

    shapelist = pyregion.ShapeList([region])
    if shapelist[0].coord_format not in ('physical','image'):
        # Requires astropy >0.4...
        # pixel_regions = shapelist.as_imagecoord(self.wcs.celestial.to_header())
        # convert the regions to image (pixel) coordinates
        celhdr = mywcs.sub([wcs.WCSSUB_CELESTIAL]).to_header()
        pixel_regions = shapelist.as_imagecoord(celhdr)
    else:
        # For this to work, we'd need to change the reference pixel after cropping.
        # Alternatively, we can just make the full-sized mask... todo....
        raise NotImplementedError("Can't use non-celestial coordinates with regions.")
        pixel_regions = shapelist

    # This is a hack to use mpl to determine the outer bounds of the regions
    # (but it's a legit hack - pyregion needs a major internal refactor
    # before we can approach this any other way, I think -AG)
    mpl_objs = pixel_regions.get_mpl_patches_texts()[0]

    # Find the minimal enclosing box containing all of the regions
    # (this will speed up the mask creation below)
    extent = mpl_objs[0].get_extents()
    xlo, ylo = extent.min
    xhi, yhi = extent.max
    all_extents = [obj.get_extents() for obj in mpl_objs]
    for ext in all_extents:
        xlo = xlo if xlo < ext.min[0] else ext.min[0]
        ylo = ylo if ylo < ext.min[1] else ext.min[1]
        xhi = xhi if xhi > ext.max[0] else ext.max[0]
        yhi = yhi if yhi > ext.max[1] else ext.max[1]

    log.debug("Region boundaries: ")
    log.debug("xlo={xlo}, ylo={ylo}, xhi={xhi}, yhi={yhi}".format(xlo=xlo,
                                                                  ylo=ylo,
                                                                  xhi=xhi,
                                                                  yhi=yhi))

    subwcs = mywcs[ylo:yhi, xlo:xhi]
    subhdr = subwcs.sub([wcs.WCSSUB_CELESTIAL]).to_header()
    subdata = data[ylo:yhi, xlo:xhi]

    mask = shapelist.get_mask(header=subhdr,
                              shape=subdata.shape)
    log.debug("Shapes: data={0}, subdata={2}, mask={1}".format(data.shape, mask.shape, subdata.shape))
    return (xlo,xhi,ylo,yhi),mask

def parse_frequency_support(frequency_support_str):
    """
    Given a "Frequency Support" string from ALMA queries, parse it into a set
    of frequency ranges

    Example input:

        '[86.26..88.14GHz,976.56kHz, XX YY] U [88.15..90.03GHz,976.56kHz, XX YY] U [98.19..100.07GHz,976.56kHz, XX YY] U [100.15..102.03GHz,976.56kHz, XX YY]'
    """
    supports = frequency_support_str.split("U")
    freq_ranges = [(float(sup.strip('[] ').split("..")[0]),
                    float(sup.strip('[] ').split("..")[1].split(',')[0].strip(string.ascii_letters)))
                   *u.Unit(sup.strip('[] ').split("..")[1].split(',')[0].strip(string.punctuation+string.digits))
                   for sup in supports]
    return u.Quantity(freq_ranges)

def approximate_primary_beam_sizes(frequency_support_str):
    """
    Given a frequency support string, return the approximate 12m array beam
    size using 1.22 lambda / D
    """
    freq_ranges = parse_frequency_support(frequency_support_str)
    beam_sizes = [(1.22*fr.mean().to(u.m, u.spectral())/(12*u.m)).to(u.arcsec,
                                                                     u.dimensionless_angles())
                  for fr in freq_ranges]
    return u.Quantity(beam_sizes)

def make_finder_chart(target, radius, save_prefix, service=SkyView.get_images,
                      service_kwargs={'survey':['2MASS-K'], 'pixels':500},
                      alma_kwargs={'public':False, 'science':False},
                      private_band_colors=('red','darkred','orange','brown','maroon'),
                      public_band_colors=('blue','cyan','green','turquoise','teal'),
                      integration_time_contour_levels=np.logspace(0,5,base=2, num=6),
                     ):
    """
    Create a "finder chart" showing where ALMA has pointed in various bands,
    including different color coding for public/private data and each band.

    Contours are set at various integration times.

    Parameters
    ----------
    target : `astropy.coordinates` or str
        A legitimate target name
    radius : `astropy.units.Quantity`
        A degree-equivalent radius
    save_prefix : str
        The prefix for the output files.  Both .reg and .png files will be written.
        The .reg files will have the band numbers and public/private appended,
        while the .png file will be named prefix_almafinderchart.png
    service : function
        The `get_images` function of an astroquery service, e.g. SkyView.
    service_kwargs : dict
        The keyword arguments to pass to the specified service.  For example,
        for SkyView, you can give it the survey ID (e.g., 2MASS-K) and the
        number of pixels in the resulting image.  See the documentation for the
        individual services for more details.
    alma_kwargs : dict
        Keywords to pass to the ALMA archive when querying.
    private_band_colors / public_band_colors : tuple
        A tuple or list of colors to be associated with private/public observations
        in the various bands
    integration_time_contour_levels : list or np.array
        The levels at which to draw contours in units of seconds.  Default is
        log-spaced (2^n) seconds: [  1.,   2.,   4.,   8.,  16.,  32.])
    """
    import aplpy

    import pyregion
    from pyregion.parser_helper import Shape

    log.info("Querying {0} for images".format(service))
    images = service(target, radius=radius, **service_kwargs)

    log.info("Querying ALMA around {0}".format(target))
    catalog = Alma.query_region(coordinate=target, radius=radius,
                                **alma_kwargs)

    primary_beam_radii = [approximate_primary_beam_sizes(row['Frequency support'])
                          for row in catalog]

    bands = np.unique(catalog['Band'])
    log.info("The bands used include: {0}".format(bands))
    band_colors_priv = dict(zip(bands, private_band_colors))
    band_colors_pub = dict(zip(bands, public_band_colors))

    private_circle_parameters = {band: [(row['RA'],row['Dec'],np.mean(rad).to(u.deg).value)
                                 for row,rad in zip(catalog, primary_beam_radii)
                                 if row['Release date']!='' and row['Band']==band]
                                 for band in bands}
    public_circle_parameters = {band: [(row['RA'],row['Dec'],np.mean(rad).to(u.deg).value)
                                 for row,rad in zip(catalog, primary_beam_radii)
                                 if row['Release date']=='' and row['Band']==band]
                                 for band in bands}

    unique_private_circle_parameters = {band:
                                        np.array(list(set(private_circle_parameters[band])))
                                        for band in bands}
    unique_public_circle_parameters = {band:
                                       np.array(list(set(public_circle_parameters[band])))
                                       for band in bands}

    for band in bands:
        log.info( "BAND {0}".format(band) )
        privrows = sum((catalog['Band']==band) & (catalog['Release date'] != ''))
        pubrows  = sum((catalog['Band']==band) & (catalog['Release date'] == ''))
        log.info("PUBLIC:  Number of rows: {0}.  Unique pointings: "
                 "{1}".format(pubrows,
                 len(unique_public_circle_parameters[band])))
        log.info( "PRIVATE: Number of rows: {0}.  Unique pointings: "
                 "{1}".format(privrows,
                 len(unique_private_circle_parameters[band])))

    prv_regions = {band: pyregion.ShapeList([Shape('circle',[x,y,r]) for x,y,r
                                             in
                                             private_circle_parameters[band]])
                   for band in bands}
    pub_regions = {band: pyregion.ShapeList([Shape('circle',[x,y,r]) for x,y,r
                                             in
                                             public_circle_parameters[band]])
                   for band in bands}
    for band in bands:
        circle_pars = np.vstack([x for x in (private_circle_parameters[band],
                                        public_circle_parameters[band]) if any(x)])
        for r,(x,y,c) in zip(prv_regions[band]+pub_regions[band],
                             circle_pars):
            r.coord_format = 'fk5'
            r.coord_list = [x,y,c]
            r.attr = ([], {'color': 'green',  'dash': '0 ',  'dashlist': '8 3',
                           'delete': '1 ',  'edit': '1 ', 'fixed': '0 ',
                           'font': '"helvetica 10 normal roman"',  'highlite':
                           '1 ', 'include': '1 ',  'move': '1 ',  'select': '1',
                           'source': '1',  'text': '', 'width': '1 '})

        if prv_regions[band]:
            prv_regions[band].write('{0}_band{1}_private.reg'.format(save_prefix, band))
        if pub_regions[band]:
            pub_regions[band].write('{0}_band{1}_public.reg'.format(save_prefix, band))

    prv_mask = {band: fits.PrimaryHDU(prv_regions[band].get_mask(images[0][0]).astype('int'),
                               header=images[0][0].header) for band in bands
                if prv_regions[band]}
    pub_mask = {band: fits.PrimaryHDU(pub_regions[band].get_mask(images[0][0]).astype('int'),
                               header=images[0][0].header) for band in bands
                if pub_regions[band]}

    hit_mask_public = {band: np.zeros_like(images[0][0].data) for band in pub_mask}
    hit_mask_private = {band: np.zeros_like(images[0][0].data) for band in prv_mask}
    mywcs = wcs.WCS(images[0][0].header)

    for band in bands:
        log.debug('Band: {0}'.format(band))
        for row,rad in zip(catalog, primary_beam_radii):
            shape = Shape('circle', (row['RA'], row['Dec'],np.mean(rad).to(u.deg).value))
            shape.coord_format = 'fk5'
            shape.coord_list = (row['RA'], row['Dec'],np.mean(rad).to(u.deg).value)
            shape.attr = ([], {'color': 'green',  'dash': '0 ',  'dashlist': '8 3 ',
                               'delete': '1 ',  'edit': '1 ', 'fixed': '0 ',
                               'font': '"helvetica 10 normal roman"',
                               'highlite': '1 ', 'include': '1 ',  'move': '1 ',
                               'select': '1 ',  'source': '1',  'text': '',
                               'width': '1 '})
            log.debug('{1} {2}: {0}'.format(shape, row['Release date'], row['Band']))
            if row['Release date']!='' and row['Band']==band and band in prv_mask:
                (xlo,xhi,ylo,yhi),mask = pyregion_subset(shape,
                                                         hit_mask_private[band],
                                                         mywcs)
                log.debug("{0},{1},{2},{3}: {4}".format(xlo,xhi,ylo,yhi,mask.sum()))
                hit_mask_private[band][ylo:yhi,xlo:xhi] += row['Integration']*mask
            if row['Release date']=='' and row['Band']==band and band in pub_mask:
                (xlo,xhi,ylo,yhi),mask = pyregion_subset(shape,
                                                         hit_mask_public[band],
                                                         mywcs)
                log.debug("{0},{1},{2},{3}: {4}".format(xlo,xhi,ylo,yhi,mask.sum()))
                hit_mask_public[band][ylo:yhi,xlo:xhi] += row['Integration']*mask

    fig = aplpy.FITSFigure(images[0])
    fig.show_grayscale(stretch='arcsinh')
    for band in bands:
        if band in pub_mask:
            fig.show_contour(fits.PrimaryHDU(data=hit_mask_public[band],
                                             header=images[0][0].header),
                             levels=integration_time_contour_levels,
                             colors=[band_colors_pub[band]]*6)
        if band in prv_mask:
            fig.show_contour(fits.PrimaryHDU(data=hit_mask_private[band],
                                             header=images[0][0].header),
                             levels=integration_time_contour_levels,
                             colors=[band_colors_priv[band]]*6)

    fig.save('{0}_almafinderchart.png'.format(save_prefix))

    return images, catalog, hit_mask_public, hit_mask_private
<commit_msg>python3 can't handle strings<commit_after>"""
Utilities for making finder charts and overlay images for ALMA proposing
"""
import string

import numpy as np

from astropy import wcs
from astropy import log
from astropy import units as u
from astropy.io import fits

from astroquery.skyview import SkyView
from astroquery.alma import Alma

def pyregion_subset(region, data, mywcs):
    """
    Return a subset of an image (`data`) given a region.

    Parameters
    ----------
    region : `pyregion.parser_helper.Shape`
        A Shape from a pyregion-parsed region file
    data : np.ndarray
        An array with shape described by WCS
    mywcs : `astropy.wcs.WCS`
        A world coordinate system describing the data
    """
    import pyregion

    shapelist = pyregion.ShapeList([region])
    if shapelist[0].coord_format not in ('physical','image'):
        # Requires astropy >0.4...
        # pixel_regions = shapelist.as_imagecoord(self.wcs.celestial.to_header())
        # convert the regions to image (pixel) coordinates
        celhdr = mywcs.sub([wcs.WCSSUB_CELESTIAL]).to_header()
        pixel_regions = shapelist.as_imagecoord(celhdr)
    else:
        # For this to work, we'd need to change the reference pixel after cropping.
        # Alternatively, we can just make the full-sized mask... todo....
        raise NotImplementedError("Can't use non-celestial coordinates with regions.")
        pixel_regions = shapelist

    # This is a hack to use mpl to determine the outer bounds of the regions
    # (but it's a legit hack - pyregion needs a major internal refactor
    # before we can approach this any other way, I think -AG)
    mpl_objs = pixel_regions.get_mpl_patches_texts()[0]

    # Find the minimal enclosing box containing all of the regions
    # (this will speed up the mask creation below)
    extent = mpl_objs[0].get_extents()
    xlo, ylo = extent.min
    xhi, yhi = extent.max
    all_extents = [obj.get_extents() for obj in mpl_objs]
    for ext in all_extents:
        xlo = xlo if xlo < ext.min[0] else ext.min[0]
        ylo = ylo if ylo < ext.min[1] else ext.min[1]
        xhi = xhi if xhi > ext.max[0] else ext.max[0]
        yhi = yhi if yhi > ext.max[1] else ext.max[1]

    log.debug("Region boundaries: ")
    log.debug("xlo={xlo}, ylo={ylo}, xhi={xhi}, yhi={yhi}".format(xlo=xlo,
                                                                  ylo=ylo,
                                                                  xhi=xhi,
                                                                  yhi=yhi))

    subwcs = mywcs[ylo:yhi, xlo:xhi]
    subhdr = subwcs.sub([wcs.WCSSUB_CELESTIAL]).to_header()
    subdata = data[ylo:yhi, xlo:xhi]

    mask = shapelist.get_mask(header=subhdr,
                              shape=subdata.shape)
    log.debug("Shapes: data={0}, subdata={2}, mask={1}".format(data.shape, mask.shape, subdata.shape))
    return (xlo,xhi,ylo,yhi),mask

def parse_frequency_support(frequency_support_str):
    """
    Given a "Frequency Support" string from ALMA queries, parse it into a set
    of frequency ranges

    Example input:

        '[86.26..88.14GHz,976.56kHz, XX YY] U [88.15..90.03GHz,976.56kHz, XX YY] U [98.19..100.07GHz,976.56kHz, XX YY] U [100.15..102.03GHz,976.56kHz, XX YY]'
    """
    try:
        supports = frequency_support_str.split("U")
    except TypeError:
        # python 3 problem
        frequency_support_str = frequency_support_str.tostring().decode('utf-8')
        supports = frequency_support_str.split("U")

    freq_ranges = [(float(sup.strip('[] ').split("..")[0]),
                    float(sup.strip('[] ').split("..")[1].split(',')[0].strip(string.ascii_letters)))
                   *u.Unit(sup.strip('[] ').split("..")[1].split(',')[0].strip(string.punctuation+string.digits))
                   for sup in supports]
    return u.Quantity(freq_ranges)

def approximate_primary_beam_sizes(frequency_support_str):
    """
    Given a frequency support string, return the approximate 12m array beam
    size using 1.22 lambda / D
    """
    freq_ranges = parse_frequency_support(frequency_support_str)
    beam_sizes = [(1.22*fr.mean().to(u.m, u.spectral())/(12*u.m)).to(u.arcsec,
                                                                     u.dimensionless_angles())
                  for fr in freq_ranges]
    return u.Quantity(beam_sizes)

def make_finder_chart(target, radius, save_prefix, service=SkyView.get_images,
                      service_kwargs={'survey':['2MASS-K'], 'pixels':500},
                      alma_kwargs={'public':False, 'science':False},
                      private_band_colors=('red','darkred','orange','brown','maroon'),
                      public_band_colors=('blue','cyan','green','turquoise','teal'),
                      integration_time_contour_levels=np.logspace(0,5,base=2, num=6),
                     ):
    """
    Create a "finder chart" showing where ALMA has pointed in various bands,
    including different color coding for public/private data and each band.

    Contours are set at various integration times.

    Parameters
    ----------
    target : `astropy.coordinates` or str
        A legitimate target name
    radius : `astropy.units.Quantity`
        A degree-equivalent radius
    save_prefix : str
        The prefix for the output files.  Both .reg and .png files will be written.
        The .reg files will have the band numbers and public/private appended,
        while the .png file will be named prefix_almafinderchart.png
    service : function
        The `get_images` function of an astroquery service, e.g. SkyView.
    service_kwargs : dict
        The keyword arguments to pass to the specified service.  For example,
        for SkyView, you can give it the survey ID (e.g., 2MASS-K) and the
        number of pixels in the resulting image.  See the documentation for the
        individual services for more details.
    alma_kwargs : dict
        Keywords to pass to the ALMA archive when querying.
    private_band_colors / public_band_colors : tuple
        A tuple or list of colors to be associated with private/public observations
        in the various bands
    integration_time_contour_levels : list or np.array
        The levels at which to draw contours in units of seconds.  Default is
        log-spaced (2^n) seconds: [  1.,   2.,   4.,   8.,  16.,  32.])
    """
    import aplpy

    import pyregion
    from pyregion.parser_helper import Shape

    log.info("Querying {0} for images".format(service))
    images = service(target, radius=radius, **service_kwargs)

    log.info("Querying ALMA around {0}".format(target))
    catalog = Alma.query_region(coordinate=target, radius=radius,
                                **alma_kwargs)

    primary_beam_radii = [approximate_primary_beam_sizes(row['Frequency support'])
                          for row in catalog]

    bands = np.unique(catalog['Band'])
    log.info("The bands used include: {0}".format(bands))
    band_colors_priv = dict(zip(bands, private_band_colors))
    band_colors_pub = dict(zip(bands, public_band_colors))

    private_circle_parameters = {band: [(row['RA'],row['Dec'],np.mean(rad).to(u.deg).value)
                                 for row,rad in zip(catalog, primary_beam_radii)
                                 if row['Release date']!='' and row['Band']==band]
                                 for band in bands}
    public_circle_parameters = {band: [(row['RA'],row['Dec'],np.mean(rad).to(u.deg).value)
                                 for row,rad in zip(catalog, primary_beam_radii)
                                 if row['Release date']=='' and row['Band']==band]
                                 for band in bands}

    unique_private_circle_parameters = {band:
                                        np.array(list(set(private_circle_parameters[band])))
                                        for band in bands}
    unique_public_circle_parameters = {band:
                                       np.array(list(set(public_circle_parameters[band])))
                                       for band in bands}

    for band in bands:
        log.info( "BAND {0}".format(band) )
        privrows = sum((catalog['Band']==band) & (catalog['Release date'] != ''))
        pubrows  = sum((catalog['Band']==band) & (catalog['Release date'] == ''))
        log.info("PUBLIC:  Number of rows: {0}.  Unique pointings: "
                 "{1}".format(pubrows,
                 len(unique_public_circle_parameters[band])))
        log.info( "PRIVATE: Number of rows: {0}.  Unique pointings: "
                 "{1}".format(privrows,
                 len(unique_private_circle_parameters[band])))

    prv_regions = {band: pyregion.ShapeList([Shape('circle',[x,y,r]) for x,y,r
                                             in
                                             private_circle_parameters[band]])
                   for band in bands}
    pub_regions = {band: pyregion.ShapeList([Shape('circle',[x,y,r]) for x,y,r
                                             in
                                             public_circle_parameters[band]])
                   for band in bands}
    for band in bands:
        circle_pars = np.vstack([x for x in (private_circle_parameters[band],
                                        public_circle_parameters[band]) if any(x)])
        for r,(x,y,c) in zip(prv_regions[band]+pub_regions[band],
                             circle_pars):
            r.coord_format = 'fk5'
            r.coord_list = [x,y,c]
            r.attr = ([], {'color': 'green',  'dash': '0 ',  'dashlist': '8 3',
                           'delete': '1 ',  'edit': '1 ', 'fixed': '0 ',
                           'font': '"helvetica 10 normal roman"',  'highlite':
                           '1 ', 'include': '1 ',  'move': '1 ',  'select': '1',
                           'source': '1',  'text': '', 'width': '1 '})

        if prv_regions[band]:
            prv_regions[band].write('{0}_band{1}_private.reg'.format(save_prefix, band))
        if pub_regions[band]:
            pub_regions[band].write('{0}_band{1}_public.reg'.format(save_prefix, band))

    prv_mask = {band: fits.PrimaryHDU(prv_regions[band].get_mask(images[0][0]).astype('int'),
                               header=images[0][0].header) for band in bands
                if prv_regions[band]}
    pub_mask = {band: fits.PrimaryHDU(pub_regions[band].get_mask(images[0][0]).astype('int'),
                               header=images[0][0].header) for band in bands
                if pub_regions[band]}

    hit_mask_public = {band: np.zeros_like(images[0][0].data) for band in pub_mask}
    hit_mask_private = {band: np.zeros_like(images[0][0].data) for band in prv_mask}
    mywcs = wcs.WCS(images[0][0].header)

    for band in bands:
        log.debug('Band: {0}'.format(band))
        for row,rad in zip(catalog, primary_beam_radii):
            shape = Shape('circle', (row['RA'], row['Dec'],np.mean(rad).to(u.deg).value))
            shape.coord_format = 'fk5'
            shape.coord_list = (row['RA'], row['Dec'],np.mean(rad).to(u.deg).value)
            shape.attr = ([], {'color': 'green',  'dash': '0 ',  'dashlist': '8 3 ',
                               'delete': '1 ',  'edit': '1 ', 'fixed': '0 ',
                               'font': '"helvetica 10 normal roman"',
                               'highlite': '1 ', 'include': '1 ',  'move': '1 ',
                               'select': '1 ',  'source': '1',  'text': '',
                               'width': '1 '})
            log.debug('{1} {2}: {0}'.format(shape, row['Release date'], row['Band']))
            if row['Release date']!='' and row['Band']==band and band in prv_mask:
                (xlo,xhi,ylo,yhi),mask = pyregion_subset(shape,
                                                         hit_mask_private[band],
                                                         mywcs)
                log.debug("{0},{1},{2},{3}: {4}".format(xlo,xhi,ylo,yhi,mask.sum()))
                hit_mask_private[band][ylo:yhi,xlo:xhi] += row['Integration']*mask
            if row['Release date']=='' and row['Band']==band and band in pub_mask:
                (xlo,xhi,ylo,yhi),mask = pyregion_subset(shape,
                                                         hit_mask_public[band],
                                                         mywcs)
                log.debug("{0},{1},{2},{3}: {4}".format(xlo,xhi,ylo,yhi,mask.sum()))
                hit_mask_public[band][ylo:yhi,xlo:xhi] += row['Integration']*mask

    fig = aplpy.FITSFigure(images[0])
    fig.show_grayscale(stretch='arcsinh')
    for band in bands:
        if band in pub_mask:
            fig.show_contour(fits.PrimaryHDU(data=hit_mask_public[band],
                                             header=images[0][0].header),
                             levels=integration_time_contour_levels,
                             colors=[band_colors_pub[band]]*6)
        if band in prv_mask:
            fig.show_contour(fits.PrimaryHDU(data=hit_mask_private[band],
                                             header=images[0][0].header),
                             levels=integration_time_contour_levels,
                             colors=[band_colors_priv[band]]*6)

    fig.save('{0}_almafinderchart.png'.format(save_prefix))

    return images, catalog, hit_mask_public, hit_mask_private
<|endoftext|>


<commit_before>
from os.path import dirname, join

import core
import atlas

from atlas.settings.local import defaultDatabase, MY_SECRET_KEY, dbaccess

### VIRTUALENV
VIRTUALENV_PATH = '/data/virtualenv/django1.6.1__python2.6.6__atlas'

### WSGI
WSGI_PATH = VIRTUALENV_PATH + '/pythonpath'

DATABASE_ROUTERS = ['dbrouter.ProdMonDBRouter']

STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    join(dirname(core.common.__file__), 'static'),
#    join(join(dirname(core.__file__),'datatables'), 'static'),
#    join(dirname(atlas.__file__), 'static'),
)

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    join(dirname(core.common.__file__), 'templates'),
    join(dirname(atlas.__file__), 'templates'),
)

STATIC_ROOT = join(dirname(atlas.__file__), 'static')
#STATIC_ROOT = None
MEDIA_ROOT = join(dirname(atlas.__file__), 'media')
STATIC_URL_BASE = '/static/'
MEDIA_URL_BASE = '/media/'

# Make this unique, and don't share it with anybody.
SECRET_KEY = MY_SECRET_KEY

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases
DATABASES = dbaccess

### URL_PATH_PREFIX for multi-developer apache/wsgi instance
### on EC2: URL_PATH_PREFIX = '/bigpandamon' or URL_PATH_PREFIX = '/developersprefix'
#URL_PATH_PREFIX = '/atlas'
URL_PATH_PREFIX = ''
### on localhost:8000: URL_PATH_PREFIX = '/.'
#URL_PATH_PREFIX = ''
MEDIA_URL = URL_PATH_PREFIX + MEDIA_URL_BASE
STATIC_URL = URL_PATH_PREFIX + STATIC_URL_BASE

LOG_ROOT = '/tmp/pandalog'
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
#    'disable_existing_loggers': True,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'null': {
            'level':'DEBUG',
            'class':'django.utils.log.NullHandler',
        },
        'logfile-bigpandamon': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': LOG_ROOT + "/logfile.bigpandamon",
            'maxBytes': 1000000000,
            'backupCount': 2,
            'formatter': 'verbose',
        },
        'logfile-prodtaskwebui': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': LOG_ROOT + "/logfile.prodtaskwebui",
            'maxBytes': 1000000000,
            'backupCount': 2,
            'formatter': 'verbose',
        },
        'logfile-django': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': LOG_ROOT + "/logfile.django",
            'maxBytes': 1000000000,
            'backupCount': 2,
            'formatter': 'verbose',
        },
        'logfile-viewdatatables': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': LOG_ROOT + "/logfile.viewdatatables",
            'maxBytes': 1000000000,
            'backupCount': 2,
            'formatter': 'verbose',
        },
        'logfile-rest': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': LOG_ROOT + "/logfile.rest",
            'maxBytes': 1000000000,
            'backupCount': 2,
            'formatter': 'verbose',
        },
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
#            'class': 'django.utils.log.AdminEmailHandler'
            'class':'logging.StreamHandler',
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
#            'level': 'ERROR',
            'level': 'DEBUG',
            'propagate': True,
        },
        'django': {
            'handlers':['logfile-django'],
            'propagate': True,
            'level':'DEBUG',
        },
        'django_datatables_view': {
            'handlers':['logfile-viewdatatables'],
            'propagate': True,
            'level':'DEBUG',
        },
        'rest_framework': {
            'handlers':['logfile-rest'],
            'propagate': True,
            'level':'DEBUG',
        },
        'bigpandamon': {
            'handlers': ['logfile-bigpandamon'],
            'level': 'DEBUG',
        },
        'prodtaskwebui': {
            'handlers': ['logfile-prodtaskwebui'],
            'level': 'DEBUG',
        },
    },
    'formatters': {
        'verbose': {
#            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
            'format': '%(asctime)s %(module)s %(name)-12s:%(lineno)d %(levelname)-5s %(message)s'
        },
        'simple': {
            'format': '%(levelname)s %(name)-12s:%(lineno)d %(message)s'
        },
    },
    'logfile': {
        'level':'DEBUG',
        'class':'logging.handlers.RotatingFileHandler',
        'filename': LOG_ROOT + "/logfile",
        'maxBytes': 10000000,
        'backupCount': 5,
        'formatter': 'verbose',
    },
}<commit_msg>Move log path to local<commit_after>
from os.path import dirname, join

import core
import atlas

from atlas.settings.local import defaultDatabase, MY_SECRET_KEY, dbaccess, local_log_root

### VIRTUALENV
VIRTUALENV_PATH = '/data/virtualenv/django1.6.1__python2.6.6__atlas'

### WSGI
WSGI_PATH = VIRTUALENV_PATH + '/pythonpath'

DATABASE_ROUTERS = ['dbrouter.ProdMonDBRouter']

STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    join(dirname(core.common.__file__), 'static'),
#    join(join(dirname(core.__file__),'datatables'), 'static'),
#    join(dirname(atlas.__file__), 'static'),
)

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    join(dirname(core.common.__file__), 'templates'),
    join(dirname(atlas.__file__), 'templates'),
)

STATIC_ROOT = join(dirname(atlas.__file__), 'static')
#STATIC_ROOT = None
MEDIA_ROOT = join(dirname(atlas.__file__), 'media')
STATIC_URL_BASE = '/static/'
MEDIA_URL_BASE = '/media/'

# Make this unique, and don't share it with anybody.
SECRET_KEY = MY_SECRET_KEY

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases
DATABASES = dbaccess

### URL_PATH_PREFIX for multi-developer apache/wsgi instance
### on EC2: URL_PATH_PREFIX = '/bigpandamon' or URL_PATH_PREFIX = '/developersprefix'
#URL_PATH_PREFIX = '/atlas'
URL_PATH_PREFIX = ''
### on localhost:8000: URL_PATH_PREFIX = '/.'
#URL_PATH_PREFIX = ''
MEDIA_URL = URL_PATH_PREFIX + MEDIA_URL_BASE
STATIC_URL = URL_PATH_PREFIX + STATIC_URL_BASE

LOG_ROOT = local_log_root
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
#    'disable_existing_loggers': True,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'null': {
            'level':'DEBUG',
            'class':'django.utils.log.NullHandler',
        },
        'logfile-bigpandamon': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': LOG_ROOT + "/logfile.bigpandamon",
            'maxBytes': 1000000000,
            'backupCount': 2,
            'formatter': 'verbose',
        },
        'logfile-prodtaskwebui': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': LOG_ROOT + "/logfile.prodtaskwebui",
            'maxBytes': 1000000000,
            'backupCount': 2,
            'formatter': 'verbose',
        },
        'logfile-django': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': LOG_ROOT + "/logfile.django",
            'maxBytes': 1000000000,
            'backupCount': 2,
            'formatter': 'verbose',
        },
        'logfile-viewdatatables': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': LOG_ROOT + "/logfile.viewdatatables",
            'maxBytes': 1000000000,
            'backupCount': 2,
            'formatter': 'verbose',
        },
        'logfile-rest': {
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': LOG_ROOT + "/logfile.rest",
            'maxBytes': 1000000000,
            'backupCount': 2,
            'formatter': 'verbose',
        },
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
#            'class': 'django.utils.log.AdminEmailHandler'
            'class':'logging.StreamHandler',
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
#            'level': 'ERROR',
            'level': 'DEBUG',
            'propagate': True,
        },
        'django': {
            'handlers':['logfile-django'],
            'propagate': True,
            'level':'DEBUG',
        },
        'django_datatables_view': {
            'handlers':['logfile-viewdatatables'],
            'propagate': True,
            'level':'DEBUG',
        },
        'rest_framework': {
            'handlers':['logfile-rest'],
            'propagate': True,
            'level':'DEBUG',
        },
        'bigpandamon': {
            'handlers': ['logfile-bigpandamon'],
            'level': 'DEBUG',
        },
        'prodtaskwebui': {
            'handlers': ['logfile-prodtaskwebui'],
            'level': 'DEBUG',
        },
    },
    'formatters': {
        'verbose': {
#            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
            'format': '%(asctime)s %(module)s %(name)-12s:%(lineno)d %(levelname)-5s %(message)s'
        },
        'simple': {
            'format': '%(levelname)s %(name)-12s:%(lineno)d %(message)s'
        },
    },
    'logfile': {
        'level':'DEBUG',
        'class':'logging.handlers.RotatingFileHandler',
        'filename': LOG_ROOT + "/logfile",
        'maxBytes': 10000000,
        'backupCount': 5,
        'formatter': 'verbose',
    },
}<|endoftext|>


<commit_before>from flask import Blueprint, render_template
from organise.models import Todo
from organise import db
from flask import render_template, request, redirect, url_for, flash

todos = Blueprint('todos', __name__, template_folder='/../templates')

@todos.route('/')
def index():
    all_todos = Todo.query.all()
    return render_template('todos.html', all_todos=all_todos)

@todos.route('/add', methods=['POST'])
def add_todo():
    todo = Todo(request.form['title'], request.form['description'])
    db.session.add(todo)
    db.session.commit()
    flash('New todo was added!')
    return redirect(url_for('todos.index'))

@todos.route('/<int:t_id>')
def show_todo(t_id):
    pass

@todos.route('/<int:t_id>/edit')
def edit_todo(t_id):
    pass

@todos.route('/<int:t_id>/delete', methods=['POST'])
def delete_todo(t_id):
    pass
<commit_msg>Return todos in descending order<commit_after>from flask import Blueprint, render_template
from organise.models import Todo
from organise import db
from flask import render_template, request, redirect, url_for, flash

todos = Blueprint('todos', __name__, template_folder='/../templates')

@todos.route('/')
def index():
    all_todos = Todo.query.order_by(Todo.id.desc()).all()
    return render_template('todos.html', all_todos=all_todos)

@todos.route('/add', methods=['POST'])
def add_todo():
    todo = Todo(request.form['title'], request.form['description'])
    db.session.add(todo)
    db.session.commit()
    flash('New todo was added!')
    return redirect(url_for('todos.index'))

@todos.route('/<int:t_id>')
def show_todo(t_id):
    pass

@todos.route('/<int:t_id>/edit')
def edit_todo(t_id):
    pass

@todos.route('/<int:t_id>/delete', methods=['POST'])
def delete_todo(t_id):
    pass
<|endoftext|>


<commit_before>#!/usr/bin/python
import struct
from sys import argv, exit

def ensure(condition, message, exitCode=0):
    if not condition:
        print(message)
        exit(exitCode)

# Check argument existence
ensure(len(argv) > 1,
'''
    swatch.py - Adobe ASE swatch reader

    Usage: swatch.py <in_file>

    https://github.com/rschiang/swatch-tool
''')

f = open(argv[1], 'r')

# Check ASE signature
ensure(f.read(4) == 'ASEF', 'ERROR: Signature mismatch')

# Check file version
version = struct.unpack_from('>hh', f)
if version != (1, 0):
    print('WARNING: File version mismatch (%d.%d)' % version)

# Read blocks
blocks = struct.unpack_from('>i', f)
for i in range(blocks):
    block_type = struct.unpack_from('>h', f)

    if block_type == 0xc001:
        print('[GROUP START]')
    elif block_type != 0x0001:
        print('WARNING: Unknown block type %d' % block_type)

    block_len = struct.unpack_from('>i', f)
    ensure(block_len > 0, 'ERROR: Invalid block size')

    name_len = struct.unpack_from('>H', f)
    ensure(0 < name_len < block_len, 'ERROR: Invalid name string length')

    name = u''
    for j in range(name_len):
        val = struct.unpack_from('>H', f)
        name += unichr(val)
    block_len -= name_len

    color_mode = struct.unpack_from('4s', f)
    block_len -= 4

    if color_mode == 'CMYK':
        C, M, Y, K = struct.unpack_from('4f', f)
        block_len -= 16

    elif color_mode == 'RGB ':
        R, G, B = struct.unpack_from('3f', f)
        block_len -= 12

    elif color_mode == 'LAB ':
        R, G, B = struct.unpack_from('3f', f)
        block_len -= 12

    elif color_mode == 'Gray':
        B = struct.unpack_from('f', f)
        block_len -= 4

    else:
        ensure(False, 'ERROR: Invalid color mode')

    ensure(block_len > 0, 'ERROR: Invalid block size')

    color_type = struct.unpack_from('>h', f)
    block_len -= 2

    ensure(block_len == 0, 'ERROR: Block size mismatch')

    if block_type == 0xc002:
        print('[GROUP END]')
<commit_msg>Fix unpacking function<commit_after>#!/usr/bin/python
import struct
from sys import argv, exit

def ensure(condition, message, exitCode=0):
    if not condition:
        print(message)
        exit(exitCode)

# Check argument existence
ensure(len(argv) > 1,
'''
    swatch.py - Adobe ASE swatch reader

    Usage: swatch.py <in_file>

    https://github.com/rschiang/swatch-tool
''')

f = open(argv[1], 'r')

# Check ASE signature
ensure(f.read(4) == 'ASEF', 'ERROR: Signature mismatch')

buf = f.read()
f.close()

def unpack(fmt):
    global buf
    val = struct.unpack_from(fmt, buf)
    buf = buf[struct.calcsize(fmt):]
    return val if len(val) > 1 else val[0]

# Check file version
version = unpack('>hh')
if version != (1, 0):
    print('WARNING: File version mismatch (%d.%d)' % version)

# Read blocks
blocks = unpack('>i')
for i in range(blocks):
    block_type = unpack('>h')

    if block_type == 0xc001:
        print('[GROUP START]')
    elif block_type != 0x0001:
        print('WARNING: Unknown block type %d' % block_type)

    block_len = unpack('>i')
    ensure(block_len > 0, 'ERROR: Invalid block size')

    name_len = unpack('>H')
    ensure(0 < name_len < block_len, 'ERROR: Invalid name string length')

    name = u''
    for j in range(name_len):
        val = unpack('>H')
        name += unichr(val)
    block_len -= name_len

    color_mode = unpack('4s')
    block_len -= 4

    if color_mode == 'CMYK':
        C, M, Y, K = unpack('4f')
        block_len -= 16

    elif color_mode == 'RGB ':
        R, G, B = unpack('3f')
        block_len -= 12

    elif color_mode == 'LAB ':
        R, G, B = unpack('3f')
        block_len -= 12

    elif color_mode == 'Gray':
        B = unpack('f')
        block_len -= 4

    else:
        ensure(False, 'ERROR: Invalid color mode')

    ensure(block_len > 0, 'ERROR: Invalid block size')

    color_type = unpack('>h')
    block_len -= 2

    ensure(block_len == 0, 'ERROR: Block size mismatch')

    if block_type == 0xc002:
        print('[GROUP END]')
<|endoftext|>


<commit_before>from django.conf.urls.defaults import *

urlpatterns = patterns('',
    # clusters
    url(r'^$', 'clusters.views.clusters', name="clusters"),
    url(r'^cluster/(?P<cluster_id>\d+)/$', 'clusters.views.cluster', name="cluster"),
    url(r'^rgraph/(?P<cluster_id>\d+)/$', 'clusters.views.radial_graph', name="radial_graph"),
    url(r'^iotable/(?P<cluster_id>\d+)/$', 'clusters.views.iotable', name="iotable"),
    url(r'^frtable/(?P<cluster_id>\d+)/(?P<toggle>\w+)/$', 'clusters.views.fr_table', name="fr_table"),
    url(r'^diagnostics/(?P<cluster_id>\d+)/$', 'clusters.views.diagnostics', name="diagnostics"),
    url(r'^modelerrors/(?P<cluster_id>\d+)/$', 'clusters.views.model_errors', name="model_errors"),
    url(r'^network/(?P<cluster_id>\d+)/$', 'clusters.views.network', name="network"),
    url(r'^network/(?P<cluster_id>\d+)/(?P<toggle>\w+)/$', 'clusters.views.network', name="network"),
    url(r'^flows/(?P<cluster_id>\d+)/$', 'clusters.views.flows', name="flows"),
    url(r'^flows/(?P<cluster_id>\d+)/(?P<toggle>\w+)/$', 'clusters.views.flows', name="flows"),

    # agents
    url(r'^clusteragents/(?P<cluster_id>\d+)/$', 'clusters.views.cluster_agents', name="cluster_agents"),
    url(r'^newclusteragent/(?P<cluster_id>\d+)/$', 'clusters.views.new_cluster_agent', name="new_cluster_agent"),
    url(r'^editclusteragent/(?P<cluster_id>\d+)/(?P<agent_id>\d+)/$', 'clusters.views.edit_cluster_agent', name="edit_cluster_agent"),
    url(r'^inlinenewagentfunction/(?P<cluster_id>\d+)/(?P<agent_id>\d+)/$', 'clusters.views.inline_new_agent_function', name="inline_new_agent_function"),
    url(r'^jsonagentaddress/(?P<agent_name>.+)/$', 'clusters.views.json_agent_address', name="json_agent_address"),

    # functions
    url(r'^functions/(?P<cluster_id>\d+)/$', 'clusters.views.economic_functions', name="economic_functions"),
    url(r'^function/(?P<function_id>\d+)/$', 'clusters.views.economic_function', name="economic_function"),
    url(r'^newfunction/(?P<cluster_id>\d+)/$', 'clusters.views.new_function', name="new_function"),
    url(r'^inlinenewfunction/(?P<cluster_id>\d+)/$', 'clusters.views.inline_new_function', name="inline_new_function"),
    url(r'^editclusterfunctions/(?P<cluster_id>\d+)/$', 'clusters.views.edit_cluster_functions', name="edit_cluster_functions"),
    url(r'^editflows/(?P<cluster_id>\d+)/$', 'clusters.views.edit_flows', name="edit_flows"),

    # resources
    url(r'^inlinenewresource/(?P<cluster_id>\d+)/$', 'clusters.views.inline_new_resource', name="inline_new_resource"),
    url(r'^inlineagentresource/(?P<cluster_id>\d+)/(?P<agent_id>\d+)/(?P<parent_id>\d+)/$', 'clusters.views.inline_agent_resource', name="inline_agent_resource"),
    url(r'^newresource/(?P<cluster_id>\d+)/$', 'clusters.views.new_resource', name="new_resource"),

    #function resources
    url(r'^newfunctionresource/(?P<function_id>\d+)/$', 'clusters.views.new_function_resource', name="new_function_resource"),
    url(r'^deletefunctionresource/(?P<id>\d+)/$', 'clusters.views.delete_function_resource', name="delete_function_resource"),
    url(r'^changefunctionresourceamount/$', 'clusters.views.change_function_resource_amount',
        name="change_function_resource_amount"),
    url(r'^changefunctionresourcevalue/$', 'clusters.views.change_function_resource_value',
        name="change_function_resource_value"),

    url(r'^deleteagentfunctionresource/(?P<id>\d+)/$', 'clusters.views.delete_agent_function_resource',
        name="delete_agent_function_resource"),
    url(r'^changeagentfunctionresourceamount/$', 'clusters.views.change_agent_function_resource_amount',
        name="change_agent_function_resource_amount"),
    url(r'^changeagentfunctionresourcevaalue/$', 'clusters.views.change_agent_function_resource_value',
        name="change_agent_function_resource_value"),
)

<commit_msg>add toggle to fr_table<commit_after>from django.conf.urls.defaults import *

urlpatterns = patterns('',
    # clusters
    url(r'^$', 'clusters.views.clusters', name="clusters"),
    url(r'^cluster/(?P<cluster_id>\d+)/$', 'clusters.views.cluster', name="cluster"),
    url(r'^rgraph/(?P<cluster_id>\d+)/$', 'clusters.views.radial_graph', name="radial_graph"),
    url(r'^iotable/(?P<cluster_id>\d+)/$', 'clusters.views.iotable', name="iotable"),
    url(r'^frtable/(?P<cluster_id>\d+)/$', 'clusters.views.fr_table', name="fr_table"),
    url(r'^frtable/(?P<cluster_id>\d+)/(?P<toggle>\w+)/$', 'clusters.views.fr_table', name="fr_table"),
    url(r'^diagnostics/(?P<cluster_id>\d+)/$', 'clusters.views.diagnostics', name="diagnostics"),
    url(r'^modelerrors/(?P<cluster_id>\d+)/$', 'clusters.views.model_errors', name="model_errors"),
    url(r'^network/(?P<cluster_id>\d+)/$', 'clusters.views.network', name="network"),
    url(r'^network/(?P<cluster_id>\d+)/(?P<toggle>\w+)/$', 'clusters.views.network', name="network"),
    url(r'^flows/(?P<cluster_id>\d+)/$', 'clusters.views.flows', name="flows"),
    url(r'^flows/(?P<cluster_id>\d+)/(?P<toggle>\w+)/$', 'clusters.views.flows', name="flows"),

    # agents
    url(r'^clusteragents/(?P<cluster_id>\d+)/$', 'clusters.views.cluster_agents', name="cluster_agents"),
    url(r'^newclusteragent/(?P<cluster_id>\d+)/$', 'clusters.views.new_cluster_agent', name="new_cluster_agent"),
    url(r'^editclusteragent/(?P<cluster_id>\d+)/(?P<agent_id>\d+)/$', 'clusters.views.edit_cluster_agent', name="edit_cluster_agent"),
    url(r'^inlinenewagentfunction/(?P<cluster_id>\d+)/(?P<agent_id>\d+)/$', 'clusters.views.inline_new_agent_function', name="inline_new_agent_function"),
    url(r'^jsonagentaddress/(?P<agent_name>.+)/$', 'clusters.views.json_agent_address', name="json_agent_address"),

    # functions
    url(r'^functions/(?P<cluster_id>\d+)/$', 'clusters.views.economic_functions', name="economic_functions"),
    url(r'^function/(?P<function_id>\d+)/$', 'clusters.views.economic_function', name="economic_function"),
    url(r'^newfunction/(?P<cluster_id>\d+)/$', 'clusters.views.new_function', name="new_function"),
    url(r'^inlinenewfunction/(?P<cluster_id>\d+)/$', 'clusters.views.inline_new_function', name="inline_new_function"),
    url(r'^editclusterfunctions/(?P<cluster_id>\d+)/$', 'clusters.views.edit_cluster_functions', name="edit_cluster_functions"),
    url(r'^editflows/(?P<cluster_id>\d+)/$', 'clusters.views.edit_flows', name="edit_flows"),

    # resources
    url(r'^inlinenewresource/(?P<cluster_id>\d+)/$', 'clusters.views.inline_new_resource', name="inline_new_resource"),
    url(r'^inlineagentresource/(?P<cluster_id>\d+)/(?P<agent_id>\d+)/(?P<parent_id>\d+)/$', 'clusters.views.inline_agent_resource', name="inline_agent_resource"),
    url(r'^newresource/(?P<cluster_id>\d+)/$', 'clusters.views.new_resource', name="new_resource"),

    #function resources
    url(r'^newfunctionresource/(?P<function_id>\d+)/$', 'clusters.views.new_function_resource', name="new_function_resource"),
    url(r'^deletefunctionresource/(?P<id>\d+)/$', 'clusters.views.delete_function_resource', name="delete_function_resource"),
    url(r'^changefunctionresourceamount/$', 'clusters.views.change_function_resource_amount',
        name="change_function_resource_amount"),
    url(r'^changefunctionresourcevalue/$', 'clusters.views.change_function_resource_value',
        name="change_function_resource_value"),

    url(r'^deleteagentfunctionresource/(?P<id>\d+)/$', 'clusters.views.delete_agent_function_resource',
        name="delete_agent_function_resource"),
    url(r'^changeagentfunctionresourceamount/$', 'clusters.views.change_agent_function_resource_amount',
        name="change_agent_function_resource_amount"),
    url(r'^changeagentfunctionresourcevaalue/$', 'clusters.views.change_agent_function_resource_value',
        name="change_agent_function_resource_value"),
)

<|endoftext|>


<commit_before>"""

So, here's the idea.

We have a repository of a filesystem stored somewhere.
The repository contains the following:
- the files with their checksum (SHA1, for example).
  Files are split into chunks and each chunk's checksum is computed. So if
  there are common parts of the files, they will have the same checksum so
  the content needs to be stored only once.

- the directory tree, where file objects are pointing to the files.

The client may requests one of the following:
- directory metadata. In this case it will receive some part of the
  directory tree. It will contain the POSIX attributes, the filenames, etc.

- bitmap of a specified file. The bitmap is a series of checksums for the
  chunks in the file.

- the chunk itself. The chunk is identified by its checksum.

So the client can re-construct both the directory structure and the files.

Also, and here comes the main point, the client may store both the metadata
and the bitmap (and the chunks) on the disk, locally.

So when a 3rd party makes some change on the filesystem, the server notifies
all the clients which are interested in this change (eg which cached those
data locally), so the clients can invalidate these objects.

The client may decide to keep the file content - this may be useful in the
future if the file content is reverted - or to remove the content as well.
But the main point is that if the user needs the invalidated file again, it
downloads it again.

If the change is initiated from the client, it notifies the server about the
invalidation, pretty like the same was as the server does for the client.

"""

import pdb

import os
import hashlib
from keyword import iskeyword
import binascii
import math
import collections

pjoin = os.path.join

def read_buffer(file, maxsize):
    """
    Reads maxsize sized buffers from the file.
    Yields the buffers.

    """
    buff = None
    while True:
        buff = file.read(maxsize)
        if buff == "":
            break
        else:
            yield buff

class Bitmap(list):
    """
    Represents one bitmap for a given file.

    Abstract class, needs to be inherited.

    Represents checksum in its binary form (not hex), inherits from list.

    """
    def __init__(self):
        super(Bitmap, self).__init__()
        self.chunk_size = None

    def calculate_digest(self, chunk):
        digest = self.alg(chunk).digest()
        return digest

    def add_chunk(self, chunk):
        digest = self.calculate_digest(chunk)
        self.append(digest)
        return digest

class SHA1Bitmap(Bitmap):
    """
    Specifies sha1 as an algorithm for the bitmap.

    """
    alg = hashlib.sha1

class Store(object):
    """

    Implements bitmap + content store.
    Stores bitmaps and chunks.

    """
    def __init__(self, output_dir, bitmap_class=SHA1Bitmap):
        self.output_dir = output_dir
        self.bitmap_class = bitmap_class

    def store_file(self, path, chunk_size=None):
        bitmap = self.bitmap_class()

        if chunk_size is None:
            chunk_size = self.calculate_chunk_size(os.path.getsize(path))

        file = open(path, "rb")

        for buff in read_buffer(file, chunk_size):
            digest = bitmap.add_chunk(buff)
            self.store_chunk(buff, digest)

        file.close()
        bitmap.chunk_size = chunk_size

        return bitmap

    def store_chunk(self, chunk, digest):
        if not os.path.isdir(self.output_dir):
            raise ValueError("No such directory: %r" % self.output_dir)

        hash_file = self.get_chunk_path(digest, True)

        if not os.path.isfile(hash_file):
            open(hash_file, "w").write(chunk)

    def get_chunk_path(self, digest, create_dir=True):
        hex_digest = binascii.b2a_hex(digest)
        hash_dir = pjoin(self.output_dir, hex_digest[:2], hex_digest[2:4])
        if create_dir:
            try:
                os.makedirs(hash_dir)
            except OSError, err:
                if err.errno != 17:
                    raise

        hash_file = pjoin(hash_dir, hex_digest)
        return hash_file

    def calculate_chunk_size(self, size):
        if size == 0:
            return 0

        exp = int(math.log(size, 2))
        ret_exp = int(exp*0.8)
        return min(max(2**int(exp*0.8), 4096), 131072)

    def iter_bitmap(self, bitmap):
        for digest in bitmap:
            yield self.read_chunk(digest)

    def read_chunk(self, digest):
        path = self.get_chunk_path(digest, False)
        # FIXME - the whole chunk?
        return open(path, "rb").read()

class Struct(object):
    """
    Implements a general struct object.

    Very similar to named tuples, but it's more flexible.

    """

    params = ()
    def __init__(self, *args, **kwargs):
        for idx, kv in enumerate(self.params):
            if isinstance(kv, tuple) and len(kv) == 2:
                key = kv[0]
                value = kv[1]
            elif isinstance(kv, basestring) and not iskeyword(kv):
                key = kv
                value = None
            else:
                raise ValueError("invalid param: %s" % repr(kv))

            if hasattr(self, key):
                raise ValueError("Invalid param, attribute already exists: %r" % key)

            if idx<len(args):
                value = args[idx]
            else:
                value = kwargs.get(key, value)

#            print "%s -> %s" % (key, value)
            setattr(self, key, value)

    def __getitem__(self, name):
        if name in self.params:
            return getattr(self, name)
        else:
            raise KeyError("No such key: %r" % name)

    def __setitem__(self, name, value):
        if name not in self.params:
            raise KeyError("Invalid key: %r" % name)
        setattr(self, name, value)

    def iteritems(self):
        for key in self.params:
            yield (key, getattr(self, key))

    def keys(self):
        return self.params

    def as_dict(self):
        return {k:getattr(self, k) for k in self.params}

    def __cmp__(self, other):
        if other is self:
            return 0 # same
        else:
            if isinstance(other, dict):
                return cmp(self.as_dict(), other)
            elif isinstance(other, Struct):
                return cmp(self.as_dict(), other.as_dict())
            else:
                return NotImplemented

class FileMeta(Struct):
    """
    Stores meta information about a file.
    Note the "chunk_size" which contains the size of the chunk in bytes.

    """
    params = ("name",
              "mode",
              "owner",
              "group",
              "mtime",
              "ctime",
              "atime",
              "size",
              "chunk_size")

class File(object):
    """
    Represents a single file in the filesystem.

    """
    def __init__(self, meta=None, bitmap=None):
        if meta is None:
            self.meta = FileMeta()
        elif not isinstance(meta, FileMeta):
            raise TypeError("meta argument must be an instance of FileMeta")
        else:
            self.meta = meta

        if bitmap:
            self.bitmap = bitmap
        else:
            # FIXME - use bitmap class instead?
            self.bitmap = []

    @classmethod
    def from_dict(cls, kwargs):
        return cls(**kwargs)

    def update_from_path(self, path, store):
        name = os.path.basename(path)
        stat = os.stat(path)
        meta = FileMeta(name, stat.st_mode, stat.st_uid, stat.st_gid, stat.st_mtime, stat.st_ctime, stat.st_atime, stat.st_size)

        bitmap = store.store_file(path)

        # FIXME
        meta.chunk_size = bitmap.chunk_size

        self.meta = meta
        self.bitmap = bitmap

    def __eq__(self, other):
        if id(self) == id(other):
            return True

        if isinstance(other, File):
            return self.meta == other.meta and self.bitmap == other.bitmap
        else:
            return NotImplemented

    def __ne__(self, other):
        if id(self) == id(other):
            return False

        if isinstance(other, File):
            return self.meta != other.meta or self.bitmap != other.bitmap
        else:
            return NotImplemented

    def get_name(self):
        return self.meta.name

    def set_name(self, value):
        self.meta.name=value

    name = property(get_name, set_name)

class Directory(object):
    """
    Represents a directory in the filesystem.

    Contains the entries (File or Directory objects), which can be queried.
    Entries are stored in a dict, hashed by their key to make faster lookups.

    """
    def __init__(self, name=None, entries=None, parent=None):
        self.name = name
        self.parent = parent # this one is currently unused

        if entries is None:
            self.entries = {}
        else:
            self.entries = entries

    def create(self, file):
        if not isinstance(file, (File, Directory)):
            raise TypeError("file parameter must be File or Directory instance")

        if file.name in self.entries:
            raise KeyError("File already exists: %r" % file.name)

        self.entries[file.name] = file

    def remove(self, file):
        if isinstance(file, (File, Directory)):
            filename = file.name
        elif isinstance(file, basestring):
            filename = file
        else:
            raise TypeError("file parameter must be File, Directory or string instance")

        if filename not in self.entries:
            raise KeyError("File does not exist: %r" % filename)

        file = self.entries[filename]

        if isinstance(file, Directory) and len(file) > 0:
            raise KeyError("Directory is not empty: %r" % file.name)

        del self.entries[file.name]

    def __len__(self):
        return len(self.entries)

    def get_dirs_files(self):
        """
        Returns the directories and files in a 2-element tuple.
        First element contains the directories as a list, the other contains the files as list.

        Note: objects are references to the directory structure, be careful when changing their name.
        """
        dirs = []
        files = []
        for entry in self.entries.values():
            if isinstance(entry, File):
                files.append(entry)
            elif isinstance(entry, Directory):
                dirs.append(entry)
            else:
                raise TypeError("Unkown object??? %s" % repr(entry))

        return (dirs, files)

    def walk(self):
        parent = ""
        queue = collections.deque([self])

        while len(queue)>0:
            curr_dir = queue.popleft()
            if type(curr_dir) == str:
                parent = curr_dir
                curr_dir = queue.popleft()

            root = pjoin(parent, curr_dir.name)
            dirs, files = curr_dir.get_dirs_files()
            yield (root, dirs, files)
            if len(dirs) > 0:
                queue.append(root)
                queue.extend(dirs)

def scan(dir, store, ignore=None):
    """
    Adds a directory structure from the filesystem to the store, which
    must be created separately.

    Returns the root directory object.
    """
    if not os.path.isabs(dir) or not os.path.isdir(dir):
        raise ValueError("dir must be absolute path and directory")

    if ignore is None:
        ignore = set()

    root_dir = Directory(os.path.basename(dir))

    for name in os.listdir(dir):
        if name in ignore:
            continue

        full_path = pjoin(dir, name)
        if os.path.isfile(full_path):
            new_file = File()
            new_file.update_from_path(full_path, store)
            root_dir.create(new_file)
        elif os.path.isdir(full_path):
            new_dir = scan(full_path, store, ignore)
            root_dir.create(new_dir)
        else:
            raise ValueError("Unsupported file: %s" % full_path)

    return root_dir

<commit_msg>syncfs.py: added get_dir, get_file, get_entry methods.<commit_after>"""

So, here's the idea.

We have a repository of a filesystem stored somewhere.
The repository contains the following:
- the files with their checksum (SHA1, for example).
  Files are split into chunks and each chunk's checksum is computed. So if
  there are common parts of the files, they will have the same checksum so
  the content needs to be stored only once.

- the directory tree, where file objects are pointing to the files.

The client may requests one of the following:
- directory metadata. In this case it will receive some part of the
  directory tree. It will contain the POSIX attributes, the filenames, etc.

- bitmap of a specified file. The bitmap is a series of checksums for the
  chunks in the file.

- the chunk itself. The chunk is identified by its checksum.

So the client can re-construct both the directory structure and the files.

Also, and here comes the main point, the client may store both the metadata
and the bitmap (and the chunks) on the disk, locally.

So when a 3rd party makes some change on the filesystem, the server notifies
all the clients which are interested in this change (eg which cached those
data locally), so the clients can invalidate these objects.

The client may decide to keep the file content - this may be useful in the
future if the file content is reverted - or to remove the content as well.
But the main point is that if the user needs the invalidated file again, it
downloads it again.

If the change is initiated from the client, it notifies the server about the
invalidation, pretty like the same was as the server does for the client.

"""

import pdb

import os
import hashlib
from keyword import iskeyword
import binascii
import math
import collections

pjoin = os.path.join

def read_buffer(file, maxsize):
    """
    Reads maxsize sized buffers from the file.
    Yields the buffers.

    """
    buff = None
    while True:
        buff = file.read(maxsize)
        if buff == "":
            break
        else:
            yield buff

class Bitmap(list):
    """
    Represents one bitmap for a given file.

    Abstract class, needs to be inherited.

    Represents checksum in its binary form (not hex), inherits from list.

    """
    def __init__(self):
        super(Bitmap, self).__init__()
        self.chunk_size = None

    def calculate_digest(self, chunk):
        digest = self.alg(chunk).digest()
        return digest

    def add_chunk(self, chunk):
        digest = self.calculate_digest(chunk)
        self.append(digest)
        return digest

class SHA1Bitmap(Bitmap):
    """
    Specifies sha1 as an algorithm for the bitmap.

    """
    alg = hashlib.sha1

class Store(object):
    """

    Implements bitmap + content store.
    Stores bitmaps and chunks.

    """
    def __init__(self, output_dir, bitmap_class=SHA1Bitmap):
        self.output_dir = output_dir
        self.bitmap_class = bitmap_class

    def store_file(self, path, chunk_size=None):
        bitmap = self.bitmap_class()

        if chunk_size is None:
            chunk_size = self.calculate_chunk_size(os.path.getsize(path))

        file = open(path, "rb")

        for buff in read_buffer(file, chunk_size):
            digest = bitmap.add_chunk(buff)
            self.store_chunk(buff, digest)

        file.close()
        bitmap.chunk_size = chunk_size

        return bitmap

    def store_chunk(self, chunk, digest):
        if not os.path.isdir(self.output_dir):
            raise ValueError("No such directory: %r" % self.output_dir)

        hash_file = self.get_chunk_path(digest, True)

        if not os.path.isfile(hash_file):
            open(hash_file, "w").write(chunk)

    def get_chunk_path(self, digest, create_dir=True):
        hex_digest = binascii.b2a_hex(digest)
        hash_dir = pjoin(self.output_dir, hex_digest[:2], hex_digest[2:4])
        if create_dir:
            try:
                os.makedirs(hash_dir)
            except OSError, err:
                if err.errno != 17:
                    raise

        hash_file = pjoin(hash_dir, hex_digest)
        return hash_file

    def calculate_chunk_size(self, size):
        if size == 0:
            return 0

        exp = int(math.log(size, 2))
        ret_exp = int(exp*0.8)
        return min(max(2**int(exp*0.8), 4096), 131072)

    def iter_bitmap(self, bitmap):
        for digest in bitmap:
            yield self.read_chunk(digest)

    def read_chunk(self, digest):
        path = self.get_chunk_path(digest, False)
        # FIXME - the whole chunk?
        return open(path, "rb").read()

class Struct(object):
    """
    Implements a general struct object.

    Very similar to named tuples, but it's more flexible.

    """

    params = ()
    def __init__(self, *args, **kwargs):
        for idx, kv in enumerate(self.params):
            if isinstance(kv, tuple) and len(kv) == 2:
                key = kv[0]
                value = kv[1]
            elif isinstance(kv, basestring) and not iskeyword(kv):
                key = kv
                value = None
            else:
                raise ValueError("invalid param: %s" % repr(kv))

            if hasattr(self, key):
                raise ValueError("Invalid param, attribute already exists: %r" % key)

            if idx<len(args):
                value = args[idx]
            else:
                value = kwargs.get(key, value)

#            print "%s -> %s" % (key, value)
            setattr(self, key, value)

    def __getitem__(self, name):
        if name in self.params:
            return getattr(self, name)
        else:
            raise KeyError("No such key: %r" % name)

    def __setitem__(self, name, value):
        if name not in self.params:
            raise KeyError("Invalid key: %r" % name)
        setattr(self, name, value)

    def iteritems(self):
        for key in self.params:
            yield (key, getattr(self, key))

    def keys(self):
        return self.params

    def as_dict(self):
        return {k:getattr(self, k) for k in self.params}

    def __cmp__(self, other):
        if other is self:
            return 0 # same
        else:
            if isinstance(other, dict):
                return cmp(self.as_dict(), other)
            elif isinstance(other, Struct):
                return cmp(self.as_dict(), other.as_dict())
            else:
                return NotImplemented

class FileMeta(Struct):
    """
    Stores meta information about a file.
    Note the "chunk_size" which contains the size of the chunk in bytes.

    """
    params = ("name",
              "mode",
              "owner",
              "group",
              "mtime",
              "ctime",
              "atime",
              "size",
              "chunk_size")

class File(object):
    """
    Represents a single file in the filesystem.

    """
    def __init__(self, meta=None, bitmap=None):
        if meta is None:
            self.meta = FileMeta()
        elif not isinstance(meta, FileMeta):
            raise TypeError("meta argument must be an instance of FileMeta")
        else:
            self.meta = meta

        if bitmap:
            self.bitmap = bitmap
        else:
            # FIXME - use bitmap class instead?
            self.bitmap = []

    @classmethod
    def from_dict(cls, kwargs):
        return cls(**kwargs)

    def update_from_path(self, path, store):
        name = os.path.basename(path)
        stat = os.stat(path)
        meta = FileMeta(name, stat.st_mode, stat.st_uid, stat.st_gid, stat.st_mtime, stat.st_ctime, stat.st_atime, stat.st_size)

        bitmap = store.store_file(path)

        # FIXME
        meta.chunk_size = bitmap.chunk_size

        self.meta = meta
        self.bitmap = bitmap

    def __eq__(self, other):
        if id(self) == id(other):
            return True

        if isinstance(other, File):
            return self.meta == other.meta and self.bitmap == other.bitmap
        else:
            return NotImplemented

    def __ne__(self, other):
        if id(self) == id(other):
            return False

        if isinstance(other, File):
            return self.meta != other.meta or self.bitmap != other.bitmap
        else:
            return NotImplemented

    def get_name(self):
        return self.meta.name

    def set_name(self, value):
        self.meta.name=value

    name = property(get_name, set_name)

class Directory(object):
    """
    Represents a directory in the filesystem.

    Contains the entries (File or Directory objects), which can be queried.
    Entries are stored in a dict, hashed by their key to make faster lookups.

    """
    def __init__(self, name=None, entries=None, parent=None):
        self.name = name
        self.parent = parent # this one is currently unused

        if entries is None:
            self.entries = {}
        else:
            self.entries = entries

    def create(self, file):
        if not isinstance(file, (File, Directory)):
            raise TypeError("file parameter must be File or Directory instance")

        if file.name in self.entries:
            raise KeyError("File already exists: %r" % file.name)

        self.entries[file.name] = file

    def remove(self, file):
        if isinstance(file, (File, Directory)):
            filename = file.name
        elif isinstance(file, basestring):
            filename = file
        else:
            raise TypeError("file parameter must be File, Directory or string instance")

        if filename not in self.entries:
            raise KeyError("File does not exist: %r" % filename)

        file = self.entries[filename]

        if isinstance(file, Directory) and len(file) > 0:
            raise KeyError("Directory is not empty: %r" % file.name)

        del self.entries[file.name]

    def __len__(self):
        return len(self.entries)

    def get_dirs_files(self):
        """
        Returns the directories and files in a 2-element tuple.
        First element contains the directories as a list, the other contains the files as list.

        Note: objects are references to the directory structure, be careful when changing their name.
        """
        dirs = []
        files = []
        for entry in self.entries.values():
            if isinstance(entry, File):
                files.append(entry)
            elif isinstance(entry, Directory):
                dirs.append(entry)
            else:
                raise TypeError("Unkown object??? %s" % repr(entry))

        return (dirs, files)

    def get_dir(self, name):
        return self.get_entry(name, Directory)

    def get_file(self, name):
        return self.get_entry(name, File)

    def get_entry(self, name, klass):
        retval = self.entries.get(name, None)
        if isintance(retval, klass):
            return retval
        else:
            raise KeyError("No such %s: %s" % (klass.__name__, name))

    def walk(self):
        parent = ""
        queue = collections.deque([self])

        while len(queue)>0:
            curr_dir = queue.popleft()
            if type(curr_dir) == str:
                parent = curr_dir
                curr_dir = queue.popleft()

            root = pjoin(parent, curr_dir.name)
            dirs, files = curr_dir.get_dirs_files()
            yield (root, dirs, files)
            if len(dirs) > 0:
                queue.append(root)
                queue.extend(dirs)

def scan(dir, store, ignore=None):
    """
    Adds a directory structure from the filesystem to the store, which
    must be created separately.

    Returns the root directory object.
    """
    if not os.path.isabs(dir) or not os.path.isdir(dir):
        raise ValueError("dir must be absolute path and directory")

    if ignore is None:
        ignore = set()

    root_dir = Directory(os.path.basename(dir))

    for name in os.listdir(dir):
        if name in ignore:
            continue

        full_path = pjoin(dir, name)
        if os.path.isfile(full_path):
            new_file = File()
            new_file.update_from_path(full_path, store)
            root_dir.create(new_file)
        elif os.path.isdir(full_path):
            new_dir = scan(full_path, store, ignore)
            root_dir.create(new_dir)
        else:
            raise ValueError("Unsupported file: %s" % full_path)

    return root_dir

<|endoftext|>