<commit_before>#!/usr/bin/env python2
# -*- coding: utf-8 -*-

import getopt
import sys
import urllib
import xml.etree.ElementTree as ET

URL_PATTERN_ID = 'http://www.xiami.com/song/playlist/id/%d'
URL_PATTERN_SONG = '%s/object_name/default/object_id/0' % URL_PATTERN_ID
URL_PATTERN_ALBUM = '%s/type/1' % URL_PATTERN_ID

def get_playlist_from_url(url):
    return parse_playlist(urllib.urlopen(url).read())

def parse_playlist(playlist):
    xml = ET.fromstring(playlist)
    return [
        {
            'title': track.find('{http://xspf.org/ns/0/}title').text,
            'location': track.find('{http://xspf.org/ns/0/}location').text
        }
        for track in xml.iter('{http://xspf.org/ns/0/}track')
    ]

def decode_location(location):
    url = location[1:]
    urllen = len(url)
    rows = int(location[0:1])

    cols_base = urllen / rows  # basic column count
    rows_ex = urllen % rows    # count of rows that have 1 more column

    matrix = []
    for r in xrange(rows):
        length = cols_base + 1 if r < rows_ex else cols_base
        matrix.append(url[:length])
        url = url[length:]

    url = ''
    for i in xrange(urllen):
        url += matrix[i % rows][i / rows]

    return urllib.unquote(url).replace('^', '0')

def download(url, dest):
    urllib.urlretrieve(url, dest)

if __name__ == '__main__':
    print 'Xiami Music Preview Downloader'

    playlists = []

    optlist, args = getopt.getopt(sys.argv[1:], 'a:s:')
    for key, value in optlist:
        if key == '-a':
            playlists.append(URL_PATTERN_ALBUM % int(value))
        elif key == '-s':
            playlists.append(URL_PATTERN_SONG % int(value))

    tracks = []

    for playlist_url in playlists:
        for url in get_playlist_from_url(playlist_url):
            tracks.append(url)

    print '%d file(s) to download' % len(tracks)
    for i in xrange(len(tracks)):
        track = tracks[i]
        filename = '%s.mp3' % track['title']
        url = decode_location(track['location'])
        print '[%02d/%02d] Downloading %s...' % (i, len(tracks), filename)
        download(url, filename)
<commit_msg>ADD: exception handling for getopt<commit_after>#!/usr/bin/env python2
# -*- coding: utf-8 -*-

import getopt
import sys
import urllib
import xml.etree.ElementTree as ET

URL_PATTERN_ID = 'http://www.xiami.com/song/playlist/id/%d'
URL_PATTERN_SONG = '%s/object_name/default/object_id/0' % URL_PATTERN_ID
URL_PATTERN_ALBUM = '%s/type/1' % URL_PATTERN_ID

def get_playlist_from_url(url):
    return parse_playlist(urllib.urlopen(url).read())

def parse_playlist(playlist):
    xml = ET.fromstring(playlist)
    return [
        {
            'title': track.find('{http://xspf.org/ns/0/}title').text,
            'location': track.find('{http://xspf.org/ns/0/}location').text
        }
        for track in xml.iter('{http://xspf.org/ns/0/}track')
    ]

def decode_location(location):
    url = location[1:]
    urllen = len(url)
    rows = int(location[0:1])

    cols_base = urllen / rows  # basic column count
    rows_ex = urllen % rows    # count of rows that have 1 more column

    matrix = []
    for r in xrange(rows):
        length = cols_base + 1 if r < rows_ex else cols_base
        matrix.append(url[:length])
        url = url[length:]

    url = ''
    for i in xrange(urllen):
        url += matrix[i % rows][i / rows]

    return urllib.unquote(url).replace('^', '0')

def download(url, dest):
    urllib.urlretrieve(url, dest)

def usage():
    message = [
        'Usage: %s [options]' % (sys.argv[0]),
        '    -s : song id',
        '    -a : album id'
    ]
    print '\n'.join(message)

if __name__ == '__main__':
    print 'Xiami Music Preview Downloader'

    playlists = []

    try:
        optlist, args = getopt.getopt(sys.argv[1:], 'ha:s:')
    except getopt.GetoptError as e:
        print e
        usage()
        sys.exit(1)

    for key, value in optlist:
        if key == '-a':
            playlists.append(URL_PATTERN_ALBUM % int(value))
        elif key == '-s':
            playlists.append(URL_PATTERN_SONG % int(value))
        elif key == '-h':
            usage()
            sys.exit(1)

    tracks = []

    for playlist_url in playlists:
        for url in get_playlist_from_url(playlist_url):
            tracks.append(url)

    print '%d file(s) to download' % len(tracks)
    for i in xrange(len(tracks)):
        track = tracks[i]
        filename = '%s.mp3' % track['title']
        url = decode_location(track['location'])
        print '[%02d/%02d] Downloading %s...' % (i, len(tracks), filename)
        download(url, filename)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
xsync.py by zjx20
http://github.com/zjx20/xsync/

This script will watch a local directory and on change will
sync to a remote directory. The script can be easily modified
to do whatever you want on a change event.

requires: pip install watchdog

  about watchdog:
    # project site: http://github.com/gorakhargosh/watchdog
    # api document: https://pythonhosted.org/watchdog/index.html

TODO: enhance 'ignore list' feature
"""

import os, datetime, time
import sys, argparse, json
import watchdog.events, watchdog.observers

class SyncHandler(watchdog.events.FileSystemEventHandler):

    def __init__(self, conf, times=False):
        watchdog.events.FileSystemEventHandler.__init__(self)
        self.local_path = conf['local_path']
        self.remote_host = conf['remote_host']
        self.remote_path = conf['remote_path']
        self.ignore_list = []
        self.times = times
        if conf.has_key('ignore_list'):
            self.ignore_list = conf['ignore_list']
        self.ignore_list += ['.xsync']    # ignore .xsync by default

    def should_ignore(self, filename):
        for ig in self.ignore_list:
            if ig in filename:
                return True
        return False

    def on_created(self, event):
        filename = event.src_path
        if self.should_ignore(filename):
            return

        if event.is_directory:
            filename += '/'

        remote_file = filename.replace(self.local_path, '')
        remote_parent = os.path.dirname(remote_file)

        # -lptgoD is almostly equal to -a except -r
        rsync_args = '-lpgoDzq' + ('t' if self.times else '')
        cmd = " rsync %s %s %s:%s%s/ " % \
            (rsync_args, filename, self.remote_host,
             self.remote_path, remote_parent)
        display("Syncing %s " % filename)
        os.system(cmd)

    def on_deleted(self, event):
        filename = event.src_path
        if self.should_ignore(filename):
            return

        local_parent = os.path.dirname(filename) + '/'
        if not os.path.isdir(local_parent):
            # the parent dir does not exists, skip
            return

        remote_file = filename.replace(self.local_path, '')

        cmd = " ssh %s 'rm -rf %s%s'" % \
            (self.remote_host, self.remote_path, remote_file)
        display("Syncing %s " % filename)
        os.system(cmd)

    def on_modified(self, event):
        if isinstance(event, watchdog.events.DirModifiedEvent):
            # ignore dir modified event
            return

        filename = event.src_path
        if self.should_ignore(filename):
            return

        remote_file = filename.replace(self.local_path, '')

        # -lptgoDr is equal to -a
        rsync_args = '-lpgoDrzq' + ('t' if self.times else '')
        cmd = " rsync %s --delete %s %s:%s%s " % \
            (rsync_args, filename, self.remote_host,
             self.remote_path, remote_file)
        display("Syncing %s " % filename)
        os.system(cmd)

    def on_moved(self, event):
        if event.is_directory:
            self.on_deleted(watchdog.events.DirDeletedEvent(event.src_path))
            self.on_created(watchdog.events.DirCreatedEvent(event.dest_path))
        else:
            self.on_deleted(watchdog.events.FileDeletedEvent(event.src_path))
            self.on_created(watchdog.events.FileCreatedEvent(event.dest_path))

def display(str):
    now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    print "[{0}] {1}".format(now, str)

def parse_opt():
    parser = argparse.ArgumentParser(prog='xsync',
            formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('--setup', action='store_true',
            help='cp xsync to /usr/local/bin')
    parser.add_argument('--full', action='store_true',
            help='do full sync instead of starting a daemon')
    parser.add_argument('--conf', action='store',
            default=os.path.join(os.getcwd(), '.xsync'),
            help='config file path')
    parser.add_argument('--init', action='store_true',
            help='create .xsync file in current folder')
    parser.add_argument('--times', action='store_true',
            help='preserve modification times')
    args = parser.parse_args()
    return args

"""
Config sample:

  {
    "local_path": "/home/x/project1/",
    "remote_host": "dev@111.222.111.222",
    "remote_path": "/home/dev/project1/",
    "ignore_list": [".git", ".svn", ".DS_Store"]
  }

  #
  # NOTICE:
  #
  #   it will use dirname of config file path as 'local_path' if
  #   the field does not exists.
  #
  #   if you are meeting a non-standard ssh port, try
  #   "--rsh='ssh -p2222' x@127.0.0.1" as 'remote_host'.
  #
  #   'ignore_list' is optional.
  #

  Or config blocks in an array

  [
    {
      "local_path": "/home/x/project1/",
      "remote_host": "dev@111.222.111.222",
      "remote_path": "/home/dev/project1/",
      "ignore_list": [".git", ".svn", ".DS_Store"]
    },
    {
      ...
    },
  ]
"""
def parse_conf(filepath):
    if not os.path.isfile(filepath):
        print >> sys.stderr, '[WARNING] Config "' + filepath + '" not ' + \
            'exists! skipped.'
        return []

    conf = None
    with open(filepath) as f:
        try:
            conf = json.loads(f.read())
        except ValueError:
            print >> sys.stderr, '[WARNING] Couldn\'t parse config ' + \
                'from "%s"! skipped.' % filepath
            return []

    conf_list = conf
    if not isinstance(conf, list):
        conf_list = [conf]

    for conf in conf_list:
        if not conf.has_key('local_path'):
            conf['local_path'] = '%s/' % \
                os.path.dirname(os.path.abspath(filepath))
        if not conf['local_path'].endswith('/'):
            conf['local_path'] += '/'
        if not conf['remote_path'].endswith('/'):
            conf['remote_path'] += '/'

    return conf_list

def watch(conf_list, options):
    observer = watchdog.observers.Observer()

    for conf in conf_list:
        observer.schedule(SyncHandler(conf, times=options.times),
                          conf['local_path'], recursive=True)
        display('Watching for local path "%s", sync to "%s:%s".' %
                (conf['local_path'], conf['remote_host'], conf['remote_path']))

    observer.start()
    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        observer.stop()
    observer.join()

def full_sync(conf_list):
    for conf in conf_list:
        display("Full sync from '%s' to %s:%s" % (conf['local_path'],
                conf['remote_host'], conf['remote_path']))
        cmd = " rsync -azq --delete %s %s:%s " % \
            (conf['local_path'], conf['remote_host'], conf['remote_path'])
        print cmd
        os.system(cmd)

def setup():
    script = os.path.abspath(sys.argv[0])
    dest = '/usr/local/bin/xsync'
    display('Link "%s" to %s' % (script, dest))
    cmd = 'chmod +x %s && sudo ln -s -f "%s" %s' % (script, script, dest)
    os.system(cmd)

def init():
    filepath = os.path.join(os.getcwd(), '.xsync')
    if not os.path.isfile(filepath):
        with open(filepath, 'w') as conf:
            conf.write("""\
{
  "local_path": "%s/",
  "remote_host": "USER@YOUHOST",
  "remote_path": "/home/USER/REMOTE_PATH/",
  "ignore_list": [".git", ".svn", ".DS_Store"]
}
""" % (os.getcwd()))

    os.system('vi %s' % filepath)

def main():
    args = parse_opt()

    if args.setup:
        setup()
        return

    if args.init:
        init()
        return

    conf_list = parse_conf(args.conf)

    if len(conf_list) == 0:
        return

    if args.full:
        full_sync(conf_list)
    else:
        watch(conf_list, args)

if __name__ == '__main__':
    main()
<commit_msg>support ' ' in file path<commit_after>#!/usr/bin/env python
"""
xsync.py by zjx20
http://github.com/zjx20/xsync/

This script will watch a local directory and on change will
sync to a remote directory. The script can be easily modified
to do whatever you want on a change event.

requires: pip install watchdog

  about watchdog:
    # project site: http://github.com/gorakhargosh/watchdog
    # api document: https://pythonhosted.org/watchdog/index.html

TODO: enhance 'ignore list' feature
"""

import os, datetime, time
import sys, argparse, json
import watchdog.events, watchdog.observers

class SyncHandler(watchdog.events.FileSystemEventHandler):

    def __init__(self, conf, times=False):
        watchdog.events.FileSystemEventHandler.__init__(self)
        self.local_path = conf['local_path']
        self.remote_host = conf['remote_host']
        self.remote_path = conf['remote_path']
        self.ignore_list = []
        self.times = times
        if 'ignore_list' in conf:
            self.ignore_list = conf['ignore_list']
        self.ignore_list += ['.xsync']    # ignore .xsync by default

    def should_ignore(self, filename):
        for ig in self.ignore_list:
            if ig in filename:
                return True
        return False

    def on_created(self, event):
        filename = event.src_path
        if self.should_ignore(filename):
            return

        if event.is_directory:
            filename += '/'

        remote_file = filename.replace(self.local_path, '')
        remote_parent = os.path.dirname(remote_file)

        # -lptgoD is almostly equal to -a except -r
        rsync_args = '-lpgoDzq' + ('t' if self.times else '')
        cmd = " rsync %s \"%s\" \"%s:%s%s/\" " % \
            (rsync_args, filename, self.remote_host,
             self.remote_path.replace(' ', '\\\\ '),
             remote_parent.replace(' ', '\\\\ '))
        display("Syncing %s " % filename)
        os.system(cmd)

    def on_deleted(self, event):
        filename = event.src_path
        if self.should_ignore(filename):
            return

        local_parent = os.path.dirname(filename) + '/'
        if not os.path.isdir(local_parent):
            # the parent dir does not exists, skip
            return

        remote_file = filename.replace(self.local_path, '')

        cmd = " ssh %s 'rm -rf \"%s%s\"'" % \
            (self.remote_host, self.remote_path, remote_file)
        display("Syncing %s " % filename)
        os.system(cmd)

    def on_modified(self, event):
        if isinstance(event, watchdog.events.DirModifiedEvent):
            # ignore dir modified event
            return

        filename = event.src_path
        if self.should_ignore(filename):
            return

        remote_file = filename.replace(self.local_path, '')

        # -lptgoDr is equal to -a
        rsync_args = '-lpgoDrzq' + ('t' if self.times else '')
        cmd = " rsync %s --delete \"%s\" \"%s:%s%s\" " % \
            (rsync_args, filename, self.remote_host,
             self.remote_path.replace(' ', '\\\\ '),
             remote_file.replace(' ', '\\\\ '))
        display("Syncing %s " % filename)
        os.system(cmd)

    def on_moved(self, event):
        if event.is_directory:
            self.on_deleted(watchdog.events.DirDeletedEvent(event.src_path))
            self.on_created(watchdog.events.DirCreatedEvent(event.dest_path))
        else:
            self.on_deleted(watchdog.events.FileDeletedEvent(event.src_path))
            self.on_created(watchdog.events.FileCreatedEvent(event.dest_path))

def display(str):
    now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    print "[{0}] {1}".format(now, str)

def parse_opt():
    parser = argparse.ArgumentParser(prog='xsync',
            formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('--setup', action='store_true',
            help='cp xsync to /usr/local/bin')
    parser.add_argument('--full', action='store_true',
            help='do full sync instead of starting a daemon')
    parser.add_argument('--conf', action='store',
            default=os.path.join(os.getcwd(), '.xsync'),
            help='config file path')
    parser.add_argument('--init', action='store_true',
            help='create .xsync file in current folder')
    parser.add_argument('--times', action='store_true',
            help='preserve modification times')
    args = parser.parse_args()
    return args

"""
Config sample:

  {
    "local_path": "/home/x/project1/",
    "remote_host": "dev@111.222.111.222",
    "remote_path": "/home/dev/project1/",
    "ignore_list": [".git", ".svn", ".DS_Store"]
  }

  #
  # NOTICE:
  #
  #   it will use dirname of config file path as 'local_path' if
  #   the field does not exists.
  #
  #   if you are meeting a non-standard ssh port, try
  #   "--rsh='ssh -p2222' x@127.0.0.1" as 'remote_host'.
  #
  #   'ignore_list' is optional.
  #

  Or config blocks in an array

  [
    {
      "local_path": "/home/x/project1/",
      "remote_host": "dev@111.222.111.222",
      "remote_path": "/home/dev/project1/",
      "ignore_list": [".git", ".svn", ".DS_Store"]
    },
    {
      ...
    },
  ]
"""
def parse_conf(filepath):
    if not os.path.isfile(filepath):
        print >> sys.stderr, '[WARNING] Config "' + filepath + '" not ' + \
            'exists! skipped.'
        return []

    conf = None
    with open(filepath) as f:
        try:
            conf = json.loads(f.read())
        except ValueError:
            print >> sys.stderr, '[WARNING] Couldn\'t parse config ' + \
                'from "%s"! skipped.' % filepath
            return []

    conf_list = conf
    if not isinstance(conf, list):
        conf_list = [conf]

    for conf in conf_list:
        if 'local_path' not in conf:
            conf['local_path'] = '%s/' % \
                os.path.dirname(os.path.abspath(filepath))
        if not conf['local_path'].endswith('/'):
            conf['local_path'] += '/'
        if not conf['remote_path'].endswith('/'):
            conf['remote_path'] += '/'

    return conf_list

def watch(conf_list, options):
    observer = watchdog.observers.Observer()

    for conf in conf_list:
        observer.schedule(SyncHandler(conf, times=options.times),
                          conf['local_path'], recursive=True)
        display('Watching for local path "%s", sync to "%s:%s".' %
                (conf['local_path'], conf['remote_host'], conf['remote_path']))

    observer.start()
    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        observer.stop()
    observer.join()

def full_sync(conf_list):
    for conf in conf_list:
        display("Full sync from '%s' to %s:%s" % (conf['local_path'],
                conf['remote_host'], conf['remote_path']))
        cmd = " rsync -azq --delete %s %s:%s " % \
            (conf['local_path'], conf['remote_host'], conf['remote_path'])
        print cmd
        os.system(cmd)

def setup():
    script = os.path.abspath(sys.argv[0])
    dest = '/usr/local/bin/xsync'
    display('Link "%s" to %s' % (script, dest))
    cmd = 'chmod +x %s && sudo ln -s -f "%s" %s' % (script, script, dest)
    os.system(cmd)

def init():
    filepath = os.path.join(os.getcwd(), '.xsync')
    if not os.path.isfile(filepath):
        with open(filepath, 'w') as conf:
            conf.write("""\
{
  "local_path": "%s/",
  "remote_host": "USER@YOUHOST",
  "remote_path": "/home/USER/REMOTE_PATH/",
  "ignore_list": [".git", ".svn", ".DS_Store"]
}
""" % (os.getcwd()))

    os.system('vi %s' % filepath)

def main():
    args = parse_opt()

    if args.setup:
        setup()
        return

    if args.init:
        init()
        return

    conf_list = parse_conf(args.conf)

    if len(conf_list) == 0:
        return

    if args.full:
        full_sync(conf_list)
    else:
        watch(conf_list, args)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Doc serving from Python.

In production there are two modes,
* Serving from public symlinks in nginx (readthedocs.org & readthedocs.com)
* Serving from private symlinks in Python (readthedocs.com only)

In development, we have two modes:
* Serving from public symlinks in Python
* Serving from private symlinks in Python

This means we should only serve from public symlinks in dev,
and generally default to serving from private symlinks in Python only.

Privacy
-------

These views will take into account the version privacy level.

Settings
--------

PYTHON_MEDIA (False) - Set this to True to serve docs & media from Python
SERVE_DOCS (['private']) - The list of ['private', 'public'] docs to serve.
"""

import logging
import mimetypes
import os
from functools import wraps

from django.conf import settings
from django.http import Http404, HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404, render
from django.utils.encoding import iri_to_uri
from django.views.static import serve

from readthedocs.builds.models import Version
from readthedocs.core.permissions import AdminPermission
from readthedocs.core.resolver import resolve, resolve_path
from readthedocs.core.symlink import PrivateSymlink, PublicSymlink
from readthedocs.projects import constants
from readthedocs.projects.models import Project, ProjectRelationship

log = logging.getLogger(__name__)

def map_subproject_slug(view_func):
    """
    A decorator that maps a ``subproject_slug`` URL param into a Project.

    :raises: Http404 if the Project doesn't exist

    .. warning:: Does not take into account any kind of privacy settings.
    """

    @wraps(view_func)
    def inner_view(  # noqa
            request, subproject=None, subproject_slug=None, *args, **kwargs,
    ):
        if subproject is None and subproject_slug:
            # Try to fetch by subproject alias first, otherwise we might end up
            # redirected to an unrelated project.
            try:
                # Depends on a project passed into kwargs
                rel = ProjectRelationship.objects.get(
                    parent=kwargs['project'],
                    alias=subproject_slug,
                )
                subproject = rel.child
            except (ProjectRelationship.DoesNotExist, KeyError):
                subproject = get_object_or_404(Project, slug=subproject_slug)
        return view_func(request, subproject=subproject, *args, **kwargs)

    return inner_view

def map_project_slug(view_func):
    """
    A decorator that maps a ``project_slug`` URL param into a Project.

    :raises: Http404 if the Project doesn't exist

    .. warning:: Does not take into account any kind of privacy settings.
    """

    @wraps(view_func)
    def inner_view(  # noqa
            request, project=None, project_slug=None, *args, **kwargs
    ):
        if project is None:
            if not project_slug:
                project_slug = request.slug
            try:
                project = Project.objects.get(slug=project_slug)
            except Project.DoesNotExist:
                raise Http404('Project does not exist.')
        return view_func(request, project=project, *args, **kwargs)

    return inner_view

@map_project_slug
@map_subproject_slug
def redirect_project_slug(request, project, subproject):  # pylint: disable=unused-argument
    """Handle / -> /en/latest/ directs on subdomains."""
    return HttpResponseRedirect(resolve(subproject or project))

@map_project_slug
@map_subproject_slug
def redirect_page_with_filename(request, project, subproject, filename):  # pylint: disable=unused-argument  # noqa
    """Redirect /page/file.html to /en/latest/file.html."""
    return HttpResponseRedirect(
        resolve(subproject or project, filename=filename),
    )

def _serve_401(request, project):
    res = render(request, '401.html')
    res.status_code = 401
    log.debug('Unauthorized access to {} documentation'.format(project.slug))
    return res

def _serve_file(request, filename, basepath):
    # Serve the file from the proper location
    if settings.DEBUG or getattr(settings, 'PYTHON_MEDIA', False):
        # Serve from Python
        return serve(request, filename, basepath)

    # Serve from Nginx
    content_type, encoding = mimetypes.guess_type(
        os.path.join(basepath, filename),
    )
    content_type = content_type or 'application/octet-stream'
    response = HttpResponse(content_type=content_type)
    if encoding:
        response['Content-Encoding'] = encoding
    try:
        iri_path = os.path.join(
            basepath[len(settings.SITE_ROOT):],
            filename,
        )
        # NGINX does not support non-ASCII characters in the header, so we
        # convert the IRI path to URI so it's compatible with what NGINX expects
        # as the header value.
        # https://github.com/benoitc/gunicorn/issues/1448
        # https://docs.djangoproject.com/en/1.11/ref/unicode/#uri-and-iri-handling
        x_accel_redirect = iri_to_uri(iri_path)
        response['X-Accel-Redirect'] = x_accel_redirect
    except UnicodeEncodeError:
        raise Http404

    return response

@map_project_slug
@map_subproject_slug
def serve_docs(
        request,
        project,
        subproject,
        lang_slug=None,
        version_slug=None,
        filename='',
):
    """Map existing proj, lang, version, filename views to the file format."""
    if not version_slug:
        version_slug = project.get_default_version()
    try:
        version = project.versions.public(request.user).get(slug=version_slug)
    except Version.DoesNotExist:
        # Properly raise a 404 if the version doesn't exist (or is inactive) and
        # a 401 if it does
        if project.versions.filter(slug=version_slug, active=True).exists():
            return _serve_401(request, project)
        raise Http404('Version does not exist.')
    filename = resolve_path(
        subproject or project,  # Resolve the subproject if it exists
        version_slug=version_slug,
        language=lang_slug,
        filename=filename,
        subdomain=True,  # subdomain will make it a "full" path without a URL prefix
    )
    if (version.privacy_level == constants.PRIVATE and
            not AdminPermission.is_member(user=request.user, obj=project)):
        return _serve_401(request, project)
    return _serve_symlink_docs(
        request,
        filename=filename,
        project=project,
        privacy_level=version.privacy_level,
    )

@map_project_slug
def _serve_symlink_docs(request, project, privacy_level, filename=''):
    """Serve a file by symlink, or a 404 if not found."""
    # Handle indexes
    if filename == '' or filename[-1] == '/':
        filename += 'index.html'

    # This breaks path joining, by ignoring the root when given an "absolute" path
    if filename[0] == '/':
        filename = filename[1:]

    log.info('Serving %s for %s', filename, project)

    files_tried = []

    serve_docs = getattr(settings, 'SERVE_DOCS', [constants.PRIVATE])

    if (settings.DEBUG or constants.PUBLIC in serve_docs) and privacy_level != constants.PRIVATE:  # yapf: disable  # noqa
        public_symlink = PublicSymlink(project)
        basepath = public_symlink.project_root
        if os.path.exists(os.path.join(basepath, filename)):
            return _serve_file(request, filename, basepath)

        files_tried.append(os.path.join(basepath, filename))

    if (settings.DEBUG or constants.PRIVATE in serve_docs) and privacy_level == constants.PRIVATE:  # yapf: disable  # noqa
        # Handle private
        private_symlink = PrivateSymlink(project)
        basepath = private_symlink.project_root

        if os.path.exists(os.path.join(basepath, filename)):
            return _serve_file(request, filename, basepath)

        files_tried.append(os.path.join(basepath, filename))

    raise Http404(
        'File not found. Tried these files: %s' % ','.join(files_tried),
    )

@map_project_slug
def robots_txt(request, project):
    """
    Serve custom user's defined ``/robots.txt``.

    If the user added a ``robots.txt`` in the "default version" of the project,
    we serve it directly.
    """
    # Use the ``robots.txt`` file from the default version configured
    version_slug = project.get_default_version()
    version = project.versions.get(slug=version_slug)

    no_serve_robots_txt = any([
        # If project is private or,
        project.privacy_level == constants.PRIVATE,
        # default version is private or,
        version.privacy_level == constants.PRIVATE,
        # default version is not active or,
        not version.active,
        # default version is not built
        not version.built,
    ])
    if no_serve_robots_txt:
        # ... we do return a 404
        raise Http404()

    filename = resolve_path(
        project,
        version_slug=version_slug,
        filename='robots.txt',
        subdomain=True,  # subdomain will make it a "full" path without a URL prefix
    )

    # This breaks path joining, by ignoring the root when given an "absolute" path
    if filename[0] == '/':
        filename = filename[1:]

    basepath = PublicSymlink(project).project_root
    fullpath = os.path.join(basepath, filename)

    if os.path.exists(fullpath):
        return HttpResponse(open(fullpath).read(), content_type='text/plain')

    return HttpResponse('User-agent: *\nAllow: /\n', content_type='text/plain')
<commit_msg>Invalid syntax mistake caused by comma in python3<commit_after># -*- coding: utf-8 -*-

"""
Doc serving from Python.

In production there are two modes,
* Serving from public symlinks in nginx (readthedocs.org & readthedocs.com)
* Serving from private symlinks in Python (readthedocs.com only)

In development, we have two modes:
* Serving from public symlinks in Python
* Serving from private symlinks in Python

This means we should only serve from public symlinks in dev,
and generally default to serving from private symlinks in Python only.

Privacy
-------

These views will take into account the version privacy level.

Settings
--------

PYTHON_MEDIA (False) - Set this to True to serve docs & media from Python
SERVE_DOCS (['private']) - The list of ['private', 'public'] docs to serve.
"""

import logging
import mimetypes
import os
from functools import wraps

from django.conf import settings
from django.http import Http404, HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404, render
from django.utils.encoding import iri_to_uri
from django.views.static import serve

from readthedocs.builds.models import Version
from readthedocs.core.permissions import AdminPermission
from readthedocs.core.resolver import resolve, resolve_path
from readthedocs.core.symlink import PrivateSymlink, PublicSymlink
from readthedocs.projects import constants
from readthedocs.projects.models import Project, ProjectRelationship

log = logging.getLogger(__name__)

def map_subproject_slug(view_func):
    """
    A decorator that maps a ``subproject_slug`` URL param into a Project.

    :raises: Http404 if the Project doesn't exist

    .. warning:: Does not take into account any kind of privacy settings.
    """

    @wraps(view_func)
    def inner_view(  # noqa
            request, subproject=None, subproject_slug=None, *args, **kwargs
    ):
        if subproject is None and subproject_slug:
            # Try to fetch by subproject alias first, otherwise we might end up
            # redirected to an unrelated project.
            try:
                # Depends on a project passed into kwargs
                rel = ProjectRelationship.objects.get(
                    parent=kwargs['project'],
                    alias=subproject_slug,
                )
                subproject = rel.child
            except (ProjectRelationship.DoesNotExist, KeyError):
                subproject = get_object_or_404(Project, slug=subproject_slug)
        return view_func(request, subproject=subproject, *args, **kwargs)

    return inner_view

def map_project_slug(view_func):
    """
    A decorator that maps a ``project_slug`` URL param into a Project.

    :raises: Http404 if the Project doesn't exist

    .. warning:: Does not take into account any kind of privacy settings.
    """

    @wraps(view_func)
    def inner_view(  # noqa
            request, project=None, project_slug=None, *args, **kwargs
    ):
        if project is None:
            if not project_slug:
                project_slug = request.slug
            try:
                project = Project.objects.get(slug=project_slug)
            except Project.DoesNotExist:
                raise Http404('Project does not exist.')
        return view_func(request, project=project, *args, **kwargs)

    return inner_view

@map_project_slug
@map_subproject_slug
def redirect_project_slug(request, project, subproject):  # pylint: disable=unused-argument
    """Handle / -> /en/latest/ directs on subdomains."""
    return HttpResponseRedirect(resolve(subproject or project))

@map_project_slug
@map_subproject_slug
def redirect_page_with_filename(request, project, subproject, filename):  # pylint: disable=unused-argument  # noqa
    """Redirect /page/file.html to /en/latest/file.html."""
    return HttpResponseRedirect(
        resolve(subproject or project, filename=filename),
    )

def _serve_401(request, project):
    res = render(request, '401.html')
    res.status_code = 401
    log.debug('Unauthorized access to {} documentation'.format(project.slug))
    return res

def _serve_file(request, filename, basepath):
    # Serve the file from the proper location
    if settings.DEBUG or getattr(settings, 'PYTHON_MEDIA', False):
        # Serve from Python
        return serve(request, filename, basepath)

    # Serve from Nginx
    content_type, encoding = mimetypes.guess_type(
        os.path.join(basepath, filename),
    )
    content_type = content_type or 'application/octet-stream'
    response = HttpResponse(content_type=content_type)
    if encoding:
        response['Content-Encoding'] = encoding
    try:
        iri_path = os.path.join(
            basepath[len(settings.SITE_ROOT):],
            filename,
        )
        # NGINX does not support non-ASCII characters in the header, so we
        # convert the IRI path to URI so it's compatible with what NGINX expects
        # as the header value.
        # https://github.com/benoitc/gunicorn/issues/1448
        # https://docs.djangoproject.com/en/1.11/ref/unicode/#uri-and-iri-handling
        x_accel_redirect = iri_to_uri(iri_path)
        response['X-Accel-Redirect'] = x_accel_redirect
    except UnicodeEncodeError:
        raise Http404

    return response

@map_project_slug
@map_subproject_slug
def serve_docs(
        request,
        project,
        subproject,
        lang_slug=None,
        version_slug=None,
        filename='',
):
    """Map existing proj, lang, version, filename views to the file format."""
    if not version_slug:
        version_slug = project.get_default_version()
    try:
        version = project.versions.public(request.user).get(slug=version_slug)
    except Version.DoesNotExist:
        # Properly raise a 404 if the version doesn't exist (or is inactive) and
        # a 401 if it does
        if project.versions.filter(slug=version_slug, active=True).exists():
            return _serve_401(request, project)
        raise Http404('Version does not exist.')
    filename = resolve_path(
        subproject or project,  # Resolve the subproject if it exists
        version_slug=version_slug,
        language=lang_slug,
        filename=filename,
        subdomain=True,  # subdomain will make it a "full" path without a URL prefix
    )
    if (version.privacy_level == constants.PRIVATE and
            not AdminPermission.is_member(user=request.user, obj=project)):
        return _serve_401(request, project)
    return _serve_symlink_docs(
        request,
        filename=filename,
        project=project,
        privacy_level=version.privacy_level,
    )

@map_project_slug
def _serve_symlink_docs(request, project, privacy_level, filename=''):
    """Serve a file by symlink, or a 404 if not found."""
    # Handle indexes
    if filename == '' or filename[-1] == '/':
        filename += 'index.html'

    # This breaks path joining, by ignoring the root when given an "absolute" path
    if filename[0] == '/':
        filename = filename[1:]

    log.info('Serving %s for %s', filename, project)

    files_tried = []

    serve_docs = getattr(settings, 'SERVE_DOCS', [constants.PRIVATE])

    if (settings.DEBUG or constants.PUBLIC in serve_docs) and privacy_level != constants.PRIVATE:  # yapf: disable  # noqa
        public_symlink = PublicSymlink(project)
        basepath = public_symlink.project_root
        if os.path.exists(os.path.join(basepath, filename)):
            return _serve_file(request, filename, basepath)

        files_tried.append(os.path.join(basepath, filename))

    if (settings.DEBUG or constants.PRIVATE in serve_docs) and privacy_level == constants.PRIVATE:  # yapf: disable  # noqa
        # Handle private
        private_symlink = PrivateSymlink(project)
        basepath = private_symlink.project_root

        if os.path.exists(os.path.join(basepath, filename)):
            return _serve_file(request, filename, basepath)

        files_tried.append(os.path.join(basepath, filename))

    raise Http404(
        'File not found. Tried these files: %s' % ','.join(files_tried),
    )

@map_project_slug
def robots_txt(request, project):
    """
    Serve custom user's defined ``/robots.txt``.

    If the user added a ``robots.txt`` in the "default version" of the project,
    we serve it directly.
    """
    # Use the ``robots.txt`` file from the default version configured
    version_slug = project.get_default_version()
    version = project.versions.get(slug=version_slug)

    no_serve_robots_txt = any([
        # If project is private or,
        project.privacy_level == constants.PRIVATE,
        # default version is private or,
        version.privacy_level == constants.PRIVATE,
        # default version is not active or,
        not version.active,
        # default version is not built
        not version.built,
    ])
    if no_serve_robots_txt:
        # ... we do return a 404
        raise Http404()

    filename = resolve_path(
        project,
        version_slug=version_slug,
        filename='robots.txt',
        subdomain=True,  # subdomain will make it a "full" path without a URL prefix
    )

    # This breaks path joining, by ignoring the root when given an "absolute" path
    if filename[0] == '/':
        filename = filename[1:]

    basepath = PublicSymlink(project).project_root
    fullpath = os.path.join(basepath, filename)

    if os.path.exists(fullpath):
        return HttpResponse(open(fullpath).read(), content_type='text/plain')

    return HttpResponse('User-agent: *\nAllow: /\n', content_type='text/plain')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Django models for the redirects app."""

import logging
import re

from django.db import models
from django.utils.translation import ugettext
from django.utils.translation import ugettext_lazy as _

from readthedocs.core.resolver import resolve_path
from readthedocs.projects.models import Project

from .querysets import RedirectQuerySet

log = logging.getLogger(__name__)

HTTP_STATUS_CHOICES = (
    (301, _('301 - Permanent Redirect')),
    (302, _('302 - Temporary Redirect')),
)

STATUS_CHOICES = (
    (True, _('Active')),
    (False, _('Inactive')),
)

TYPE_CHOICES = (
    ('prefix', _('Prefix Redirect')),
    ('page', _('Page Redirect')),
    ('exact', _('Exact Redirect')),
    ('sphinx_html', _('Sphinx HTMLDir -> HTML')),
    ('sphinx_htmldir', _('Sphinx HTML -> HTMLDir')),
    # ('advanced', _('Advanced')),
)

# FIXME: this help_text message should be dynamic since "Absolute path" doesn't
# make sense for "Prefix Redirects" since the from URL is considered after the
# ``/$lang/$version/`` part. Also, there is a feature for the "Exact
# Redirects" that should be mentioned here: the usage of ``$rest``
from_url_helptext = _(
    'Absolute path, excluding the domain. '
    'Example: /docs/  or /install.html',
)
to_url_helptext = _(
    'Absolute or relative URL. Example: '
    '/tutorial/install.html',
)
redirect_type_helptext = _('The type of redirect you wish to use.')

class Redirect(models.Model):

    """A HTTP redirect associated with a Project."""

    project = models.ForeignKey(
        Project,
        verbose_name=_('Project'),
        related_name='redirects',
        on_delete=models.CASCADE,
    )

    redirect_type = models.CharField(
        _('Redirect Type'),
        max_length=255,
        choices=TYPE_CHOICES,
        help_text=redirect_type_helptext,
    )

    from_url = models.CharField(
        _('From URL'),
        max_length=255,
        db_index=True,
        help_text=from_url_helptext,
        blank=True,
    )

    from_url_without_rest = models.CharField(
        max_length=255,
        db_index=True,
        help_text='Only for internal querying use',
        blank=True,
        null=True,
    )

    to_url = models.CharField(
        _('To URL'),
        max_length=255,
        db_index=True,
        help_text=to_url_helptext,
        blank=True,
    )

    http_status = models.SmallIntegerField(
        _('HTTP Status'),
        choices=HTTP_STATUS_CHOICES,
        default=302,
    )
    status = models.BooleanField(choices=STATUS_CHOICES, default=True)

    create_dt = models.DateTimeField(auto_now_add=True)
    update_dt = models.DateTimeField(auto_now=True)

    objects = RedirectQuerySet.as_manager()

    class Meta:
        verbose_name = _('redirect')
        verbose_name_plural = _('redirects')
        ordering = ('-update_dt',)

    def save(self, *args, **kwargs):  # pylint: disable=arguments-differ
        if self.redirect_type == 'exact' and '$rest' in self.from_url:
            self.from_url_without_rest = self.from_url.replace('$rest', '')
        super().save(*args, **kwargs)

    def __str__(self):
        redirect_text = '{type}: {from_to_url}'
        if self.redirect_type in ['prefix', 'page', 'exact']:
            return redirect_text.format(
                type=self.get_redirect_type_display(),
                from_to_url=self.get_from_to_url_display(),
            )
        return ugettext(
            'Redirect: {}'.format(
                self.get_redirect_type_display(),
            ),
        )

    def get_from_to_url_display(self):
        if self.redirect_type in ['prefix', 'page', 'exact']:
            from_url = self.from_url
            to_url = self.to_url
            if self.redirect_type == 'prefix':
                to_url = '/{lang}/{version}/'.format(
                    lang=self.project.language,
                    version=self.project.default_version,
                )
            return '{from_url} -> {to_url}'.format(
                from_url=from_url,
                to_url=to_url,
            )
        return ''

    def get_full_path(self, filename, language=None, version_slug=None, allow_crossdomain=False):
        """
        Return a full path for a given filename.

        This will include version and language information. No protocol/domain
        is returned.
        """
        # Handle explicit http redirects
        if allow_crossdomain and re.match('^https?://', filename):
            return filename

        return resolve_path(
            project=self.project,
            language=language,
            version_slug=version_slug,
            filename=filename,
        )

    def get_redirect_path(self, path, language=None, version_slug=None):
        method = getattr(
            self,
            'redirect_{type}'.format(
                type=self.redirect_type,
            ),
        )
        return method(path, language=language, version_slug=version_slug)

    def redirect_prefix(self, path, language=None, version_slug=None):
        if path.startswith(self.from_url):
            log.debug('Redirecting %s', self)
            cut_path = re.sub('^%s' % self.from_url, '', path)

            to = self.get_full_path(
                filename=cut_path,
                language=language,
                version_slug=version_slug,
                allow_crossdomain=False,
            )
            return to

    def redirect_page(self, path, language=None, version_slug=None):
        if path == self.from_url:
            log.debug('Redirecting %s', self)
            to = self.get_full_path(
                filename=self.to_url.lstrip('/'),
                language=language,
                version_slug=version_slug,
                allow_crossdomain=True,
            )
            return to

    def redirect_exact(self, path, language=None, version_slug=None):
        full_path = path
        if language and version_slug:
            # reconstruct the full path for an exact redirect
            full_path = self.get_full_path(path, language, version_slug, allow_crossdomain=False)
        if full_path == self.from_url:
            log.debug('Redirecting %s', self)
            return self.to_url
        # Handle full sub-level redirects
        if '$rest' in self.from_url:
            match = self.from_url.split('$rest')[0]
            if full_path.startswith(match):
                cut_path = re.sub('^%s' % match, self.to_url, full_path)
                return cut_path

    def redirect_sphinx_html(self, path, language=None, version_slug=None):
        for ending in ['/', '/index.html']:
            if path.endswith(ending):
                log.debug('Redirecting %s', self)
                path = path[1:]  # Strip leading slash.
                to = re.sub(ending + '$', '.html', path)
                return self.get_full_path(
                    filename=to,
                    language=language,
                    version_slug=version_slug,
                    allow_crossdomain=False,
                )

    def redirect_sphinx_htmldir(self, path, language=None, version_slug=None):
        if path.endswith('.html'):
            log.debug('Redirecting %s', self)
            path = path[1:]  # Strip leading slash.
            to = re.sub('.html$', '/', path)
            return self.get_full_path(
                filename=to,
                language=language,
                version_slug=version_slug,
                allow_crossdomain=False,
            )
<commit_msg>Add comment explaining why/where is this needed<commit_after># -*- coding: utf-8 -*-

"""Django models for the redirects app."""

import logging
import re

from django.db import models
from django.utils.translation import ugettext
from django.utils.translation import ugettext_lazy as _

from readthedocs.core.resolver import resolve_path
from readthedocs.projects.models import Project

from .querysets import RedirectQuerySet

log = logging.getLogger(__name__)

HTTP_STATUS_CHOICES = (
    (301, _('301 - Permanent Redirect')),
    (302, _('302 - Temporary Redirect')),
)

STATUS_CHOICES = (
    (True, _('Active')),
    (False, _('Inactive')),
)

TYPE_CHOICES = (
    ('prefix', _('Prefix Redirect')),
    ('page', _('Page Redirect')),
    ('exact', _('Exact Redirect')),
    ('sphinx_html', _('Sphinx HTMLDir -> HTML')),
    ('sphinx_htmldir', _('Sphinx HTML -> HTMLDir')),
    # ('advanced', _('Advanced')),
)

# FIXME: this help_text message should be dynamic since "Absolute path" doesn't
# make sense for "Prefix Redirects" since the from URL is considered after the
# ``/$lang/$version/`` part. Also, there is a feature for the "Exact
# Redirects" that should be mentioned here: the usage of ``$rest``
from_url_helptext = _(
    'Absolute path, excluding the domain. '
    'Example: /docs/  or /install.html',
)
to_url_helptext = _(
    'Absolute or relative URL. Example: '
    '/tutorial/install.html',
)
redirect_type_helptext = _('The type of redirect you wish to use.')

class Redirect(models.Model):

    """A HTTP redirect associated with a Project."""

    project = models.ForeignKey(
        Project,
        verbose_name=_('Project'),
        related_name='redirects',
        on_delete=models.CASCADE,
    )

    redirect_type = models.CharField(
        _('Redirect Type'),
        max_length=255,
        choices=TYPE_CHOICES,
        help_text=redirect_type_helptext,
    )

    from_url = models.CharField(
        _('From URL'),
        max_length=255,
        db_index=True,
        help_text=from_url_helptext,
        blank=True,
    )

    # We are denormalizing the database here to easily query for Exact Redirects
    # with ``$rest`` on them from El Proxito
    from_url_without_rest = models.CharField(
        max_length=255,
        db_index=True,
        help_text='Only for internal querying use',
        blank=True,
        null=True,
    )

    to_url = models.CharField(
        _('To URL'),
        max_length=255,
        db_index=True,
        help_text=to_url_helptext,
        blank=True,
    )

    http_status = models.SmallIntegerField(
        _('HTTP Status'),
        choices=HTTP_STATUS_CHOICES,
        default=302,
    )
    status = models.BooleanField(choices=STATUS_CHOICES, default=True)

    create_dt = models.DateTimeField(auto_now_add=True)
    update_dt = models.DateTimeField(auto_now=True)

    objects = RedirectQuerySet.as_manager()

    class Meta:
        verbose_name = _('redirect')
        verbose_name_plural = _('redirects')
        ordering = ('-update_dt',)

    def save(self, *args, **kwargs):  # pylint: disable=arguments-differ
        if self.redirect_type == 'exact' and '$rest' in self.from_url:
            self.from_url_without_rest = self.from_url.replace('$rest', '')
        super().save(*args, **kwargs)

    def __str__(self):
        redirect_text = '{type}: {from_to_url}'
        if self.redirect_type in ['prefix', 'page', 'exact']:
            return redirect_text.format(
                type=self.get_redirect_type_display(),
                from_to_url=self.get_from_to_url_display(),
            )
        return ugettext(
            'Redirect: {}'.format(
                self.get_redirect_type_display(),
            ),
        )

    def get_from_to_url_display(self):
        if self.redirect_type in ['prefix', 'page', 'exact']:
            from_url = self.from_url
            to_url = self.to_url
            if self.redirect_type == 'prefix':
                to_url = '/{lang}/{version}/'.format(
                    lang=self.project.language,
                    version=self.project.default_version,
                )
            return '{from_url} -> {to_url}'.format(
                from_url=from_url,
                to_url=to_url,
            )
        return ''

    def get_full_path(self, filename, language=None, version_slug=None, allow_crossdomain=False):
        """
        Return a full path for a given filename.

        This will include version and language information. No protocol/domain
        is returned.
        """
        # Handle explicit http redirects
        if allow_crossdomain and re.match('^https?://', filename):
            return filename

        return resolve_path(
            project=self.project,
            language=language,
            version_slug=version_slug,
            filename=filename,
        )

    def get_redirect_path(self, path, language=None, version_slug=None):
        method = getattr(
            self,
            'redirect_{type}'.format(
                type=self.redirect_type,
            ),
        )
        return method(path, language=language, version_slug=version_slug)

    def redirect_prefix(self, path, language=None, version_slug=None):
        if path.startswith(self.from_url):
            log.debug('Redirecting %s', self)
            cut_path = re.sub('^%s' % self.from_url, '', path)

            to = self.get_full_path(
                filename=cut_path,
                language=language,
                version_slug=version_slug,
                allow_crossdomain=False,
            )
            return to

    def redirect_page(self, path, language=None, version_slug=None):
        if path == self.from_url:
            log.debug('Redirecting %s', self)
            to = self.get_full_path(
                filename=self.to_url.lstrip('/'),
                language=language,
                version_slug=version_slug,
                allow_crossdomain=True,
            )
            return to

    def redirect_exact(self, path, language=None, version_slug=None):
        full_path = path
        if language and version_slug:
            # reconstruct the full path for an exact redirect
            full_path = self.get_full_path(path, language, version_slug, allow_crossdomain=False)
        if full_path == self.from_url:
            log.debug('Redirecting %s', self)
            return self.to_url
        # Handle full sub-level redirects
        if '$rest' in self.from_url:
            match = self.from_url.split('$rest')[0]
            if full_path.startswith(match):
                cut_path = re.sub('^%s' % match, self.to_url, full_path)
                return cut_path

    def redirect_sphinx_html(self, path, language=None, version_slug=None):
        for ending in ['/', '/index.html']:
            if path.endswith(ending):
                log.debug('Redirecting %s', self)
                path = path[1:]  # Strip leading slash.
                to = re.sub(ending + '$', '.html', path)
                return self.get_full_path(
                    filename=to,
                    language=language,
                    version_slug=version_slug,
                    allow_crossdomain=False,
                )

    def redirect_sphinx_htmldir(self, path, language=None, version_slug=None):
        if path.endswith('.html'):
            log.debug('Redirecting %s', self)
            path = path[1:]  # Strip leading slash.
            to = re.sub('.html$', '/', path)
            return self.get_full_path(
                filename=to,
                language=language,
                version_slug=version_slug,
                allow_crossdomain=False,
            )
<|endoftext|>


<commit_before>"""Category operations used by neo4j graph rewriting tool."""

import regraph.neo4j.cypher_utils as cypher

def pullback(b, c, d, a=None, inplace=False):
    """Find the pullback from b -> d <- c."""
    if a is None:
        a = "pb_" + "_".join([b, c, d])
    query = ""
    carry_vars = set()

    # Match all the pair of nodes with the same image in d
    query +=\
        "OPTIONAL MATCH (n:{})-[:typing]->(:{})<-[:typing]-(m:{})\n".format(
            b, d, c)

    # For each pair, collect all the merged properties
    query += cypher.merge_properties(["n", "m"], 'new_props',
                                     method='intersection',
                                     carry_vars=carry_vars)
    # For each pair, create a new node
    query += cypher.create_node(
                        var_name="new_node_a",
                        node_id="pb",
                        node_id_var="id_var",
                        label=a,
                        carry_vars=carry_vars,
                        ignore_naming=True)[0]
    carry_vars.remove("id_var")
    query += "SET new_node_a += new_props\n"
    carry_vars.remove("new_props")

    # Add the typing edges
    query += cypher.with_vars(carry_vars) + "\n"
    query += cypher.create_edge('new_node_a', 'n', edge_label='typing') + "\n"
    query += cypher.create_edge('new_node_a', 'm', edge_label='typing') + "\n"

    # Add the graph edges
    carry_vars = set()
    query2 = ""
    query2 +=\
        "MATCH (x:{})-[:typing]->(:{})-[r1:edge]->(:{})<-[:typing]-(y:{}),\n".format(
            a, b, b, a) +\
        "(x)-[:typing]->(:{})-[r2:edge]->(:{})<-[:typing]-(y)\n".format(
            c, c)
    # Collect all the merged properties of the edges r1 and r2
    query2 += cypher.merge_properties(["r1", "r2"], 'new_props',
                                      method='intersection',
                                      carry_vars={'x', 'y'})
    query2 += "MERGE (x)-[r:edge]->(y)\n"
    query2 += "SET r += new_props"

    return query, query2

def pushout(a, b, c, d=None, inplace=False):
    pass

def pullback_complement(a, b, d, c=None, inplace=False):
    pass

def check_homomorphism(domain, codomain, total=True):
    pass
<commit_msg>Pushout: first version<commit_after>"""Category operations used by neo4j graph rewriting tool."""

import regraph.neo4j.cypher_utils as cypher

def pullback(b, c, d, a=None, inplace=False):
    """Find the pullback from b -> d <- c."""
    if a is None:
        a = "pb_" + "_".join([b, c, d])
    query1 = ""
    carry_vars = set()

    # Match all the pair of nodes with the same image in d
    query1 +=\
        "OPTIONAL MATCH (n:{})-[:typing]->(:{})<-[:typing]-(m:{})\n".format(
            b, d, c)

    # For each pair, collect all the merged properties
    query1 += cypher.merge_properties(["n", "m"], 'new_props',
                                      method='intersection',
                                      carry_vars=carry_vars)
    # For each pair, create a new node
    query1 += cypher.create_node(
                        var_name="new_node_a",
                        node_id="pb",
                        node_id_var="id_var",
                        label=a,
                        carry_vars=carry_vars,
                        ignore_naming=True)[0]
    carry_vars.remove("id_var")
    query1 += "SET new_node_a += new_props\n"
    carry_vars.remove("new_props")

    # Add the typing edges
    query1 += cypher.with_vars(carry_vars) + "\n"
    query1 += cypher.create_edge('new_node_a', 'n', edge_label='typing') + "\n"
    query1 += cypher.create_edge('new_node_a', 'm', edge_label='typing') + "\n"

    # Add the graph edges
    carry_vars = set()
    query2 = ""
    query2 +=\
        "OPTIONAL MATCH (x:{})-[:typing]->(:{})-[r1:edge]->(:{})<-[:typing]-(y:{}),\n".format(
            a, b, b, a) +\
        "(x)-[:typing]->(:{})-[r2:edge]->(:{})<-[:typing]-(y)\n".format(
            c, c)
    # Collect all the merged properties of the edges r1 and r2
    query2 += cypher.merge_properties(["r1", "r2"], 'new_props',
                                      method='intersection',
                                      carry_vars={'x', 'y'})
    query2 += "MERGE (x)-[r:edge]->(y)\n"
    query2 += "SET r += new_props"

    return query1, query2

def pushout(a, b, c, d=None, inplace=False):
    """Find the pushout of the span b <- a -> c."""
    if d is None:
        d = "pb_" + "_".join([a, b, c])
    carry_vars = set()

    query1 =\
        "OPTIONAL MATCH (n:{})\n".format(b) +\
        "WHERE n IS NOT NULL\n" +\
        "MERGE (n)-[:typing]->(m:{})\n".format(d) +\
        "SET m += properties(n)" +\
        "SET m.id = id(m)"

    query2 =\
        "//Add all the edges\n" +\
        "OPTIONAL MATCH (x:{})<-[:typing]-(:{})-[r:edge]->(:{})-[:typing]->(y:{})\n".format(
            d, b, b, d) +\
        "WITH x, y, r WHERE r IS NOT NULL\n" +\
        "MERGE (x)-[new_r:edge]->(y)\n" +\
        "SET new_r += properties(r)"
    query22 =\
        "//We search for self loops\n" +\
        "OPTIONAL MATCH (z:{})<-[:typing]-(y:{})-[r:edge]->(y)\n".format(
            d, b, b) +\
        "WITH z, r WHERE r IS NOT NULL\n" +\
        "MERGE (x)-[new_r:edge]->(x)\n" +\
        "SET new_r += properties(r)"

    query3 =\
        "OPTIONAL MATCH (n:{})\n".format(c) +\
        "WHERE n IS NOT NULL\n" +\
        "MERGE (n)-[:typing]->(m:{})\n".format(d) +\
        "SET m += properties(n)"

    query4 =\
        "OPTIONAL MATCH (x:{})<-[:typing]-(:{})-[r:edge]->(:{})-[:typing]->(y:{})\n".format(
            d, c, c, d) +\
        "WITH x, y, r WHERE r IS NOT NULL\n" +\
        "MERGE (x)-[new_r:edge]->(y)\n" +\
        "SET new_r += properties(r)"
    query44 =\
        "//We search for self loops\n" +\
        "OPTIONAL MATCH (z:{})<-[:typing]-(y:{})-[r:edge]->(y)\n".format(
            d, c, c) +\
        "WITH z, r WHERE r IS NOT NULL\n" +\
        "MERGE (z)-[new_r:edge]->(z)\n" +\
        "SET new_r += properties(r)"

    query5 =\
        "OPTIONAL MATCH (n:{})<-[:typing]-(:{})<-[:typing]-(:{})-[:typing]->(m:{})\n".format(
            d, b, a, c)

    query55 =\
        "OPTIONAL MATCH (n:{})<-[:typing]-(:{})<-[:typing]-(:{})-[:typing]->(:{})-[:typing]->(m:{})\n".format(
            d, b, a, c, d) +\
        cypher.merging_query(original_vars=["n", "m"],
                             merged_var="merged_node",
                             merged_id="id",
                             merged_id_var="new_id",
                             node_label=d,
                             edge_label=None,
                             ignore_naming=True)[0] + "\n" +\
        cypher.return_vars(['new_id'])

    return query1, query2, query22, query3, query4, query44, query5, query55

def pullback_complement(a, b, d, c=None, inplace=False):
    pass

def check_homomorphism(domain, codomain, total=True):
    pass

res = pushout('graphA', 'graphB', 'graphC', 'graphD')
print(res[0])
print('//---------------')
print(res[1])
print('//---------------')
print(res[2])
print('//---------------')
print(res[3])
print('//---------------')
print(res[4])
print('//---------------')
print(res[5])
print('//---------------')
print(res[6])
print('//---------------')
print(res[7])
print('//---------------')
<|endoftext|>


<commit_before>from regulations.generator.section_url import SectionUrl
from regulations.generator.toc import fetch_toc

def get_labels(current):
    return current.split('-')

def _add_extra(el, version):
    """Add extra fields to a TOC element -- only added to elements we will
    use for prev/next"""
    if el.get('is_section'):
        if el.get('is_section_span'):
            el['markup_prefix'] = '§§ '
        else:
            el['markup_prefix'] = '§ '
    elif el.get('is_subterp'):
        el['markup_prefix'] = 'Interpretations For '
    el['url'] = SectionUrl.of(el['index'], version, sectional=True)
    return el

def nav_sections(current, version):
    labels = get_labels(current)
    reg_part = labels[0]
    toc = fetch_toc(reg_part, version, flatten=True)
    for idx, el in enumerate(toc):
        if el['index'] == labels:
            if idx == 0:
                previous_section = None
            else:
                previous_section = _add_extra(toc[idx - 1], version)

            if idx == len(toc) - 1:
                next_section = None
            else:
                next_section = _add_extra(toc[idx + 1], version)

            return (previous_section, next_section)
    # Implicit return None if the section isn't in the TOC
<commit_msg>Flatten `if` branches to improve readbility<commit_after>from regulations.generator.section_url import SectionUrl
from regulations.generator.toc import fetch_toc

def get_labels(current):
    return current.split('-')

def _add_extra(el, version):
    """Add extra fields to a TOC element -- only added to elements we will
    use for prev/next"""
    if el.get('is_section_span'):
        el['markup_prefix'] = '§§ '
    elif el.get('is_section'):
        el['markup_prefix'] = '§ '
    elif el.get('is_subterp'):
        el['markup_prefix'] = 'Interpretations For '
    el['url'] = SectionUrl.of(el['index'], version, sectional=True)
    return el

def nav_sections(current, version):
    labels = get_labels(current)
    reg_part = labels[0]
    toc = fetch_toc(reg_part, version, flatten=True)
    for idx, el in enumerate(toc):
        if el['index'] == labels:
            if idx == 0:
                previous_section = None
            else:
                previous_section = _add_extra(toc[idx - 1], version)

            if idx == len(toc) - 1:
                next_section = None
            else:
                next_section = _add_extra(toc[idx + 1], version)

            return (previous_section, next_section)
    # Implicit return None if the section isn't in the TOC
<|endoftext|>


<commit_before>import radical.utils as ru
from radical.entk import Pipeline, Stage, Task
import os
import tarfile
import writeInputs
import time

class Replica(object):

    #A replica class to hold replica information, potentially useful for replica states in asynchronous exchange

    def __init__(self, rid, Temp, EPtot, rstate):
        self.rid    = rid     #Replica ID
        self.Temp   = Temp    #Replica Temp
        self.EPtot  = EPtot   #Replica Potential Energy
        self.rstate = rstate  #Replica State

class GROMACSTask(Task):

    # GROMACS specific MD task class

    def __init__(self, cores, mpi=True):

        super(GROMACSTask, self).__init__()
        self._executable = ['']
        self._cores      = cores
        self._pre_exec   = ['module load gromacs']
        self._post_exec = ['gmx energy < Energy.input']
        self._mpi        = mpi

class AMBERTask(Task):

    # AMBER specific MD task class.

    def __init__(self, MD_Executable, cores, mpi=False):

        super(AMBERTask, self).__init__()
        #self._executable = ['/usr/local/packages/amber/16/INTEL-140-MVAPICH2-2.0/bin/pmemd.MPI']
        self._executable = [MD_Executable]
        self._cores      = cores
        #self._pre_exec   = ['module load amber'] #For BW make a pre-exec that points to $AMBERHOME correctly  ['export AMBERHOME=$HOME/amber/amber14/']
        #self._post_exec = [''] #Post exec is not useful here, but may be useful for something like a GROMACS class...
        self._mpi        = mpi

class SynchronousExchange(object):

    """
    Defines the Synchronous Replica Exchange Workflow. InitCycle() creates the workflow for the first cycle, i.e.
    the first MD phase and the subsequent exchange computation. GeneralCycle() then creates the workflows for all
    subsequent cycles. Each cycle (MD plus immediate exchange computation) must be specified as separate workflows.
    """

    def __init__(self):

        self.Book = [] #Bookkeeping, maintains a record of all MD tasks carried out
        self.md_task_list = []
        self.ex_task_list = []
        #self.test = 1

        self._uid = ru.generate_id('radical.repex.syncex')
        self._logger = ru.get_logger('radical.repex.syncex')
        self._prof = ru.Profiler(name=self._uid)
        self._prof.prof('Initinit', uid=self._uid)

    def Replica_Init(self,Replicas):

        #Nothing actually happens to a replica object here, this is just bookkeeping
        #potentially for a state/event model in the future

        Replica_List=dict()

        for r in range(Replicas):
            Replica_List[r] = Replica
            (Replica_List[r]).rstate = 'I' #Initialize with idle state

    def InitCycle(self, Replicas, Replica_Cores, MD_Executable, ExchangeMethod, timesteps, Basename): # "Cycle" = 1 MD stage plus the subsequent exchange computation

        """
        Initial cycle consists of:
        1) Create tarball of MD input data
        2) Transfer the tarball to pilot sandbox
        3) Untar the tarball
        4) Run first Cycle
        """

        #Initialize Pipeline
        #self._prof.prof('InitTar', uid=self._uid)
        p = Pipeline()
        p.name = 'initpipeline'

        md_dict    = dict() #Bookkeeping
        tar_dict   = dict() #Bookkeeping

        ##Write the input files

        self._prof.prof('InitWriteInputs', uid=self._uid)

        writeInputs.writeInputs(max_temp=350,min_temp=250,replicas=Replicas,timesteps=timesteps, basename=Basename)

        self._prof.prof('EndWriteInputs', uid=self._uid)

        self._prof.prof('InitTar', uid=self._uid)
        #Create Tarball of input data

        tar = tarfile.open("Input_Files.tar","w")
        for name in [Basename+".prmtop",
                     Basename+".inpcrd",
                     Basename+".mdin"]:
            tar.add(name)
        for r in range (Replicas):
            tar.add('mdin_{0}'.format(r))
        tar.close()

        #delete all input files outside the tarball

        for r in range (Replicas):
            os.remove('mdin_{0}'.format(r))

        self._prof.prof('EndTar', uid=self._uid)

        #Create Untar Stage

        untar_stg = Stage()
        untar_stg.name = 'untarStg'

        #Untar Task

        untar_tsk                   = Task()
        untar_tsk.name              = 'untartsk'
        untar_tsk.executable        = ['python']

        untar_tsk.upload_input_data = ['untar_input_files.py','Input_Files.tar']
        untar_tsk.arguments         = ['untar_input_files.py','Input_Files.tar']
        untar_tsk.cores             = 1

        untar_stg.add_tasks(untar_tsk)
        p.add_stages(untar_stg)

        tar_dict[0] = '$Pipeline_%s_Stage_%s_Task_%s'%(p.name,
                                                       untar_stg.name,
                                                       untar_tsk.name)

        # First MD stage: needs to be defined separately since workflow is not built from a predetermined order

        md_stg = Stage()
        md_stg.name = 'mdstg0'
        self._prof.prof('InitMD_0', uid=self._uid)

        # MD tasks

        for r in range (Replicas):

            md_tsk                  = AMBERTask(cores=Replica_Cores, MD_Executable=MD_Executable)
            md_tsk.name             = 'mdtsk-{replica}-{cycle}'.format(replica=r,cycle=0)
            md_tsk.link_input_data += [
                                       '%s/inpcrd'%tar_dict[0],
                                       '%s/prmtop'%tar_dict[0],
                                       '%s/mdin_{0}'.format(r)%tar_dict[0]  #Use for full temperature exchange
                                       #'%s/mdin'%tar_dict[0]  #Testing only
                                       ]
            md_tsk.arguments        = ['-O','-p','prmtop', '-i', 'mdin_{0}'.format(r), # Use this for full Temperature Exchange
                                       '-c','inpcrd','-o','out_{0}'.format(r),
                                       '-inf','mdinfo_{0}'.format(r)]
            md_dict[r]              = '$Pipeline_%s_Stage_%s_Task_%s'%(p.name, md_stg.name, md_tsk.name)

            md_stg.add_tasks(md_tsk)
            self.md_task_list.append(md_tsk)
            #print md_tsk.uid
        p.add_stages(md_stg)
        #stage_uids.append(md_stg.uid)

        # First Exchange Stage

        ex_stg = Stage()
        ex_stg.name = 'exstg0'
        self._prof.prof('InitEx_0', uid=self._uid)
        #with open('logfile.log', 'a') as logfile:
         #   logfile.write( '%.5f' %time.time() + ',' + 'InitEx0' + '\n')
        # Create Exchange Task. Exchange task performs a Metropolis Hastings thermodynamic balance condition
        # check and spits out the exchangePairs.dat file that contains a sorted list of ordered pairs.
        # Said pairs then exchange configurations by linking output configuration files appropriately.

        ex_tsk                      = Task()
        ex_tsk.name                 = 'extsk0'
        ex_tsk.executable           = ['python']
        ex_tsk.upload_input_data    = [ExchangeMethod]
        for r in range (Replicas):
            ex_tsk.link_input_data     += ['%s/mdinfo_%s'%(md_dict[r],r)]
        ex_tsk.arguments            = ['TempEx.py','{0}'.format(Replicas), '0']
        ex_tsk.cores                = 1
        ex_tsk.mpi                  = False
        ex_tsk.download_output_data = ['exchangePairs_0.dat']
        ex_stg.add_tasks(ex_tsk)
        #task_uids.append(ex_tsk.uid)
        p.add_stages(ex_stg)
        self.ex_task_list.append(ex_tsk)
        #self.ex_task_uids.append(ex_tsk.uid)
        self.Book.append(md_dict)
        return p

    def GeneralCycle(self, Replicas, Replica_Cores, Cycle, MD_Executable, ExchangeMethod):

        """
        All cycles after the initial cycle
        Pulls up exchange pairs file and generates the new workflow
        """

        self._prof.prof('InitcreateMDwokflow_{0}'.format(Cycle), uid=self._uid)
        with open('exchangePairs_{0}.dat'.format(Cycle),'r') as f:  # Read exchangePairs.dat
            ExchangeArray = []
            for line in f:
                ExchangeArray.append(int(line.split()[1]))
                #ExchangeArray.append(line)
                #print ExchangeArray

        q = Pipeline()
        q.name = 'genpipeline{0}'.format(Cycle)
        #Bookkeeping
        stage_uids = list()
        task_uids = list() ## = dict()
        md_dict = dict()

        #Create initial MD stage

        md_stg = Stage()
        md_stg.name = 'mdstage{0}'.format(Cycle)

        self._prof.prof('InitMD_{0}'.format(Cycle), uid=self._uid)

        for r in range (Replicas):
            md_tsk                 = AMBERTask(cores=Replica_Cores, MD_Executable=MD_Executable)
            md_tsk.name            = 'mdtsk-{replica}-{cycle}'.format(replica=r,cycle=Cycle)
            md_tsk.link_input_data = ['%s/restrt > inpcrd'%(self.Book[Cycle-1][ExchangeArray[r]]),
                                      '%s/prmtop'%(self.Book[0][r]),
                                      #'%s/prmtop'%(self.Tarball_path[0]),
                                      '%s/mdin_{0}'.format(r)%(self.Book[0][r])]

                                      #'%s/mdin'%(self.Book[0][r])]
                                      #'%s/mdin'%(self.Tarball_path[0])]

            md_tsk.arguments      = ['-O', '-i', 'mdin_{0}'.format(r), '-p', 'prmtop', '-c', 'inpcrd', '-o', 'out_{0}'.format(r),'-inf', 'mdinfo_{0}'.format(r)]
            md_tsk.tag              = 'mdtsk-{replica}-{cycle}'.format(replica=r,cycle=0)
            #md_tsk.arguments       = ['-O', '-i', 'mdin', '-p', 'prmtop', '-c', 'inpcrd', '-o', 'out_{0}'.format(r),'-inf', 'mdinfo_{0}'.format(r)]
            md_dict[r]             = '$Pipeline_%s_Stage_%s_Task_%s'%(q.name, md_stg.name, md_tsk.name)
            self.md_task_list.append(md_tsk)
            md_stg.add_tasks(md_tsk)

        q.add_stages(md_stg)

        ex_stg = Stage()
        ex_stg.name = 'exstg{0}'.format(Cycle+1)

        #Create Exchange Task
        ex_tsk                      = Task()
        ex_tsk.name                 = 'extsk{0}'.format(Cycle+1)
        ex_tsk.executable           = ['python']
        ex_tsk.upload_input_data    = [ExchangeMethod]
        for r in range (Replicas):

            ex_tsk.link_input_data += ['%s/mdinfo_%s'%(md_dict[r],r)]

        ex_tsk.arguments            = ['TempEx.py','{0}'.format(Replicas), '{0}'.format(Cycle+1)]
        ex_tsk.cores                = 1
        ex_tsk.mpi                  = False
        ex_tsk.download_output_data = ['exchangePairs_{0}.dat'.format(Cycle+1)] # Finds exchange partners, also  Generates exchange history trace

        ex_stg.add_tasks(ex_tsk)

        #task_uids.append(ex_tsk.uid)
        self.ex_task_list.append(ex_tsk)

        q.add_stages(ex_stg)

        #stage_uids.append(ex_stg.uid)

        self.Book.append(md_dict)
        #self._prof.prof('EndEx_{0}'.format(Cycle), uid=self._uid)
        #print d
        #print self.Book
        return q

    @property
    def totalmdlist(self):
        print 'Book is', self.Book
        return self.Book

    @property
    def mdtasklist(self):
        #print 'MD Task List:', self.md_task_list
        return self.md_task_list

    @property
    def extasklist(self):
        #print 'EX Task List', self.ex_task_list
        return self.ex_task_list

<commit_msg>Sync updated<commit_after>import radical.utils as ru
from radical.entk import Pipeline, Stage, Task
import os
import tarfile
import writeInputs
import time

class Replica(object):

    #A replica class to hold replica information, potentially useful for replica states in asynchronous exchange

    def __init__(self, rid, Temp, EPtot, rstate):
        self.rid    = rid     #Replica ID
        self.Temp   = Temp    #Replica Temp
        self.EPtot  = EPtot   #Replica Potential Energy
        self.rstate = rstate  #Replica State

class GROMACSTask(Task):

    # GROMACS specific MD task class

    def __init__(self, cores, mpi=True):

        super(GROMACSTask, self).__init__()
        self._executable = ['']
        self._cores      = cores
        self._pre_exec   = ['module load gromacs']
        self._post_exec = ['gmx energy < Energy.input']
        self._mpi        = mpi

class AMBERTask(Task):

    # AMBER specific MD task class.

    def __init__(self, MD_Executable, cores):

        super(AMBERTask, self).__init__()
        #self._executable = ['/usr/local/packages/amber/16/INTEL-140-MVAPICH2-2.0/bin/pmemd.MPI']
        self._executable = [MD_Executable]
        #self._cpu_reqs['processes']   = cores
        #self._cpu_reqs['process_type'] = ''
        self._cpu_reqs = {
                            'processes': cores,
                            'process_type': '',
                            'threads_per_process': 1,
                            'thread_type': None
                        }
        self._pre_exec   = ['module load amber'] #For BW make a pre-exec that points to $AMBERHOME correctly  ['export AMBERHOME=$HOME/amber/amber14/']
        #self._post_exec = [''] #Post exec is not useful here, but may be useful for something like a GROMACS class...

class SynchronousExchange(object):

    """
    Defines the Synchronous Replica Exchange Workflow. InitCycle() creates the workflow for the first cycle, i.e.
    the first MD phase and the subsequent exchange computation. GeneralCycle() then creates the workflows for all
    subsequent cycles. Each cycle (MD plus immediate exchange computation) must be specified as separate workflows.
    """

    def __init__(self):

        self.Book = [] #Bookkeeping, maintains a record of all MD tasks carried out
        self.md_task_list = []
        self.ex_task_list = []
        #self.test = 1

        self._uid = ru.generate_id('radical.repex.syncex')
        self._logger = ru.get_logger('radical.repex.syncex')
        self._prof = ru.Profiler(name=self._uid)
        self._prof.prof('Initinit', uid=self._uid)

    def Replica_Init(self,Replicas):

        #Nothing actually happens to a replica object here, this is just bookkeeping
        #potentially for a state/event model in the future

        Replica_List=dict()

        for r in range(Replicas):
            Replica_List[r] = Replica
            (Replica_List[r]).rstate = 'I' #Initialize with idle state

    def InitCycle(self, Replicas, Replica_Cores, MD_Executable, ExchangeMethod, timesteps, Basename): # "Cycle" = 1 MD stage plus the subsequent exchange computation

        """
        Initial cycle consists of:
        1) Create tarball of MD input data
        2) Transfer the tarball to pilot sandbox
        3) Untar the tarball
        4) Run first Cycle
        """

        #Initialize Pipeline
        #self._prof.prof('InitTar', uid=self._uid)
        p = Pipeline()
        p.name = 'initpipeline'

        md_dict    = dict() #Bookkeeping
        tar_dict   = dict() #Bookkeeping

        ##Write the input files

        self._prof.prof('InitWriteInputs', uid=self._uid)

        writeInputs.writeInputs(max_temp=350,min_temp=250,replicas=Replicas,timesteps=timesteps, basename=Basename)

        self._prof.prof('EndWriteInputs', uid=self._uid)

        self._prof.prof('InitTar', uid=self._uid)
        #Create Tarball of input data

        tar = tarfile.open("Input_Files.tar","w")
        for name in [Basename+".prmtop",
                     Basename+".inpcrd",
                     Basename+".mdin"]:
            tar.add(name)
        for r in range (Replicas):
            tar.add('mdin_{0}'.format(r))
        tar.close()

        #delete all input files outside the tarball

        for r in range (Replicas):
            os.remove('mdin_{0}'.format(r))

        self._prof.prof('EndTar', uid=self._uid)

        #Create Untar Stage

        untar_stg = Stage()
        untar_stg.name = 'untarStg'

        #Untar Task

        untar_tsk                   = Task()
        untar_tsk.name              = 'untartsk'
        untar_tsk.executable        = ['python']

        untar_tsk.upload_input_data = ['untar_input_files.py','Input_Files.tar']
        untar_tsk.arguments         = ['untar_input_files.py','Input_Files.tar']
        untar_tsk.cpu_reqs          = 1

        untar_stg.add_tasks(untar_tsk)
        p.add_stages(untar_stg)

        tar_dict[0] = '$Pipeline_%s_Stage_%s_Task_%s'%(p.name,
                                                       untar_stg.name,
                                                       untar_tsk.name)

        # First MD stage: needs to be defined separately since workflow is not built from a predetermined order

        md_stg = Stage()
        md_stg.name = 'mdstg0'
        self._prof.prof('InitMD_0', uid=self._uid)

        # MD tasks

        for r in range (Replicas):

            md_tsk                  = AMBERTask(cores=Replica_Cores, MD_Executable=MD_Executable)
            md_tsk.name             = 'mdtsk-{replica}-{cycle}'.format(replica=r,cycle=0)
            md_tsk.pre_exec         = ['module load amber']
            md_tsk.link_input_data += [
                                       '%s/inpcrd'%tar_dict[0],
                                       '%s/prmtop'%tar_dict[0],
                                       '%s/mdin_{0}'.format(r)%tar_dict[0]  #Use for full temperature exchange
                                       #'%s/mdin'%tar_dict[0]  #Testing only
                                       ]
            md_tsk.arguments        = ['-O','-p','prmtop', '-i', 'mdin_{0}'.format(r), # Use this for full Temperature Exchange
                                       '-c','inpcrd','-o','out_{0}'.format(r),
                                       '-inf','mdinfo_{0}'.format(r)]
            md_dict[r]              = '$Pipeline_%s_Stage_%s_Task_%s'%(p.name, md_stg.name, md_tsk.name)

            md_stg.add_tasks(md_tsk)
            self.md_task_list.append(md_tsk)
            #print md_tsk.uid
        p.add_stages(md_stg)
        #stage_uids.append(md_stg.uid)

        # First Exchange Stage

        ex_stg = Stage()
        ex_stg.name = 'exstg0'
        self._prof.prof('InitEx_0', uid=self._uid)
        #with open('logfile.log', 'a') as logfile:
         #   logfile.write( '%.5f' %time.time() + ',' + 'InitEx0' + '\n')
        # Create Exchange Task. Exchange task performs a Metropolis Hastings thermodynamic balance condition
        # check and spits out the exchangePairs.dat file that contains a sorted list of ordered pairs.
        # Said pairs then exchange configurations by linking output configuration files appropriately.

        ex_tsk                      = Task()
        ex_tsk.name                 = 'extsk0'
        ex_tsk.pre_exec             = ['module load python/2.7.10']
        ex_tsk.executable           = ['/opt/python/bin/python']
        ex_tsk.upload_input_data    = [ExchangeMethod]
        for r in range (Replicas):
            ex_tsk.link_input_data     += ['%s/mdinfo_%s'%(md_dict[r],r)]
        ex_tsk.arguments            = ['TempEx.py','{0}'.format(Replicas), '0']
        ex_tsk.cores                = 1
        ex_tsk.mpi                  = False
        ex_tsk.download_output_data = ['exchangePairs_0.dat']
        ex_stg.add_tasks(ex_tsk)
        #task_uids.append(ex_tsk.uid)
        p.add_stages(ex_stg)
        self.ex_task_list.append(ex_tsk)
        #self.ex_task_uids.append(ex_tsk.uid)
        self.Book.append(md_dict)
        return p

    def GeneralCycle(self, Replicas, Replica_Cores, Cycle, MD_Executable, ExchangeMethod):

        """
        All cycles after the initial cycle
        Pulls up exchange pairs file and generates the new workflow
        """

        self._prof.prof('InitcreateMDwokflow_{0}'.format(Cycle), uid=self._uid)
        with open('exchangePairs_{0}.dat'.format(Cycle),'r') as f:  # Read exchangePairs.dat
            ExchangeArray = []
            for line in f:
                ExchangeArray.append(int(line.split()[1]))
                #ExchangeArray.append(line)
                #print ExchangeArray

        q = Pipeline()
        q.name = 'genpipeline{0}'.format(Cycle)
        #Bookkeeping
        stage_uids = list()
        task_uids = list() ## = dict()
        md_dict = dict()

        #Create initial MD stage

        md_stg = Stage()
        md_stg.name = 'mdstage{0}'.format(Cycle)

        self._prof.prof('InitMD_{0}'.format(Cycle), uid=self._uid)

        for r in range (Replicas):
            md_tsk                 = AMBERTask(cores=Replica_Cores, MD_Executable=MD_Executable)
            md_tsk.name            = 'mdtsk-{replica}-{cycle}'.format(replica=r,cycle=Cycle)
            md_tsk.link_input_data = ['%s/restrt > inpcrd'%(self.Book[Cycle-1][ExchangeArray[r]]),
                                      '%s/prmtop'%(self.Book[0][r]),
                                      #'%s/prmtop'%(self.Tarball_path[0]),
                                      '%s/mdin_{0}'.format(r)%(self.Book[0][r])]

                                      #'%s/mdin'%(self.Book[0][r])]
                                      #'%s/mdin'%(self.Tarball_path[0])]

            md_tsk.arguments      = ['-O', '-i', 'mdin_{0}'.format(r), '-p', 'prmtop', '-c', 'inpcrd', '-o', 'out_{0}'.format(r),'-inf', 'mdinfo_{0}'.format(r)]
            md_tsk.tag              = 'mdtsk-{replica}-{cycle}'.format(replica=r,cycle=0)
            #md_tsk.arguments       = ['-O', '-i', 'mdin', '-p', 'prmtop', '-c', 'inpcrd', '-o', 'out_{0}'.format(r),'-inf', 'mdinfo_{0}'.format(r)]
            md_dict[r]             = '$Pipeline_%s_Stage_%s_Task_%s'%(q.name, md_stg.name, md_tsk.name)
            self.md_task_list.append(md_tsk)
            md_stg.add_tasks(md_tsk)

        q.add_stages(md_stg)

        ex_stg = Stage()
        ex_stg.name = 'exstg{0}'.format(Cycle+1)

        #Create Exchange Task
        ex_tsk                      = Task()
        ex_tsk.name                 = 'extsk{0}'.format(Cycle+1)
        ex_tsk.executable           = ['/opt/python/bin/python']
        ex_tsk.upload_input_data    = [ExchangeMethod]
        for r in range (Replicas):

            ex_tsk.link_input_data += ['%s/mdinfo_%s'%(md_dict[r],r)]

        ex_tsk.arguments            = ['TempEx.py','{0}'.format(Replicas), '{0}'.format(Cycle+1)]
        ex_tsk.cores                = 1
        ex_tsk.mpi                  = False
        ex_tsk.download_output_data = ['exchangePairs_{0}.dat'.format(Cycle+1)] # Finds exchange partners, also  Generates exchange history trace

        ex_stg.add_tasks(ex_tsk)

        #task_uids.append(ex_tsk.uid)
        self.ex_task_list.append(ex_tsk)

        q.add_stages(ex_stg)

        #stage_uids.append(ex_stg.uid)

        self.Book.append(md_dict)
        #self._prof.prof('EndEx_{0}'.format(Cycle), uid=self._uid)
        #print d
        #print self.Book
        return q

    @property
    def totalmdlist(self):
        print 'Book is', self.Book
        return self.Book

    @property
    def mdtasklist(self):
        #print 'MD Task List:', self.md_task_list
        return self.md_task_list

    @property
    def extasklist(self):
        #print 'EX Task List', self.ex_task_list
        return self.ex_task_list

<|endoftext|>


<commit_before>import sys
import rrdtool
import math
import time

class RRDManager(object):

	def getActualTime(self):
		return int(math.floor(time.time()))

	#se eth_type non corrisponde a nulla (IP, MPLS, ...) inserire 0000
	def constructDataSource(self, dev_id, port_n, eth_type):
		return str(str(port_n)+'_'+str(eth_type))

	def __init__(self, filename, device, eth_types):
		# define rrd filename
		self.filename = filename
		self.eth_types = eth_types
		#self.eth_types.append(eth_types)
		# import port numbers for each device id
		self.device = device

		# build ALL data sources DS:deviceID_portN_ETH_TYPE:COUNTER:600:U:U
		self.data_sources = []
		self.raw_data_sources = []

		for dev_id in sorted(self.device):
			for port_n in self.device[dev_id]:
				for eth_type in self.eth_types:
					temp = str(self.constructDataSource(dev_id, port_n, eth_type))
					self.raw_data_sources.append(temp)
					self.data_sources.append('DS:' + temp + ':GAUGE:600:U:U')
		# create rrd w/ default step = 300 sec
		rrdtool.create(self.filename,
			       '--step',
			       '300',
		               '--start',
		               str(self.getActualTime()),
		               self.data_sources,
		               'RRA:AVERAGE:0.5:1:24',	 #i dati raccolti ogni 5 minuti per 2 ore
		               'RRA:AVERAGE:0.5:6:10',	 #i dati raccolti ogni 30 minuti per 5 ore
		               'RRA:AVERAGE:0.5:12:24',	 #i dati raccolti ogni ora per un giorno
			       'RRA:AVERAGE:0.5:288:7',  #i dati raccolti ogni giorno per una settimana
			       'RRA:AVERAGE:0.5:2016:4') #i dati raccolti ogni settimana per 4 settimane
		print self.raw_data_sources

	# insert values w/ timestamp NOW for a set of given DS
	def update(self, data_sources, values):
		if (len(data_sources) != len(values)) or len(data_sources) <= 0 or (len(data_sources) >= self.data_sources):
			raise Exception('Wrong number of data_sources or values')
		for DS in data_sources:
			if DS not in self.raw_data_sources:
				raise Exception('Data source not available in RRD')
		template = ':'.join(data_sources)
		values = ':'.join(str(value) for value in values)
		rrdtool.update(self.filename, '-t', template, str(self.getActualTime()) + ':' + values)

################
#   T E S T    #
################
'''
device = {}
device['PEO1'] = [1,2,3,4]
device['PEO2'] = [1,2]
device['PEO3'] = [1,4,5,6]
device['PEO4'] = [2,3,5]

rrdman = RRDManager('test.rrd', device, ["0800", "0806", "0000"])
time.sleep(10)
rrdman.update([rrdman.constructDataSource('PEO1', 1, "0800"), rrdman.constructDataSource('PEO2', 1, "0806"), rrdman.constructDataSource('PEO3', 1, "0000")],[13000,15005,13501])
'''<commit_msg>inserite costanti per setting rrd<commit_after>import sys
import rrdtool
import math
import time

class RRDManager(object):
    #COSTANTI RRD------
	XFF1="0.5"
	XFF2="0.5"
	XFF3="0.5"
	XFF4="0.5"
	XFF5="0.5"
	STEP1="1"
	STEP2="6"
	STEP3="12"
	STEP4="288"
	STEP5="2016"
	ROWS1="24"
	ROWS2="10"
	ROWS3="24"
	ROWS4="7"
	ROWS5="4"
	#--------------
	def getActualTime(self):
		return int(math.floor(time.time()))

	#se eth_type non corrisponde a nulla (IP, MPLS, ...) inserire 0000
	def constructDataSource(self, dev_id, port_n, eth_type):
		return str(str(port_n)+'_'+str(eth_type))

	def __init__(self, filename, device, eth_types):
		# define rrd filename
		self.filename = filename
		self.eth_types = eth_types
		#self.eth_types.append(eth_types)
		# import port numbers for each device id
		self.device = device

		# build ALL data sources DS:deviceID_portN_ETH_TYPE:COUNTER:600:U:U
		self.data_sources = []
		self.raw_data_sources = []

		for dev_id in sorted(self.device):
			for port_n in self.device[dev_id]:
				for eth_type in self.eth_types:
					temp = str(self.constructDataSource(dev_id, port_n, eth_type))
					self.raw_data_sources.append(temp)
					self.data_sources.append('DS:' + temp + ':GAUGE:600:U:U')
		# create rrd w/ default step = 300 sec
		rrdtool.create(self.filename,
			       '--step',
			       '300',
		               '--start',
		               str(self.getActualTime()),
		               self.data_sources,
		               'RRA:AVERAGE:'+XFF1+':'+STEP1+':'+ROWS1,	 #i dati raccolti ogni 5 minuti per 2 ore
		               'RRA:AVERAGE:'+XFF2+':'+STEP2+':'+ROWS2,	 #i dati raccolti ogni 30 minuti per 5 ore
		               'RRA:AVERAGE:'+XFF3+':'+STEP3+':'+ROWS3,	 #i dati raccolti ogni ora per un giorno
			       'RRA:AVERAGE:'+XFF4+':'+STEP4+':'+ROWS4,  #i dati raccolti ogni giorno per una settimana
			       'RRA:AVERAGE:'+XFF5+':'+STEP5+':'+ROWS5) #i dati raccolti ogni settimana per 4 settimane
		print self.raw_data_sources

	# insert values w/ timestamp NOW for a set of given DS
	def update(self, data_sources, values):
		if (len(data_sources) != len(values)) or len(data_sources) <= 0 or (len(data_sources) >= self.data_sources):
			raise Exception('Wrong number of data_sources or values')
		for DS in data_sources:
			if DS not in self.raw_data_sources:
				raise Exception('Data source not available in RRD')
		template = ':'.join(data_sources)
		values = ':'.join(str(value) for value in values)
		rrdtool.update(self.filename, '-t', template, str(self.getActualTime()) + ':' + values)

################
#   T E S T    #
################
'''
device = {}
device['PEO1'] = [1,2,3,4]
device['PEO2'] = [1,2]
device['PEO3'] = [1,4,5,6]
device['PEO4'] = [2,3,5]

rrdman = RRDManager('test.rrd', device, ["0800", "0806", "0000"])
time.sleep(10)
rrdman.update([rrdman.constructDataSource('PEO1', 1, "0800"), rrdman.constructDataSource('PEO2', 1, "0806"), rrdman.constructDataSource('PEO3', 1, "0000")],[13000,15005,13501])
'''
<|endoftext|>