<commit_before>"""spacewiki database models"""
import bs4
import re
import difflib
import datetime
import logging
import hashlib
import mistune
import peewee
import playhouse.migrate
from playhouse.db_url import connect
import settings
import shutil
import urlparse
import urllib
import os
import slugify

import wikiformat

database = peewee.Proxy()
currentURI = None

def setURI(u):
    database.initialize(connect(u))
    database.connect()

class BaseModel(peewee.Model):
    class Meta:
        database = database

class SlugField(peewee.CharField):
    def coerce(self, value):
        return self.slugify(value)

    @staticmethod
    def slugify(title):
        """Translates a string into a reduced character set"""
        return slugify.slugify(unicode(title))

class Page(BaseModel):
    title = peewee.CharField(unique=True)
    slug = SlugField(unique=True)

    @staticmethod
    def parsePreviousSlugFromRequest(req, default):
        if 'Referer' in req.headers:
            referer = req.headers['Referer']
            referUrl = urlparse.urlparse(referer)
            if 'Host' in req.headers:
                if referUrl.netloc == req.headers['Host']:
                    script_name = req.environ['SCRIPT_NAME']

                    lastPageSlug = urllib.unquote(referUrl.path[len(script_name)+1:])
                    if '/' in lastPageSlug:
                      lastPageSlug, _ = lastPageSlug.split('/', 1)
                    if lastPageSlug == "":
                        lastPageSlug = default
                    req.lastSlug = lastPageSlug
                    return lastPageSlug
        if 'lastSlug' in req.args:
            req.lastSlug = req.args.get('lastSlug')
            return req.lastSlug
        req.lastSlug = None
        return None

    def newRevision(self, body, message):
        """Creates a new Revision of this Page with the given body"""
        return Revision.create(page=self, body=body, message=message)

    def makeSoftlinkFrom(self, prev):
        logging.debug("Linking from %s to %s", prev.slug, self.slug)
        try:
            Softlink.get(Softlink.src == prev, Softlink.dest == self)
            logging.debug("Link exists!")
        except peewee.DoesNotExist:
            Softlink.create(src=prev, dest=self)
            logging.debug("New link!")
        Softlink.update(hits = Softlink.hits + 1).where(Softlink.src ==
            prev, Softlink.dest == self).execute()

    def attachUpload(self, src, filename, uploadPath):
        assert(isinstance(src, basestring))
        assert(isinstance(filename, basestring))
        assert(isinstance(uploadPath, basestring))

        hexSha = Attachment.hashFile(src)
        savedName = os.path.join(uploadPath, Attachment.hashPath(hexSha, filename))
        if not os.path.exists(os.path.dirname(savedName)):
            os.makedirs(os.path.dirname(savedName))
        shutil.move(src, savedName)
        """FIXME: These db queries should be handled by the model"""
        try:
            attachment = Attachment.get(page=self, slug=filename)
        except peewee.DoesNotExist:
            attachment = Attachment.create(page=self, filename=filename,
                slug=filename)
        try:
            AttachmentRevision.create(attachment=attachment, sha=hexSha)
        except peewee.IntegrityError:
            print "Duplicate upload!"
        print "Uploaded file %s to %s"%(filename, savedName)

    @classmethod
    def latestRevision(cls, slug):
        try:
            return Revision.select() \
                .join(cls) \
                .where(cls.slug == slug) \
                .order_by(Revision.id.desc())[0]
        except IndexError:
            return None

class Softlink(BaseModel):
    src = peewee.ForeignKeyField(Page, related_name='softlinks_out')
    dest = peewee.ForeignKeyField(Page, related_name='softlinks_in')
    hits = peewee.IntegerField(default=0)

class WikiRenderer(mistune.Renderer):
    def block_html(self, html):
        firstLine, rest = html.split('\n', 1)
        tag, tagTail = re.match('^<(.+?)>(.*)', html).groups()
        rest = tagTail + rest
        renderer = WikiRenderer()
        md = mistune.Markdown(renderer=renderer)
        submd = md.render(unicode(rest))
        ret = "<%s>%s"%(tag, submd)
        return ret

class Revision(BaseModel):
    page = peewee.ForeignKeyField(Page, related_name='revisions')
    body = peewee.TextField()
    message = peewee.TextField(default='')
    timestamp = peewee.DateTimeField(default=datetime.datetime.now)

    @staticmethod
    def markdown(s):
        renderer = WikiRenderer()
        md = mistune.Markdown(renderer=renderer)
        return md.render(s)

    @property
    def html(self):
        return  \
            wikiformat.safetags(\
            self.markdown(\
            wikiformat.wikilinks(\
            wikiformat.wikitemplates(self.body, self.page.slug))))

    @property
    def is_latest(self):
      return Page.latestRevision(self.page.slug) == self

    @property
    def prev(self):
      try:
        return Revision.select().where(Revision.page == self.page, Revision.id <
            self.id).order_by(Revision.id.desc()).limit(1)[0]
      except IndexError:
        return None

    @classmethod
    def _makeDiff(cls, r1, r2):
        if r1 is None:
            d = difflib.unified_diff(
                "",
                r2.body.split("\n"),
                lineterm="",
                fromfile="%s@%s"%(r2.page.slug, 0),
                tofile="%s@%s"%(r2.page.slug, r2.id))
        elif r2 is None:
            d = difflib.unified_diff(
                r1.body.split("\n"),
                "",
                lineterm="",
                fromfile="%s@%s"%(r1.page.slug, r1.id),
                tofile="%s@%s"%(r1.page.slug, 0))
        else:
            d = difflib.unified_diff(
                r1.body.split("\n"),
                r2.body.split("\n"),
                lineterm="",
                fromfile="%s@%s"%(r1.page.slug, r1.id),
                tofile="%s@%s"%(r2.page.slug, r2.id))
        return cls._parseDiff(d)

    @staticmethod
    def _parseDiff(diff):
        ret = []
        for line in diff:
            if line.startswith('+++') or line.startswith('---'):
                lineType = 'meta'
            elif line.startswith('@@'):
                lineType = 'context'
            elif line.startswith('+'):
                lineType = 'addition'
            elif line.startswith('-'):
                lineType = 'subtraction'
            ret.append({'contents': line, 'type': lineType})
        return ret

    def diffTo(self, prev):
        return self._makeDiff(self, prev)

    @property
    def diffToPrev(self):
        prev = self.prev
        if prev is not None:
            return self.diffTo(prev)
        return self._makeDiff(None, self)

    @property
    def diffToNext(self):
        next = self.next
        if next is not None:
          return self.diffTo(next)
        return self._makeDiff(self, None)

    def diffStatsToPrev(self):
        meta = {'additions': 0, 'subtractions': 0}
        for line in self.diffToPrev:
            if line['type'] == 'addition':
                meta['additions'] += 1
            if line['type'] == 'subtraction':
                meta['subtractions'] += 1
        return meta

    @property
    def next(self):
      try:
        return Revision.select().where(Revision.page == self.page, Revision.id >
            self.id).order_by(Revision.id).limit(1)[0]
      except IndexError:
        return None

class Attachment(BaseModel):
    page = peewee.ForeignKeyField(Page, related_name='attachments')
    filename = peewee.CharField(unique=True)
    slug = SlugField(unique=True)

    @staticmethod
    def hashFile(src):
        with open(src, 'r') as f:
            sha = hashlib.sha256()
            sha.update(f.read())
        return sha.hexdigest()

    @staticmethod
    def hashPath(sha, src):
        return "%s/%s/%s-%s"%(sha[0:2], sha[2:4], sha, src)

    class Meta:
        indexes = (
            (('page', 'slug'), True),
        )

class AttachmentRevision(BaseModel):
    attachment = peewee.ForeignKeyField(Attachment, related_name='revisions')
    sha = peewee.CharField()

    class Meta:
        indexes = (
            (('attachment', 'sha'), True),
        )

class DatabaseVersion(BaseModel):
    schema_version = peewee.IntegerField(default=0)

def syncdb():
    logging.info("Creating tables...")
    try:
        DatabaseVersion.select().execute()
    except peewee.OperationalError, peewee.ProgrammingError:
        database.create_tables([DatabaseVersion])
    try:
        v = DatabaseVersion.select()[0]
    except IndexError:
        v = DatabaseVersion.create(schema_version=0)
    v.schema_version = migrate(v.schema_version)
    v.save()
    logging.info("OK!")

def migrate(currentRevision):
    migrator = playhouse.migrate.SqliteMigrator(database)
    with database.transaction():
        if currentRevision == 0:
            database.create_tables([Page, Revision, Softlink, Attachment,
              AttachmentRevision])
            return migrate(1)
    return currentRevision
<commit_msg>computer :(<commit_after>"""spacewiki database models"""
import bs4
import re
import difflib
import datetime
import logging
import hashlib
import mistune
import peewee
import playhouse.migrate
from playhouse.db_url import connect
import settings
import shutil
import urlparse
import urllib
import os
import slugify

import wikiformat

database = peewee.Proxy()
currentURI = None

def setURI(u):
    database.initialize(connect(u))
    database.connect()

class BaseModel(peewee.Model):
    class Meta:
        database = database

class SlugField(peewee.CharField):
    def coerce(self, value):
        return self.slugify(value)

    @staticmethod
    def slugify(title):
        """Translates a string into a reduced character set"""
        return slugify.slugify(unicode(title))

class Page(BaseModel):
    title = peewee.CharField(unique=True)
    slug = SlugField(unique=True)

    @staticmethod
    def parsePreviousSlugFromRequest(req, default):
        if 'Referer' in req.headers:
            referer = req.headers['Referer']
            referUrl = urlparse.urlparse(referer)
            if 'Host' in req.headers:
                if referUrl.netloc == req.headers['Host']:
                    script_name = req.environ['SCRIPT_NAME']

                    lastPageSlug = urllib.unquote(referUrl.path[len(script_name)+1:])
                    if '/' in lastPageSlug:
                      lastPageSlug, _ = lastPageSlug.split('/', 1)
                    if lastPageSlug == "":
                        lastPageSlug = default
                    req.lastSlug = lastPageSlug
                    return lastPageSlug
        if 'lastSlug' in req.args:
            req.lastSlug = req.args.get('lastSlug')
            return req.lastSlug
        req.lastSlug = None
        return None

    def newRevision(self, body, message):
        """Creates a new Revision of this Page with the given body"""
        return Revision.create(page=self, body=body, message=message)

    def makeSoftlinkFrom(self, prev):
        logging.debug("Linking from %s to %s", prev.slug, self.slug)
        try:
            Softlink.get(Softlink.src == prev, Softlink.dest == self)
            logging.debug("Link exists!")
        except peewee.DoesNotExist:
            Softlink.create(src=prev, dest=self)
            logging.debug("New link!")
        Softlink.update(hits = Softlink.hits + 1).where(Softlink.src ==
            prev, Softlink.dest == self).execute()

    def attachUpload(self, src, filename, uploadPath):
        assert(isinstance(src, basestring))
        assert(isinstance(filename, basestring))
        assert(isinstance(uploadPath, basestring))

        hexSha = Attachment.hashFile(src)
        savedName = os.path.join(uploadPath, Attachment.hashPath(hexSha, filename))
        if not os.path.exists(os.path.dirname(savedName)):
            os.makedirs(os.path.dirname(savedName))
        shutil.move(src, savedName)
        """FIXME: These db queries should be handled by the model"""
        try:
            attachment = Attachment.get(page=self, slug=filename)
        except peewee.DoesNotExist:
            attachment = Attachment.create(page=self, filename=filename,
                slug=filename)
        try:
            AttachmentRevision.create(attachment=attachment, sha=hexSha)
        except peewee.IntegrityError:
            print "Duplicate upload!"
        print "Uploaded file %s to %s"%(filename, savedName)

    @classmethod
    def latestRevision(cls, slug):
        try:
            return Revision.select() \
                .join(cls) \
                .where(cls.slug == slug) \
                .order_by(Revision.id.desc())[0]
        except IndexError:
            return None

class Softlink(BaseModel):
    src = peewee.ForeignKeyField(Page, related_name='softlinks_out')
    dest = peewee.ForeignKeyField(Page, related_name='softlinks_in')
    hits = peewee.IntegerField(default=0)

class WikiRenderer(mistune.Renderer):
    def block_html(self, html):
        firstLine, rest = html.split('\n', 1)
        tag, tagTail = re.match('^<(.+?)>(.*)', html).groups()
        rest = tagTail + rest
        renderer = WikiRenderer()
        md = mistune.Markdown(renderer=renderer)
        submd = md.render(unicode(rest))
        ret = "<%s>%s"%(tag, submd)
        return ret

class Revision(BaseModel):
    page = peewee.ForeignKeyField(Page, related_name='revisions')
    body = peewee.TextField()
    message = peewee.TextField(default='')
    timestamp = peewee.DateTimeField(default=datetime.datetime.now)

    @staticmethod
    def markdown(s):
        renderer = WikiRenderer()
        md = mistune.Markdown(renderer=renderer)
        return md.render(s)

    @property
    def html(self):
        return  \
            wikiformat.safetags(\
            self.markdown(\
            wikiformat.wikilinks(\
            wikiformat.wikitemplates(self.body, self.page.slug))))

    @property
    def is_latest(self):
      return Page.latestRevision(self.page.slug) == self

    @property
    def prev(self):
      try:
        return Revision.select().where(Revision.page == self.page, Revision.id <
            self.id).order_by(Revision.id.desc()).limit(1)[0]
      except IndexError:
        return None

    @classmethod
    def _makeDiff(cls, r1, r2):
        if r1 is None:
            d = difflib.unified_diff(
                "",
                r2.body.split("\n"),
                lineterm="",
                fromfile="%s@%s"%(r2.page.slug, 0),
                tofile="%s@%s"%(r2.page.slug, r2.id))
        elif r2 is None:
            d = difflib.unified_diff(
                r1.body.split("\n"),
                "",
                lineterm="",
                fromfile="%s@%s"%(r1.page.slug, r1.id),
                tofile="%s@%s"%(r1.page.slug, 0))
        else:
            d = difflib.unified_diff(
                r1.body.split("\n"),
                r2.body.split("\n"),
                lineterm="",
                fromfile="%s@%s"%(r1.page.slug, r1.id),
                tofile="%s@%s"%(r2.page.slug, r2.id))
        return cls._parseDiff(d)

    @staticmethod
    def _parseDiff(diff):
        ret = []
        for line in diff:
            if line.startswith('+++') or line.startswith('---'):
                lineType = 'meta'
            elif line.startswith('@@'):
                lineType = 'context'
            elif line.startswith('+'):
                lineType = 'addition'
            elif line.startswith('-'):
                lineType = 'subtraction'
            ret.append({'contents': line, 'type': lineType})
        return ret

    def diffTo(self, prev):
        return self._makeDiff(self, prev)

    @property
    def diffToPrev(self):
        prev = self.prev
        if prev is not None:
            return self.diffTo(prev)
        return self._makeDiff(None, self)

    @property
    def diffToNext(self):
        next = self.next
        if next is not None:
          return self.diffTo(next)
        return self._makeDiff(self, None)

    def diffStatsToPrev(self):
        meta = {'additions': 0, 'subtractions': 0}
        for line in self.diffToPrev:
            if line['type'] == 'addition':
                meta['additions'] += 1
            if line['type'] == 'subtraction':
                meta['subtractions'] += 1
        return meta

    @property
    def next(self):
      try:
        return Revision.select().where(Revision.page == self.page, Revision.id >
            self.id).order_by(Revision.id).limit(1)[0]
      except IndexError:
        return None

class Attachment(BaseModel):
    page = peewee.ForeignKeyField(Page, related_name='attachments')
    filename = peewee.CharField(unique=True)
    slug = SlugField(unique=True)

    @staticmethod
    def hashFile(src):
        with open(src, 'r') as f:
            sha = hashlib.sha256()
            sha.update(f.read())
        return sha.hexdigest()

    @staticmethod
    def hashPath(sha, src):
        return "%s/%s/%s-%s"%(sha[0:2], sha[2:4], sha, src)

    class Meta:
        indexes = (
            (('page', 'slug'), True),
        )

class AttachmentRevision(BaseModel):
    attachment = peewee.ForeignKeyField(Attachment, related_name='revisions')
    sha = peewee.CharField()

    class Meta:
        indexes = (
            (('attachment', 'sha'), True),
        )

class DatabaseVersion(BaseModel):
    schema_version = peewee.IntegerField(default=0)

def syncdb():
    logging.info("Creating tables...")
    try:
        DatabaseVersion.select().execute()
    except peewee.ProgrammingError:
        database.create_tables([DatabaseVersion])
    except peewee.OperationalError:
        database.create_tables([DatabaseVersion])
    try:
        v = DatabaseVersion.select()[0]
    except IndexError:
        v = DatabaseVersion.create(schema_version=0)
    v.schema_version = migrate(v.schema_version)
    v.save()
    logging.info("OK!")

def migrate(currentRevision):
    migrator = playhouse.migrate.SqliteMigrator(database)
    with database.transaction():
        if currentRevision == 0:
            database.create_tables([Page, Revision, Softlink, Attachment,
              AttachmentRevision])
            return migrate(1)
    return currentRevision
<|endoftext|>


<commit_before>"""
Copyright 2015 Udey Rishi

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import abc
import json
import time
import stat
import sys
from datetime import datetime

SETUP_DIR = '/usr/local/bin/hours_data'
CONFIG_FILE_PATH = os.path.join(SETUP_DIR, 'config.json')
OPEN_LOG_SCRIPT_PATH = os.path.join(SETUP_DIR, 'open_log.sh')
START_SCRIPT_PATH = os.path.join(SETUP_DIR, 'start.sh')
END_SCRIPT_PATH = os.path.join(SETUP_DIR, 'end.sh')
END_LINE = '--------------------------------------------'

class Mode:
    __metaclass__ = abc.ABCMeta

    @abc.abstractmethod
    def run(self):
        pass

def get_mode(options):
    if options is None or len(options) < 1:
        raise ValueError('Command line options needed.')

    if options[0] == '--config':
        if len(options) < 2:
            return ReadConfigMode()
        else:
            return ConfigMode(options[1:])

    elif options[0] == '--rconfig':
        return DeleteFileMode(CONFIG_FILE_PATH)

    elif options[0] == '--clear':
        return DeleteFileMode(get_logfile_path())

    elif options[0] == '--reset':
        # Order is important, clear mode won't work without any config
        return CompositeMode(DeleteFileMode(get_logfile_path()), DeleteFileMode(CONFIG_FILE_PATH))

    elif options[0] == '--start':
        return StartMode()

    elif options[0] == '--end' or options[0] == '--stop':
        return EndMode()

    elif options[0] == '--payment':
        if len(options) < 2:
            raise ValueError('Payment amount needed as argument.')

        return PaymentMode(options[1])

    elif options[0] == '--status':
        return StatusMode()

    elif options[0] == '--bitbar':
        return BitbarStatusMode()

    else:
        raise ValueError("Unknown option: '{0}'".format(options[0]))

#Source: http://stackoverflow.com/questions/12791997/how-do-you-do-a-simple-chmod-x-from-within-python
def make_executable(path):
    st = os.stat(path)
    os.chmod(path, st.st_mode | stat.S_IEXEC)

def generate_open_logfile_script():
    if not os.path.isfile(OPEN_LOG_SCRIPT_PATH):
        with open(OPEN_LOG_SCRIPT_PATH, 'w') as script:
            script_contents = '#!/bin/bash\n'

            if sys.platform == 'linux' or sys.platform == 'linux2':
               script_contents += 'xdg-open {0}\n'.format(get_logfile_path())
            elif sys.platform == 'darwin':
               script_contents += 'open {0}\n'.format(get_logfile_path())
            # else, unsupported OS

            script.write(script_contents)

        make_executable(OPEN_LOG_SCRIPT_PATH)

def is_configured():
    if not os.path.isfile(CONFIG_FILE_PATH):
        return False

    with open(CONFIG_FILE_PATH, 'r') as config_file:
        try:
            json.load(config_file)
            return True
        except json.decoder.JSONDecodeError:
            return False

def get_configuration():
    if not is_configured():
        raise NotYetConfiguredError('Program has not been configured yet. '
                                    'Use --config option to configure.')

    with open(CONFIG_FILE_PATH, 'r') as config_file:
        return json.load(config_file)

def get_logfile_path():
    config = get_configuration()

    if 'logfile' not in config:
        raise NotYetConfiguredError("logfile not yet configured. Use '--config "
                                    "-logfile path/to/logfile.txt' flag to "
                                    "configure")
    return config['logfile']

def get_wage_rate():
    config = get_configuration()

    if 'rate' not in config:
        raise NotYetConfiguredError('Please configure the earning hourly wage. e.g.: --config -rate 13.45')

    return float(config['rate'])

def get_last_non_payment_line(ignore_end_line=False):
    logfile_path = get_logfile_path()

    if not os.path.isfile(logfile_path):
        return None

    with open(logfile_path, 'r') as logfile:
        logfile.seek(0)
        last_line = None
        for line in logfile:
            if line is not '\n' and not line.startswith('Payment'):
                if line.strip() == END_LINE and ignore_end_line:
                    continue
                else:
                    last_line = line

        if last_line is None:
            return last_line

        return last_line.strip()

def get_duration_hours(start, end):
    delta = (end - start)
    return (delta.days * 24) + (delta.seconds / 3600)

def parse_time(time_string, initial_keyword):
    return datetime.strptime(time_string.strip(initial_keyword).strip(), "%Y-%m-%d %H:%M:%S")

def get_pending_payment():
    rate = get_wage_rate()
    logfile_path = get_logfile_path()

    pending_payment = 0

    if not os.path.isfile(logfile_path):
        return 0

    with open(logfile_path, 'r') as logfile:
        last_start = None

        for line in logfile.readlines():
            if line.startswith('Start: '):
                if last_start is None:
                    last_start = parse_time(line, 'Start: ')
                else:
                    raise ValueError("Logfile corrupted. Two successive 'Start' statements without an 'End' one.")
            elif line.startswith('End: '):
                if last_start is None:
                    raise ValueError("Logfile corrupted. Two successive 'End' statements without a 'Start' one.")
                else:
                    end_time = parse_time(line, 'End: ')
                    duration = get_duration_hours(last_start, end_time)
                    last_start = None
                    pending_payment += rate * duration
            elif line.startswith('Payment Made: '):
                payment_made = float(line[line.rfind('$') + 1:])
                pending_payment -= payment_made

    return pending_payment

def get_ongoing_shift_hours(last_start_line):
    start = parse_time(last_start_line, 'Start: ')
    end = datetime.today()
    return get_duration_hours(start, end)

class NotYetConfiguredError(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

class ReadConfigMode(Mode):
    def run(self):
        super()

        for key, value in get_configuration().items():
            print('{0}: {1}'.format(key, value))

class ConfigMode(Mode):
    def __init__(self, config):
        self.__parse_config_args(config)

    def run(self):
        super()

        os.makedirs(SETUP_DIR, exist_ok=True)

        if is_configured():
            already_existing_config = get_configuration()

            settings_to_be_kept = {
                key: value
                for key, value in already_existing_config.items()
                if key not in self.__config
                }
            self.__config.update(settings_to_be_kept)

        with open(CONFIG_FILE_PATH, 'w') as config_file:
            json.dump(self.__config, config_file, indent=4)

    def __parse_config_args(self, config_args):
        if len(config_args) % 2 is not 0:
            raise ValueError("No value for config option '{0}'".format(config_args[-1]))

        self.__config = {}
        for key, value in zip(*[iter(config_args)] * 2):

            if key[0] == '-':
                self.__config[key[1:]] = value
            else:
                raise ValueError("Bad command line parameter: '{0}.'".format(key))

class DeleteFileMode(Mode):
    def __init__(self, file_path):
        self.__file_path = file_path

    def run(self):
        super()

        if os.path.isfile(self.__file_path):
            os.remove(self.__file_path)

class StartMode(Mode):
    def run(self):
        super()
        last_line = get_last_non_payment_line()

        if last_line is None or last_line == END_LINE:
            logfile_path = get_logfile_path()
            with open(logfile_path, 'a+') as logfile:
                logfile.write('Start: ' + time.strftime('%Y-%m-%d %H:%M:%S') + '\n')
        else:
            if last_line.startswith('Start: '):
                raise ValueError('Previous shift not ended yet.')
            else:
                raise ValueError('Log file corrupted. Last line is not recognizable.')

class EndMode(Mode):
    def run(self):
        super()
        last_line = get_last_non_payment_line()

        if last_line is None:
            raise ValueError('No shift time started yet.')
        elif last_line.startswith('Start: '):
            start_time = parse_time(last_line, 'Start: ')
            end_time = datetime.now()

            duration = get_duration_hours(start_time, end_time)

            with open(get_logfile_path(), 'a+') as logfile:
                logfile.write('End: ' + end_time.strftime('%Y-%m-%d %H:%M:%S') + '\n')
                logfile.write('Duration: ' + '%.4f' % duration + ' hours \n')
                config = get_configuration()
                if 'rate' in config:
                    rate = float(config['rate'])
                    money = rate * duration
                    logfile.write('Money earned: $%.2f\n' % money)

                logfile.write(END_LINE + '\n')
        elif last_line == END_LINE:
            raise ValueError('No shift time started yet.')
        else:
            raise ValueError('Log file corrupted. Last line is not recognizable.')

class PaymentMode(Mode):
    def __init__(self, payment):
        self.__payment = float(payment)

    def run(self):
        super()
        pending_payment = get_pending_payment()

        logfile_path = get_logfile_path()
        with open(logfile_path, 'a+') as logfile:
            logfile.write('Payment Made: {0:s} : ${1:.2f}\n'.format(time.strftime('%Y-%m-%d %H:%M:%S'), self.__payment))
            logfile.write('Payment Pending: $%.2f\n' % (pending_payment - self.__payment))

class CompositeMode(Mode):
    def __init__(self, *modes):
        self.__modes = modes

    def run(self):
        super()

        for mode in self.__modes:
            mode.run()

class StatusMode(Mode):
    def run(self):
        super()

        if is_configured():

            status = self.__get_status()
            pending_payment = self.__get_pending_payment(status)
            today_hours = self.__get_today_hours()
            self.output_status(status, pending_payment, today_hours)
        else:
            self.output_not_configured()

    def output_status(self, status, pending_payment, today_hours):
        print('Status: {0}. Total pending payments: {1}. Hours '
                  'worked today: {2:.4f}'.format(status, pending_payment, today_hours))

    def output_not_configured(self):
        print('App not configured yet. Use the --config option to configure.')

    def __get_today_hours(self):
        today_hours = 0
        today = datetime.now()

        if os.path.isfile(get_logfile_path()):
            with open(get_logfile_path(), 'r') as logfile:
                last_start = None

                for line in logfile.readlines():
                    if line.startswith('Start: '):
                        if last_start is None:
                            last_start = parse_time(line, 'Start: ')
                        else:
                            raise ValueError("Logfile corrupted. Two successive 'Start' statements without an 'End' one.")
                    elif line.startswith('End: '):
                        if last_start is None:
                            raise ValueError("Logfile corrupted. Two successive 'End' statements without a 'Start' one.")
                        else:
                            end_time = parse_time(line, 'End: ')

                            if end_time.date() == today.date():
                                duration = get_duration_hours(last_start, end_time)
                                today_hours += duration

                            last_start = None

                if last_start is not None:
                    today_hours += get_ongoing_shift_hours(self.__last_start_line)

        return today_hours

    def __get_pending_payment(self, status):
        pending_payment = 'Unknown (rate not configured)'

        if 'rate' in get_configuration():
            base_payment = get_pending_payment()

            if status == 'Shift Ongoing':
                ongoing_shift_hours = get_ongoing_shift_hours(self.__last_start_line)
                base_payment += get_wage_rate() * ongoing_shift_hours

            pending_payment = '${0:.2f}'.format(base_payment)

        return pending_payment

    def __get_status(self):
        last_line = get_last_non_payment_line(True)

        if last_line is None:
            return 'Shift Not Ongoing'
        elif last_line.startswith('Start: '):
            self.__last_start_line = last_line
            return 'Shift Ongoing'
        elif last_line.startswith('Duration: ') or last_line.startswith('Money earned: '):
            return 'Shift Not Ongoing'
        else:
            raise ValueError("Corrupted log file. Unexpected line: '{0}'".format(last_line))

class BitbarStatusMode(StatusMode):
    def output_status(self, status, pending_payment, today_hours):

        status_mini = '{0:.2f}h : {1} | '.format(today_hours, pending_payment)
        if status == 'Shift Ongoing':
            status_mini += 'color=green'
        else:
            status_mini += 'color=red'

        print(status_mini)
        print('---')
        print('Hours worked today: {0:.2f}'.format(today_hours))
        print('Payment since last paycheck: {0}'.format(pending_payment))
        print(status)
        print('---')

        if status == 'Shift Ongoing':
            print('End Shift | refresh=true bash=' + END_SCRIPT_PATH+ ' terminal=false')
        else:
            print('Start Shift | refresh=true bash=' + START_SCRIPT_PATH + ' terminal=false')

        print('---')
        generate_open_logfile_script()
        print('View Logfile | refresh=true bash=' + OPEN_LOG_SCRIPT_PATH + ' terminal=false')

    def output_not_configured(self):
        print('Not configured')

<commit_msg>Bug fix: open logfile script for BitBar wasn't being updated when config changes<commit_after>"""
Copyright 2015 Udey Rishi

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import abc
import json
import time
import stat
import sys
from datetime import datetime

SETUP_DIR = '/usr/local/bin/hours_data'
CONFIG_FILE_PATH = os.path.join(SETUP_DIR, 'config.json')
OPEN_LOG_SCRIPT_PATH = os.path.join(SETUP_DIR, 'open_log.sh')
START_SCRIPT_PATH = os.path.join(SETUP_DIR, 'start.sh')
END_SCRIPT_PATH = os.path.join(SETUP_DIR, 'end.sh')
END_LINE = '--------------------------------------------'

class Mode:
    __metaclass__ = abc.ABCMeta

    @abc.abstractmethod
    def run(self):
        pass

def get_mode(options):
    if options is None or len(options) < 1:
        raise ValueError('Command line options needed.')

    if options[0] == '--config':
        if len(options) < 2:
            return ReadConfigMode()
        else:
            return ConfigMode(options[1:])

    elif options[0] == '--rconfig':
        return DeleteFileMode(CONFIG_FILE_PATH)

    elif options[0] == '--clear':
        return DeleteFileMode(get_logfile_path())

    elif options[0] == '--reset':
        # Order is important, clear mode won't work without any config
        return CompositeMode(DeleteFileMode(get_logfile_path()), DeleteFileMode(CONFIG_FILE_PATH))

    elif options[0] == '--start':
        return StartMode()

    elif options[0] == '--end' or options[0] == '--stop':
        return EndMode()

    elif options[0] == '--payment':
        if len(options) < 2:
            raise ValueError('Payment amount needed as argument.')

        return PaymentMode(options[1])

    elif options[0] == '--status':
        return StatusMode()

    elif options[0] == '--bitbar':
        return BitbarStatusMode()

    else:
        raise ValueError("Unknown option: '{0}'".format(options[0]))

#Source: http://stackoverflow.com/questions/12791997/how-do-you-do-a-simple-chmod-x-from-within-python
def make_executable(path):
    st = os.stat(path)
    os.chmod(path, st.st_mode | stat.S_IEXEC)

def generate_open_logfile_script(force=False):
    if not os.path.isfile(OPEN_LOG_SCRIPT_PATH) or force:
        with open(OPEN_LOG_SCRIPT_PATH, 'w') as script:
            script_contents = '#!/bin/bash\n'

            if sys.platform == 'linux' or sys.platform == 'linux2':
               script_contents += 'xdg-open {0}\n'.format(get_logfile_path())
            elif sys.platform == 'darwin':
               script_contents += 'open {0}\n'.format(get_logfile_path())
            # else, unsupported OS

            script.write(script_contents)

        make_executable(OPEN_LOG_SCRIPT_PATH)

def is_configured():
    if not os.path.isfile(CONFIG_FILE_PATH):
        return False

    with open(CONFIG_FILE_PATH, 'r') as config_file:
        try:
            json.load(config_file)
            return True
        except json.decoder.JSONDecodeError:
            return False

def get_configuration():
    if not is_configured():
        raise NotYetConfiguredError('Program has not been configured yet. '
                                    'Use --config option to configure.')

    with open(CONFIG_FILE_PATH, 'r') as config_file:
        return json.load(config_file)

def get_logfile_path():
    config = get_configuration()

    if 'logfile' not in config:
        raise NotYetConfiguredError("logfile not yet configured. Use '--config "
                                    "-logfile path/to/logfile.txt' flag to "
                                    "configure")
    return config['logfile']

def get_wage_rate():
    config = get_configuration()

    if 'rate' not in config:
        raise NotYetConfiguredError('Please configure the earning hourly wage. e.g.: --config -rate 13.45')

    return float(config['rate'])

def get_last_non_payment_line(ignore_end_line=False):
    logfile_path = get_logfile_path()

    if not os.path.isfile(logfile_path):
        return None

    with open(logfile_path, 'r') as logfile:
        logfile.seek(0)
        last_line = None
        for line in logfile:
            if line is not '\n' and not line.startswith('Payment'):
                if line.strip() == END_LINE and ignore_end_line:
                    continue
                else:
                    last_line = line

        if last_line is None:
            return last_line

        return last_line.strip()

def get_duration_hours(start, end):
    delta = (end - start)
    return (delta.days * 24) + (delta.seconds / 3600)

def parse_time(time_string, initial_keyword):
    return datetime.strptime(time_string.strip(initial_keyword).strip(), "%Y-%m-%d %H:%M:%S")

def get_pending_payment():
    rate = get_wage_rate()
    logfile_path = get_logfile_path()

    pending_payment = 0

    if not os.path.isfile(logfile_path):
        return 0

    with open(logfile_path, 'r') as logfile:
        last_start = None

        for line in logfile.readlines():
            if line.startswith('Start: '):
                if last_start is None:
                    last_start = parse_time(line, 'Start: ')
                else:
                    raise ValueError("Logfile corrupted. Two successive 'Start' statements without an 'End' one.")
            elif line.startswith('End: '):
                if last_start is None:
                    raise ValueError("Logfile corrupted. Two successive 'End' statements without a 'Start' one.")
                else:
                    end_time = parse_time(line, 'End: ')
                    duration = get_duration_hours(last_start, end_time)
                    last_start = None
                    pending_payment += rate * duration
            elif line.startswith('Payment Made: '):
                payment_made = float(line[line.rfind('$') + 1:])
                pending_payment -= payment_made

    return pending_payment

def get_ongoing_shift_hours(last_start_line):
    start = parse_time(last_start_line, 'Start: ')
    end = datetime.today()
    return get_duration_hours(start, end)

class NotYetConfiguredError(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

class ReadConfigMode(Mode):
    def run(self):
        super()

        for key, value in get_configuration().items():
            print('{0}: {1}'.format(key, value))

class ConfigMode(Mode):
    def __init__(self, config):
        self.__parse_config_args(config)

    def run(self):
        super()

        os.makedirs(SETUP_DIR, exist_ok=True)

        if is_configured():
            already_existing_config = get_configuration()

            settings_to_be_kept = {
                key: value
                for key, value in already_existing_config.items()
                if key not in self.__config
                }
            self.__config.update(settings_to_be_kept)

        with open(CONFIG_FILE_PATH, 'w') as config_file:
            json.dump(self.__config, config_file, indent=4)

        self.__fix_scripts()

    def __fix_scripts(self):
        # Fix all the scripts that might have become invalid by changing
        # config file
        generate_open_logfile_script(force=True)

    def __parse_config_args(self, config_args):
        if len(config_args) % 2 is not 0:
            raise ValueError("No value for config option '{0}'".format(config_args[-1]))

        self.__config = {}
        for key, value in zip(*[iter(config_args)] * 2):

            if key[0] == '-':
                self.__config[key[1:]] = value
            else:
                raise ValueError("Bad command line parameter: '{0}.'".format(key))

class DeleteFileMode(Mode):
    def __init__(self, file_path):
        self.__file_path = file_path

    def run(self):
        super()

        if os.path.isfile(self.__file_path):
            os.remove(self.__file_path)

class StartMode(Mode):
    def run(self):
        super()
        last_line = get_last_non_payment_line()

        if last_line is None or last_line == END_LINE:
            logfile_path = get_logfile_path()
            with open(logfile_path, 'a+') as logfile:
                logfile.write('Start: ' + time.strftime('%Y-%m-%d %H:%M:%S') + '\n')
        else:
            if last_line.startswith('Start: '):
                raise ValueError('Previous shift not ended yet.')
            else:
                raise ValueError('Log file corrupted. Last line is not recognizable.')

class EndMode(Mode):
    def run(self):
        super()
        last_line = get_last_non_payment_line()

        if last_line is None:
            raise ValueError('No shift time started yet.')
        elif last_line.startswith('Start: '):
            start_time = parse_time(last_line, 'Start: ')
            end_time = datetime.now()

            duration = get_duration_hours(start_time, end_time)

            with open(get_logfile_path(), 'a+') as logfile:
                logfile.write('End: ' + end_time.strftime('%Y-%m-%d %H:%M:%S') + '\n')
                logfile.write('Duration: ' + '%.4f' % duration + ' hours \n')
                config = get_configuration()
                if 'rate' in config:
                    rate = float(config['rate'])
                    money = rate * duration
                    logfile.write('Money earned: $%.2f\n' % money)

                logfile.write(END_LINE + '\n')
        elif last_line == END_LINE:
            raise ValueError('No shift time started yet.')
        else:
            raise ValueError('Log file corrupted. Last line is not recognizable.')

class PaymentMode(Mode):
    def __init__(self, payment):
        self.__payment = float(payment)

    def run(self):
        super()
        pending_payment = get_pending_payment()

        logfile_path = get_logfile_path()
        with open(logfile_path, 'a+') as logfile:
            logfile.write('Payment Made: {0:s} : ${1:.2f}\n'.format(time.strftime('%Y-%m-%d %H:%M:%S'), self.__payment))
            logfile.write('Payment Pending: $%.2f\n' % (pending_payment - self.__payment))

class CompositeMode(Mode):
    def __init__(self, *modes):
        self.__modes = modes

    def run(self):
        super()

        for mode in self.__modes:
            mode.run()

class StatusMode(Mode):
    def run(self):
        super()

        if is_configured():

            status = self.__get_status()
            pending_payment = self.__get_pending_payment(status)
            today_hours = self.__get_today_hours()
            self.output_status(status, pending_payment, today_hours)
        else:
            self.output_not_configured()

    def output_status(self, status, pending_payment, today_hours):
        print('Status: {0}. Total pending payments: {1}. Hours '
                  'worked today: {2:.4f}'.format(status, pending_payment, today_hours))

    def output_not_configured(self):
        print('App not configured yet. Use the --config option to configure.')

    def __get_today_hours(self):
        today_hours = 0
        today = datetime.now()

        if os.path.isfile(get_logfile_path()):
            with open(get_logfile_path(), 'r') as logfile:
                last_start = None

                for line in logfile.readlines():
                    if line.startswith('Start: '):
                        if last_start is None:
                            last_start = parse_time(line, 'Start: ')
                        else:
                            raise ValueError("Logfile corrupted. Two successive 'Start' statements without an 'End' one.")
                    elif line.startswith('End: '):
                        if last_start is None:
                            raise ValueError("Logfile corrupted. Two successive 'End' statements without a 'Start' one.")
                        else:
                            end_time = parse_time(line, 'End: ')

                            if end_time.date() == today.date():
                                duration = get_duration_hours(last_start, end_time)
                                today_hours += duration

                            last_start = None

                if last_start is not None:
                    today_hours += get_ongoing_shift_hours(self.__last_start_line)

        return today_hours

    def __get_pending_payment(self, status):
        pending_payment = 'Unknown (rate not configured)'

        if 'rate' in get_configuration():
            base_payment = get_pending_payment()

            if status == 'Shift Ongoing':
                ongoing_shift_hours = get_ongoing_shift_hours(self.__last_start_line)
                base_payment += get_wage_rate() * ongoing_shift_hours

            pending_payment = '${0:.2f}'.format(base_payment)

        return pending_payment

    def __get_status(self):
        last_line = get_last_non_payment_line(True)

        if last_line is None:
            return 'Shift Not Ongoing'
        elif last_line.startswith('Start: '):
            self.__last_start_line = last_line
            return 'Shift Ongoing'
        elif last_line.startswith('Duration: ') or last_line.startswith('Money earned: '):
            return 'Shift Not Ongoing'
        else:
            raise ValueError("Corrupted log file. Unexpected line: '{0}'".format(last_line))

class BitbarStatusMode(StatusMode):
    def output_status(self, status, pending_payment, today_hours):

        status_mini = '{0:.2f}h : {1} | '.format(today_hours, pending_payment)
        if status == 'Shift Ongoing':
            status_mini += 'color=green'
        else:
            status_mini += 'color=red'

        print(status_mini)
        print('---')
        print('Hours worked today: {0:.2f}'.format(today_hours))
        print('Payment since last paycheck: {0}'.format(pending_payment))
        print(status)
        print('---')

        if status == 'Shift Ongoing':
            print('End Shift | refresh=true bash=' + END_SCRIPT_PATH+ ' terminal=false')
        else:
            print('Start Shift | refresh=true bash=' + START_SCRIPT_PATH + ' terminal=false')

        print('---')
        generate_open_logfile_script()
        print('View Logfile | refresh=true bash=' + OPEN_LOG_SCRIPT_PATH + ' terminal=false')

    def output_not_configured(self):
        print('Not configured')

<|endoftext|>


<commit_before>"""
Python bindings for the Moira library

Moira is the Athena Service Management system.  It serves as the
central repository for information about users, groups hosts, print
queues, and several other aspects of the Athena environment.
"""

import re

import _moira
from _moira import (connect, disconnect, auth, host, motd, noop,
                    MoiraException)

help_re = re.compile('([a-z0-9_, ]*) \(([a-z0-9_, ]*)\)(?: => ([a-z0-9_, ]*))?',
                     re.I)

_arg_cache = {}
_return_cache = {}

def _load_help(handle):
    """Fetch info about the arguments and return values for a query.

    This uses the "_help" Moira query to retrieve names for the
    arguments and return values to and from a particular Moira
    query. These values are cached and used for translating arguments
    and return values into and out of dictionaries and into and out of
    tuples.
    """
    help_string = ', '.join(query('_help', handle)[0]).strip()

    handle_str, arg_str, return_str = help_re.match(help_string).groups('')

    handles = handle_str.split(', ')
    args = arg_str.split(', ')
    returns = return_str.split(', ')

    for h in handles:
        _arg_cache[h] = args
        _return_cache[h] = returns

def _list_query(handle, *args):
    """
    Execute a Moira query and return the result as a list of tuples.

    This bypasses the tuple -> dict conversion done in moira.query()
    """
    results = []
    _moira._query(handle, results.append, *args)
    return results

def query(handle, *args, **kwargs):
    """
    Execute a Moira query and return the result as a list of dicts.

    Arguments can be specified either as positional or keyword
    arguments. If specified by keyword, they are crossreferenced with
    the argument name given by the query "_help handle".

    All of the real work of Moira is done in queries. There are over
    100 queries, each of which requires different arguments. The
    arguments to the queries should be passed as separate arguments to
    the function.
    """
    if handle.startswith('_'):
        return _list_query(handle, *args)
    else:
        if handle not in _return_cache or \
                not _return_cache[handle]:
            _load_help(handle)

        fmt = kwargs.pop('fmt', dict)

        if kwargs:
            args = tuple(kwargs.get(i, '*') \
                             for i in _arg_cache[handle])

        plain_results = _list_query(handle, *args)
        results = []

        for r in plain_results:
            results.append(fmt(zip(_return_cache[handle], r)))

        return results

__all__ = ['connect', 'disconnect', 'auth', 'host', 'motd', 'noop', 'query',
           '_list_query']
<commit_msg>Actually export MoiraException in the moira module.<commit_after>"""
Python bindings for the Moira library

Moira is the Athena Service Management system.  It serves as the
central repository for information about users, groups hosts, print
queues, and several other aspects of the Athena environment.
"""

import re

import _moira
from _moira import (connect, disconnect, auth, host, motd, noop,
                    MoiraException)

help_re = re.compile('([a-z0-9_, ]*) \(([a-z0-9_, ]*)\)(?: => ([a-z0-9_, ]*))?',
                     re.I)

_arg_cache = {}
_return_cache = {}

def _load_help(handle):
    """Fetch info about the arguments and return values for a query.

    This uses the "_help" Moira query to retrieve names for the
    arguments and return values to and from a particular Moira
    query. These values are cached and used for translating arguments
    and return values into and out of dictionaries and into and out of
    tuples.
    """
    help_string = ', '.join(query('_help', handle)[0]).strip()

    handle_str, arg_str, return_str = help_re.match(help_string).groups('')

    handles = handle_str.split(', ')
    args = arg_str.split(', ')
    returns = return_str.split(', ')

    for h in handles:
        _arg_cache[h] = args
        _return_cache[h] = returns

def _list_query(handle, *args):
    """
    Execute a Moira query and return the result as a list of tuples.

    This bypasses the tuple -> dict conversion done in moira.query()
    """
    results = []
    _moira._query(handle, results.append, *args)
    return results

def query(handle, *args, **kwargs):
    """
    Execute a Moira query and return the result as a list of dicts.

    Arguments can be specified either as positional or keyword
    arguments. If specified by keyword, they are crossreferenced with
    the argument name given by the query "_help handle".

    All of the real work of Moira is done in queries. There are over
    100 queries, each of which requires different arguments. The
    arguments to the queries should be passed as separate arguments to
    the function.
    """
    if handle.startswith('_'):
        return _list_query(handle, *args)
    else:
        if handle not in _return_cache or \
                not _return_cache[handle]:
            _load_help(handle)

        fmt = kwargs.pop('fmt', dict)

        if kwargs:
            args = tuple(kwargs.get(i, '*') \
                             for i in _arg_cache[handle])

        plain_results = _list_query(handle, *args)
        results = []

        for r in plain_results:
            results.append(fmt(zip(_return_cache[handle], r)))

        return results

__all__ = ['connect', 'disconnect', 'auth', 'host', 'motd', 'noop', 'query',
           '_list_query', 'MoiraException']
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import os
import os.path
import time

from base64 import b64encode, b64decode
import re
import string

import lxml.html
import lxml.html.clean
import lxml.builder

import feeds

import urllib2
import socket
import chardet
import urlparse

from gzip import GzipFile
from StringIO import StringIO

from readability import readability

LIM_ITEM = 100	# deletes what's beyond
MAX_ITEM = 50	# cache-only beyond
MAX_TIME = 7	# cache-only after
DELAY = 10	# xml cache
TIMEOUT = 2	# http timeout

OPTIONS = ['progress', 'cache']

UA_RSS = 'Liferea/1.8.12 (Linux; fr_FR.utf8; http://liferea.sf.net/)'
UA_HML = 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.11) Gecko/20101012 Firefox/3.6.11'

PROTOCOL = ['http', 'https', 'ftp']

if 'REQUEST_URI' in os.environ:
	import httplib
	httplib.HTTPConnection.debuglevel = 1

	import cgitb
	cgitb.enable()

def log(txt):
	if not 'REQUEST_URI' in os.environ:
		if os.getenv('DEBUG', False):
			print repr(txt)
	else:
		with open('morss.log', 'a') as file:
			file.write(repr(txt).encode('utf-8') + "\n")

def lenHTML(txt):
	if len(txt):
		return len(lxml.html.fromstring(txt).text_content())
	else:
		return 0

def countWord(txt):
	if len(txt):
		return len(lxml.html.fromstring(txt).text_content().split())
	else:
		return 0

def makeDesc(txt, length, suffix='...'):
	' '.join(txt.split()[:length]) + suffix

def setContent(item, txt):
	if not item.desc:
		if item.content:
			log('content alone')
			item.desc = item.content
			item.content = txt
		else:
			log('empty')
			item.desc = makeDesc(txt, 30)
			item.content = txt
	else:
		item.content = txt

def parseOptions(available):
	options = None
	if 'REQUEST_URI' in os.environ:
		if 'REDIRECT_URL' in os.environ:
			url = os.environ['REQUEST_URI'][1:]
		else:
			url = os.environ['REQUEST_URI'][len(os.environ['SCRIPT_NAME'])+1:]

		if urlparse.urlparse(url).scheme not in PROTOCOL:
			split = url.split('/', 1)
			if len(split) and split[0] in available:
				options = split[0]
				url = split[1]
			url = 'http://' + url

	else:
		if len(sys.argv) == 3:
			if sys.argv[1] in available:
				options = sys.argv[1]
			url = sys.argv[2]
		elif len(sys.argv) == 2:
			url = sys.argv[1]
		else:
			return (None, None)

		if urlparse.urlparse(url).scheme not in PROTOCOL:
			url = 'http://' + url

	return (url, options)

class Cache:
	"""Light, error-prone caching system."""
	def __init__(self, folder, key):
		self._key = key
		self._hash = str(hash(self._key))

		self._dir = folder
		self._file = self._dir + '/' + self._hash

		self._cached = {} # what *was* cached
		self._cache = {} # new things to put in cache

		if os.path.isfile(self._file):
			data = open(self._file).readlines()
			for line in data:
				if "\t" in line:
					key, bdata = line.split("\t", 1)
					self._cached[key] = bdata

		log(self._hash)

	def __del__(self):
		self.save()

	def __contains__(self, key):
		return key in self._cached

	def get(self, key):
		if key in self._cached:
			self._cache[key] = self._cached[key]
			return b64decode(self._cached[key])
		else:
			return None

	def set(self, key, content):
		self._cache[key] = b64encode(content or '')

	def save(self):
		if len(self._cache) == 0:
			return

		out = []
		for (key, bdata) in self._cache.iteritems():
			out.append(str(key) + "\t" + bdata)
		txt = "\n".join(out)

		if not os.path.exists(self._dir):
			os.makedirs(self._dir)

		with open(self._file, 'w') as file:
			file.write(txt)

	def isYoungerThan(self, sec):
		if not os.path.exists(self._file):
			return False

		return time.time() - os.path.getmtime(self._file) < sec

class HTMLDownloader(urllib2.HTTPCookieProcessor):
	"""
	Custom urllib2 handler to download html pages, following <meta> redirects,
	using a browser user-agent and storing cookies.
	"""
	def __init__(self, useragent=UA_HML, cookiejar=None):
		urllib2.HTTPCookieProcessor.__init__(self, cookiejar)
		self.useragent = useragent

	def http_request(self, req):
		urllib2.HTTPCookieProcessor.http_request(self, req)
		req.add_unredirected_header('Accept-Encoding', 'gzip')
		req.add_unredirected_header('User-Agent', self.useragent)
		return req

	def http_response(self, req, resp):
		urllib2.HTTPCookieProcessor.http_response(self, req, resp)

		if 200 <= resp.code < 300 and resp.info().maintype == 'text':
			data = resp.read()

			# gzip
			if resp.headers.get('Content-Encoding') == 'gzip':
				log('un-gzip')
				data = GzipFile(fileobj=StringIO(data), mode='r').read()

			# <meta> redirect
			match = re.search(r'(?i)<meta http-equiv=.refresh[^>]*?url=(http.*?)["\']', data)
			if match:
				newurl = match.groups()[0]
				log('redirect: %s' % newurl)

				newheaders = dict((k,v) for k,v in req.headers.items()
					if k.lower() not in ('content-length', 'content-type'))
				new = urllib2.Request(newurl,
					headers=newheaders,
					origin_req_host=req.get_origin_req_host(),
					unverifiable=True)

				return self.parent.open(new, timeout=req.timeout)

			# decode
			data = decodeHTML(resp, data)

			fp = StringIO(data)
			old_resp = resp
			resp = urllib2.addinfourl(fp, old_resp.headers, old_resp.url, old_resp.code)
			resp.msg = old_resp.msg
		return resp

	https_response = http_response
	https_request = http_request

class CacheDownload(urllib2.BaseHandler):
	"""
	Custom urllib2 handler to download a page, using etag/last-modified headers,
	to save bandwidth. The given headers are added back into the header on error
	304 for easier use.
	"""
	def __init__(self, cache="", etag=None, lastmodified=None, useragent=UA_RSS):
		self.cache = cache
		self.etag = etag
		self.lastmodified = lastmodified
		self.useragent = useragent

	def http_request(self, req):
		req.add_unredirected_header('User-Agent', self.useragent)
		if self.cache:
			if self.etag:
				req.add_unredirected_header('If-None-Match', self.etag)
			if self.lastmodified:
				req.add_unredirected_header('If-Modified-Since', self.lastmodified)
		return req

	def http_error_304(self, req, fp, code, msg, headers):
		log('http cached')
		if self.etag:
			headers.addheader('etag', self.etag)
		if self.lastmodified:
			headers.addheader('last-modified', self.lastmodified)
		resp = urllib2.addinfourl(StringIO(self.cache), headers, req.get_full_url(), 200)
		return resp

	https_request = http_request

def decodeHTML(con, data):
	if con.headers.getparam('charset'):
		log('header')
		enc = con.headers.getparam('charset')
	else:
		match = re.search('charset=["\']?([0-9a-zA-Z-]+)', data)
		if match:
			log('meta.re')
			enc = match.groups()[0]
		else:
			log('chardet')
			enc = chardet.detect(data)['encoding']

	log(enc)
	return data.decode(enc, 'replace')

def Fill(item, cache, feedurl='/', fast=False):
	""" Returns True when it has done its best """

	if not item.link:
		log('no link')
		return True

	log(item.link)

	# feedburner
	feeds.NSMAP['feedburner'] = 'http://rssnamespace.org/feedburner/ext/1.0'
	match = item.xval('feedburner:origLink')
	if match:
		item.link = match
		log(item.link)

	# feedsportal
	match = re.search('/([0-9a-zA-Z]{20,})/story01.htm$', item.link)
	if match:
		url = match.groups()[0].split('0')
		t = {'A':'0', 'B':'.', 'C':'/', 'D':'?', 'E':'-', 'I':'_', 'L':'http://', 'S':'www.', 'N':'.com', 'O':'.co.uk'}
		item.link = ''.join([(t[s[0]] if s[0] in t else '=') + s[1:] for s in url[1:]])
		log(item.link)

	# reddit
	if urlparse.urlparse(item.link).netloc == 'www.reddit.com':
		match = lxml.html.fromstring(item.desc).xpath('//a[text()="[link]"]/@href')
		if len(match):
			item.link = match[0]
			log(item.link)

	# check relative urls
	if urlparse.urlparse(item.link).netloc is '':
		item.link = urlparse.urljoin(feedurl, item.link)

	# check unwanted uppercase title
	if len(item.title) > 20 and item.title.isupper():
		item.title = item.title.title()

	# content already provided?
	count_content = countWord(item.content)
	count_desc = countWord(item.desc)

	if max(count_content, count_desc) > 500:
		log('long enough')
		return True

	if count_content > 5*count_desc > 0 and count_content > 50:
		log('content bigger enough')
		return True

	# check cache and previous errors
	if item.link in cache:
		content = cache.get(item.link)
		match = re.search(r'^error-([a-z]{2,10})$', content)
		if match:
			if cache.isYoungerThan(DELAY*60):
				log('cached error: %s' % match.groups()[0])
				return True
			else:
				log('old error')
		else:
			log('cached')
			setContent(item, cache.get(item.link))
			return True

	# super-fast mode
	if fast:
		log('skipped')
		return False

	# download
	try:
		url = item.link.encode('utf-8')
		con = urllib2.build_opener(HTMLDownloader()).open(url, timeout=TIMEOUT)
		data = con.read()
	except (urllib2.HTTPError, urllib2.URLError, socket.timeout) as error:
		log('http error')
		cache.set(item.link, 'error-http')
		return True

	out = readability.Document(data, url=con.url).summary(True)

	if countWord(out) > max(count_content, count_desc) > 0:
		setContent(item, out)
		cache.set(item.link, out)
	else:
		log('not bigger enough')
		cache.set(item.link, 'error-length')
		return True

	return True

def Gather(url, cachePath, mode='feed'):
	cache = Cache(cachePath, url)

	# fetch feed
	if cache.isYoungerThan(DELAY*60) and url in cache:
		log('xml cached')
		xml = cache.get(url)
	else:
		try:
			opener = CacheDownload(cache.get(url), cache.get('etag'), cache.get('lastmodified'))
			con = urllib2.build_opener(opener).open(url)
			xml = con.read()
			cache.set(url, xml)
			cache.set('etag', con.headers.getheader('etag'))
			cache.set('lastmodified', con.headers.getheader('last-modified'))
		except (urllib2.HTTPError, urllib2.URLError):
			return False

	rss = feeds.parse(xml)
	size = len(rss)

	# set
	startTime = time.time()
	for i, item in enumerate(rss.items):
		if mode == 'progress':
			if MAX_ITEM == 0:
				print '%s/%s' % (i+1, size)
			else:
				print '%s/%s' % (i+1, min(MAX_ITEM, size))
			sys.stdout.flush()

		if i+1 > LIM_ITEM > 0:
			item.remove()
		elif time.time() - startTime > MAX_TIME >= 0 or i+1 > MAX_ITEM > 0:
			if Fill(item, cache, url, True) is False:
				item.remove()
		else:
			Fill(item, cache, url)

	log(len(rss))

	return rss.tostring(xml_declaration=True, encoding='UTF-8')

if __name__ == '__main__':
	url, options = parseOptions(OPTIONS)

	if 'REQUEST_URI' in os.environ:
		print 'Status: 200'

		if options == 'progress':
			print 'Content-Type: application/octet-stream'
		else:
			print 'Content-Type: text/xml'
		print

		cache = os.getcwd() + '/cache'
		log(url)
	else:
		cache =	os.path.expanduser('~') + '/.cache/morss'

	if url is None:
		print 'Please provide url.'
		sys.exit(1)

	if options == 'progress':
		MAX_TIME = -1
	if options == 'cache':
		MAX_TIME = 0

	RSS = Gather(url, cache, options)

	if RSS is not False and options != 'progress':
		if 'REQUEST_URI' in os.environ or not os.getenv('DEBUG', False):
			print RSS

	if RSS is False and options != 'progress':
		print 'Error fetching feed.'

	log('done')
<commit_msg>Use seconds for every parameter<commit_after>#!/usr/bin/env python
import sys
import os
import os.path
import time

from base64 import b64encode, b64decode
import re
import string

import lxml.html
import lxml.html.clean
import lxml.builder

import feeds

import urllib2
import socket
import chardet
import urlparse

from gzip import GzipFile
from StringIO import StringIO

from readability import readability

LIM_ITEM = 100	# deletes what's beyond
MAX_ITEM = 50	# cache-only beyond
MAX_TIME = 7	# cache-only after (in sec)
DELAY = 10*60	# xml cache (in sec)
TIMEOUT = 2	# http timeout (in sec)

OPTIONS = ['progress', 'cache']

UA_RSS = 'Liferea/1.8.12 (Linux; fr_FR.utf8; http://liferea.sf.net/)'
UA_HML = 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.11) Gecko/20101012 Firefox/3.6.11'

PROTOCOL = ['http', 'https', 'ftp']

if 'REQUEST_URI' in os.environ:
	import httplib
	httplib.HTTPConnection.debuglevel = 1

	import cgitb
	cgitb.enable()

def log(txt):
	if not 'REQUEST_URI' in os.environ:
		if os.getenv('DEBUG', False):
			print repr(txt)
	else:
		with open('morss.log', 'a') as file:
			file.write(repr(txt).encode('utf-8') + "\n")

def lenHTML(txt):
	if len(txt):
		return len(lxml.html.fromstring(txt).text_content())
	else:
		return 0

def countWord(txt):
	if len(txt):
		return len(lxml.html.fromstring(txt).text_content().split())
	else:
		return 0

def makeDesc(txt, length, suffix='...'):
	' '.join(txt.split()[:length]) + suffix

def setContent(item, txt):
	if not item.desc:
		if item.content:
			log('content alone')
			item.desc = item.content
			item.content = txt
		else:
			log('empty')
			item.desc = makeDesc(txt, 30)
			item.content = txt
	else:
		item.content = txt

def parseOptions(available):
	options = None
	if 'REQUEST_URI' in os.environ:
		if 'REDIRECT_URL' in os.environ:
			url = os.environ['REQUEST_URI'][1:]
		else:
			url = os.environ['REQUEST_URI'][len(os.environ['SCRIPT_NAME'])+1:]

		if urlparse.urlparse(url).scheme not in PROTOCOL:
			split = url.split('/', 1)
			if len(split) and split[0] in available:
				options = split[0]
				url = split[1]
			url = 'http://' + url

	else:
		if len(sys.argv) == 3:
			if sys.argv[1] in available:
				options = sys.argv[1]
			url = sys.argv[2]
		elif len(sys.argv) == 2:
			url = sys.argv[1]
		else:
			return (None, None)

		if urlparse.urlparse(url).scheme not in PROTOCOL:
			url = 'http://' + url

	return (url, options)

class Cache:
	"""Light, error-prone caching system."""
	def __init__(self, folder, key):
		self._key = key
		self._hash = str(hash(self._key))

		self._dir = folder
		self._file = self._dir + '/' + self._hash

		self._cached = {} # what *was* cached
		self._cache = {} # new things to put in cache

		if os.path.isfile(self._file):
			data = open(self._file).readlines()
			for line in data:
				if "\t" in line:
					key, bdata = line.split("\t", 1)
					self._cached[key] = bdata

		log(self._hash)

	def __del__(self):
		self.save()

	def __contains__(self, key):
		return key in self._cached

	def get(self, key):
		if key in self._cached:
			self._cache[key] = self._cached[key]
			return b64decode(self._cached[key])
		else:
			return None

	def set(self, key, content):
		self._cache[key] = b64encode(content or '')

	def save(self):
		if len(self._cache) == 0:
			return

		out = []
		for (key, bdata) in self._cache.iteritems():
			out.append(str(key) + "\t" + bdata)
		txt = "\n".join(out)

		if not os.path.exists(self._dir):
			os.makedirs(self._dir)

		with open(self._file, 'w') as file:
			file.write(txt)

	def isYoungerThan(self, sec):
		if not os.path.exists(self._file):
			return False

		return time.time() - os.path.getmtime(self._file) < sec

class HTMLDownloader(urllib2.HTTPCookieProcessor):
	"""
	Custom urllib2 handler to download html pages, following <meta> redirects,
	using a browser user-agent and storing cookies.
	"""
	def __init__(self, useragent=UA_HML, cookiejar=None):
		urllib2.HTTPCookieProcessor.__init__(self, cookiejar)
		self.useragent = useragent

	def http_request(self, req):
		urllib2.HTTPCookieProcessor.http_request(self, req)
		req.add_unredirected_header('Accept-Encoding', 'gzip')
		req.add_unredirected_header('User-Agent', self.useragent)
		return req

	def http_response(self, req, resp):
		urllib2.HTTPCookieProcessor.http_response(self, req, resp)

		if 200 <= resp.code < 300 and resp.info().maintype == 'text':
			data = resp.read()

			# gzip
			if resp.headers.get('Content-Encoding') == 'gzip':
				log('un-gzip')
				data = GzipFile(fileobj=StringIO(data), mode='r').read()

			# <meta> redirect
			match = re.search(r'(?i)<meta http-equiv=.refresh[^>]*?url=(http.*?)["\']', data)
			if match:
				newurl = match.groups()[0]
				log('redirect: %s' % newurl)

				newheaders = dict((k,v) for k,v in req.headers.items()
					if k.lower() not in ('content-length', 'content-type'))
				new = urllib2.Request(newurl,
					headers=newheaders,
					origin_req_host=req.get_origin_req_host(),
					unverifiable=True)

				return self.parent.open(new, timeout=req.timeout)

			# decode
			data = decodeHTML(resp, data)

			fp = StringIO(data)
			old_resp = resp
			resp = urllib2.addinfourl(fp, old_resp.headers, old_resp.url, old_resp.code)
			resp.msg = old_resp.msg
		return resp

	https_response = http_response
	https_request = http_request

class CacheDownload(urllib2.BaseHandler):
	"""
	Custom urllib2 handler to download a page, using etag/last-modified headers,
	to save bandwidth. The given headers are added back into the header on error
	304 for easier use.
	"""
	def __init__(self, cache="", etag=None, lastmodified=None, useragent=UA_RSS):
		self.cache = cache
		self.etag = etag
		self.lastmodified = lastmodified
		self.useragent = useragent

	def http_request(self, req):
		req.add_unredirected_header('User-Agent', self.useragent)
		if self.cache:
			if self.etag:
				req.add_unredirected_header('If-None-Match', self.etag)
			if self.lastmodified:
				req.add_unredirected_header('If-Modified-Since', self.lastmodified)
		return req

	def http_error_304(self, req, fp, code, msg, headers):
		log('http cached')
		if self.etag:
			headers.addheader('etag', self.etag)
		if self.lastmodified:
			headers.addheader('last-modified', self.lastmodified)
		resp = urllib2.addinfourl(StringIO(self.cache), headers, req.get_full_url(), 200)
		return resp

	https_request = http_request

def decodeHTML(con, data):
	if con.headers.getparam('charset'):
		log('header')
		enc = con.headers.getparam('charset')
	else:
		match = re.search('charset=["\']?([0-9a-zA-Z-]+)', data)
		if match:
			log('meta.re')
			enc = match.groups()[0]
		else:
			log('chardet')
			enc = chardet.detect(data)['encoding']

	log(enc)
	return data.decode(enc, 'replace')

def Fill(item, cache, feedurl='/', fast=False):
	""" Returns True when it has done its best """

	if not item.link:
		log('no link')
		return True

	log(item.link)

	# feedburner
	feeds.NSMAP['feedburner'] = 'http://rssnamespace.org/feedburner/ext/1.0'
	match = item.xval('feedburner:origLink')
	if match:
		item.link = match
		log(item.link)

	# feedsportal
	match = re.search('/([0-9a-zA-Z]{20,})/story01.htm$', item.link)
	if match:
		url = match.groups()[0].split('0')
		t = {'A':'0', 'B':'.', 'C':'/', 'D':'?', 'E':'-', 'I':'_', 'L':'http://', 'S':'www.', 'N':'.com', 'O':'.co.uk'}
		item.link = ''.join([(t[s[0]] if s[0] in t else '=') + s[1:] for s in url[1:]])
		log(item.link)

	# reddit
	if urlparse.urlparse(item.link).netloc == 'www.reddit.com':
		match = lxml.html.fromstring(item.desc).xpath('//a[text()="[link]"]/@href')
		if len(match):
			item.link = match[0]
			log(item.link)

	# check relative urls
	if urlparse.urlparse(item.link).netloc is '':
		item.link = urlparse.urljoin(feedurl, item.link)

	# check unwanted uppercase title
	if len(item.title) > 20 and item.title.isupper():
		item.title = item.title.title()

	# content already provided?
	count_content = countWord(item.content)
	count_desc = countWord(item.desc)

	if max(count_content, count_desc) > 500:
		log('long enough')
		return True

	if count_content > 5*count_desc > 0 and count_content > 50:
		log('content bigger enough')
		return True

	# check cache and previous errors
	if item.link in cache:
		content = cache.get(item.link)
		match = re.search(r'^error-([a-z]{2,10})$', content)
		if match:
			if cache.isYoungerThan(DELAY):
				log('cached error: %s' % match.groups()[0])
				return True
			else:
				log('old error')
		else:
			log('cached')
			setContent(item, cache.get(item.link))
			return True

	# super-fast mode
	if fast:
		log('skipped')
		return False

	# download
	try:
		url = item.link.encode('utf-8')
		con = urllib2.build_opener(HTMLDownloader()).open(url, timeout=TIMEOUT)
		data = con.read()
	except (urllib2.HTTPError, urllib2.URLError, socket.timeout) as error:
		log('http error')
		cache.set(item.link, 'error-http')
		return True

	out = readability.Document(data, url=con.url).summary(True)

	if countWord(out) > max(count_content, count_desc) > 0:
		setContent(item, out)
		cache.set(item.link, out)
	else:
		log('not bigger enough')
		cache.set(item.link, 'error-length')
		return True

	return True

def Gather(url, cachePath, mode='feed'):
	url = url.replace(' ', '%20')
	cache = Cache(cachePath, url)

	# fetch feed
	if cache.isYoungerThan(DELAY) and url in cache:
		log('xml cached')
		xml = cache.get(url)
	else:
		try:
			opener = CacheDownload(cache.get(url), cache.get('etag'), cache.get('lastmodified'))
			con = urllib2.build_opener(opener).open(url)
			xml = con.read()
			cache.set(url, xml)
			cache.set('etag', con.headers.getheader('etag'))
			cache.set('lastmodified', con.headers.getheader('last-modified'))
		except (urllib2.HTTPError, urllib2.URLError):
			return False

	rss = feeds.parse(xml)
	size = len(rss)

	# set
	startTime = time.time()
	for i, item in enumerate(rss.items):
		if mode == 'progress':
			if MAX_ITEM == 0:
				print '%s/%s' % (i+1, size)
			else:
				print '%s/%s' % (i+1, min(MAX_ITEM, size))
			sys.stdout.flush()

		if i+1 > LIM_ITEM > 0:
			item.remove()
		elif time.time() - startTime > MAX_TIME >= 0 or i+1 > MAX_ITEM > 0:
			if Fill(item, cache, url, True) is False:
				item.remove()
		else:
			Fill(item, cache, url)

	log(len(rss))

	return rss.tostring(xml_declaration=True, encoding='UTF-8')

if __name__ == '__main__':
	url, options = parseOptions(OPTIONS)

	if 'REQUEST_URI' in os.environ:
		print 'Status: 200'

		if options == 'progress':
			print 'Content-Type: application/octet-stream'
		else:
			print 'Content-Type: text/xml'
		print

		cache = os.getcwd() + '/cache'
		log(url)
	else:
		cache =	os.path.expanduser('~') + '/.cache/morss'

	if url is None:
		print 'Please provide url.'
		sys.exit(1)

	if options == 'progress':
		MAX_TIME = -1
	if options == 'cache':
		MAX_TIME = 0

	RSS = Gather(url, cache, options)

	if RSS is not False and options != 'progress':
		if 'REQUEST_URI' in os.environ or not os.getenv('DEBUG', False):
			print RSS

	if RSS is False and options != 'progress':
		print 'Error fetching feed.'

	log('done')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
__all__ = ['mount_blueprint']
import re
from itertools import product
from datetime import datetime, timedelta

# dateutil
from dateutil.parser import parse as dtparse

# flask
from flask import (
    Flask, request, session, g,
    redirect, url_for, abort,
    render_template, flash, jsonify,
    Blueprint, abort,
    send_from_directory,
    current_app,
)

# flask login
from flask.ext.login import login_required, fresh_login_required, current_user

# flask-wtf
from flask.ext.wtf import Form
from wtforms import validators
from wtforms import TextField, PasswordField, SelectField, BooleanField
from wtforms_html5 import EmailField

# model
from model.db import db
from model.db import object_to_dict, objects_to_list, update_object_with_dict
from model.user import UserAccount, UserQuota
from model.host import Host
from model.mount import MountPoint

mount_blueprint = Blueprint('mount_blueprint', __name__)

@mount_blueprint.route('/mount/points', methods=['GET'])
@login_required
def mount_points():
    username = current_user.username
    print 'mount_points:', locals()

    # get user account properties
    user_account = UserAccount.query.filter_by(username=username).one()
    dct = object_to_dict(user_account)

    return render_template(
        'mount-points.html',
        **dct
    )

@mount_blueprint.route('/mount/points/all', methods=['POST'])
@login_required
def mount_points_all():
    username = current_user.username
    usertype = current_user.usertype
    print 'mount_points_all:', locals()

    # FIXME:
    if usertype != 'super':
        data = {}
        return jsonify(data)

    mounts = MountPoint.query.all()
    _mounts = objects_to_list(mounts)

    # insert host_name
    for _mount in _mounts:
        host = Host.query.get(_mount['host_id'])
        _mount['host_name'] = host.name

    data = {
        'mounts': _mounts,
    }

    return jsonify(data)

@mount_blueprint.route('/mount/point/create', methods=['POST'])
@login_required
def mount_create():
    username = current_user.username
    usertype = current_user.usertype
    _mount = request.json['mount']
    print 'mount_add:', locals()

    # FIXME:
    if usertype != 'super':
        data = {}
        return jsonify(data)

    # try to find range patterns
    if 'host' in _mount:
        host = _mount['host']
        ranges = list(re.findall('\[\w+\-\w+\]', host))
    else:
        ranges = None

    if ranges:
        ## FIXME: support arbitrary number of ranges
        # assert len(ranges) == 2
        patterns = ranges
        ranges = [range.strip('[]').split('-') for range in ranges]

        for i, r in enumerate(ranges):
            s, e = r

            if s.isdigit() and e.isdigit():
                s, e = map(int, r)
                r = range(s, e + 1)
                ranges[i] = r
            else:
                # FIXME: support alpha-num ranges for multi-character strings
                #        with same size
                assert len(s) == len(e) == 1
                s, e = map(ord, r)
                r = range(s, e + 1)
                r = map(chr, r)
                r = ''.join(r)
                ranges[i] = r

        # generate mount points
        name = _mount['name']
        device = _mount['device']
        mountpoint = _mount['mountpoint']
        filesystem = _mount['filesystem']
        capacity = _mount['capacity']

        # rcomb = product(*ranges)
        # for p, r in zip(patterns, ranges):
        combs = product(*ranges)

        for comb in combs:
            _host_id = host
            _name = name
            _device = device
            _mountpoint = mountpoint

            for p, c in zip(patterns, comb):
                _host_id = _host_id.replace(p, c)
                _name = _name.replace(p, c)
                _device = _device.replace(p, c)
                _mountpoint = _mountpoint.replace(p, c)
    else:
        mount = MountPoint(**_mount)
        _mount['created'] = _mount['updated'] = datetime.utcnow()
        db.session.add(mount)
        db.session.commit()

    _mount = object_to_dict(mount)

    # insert host_name
    host = Host.query.get(_mount['host_id'])
    _mount['host_name'] = host.name

    data = {
        'mount': _mount,
    }

    return jsonify(data)

@mount_blueprint.route('/mount/point/update', methods=['POST'])
@login_required
def mount_update():
    username = current_user.username
    usertype = current_user.usertype
    _mount = request.json['mount']
    print 'mount_update:', locals()

    # FIXME:
    if usertype != 'super':
        data = {}
        return jsonify(data)

    mount = MountPoint.query.get(_mount['id'])
    _mount['updated'] = datetime.utcnow()
    update_object_with_dict(mount, _mount)
    db.session.commit()

    _mount = object_to_dict(mount)

    # insert host_name
    host = Host.query.get(_mount['host_id'])
    _mount['host_name'] = host.name

    data = {
        'mount': _mount,
    }

    return jsonify(data)

@mount_blueprint.route('/mount/point/remove', methods=['POST'])
@login_required
def mount_remove():
    username = current_user.username
    usertype = current_user.usertype
    id = request.json['id']
    print 'mount_remove:', locals()

    # FIXME:
    if usertype != 'super':
        data = {}
        return jsonify(data)

    mount = MountPoint.query.get(id)
    db.session.delete(mount)
    db.session.commit()

    data = {}
    return jsonify(data)
<commit_msg>mount points, slave<commit_after># -*- coding: utf-8 -*-
__all__ = ['mount_blueprint']
import re
from itertools import product
from datetime import datetime, timedelta

# dateutil
from dateutil.parser import parse as dtparse

# flask
from flask import (
    Flask, request, session, g,
    redirect, url_for, abort,
    render_template, flash, jsonify,
    Blueprint, abort,
    send_from_directory,
    current_app,
)

# flask login
from flask.ext.login import login_required, fresh_login_required, current_user

# flask-wtf
from flask.ext.wtf import Form
from wtforms import validators
from wtforms import TextField, PasswordField, SelectField, BooleanField
from wtforms_html5 import EmailField

# model
from model.db import db
from model.db import object_to_dict, objects_to_list, update_object_with_dict
from model.user import UserAccount, UserQuota
from model.host import Host
from model.mount import MountPoint

mount_blueprint = Blueprint('mount_blueprint', __name__)

@mount_blueprint.route('/mount/points', methods=['GET'])
@login_required
def mount_points():
    username = current_user.username
    print 'mount_points:', locals()

    # get user account properties
    user_account = UserAccount.query.filter_by(username=username).one()
    dct = object_to_dict(user_account)

    return render_template(
        'mount-points.html',
        **dct
    )

@mount_blueprint.route('/mount/points/all', methods=['POST'])
@login_required
def mount_points_all():
    username = current_user.username
    usertype = current_user.usertype
    print 'mount_points_all:', locals()

    # FIXME:
    if usertype != 'super':
        data = {}
        return jsonify(data)

    mounts = MountPoint.query.all()
    _mounts = objects_to_list(mounts)

    # insert host_name
    for _mount in _mounts:
        host = Host.query.get(_mount['host_id'])
        _mount['host_name'] = host.name

    data = {
        'mounts': _mounts,
    }

    return jsonify(data)

@mount_blueprint.route('/mount/point/create', methods=['POST'])
@login_required
def mount_create():
    username = current_user.username
    usertype = current_user.usertype
    _mount = request.json['mount']
    print 'mount_add:', locals()

    # FIXME:
    if usertype != 'super':
        data = {}
        return jsonify(data)

    # try to find range patterns
    if 'host' in _mount:
        host = _mount['host']
        ranges = list(re.findall('\[\w+\-\w+\]', host))
    else:
        ranges = None

    if ranges:
        # generate all combinations
        patterns = ranges
        ranges = [range.strip('[]').split('-') for range in ranges]

        for i, r in enumerate(ranges):
            s, e = r

            if s.isdigit() and e.isdigit():
                s, e = map(int, r)
                r = range(s, e + 1)
                ranges[i] = r
            else:
                # FIXME: support alpha-num ranges for multi-character strings
                #        with same size
                assert len(s) == len(e) == 1
                s, e = map(ord, r)
                r = range(s, e + 1)
                r = map(chr, r)
                r = ''.join(r)
                ranges[i] = r

        combs = product(*ranges)

        # generate mount points
        name = _mount['name']
        device = _mount['device']
        mountpoint = _mount['mountpoint']
        filesystem = _mount['filesystem']
        capacity = _mount['capacity']

        for comb in combs:
            _host_id = host
            _name = name
            _device = device
            _mountpoint = mountpoint
            _capacity = capacity

            for p, c in zip(patterns, comb):
                _host_id = _host_id.replace(p, c)
                _name = _name.replace(p, c)
                _device = _device.replace(p, c)
                _mountpoint = _mountpoint.replace(p, c)
                _capacity = _capacity.replace(p, c)

                __mount = {
                    'host_id': _host_id,
                    'name': _name,
                    'device': _device,
                    'mountpoint': _mountpoint,
                    'capacity': _capacity,
                }

                print __mount
    else:
        mount = MountPoint(**_mount)
        _mount['created'] = _mount['updated'] = datetime.utcnow()
        db.session.add(mount)
        db.session.commit()

    _mount = object_to_dict(mount)

    # insert host_name
    host = Host.query.get(_mount['host_id'])
    _mount['host_name'] = host.name

    data = {
        'mount': _mount,
    }

    return jsonify(data)

@mount_blueprint.route('/mount/point/update', methods=['POST'])
@login_required
def mount_update():
    username = current_user.username
    usertype = current_user.usertype
    _mount = request.json['mount']
    print 'mount_update:', locals()

    # FIXME:
    if usertype != 'super':
        data = {}
        return jsonify(data)

    mount = MountPoint.query.get(_mount['id'])
    _mount['updated'] = datetime.utcnow()
    update_object_with_dict(mount, _mount)
    db.session.commit()

    _mount = object_to_dict(mount)

    # insert host_name
    host = Host.query.get(_mount['host_id'])
    _mount['host_name'] = host.name

    data = {
        'mount': _mount,
    }

    return jsonify(data)

@mount_blueprint.route('/mount/point/remove', methods=['POST'])
@login_required
def mount_remove():
    username = current_user.username
    usertype = current_user.usertype
    id = request.json['id']
    print 'mount_remove:', locals()

    # FIXME:
    if usertype != 'super':
        data = {}
        return jsonify(data)

    mount = MountPoint.query.get(id)
    db.session.delete(mount)
    db.session.commit()

    data = {}
    return jsonify(data)
<|endoftext|>


<commit_before>import yaml
from boto.mturk.connection import MTurkConnection, MTurkRequestError
from mturk_tmpl import MturkTmpl, ReceiversTooMuch, TokenLength

class Mturk():
    def __init__(self):
        self.config = self.set_config()
        self.mturk = MTurkConnection(aws_access_key_id = self.config['aws_access_key_id'],
                                     aws_secret_access_key = self.config['aws_secret_access_key'],
                                     host = self.config['host'])
        self.mturk_tmpl  = MturkTmpl()

    def set_config(self, config_path = "config.yml"):
        with open(config_path, 'r') as file:
            config = yaml.load(file)
        return config

    def account_balance(self):
        account_balance = self.mturk.get_account_balance()
        print("Testing connection: You have a balance of: {}".format(account_balance))

    def remove_old_hits(self):
        # Disable old hits.
        for hit in self.mturk.get_all_hits():
            print("Hit {} has been removed.".format(hit.HITId))
            self.mturk.disable_hit(hit.HITId)

    def create_hit(self, data):
        # These parameters define the HIT that will be created
        # question is what we defined above
        # max_assignments is the # of unique Workers you're requesting
        # title, description, and keywords help Workers find your HIT
        # duration is the # of seconds Workers have to complete your HIT
        # reward is what Workers will be paid when you approve their work
        # Check out the documentation on CreateHIT for more details
        response = self.mturk.create_hit(
            question = self.mturk_tmpl.html_question(data),
            max_assignments=1,
            title = self.config['title'],
            description = self.config['description'],
            keywords = self.config['keywords'],
            duration = 120,
            reward = self.config['reward_amount']
        )
        return response

if __name__ == '__main__':
    m = Mturk()
    print(m.mturk.get_account_balance())<commit_msg>Wrap the api of getting all hits<commit_after>import yaml
from boto.mturk.connection import MTurkConnection, MTurkRequestError
from mturk_tmpl import MturkTmpl, ReceiversTooMuch, TokenLength

class Mturk():
    def __init__(self):
        self.config = self.set_config()
        self.mturk = MTurkConnection(aws_access_key_id = self.config['aws_access_key_id'],
                                     aws_secret_access_key = self.config['aws_secret_access_key'],
                                     host = self.config['host'])
        self.mturk_tmpl  = MturkTmpl()

    def set_config(self, config_path = "config.yml"):
        with open(config_path, 'r') as file:
            config = yaml.load(file)
        return config

    def account_balance(self):
        account_balance = self.mturk.get_account_balance()
        print("Testing connection: You have a balance of: {}".format(account_balance))

    def get_hits(self):
        return self.mturk.get_all_hits()

    def remove_old_hits(self):
        # Disable old hits.
        for hit in self.get_all_hits():
            print("Hit {} has been removed.".format(hit.HITId))
            self.mturk.disable_hit(hit.HITId)

    def create_hit(self, data):
        # These parameters define the HIT that will be created
        # question is what we defined above
        # max_assignments is the # of unique Workers you're requesting
        # title, description, and keywords help Workers find your HIT
        # duration is the # of seconds Workers have to complete your HIT
        # reward is what Workers will be paid when you approve their work
        # Check out the documentation on CreateHIT for more details
        response = self.mturk.create_hit(
            question = self.mturk_tmpl.html_question(data),
            max_assignments=1,
            title = self.config['title'],
            description = self.config['description'],
            keywords = self.config['keywords'],
            duration = 120,
            reward = self.config['reward_amount']
        )
        return response

if __name__ == '__main__':
    m = Mturk()
    print(m.mturk.get_account_balance())<|endoftext|>


<commit_before>import sys
import socket
import urllib
import re
import cgi
import xml.sax.saxutils
import webracer.agent

class Redirect(StandardError):
    pass

class OpenDnsDetected(StandardError):
    pass

opendns_ips = ['208.67.222.222', '208.67.220.220']

def check_opendns():
    with open('/etc/resolv.conf') as f:
        conf = f.read()
    words = re.split(r'\s+', conf)
    for ip in opendns_ips:
        if ip in words:
            return True
    return False

def replace(match):
    url = match.group(1)
    url = xml.sax.saxutils.unescape(url)
    url = url[:url.find('&')]
    #url = url[:url.find('&')]
    url = urllib.unquote(url)
    #print url
    url = xml.sax.saxutils.quoteattr(url)
    return 'href=%s' % url

def index(query_args):
    # workaround for http://my.opera.com/community/forums/topic.dml?id=486461
    if 'q' in query_args and 'start' not in query_args and \
        'sourceid' in query_args and query_args['sourceid'] == 'opera':
            host = query_args['q']
            if re.match(r'\w+$', host):
                if check_opendns():
                    return 'opendns detected, please fix'
                exc = None
                try:
                    hostname = socket.gethostbyname(host)
                    if hostname:
                        exc = Redirect('http://%s' % host)
                except:
                    #print sys.exc_info()
                    pass
                if exc is not None:
                    raise exc

    args = {}
    for key in ['q', 'start']:
        value = query_args.get(key)
        if value is not None:
            args[key] = value
    query = urllib.urlencode(args)
    if query:
        url = 'http://www.google.com/search?%s' % query
    else:
        url = 'http://www.google.com/'

    ua = webracer.agent.Agent(use_cookie_jar=False)
    # http://curl.haxx.se/mail/curlpython-2007-07/0001.html
    # curl insists on a str, not unicode, on python 2
    #url = url.encode('utf8')
    response = ua.get(url)
    content = response.body
    content = re.compile(r'<script[^>]*>.*?</script>', re.S).sub('', content)
    content = re.sub(r'href="/url\?q=([^"]+)"', replace, content)
    content = re.sub(r'href="/interstitial\?url=([^"]+)"', replace, content)
    # this causes forms to submit to google, not good
    #content = content.replace(r'<head>', '<head><base href="http://www.google.com/">')
    content = re.sub(r'url\(/(?!/)', 'url(http://www.google.com/', content)
    return content
<commit_msg>Dashes are allowed in dns names<commit_after>import sys
import socket
import urllib
import re
import cgi
import xml.sax.saxutils
import webracer.agent

class Redirect(StandardError):
    pass

class OpenDnsDetected(StandardError):
    pass

opendns_ips = ['208.67.222.222', '208.67.220.220']

def check_opendns():
    with open('/etc/resolv.conf') as f:
        conf = f.read()
    words = re.split(r'\s+', conf)
    for ip in opendns_ips:
        if ip in words:
            return True
    return False

def replace(match):
    url = match.group(1)
    url = xml.sax.saxutils.unescape(url)
    url = url[:url.find('&')]
    #url = url[:url.find('&')]
    url = urllib.unquote(url)
    #print url
    url = xml.sax.saxutils.quoteattr(url)
    return 'href=%s' % url

def index(query_args):
    # workaround for http://my.opera.com/community/forums/topic.dml?id=486461
    if 'q' in query_args and 'start' not in query_args and \
        'sourceid' in query_args and query_args['sourceid'] == 'opera':
            host = query_args['q']
            if re.match(r'[a-zA-Z0-9\-]+$', host):
                if check_opendns():
                    return 'opendns detected, please fix'
                exc = None
                try:
                    hostname = socket.gethostbyname(host)
                    if hostname:
                        exc = Redirect('http://%s' % host)
                except:
                    #print sys.exc_info()
                    pass
                if exc is not None:
                    raise exc

    args = {}
    for key in ['q', 'start']:
        value = query_args.get(key)
        if value is not None:
            args[key] = value
    query = urllib.urlencode(args)
    if query:
        url = 'http://www.google.com/search?%s' % query
    else:
        url = 'http://www.google.com/'

    ua = webracer.agent.Agent(use_cookie_jar=False)
    # http://curl.haxx.se/mail/curlpython-2007-07/0001.html
    # curl insists on a str, not unicode, on python 2
    #url = url.encode('utf8')
    response = ua.get(url)
    content = response.body
    content = re.compile(r'<script[^>]*>.*?</script>', re.S).sub('', content)
    content = re.sub(r'href="/url\?q=([^"]+)"', replace, content)
    content = re.sub(r'href="/interstitial\?url=([^"]+)"', replace, content)
    # this causes forms to submit to google, not good
    #content = content.replace(r'<head>', '<head><base href="http://www.google.com/">')
    content = re.sub(r'url\(/(?!/)', 'url(http://www.google.com/', content)
    return content
<|endoftext|>