<commit_before>from __future__ import absolute_import
from __future__ import with_statement

from celery.utils.compat import OrderedDict

class BaseModel(object):
    def __init__(self, app):
        self.app = app

    def __eq__(self, other):
        raise NotImplementedError

    def __ne__(self, other):
        return not self.__eq__(other)

class WorkersModel(BaseModel):
    def __init__(self, app):
        super(WorkersModel, self).__init__(app)
        self.workers = OrderedDict()

        state = self.app.state
        for workername, stat in sorted(state.stats.iteritems()):
            pool = stat.get('pool') or {}
            self.workers[workername] = dict(
                    status=(workername in state.ping),
                    concurrency=pool.get('max-concurrency'),
                    completed_tasks=sum(stat['total'].itervalues()),
                    running_tasks=state.active_tasks.get(workername, []),
                    queues=map(lambda x: x['name'],
                               state.active_queues.get(workername, [])),
                    )

    @classmethod
    def get_latest(cls, app):
        return WorkersModel(app)

    @classmethod
    def get_workers(cls, app):
        return app.state.stats.keys()

    @classmethod
    def is_worker(cls, app, workername):
        return WorkerModel.get_worker(app, workername) is not None

    def __eq__(self, other):
        return other is not None and self.workers == other.workers

class WorkerModel(BaseModel):
    def __init__(self, app, name):
        super(WorkerModel, self).__init__(app)

        state = self.app.state
        self.name = name
        self.stats = state.stats[name]
        self.active_tasks = state.active_tasks.get(name, {})
        self.scheduled_tasks = state.scheduled_tasks.get(name, {})
        self.active_queues = state.active_queues.get(name, {})
        self.revoked_tasks = state.revoked_tasks.get(name, [])
        self.registered_tasks = filter(lambda x: not x.startswith('celery.'),
                                       state.registered_tasks.get(name, {}))
        self.reserved_tasks = state.reserved_tasks.get(name, {})
        self.conf = state.conf.get(name, {})

    @classmethod
    def get_worker(self, app, name):
        if name not in app.state.stats:
            return None
        return WorkerModel(app, name)

    def __eq__(self, other):
        return self.name == other.name and self.stats == other.stats and\
               self.active_tasks == other.active_tasks and\
               self.active_queues == other.active_queues and\
               self.revoked_tasks == other.revoked_tasks and\
               self.registered_tasks == other.registered_tasks and\
               self.scheduled_tasks == other.scheduled_tasks and\
               self.reserved_tasks == other.reserved_tasks and\
               self.conf == other.conf

class TaskModel(BaseModel):
    def __init__(self, app, task_id):
        super(TaskModel, self).__init__(app)

        task = app.events.state.tasks[task_id]

        self._fields = task._defaults.keys()
        for name, value in task.info(fields=self._fields).iteritems():
            setattr(self, name, value)

    @classmethod
    def get_task_by_id(cls, app, task_id):
        try:
            return TaskModel(app, task_id)
        except KeyError:
            return None

    @classmethod
    def iter_tasks(cls, app, limit=None, type=None, worker=None, state=None):
        i = 0
        events_state = app.events.state
        for uuid, task in events_state._sort_tasks_by_time(
                events_state.itertasks()):
            if type and task.name != type:
                continue
            if worker and task.worker.hostname != worker:
                continue
            if state and task.state != state:
                continue
            yield uuid, task
            i += 1
            if i == limit:
                break

    @classmethod
    def seen_task_types(cls, app):
        return app.events.state.task_types()

    def __dir__(self):
        return self._fields
<commit_msg>Uses tasks sorted by logical clocks<commit_after>from __future__ import absolute_import
from __future__ import with_statement

from celery.utils.compat import OrderedDict

class BaseModel(object):
    def __init__(self, app):
        self.app = app

    def __eq__(self, other):
        raise NotImplementedError

    def __ne__(self, other):
        return not self.__eq__(other)

class WorkersModel(BaseModel):
    def __init__(self, app):
        super(WorkersModel, self).__init__(app)
        self.workers = OrderedDict()

        state = self.app.state
        for workername, stat in sorted(state.stats.iteritems()):
            pool = stat.get('pool') or {}
            self.workers[workername] = dict(
                    status=(workername in state.ping),
                    concurrency=pool.get('max-concurrency'),
                    completed_tasks=sum(stat['total'].itervalues()),
                    running_tasks=state.active_tasks.get(workername, []),
                    queues=map(lambda x: x['name'],
                               state.active_queues.get(workername, [])),
                    )

    @classmethod
    def get_latest(cls, app):
        return WorkersModel(app)

    @classmethod
    def get_workers(cls, app):
        return app.state.stats.keys()

    @classmethod
    def is_worker(cls, app, workername):
        return WorkerModel.get_worker(app, workername) is not None

    def __eq__(self, other):
        return other is not None and self.workers == other.workers

class WorkerModel(BaseModel):
    def __init__(self, app, name):
        super(WorkerModel, self).__init__(app)

        state = self.app.state
        self.name = name
        self.stats = state.stats[name]
        self.active_tasks = state.active_tasks.get(name, {})
        self.scheduled_tasks = state.scheduled_tasks.get(name, {})
        self.active_queues = state.active_queues.get(name, {})
        self.revoked_tasks = state.revoked_tasks.get(name, [])
        self.registered_tasks = filter(lambda x: not x.startswith('celery.'),
                                       state.registered_tasks.get(name, {}))
        self.reserved_tasks = state.reserved_tasks.get(name, {})
        self.conf = state.conf.get(name, {})

    @classmethod
    def get_worker(self, app, name):
        if name not in app.state.stats:
            return None
        return WorkerModel(app, name)

    def __eq__(self, other):
        return self.name == other.name and self.stats == other.stats and\
               self.active_tasks == other.active_tasks and\
               self.active_queues == other.active_queues and\
               self.revoked_tasks == other.revoked_tasks and\
               self.registered_tasks == other.registered_tasks and\
               self.scheduled_tasks == other.scheduled_tasks and\
               self.reserved_tasks == other.reserved_tasks and\
               self.conf == other.conf

class TaskModel(BaseModel):
    def __init__(self, app, task_id):
        super(TaskModel, self).__init__(app)

        task = app.events.state.tasks[task_id]

        self._fields = task._defaults.keys()
        for name, value in task.info(fields=self._fields).iteritems():
            setattr(self, name, value)

    @classmethod
    def get_task_by_id(cls, app, task_id):
        try:
            return TaskModel(app, task_id)
        except KeyError:
            return None

    @classmethod
    def iter_tasks(cls, app, limit=None, type=None, worker=None, state=None):
        i = 0
        events_state = app.events.state
        for uuid, task in events_state.tasks_by_time():
            if type and task.name != type:
                continue
            if worker and task.worker.hostname != worker:
                continue
            if state and task.state != state:
                continue
            yield uuid, task
            i += 1
            if i == limit:
                break

    @classmethod
    def seen_task_types(cls, app):
        return app.events.state.task_types()

    def __dir__(self):
        return self._fields
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')

import nltk
import os
import re
import string
import subprocess
import time
import threading
import urllib2

class GoogleTranslateAPI(threading.Thread):
    """
    doc to be made
    """
    def __init__(self, language='en', text='Service provided by Baidu'):
        threading.Thread.__init__(self)
        self.language = language
        self.text = text
        self.result = None

    def run(self):
        response = subprocess.Popen('''curl -A Mozilla "http://translate.google.com/translate_tts?tl=%s&q=%s"''' % (self.language, urlib2.quote(self.text)), stdout=subprocess.PIPE, shell=True)
        content, error = response.communicate()
        if not error and content:
            if 'error' not in content or 'permission' not in content:
                self.result = content
        raise Exception

# Todos
# rename the file and variables
# remove accepting command line calls
def google(language='en', query='Service provided by Baidu', output_path='out.mp3'):
    """
    1. download mp3 from google tts api
    2. convert it to wav
    3. speed up the wav file, if necessary
    4. convert to mp3
    5. store in some location
    6. return the path
    """
    # generate out.mp3
    download(language, query, output_path)
    subprocess.Popen(
        'tts="%s"; lame --decode $tts - | sox -t wav - -t wav - speed 1.06 | lame - $tts' % output_path, stderr=subprocess.PIPE, shell=True)

# Todos
# to write some boundary checkers
def query_segment(language='en', query='Service provided by Baidu'):
    '''
    remove after implementing line 91: the algorithm only now works for latins
    '''
    query = query.strip()
    sentences = nltk.sent_tokenize(query)
    parts = []
    for sentence in sentences:
        if len(sentence) < 99:
            # none of len(item) in parts will exceed 100
            parts.append(sentence)  # parts: ['xxx, xxx', 'yyy zzz aaa bbb.']
        else:
            phrases = sentence.split(',') # phrases: ['xxx -- xxx', 'yyy zzz aaa']
            for phrase in phrases:
                if len(phrase) < 99:
                    parts.append(phrase)
                else:
                    if language == 'en' | language == 'pt' | language == 'id':
                        words = phrase.split(' ')
                        # none of len(item) in combined_words will exceed 100
                        combined_words = ""  # combined_words = ['yyy zzz. aaa bbb']
                        for word in words:
                            if len(combined_words) + len(word) + 1 < 100: # +1 for possible space
                                combined_words = ("""%s %s""" if word not in string.punctuation else """%s%s""") % (combined_words, word)
                            else:
                                parts.append(combined_words)
                                combined_words = word
                        if combined_words:
                            parts.append(combined_words)
                    # -------------------------- #
                    #  \           |           / #
                    #  _  IMPLEMENT THIS PART _  #
                    #    AS SOON AS POSSIBLE!    #
                    #  /           |           \ #
                    # -------------------------- #
                    else: # ja, th, ar
                        # Todos
                        # determine how do these languages separate words
                        pass
    segments = []
    # a higher-level version of 'combined_words' algorithm
    segment = ""
    for part in parts:
        if len(segment) + len(part) + 1 < 100:
            segment = """%s %s""" % (segment, part)
        else:
            segments.append(segment)
            segment = part
    segments.append(segment)
    print 'after some serious thoughts, we get these:'
    for segment in segments:
        print segment
    return segments

# Todos
# Test! Test! Test!
# docs!
def download(language='en', query='Service provided by Baidu', output='do_not_exists.mp3'):
    '''
    docs needed!
    other ways to write download
    1. https://github.com/hungtruong/Google-Translate-TTS/blob/master/GoogleTTS.py
    2. https://github.com/gavinmh/tts-api/blob/master/text_segmenter.py
    '''
    segments = query_segment(language, query)

    # download chunks and write them to the output file
    try:
        threads = []
        for segment in segments:
            if segment:
                print 'transmitting ... %s' % segment
                gt_request = GoogleTranslateAPI(language, segment)
                threads.append(gt_request)
                gt_request.start()
                gt_request.join(2*1000)
        out = open(output, 'a')
        for th in threads:
            sys.stdout.write('.')
            sys.stdout.flush()
            if th.result:
                out.write(th.result)
            else:
                raise Exception
        out.close()
    except Exception as e:
        if os.path.exists(output):
            os.remove(output)
    print
<commit_msg>autopep8ed<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')

import nltk
import os
import re
import string
import subprocess
import time
import threading
import urllib2

class GoogleTranslateAPI(threading.Thread):

    """
    doc to be made
    """

    def __init__(self, language='en', text='Service provided by Baidu'):
        threading.Thread.__init__(self)
        self.language = language
        self.text = text
        self.result = None

    def run(self):
        response = subprocess.Popen(
            '''curl -A Mozilla "http://translate.google.com/translate_tts?tl=%s&q=%s"''' %
            (self.language, urlib2.quote(self.text)), stdout=subprocess.PIPE, shell=True)
        content, error = response.communicate()
        if not error and content:
            if 'error' not in content or 'permission' not in content:
                self.result = content
        raise Exception

# Todos
# rename the file and variables
# remove accepting command line calls
def google(language='en', query='Service provided by Baidu', output_path='out.mp3'):
    """
    1. download mp3 from google tts api
    2. convert it to wav
    3. speed up the wav file, if necessary
    4. convert to mp3
    5. store in some location
    6. return the path
    """
    # generate out.mp3
    download(language, query, output_path)
    subprocess.Popen(
        'tts="%s"; lame --decode $tts - | sox -t wav - -t wav - speed 1.06 | lame - $tts' % output_path, stderr=subprocess.PIPE, shell=True)

# Todos
# to write some boundary checkers
def query_segment(language='en', query='Service provided by Baidu'):
    '''
    remove after implementing line 91: the algorithm only now works for latins
    '''
    query = query.strip()
    sentences = nltk.sent_tokenize(query)
    parts = []
    for sentence in sentences:
        if len(sentence) < 99:
            # none of len(item) in parts will exceed 100
            parts.append(sentence)  # parts: ['xxx, xxx', 'yyy zzz aaa bbb.']
        else:
            # phrases: ['xxx -- xxx', 'yyy zzz aaa']
            phrases = sentence.split(',')
            for phrase in phrases:
                if len(phrase) < 99:
                    parts.append(phrase)
                else:
                    if language == 'en' | language == 'pt' | language == 'id':
                        words = phrase.split(' ')
                        # none of len(item) in combined_words will exceed 100
                        # combined_words = ['yyy zzz. aaa bbb']
                        combined_words = ""
                        for word in words:
                            # +1 for possible space
                            if len(combined_words) + len(word) + 1 < 100:
                                combined_words = ("""%s %s""" if word not in string.punctuation else """%s%s""") % (
                                    combined_words, word)
                            else:
                                parts.append(combined_words)
                                combined_words = word
                        if combined_words:
                            parts.append(combined_words)
                    # -------------------------- #
                    # \           |           / #
                    # _  IMPLEMENT THIS PART _  #
                    # AS SOON AS POSSIBLE!    #
                    # /           |           \ #
                    # -------------------------- #
                    else:  # ja, th, ar
                        # Todos
                        # determine how do these languages separate words
                        pass
    segments = []
    # a higher-level version of 'combined_words' algorithm
    segment = ""
    for part in parts:
        if len(segment) + len(part) + 1 < 100:
            segment = """%s %s""" % (segment, part)
        else:
            segments.append(segment)
            segment = part
    segments.append(segment)
    print 'after some serious thoughts, we get these:'
    for segment in segments:
        print segment
    return segments

# Todos
# Test! Test! Test!
# docs!
def download(language='en', query='Service provided by Baidu', output='do_not_exists.mp3'):
    '''
    docs needed!
    other ways to write download
    1. https://github.com/hungtruong/Google-Translate-TTS/blob/master/GoogleTTS.py
    2. https://github.com/gavinmh/tts-api/blob/master/text_segmenter.py
    '''
    segments = query_segment(language, query)

    # download chunks and write them to the output file
    try:
        threads = []
        for segment in segments:
            if segment:
                print 'transmitting ... %s' % segment
                gt_request = GoogleTranslateAPI(language, segment)
                threads.append(gt_request)
                gt_request.start()
                gt_request.join(2 * 1000)
        out = open(output, 'a')
        for th in threads:
            sys.stdout.write('.')
            sys.stdout.flush()
            if th.result:
                out.write(th.result)
            else:
                raise Exception
        out.close()
    except Exception as e:
        if os.path.exists(output):
            os.remove(output)
    print
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) Björn Edström <be@bjrn.se> 2015. See LICENSE for details.

# WORK IN PROGRESS

import base64
import bcrypt
import ed25519
import hashlib
import os
import re
import struct

class SignifyError(Exception):
    pass

class InvalidSignature(SignifyError):
    pass

def xorbuf(buf1, buf2):
    res = []
    for i in range(len(buf1)):
        res.append(chr(ord(buf1[i]) ^ ord(buf2[i])))
    return ''.join(res)

class Signify(object):
    def __init__(self):
        pass

    def _decrypt_secret_key(self, blob, password):
        try:
            comment, b64 = blob.split('\n', 1)
        except:
            raise SignifyError('malformed blob')

        extracted_comment = re.findall(r'^untrusted comment: (.*?) secret key$', comment)
        if extracted_comment:
            extracted_comment = extracted_comment[0]
        else:
            extracted_comment = 'signify'

        try:
            buf = base64.b64decode(b64)
            pkalg, kdfalg, kdfrounds, salt, checksum, keynum, seckey = \
                struct.unpack('!2s2sL16s8s8s64s', buf)

            assert pkalg == 'Ed'
            assert kdfalg == 'BK'
            assert kdfrounds in [0, 42]
        except Exception, e:
            raise SignifyError(e)

        if kdfrounds == 0:
            xorkey = '\x00' * 64
        else:
            xorkey = bcrypt.kdf(password, salt, 64, kdfrounds)
        priv = xorbuf(seckey, xorkey)

        checksum_ref = hashlib.sha512(priv).digest()[0:8]
        if checksum != checksum_ref:
            raise KeyError('incorrect password')

        return priv, keynum, extracted_comment

    def _parse_public_key(self, blob):
        try:
            comment, b64 = blob.split('\n', 1)

            buf = base64.b64decode(b64)
            pkalg, keynum, pubkey = \
                struct.unpack('!2s8s32s', buf)

            assert pkalg == 'Ed'
        except Exception, e:
            raise SignifyError('malformed public key')

        return pubkey, keynum

    def _parse_sigfile(self, blob):
        try:
            comment, b64 = blob.split('\n', 1)

            buf = base64.b64decode(b64)
            pkalg, keynum, sig = \
                struct.unpack('!2s8s64s', buf)

            assert pkalg == 'Ed'
        except Exception, e:
            raise SignifyError('malformed signature blob')

        return sig, keynum

    def generate(self, comment, password):
        """Generate a signify keypair.

        @param comment: A comment to name the keypair, or None.
        @param password: A password to protect the private key, or None.
        """

        if comment is None:
            comment = 'signify'

        sk, vk = ed25519.keys.create_keypair()
        keynum = os.urandom(8)

        # private key
        sk_buf = sk.to_bytes()
        #print [sk_buf]
        kdfrounds = 42
        salt = os.urandom(16)

        if password is None:
            kdfrounds = 0
            xorkey = '\x00' * 64
        else:
            xorkey = bcrypt.kdf(password, salt, 64, kdfrounds)
        protected_key = xorbuf(xorkey, sk_buf)
        checksum = hashlib.sha512(sk_buf).digest()[0:8]

        priv = 'untrusted comment: ' + comment + ' secret key\n' + \
               base64.b64encode('Ed' + 'BK' + struct.pack('!L', kdfrounds) + \
                                salt + checksum + keynum + protected_key) + '\n'

        # public key
        vk_buf = vk.to_bytes()
        pub = 'untrusted comment: ' + comment + ' public key\n' + \
              base64.b64encode('Ed' + keynum + vk_buf) + '\n'

        return pub, priv

    def sign_simple(self, priv, password, message):
        """Sign message with the private key.

        @param priv: private key blob
        @param password: The password that protects the private key, or None.
        @param message: The message to be signed.
        """

        priv_buf, priv_keynum, comment = self._decrypt_secret_key(priv, password)
        key_obj = ed25519.keys.SigningKey(priv_buf)

        sig_buf = key_obj.sign(message)

        return 'untrusted comment: signature from ' + comment + ' secret key\n' + \
            base64.b64encode('Ed' + priv_keynum + sig_buf) + '\n'

    def verify_simple(self, pubkey, sig, message):
        """Perform signature verification.

        throws InvalidSignature on error.

        @param pubkey: The public key to verify against.
        @param sig: The signature blob.
        @param message: The message that was signed.
        """

        sig_buf, sig_keynum = self._parse_sigfile(sig)
        pub_buf, pub_keynum = self._parse_public_key(pubkey)

        key_obj = ed25519.keys.VerifyingKey(pub_buf)
        try:
            key_obj.verify(sig_buf, message)
            return True
        except ed25519.BadSignatureError, e:
            raise InvalidSignature('signify: signature verification failed')
<commit_msg>Refactor message parsing and writing.<commit_after># -*- coding: utf-8 -*-
# Copyright (c) Björn Edström <be@bjrn.se> 2015. See LICENSE for details.

# WORK IN PROGRESS

import base64
import bcrypt
import ed25519
import hashlib
import os
import re
import struct

class SignifyError(Exception):
    pass

class InvalidSignature(SignifyError):
    pass

def xorbuf(buf1, buf2):
    res = []
    for i in range(len(buf1)):
        res.append(chr(ord(buf1[i]) ^ ord(buf2[i])))
    return ''.join(res)

def write_message(comment, blob):
    return 'untrusted comment: %s\n%s\n' % (comment, base64.b64encode(blob))

def read_message(msg):
    try:
        comment_line, b64 = msg.split('\n', 1)
    except:
        raise SignifyError('malformed message')

    try:
        blob = base64.b64decode(b64)
    except:
        raise SignifyError('malformed message: base64 error')

    try:
        comment = re.findall(r'^untrusted comment: (.*?)$', comment_line)[0]
    except:
        raise SignifyError('malformed message: expected a comment line')

    return (comment, blob)

class Signify(object):
    def __init__(self):
        pass

    def _decrypt_secret_key(self, msg, password):
        comment, blob = read_message(msg)

        try:
            pkalg, kdfalg, kdfrounds, salt, checksum, keynum, seckey = \
                struct.unpack('!2s2sL16s8s8s64s', blob)

            assert pkalg == 'Ed'
            assert kdfalg == 'BK'
            assert kdfrounds in [0, 42]
        except Exception, e:
            raise SignifyError(e)

        if kdfrounds == 0:
            xorkey = '\x00' * 64
        else:
            xorkey = bcrypt.kdf(password, salt, 64, kdfrounds)
        priv = xorbuf(seckey, xorkey)

        checksum_ref = hashlib.sha512(priv).digest()[0:8]
        if checksum != checksum_ref:
            raise KeyError('incorrect password')

        return priv, keynum, comment

    def _parse_public_key(self, msg):
        comment, blob = read_message(msg)

        try:
            pkalg, keynum, pubkey = \
                struct.unpack('!2s8s32s', blob)

            assert pkalg == 'Ed'
        except Exception, e:
            raise SignifyError('malformed public key')

        return pubkey, keynum

    def _parse_sigfile(self, msg):
        comment, blob = read_message(msg)

        try:
            pkalg, keynum, sig = \
                struct.unpack('!2s8s64s', blob)

            assert pkalg == 'Ed'
        except Exception, e:
            raise SignifyError('malformed signature blob')

        return sig, keynum

    def generate(self, comment, password):
        """Generate a signify keypair.

        @param comment: A comment to name the keypair, or None.
        @param password: A password to protect the private key, or None.
        """

        if comment is None:
            comment = 'signify'

        sk, vk = ed25519.keys.create_keypair()
        keynum = os.urandom(8)

        # private key
        sk_buf = sk.to_bytes()
        #print [sk_buf]
        kdfrounds = 42
        salt = os.urandom(16)

        if password is None:
            kdfrounds = 0
            xorkey = '\x00' * 64
        else:
            xorkey = bcrypt.kdf(password, salt, 64, kdfrounds)
        protected_key = xorbuf(xorkey, sk_buf)
        checksum = hashlib.sha512(sk_buf).digest()[0:8]

        priv_blob = 'Ed' + 'BK' + struct.pack('!L', kdfrounds) + \
                    salt + checksum + keynum + protected_key
        priv = write_message('%s secret key' % (comment,), priv_blob)

        # public key
        vk_buf = vk.to_bytes()
        pub_blob = 'Ed' + keynum + vk_buf
        pub = write_message('%s public key' % (comment,), pub_blob)

        return pub, priv

    def sign_simple(self, priv, password, message):
        """Sign message with the private key.

        @param priv: private key blob
        @param password: The password that protects the private key, or None.
        @param message: The message to be signed.
        """

        priv_buf, priv_keynum, comment = self._decrypt_secret_key(priv, password)
        key_obj = ed25519.keys.SigningKey(priv_buf)

        sig_buf = key_obj.sign(message)

        sig_blob = 'Ed' + priv_keynum + sig_buf
        return write_message('signature from %s' % (comment,), sig_blob)

    def verify_simple(self, pubkey, sig, message):
        """Perform signature verification.

        throws InvalidSignature on error.

        @param pubkey: The public key to verify against.
        @param sig: The signature blob.
        @param message: The message that was signed.
        """

        sig_buf, sig_keynum = self._parse_sigfile(sig)
        pub_buf, pub_keynum = self._parse_public_key(pubkey)

        key_obj = ed25519.keys.VerifyingKey(pub_buf)
        try:
            key_obj.verify(sig_buf, message)
            return True
        except ed25519.BadSignatureError, e:
            raise InvalidSignature('signify: signature verification failed')
<|endoftext|>


<commit_before>toc_sam = [
    # {
    #     'name': 'header',
    #     'content': 'README.md',
    #     'fingerprints': {
    #         'starter': None,
    #         'stopper': '### '
    #     }
    # },
    {
        "name": "description",
        "content": "SMART_AGENT_MONITOR.md",
        "fingerprints": {"starter": "## Description", "stopper": "## "},
    },
    {
        "name": "requirements and dependencies",
        "content": "README.md",
        "fingerprints": {"starter": "### REQUIREMENTS AND DEPENDENCIES", "stopper": "### "},
    },
    {
        "name": "installation",
        "content": "SMART_AGENT_MONITOR.md",
        "fingerprints": {
            "starter": "### INSTALLATION",
            "stopper": "<!--- GENERATED BY gomplate from scripts/docs/monitor-page.md.tmpl --->",
        },
    },
    {
        "name": "configuration",
        "content": "SMART_AGENT_MONITOR.md",
        "fingerprints": {"starter": "## Configuration", "stopper": "## "},
    },
    {"name": "usage", "content": "README.md", "fingerprints": {"starter": "### USAGE", "stopper": "### "}},
    {
        "name": "metrics",
        "content": "SMART_AGENT_MONITOR.md",
        "fingerprints": {"starter": "## Metrics", "stopper": "## "},
    },
]

toc_non_sam = [
    # {
    #     'name': 'header',
    #     'content': 'README.md',
    #     'fingerprints': {
    #         'starter': None,
    #         'stopper': '### DESCRIPTION'
    #     }
    # },
    {"name": "description", "content": "README.md", "fingerprints": {"starter": "### DESCRIPTION", "stopper": "### "}},
    {
        "name": "requirements and dependencies",
        "content": "README.md",
        "fingerprints": {"starter": "### REQUIREMENTS AND DEPENDENCIES", "stopper": "### "},
    },
    {
        "name": "installation",
        "content": "README.md",
        "fingerprints": {"starter": "### INSTALLATION", "stopper": "### "},
    },
    {
        "name": "configuration",
        "content": "README.md",
        "fingerprints": {"starter": "### CONFIGURATION", "stopper": "### "},
    },
    {"name": "usage", "content": "README.md", "fingerprints": {"starter": "### USAGE", "stopper": "### "}},
    {"name": "metrics", "content": "README.md", "fingerprints": {"starter": "### METRICS", "stopper": "### "}},
]
<commit_msg>Handle new template in to-product-docs (#760)<commit_after>toc_sam = [
    # {
    #     'name': 'header',
    #     'content': 'README.md',
    #     'fingerprints': {
    #         'starter': None,
    #         'stopper': '### '
    #     }
    # },
    {
        "name": "description",
        "content": "SMART_AGENT_MONITOR.md",
        "fingerprints": {"starter": "## Description", "stopper": "## "},
    },
    {
        "name": "requirements and dependencies",
        "content": "README.md",
        "fingerprints": {"starter": "### REQUIREMENTS AND DEPENDENCIES", "stopper": "### "},
    },
    {
        "name": "installation",
        "content": "SMART_AGENT_MONITOR.md",
        "fingerprints": {
            "starter": "### INSTALLATION",
            "stopper": "<!--- GENERATED BY ",
        },
    },
    {
        "name": "configuration",
        "content": "SMART_AGENT_MONITOR.md",
        "fingerprints": {"starter": "## Configuration", "stopper": "## "},
    },
    {"name": "usage", "content": "README.md", "fingerprints": {"starter": "### USAGE", "stopper": "### "}},
    {
        "name": "metrics",
        "content": "SMART_AGENT_MONITOR.md",
        "fingerprints": {"starter": "## Metrics", "stopper": "## "},
    },
]

toc_non_sam = [
    # {
    #     'name': 'header',
    #     'content': 'README.md',
    #     'fingerprints': {
    #         'starter': None,
    #         'stopper': '### DESCRIPTION'
    #     }
    # },
    {"name": "description", "content": "README.md", "fingerprints": {"starter": "### DESCRIPTION", "stopper": "### "}},
    {
        "name": "requirements and dependencies",
        "content": "README.md",
        "fingerprints": {"starter": "### REQUIREMENTS AND DEPENDENCIES", "stopper": "### "},
    },
    {
        "name": "installation",
        "content": "README.md",
        "fingerprints": {"starter": "### INSTALLATION", "stopper": "### "},
    },
    {
        "name": "configuration",
        "content": "README.md",
        "fingerprints": {"starter": "### CONFIGURATION", "stopper": "### "},
    },
    {"name": "usage", "content": "README.md", "fingerprints": {"starter": "### USAGE", "stopper": "### "}},
    {"name": "metrics", "content": "README.md", "fingerprints": {"starter": "### METRICS", "stopper": "### "}},
]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# @Author: ZwEin
# @Date:   2016-08-11 14:17:25
# @Last Modified by:   ZwEin
# @Last Modified time: 2016-08-14 15:31:46

DC_LABEL_NAME = 'label'

class Labeller(object):

    @staticmethod
    def label(data_path):
        reader = csv.reader(codecs.open(data_path, 'r'))
        # print reader.fieldnames
        header = next(reader)
        dataset = []
        for row in reader:
            os.system('cls' if os.name == 'nt' else 'clear')
            data = {}
            if int(row[0]) == -1:
                # print info
                print '#'*50
                print '# Post Features'
                print '#'*50
                for i in range(2, len(header)):
                    print header[i]+':', row[header.index(header[i])]
                print '#'*50
                print '# Post Content'
                print '#'*50
                print row[1]
                print '#'*50

                # ask user to label
                user_defined_label = raw_input('Please enter the label: (others: 1, massage: 2, escort: 3, job_ads: 4)\n')
                print 'You enter:', user_defined_label
                print '#'*50

                data.setdefault(DC_LABEL_NAME, user_defined_label)
            else:
                data.setdefault(DC_LABEL_NAME, row[0])

            for field in header:
                if field == DC_LABEL_NAME:
                    continue
                data.setdefault(field, row[header.index(field)])
            dataset.append(data)

        with open(data_path, 'w') as csvfile:
            writer = csv.DictWriter(csvfile, fieldnames=[DC_LABEL_NAME]+header)
            writer.writeheader()
            for data in dataset:
                writer.writerow(data)

if __name__ == '__main__':
    from loader import *
    Labeller.label(DC_DEFAULT_TRAINING_DATA_FILEPATH)
<commit_msg>clean up labelled<commit_after># -*- coding: utf-8 -*-
# @Author: ZwEin
# @Date:   2016-08-11 14:17:25
# @Last Modified by:   ZwEin
# @Last Modified time: 2016-08-21 20:54:17

DC_LABEL_NAME = 'label'

class Labeller(object):

    @staticmethod
    def label(data_path):
        reader = csv.reader(codecs.open(data_path, 'r'))

        header = next(reader)
        dataset = []
        for row in reader:
            os.system('cls' if os.name == 'nt' else 'clear')
            data = {}
            if int(row[0]) == -1:

                print '#'*50
                print '# Post Features'
                print '#'*50
                for i in range(2, len(header)):
                    print header[i]+':', row[header.index(header[i])]
                print '#'*50
                print '# Post Content'
                print '#'*50
                print row[1]
                print '#'*50

                # ask user to label
                user_defined_label = raw_input('Please enter the label: (others: 1, massage: 2, escort: 3, job_ads: 4)\n')
                print 'You enter:', user_defined_label
                print '#'*50

                data.setdefault(DC_LABEL_NAME, user_defined_label)
            else:
                data.setdefault(DC_LABEL_NAME, row[0])

            for field in header:
                if field == DC_LABEL_NAME:
                    continue
                data.setdefault(field, row[header.index(field)])
            dataset.append(data)

        with open(data_path, 'w') as csvfile:
            writer = csv.DictWriter(csvfile, fieldnames=[DC_LABEL_NAME]+header)
            writer.writeheader()
            for data in dataset:
                writer.writerow(data)

if __name__ == '__main__':
    from loader import *
    Labeller.label(DC_DEFAULT_TRAINING_DATA_FILEPATH)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

__LIBRARY_VERSION__ = '1.1.4'
__SII_VERSION__ = '1.0'
<commit_msg>Bump to v1.1.5<commit_after># -*- coding: utf-8 -*-

__LIBRARY_VERSION__ = '1.1.5'
__SII_VERSION__ = '1.0'
<|endoftext|>


<commit_before># Miro Community - Easiest way to make a video website
#
# Copyright (C) 2010, 2011, 2012 Participatory Culture Foundation
#
# Miro Community is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# Miro Community is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Miro Community.  If not, see <http://www.gnu.org/licenses/>.

import urllib
import logging

from django.contrib.auth.models import User
from django.core.mail import EmailMessage
from django.core.files.base import ContentFile
from django.db import models
from django.db.models import signals
from django.template import Context, loader
from django.utils.safestring import mark_safe
from socialauth.models import TwitterUserProfile, FacebookUserProfile

class Profile(models.Model):
    """
    Some extra data that we store about users.  Gets linked to a User object
    through the Django authentication system.
    """
    user = models.ForeignKey('auth.User')
    logo = models.ImageField(upload_to="localtv/profile_logos", blank=True,
                             verbose_name='Image')
    location = models.CharField(max_length=200, blank=True, default='')
    description = models.TextField(blank=True, default='')
    website = models.URLField(blank=True, default='')

    class Meta:
        db_table = 'localtv_profile'

    def __unicode__(self):
        return unicode(self.user)

def twitteruserprofile_created(sender, instance=None, raw=None, created=False,
                               **kwargs):
    if not created:
        return # we don't care about updates
    profile = Profile.objects.create(
        user=instance.user,
        location=instance.location or '',
        description=instance.description or '',
        website=instance.url or '')
    if instance.profile_image_url:
        try:
            cf = ContentFile(urllib.urlopen(
                    instance.profile_image_url).read())
        except Exception:
            pass
        else:
            cf.name = instance.profile_image_url
            profile.logo = cf
            profile.save()

def facebookuserprofile_created(sender, instance=None, raw=None, created=False,
                                **kwargs):
    if not created:
        return # we don't care about updates
    Profile.objects.create(
        user=instance.user,
        location=instance.location or '',
        description=instance.about_me or '',
        website=instance.url or '')

signals.post_save.connect(twitteruserprofile_created,
                          sender=TwitterUserProfile)
signals.post_save.connect(facebookuserprofile_created,
                          sender=FacebookUserProfile)

### On creating a new user, if the user has an email address
### XXX If you make changes to the way users are auto-created on video import,
### pay attention to this to make sure that we don't email people who don't care.
def on_user_create_send_welcomed_email(sender, instance=None, raw=None, created=False, **kwargs):
    if not created:
        return # We only care about *new* users.

    # If this is the only user, then skip the email sending.
    if User.objects.all().count() <= 1:
        # We stop right here, and refuse to send email.
        return

    # Note: We're extra careful here: if the user does not have a login-able password,
    # perhaps because the user was created through OpenID or Twitter or Facebook,
    # bail out.
    if not instance.has_usable_password():
        return

    if not instance.email:
        return # We only care about users with email addresses.

    ### Well, in that case, let's send the user a welcome email.
    import localtv.models

    site = localtv.models.SiteSettings.objects.get_current().site

    t = loader.get_template('localtv/user_profile/welcome_message.txt')
    c = Context({'site': site,
                 'user': instance})
    subject = "Welcome to %s" % site.name
    message = t.render(c)

    from django.conf import settings
    EmailMessage(subject, message, settings.DEFAULT_FROM_EMAIL,
                 [instance.email]).send(fail_silently=True)

signals.post_save.connect(on_user_create_send_welcomed_email,
                          sender=User)
<commit_msg>Abstracted out a BaseProfile model and tweaked the profile signals to use whatever profile model is used by the project.<commit_after># Miro Community - Easiest way to make a video website
#
# Copyright (C) 2010, 2011, 2012 Participatory Culture Foundation
#
# Miro Community is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# Miro Community is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Miro Community.  If not, see <http://www.gnu.org/licenses/>.

import urllib
import logging

from django.contrib.auth.models import User
from django.core.mail import EmailMessage
from django.core.files.base import ContentFile
from django.db import models
from django.db.models import signals
from django.template import Context, loader
from django.utils.safestring import mark_safe
from socialauth.models import TwitterUserProfile, FacebookUserProfile

from localtv.utils import get_profile_model

class BaseProfile(models.Model):
    """
    Some extra data that we store about users.  Gets linked to a User object
    through the Django authentication system.
    """
    user = models.ForeignKey('auth.User')
    logo = models.ImageField(upload_to="localtv/profile_logos", blank=True,
                             verbose_name='Image')
    location = models.CharField(max_length=200, blank=True, default='')
    description = models.TextField(blank=True, default='')
    website = models.URLField(blank=True, default='')

    class Meta:
        abstract = True

    def __unicode__(self):
        return u"%s for %s" % (self.__class__.__name__, unicode(self.user))

class Profile(BaseProfile):
    class Meta:
        db_table = 'localtv_profile'

def twitteruserprofile_created(sender, instance=None, raw=None, created=False,
                               **kwargs):
    if not created:
        return # we don't care about updates
    profile = get_profile_model().objects.create(
        user_id=instance.user_id,
        location=instance.location or '',
        description=instance.description or '',
        website=instance.url or '')
    if instance.profile_image_url:
        try:
            cf = ContentFile(urllib.urlopen(
                    instance.profile_image_url).read())
        except Exception:
            pass
        else:
            cf.name = instance.profile_image_url
            profile.logo = cf
            profile.save()

def facebookuserprofile_created(sender, instance=None, raw=None, created=False,
                                **kwargs):
    if not created:
        return # we don't care about updates
    get_profile_model().objects.create(
        user_id=instance.user_id,
        location=instance.location or '',
        description=instance.about_me or '',
        website=instance.url or '')

signals.post_save.connect(twitteruserprofile_created,
                          sender=TwitterUserProfile)
signals.post_save.connect(facebookuserprofile_created,
                          sender=FacebookUserProfile)

### On creating a new user, if the user has an email address
### XXX If you make changes to the way users are auto-created on video import,
### pay attention to this to make sure that we don't email people who don't care.
def on_user_create_send_welcomed_email(sender, instance=None, raw=None, created=False, **kwargs):
    if not created:
        return # We only care about *new* users.

    # If this is the only user, then skip the email sending.
    if User.objects.all().count() <= 1:
        # We stop right here, and refuse to send email.
        return

    # Note: We're extra careful here: if the user does not have a login-able password,
    # perhaps because the user was created through OpenID or Twitter or Facebook,
    # bail out.
    if not instance.has_usable_password():
        return

    if not instance.email:
        return # We only care about users with email addresses.

    ### Well, in that case, let's send the user a welcome email.
    import localtv.models

    site = localtv.models.SiteSettings.objects.get_current().site

    t = loader.get_template('localtv/user_profile/welcome_message.txt')
    c = Context({'site': site,
                 'user': instance})
    subject = "Welcome to %s" % site.name
    message = t.render(c)

    from django.conf import settings
    EmailMessage(subject, message, settings.DEFAULT_FROM_EMAIL,
                 [instance.email]).send(fail_silently=True)

signals.post_save.connect(on_user_create_send_welcomed_email,
                          sender=User)
<|endoftext|>


<commit_before>import os
import posixpath
import uuid

from django.core.urlresolvers import reverse
from django.db import models
from django.db.models import Sum
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.utils.translation import ugettext_lazy as _

from model_utils import Choices
from model_utils.fields import AutoCreatedField, AutoLastModifiedField
from model_utils.models import TimeStampedModel

from crate.fields import JSONField
from packages.utils import verlib

def release_file_upload_to(instance, filename):
    dsplit = instance.digest.split("$")
    if len(dsplit) == 2:
        directory = dsplit[1]
    else:
        directory = str(uuid.uuid4()).replace("-", "")

    return posixpath.join("packages", directory, filename)

# @@@ These are by Nature Hierarchical. Would we benefit from a tree structure?
class TroveClassifier(models.Model):
    trove = models.CharField(max_length=350, unique=True)

    def __unicode__(self):
        return self.trove

class Package(TimeStampedModel):
    name = models.SlugField(max_length=150, unique=True)

    def __unicode__(self):
        return self.name

    def get_absolute_url(self):
        return reverse("package_detail", kwargs={"package": self.name})

    @property
    def downloads(self):
        total_downloads = ReleaseFile.objects.filter(release__package__pk=self.pk).aggregate(total_downloads=Sum("downloads"))["total_downloads"]
        if total_downloads is None:
            return 0
        return total_downloads

    @property
    def latest(self):
        if not hasattr(self, "_latest_release"):
            releases = self.releases.order_by("-order")[:1]
            if releases:
                self._latest_release = releases[0]
            else:
                self._latest_release = None
        return self._latest_release

    @property
    def install_command(self):
        return "pip install %(package)s" % {"package": self.name}

    @property
    def requirement_line(self):
        if self.latest is not None:
            # @@@ Should This Be Major/Minor/Patch/Exact Version?
            #       For Now we'll use Minor if verlib can parse it, else exact
            normalized = verlib.suggest_normalized_version(self.latest.version)
            if normalized is not None:
                ver = verlib.NormalizedVersion(normalized)
                next_version = "%(major)s.%(minor)s" % {"major": ver.parts[0][0], "minor": ver.parts[0][1] + 1}
                return "%(package)s>=%(current_version)s,<%(next_version)s" % {
                    "package": self.name,
                    "current_version": self.latest.version,
                    "next_version": next_version,
                }
            return "%(package)s==%(version)s" % {"package": self.name, "version": self.latest.version}

class Release(models.Model):
    created = AutoCreatedField(_("created"), db_index=True)
    modified = AutoLastModifiedField(_("modified"))

    package = models.ForeignKey(Package, related_name="releases")
    version = models.CharField(max_length=512)

    hidden = models.BooleanField(default=False)

    order = models.IntegerField(default=0)

    platform = models.TextField(blank=True)

    summary = models.TextField()
    description = models.TextField(blank=True)

    keywords = models.TextField(blank=True)

    license = models.TextField(blank=True)

    author = models.TextField(blank=True)
    author_email = models.TextField(blank=True)

    maintainer = models.TextField(blank=True)
    maintainer_email = models.TextField(blank=True)

    requires_python = models.CharField(max_length=25, blank=True)

    download_uri = models.URLField(max_length=1024, blank=True)
    # uris = hstore.DictionaryField()

    classifiers = models.ManyToManyField(TroveClassifier, related_name="releases", blank=True)

    raw_data = JSONField(null=True, blank=True)

    class Meta:
        unique_together = ("package", "version")

    def __unicode__(self):
        return u"%(package)s %(version)s" % {"package": self.package.name, "version": self.version}

    def get_absolute_url(self):
        return reverse("package_detail", kwargs={"package": self.package.name, "version": self.version})

    @property
    def downloads(self):
        total_downloads = ReleaseFile.objects.filter(release__pk=self.pk).aggregate(total_downloads=Sum("downloads"))["total_downloads"]
        if total_downloads is None:
            return 0
        return total_downloads

    @property
    def install_command(self):
        return "pip install %(package)s==%(version)s" % {"package": self.package.name, "version": self.version}

    @property
    def requirement_line(self):
        # @@@ Should This Be Major/Minor/Patch/Exact Version?
        #       For Now we'll use Minor if verlib can parse it, else exact
        normalized = verlib.suggest_normalized_version(self.version)
        if normalized is not None:
            ver = verlib.NormalizedVersion(normalized)
            next_version = "%(major)s.%(minor)s" % {"major": ver.parts[0][0], "minor": ver.parts[0][1] + 1}
            return "%(package)s>=%(current_version)s,<%(next_version)s" % {
                "package": self.package.name,
                "current_version": self.version,
                "next_version": next_version,
            }
        return "%(package)s==%(version)s" % {"package": self.package.name, "version": self.version}

class ReleaseFile(models.Model):

    TYPES = Choices(
        ("sdist", "Source"),
        ("bdist_egg", "Egg"),
        ("bdist_msi", "MSI"),
        ("bdist_dmg", "DMG"),
        ("bdist_rpm", "RPM"),
        ("bdist_dumb", "bdist_dumb"),
        ("bdist_wininst", "bdist_wininst"),
    )

    created = AutoCreatedField(_("created"), db_index=True)
    modified = AutoLastModifiedField(_("modified"))

    release = models.ForeignKey(Release, related_name="files")

    type = models.CharField(max_length=25, choices=TYPES)
    file = models.FileField(upload_to=release_file_upload_to, max_length=512)
    filename = models.CharField(max_length=200, help_text="This is the file name given to us by PyPI", blank=True, null=True, default=None)
    digest = models.CharField(max_length=512)

    python_version = models.CharField(max_length=25)

    downloads = models.PositiveIntegerField(default=0)
    comment = models.TextField(blank=True)

    class Meta:
        unique_together = ("release", "type", "python_version", "filename")

    def __unicode__(self):
        return os.path.basename(self.file.name)

    def get_absolute_url(self):
        return self.file.url

    def get_python_version_display(self):
        if self.python_version.lower() == "source":
            return ""
        return self.python_version

class ReleaseURI(models.Model):
    release = models.ForeignKey(Release, related_name="uris")
    label = models.CharField(max_length=64)
    uri = models.URLField(max_length=500)

class ReleaseRequire(models.Model):

    KIND = Choices(
        ("requires", "Requirement"),
        ("requires_dist", "Dist Requirement"),
        ("external", "External Requirement"),
    )

    release = models.ForeignKey(Release, related_name="requires")

    kind = models.CharField(max_length=50, choices=KIND)
    name = models.CharField(max_length=150)
    version = models.CharField(max_length=50)

    environment = models.TextField(blank=True)

    def __unicode__(self):
        return self.name

class ReleaseProvide(models.Model):

    KIND = Choices(
        ("provides", "Provides"),
        ("provides_dist", "Dist Provides"),
    )

    release = models.ForeignKey(Release, related_name="provides")

    kind = models.CharField(max_length=50, choices=KIND)
    name = models.CharField(max_length=150)
    version = models.CharField(max_length=50)

    environment = models.TextField(blank=True)

    def __unicode__(self):
        return self.name

class ReleaseObsolete(models.Model):

    KIND = Choices(
        ("obsoletes", "Obsoletes"),
        ("obsoletes_dist", "Dist Obsoletes"),
    )

    release = models.ForeignKey(Release, related_name="obsoletes")

    kind = models.CharField(max_length=50, choices=KIND)
    name = models.CharField(max_length=150)
    version = models.CharField(max_length=50)

    environment = models.TextField(blank=True)

    def __unicode__(self):
        return self.name

@receiver(post_save, sender=Release)
def version_ordering(sender, **kwargs):
    instance = kwargs.get("instance")
    if instance is not None and kwargs.get("created", True):
        releases = Release.objects.filter(package__pk=instance.package.pk)

        versions = []
        dated = []

        for release in releases:
            normalized = verlib.suggest_normalized_version(release.version)
            if normalized is not None:
                versions.append(release)
            else:
                dated.append(release)

        versions.sort(key=lambda x: verlib.NormalizedVersion(verlib.suggest_normalized_version(x.version)))
        dated.sort(key=lambda x: x.created)

        for i, release in enumerate(dated + versions):
            if release.order != i:
                Release.objects.filter(pk=release.pk).update(order=i)
<commit_msg>remove the commented out hstore field<commit_after>import os
import posixpath
import uuid

from django.core.urlresolvers import reverse
from django.db import models
from django.db.models import Sum
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.utils.translation import ugettext_lazy as _

from model_utils import Choices
from model_utils.fields import AutoCreatedField, AutoLastModifiedField
from model_utils.models import TimeStampedModel

from crate.fields import JSONField
from packages.utils import verlib

def release_file_upload_to(instance, filename):
    dsplit = instance.digest.split("$")
    if len(dsplit) == 2:
        directory = dsplit[1]
    else:
        directory = str(uuid.uuid4()).replace("-", "")

    return posixpath.join("packages", directory, filename)

# @@@ These are by Nature Hierarchical. Would we benefit from a tree structure?
class TroveClassifier(models.Model):
    trove = models.CharField(max_length=350, unique=True)

    def __unicode__(self):
        return self.trove

class Package(TimeStampedModel):
    name = models.SlugField(max_length=150, unique=True)

    def __unicode__(self):
        return self.name

    def get_absolute_url(self):
        return reverse("package_detail", kwargs={"package": self.name})

    @property
    def downloads(self):
        total_downloads = ReleaseFile.objects.filter(release__package__pk=self.pk).aggregate(total_downloads=Sum("downloads"))["total_downloads"]
        if total_downloads is None:
            return 0
        return total_downloads

    @property
    def latest(self):
        if not hasattr(self, "_latest_release"):
            releases = self.releases.order_by("-order")[:1]
            if releases:
                self._latest_release = releases[0]
            else:
                self._latest_release = None
        return self._latest_release

    @property
    def install_command(self):
        return "pip install %(package)s" % {"package": self.name}

    @property
    def requirement_line(self):
        if self.latest is not None:
            # @@@ Should This Be Major/Minor/Patch/Exact Version?
            #       For Now we'll use Minor if verlib can parse it, else exact
            normalized = verlib.suggest_normalized_version(self.latest.version)
            if normalized is not None:
                ver = verlib.NormalizedVersion(normalized)
                next_version = "%(major)s.%(minor)s" % {"major": ver.parts[0][0], "minor": ver.parts[0][1] + 1}
                return "%(package)s>=%(current_version)s,<%(next_version)s" % {
                    "package": self.name,
                    "current_version": self.latest.version,
                    "next_version": next_version,
                }
            return "%(package)s==%(version)s" % {"package": self.name, "version": self.latest.version}

class Release(models.Model):
    created = AutoCreatedField(_("created"), db_index=True)
    modified = AutoLastModifiedField(_("modified"))

    package = models.ForeignKey(Package, related_name="releases")
    version = models.CharField(max_length=512)

    hidden = models.BooleanField(default=False)

    order = models.IntegerField(default=0)

    platform = models.TextField(blank=True)

    summary = models.TextField()
    description = models.TextField(blank=True)

    keywords = models.TextField(blank=True)

    license = models.TextField(blank=True)

    author = models.TextField(blank=True)
    author_email = models.TextField(blank=True)

    maintainer = models.TextField(blank=True)
    maintainer_email = models.TextField(blank=True)

    requires_python = models.CharField(max_length=25, blank=True)

    download_uri = models.URLField(max_length=1024, blank=True)

    classifiers = models.ManyToManyField(TroveClassifier, related_name="releases", blank=True)

    raw_data = JSONField(null=True, blank=True)

    class Meta:
        unique_together = ("package", "version")

    def __unicode__(self):
        return u"%(package)s %(version)s" % {"package": self.package.name, "version": self.version}

    def get_absolute_url(self):
        return reverse("package_detail", kwargs={"package": self.package.name, "version": self.version})

    @property
    def downloads(self):
        total_downloads = ReleaseFile.objects.filter(release__pk=self.pk).aggregate(total_downloads=Sum("downloads"))["total_downloads"]
        if total_downloads is None:
            return 0
        return total_downloads

    @property
    def install_command(self):
        return "pip install %(package)s==%(version)s" % {"package": self.package.name, "version": self.version}

    @property
    def requirement_line(self):
        # @@@ Should This Be Major/Minor/Patch/Exact Version?
        #       For Now we'll use Minor if verlib can parse it, else exact
        normalized = verlib.suggest_normalized_version(self.version)
        if normalized is not None:
            ver = verlib.NormalizedVersion(normalized)
            next_version = "%(major)s.%(minor)s" % {"major": ver.parts[0][0], "minor": ver.parts[0][1] + 1}
            return "%(package)s>=%(current_version)s,<%(next_version)s" % {
                "package": self.package.name,
                "current_version": self.version,
                "next_version": next_version,
            }
        return "%(package)s==%(version)s" % {"package": self.package.name, "version": self.version}

class ReleaseFile(models.Model):

    TYPES = Choices(
        ("sdist", "Source"),
        ("bdist_egg", "Egg"),
        ("bdist_msi", "MSI"),
        ("bdist_dmg", "DMG"),
        ("bdist_rpm", "RPM"),
        ("bdist_dumb", "bdist_dumb"),
        ("bdist_wininst", "bdist_wininst"),
    )

    created = AutoCreatedField(_("created"), db_index=True)
    modified = AutoLastModifiedField(_("modified"))

    release = models.ForeignKey(Release, related_name="files")

    type = models.CharField(max_length=25, choices=TYPES)
    file = models.FileField(upload_to=release_file_upload_to, max_length=512)
    filename = models.CharField(max_length=200, help_text="This is the file name given to us by PyPI", blank=True, null=True, default=None)
    digest = models.CharField(max_length=512)

    python_version = models.CharField(max_length=25)

    downloads = models.PositiveIntegerField(default=0)
    comment = models.TextField(blank=True)

    class Meta:
        unique_together = ("release", "type", "python_version", "filename")

    def __unicode__(self):
        return os.path.basename(self.file.name)

    def get_absolute_url(self):
        return self.file.url

    def get_python_version_display(self):
        if self.python_version.lower() == "source":
            return ""
        return self.python_version

class ReleaseURI(models.Model):
    release = models.ForeignKey(Release, related_name="uris")
    label = models.CharField(max_length=64)
    uri = models.URLField(max_length=500)

class ReleaseRequire(models.Model):

    KIND = Choices(
        ("requires", "Requirement"),
        ("requires_dist", "Dist Requirement"),
        ("external", "External Requirement"),
    )

    release = models.ForeignKey(Release, related_name="requires")

    kind = models.CharField(max_length=50, choices=KIND)
    name = models.CharField(max_length=150)
    version = models.CharField(max_length=50)

    environment = models.TextField(blank=True)

    def __unicode__(self):
        return self.name

class ReleaseProvide(models.Model):

    KIND = Choices(
        ("provides", "Provides"),
        ("provides_dist", "Dist Provides"),
    )

    release = models.ForeignKey(Release, related_name="provides")

    kind = models.CharField(max_length=50, choices=KIND)
    name = models.CharField(max_length=150)
    version = models.CharField(max_length=50)

    environment = models.TextField(blank=True)

    def __unicode__(self):
        return self.name

class ReleaseObsolete(models.Model):

    KIND = Choices(
        ("obsoletes", "Obsoletes"),
        ("obsoletes_dist", "Dist Obsoletes"),
    )

    release = models.ForeignKey(Release, related_name="obsoletes")

    kind = models.CharField(max_length=50, choices=KIND)
    name = models.CharField(max_length=150)
    version = models.CharField(max_length=50)

    environment = models.TextField(blank=True)

    def __unicode__(self):
        return self.name

@receiver(post_save, sender=Release)
def version_ordering(sender, **kwargs):
    instance = kwargs.get("instance")
    if instance is not None and kwargs.get("created", True):
        releases = Release.objects.filter(package__pk=instance.package.pk)

        versions = []
        dated = []

        for release in releases:
            normalized = verlib.suggest_normalized_version(release.version)
            if normalized is not None:
                versions.append(release)
            else:
                dated.append(release)

        versions.sort(key=lambda x: verlib.NormalizedVersion(verlib.suggest_normalized_version(x.version)))
        dated.sort(key=lambda x: x.created)

        for i, release in enumerate(dated + versions):
            if release.order != i:
                Release.objects.filter(pk=release.pk).update(order=i)
<|endoftext|>


<commit_before>from bettercache.utils import CachingMixin, strip_wsgi
from bettercache.tasks import GeneratePage
from bettercache.proxy import proxy

import logging
logger = logging.getLogger()

class BetterView(CachingMixin):
    def get(self, request):
        response = None
        #should this bypass this replicates part of the irule
        if not self.should_bypass_cache(request):
            response, expired = self.get_cache(request)
            # send off the celery task if it's expired
            if expired:
                self.send_task(request, response)

        # if response is still none we have to proxy
        if response is None:
            logger.info('request %s proxied' %request.build_absolute_uri)
            response = proxy(request)
            #TODO: delete the following two lines
            #self.set_cache(request, response)
            response['X-Bettercache-Proxy'] = 'true'
        else:
            response['X-Bettercache-Proxy'] = 'false'
            logger.info('request %s from cache' %request.build_absolute_uri)

        return response

#TODO: properly implement a class based view
BV = BetterView()
cache_view = BV.get
<commit_msg>[CMSPERF-189] added some debug logging<commit_after>from bettercache.utils import CachingMixin, strip_wsgi
from bettercache.tasks import GeneratePage
from bettercache.proxy import proxy

import logging
logger = logging.getLogger()

class BetterView(CachingMixin):
    def get(self, request):
        response = None
        #should this bypass this replicates part of the irule
        if not self.should_bypass_cache(request):
            response, expired = self.get_cache(request)
            # send off the celery task if it's expired
            if expired:
                logger.debug("sending task for %s" %request.build_absolute_uri)
                self.send_task(request, response)
            else:
                logger.debug("sending task for %s" %request.build_absolute_uri)

        # if response is still none we have to proxy
        if response is None:
            logger.info('request %s proxied' %request.build_absolute_uri)
            response = proxy(request)
            #TODO: delete the following two lines
            #self.set_cache(request, response)
            response['X-Bettercache-Proxy'] = 'true'
        else:
            response['X-Bettercache-Proxy'] = 'false'
            logger.info('request %s from cache' %request.build_absolute_uri)

        return response

#TODO: properly implement a class based view
BV = BetterView()
cache_view = BV.get
<|endoftext|>


<commit_before>"""Backing implementation for InstallRequirement's various constructors

The idea here is that these formed a major chunk of InstallRequirement's size
so, moving them and support code dedicated to them outside of that class
helps creates for better understandability for the rest of the code.

These are meant to be used elsewhere within pip to create instances of
InstallRequirement.
"""

import logging
import os
import re

from pip._vendor.packaging.markers import Marker
from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
from pip._vendor.packaging.specifiers import Specifier
from pip._vendor.pkg_resources import RequirementParseError, parse_requirements

from pip._internal.exceptions import InstallationError
from pip._internal.models.index import PyPI, TestPyPI
from pip._internal.models.link import Link
from pip._internal.models.wheel import Wheel
from pip._internal.pyproject import make_pyproject_path
from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils.deprecation import deprecated
from pip._internal.utils.filetypes import is_archive_file
from pip._internal.utils.misc import is_installable_dir
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
from pip._internal.utils.urls import path_to_url
from pip._internal.vcs import is_url, vcs

if MYPY_CHECK_RUNNING:
    from typing import Any, Dict, Optional, Set, Tuple, Union

    from pip._internal.req.req_file import ParsedRequirement

__all__ = [
    "install_req_from_editable", "install_req_from_line",
    "parse_editable"
]

logger = logging.getLogger(__name__)
operators = Specifier._operators.keys()

def _strip_extras(path):
    # type: (str) -> Tuple[str, Optional[str]]
    m = re.match(r'^(.+)(\[[^\]]+\])$', path)
    extras = None
    if m:
        path_no_extras = m.group(1)
        extras = m.group(2)
    else:
        path_no_extras = path

    return path_no_extras, extras

def convert_extras(extras):
    # type: (Optional[str]) -> Set[str]
    if not extras:
        return set()
    return Requirement("placeholder" + extras.lower()).extras

def parse_editable(editable_req):
    # type: (str) -> Tuple[Optional[str], str, Set[str]]
    """Parses an editable requirement into:
        - a requirement name
        - an URL
        - extras
        - editable options
    Accepted requirements:
        svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir
        .[some_extra]
    """

    url = editable_req

    # If a file path is specified with extras, strip off the extras.
    url_no_extras, extras = _strip_extras(url)

    if os.path.isdir(url_no_extras):
        if not os.path.exists(os.path.join(url_no_extras, 'setup.py')):
            msg = (
                'File "setup.py" not found. Directory cannot be installed '
                'in editable mode: {}'.format(os.path.abspath(url_no_extras))
            )
            pyproject_path = make_pyproject_path(url_no_extras)
            if os.path.isfile(pyproject_path):
                msg += (
                    '\n(A "pyproject.toml" file was found, but editable '
                    'mode currently requires a setup.py based build.)'
                )
            raise InstallationError(msg)

        # Treating it as code that has already been checked out
        url_no_extras = path_to_url(url_no_extras)

    if url_no_extras.lower().startswith('file:'):
        package_name = Link(url_no_extras).egg_fragment
        if extras:
            return (
                package_name,
                url_no_extras,
                Requirement("placeholder" + extras.lower()).extras,
            )
        else:
            return package_name, url_no_extras, set()

    for version_control in vcs:
        if url.lower().startswith(f'{version_control}:'):
            url = f'{version_control}+{url}'
            break

    link = Link(url)

    if not link.is_vcs:
        backends = ", ".join([backend.name + '+' for backend in vcs.backends])
        raise InstallationError(
            f'{editable_req} is not a valid editable requirement. '
            f'It should either be a path to a local project or a VCS URL '
            f'(beginning with {backends}).'
        )

    package_name = link.egg_fragment
    if not package_name:
        raise InstallationError(
            "Could not detect requirement name for '{}', please specify one "
            "with #egg=your_package_name".format(editable_req)
        )
    return package_name, url, set()

def deduce_helpful_msg(req):
    # type: (str) -> str
    """Returns helpful msg in case requirements file does not exist,
    or cannot be parsed.

    :params req: Requirements file path
    """
    msg = ""
    if os.path.exists(req):
        msg = " The path does exist. "
        # Try to parse and check if it is a requirements file.
        try:
            with open(req, 'r') as fp:
                # parse first line only
                next(parse_requirements(fp.read()))
                msg += (
                    "The argument you provided "
                    "({}) appears to be a"
                    " requirements file. If that is the"
                    " case, use the '-r' flag to install"
                    " the packages specified within it."
                ).format(req)
        except RequirementParseError:
            logger.debug(
                "Cannot parse '%s' as requirements file", req, exc_info=True
            )
    else:
        msg += f" File '{req}' does not exist."
    return msg

class RequirementParts:
    def __init__(
            self,
            requirement,  # type: Optional[Requirement]
            link,         # type: Optional[Link]
            markers,      # type: Optional[Marker]
            extras,       # type: Set[str]
    ):
        self.requirement = requirement
        self.link = link
        self.markers = markers
        self.extras = extras

def parse_req_from_editable(editable_req):
    # type: (str) -> RequirementParts
    name, url, extras_override = parse_editable(editable_req)

    if name is not None:
        try:
            req = Requirement(name)
        except InvalidRequirement:
            raise InstallationError(f"Invalid requirement: '{name}'")
    else:
        req = None

    link = Link(url)

    return RequirementParts(req, link, None, extras_override)

# ---- The actual constructors follow ----

def install_req_from_editable(
    editable_req,  # type: str
    comes_from=None,  # type: Optional[Union[InstallRequirement, str]]
    use_pep517=None,  # type: Optional[bool]
    isolated=False,  # type: bool
    options=None,  # type: Optional[Dict[str, Any]]
    constraint=False,  # type: bool
    user_supplied=False,  # type: bool
):
    # type: (...) -> InstallRequirement

    parts = parse_req_from_editable(editable_req)

    return InstallRequirement(
        parts.requirement,
        comes_from=comes_from,
        user_supplied=user_supplied,
        editable=True,
        link=parts.link,
        constraint=constraint,
        use_pep517=use_pep517,
        isolated=isolated,
        install_options=options.get("install_options", []) if options else [],
        global_options=options.get("global_options", []) if options else [],
        hash_options=options.get("hashes", {}) if options else {},
        extras=parts.extras,
    )

def _looks_like_path(name):
    # type: (str) -> bool
    """Checks whether the string "looks like" a path on the filesystem.

    This does not check whether the target actually exists, only judge from the
    appearance.

    Returns true if any of the following conditions is true:
    * a path separator is found (either os.path.sep or os.path.altsep);
    * a dot is found (which represents the current directory).
    """
    if os.path.sep in name:
        return True
    if os.path.altsep is not None and os.path.altsep in name:
        return True
    if name.startswith("."):
        return True
    return False

def _get_url_from_path(path, name):
    # type: (str, str) -> Optional[str]
    """
    First, it checks whether a provided path is an installable directory
    (e.g. it has a setup.py). If it is, returns the path.

    If false, check if the path is an archive file (such as a .whl).
    The function checks if the path is a file. If false, if the path has
    an @, it will treat it as a PEP 440 URL requirement and return the path.
    """
    if _looks_like_path(name) and os.path.isdir(path):
        if is_installable_dir(path):
            return path_to_url(path)
        raise InstallationError(
            "Directory {name!r} is not installable. Neither 'setup.py' "
            "nor 'pyproject.toml' found.".format(**locals())
        )
    if not is_archive_file(path):
        return None
    if os.path.isfile(path):
        return path_to_url(path)
    urlreq_parts = name.split('@', 1)
    if len(urlreq_parts) >= 2 and not _looks_like_path(urlreq_parts[0]):
        # If the path contains '@' and the part before it does not look
        # like a path, try to treat it as a PEP 440 URL req instead.
        return None
    logger.warning(
        'Requirement %r looks like a filename, but the '
        'file does not exist',
        name
    )
    return path_to_url(path)

def parse_req_from_line(name, line_source):
    # type: (str, Optional[str]) -> RequirementParts
    if is_url(name):
        marker_sep = '; '
    else:
        marker_sep = ';'
    if marker_sep in name:
        name, markers_as_string = name.split(marker_sep, 1)
        markers_as_string = markers_as_string.strip()
        if not markers_as_string:
            markers = None
        else:
            markers = Marker(markers_as_string)
    else:
        markers = None
    name = name.strip()
    req_as_string = None
    path = os.path.normpath(os.path.abspath(name))
    link = None
    extras_as_string = None

    if is_url(name):
        link = Link(name)
    else:
        p, extras_as_string = _strip_extras(path)
        url = _get_url_from_path(p, name)
        if url is not None:
            link = Link(url)

    # it's a local file, dir, or url
    if link:
        # Handle relative file URLs
        if link.scheme == 'file' and re.search(r'\.\./', link.url):
            link = Link(
                path_to_url(os.path.normpath(os.path.abspath(link.path))))
        # wheel file
        if link.is_wheel:
            wheel = Wheel(link.filename)  # can raise InvalidWheelFilename
            req_as_string = "{wheel.name}=={wheel.version}".format(**locals())
        else:
            # set the req to the egg fragment.  when it's not there, this
            # will become an 'unnamed' requirement
            req_as_string = link.egg_fragment

    # a requirement specifier
    else:
        req_as_string = name

    extras = convert_extras(extras_as_string)

    def with_source(text):
        # type: (str) -> str
        if not line_source:
            return text
        return f'{text} (from {line_source})'

    if req_as_string is not None:
        try:
            req = Requirement(req_as_string)
        except InvalidRequirement:
            if os.path.sep in req_as_string:
                add_msg = "It looks like a path."
                add_msg += deduce_helpful_msg(req_as_string)
            elif ('=' in req_as_string and
                  not any(op in req_as_string for op in operators)):
                add_msg = "= is not a valid operator. Did you mean == ?"
            else:
                add_msg = ''
            msg = with_source(
                f'Invalid requirement: {req_as_string!r}'
            )
            if add_msg:
                msg += f'\nHint: {add_msg}'
            raise InstallationError(msg)
        else:
            # Deprecate extras after specifiers: "name>=1.0[extras]"
            # This currently works by accident because _strip_extras() parses
            # any extras in the end of the string and those are saved in
            # RequirementParts
            for spec in req.specifier:
                spec_str = str(spec)
                if spec_str.endswith(']'):
                    msg = f"Extras after version '{spec_str}'."
                    replace = "moving the extras before version specifiers"
                    deprecated(msg, replacement=replace, gone_in="21.0")
    else:
        req = None

    return RequirementParts(req, link, markers, extras)

def install_req_from_line(
    name,  # type: str
    comes_from=None,  # type: Optional[Union[str, InstallRequirement]]
    use_pep517=None,  # type: Optional[bool]
    isolated=False,  # type: bool
    options=None,  # type: Optional[Dict[str, Any]]
    constraint=False,  # type: bool
    line_source=None,  # type: Optional[str]
    user_supplied=False,  # type: bool
):
    # type: (...) -> InstallRequirement
    """Creates an InstallRequirement from a name, which might be a
    requirement, directory containing 'setup.py', filename, or URL.

    :param line_source: An optional string describing where the line is from,
        for logging purposes in case of an error.
    """
    parts = parse_req_from_line(name, line_source)

    return InstallRequirement(
        parts.requirement, comes_from, link=parts.link, markers=parts.markers,
        use_pep517=use_pep517, isolated=isolated,
        install_options=options.get("install_options", []) if options else [],
        global_options=options.get("global_options", []) if options else [],
        hash_options=options.get("hashes", {}) if options else {},
        constraint=constraint,
        extras=parts.extras,
        user_supplied=user_supplied,
    )

def install_req_from_req_string(
    req_string,  # type: str
    comes_from=None,  # type: Optional[InstallRequirement]
    isolated=False,  # type: bool
    use_pep517=None,  # type: Optional[bool]
    user_supplied=False,  # type: bool
):
    # type: (...) -> InstallRequirement
    try:
        req = Requirement(req_string)
    except InvalidRequirement:
        raise InstallationError(f"Invalid requirement: '{req_string}'")

    domains_not_allowed = [
        PyPI.file_storage_domain,
        TestPyPI.file_storage_domain,
    ]
    if (req.url and comes_from and comes_from.link and
            comes_from.link.netloc in domains_not_allowed):
        # Explicitly disallow pypi packages that depend on external urls
        raise InstallationError(
            "Packages installed from PyPI cannot depend on packages "
            "which are not also hosted on PyPI.\n"
            "{} depends on {} ".format(comes_from.name, req)
        )

    return InstallRequirement(
        req,
        comes_from,
        isolated=isolated,
        use_pep517=use_pep517,
        user_supplied=user_supplied,
    )

def install_req_from_parsed_requirement(
    parsed_req,  # type: ParsedRequirement
    isolated=False,  # type: bool
    use_pep517=None,  # type: Optional[bool]
    user_supplied=False,  # type: bool
):
    # type: (...) -> InstallRequirement
    if parsed_req.is_editable:
        req = install_req_from_editable(
            parsed_req.requirement,
            comes_from=parsed_req.comes_from,
            use_pep517=use_pep517,
            constraint=parsed_req.constraint,
            isolated=isolated,
            user_supplied=user_supplied,
        )

    else:
        req = install_req_from_line(
            parsed_req.requirement,
            comes_from=parsed_req.comes_from,
            use_pep517=use_pep517,
            isolated=isolated,
            options=parsed_req.options,
            constraint=parsed_req.constraint,
            line_source=parsed_req.line_source,
            user_supplied=user_supplied,
        )
    return req
<commit_msg>Improve syntax<commit_after>"""Backing implementation for InstallRequirement's various constructors

The idea here is that these formed a major chunk of InstallRequirement's size
so, moving them and support code dedicated to them outside of that class
helps creates for better understandability for the rest of the code.

These are meant to be used elsewhere within pip to create instances of
InstallRequirement.
"""

import logging
import os
import re

from pip._vendor.packaging.markers import Marker
from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
from pip._vendor.packaging.specifiers import Specifier
from pip._vendor.pkg_resources import RequirementParseError, parse_requirements

from pip._internal.exceptions import InstallationError
from pip._internal.models.index import PyPI, TestPyPI
from pip._internal.models.link import Link
from pip._internal.models.wheel import Wheel
from pip._internal.pyproject import make_pyproject_path
from pip._internal.req.req_install import InstallRequirement
from pip._internal.utils.deprecation import deprecated
from pip._internal.utils.filetypes import is_archive_file
from pip._internal.utils.misc import is_installable_dir
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
from pip._internal.utils.urls import path_to_url
from pip._internal.vcs import is_url, vcs

if MYPY_CHECK_RUNNING:
    from typing import Any, Dict, Optional, Set, Tuple, Union

    from pip._internal.req.req_file import ParsedRequirement

__all__ = [
    "install_req_from_editable", "install_req_from_line",
    "parse_editable"
]

logger = logging.getLogger(__name__)
operators = Specifier._operators.keys()

def _strip_extras(path):
    # type: (str) -> Tuple[str, Optional[str]]
    m = re.match(r'^(.+)(\[[^\]]+\])$', path)
    extras = None
    if m:
        path_no_extras = m.group(1)
        extras = m.group(2)
    else:
        path_no_extras = path

    return path_no_extras, extras

def convert_extras(extras):
    # type: (Optional[str]) -> Set[str]
    if not extras:
        return set()
    return Requirement("placeholder" + extras.lower()).extras

def parse_editable(editable_req):
    # type: (str) -> Tuple[Optional[str], str, Set[str]]
    """Parses an editable requirement into:
        - a requirement name
        - an URL
        - extras
        - editable options
    Accepted requirements:
        svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir
        .[some_extra]
    """

    url = editable_req

    # If a file path is specified with extras, strip off the extras.
    url_no_extras, extras = _strip_extras(url)

    if os.path.isdir(url_no_extras):
        if not os.path.exists(os.path.join(url_no_extras, 'setup.py')):
            msg = (
                'File "setup.py" not found. Directory cannot be installed '
                'in editable mode: {}'.format(os.path.abspath(url_no_extras))
            )
            pyproject_path = make_pyproject_path(url_no_extras)
            if os.path.isfile(pyproject_path):
                msg += (
                    '\n(A "pyproject.toml" file was found, but editable '
                    'mode currently requires a setup.py based build.)'
                )
            raise InstallationError(msg)

        # Treating it as code that has already been checked out
        url_no_extras = path_to_url(url_no_extras)

    if url_no_extras.lower().startswith('file:'):
        package_name = Link(url_no_extras).egg_fragment
        if extras:
            return (
                package_name,
                url_no_extras,
                Requirement("placeholder" + extras.lower()).extras,
            )
        else:
            return package_name, url_no_extras, set()

    for version_control in vcs:
        if url.lower().startswith(f'{version_control}:'):
            url = f'{version_control}+{url}'
            break

    link = Link(url)

    if not link.is_vcs:
        backends = ", ".join(f"{backend.name}+" for backend in vcs.backends)
        raise InstallationError(
            f'{editable_req} is not a valid editable requirement. '
            f'It should either be a path to a local project or a VCS URL '
            f'(beginning with {backends}).'
        )

    package_name = link.egg_fragment
    if not package_name:
        raise InstallationError(
            "Could not detect requirement name for '{}', please specify one "
            "with #egg=your_package_name".format(editable_req)
        )
    return package_name, url, set()

def deduce_helpful_msg(req):
    # type: (str) -> str
    """Returns helpful msg in case requirements file does not exist,
    or cannot be parsed.

    :params req: Requirements file path
    """
    msg = ""
    if os.path.exists(req):
        msg = " The path does exist. "
        # Try to parse and check if it is a requirements file.
        try:
            with open(req, 'r') as fp:
                # parse first line only
                next(parse_requirements(fp.read()))
                msg += (
                    "The argument you provided "
                    "({}) appears to be a"
                    " requirements file. If that is the"
                    " case, use the '-r' flag to install"
                    " the packages specified within it."
                ).format(req)
        except RequirementParseError:
            logger.debug(
                "Cannot parse '%s' as requirements file", req, exc_info=True
            )
    else:
        msg += f" File '{req}' does not exist."
    return msg

class RequirementParts:
    def __init__(
            self,
            requirement,  # type: Optional[Requirement]
            link,         # type: Optional[Link]
            markers,      # type: Optional[Marker]
            extras,       # type: Set[str]
    ):
        self.requirement = requirement
        self.link = link
        self.markers = markers
        self.extras = extras

def parse_req_from_editable(editable_req):
    # type: (str) -> RequirementParts
    name, url, extras_override = parse_editable(editable_req)

    if name is not None:
        try:
            req = Requirement(name)
        except InvalidRequirement:
            raise InstallationError(f"Invalid requirement: '{name}'")
    else:
        req = None

    link = Link(url)

    return RequirementParts(req, link, None, extras_override)

# ---- The actual constructors follow ----

def install_req_from_editable(
    editable_req,  # type: str
    comes_from=None,  # type: Optional[Union[InstallRequirement, str]]
    use_pep517=None,  # type: Optional[bool]
    isolated=False,  # type: bool
    options=None,  # type: Optional[Dict[str, Any]]
    constraint=False,  # type: bool
    user_supplied=False,  # type: bool
):
    # type: (...) -> InstallRequirement

    parts = parse_req_from_editable(editable_req)

    return InstallRequirement(
        parts.requirement,
        comes_from=comes_from,
        user_supplied=user_supplied,
        editable=True,
        link=parts.link,
        constraint=constraint,
        use_pep517=use_pep517,
        isolated=isolated,
        install_options=options.get("install_options", []) if options else [],
        global_options=options.get("global_options", []) if options else [],
        hash_options=options.get("hashes", {}) if options else {},
        extras=parts.extras,
    )

def _looks_like_path(name):
    # type: (str) -> bool
    """Checks whether the string "looks like" a path on the filesystem.

    This does not check whether the target actually exists, only judge from the
    appearance.

    Returns true if any of the following conditions is true:
    * a path separator is found (either os.path.sep or os.path.altsep);
    * a dot is found (which represents the current directory).
    """
    if os.path.sep in name:
        return True
    if os.path.altsep is not None and os.path.altsep in name:
        return True
    if name.startswith("."):
        return True
    return False

def _get_url_from_path(path, name):
    # type: (str, str) -> Optional[str]
    """
    First, it checks whether a provided path is an installable directory
    (e.g. it has a setup.py). If it is, returns the path.

    If false, check if the path is an archive file (such as a .whl).
    The function checks if the path is a file. If false, if the path has
    an @, it will treat it as a PEP 440 URL requirement and return the path.
    """
    if _looks_like_path(name) and os.path.isdir(path):
        if is_installable_dir(path):
            return path_to_url(path)
        raise InstallationError(
            "Directory {name!r} is not installable. Neither 'setup.py' "
            "nor 'pyproject.toml' found.".format(**locals())
        )
    if not is_archive_file(path):
        return None
    if os.path.isfile(path):
        return path_to_url(path)
    urlreq_parts = name.split('@', 1)
    if len(urlreq_parts) >= 2 and not _looks_like_path(urlreq_parts[0]):
        # If the path contains '@' and the part before it does not look
        # like a path, try to treat it as a PEP 440 URL req instead.
        return None
    logger.warning(
        'Requirement %r looks like a filename, but the '
        'file does not exist',
        name
    )
    return path_to_url(path)

def parse_req_from_line(name, line_source):
    # type: (str, Optional[str]) -> RequirementParts
    if is_url(name):
        marker_sep = '; '
    else:
        marker_sep = ';'
    if marker_sep in name:
        name, markers_as_string = name.split(marker_sep, 1)
        markers_as_string = markers_as_string.strip()
        if not markers_as_string:
            markers = None
        else:
            markers = Marker(markers_as_string)
    else:
        markers = None
    name = name.strip()
    req_as_string = None
    path = os.path.normpath(os.path.abspath(name))
    link = None
    extras_as_string = None

    if is_url(name):
        link = Link(name)
    else:
        p, extras_as_string = _strip_extras(path)
        url = _get_url_from_path(p, name)
        if url is not None:
            link = Link(url)

    # it's a local file, dir, or url
    if link:
        # Handle relative file URLs
        if link.scheme == 'file' and re.search(r'\.\./', link.url):
            link = Link(
                path_to_url(os.path.normpath(os.path.abspath(link.path))))
        # wheel file
        if link.is_wheel:
            wheel = Wheel(link.filename)  # can raise InvalidWheelFilename
            req_as_string = "{wheel.name}=={wheel.version}".format(**locals())
        else:
            # set the req to the egg fragment.  when it's not there, this
            # will become an 'unnamed' requirement
            req_as_string = link.egg_fragment

    # a requirement specifier
    else:
        req_as_string = name

    extras = convert_extras(extras_as_string)

    def with_source(text):
        # type: (str) -> str
        if not line_source:
            return text
        return f'{text} (from {line_source})'

    if req_as_string is not None:
        try:
            req = Requirement(req_as_string)
        except InvalidRequirement:
            if os.path.sep in req_as_string:
                add_msg = "It looks like a path."
                add_msg += deduce_helpful_msg(req_as_string)
            elif ('=' in req_as_string and
                  not any(op in req_as_string for op in operators)):
                add_msg = "= is not a valid operator. Did you mean == ?"
            else:
                add_msg = ''
            msg = with_source(
                f'Invalid requirement: {req_as_string!r}'
            )
            if add_msg:
                msg += f'\nHint: {add_msg}'
            raise InstallationError(msg)
        else:
            # Deprecate extras after specifiers: "name>=1.0[extras]"
            # This currently works by accident because _strip_extras() parses
            # any extras in the end of the string and those are saved in
            # RequirementParts
            for spec in req.specifier:
                spec_str = str(spec)
                if spec_str.endswith(']'):
                    msg = f"Extras after version '{spec_str}'."
                    replace = "moving the extras before version specifiers"
                    deprecated(msg, replacement=replace, gone_in="21.0")
    else:
        req = None

    return RequirementParts(req, link, markers, extras)

def install_req_from_line(
    name,  # type: str
    comes_from=None,  # type: Optional[Union[str, InstallRequirement]]
    use_pep517=None,  # type: Optional[bool]
    isolated=False,  # type: bool
    options=None,  # type: Optional[Dict[str, Any]]
    constraint=False,  # type: bool
    line_source=None,  # type: Optional[str]
    user_supplied=False,  # type: bool
):
    # type: (...) -> InstallRequirement
    """Creates an InstallRequirement from a name, which might be a
    requirement, directory containing 'setup.py', filename, or URL.

    :param line_source: An optional string describing where the line is from,
        for logging purposes in case of an error.
    """
    parts = parse_req_from_line(name, line_source)

    return InstallRequirement(
        parts.requirement, comes_from, link=parts.link, markers=parts.markers,
        use_pep517=use_pep517, isolated=isolated,
        install_options=options.get("install_options", []) if options else [],
        global_options=options.get("global_options", []) if options else [],
        hash_options=options.get("hashes", {}) if options else {},
        constraint=constraint,
        extras=parts.extras,
        user_supplied=user_supplied,
    )

def install_req_from_req_string(
    req_string,  # type: str
    comes_from=None,  # type: Optional[InstallRequirement]
    isolated=False,  # type: bool
    use_pep517=None,  # type: Optional[bool]
    user_supplied=False,  # type: bool
):
    # type: (...) -> InstallRequirement
    try:
        req = Requirement(req_string)
    except InvalidRequirement:
        raise InstallationError(f"Invalid requirement: '{req_string}'")

    domains_not_allowed = [
        PyPI.file_storage_domain,
        TestPyPI.file_storage_domain,
    ]
    if (req.url and comes_from and comes_from.link and
            comes_from.link.netloc in domains_not_allowed):
        # Explicitly disallow pypi packages that depend on external urls
        raise InstallationError(
            "Packages installed from PyPI cannot depend on packages "
            "which are not also hosted on PyPI.\n"
            "{} depends on {} ".format(comes_from.name, req)
        )

    return InstallRequirement(
        req,
        comes_from,
        isolated=isolated,
        use_pep517=use_pep517,
        user_supplied=user_supplied,
    )

def install_req_from_parsed_requirement(
    parsed_req,  # type: ParsedRequirement
    isolated=False,  # type: bool
    use_pep517=None,  # type: Optional[bool]
    user_supplied=False,  # type: bool
):
    # type: (...) -> InstallRequirement
    if parsed_req.is_editable:
        req = install_req_from_editable(
            parsed_req.requirement,
            comes_from=parsed_req.comes_from,
            use_pep517=use_pep517,
            constraint=parsed_req.constraint,
            isolated=isolated,
            user_supplied=user_supplied,
        )

    else:
        req = install_req_from_line(
            parsed_req.requirement,
            comes_from=parsed_req.comes_from,
            use_pep517=use_pep517,
            isolated=isolated,
            options=parsed_req.options,
            constraint=parsed_req.constraint,
            line_source=parsed_req.line_source,
            user_supplied=user_supplied,
        )
    return req
<|endoftext|>